1"""
2Doxyrunner Sphinx Plugin
3########################
4
5Copyright (c) 2021 Nordic Semiconductor ASA
6SPDX-License-Identifier: Apache-2.0
7
8Introduction
9============
10
11This Sphinx plugin can be used to run Doxygen build as part of the Sphinx build
12process. It is meant to be used with other plugins such as ``breathe`` in order
13to improve the user experience. The principal features offered by this plugin
14are:
15
16- Doxygen build is run before Sphinx reads input files
17- Doxyfile can be optionally pre-processed so that variables can be inserted
18- Changes in the Doxygen input files are tracked so that Doxygen build is only
19  run if necessary.
20- Synchronizes Doxygen XML output so that even if Doxygen is run only changed,
21  deleted or added files are modified.
22
23References:
24
25-  https://github.com/michaeljones/breathe/issues/420
26
27Configuration options
28=====================
29
30- ``doxyrunner_doxygen``: Path to the Doxygen binary.
31- ``doxyrunner_doxyfile``: Path to Doxyfile.
32- ``doxyrunner_outdir``: Doxygen build output directory (inserted to
33  ``OUTPUT_DIRECTORY``)
34- ``doxyrunner_outdir_var``: Variable representing the Doxygen build output
35  directory, as used by ``OUTPUT_DIRECTORY``. This can be useful if other
36  Doxygen variables reference to the output directory.
37- ``doxyrunner_fmt``: Flag to indicate if Doxyfile should be formatted.
38- ``doxyrunner_fmt_vars``: Format variables dictionary (name: value).
39- ``doxyrunner_fmt_pattern``: Format pattern.
40- ``doxyrunner_silent``: If Doxygen output should be logged or not. Note that
41  this option may not have any effect if ``QUIET`` is set to ``YES``.
42"""
43
44import filecmp
45import hashlib
46from pathlib import Path
47import re
48import shlex
49import shutil
50from subprocess import Popen, PIPE, STDOUT
51import tempfile
52from typing import List, Dict, Optional, Any
53
54from sphinx.application import Sphinx
55from sphinx.environment import BuildEnvironment
56from sphinx.util import logging
57
58
59__version__ = "0.1.0"
60
61
62logger = logging.getLogger(__name__)
63
64
65def hash_file(file: Path) -> str:
66    """Compute the hash (SHA256) of a file in text mode.
67
68    Args:
69        file: File to be hashed.
70
71    Returns:
72        Hash.
73    """
74
75    with open(file, encoding="utf-8") as f:
76        sha256 = hashlib.sha256(f.read().encode("utf-8"))
77
78    return sha256.hexdigest()
79
80def get_doxygen_option(doxyfile: str, option: str) -> List[str]:
81    """Obtain the value of a Doxygen option.
82
83    Args:
84        doxyfile: Content of the Doxyfile.
85        option: Option to be retrieved.
86
87    Notes:
88        Does not support appended values.
89
90    Returns:
91        Option values.
92    """
93
94    option_re = re.compile(r"^\s*([A-Z0-9_]+)\s*=\s*(.*)$")
95    multiline_re = re.compile(r"^\s*(.*)$")
96
97    values = []
98    found = False
99    finished = False
100    for line in doxyfile.splitlines():
101        if not found:
102            m = option_re.match(line)
103            if not m or m.group(1) != option:
104                continue
105
106            found = True
107            value = m.group(2)
108        else:
109            m = multiline_re.match(line)
110            if not m:
111                raise ValueError(f"Unexpected line content: {line}")
112
113            value = m.group(1)
114
115        # check if it is a multiline value
116        finished = not value.endswith("\\")
117
118        # strip backslash
119        if not finished:
120            value = value[:-1]
121
122        # split values
123        values += shlex.split(value.replace("\\", "\\\\"))
124
125        if finished:
126            break
127
128    return values
129
130
131def process_doxyfile(
132    doxyfile: str,
133    outdir: Path,
134    silent: bool,
135    fmt: bool = False,
136    fmt_pattern: Optional[str] = None,
137    fmt_vars: Optional[Dict[str, str]] = None,
138    outdir_var: Optional[str] = None,
139) -> str:
140    """Process Doxyfile.
141
142    Notes:
143        OUTPUT_DIRECTORY, WARN_FORMAT and QUIET are overridden to satisfy
144        extension operation needs.
145
146    Args:
147        doxyfile: Path to the Doxyfile.
148        outdir: Output directory of the Doxygen build.
149        silent: If Doxygen should be run in quiet mode or not.
150        fmt: If Doxyfile should be formatted.
151        fmt_pattern: Format pattern.
152        fmt_vars: Format variables.
153        outdir_var: Variable representing output directory.
154
155     Returns:
156        Processed Doxyfile content.
157    """
158
159    with open(doxyfile) as f:
160        content = f.read()
161
162    content = re.sub(
163        r"^\s*OUTPUT_DIRECTORY\s*=.*$",
164        f"OUTPUT_DIRECTORY={outdir.as_posix()}",
165        content,
166        flags=re.MULTILINE,
167    )
168
169    content = re.sub(
170        r"^\s*WARN_FORMAT\s*=.*$",
171        'WARN_FORMAT="$file:$line: $text"',
172        content,
173        flags=re.MULTILINE,
174    )
175
176    content = re.sub(
177        r"^\s*QUIET\s*=.*$",
178        "QUIET=" + "YES" if silent else "NO",
179        content,
180        flags=re.MULTILINE,
181    )
182
183    if fmt:
184        if not fmt_pattern or not fmt_vars:
185            raise ValueError("Invalid formatting pattern or variables")
186
187        if outdir_var:
188            fmt_vars = fmt_vars.copy()
189            fmt_vars[outdir_var] = outdir.as_posix()
190
191        for var, value in fmt_vars.items():
192            content = content.replace(fmt_pattern.format(var), value)
193
194    return content
195
196
197def doxygen_input_has_changed(env: BuildEnvironment, doxyfile: str) -> bool:
198    """Check if Doxygen input files have changed.
199
200    Args:
201        env: Sphinx build environment instance.
202        doxyfile: Doxyfile content.
203
204    Returns:
205        True if changed, False otherwise.
206    """
207
208    # obtain Doxygen input files and patterns
209    input_files = get_doxygen_option(doxyfile, "INPUT")
210    if not input:
211        raise ValueError("No INPUT set in Doxyfile")
212
213    file_patterns = get_doxygen_option(doxyfile, "FILE_PATTERNS")
214    if not file_patterns:
215        raise ValueError("No FILE_PATTERNS set in Doxyfile")
216
217    # build a set with input files hash
218    cache = set()
219    for file in input_files:
220        path = Path(file)
221        if path.is_file():
222            cache.add(hash_file(path))
223        else:
224            for pattern in file_patterns:
225                for p_file in path.glob("**/" + pattern):
226                    cache.add(hash_file(p_file))
227
228    # check if any file has changed
229    if hasattr(env, "doxyrunner_cache") and env.doxyrunner_cache == cache:
230        return False
231
232    # store current state
233    env.doxyrunner_cache = cache
234
235    return True
236
237
238def process_doxygen_output(line: str, silent: bool) -> None:
239    """Process a line of Doxygen program output.
240
241    This function will map Doxygen output to the Sphinx logger output. Errors
242    and warnings will be converted to Sphinx errors and warnings. Other
243    messages, if not silent, will be mapped to the info logger channel.
244
245    Args:
246        line: Doxygen program line.
247        silent: True if regular messages should be logged, False otherwise.
248    """
249
250    m = re.match(r"(.*):(\d+): ([a-z]+): (.*)", line)
251    if m:
252        type = m.group(3)
253        message = f"{m.group(1)}:{m.group(2)}: {m.group(4)}"
254        if type == "error":
255            logger.error(message)
256        elif type == "warning":
257            logger.warning(message)
258        else:
259            logger.info(message)
260    elif not silent:
261        logger.info(line)
262
263
264def run_doxygen(doxygen: str, doxyfile: str, silent: bool = False) -> None:
265    """Run Doxygen build.
266
267    Args:
268        doxygen: Path to Doxygen binary.
269        doxyfile: Doxyfile content.
270        silent: If Doxygen output should be logged or not.
271    """
272
273    f_doxyfile = tempfile.NamedTemporaryFile("w", delete=False)
274    f_doxyfile.write(doxyfile)
275    f_doxyfile.close()
276
277    p = Popen([doxygen, f_doxyfile.name], stdout=PIPE, stderr=STDOUT, encoding="utf-8")
278    while True:
279        line = p.stdout.readline()  # type: ignore
280        if line:
281            process_doxygen_output(line.rstrip(), silent)
282        if p.poll() is not None:
283            break
284
285    Path(f_doxyfile.name).unlink()
286
287    if p.returncode:
288        raise IOError(f"Doxygen process returned non-zero ({p.returncode})")
289
290
291def sync_doxygen(doxyfile: str, new: Path, prev: Path) -> None:
292    """Synchronize Doxygen output with a previous build.
293
294    This function makes sure that only new, deleted or changed files are
295    actually modified in the Doxygen XML output. Latest HTML content is just
296    moved.
297
298    Args:
299        doxyfile: Contents of the Doxyfile.
300        new: Newest Doxygen build output directory.
301        prev: Previous Doxygen build output directory.
302    """
303
304    generate_html = get_doxygen_option(doxyfile, "GENERATE_HTML")
305    if generate_html[0] == "YES":
306        html_output = get_doxygen_option(doxyfile, "HTML_OUTPUT")
307        if not html_output:
308            raise ValueError("No HTML_OUTPUT set in Doxyfile")
309
310        new_htmldir = new / html_output[0]
311        prev_htmldir = prev / html_output[0]
312
313        if prev_htmldir.exists():
314            shutil.rmtree(prev_htmldir)
315        new_htmldir.rename(prev_htmldir)
316
317    xml_output = get_doxygen_option(doxyfile, "XML_OUTPUT")
318    if not xml_output:
319        raise ValueError("No XML_OUTPUT set in Doxyfile")
320
321    new_xmldir = new / xml_output[0]
322    prev_xmldir = prev / xml_output[0]
323
324    if prev_xmldir.exists():
325        dcmp = filecmp.dircmp(new_xmldir, prev_xmldir)
326
327        for file in dcmp.right_only:
328            (Path(dcmp.right) / file).unlink()
329
330        for file in dcmp.left_only + dcmp.diff_files:
331            shutil.copy(Path(dcmp.left) / file, Path(dcmp.right) / file)
332
333        shutil.rmtree(new_xmldir)
334    else:
335        new_xmldir.rename(prev_xmldir)
336
337
338def doxygen_build(app: Sphinx) -> None:
339    """Doxyrunner entry point.
340
341    Args:
342        app: Sphinx application instance.
343    """
344
345    if app.config.doxyrunner_outdir:
346        outdir = Path(app.config.doxyrunner_outdir)
347    else:
348        outdir = Path(app.outdir) / "_doxygen"
349
350    outdir.mkdir(exist_ok=True)
351    tmp_outdir = outdir / "tmp"
352
353    logger.info("Preparing Doxyfile...")
354    doxyfile = process_doxyfile(
355        app.config.doxyrunner_doxyfile,
356        tmp_outdir,
357        app.config.doxyrunner_silent,
358        app.config.doxyrunner_fmt,
359        app.config.doxyrunner_fmt_pattern,
360        app.config.doxyrunner_fmt_vars,
361        app.config.doxyrunner_outdir_var,
362    )
363
364    logger.info("Checking if Doxygen needs to be run...")
365    changed = doxygen_input_has_changed(app.env, doxyfile)
366    if not changed:
367        logger.info("Doxygen build will be skipped (no changes)!")
368        return
369
370    logger.info("Running Doxygen...")
371    run_doxygen(
372        app.config.doxyrunner_doxygen,
373        doxyfile,
374        app.config.doxyrunner_silent,
375    )
376
377    logger.info("Syncing Doxygen output...")
378    sync_doxygen(doxyfile, tmp_outdir, outdir)
379
380    shutil.rmtree(tmp_outdir)
381
382
383def setup(app: Sphinx) -> Dict[str, Any]:
384    app.add_config_value("doxyrunner_doxygen", "doxygen", "env")
385    app.add_config_value("doxyrunner_doxyfile", None, "env")
386    app.add_config_value("doxyrunner_outdir", None, "env")
387    app.add_config_value("doxyrunner_outdir_var", None, "env")
388    app.add_config_value("doxyrunner_fmt", False, "env")
389    app.add_config_value("doxyrunner_fmt_vars", {}, "env")
390    app.add_config_value("doxyrunner_fmt_pattern", "@{}@", "env")
391    app.add_config_value("doxyrunner_silent", True, "")
392
393    app.connect("builder-inited", doxygen_build)
394
395    return {
396        "version": __version__,
397        "parallel_read_safe": True,
398        "parallel_write_safe": True,
399    }
400