1"""
2Doxyrunner Sphinx Plugin
3########################
4
5Copyright (c) 2021 Nordic Semiconductor ASA
6SPDX-License-Identifier: Apache-2.0
7
8Introduction
9============
10
11This Sphinx plugin can be used to run Doxygen build as part of the Sphinx build
12process. It is meant to be used with other plugins such as ``breathe`` in order
13to improve the user experience. The principal features offered by this plugin
14are:
15
16- Doxygen build is run before Sphinx reads input files
17- Doxyfile can be optionally pre-processed so that variables can be inserted
18- Changes in the Doxygen input files are tracked so that Doxygen build is only
19  run if necessary.
20- Synchronizes Doxygen XML output so that even if Doxygen is run only changed,
21  deleted or added files are modified.
22
23References:
24
25-  https://github.com/michaeljones/breathe/issues/420
26
27Configuration options
28=====================
29
30- ``doxyrunner_doxygen``: Path to the Doxygen binary.
31- ``doxyrunner_silent``: If Doxygen output should be logged or not. Note that
32  this option may not have any effect if ``QUIET`` is set to ``YES``.
33- ``doxyrunner_projects``: Dictionary specifying projects, keys being project
34  name and values a dictionary with the following keys:
35
36  - ``doxyfile``: Path to Doxyfile.
37  - ``outdir``: Doxygen build output directory (inserted to ``OUTPUT_DIRECTORY``)
38  - ``outdir_var``: Variable representing the Doxygen build output directory,
39    as used by ``OUTPUT_DIRECTORY``. This can be useful if other Doxygen
40    variables reference to the output directory.
41  - ``fmt``: Flag to indicate if Doxyfile should be formatted.
42  - ``fmt_vars``: Format variables dictionary (name: value).
43  - ``fmt_pattern``: Format pattern.
44"""
45
46import filecmp
47import hashlib
48import re
49import shlex
50import shutil
51import tempfile
52from pathlib import Path
53from subprocess import PIPE, STDOUT, Popen
54from typing import Any
55
56from sphinx.application import Sphinx
57from sphinx.environment import BuildEnvironment
58from sphinx.util import logging
59
60__version__ = "0.1.0"
61
62
63logger = logging.getLogger(__name__)
64
65
66def hash_file(file: Path) -> str:
67    """Compute the hash (SHA256) of a file in text mode.
68
69    Args:
70        file: File to be hashed.
71
72    Returns:
73        Hash.
74    """
75
76    with open(file, encoding="utf-8") as f:
77        sha256 = hashlib.sha256(f.read().encode("utf-8"))
78
79    return sha256.hexdigest()
80
81def get_doxygen_option(doxyfile: str, option: str) -> list[str]:
82    """Obtain the value of a Doxygen option.
83
84    Args:
85        doxyfile: Content of the Doxyfile.
86        option: Option to be retrieved.
87
88    Notes:
89        Does not support appended values.
90
91    Returns:
92        Option values.
93    """
94
95    option_re = re.compile(r"^\s*([A-Z0-9_]+)\s*=\s*(.*)$")
96    multiline_re = re.compile(r"^\s*(.*)$")
97
98    values = []
99    found = False
100    finished = False
101    for line in doxyfile.splitlines():
102        if not found:
103            m = option_re.match(line)
104            if not m or m.group(1) != option:
105                continue
106
107            found = True
108            value = m.group(2)
109        else:
110            m = multiline_re.match(line)
111            if not m:
112                raise ValueError(f"Unexpected line content: {line}")
113
114            value = m.group(1)
115
116        # check if it is a multiline value
117        finished = not value.endswith("\\")
118
119        # strip backslash
120        if not finished:
121            value = value[:-1]
122
123        # split values
124        values += shlex.split(value.replace("\\", "\\\\"))
125
126        if finished:
127            break
128
129    return values
130
131
132def process_doxyfile(
133    doxyfile: str,
134    outdir: Path,
135    silent: bool,
136    fmt: bool = False,
137    fmt_pattern: str | None = None,
138    fmt_vars: dict[str, str] | None = None,
139    outdir_var: str | None = None,
140) -> str:
141    """Process Doxyfile.
142
143    Notes:
144        OUTPUT_DIRECTORY, WARN_FORMAT and QUIET are overridden to satisfy
145        extension operation needs.
146
147    Args:
148        doxyfile: Path to the Doxyfile.
149        outdir: Output directory of the Doxygen build.
150        silent: If Doxygen should be run in quiet mode or not.
151        fmt: If Doxyfile should be formatted.
152        fmt_pattern: Format pattern.
153        fmt_vars: Format variables.
154        outdir_var: Variable representing output directory.
155
156     Returns:
157        Processed Doxyfile content.
158    """
159
160    with open(doxyfile) as f:
161        content = f.read()
162
163    content = re.sub(
164        r"^\s*OUTPUT_DIRECTORY\s*=.*$",
165        f"OUTPUT_DIRECTORY={outdir.as_posix()}",
166        content,
167        flags=re.MULTILINE,
168    )
169
170    content = re.sub(
171        r"^\s*WARN_FORMAT\s*=.*$",
172        'WARN_FORMAT="$file:$line: $text"',
173        content,
174        flags=re.MULTILINE,
175    )
176
177    content = re.sub(
178        r"^\s*QUIET\s*=.*$",
179        "QUIET=" + "YES" if silent else "NO",
180        content,
181        flags=re.MULTILINE,
182    )
183
184    if fmt:
185        if not fmt_pattern or not fmt_vars:
186            raise ValueError("Invalid formatting pattern or variables")
187
188        if outdir_var:
189            fmt_vars = fmt_vars.copy()
190            fmt_vars[outdir_var] = outdir.as_posix()
191
192        for var, value in fmt_vars.items():
193            content = content.replace(fmt_pattern.format(var), value)
194
195    return content
196
197
198def doxygen_input_has_changed(env: BuildEnvironment, name, doxyfile: str) -> bool:
199    """Check if Doxygen input files have changed.
200
201    Args:
202        env: Sphinx build environment instance.
203        doxyfile: Doxyfile content.
204
205    Returns:
206        True if changed, False otherwise.
207    """
208
209    # obtain Doxygen input files and patterns
210    input_files = get_doxygen_option(doxyfile, "INPUT")
211    if not input:
212        raise ValueError("No INPUT set in Doxyfile")
213
214    file_patterns = get_doxygen_option(doxyfile, "FILE_PATTERNS")
215    if not file_patterns:
216        raise ValueError("No FILE_PATTERNS set in Doxyfile")
217
218    # build a set with input files hash
219    cache = set()
220    for file in input_files:
221        path = Path(file)
222        if path.is_file():
223            cache.add(hash_file(path))
224        else:
225            for pattern in file_patterns:
226                for p_file in path.glob("**/" + pattern):
227                    cache.add(hash_file(p_file))
228
229    if not hasattr(env, "doxyrunner_cache"):
230        env.doxyrunner_cache = dict()
231
232    # check if any file has changed
233    if env.doxyrunner_cache.get(name) == cache:
234        return False
235
236    # store current state
237    env.doxyrunner_cache[name] = cache
238
239    return True
240
241
242def process_doxygen_output(line: str, silent: bool) -> None:
243    """Process a line of Doxygen program output.
244
245    This function will map Doxygen output to the Sphinx logger output. Errors
246    and warnings will be converted to Sphinx errors and warnings. Other
247    messages, if not silent, will be mapped to the info logger channel.
248
249    Args:
250        line: Doxygen program line.
251        silent: True if regular messages should be logged, False otherwise.
252    """
253
254    m = re.match(r"(.*):(\d+): ([a-z]+): (.*)", line)
255    if m:
256        type = m.group(3)
257        message = f"{m.group(1)}:{m.group(2)}: {m.group(4)}"
258        if type == "error":
259            logger.error(message)
260        elif type == "warning":
261            logger.warning(message)
262        else:
263            logger.info(message)
264    elif not silent:
265        logger.info(line)
266
267
268def run_doxygen(doxygen: str, doxyfile: str, silent: bool = False) -> None:
269    """Run Doxygen build.
270
271    Args:
272        doxygen: Path to Doxygen binary.
273        doxyfile: Doxyfile content.
274        silent: If Doxygen output should be logged or not.
275    """
276
277    with tempfile.NamedTemporaryFile("w", delete=False) as f_doxyfile:
278        f_doxyfile.write(doxyfile)
279        f_doxyfile_name = f_doxyfile.name
280
281    p = Popen([doxygen, f_doxyfile_name], stdout=PIPE, stderr=STDOUT, encoding="utf-8")
282    while True:
283        line = p.stdout.readline()  # type: ignore
284        if line:
285            process_doxygen_output(line.rstrip(), silent)
286        if p.poll() is not None:
287            break
288
289    Path(f_doxyfile_name).unlink()
290
291    if p.returncode:
292        raise OSError(f"Doxygen process returned non-zero ({p.returncode})")
293
294
295def sync_doxygen(doxyfile: str, new: Path, prev: Path) -> None:
296    """Synchronize Doxygen output with a previous build.
297
298    This function makes sure that only new, deleted or changed files are
299    actually modified in the Doxygen XML output. Latest HTML content is just
300    moved.
301
302    Args:
303        doxyfile: Contents of the Doxyfile.
304        new: Newest Doxygen build output directory.
305        prev: Previous Doxygen build output directory.
306    """
307
308    generate_html = get_doxygen_option(doxyfile, "GENERATE_HTML")
309    if generate_html[0] == "YES":
310        html_output = get_doxygen_option(doxyfile, "HTML_OUTPUT")
311        if not html_output:
312            raise ValueError("No HTML_OUTPUT set in Doxyfile")
313
314        new_htmldir = new / html_output[0]
315        prev_htmldir = prev / html_output[0]
316
317        if prev_htmldir.exists():
318            shutil.rmtree(prev_htmldir)
319        new_htmldir.rename(prev_htmldir)
320
321    xml_output = get_doxygen_option(doxyfile, "XML_OUTPUT")
322    if not xml_output:
323        raise ValueError("No XML_OUTPUT set in Doxyfile")
324
325    new_xmldir = new / xml_output[0]
326    prev_xmldir = prev / xml_output[0]
327
328    if prev_xmldir.exists():
329        dcmp = filecmp.dircmp(new_xmldir, prev_xmldir)
330
331        for file in dcmp.right_only:
332            (Path(dcmp.right) / file).unlink()
333
334        for file in dcmp.left_only + dcmp.diff_files:
335            shutil.copy(Path(dcmp.left) / file, Path(dcmp.right) / file)
336
337        shutil.rmtree(new_xmldir)
338    else:
339        new_xmldir.rename(prev_xmldir)
340
341
342def doxygen_build(app: Sphinx) -> None:
343    """Doxyrunner entry point.
344
345    Args:
346        app: Sphinx application instance.
347    """
348
349    for name, config in app.config.doxyrunner_projects.items():
350        if config.get("outdir"):
351            outdir = Path(config["outdir"])
352        else:
353            outdir = Path(app.outdir) / "_doxygen" / name
354
355        outdir.mkdir(exist_ok=True)
356        tmp_outdir = outdir / "tmp"
357
358        logger.info("Preparing Doxyfile...")
359        doxyfile = process_doxyfile(
360            config["doxyfile"],
361            tmp_outdir,
362            app.config.doxyrunner_silent,
363            config.get("fmt", False),
364            config.get("fmt_pattern", "@{}@"),
365            config.get("fmt_vars", {}),
366            config.get("outdir_var"),
367        )
368
369        logger.info(f"Checking if Doxygen needs to be run for {name}...")
370        if not hasattr(app.env, "doxygen_input_changed"):
371            app.env.doxygen_input_changed = dict()
372
373        app.env.doxygen_input_changed[name] = doxygen_input_has_changed(app.env, name, doxyfile)
374        if not app.env.doxygen_input_changed[name]:
375            logger.info(f"Doxygen build for {name} will be skipped (no changes)!")
376            return
377
378        logger.info(f"Running Doxygen for {name}...")
379        run_doxygen(
380            app.config.doxyrunner_doxygen,
381            doxyfile,
382            app.config.doxyrunner_silent,
383        )
384
385        logger.info(f"Syncing Doxygen output for {name}...")
386        sync_doxygen(doxyfile, tmp_outdir, outdir)
387
388        shutil.rmtree(tmp_outdir)
389
390
391def setup(app: Sphinx) -> dict[str, Any]:
392    app.add_config_value("doxyrunner_doxygen", "doxygen", "env")
393    app.add_config_value("doxyrunner_silent", True, "")
394    app.add_config_value("doxyrunner_projects", {}, "")
395
396    app.connect("builder-inited", doxygen_build)
397
398    return {
399        "version": __version__,
400        "parallel_read_safe": True,
401        "parallel_write_safe": True,
402    }
403