1"""
2Doxyrunner Sphinx Plugin
3########################
4
5Copyright (c) 2021 Nordic Semiconductor ASA
6SPDX-License-Identifier: Apache-2.0
7
8Introduction
9============
10
11This Sphinx plugin can be used to run Doxygen build as part of the Sphinx build
12process. It is meant to be used with other plugins such as ``breathe`` in order
13to improve the user experience. The principal features offered by this plugin
14are:
15
16- Doxygen build is run before Sphinx reads input files
17- Doxyfile can be optionally pre-processed so that variables can be inserted
18- Changes in the Doxygen input files are tracked so that Doxygen build is only
19  run if necessary.
20- Synchronizes Doxygen XML output so that even if Doxygen is run only changed,
21  deleted or added files are modified.
22
23References:
24
25-  https://github.com/michaeljones/breathe/issues/420
26
27Configuration options
28=====================
29
30- ``doxyrunner_doxygen``: Path to the Doxygen binary.
31- ``doxyrunner_doxyfile``: Path to Doxyfile.
32- ``doxyrunner_outdir``: Doxygen build output directory (inserted to
33  ``OUTPUT_DIRECTORY``)
34- ``doxyrunner_outdir_var``: Variable representing the Doxygen build output
35  directory, as used by ``OUTPUT_DIRECTORY``. This can be useful if other
36  Doxygen variables reference to the output directory.
37- ``doxyrunner_fmt``: Flag to indicate if Doxyfile should be formatted.
38- ``doxyrunner_fmt_vars``: Format variables dictionary (name: value).
39- ``doxyrunner_fmt_pattern``: Format pattern.
40- ``doxyrunner_silent``: If Doxygen output should be logged or not. Note that
41  this option may not have any effect if ``QUIET`` is set to ``YES``.
42"""
43
44import filecmp
45import hashlib
46import re
47import shlex
48import shutil
49import tempfile
50from pathlib import Path
51from subprocess import PIPE, STDOUT, Popen
52from typing import Any
53
54from sphinx.application import Sphinx
55from sphinx.environment import BuildEnvironment
56from sphinx.util import logging
57
58__version__ = "0.1.0"
59
60
61logger = logging.getLogger(__name__)
62
63
64def hash_file(file: Path) -> str:
65    """Compute the hash (SHA256) of a file in text mode.
66
67    Args:
68        file: File to be hashed.
69
70    Returns:
71        Hash.
72    """
73
74    with open(file, encoding="utf-8") as f:
75        sha256 = hashlib.sha256(f.read().encode("utf-8"))
76
77    return sha256.hexdigest()
78
79def get_doxygen_option(doxyfile: str, option: str) -> list[str]:
80    """Obtain the value of a Doxygen option.
81
82    Args:
83        doxyfile: Content of the Doxyfile.
84        option: Option to be retrieved.
85
86    Notes:
87        Does not support appended values.
88
89    Returns:
90        Option values.
91    """
92
93    option_re = re.compile(r"^\s*([A-Z0-9_]+)\s*=\s*(.*)$")
94    multiline_re = re.compile(r"^\s*(.*)$")
95
96    values = []
97    found = False
98    finished = False
99    for line in doxyfile.splitlines():
100        if not found:
101            m = option_re.match(line)
102            if not m or m.group(1) != option:
103                continue
104
105            found = True
106            value = m.group(2)
107        else:
108            m = multiline_re.match(line)
109            if not m:
110                raise ValueError(f"Unexpected line content: {line}")
111
112            value = m.group(1)
113
114        # check if it is a multiline value
115        finished = not value.endswith("\\")
116
117        # strip backslash
118        if not finished:
119            value = value[:-1]
120
121        # split values
122        values += shlex.split(value.replace("\\", "\\\\"))
123
124        if finished:
125            break
126
127    return values
128
129
130def process_doxyfile(
131    doxyfile: str,
132    outdir: Path,
133    silent: bool,
134    fmt: bool = False,
135    fmt_pattern: str | None = None,
136    fmt_vars: dict[str, str] | None = None,
137    outdir_var: str | None = None,
138) -> str:
139    """Process Doxyfile.
140
141    Notes:
142        OUTPUT_DIRECTORY, WARN_FORMAT and QUIET are overridden to satisfy
143        extension operation needs.
144
145    Args:
146        doxyfile: Path to the Doxyfile.
147        outdir: Output directory of the Doxygen build.
148        silent: If Doxygen should be run in quiet mode or not.
149        fmt: If Doxyfile should be formatted.
150        fmt_pattern: Format pattern.
151        fmt_vars: Format variables.
152        outdir_var: Variable representing output directory.
153
154     Returns:
155        Processed Doxyfile content.
156    """
157
158    with open(doxyfile) as f:
159        content = f.read()
160
161    content = re.sub(
162        r"^\s*OUTPUT_DIRECTORY\s*=.*$",
163        f"OUTPUT_DIRECTORY={outdir.as_posix()}",
164        content,
165        flags=re.MULTILINE,
166    )
167
168    content = re.sub(
169        r"^\s*WARN_FORMAT\s*=.*$",
170        'WARN_FORMAT="$file:$line: $text"',
171        content,
172        flags=re.MULTILINE,
173    )
174
175    content = re.sub(
176        r"^\s*QUIET\s*=.*$",
177        "QUIET=" + "YES" if silent else "NO",
178        content,
179        flags=re.MULTILINE,
180    )
181
182    if fmt:
183        if not fmt_pattern or not fmt_vars:
184            raise ValueError("Invalid formatting pattern or variables")
185
186        if outdir_var:
187            fmt_vars = fmt_vars.copy()
188            fmt_vars[outdir_var] = outdir.as_posix()
189
190        for var, value in fmt_vars.items():
191            content = content.replace(fmt_pattern.format(var), value)
192
193    return content
194
195
196def doxygen_input_has_changed(env: BuildEnvironment, doxyfile: str) -> bool:
197    """Check if Doxygen input files have changed.
198
199    Args:
200        env: Sphinx build environment instance.
201        doxyfile: Doxyfile content.
202
203    Returns:
204        True if changed, False otherwise.
205    """
206
207    # obtain Doxygen input files and patterns
208    input_files = get_doxygen_option(doxyfile, "INPUT")
209    if not input:
210        raise ValueError("No INPUT set in Doxyfile")
211
212    file_patterns = get_doxygen_option(doxyfile, "FILE_PATTERNS")
213    if not file_patterns:
214        raise ValueError("No FILE_PATTERNS set in Doxyfile")
215
216    # build a set with input files hash
217    cache = set()
218    for file in input_files:
219        path = Path(file)
220        if path.is_file():
221            cache.add(hash_file(path))
222        else:
223            for pattern in file_patterns:
224                for p_file in path.glob("**/" + pattern):
225                    cache.add(hash_file(p_file))
226
227    # check if any file has changed
228    if hasattr(env, "doxyrunner_cache") and env.doxyrunner_cache == cache:
229        return False
230
231    # store current state
232    env.doxyrunner_cache = cache
233
234    return True
235
236
237def process_doxygen_output(line: str, silent: bool) -> None:
238    """Process a line of Doxygen program output.
239
240    This function will map Doxygen output to the Sphinx logger output. Errors
241    and warnings will be converted to Sphinx errors and warnings. Other
242    messages, if not silent, will be mapped to the info logger channel.
243
244    Args:
245        line: Doxygen program line.
246        silent: True if regular messages should be logged, False otherwise.
247    """
248
249    m = re.match(r"(.*):(\d+): ([a-z]+): (.*)", line)
250    if m:
251        type = m.group(3)
252        message = f"{m.group(1)}:{m.group(2)}: {m.group(4)}"
253        if type == "error":
254            logger.error(message)
255        elif type == "warning":
256            logger.warning(message)
257        else:
258            logger.info(message)
259    elif not silent:
260        logger.info(line)
261
262
263def run_doxygen(doxygen: str, doxyfile: str, silent: bool = False) -> None:
264    """Run Doxygen build.
265
266    Args:
267        doxygen: Path to Doxygen binary.
268        doxyfile: Doxyfile content.
269        silent: If Doxygen output should be logged or not.
270    """
271
272    with tempfile.NamedTemporaryFile("w", delete=False) as f_doxyfile:
273        f_doxyfile.write(doxyfile)
274        f_doxyfile_name = f_doxyfile.name
275
276    p = Popen([doxygen, f_doxyfile_name], stdout=PIPE, stderr=STDOUT, encoding="utf-8")
277    while True:
278        line = p.stdout.readline()  # type: ignore
279        if line:
280            process_doxygen_output(line.rstrip(), silent)
281        if p.poll() is not None:
282            break
283
284    Path(f_doxyfile_name).unlink()
285
286    if p.returncode:
287        raise OSError(f"Doxygen process returned non-zero ({p.returncode})")
288
289
290def sync_doxygen(doxyfile: str, new: Path, prev: Path) -> None:
291    """Synchronize Doxygen output with a previous build.
292
293    This function makes sure that only new, deleted or changed files are
294    actually modified in the Doxygen XML output. Latest HTML content is just
295    moved.
296
297    Args:
298        doxyfile: Contents of the Doxyfile.
299        new: Newest Doxygen build output directory.
300        prev: Previous Doxygen build output directory.
301    """
302
303    generate_html = get_doxygen_option(doxyfile, "GENERATE_HTML")
304    if generate_html[0] == "YES":
305        html_output = get_doxygen_option(doxyfile, "HTML_OUTPUT")
306        if not html_output:
307            raise ValueError("No HTML_OUTPUT set in Doxyfile")
308
309        new_htmldir = new / html_output[0]
310        prev_htmldir = prev / html_output[0]
311
312        if prev_htmldir.exists():
313            shutil.rmtree(prev_htmldir)
314        new_htmldir.rename(prev_htmldir)
315
316    xml_output = get_doxygen_option(doxyfile, "XML_OUTPUT")
317    if not xml_output:
318        raise ValueError("No XML_OUTPUT set in Doxyfile")
319
320    new_xmldir = new / xml_output[0]
321    prev_xmldir = prev / xml_output[0]
322
323    if prev_xmldir.exists():
324        dcmp = filecmp.dircmp(new_xmldir, prev_xmldir)
325
326        for file in dcmp.right_only:
327            (Path(dcmp.right) / file).unlink()
328
329        for file in dcmp.left_only + dcmp.diff_files:
330            shutil.copy(Path(dcmp.left) / file, Path(dcmp.right) / file)
331
332        shutil.rmtree(new_xmldir)
333    else:
334        new_xmldir.rename(prev_xmldir)
335
336
337def doxygen_build(app: Sphinx) -> None:
338    """Doxyrunner entry point.
339
340    Args:
341        app: Sphinx application instance.
342    """
343
344    if app.config.doxyrunner_outdir:
345        outdir = Path(app.config.doxyrunner_outdir)
346    else:
347        outdir = Path(app.outdir) / "_doxygen"
348
349    outdir.mkdir(exist_ok=True)
350    tmp_outdir = outdir / "tmp"
351
352    logger.info("Preparing Doxyfile...")
353    doxyfile = process_doxyfile(
354        app.config.doxyrunner_doxyfile,
355        tmp_outdir,
356        app.config.doxyrunner_silent,
357        app.config.doxyrunner_fmt,
358        app.config.doxyrunner_fmt_pattern,
359        app.config.doxyrunner_fmt_vars,
360        app.config.doxyrunner_outdir_var,
361    )
362
363    logger.info("Checking if Doxygen needs to be run...")
364    app.env.doxygen_input_changed = doxygen_input_has_changed(app.env, doxyfile)
365    if not app.env.doxygen_input_changed:
366        logger.info("Doxygen build will be skipped (no changes)!")
367        return
368
369    logger.info("Running Doxygen...")
370    run_doxygen(
371        app.config.doxyrunner_doxygen,
372        doxyfile,
373        app.config.doxyrunner_silent,
374    )
375
376    logger.info("Syncing Doxygen output...")
377    sync_doxygen(doxyfile, tmp_outdir, outdir)
378
379    shutil.rmtree(tmp_outdir)
380
381
382def setup(app: Sphinx) -> dict[str, Any]:
383    app.add_config_value("doxyrunner_doxygen", "doxygen", "env")
384    app.add_config_value("doxyrunner_doxyfile", None, "env")
385    app.add_config_value("doxyrunner_outdir", None, "env")
386    app.add_config_value("doxyrunner_outdir_var", None, "env")
387    app.add_config_value("doxyrunner_fmt", False, "env")
388    app.add_config_value("doxyrunner_fmt_vars", {}, "env")
389    app.add_config_value("doxyrunner_fmt_pattern", "@{}@", "env")
390    app.add_config_value("doxyrunner_silent", True, "")
391
392    app.connect("builder-inited", doxygen_build)
393
394    return {
395        "version": __version__,
396        "parallel_read_safe": True,
397        "parallel_write_safe": True,
398    }
399