1#!/usr/bin/env python3
2#
3# Copyright (c) 2024 Raspberry Pi (Trading) Ltd.
4#
5# SPDX-License-Identifier: BSD-3-Clause
6#
7# Check Bazel build file source coverage. Reports files that:
8# - Are in the repo but not included in a BUILD.bazel file.
9# - Are referenced in a BUILD.bazel file but are not present.
10#
11# Usage:
12#   python tools/check_source_files_in_bazel_build.py
13#
14# Run from anywhere in the pico-sdk repo.
15
16import logging
17from pathlib import Path
18import shlex
19import subprocess
20from typing import (
21    Container,
22    Iterable,
23    List,
24    Optional,
25    Set,
26)
27import sys
28
29from bazel_common import (
30    SDK_ROOT,
31    bazel_command,
32    override_picotool_arg,
33    parse_common_args,
34    setup_logging,
35)
36
37_LOG = logging.getLogger(__file__)
38
39CPP_HEADER_EXTENSIONS = (
40    ".h",
41    ".hpp",
42    ".hxx",
43    ".h++",
44    ".hh",
45    ".H",
46)
47CPP_SOURCE_EXTENSIONS = (
48    ".c",
49    ".cpp",
50    ".cxx",
51    ".c++",
52    ".cc",
53    ".C",
54    ".S",
55    ".inc",
56    ".inl",
57)
58
59IGNORED_FILE_PATTERNS = (
60    # Doxygen only files.
61    "**/index.h",
62    "**/doc.h",
63)
64
65
66def get_paths_from_command(source_dir: Path, *args, **kwargs) -> Set[Path]:
67    """Runs a command and reads Bazel //-style paths from it."""
68    process = subprocess.run(
69        args, check=False, capture_output=True, cwd=source_dir, **kwargs
70    )
71
72    if process.returncode:
73        _LOG.error("Command invocation failed with return code %d!", process.returncode)
74        _LOG.error(
75            "Command: %s",
76            " ".join(shlex.quote(str(arg)) for arg in args),
77        )
78        _LOG.error(
79            "Output:\n%s",
80            process.stderr.decode(),
81        )
82        sys.exit(1)
83
84    files = set()
85
86    for line in process.stdout.splitlines():
87        path = line.strip().lstrip(b"/").replace(b":", b"/").decode()
88        files.add(Path(path))
89
90    return files
91
92
93def check_bazel_build_for_files(
94    bazel_extensions_to_check: Container[str],
95    files: Iterable[Path],
96    bazel_dirs: Iterable[Path],
97    picotool_dir: Optional[Path],
98) -> List[Path]:
99    """Checks that source files are in the Bazel builds.
100
101    Args:
102        bazel_extensions_to_check: which file suffixes to look for in Bazel
103        files: the files that should be checked
104        bazel_dirs: directories in which to run bazel query
105
106    Returns:
107        a list of missing files; will be empty if there were no missing files
108    """
109
110    # Collect all paths in the Bazel builds files.
111    bazel_build_source_files: Set[Path] = set()
112    pictool_override = override_picotool_arg(picotool_dir) if picotool_dir else ""
113    for directory in bazel_dirs:
114        bazel_build_source_files.update(
115            get_paths_from_command(
116                directory, bazel_command(), "query", pictool_override, 'kind("source file", //...:*)',
117            )
118        )
119    missing_from_bazel: List[Path] = []
120    referenced_in_bazel_missing: List[Path] = []
121
122    if not bazel_dirs:
123        _LOG.error("No bazel directories to check.")
124        raise RuntimeError
125
126    for path in (p for p in files if p.suffix in bazel_extensions_to_check):
127        if path not in bazel_build_source_files:
128            missing_from_bazel.append(path)
129
130    for path in (
131        p for p in bazel_build_source_files if p.suffix in bazel_extensions_to_check
132    ):
133        if path not in files:
134            referenced_in_bazel_missing.append(path)
135
136    if missing_from_bazel:
137        _LOG.warning(
138            "Files not included in the Bazel build:\n\n%s\n",
139            "\n".join("  " + str(x) for x in sorted(missing_from_bazel)),
140        )
141
142    if referenced_in_bazel_missing:
143        _LOG.warning(
144            "Files referenced in the Bazel build that are missing:\n\n%s\n",
145            "\n".join("  " + str(x) for x in sorted(referenced_in_bazel_missing)),
146        )
147
148    return missing_from_bazel + referenced_in_bazel_missing
149
150
151def git_ls_files_by_extension(file_suffixes: Iterable[str]) -> Iterable[Path]:
152    """List git source files.
153
154    Returns: A list of files matching the provided extensions.
155    """
156    git_command = ["git", "ls-files"]
157    for pattern in file_suffixes:
158        git_command.append("*" + pattern)
159
160    bazel_file_list = subprocess.run(
161        git_command,
162        cwd=SDK_ROOT,
163        text=True,
164        check=True,
165        capture_output=True,
166    ).stdout
167
168    bazel_files = [Path(f) for f in bazel_file_list.splitlines()]
169    return bazel_files
170
171
172def check_sources_in_bazel_build(picotool_dir) -> int:
173    # List files using git ls-files
174    all_source_files = git_ls_files_by_extension(
175        CPP_HEADER_EXTENSIONS + CPP_SOURCE_EXTENSIONS
176    )
177
178    # Filter out any unwanted files.
179    ignored_files = []
180    for source in all_source_files:
181        for pattern in IGNORED_FILE_PATTERNS:
182            if source.match(pattern):
183                ignored_files.append(source)
184    _LOG.debug(
185        "Ignoring files:\n\n%s\n", "\n".join("  " + str(f) for f in ignored_files)
186    )
187
188    source_files = list(set(all_source_files) - set(ignored_files))
189
190    # Check for missing files.
191    _LOG.info("Checking all source files are accounted for in Bazel.")
192    missing_files = check_bazel_build_for_files(
193        bazel_extensions_to_check=CPP_HEADER_EXTENSIONS + CPP_SOURCE_EXTENSIONS,
194        files=source_files,
195        bazel_dirs=[Path(SDK_ROOT)],
196        picotool_dir=picotool_dir,
197    )
198
199    if missing_files:
200        _LOG.error("Missing files found.")
201        return 1
202
203    _LOG.info("\x1b[32mSuccess!\x1b[0m All files accounted for in Bazel.")
204    return 0
205
206
207if __name__ == "__main__":
208    setup_logging()
209    args = parse_common_args()
210    sys.exit(check_sources_in_bazel_build(args.picotool_dir))
211