1# Copyright (c) 2024-2025 The Linux Foundation
2# SPDX-License-Identifier: Apache-2.0
3
4import logging
5import os
6import pickle
7import re
8import subprocess
9import sys
10from collections import namedtuple
11from pathlib import Path
12
13import list_boards
14import list_hardware
15import yaml
16import zephyr_module
17from gen_devicetree_rest import VndLookup
18
19ZEPHYR_BASE = Path(__file__).parents[2]
20ZEPHYR_BINDINGS = ZEPHYR_BASE / "dts/bindings"
21EDT_PICKLE_PATH = "zephyr/edt.pickle"
22
23logger = logging.getLogger(__name__)
24
25
26class DeviceTreeUtils:
27    _compat_description_cache = {}
28
29    @classmethod
30    def get_first_sentence(cls, text):
31        """Extract the first sentence from a text block (typically a node description).
32
33        Args:
34            text: The text to extract the first sentence from.
35
36        Returns:
37            The first sentence found in the text, or the entire text if no sentence
38            boundary is found.
39        """
40        if not text:
41            return ""
42
43        text = text.replace('\n', ' ')
44        # Split by double spaces to get paragraphs
45        paragraphs = text.split('  ')
46        first_paragraph = paragraphs[0].strip()
47
48        # Look for a period followed by a space in the first paragraph
49        period_match = re.search(r'(.*?)\.(?:\s|$)', first_paragraph)
50        if period_match:
51            return period_match.group(1).strip()
52
53        # If no period in the first paragraph, return the entire first paragraph
54        return first_paragraph
55
56    @classmethod
57    def get_cached_description(cls, node):
58        """Get the cached description for a devicetree node.
59
60        Args:
61            node: A devicetree node object with matching_compat and description attributes.
62
63        Returns:
64            The cached description for the node's compatible, creating it if needed.
65        """
66        return cls._compat_description_cache.setdefault(
67            node.matching_compat,
68            cls.get_first_sentence(node.description)
69        )
70
71
72def guess_file_from_patterns(directory, patterns, name, extensions):
73    for pattern in patterns:
74        for ext in extensions:
75            matching_file = next(directory.glob(pattern.format(name=name, ext=ext)), None)
76            if matching_file:
77                return matching_file
78    return None
79
80
81def guess_image(board_or_shield):
82    img_exts = ["jpg", "jpeg", "webp", "png"]
83    patterns = [
84        "**/{name}.{ext}",
85        "**/*{name}*.{ext}",
86        "**/*.{ext}",
87    ]
88    img_file = guess_file_from_patterns(
89        board_or_shield.dir, patterns, board_or_shield.name, img_exts
90    )
91
92    return (img_file.relative_to(ZEPHYR_BASE)).as_posix() if img_file else None
93
94
95def guess_doc_page(board_or_shield):
96    patterns = [
97        "doc/index.{ext}",
98        "**/{name}.{ext}",
99        "**/*{name}*.{ext}",
100        "**/*.{ext}",
101    ]
102    doc_file = guess_file_from_patterns(
103        board_or_shield.dir, patterns, board_or_shield.name, ["rst"]
104    )
105    return doc_file
106
107
108def gather_board_devicetrees(twister_out_dir):
109    """Gather EDT objects for each board from twister output directory.
110
111    Args:
112        twister_out_dir: Path object pointing to twister output directory
113
114    Returns:
115        A dictionary mapping board names to a dictionary of board targets and their EDT objects.
116        The structure is: {board_name: {board_target: edt_object}}
117    """
118    board_devicetrees = {}
119
120    if not twister_out_dir.exists():
121        return board_devicetrees
122
123    # Find all build_info.yml files in twister-out
124    build_info_files = list(twister_out_dir.glob("*/**/build_info.yml"))
125
126    for build_info_file in build_info_files:
127        # Look for corresponding zephyr.dts
128        edt_pickle_file = build_info_file.parent / EDT_PICKLE_PATH
129        if not edt_pickle_file.exists():
130            continue
131
132        try:
133            with open(build_info_file) as f:
134                build_info = yaml.safe_load(f)
135                board_info = build_info.get('cmake', {}).get('board', {})
136                board_name = board_info.get('name')
137                qualifier = board_info.get('qualifiers', '')
138                revision = board_info.get('revision', '')
139
140                board_target = board_name
141                if qualifier:
142                    board_target = f"{board_name}/{qualifier}"
143                if revision:
144                    board_target = f"{board_target}@{revision}"
145
146                with open(edt_pickle_file, 'rb') as f:
147                    edt = pickle.load(f)
148                    board_devicetrees.setdefault(board_name, {})[board_target] = edt
149
150        except Exception as e:
151            logger.error(f"Error processing build info file {build_info_file}: {e}")
152
153    return board_devicetrees
154
155
156def run_twister_cmake_only(outdir):
157    """Run twister in cmake-only mode to generate build info files.
158
159    Args:
160        outdir: Directory where twister should output its files
161    """
162    twister_cmd = [
163        sys.executable,
164        f"{ZEPHYR_BASE}/scripts/twister",
165        "-T", "samples/hello_world/",
166        "--all",
167        "-M",
168        "--keep-artifacts", "zephyr/edt.pickle",
169        "--cmake-only",
170        "--outdir", str(outdir),
171    ]
172
173    minimal_env = {
174        'PATH': os.environ.get('PATH', ''),
175        'ZEPHYR_BASE': str(ZEPHYR_BASE),
176        'HOME': os.environ.get('HOME', ''),
177        'PYTHONPATH': os.environ.get('PYTHONPATH', '')
178    }
179
180    try:
181        subprocess.run(twister_cmd, check=True, cwd=ZEPHYR_BASE, env=minimal_env)
182    except subprocess.CalledProcessError as e:
183        logger.warning(f"Failed to run Twister, list of hw features might be incomplete.\n{e}")
184
185
186def get_catalog(generate_hw_features=False):
187    """Get the board catalog.
188
189    Args:
190        generate_hw_features: If True, run twister to generate hardware features information.
191    """
192    import tempfile
193
194    vnd_lookup = VndLookup(ZEPHYR_BASE / "dts/bindings/vendor-prefixes.txt", [])
195
196    module_settings = {
197        "arch_root": [ZEPHYR_BASE],
198        "board_root": [ZEPHYR_BASE],
199        "soc_root": [ZEPHYR_BASE],
200    }
201
202    for module in zephyr_module.parse_modules(ZEPHYR_BASE):
203        for key in module_settings:
204            root = module.meta.get("build", {}).get("settings", {}).get(key)
205            if root is not None:
206                module_settings[key].append(Path(module.project) / root)
207
208    Args = namedtuple("args", ["arch_roots", "board_roots", "soc_roots", "board_dir", "board"])
209    args_find_boards = Args(
210        arch_roots=module_settings["arch_root"],
211        board_roots=module_settings["board_root"],
212        soc_roots=module_settings["soc_root"],
213        board_dir=[],
214        board=None,
215    )
216
217    boards = list_boards.find_v2_boards(args_find_boards)
218    systems = list_hardware.find_v2_systems(args_find_boards)
219    board_catalog = {}
220    board_devicetrees = {}
221
222    if generate_hw_features:
223        logger.info("Running twister in cmake-only mode to get Devicetree files for all boards")
224        with tempfile.TemporaryDirectory() as tmp_dir:
225            run_twister_cmake_only(tmp_dir)
226            board_devicetrees = gather_board_devicetrees(Path(tmp_dir))
227    else:
228        logger.info("Skipping generation of supported hardware features.")
229
230    for board in boards.values():
231        # We could use board.vendor but it is often incorrect. Instead, deduce vendor from
232        # containing folder. There are a few exceptions, like the "native" and "others" folders
233        # which we know are not actual vendors so treat them as such.
234        for folder in board.dir.parents:
235            if folder.name in ["native", "others"]:
236                vendor = "others"
237                break
238            elif vnd_lookup.vnd2vendor.get(folder.name):
239                vendor = folder.name
240                break
241
242        socs = {soc.name for soc in board.socs}
243        full_name = board.full_name or board.name
244        doc_page = guess_doc_page(board)
245
246        supported_features = {}
247
248        # Use pre-gathered build info and DTS files
249        if board.name in board_devicetrees:
250            for board_target, edt in board_devicetrees[board.name].items():
251                features = {}
252                for node in edt.nodes:
253                    if node.binding_path is None:
254                        continue
255
256                    binding_path = Path(node.binding_path)
257                    is_custom_binding = False
258                    if binding_path.is_relative_to(ZEPHYR_BINDINGS):
259                        binding_type = binding_path.relative_to(ZEPHYR_BINDINGS).parts[0]
260                    else:
261                        binding_type = "misc"
262                        is_custom_binding = True
263
264
265                    if node.matching_compat is None:
266                        continue
267
268                    # skip "zephyr,xxx" compatibles
269                    if node.matching_compat.startswith("zephyr,"):
270                        continue
271
272                    description = DeviceTreeUtils.get_cached_description(node)
273                    filename = node.filename
274                    lineno = node.lineno
275                    locations = set()
276                    if Path(filename).is_relative_to(ZEPHYR_BASE):
277                        filename = Path(filename).relative_to(ZEPHYR_BASE)
278                        if filename.parts[0] == "boards":
279                            locations.add("board")
280                        else:
281                            locations.add("soc")
282
283                    existing_feature = features.get(binding_type, {}).get(
284                        node.matching_compat
285                    )
286
287                    node_info = {"filename": str(filename), "lineno": lineno}
288                    node_list_key = "okay_nodes" if node.status == "okay" else "disabled_nodes"
289
290                    if existing_feature:
291                        locations.update(existing_feature["locations"])
292                        existing_feature.setdefault(node_list_key, []).append(node_info)
293                        continue
294
295                    feature_data = {
296                        "description": description,
297                        "custom_binding": is_custom_binding,
298                        "locations": locations,
299                        "okay_nodes": [],
300                        "disabled_nodes": [],
301                    }
302                    feature_data[node_list_key].append(node_info)
303
304                    features.setdefault(binding_type, {})[node.matching_compat] = feature_data
305
306                # Store features for this specific target
307                supported_features[board_target] = features
308
309        # Grab all the twister files for this board and use them to figure out all the archs it
310        # supports.
311        archs = set()
312        pattern = f"{board.name}*.yaml"
313        for twister_file in board.dir.glob(pattern):
314            try:
315                with open(twister_file) as f:
316                    board_data = yaml.safe_load(f)
317                    archs.add(board_data.get("arch"))
318            except Exception as e:
319                logger.error(f"Error parsing twister file {twister_file}: {e}")
320
321        board_catalog[board.name] = {
322            "name": board.name,
323            "full_name": full_name,
324            "doc_page": doc_page.relative_to(ZEPHYR_BASE).as_posix() if doc_page else None,
325            "vendor": vendor,
326            "archs": list(archs),
327            "socs": list(socs),
328            "supported_features": supported_features,
329            "image": guess_image(board),
330        }
331
332    socs_hierarchy = {}
333    for soc in systems.get_socs():
334        family = soc.family or "<no family>"
335        series = soc.series or "<no series>"
336        socs_hierarchy.setdefault(family, {}).setdefault(series, []).append(soc.name)
337
338    return {
339        "boards": board_catalog,
340        "vendors": {**vnd_lookup.vnd2vendor, "others": "Other/Unknown"},
341        "socs": socs_hierarchy,
342    }
343