1#!/usr/bin/env python3
2
3# Copyright 2023 Nordic Semiconductor ASA
4# SPDX-License-Identifier: Apache-2.0
5
6from collections import defaultdict
7from dataclasses import dataclass
8from pathlib import Path
9from typing import Any, Callable, Dict, List, Optional, Set, Union
10import argparse
11import contextlib
12import glob
13import os
14import subprocess
15import sys
16import tempfile
17
18# TODO: include changes to child bindings
19
20HERE = Path(__file__).parent.resolve()
21ZEPHYR_BASE = HERE.parent.parent
22SCRIPTS = ZEPHYR_BASE / 'scripts'
23
24sys.path.insert(0, str(SCRIPTS / 'dts' / 'python-devicetree' / 'src'))
25
26from devicetree.edtlib import Binding, bindings_from_paths, load_vendor_prefixes_txt
27
28# The Compat type is a (compatible, on_bus) pair, which is used as a
29# lookup key for bindings. The name "compat" matches edtlib's internal
30# variable for this; it's a bit of a misnomer, but let's be
31# consistent.
32@dataclass
33class Compat:
34    compatible: str
35    on_bus: Optional[str]
36
37    def __hash__(self):
38        return hash((self.compatible, self.on_bus))
39
40class BindingChange:
41    '''Marker type for an individual change that happened to a
42    binding between the start and end commits. See subclasses
43    below for concrete changes.
44    '''
45
46Compat2Binding = Dict[Compat, Binding]
47Binding2Changes = Dict[Binding, List[BindingChange]]
48
49@dataclass
50class Changes:
51    '''Container for all the changes that happened between the
52    start and end commits.'''
53
54    vnds: List[str]
55    vnd2added: Dict[str, Compat2Binding]
56    vnd2removed: Dict[str, Compat2Binding]
57    vnd2changes: Dict[str, Binding2Changes]
58
59@dataclass
60class ModifiedSpecifier2Cells(BindingChange):
61    space: str
62    start: List[str]
63    end: List[str]
64
65@dataclass
66class ModifiedBuses(BindingChange):
67    start: List[str]
68    end: List[str]
69
70@dataclass
71class AddedProperty(BindingChange):
72    property: str
73
74@dataclass
75class RemovedProperty(BindingChange):
76    property: str
77
78@dataclass
79class ModifiedPropertyType(BindingChange):
80    property: str
81    start: str
82    end: str
83
84@dataclass
85class ModifiedPropertyEnum(BindingChange):
86    property: str
87    start: Any
88    end: Any
89
90@dataclass
91class ModifiedPropertyConst(BindingChange):
92    property: str
93    start: Any
94    end: Any
95
96@dataclass
97class ModifiedPropertyDefault(BindingChange):
98    property: str
99    start: Any
100    end: Any
101
102@dataclass
103class ModifiedPropertyDeprecated(BindingChange):
104    property: str
105    start: bool
106    end: bool
107
108@dataclass
109class ModifiedPropertyRequired(BindingChange):
110    property: str
111    start: bool
112    end: bool
113
114def get_changes_between(
115        compat2binding_start: Compat2Binding,
116        compat2binding_end: Compat2Binding
117) -> Changes:
118    vnd2added: Dict[str, Compat2Binding] = \
119        group_compat2binding_by_vnd({
120            compat: compat2binding_end[compat]
121            for compat in compat2binding_end
122            if compat not in compat2binding_start
123        })
124
125    vnd2removed: Dict[str, Compat2Binding] = \
126        group_compat2binding_by_vnd({
127            compat: compat2binding_start[compat]
128            for compat in compat2binding_start
129            if compat not in compat2binding_end
130        })
131
132    vnd2changes = group_binding2changes_by_vnd(
133        get_binding2changes(compat2binding_start,
134                            compat2binding_end))
135
136    vnds_set: Set[str] = set()
137    vnds_set.update(set(vnd2added.keys()),
138                    set(vnd2removed.keys()),
139                    set(vnd2changes.keys()))
140
141    return Changes(vnds=sorted(vnds_set),
142                   vnd2added=vnd2added,
143                   vnd2removed=vnd2removed,
144                   vnd2changes=vnd2changes)
145
146def group_compat2binding_by_vnd(
147        compat2binding: Compat2Binding
148) -> Dict[str, Compat2Binding]:
149    '''Convert *compat2binding* to a dict mapping vendor prefixes
150    to the subset of *compat2binding* with that vendor prefix.'''
151    ret: Dict[str, Compat2Binding] = defaultdict(dict)
152
153    for compat, binding in compat2binding.items():
154        ret[get_vnd(binding.compatible)][compat] = binding
155
156    return ret
157
158def group_binding2changes_by_vnd(
159        binding2changes: Binding2Changes
160) -> Dict[str, Binding2Changes]:
161    '''Convert *binding2chages* to a dict mapping vendor prefixes
162    to the subset of *binding2changes* with that vendor prefix.'''
163    ret: Dict[str, Binding2Changes] = defaultdict(dict)
164
165    for binding, changes in binding2changes.items():
166        ret[get_vnd(binding.compatible)][binding] = changes
167
168    return ret
169
170def get_vnd(compatible: str) -> str:
171    '''Return the vendor prefix or the empty string.'''
172    if ',' not in compatible:
173        return ''
174
175    return compatible.split(',')[0]
176
177def get_binding2changes(
178        compat2binding_start: Compat2Binding,
179        compat2binding_end: Compat2Binding
180) -> Binding2Changes:
181    ret: Binding2Changes = {}
182
183    for compat, binding in compat2binding_end.items():
184        if compat not in compat2binding_start:
185            continue
186
187        binding_start = compat2binding_start[compat]
188        binding_end = compat2binding_end[compat]
189
190        binding_changes: List[BindingChange] = \
191            get_binding_changes(binding_start, binding_end)
192        if binding_changes:
193            ret[binding] = binding_changes
194
195    return ret
196
197def get_binding_changes(
198        binding_start: Binding,
199        binding_end: Binding
200) -> List[BindingChange]:
201    '''Enumerate the changes to a binding given its start and end values.'''
202    ret: List[BindingChange] = []
203
204    assert binding_start.compatible == binding_end.compatible
205    assert binding_start.on_bus == binding_end.on_bus
206
207    common_props: Set[str] = set(binding_start.prop2specs).intersection(
208        set(binding_end.prop2specs))
209
210    ret.extend(get_modified_specifier2cells(binding_start, binding_end))
211    ret.extend(get_modified_buses(binding_start, binding_end))
212    ret.extend(get_added_properties(binding_start, binding_end))
213    ret.extend(get_removed_properties(binding_start, binding_end))
214    ret.extend(get_modified_property_type(binding_start, binding_end,
215                                          common_props))
216    ret.extend(get_modified_property_enum(binding_start, binding_end,
217                                          common_props))
218    ret.extend(get_modified_property_const(binding_start, binding_end,
219                                           common_props))
220    ret.extend(get_modified_property_default(binding_start, binding_end,
221                                             common_props))
222    ret.extend(get_modified_property_deprecated(binding_start, binding_end,
223                                                common_props))
224    ret.extend(get_modified_property_required(binding_start, binding_end,
225                                              common_props))
226
227    return ret
228
229def get_modified_specifier2cells(
230        binding_start: Binding,
231        binding_end: Binding
232) -> List[BindingChange]:
233    ret: List[BindingChange] = []
234    start = binding_start.specifier2cells
235    end = binding_end.specifier2cells
236
237    if start == end:
238        return []
239
240    for space, cells_end in end.items():
241        cells_start = start.get(space)
242        if cells_start != cells_end:
243            ret.append(ModifiedSpecifier2Cells(space,
244                                               start=cells_start,
245                                               end=cells_end))
246    for space, cells_start in start.items():
247        if space not in end:
248            ret.append(ModifiedSpecifier2Cells(space,
249                                               start=cells_start,
250                                               end=None))
251
252    return ret
253
254def get_modified_buses(
255        binding_start: Binding,
256        binding_end: Binding
257) -> List[BindingChange]:
258    start = binding_start.buses
259    end = binding_end.buses
260
261    if start == end:
262        return []
263
264    return [ModifiedBuses(start=start, end=end)]
265
266def get_added_properties(
267        binding_start: Binding,
268        binding_end: Binding
269) -> List[BindingChange]:
270    return [AddedProperty(prop) for prop in binding_end.prop2specs
271            if prop not in binding_start.prop2specs]
272
273def get_removed_properties(
274        binding_start: Binding,
275        binding_end: Binding
276) -> List[BindingChange]:
277    return [RemovedProperty(prop) for prop in binding_start.prop2specs
278            if prop not in binding_end.prop2specs]
279
280def get_modified_property_type(
281        binding_start: Binding,
282        binding_end: Binding,
283        common_props: Set[str]
284) -> List[BindingChange]:
285    return get_modified_property_helper(
286        common_props,
287        lambda prop: binding_start.prop2specs[prop].type,
288        lambda prop: binding_end.prop2specs[prop].type,
289        ModifiedPropertyType)
290
291def get_modified_property_enum(
292        binding_start: Binding,
293        binding_end: Binding,
294        common_props: Set[str]
295) -> List[BindingChange]:
296    return get_modified_property_helper(
297        common_props,
298        lambda prop: binding_start.prop2specs[prop].enum,
299        lambda prop: binding_end.prop2specs[prop].enum,
300        ModifiedPropertyEnum)
301
302def get_modified_property_const(
303        binding_start: Binding,
304        binding_end: Binding,
305        common_props: Set[str]
306) -> List[BindingChange]:
307    return get_modified_property_helper(
308        common_props,
309        lambda prop: binding_start.prop2specs[prop].const,
310        lambda prop: binding_end.prop2specs[prop].const,
311        ModifiedPropertyConst)
312
313def get_modified_property_default(
314        binding_start: Binding,
315        binding_end: Binding,
316        common_props: Set[str]
317) -> List[BindingChange]:
318    return get_modified_property_helper(
319        common_props,
320        lambda prop: binding_start.prop2specs[prop].default,
321        lambda prop: binding_end.prop2specs[prop].default,
322        ModifiedPropertyDefault)
323
324def get_modified_property_deprecated(
325        binding_start: Binding,
326        binding_end: Binding,
327        common_props: Set[str]
328) -> List[BindingChange]:
329    return get_modified_property_helper(
330        common_props,
331        lambda prop: binding_start.prop2specs[prop].deprecated,
332        lambda prop: binding_end.prop2specs[prop].deprecated,
333        ModifiedPropertyDeprecated)
334
335def get_modified_property_required(
336        binding_start: Binding,
337        binding_end: Binding,
338        common_props: Set[str]
339) -> List[BindingChange]:
340    return get_modified_property_helper(
341        common_props,
342        lambda prop: binding_start.prop2specs[prop].required,
343        lambda prop: binding_end.prop2specs[prop].required,
344        ModifiedPropertyRequired)
345
346def get_modified_property_helper(
347        common_props: Set[str],
348        start_fn: Callable[[str], Any],
349        end_fn: Callable[[str], Any],
350        change_constructor: Callable[[str, Any, Any], BindingChange]
351) -> List[BindingChange]:
352
353    ret = []
354    for prop in common_props:
355        start = start_fn(prop)
356        end = end_fn(prop)
357        if start != end:
358            ret.append(change_constructor(prop, start, end))
359    return ret
360
361def load_compat2binding(commit: str) -> Compat2Binding:
362    '''Load a map from compatible to binding with that compatible,
363    based on the bindings in zephyr at the given commit.'''
364
365    @contextlib.contextmanager
366    def git_worktree(directory: os.PathLike, commit: str):
367        fspath = os.fspath(directory)
368        subprocess.run(['git', 'worktree', 'add', '--detach', fspath, commit],
369                       check=True)
370        yield
371        print('removing worktree...')
372        subprocess.run(['git', 'worktree', 'remove', fspath], check=True)
373
374    ret: Compat2Binding = {}
375    with tempfile.TemporaryDirectory(prefix='dt_bindings_worktree') as tmpdir:
376        with git_worktree(tmpdir, commit):
377            tmpdir_bindings = Path(tmpdir) / 'dts' / 'bindings'
378            binding_files = []
379            binding_files.extend(glob.glob(f'{tmpdir_bindings}/**/*.yml',
380                                           recursive=True))
381            binding_files.extend(glob.glob(f'{tmpdir_bindings}/**/*.yaml',
382                                           recursive=True))
383            bindings: List[Binding] = bindings_from_paths(
384                binding_files, ignore_errors=True)
385            for binding in bindings:
386                compat = Compat(binding.compatible, binding.on_bus)
387                assert compat not in ret
388                ret[compat] = binding
389
390    return ret
391
392def compatible_sort_key(data: Union[Compat, Binding]) -> str:
393    '''Sort key used by Printer.'''
394    return (data.compatible, data.on_bus or '')
395
396class Printer:
397    '''Helper class for formatting output.'''
398
399    def __init__(self, outfile):
400        self.outfile = outfile
401        self.vnd2vendor_name = load_vendor_prefixes_txt(
402            ZEPHYR_BASE / 'dts' / 'bindings' / 'vendor-prefixes.txt')
403
404    def print(self, *args, **kwargs):
405        kwargs['file'] = self.outfile
406        print(*args, **kwargs)
407
408    def print_changes(self, changes: Changes):
409        for vnd in changes.vnds:
410            if vnd:
411                vnd_fmt = f' ({vnd})'
412            else:
413                vnd_fmt = ''
414            self.print(f'* {self.vendor_name(vnd)}{vnd_fmt}:\n')
415
416            added = changes.vnd2added[vnd]
417            if added:
418                self.print('  * New bindings:\n')
419                self.print_compat2binding(
420                    added,
421                    lambda binding: f':dtcompatible:`{binding.compatible}`'
422                )
423
424            removed = changes.vnd2removed[vnd]
425            if removed:
426                self.print('  * Removed bindings:\n')
427                self.print_compat2binding(
428                    removed,
429                    lambda binding: f'``{binding.compatible}``'
430                )
431
432            modified = changes.vnd2changes[vnd]
433            if modified:
434                self.print('  * Modified bindings:\n')
435                self.print_binding2changes(modified)
436
437    def print_compat2binding(
438            self,
439            compat2binding: Compat2Binding,
440            formatter: Callable[[Binding], str]
441    ) -> None:
442        for compat in sorted(compat2binding, key=compatible_sort_key):
443            self.print(f'    * {formatter(compat2binding[compat])}')
444        self.print()
445
446    def print_binding2changes(self, binding2changes: Binding2Changes) -> None:
447        for binding, changes in binding2changes.items():
448            on_bus = f' (on {binding.on_bus} bus)' if binding.on_bus else ''
449            self.print(f'    * :dtcompatible:`{binding.compatible}`{on_bus}:\n')
450            for change in changes:
451                self.print_change(change)
452            self.print()
453
454    def print_change(self, change: BindingChange) -> None:
455        def print(msg):
456            self.print(f'          * {msg}')
457        def print_prop_change(details):
458            print(f'property ``{change.property}`` {details} changed from '
459                  f'{change.start} to {change.end}')
460        if isinstance(change, ModifiedSpecifier2Cells):
461            print(f'specifier cells for space "{change.space}" '
462                  f'are now named: {change.end} (old value: {change.start})')
463        elif isinstance(change, ModifiedBuses):
464            print(f'bus list changed from {change.start} to {change.end}')
465        elif isinstance(change, AddedProperty):
466            print(f'new property: ``{change.property}``')
467        elif isinstance(change, RemovedProperty):
468            print(f'removed property: ``{change.property}``')
469        elif isinstance(change, ModifiedPropertyType):
470            print_prop_change('type')
471        elif isinstance(change, ModifiedPropertyEnum):
472            print_prop_change('enum value')
473        elif isinstance(change, ModifiedPropertyConst):
474            print_prop_change('const value')
475        elif isinstance(change, ModifiedPropertyDefault):
476            print_prop_change('default value')
477        elif isinstance(change, ModifiedPropertyDeprecated):
478            print_prop_change('deprecation status')
479        elif isinstance(change, ModifiedPropertyRequired):
480            if not change.start and change.end:
481                print(f'property ``{change.property}`` is now required')
482            else:
483                print(f'property ``{change.property}`` is no longer required')
484        else:
485            raise ValueError(f'unknown type for {change}: {type(change)}')
486
487    def vendor_name(self, vnd: str) -> str:
488        # Necessary due to the patch for openthread.
489
490        if vnd == 'openthread':
491            # FIXME: we have to go beyond the dict since this
492            # compatible isn't in vendor-prefixes.txt, but we have
493            # binding(s) for it. We need to fix this in CI by
494            # rejecting unknown vendors in a bindings check.
495            return 'OpenThread'
496        if vnd == '':
497            return 'Generic or vendor-independent'
498        return self.vnd2vendor_name[vnd]
499
500def parse_args() -> argparse.Namespace:
501    parser = argparse.ArgumentParser(
502        allow_abbrev=False,
503        description='''
504Print human-readable descriptions of changes to devicetree
505bindings between two commits, in .rst format suitable for copy/pasting
506into the release notes.
507''',
508        formatter_class=argparse.RawDescriptionHelpFormatter
509    )
510    parser.add_argument('start', metavar='START-COMMIT',
511                        help='''what you want to compare bindings against
512                        (typically the previous release's tag)''')
513    parser.add_argument('end', metavar='END-COMMIT',
514                        help='''what you want to know bindings changes for
515                        (typically 'main')''')
516    parser.add_argument('file', help='where to write the .rst output to')
517    return parser.parse_args()
518
519def main():
520    args = parse_args()
521
522    compat2binding_start = load_compat2binding(args.start)
523    compat2binding_end = load_compat2binding(args.end)
524    changes = get_changes_between(compat2binding_start,
525                                  compat2binding_end)
526
527    with open(args.file, 'w') as outfile:
528        Printer(outfile).print_changes(changes)
529
530if __name__ == '__main__':
531    main()
532