1#!/usr/bin/env python
2#
3# Copyright 2018 Espressif Systems (Shanghai) PTE LTD
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17from __future__ import print_function, unicode_literals
18
19import argparse
20import os
21import re
22import sys
23from io import open
24
25from idf_ci_utils import IDF_PATH, get_submodule_dirs
26
27# regular expression for matching Kconfig files
28RE_KCONFIG = r'^Kconfig(\.projbuild)?(\.in)?$'
29
30# ouput file with suggestions will get this suffix
31OUTPUT_SUFFIX = '.new'
32
33# ignored directories (makes sense only when run on IDF_PATH)
34# Note: IGNORE_DIRS is a tuple in order to be able to use it directly with the startswith() built-in function which
35# accepts tuples but no lists.
36IGNORE_DIRS = (
37    # Kconfigs from submodules need to be ignored:
38    os.path.join(IDF_PATH, 'components', 'mqtt', 'esp-mqtt'),
39    # Test Kconfigs are also ignored
40    os.path.join(IDF_PATH, 'tools', 'ldgen', 'test', 'data'),
41    os.path.join(IDF_PATH, 'tools', 'kconfig_new', 'test'),
42)
43
44SPACES_PER_INDENT = 4
45
46CONFIG_NAME_MAX_LENGTH = 40
47
48CONFIG_NAME_MIN_PREFIX_LENGTH = 3
49
50# The checker will not fail if it encounters this string (it can be used for temporarily resolve conflicts)
51RE_NOERROR = re.compile(r'\s+#\s+NOERROR\s+$')
52
53# list or rules for lines
54LINE_ERROR_RULES = [
55    # (regular expression for finding,      error message,                                  correction)
56    (re.compile(r'\t'),                     'tabulators should be replaced by spaces',      r' ' * SPACES_PER_INDENT),
57    (re.compile(r'\s+\n'),                  'trailing whitespaces should be removed',       r'\n'),
58    (re.compile(r'.{120}'),                 'line should be shorter than 120 characters',   None),
59    # "\<CR><LF>" is not recognized due to a bug in tools/kconfig/zconf.l. The bug was fixed but the rebuild of
60    # mconf-idf is not enforced and an incorrect version is supplied with all previous IDF versions. Backslashes
61    # cannot be enabled unless everybody updates mconf-idf.
62    (re.compile(r'\\\n'),                   'line cannot be wrapped by backslash',          None),
63]
64
65
66class InputError(RuntimeError):
67    """
68    Represents and error on the input
69    """
70    def __init__(self, path, line_number, error_msg, suggested_line):
71        super(InputError, self).__init__('{}:{}: {}'.format(path, line_number, error_msg))
72        self.suggested_line = suggested_line
73
74
75class BaseChecker(object):
76    """
77    Base class for all checker objects
78    """
79    def __init__(self, path_in_idf):
80        self.path_in_idf = path_in_idf
81
82    def __enter__(self):
83        return self
84
85    def __exit__(self, type, value, traceback):
86        pass
87
88
89class SourceChecker(BaseChecker):
90    # allow to source only files which will be also checked by the script
91    # Note: The rules are complex and the LineRuleChecker cannot be used
92    def process_line(self, line, line_number):
93        m = re.search(r'^\s*source(\s*)"([^"]+)"', line)
94        if m:
95            if len(m.group(1)) == 0:
96                raise InputError(self.path_in_idf, line_number, '"source" has to been followed by space',
97                                 line.replace('source', 'source '))
98            path = m.group(2)
99            filename = os.path.basename(path)
100            if path in ['$COMPONENT_KCONFIGS_SOURCE_FILE', '$COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE']:
101                pass
102            elif not filename.startswith('Kconfig.'):
103                raise InputError(self.path_in_idf, line_number, 'only filenames starting with Kconfig.* can be sourced',
104                                 line.replace(path, os.path.join(os.path.dirname(path), 'Kconfig.' + filename)))
105
106
107class LineRuleChecker(BaseChecker):
108    """
109    checks LINE_ERROR_RULES for each line
110    """
111    def process_line(self, line, line_number):
112        suppress_errors = RE_NOERROR.search(line) is not None
113        errors = []
114        for rule in LINE_ERROR_RULES:
115            m = rule[0].search(line)
116            if m:
117                if suppress_errors:
118                    # just print but no failure
119                    e = InputError(self.path_in_idf, line_number, rule[1], line)
120                    print(e)
121                else:
122                    errors.append(rule[1])
123                if rule[2]:
124                    line = rule[0].sub(rule[2], line)
125        if len(errors) > 0:
126            raise InputError(self.path_in_idf, line_number, '; '.join(errors), line)
127
128
129class IndentAndNameChecker(BaseChecker):
130    """
131    checks the indentation of each line and configuration names
132    """
133    def __init__(self, path_in_idf, debug=False):
134        super(IndentAndNameChecker, self).__init__(path_in_idf)
135        self.debug = debug
136        self.min_prefix_length = CONFIG_NAME_MIN_PREFIX_LENGTH
137
138        # stack of the nested menuconfig items, e.g. ['mainmenu', 'menu', 'config']
139        self.level_stack = []
140
141        # stack common prefixes of configs
142        self.prefix_stack = []
143
144        # if the line ends with '\' then we force the indent of the next line
145        self.force_next_indent = 0
146
147        # menu items which increase the indentation of the next line
148        self.re_increase_level = re.compile(r'''^\s*
149                                          (
150                                               (menu(?!config))
151                                              |(mainmenu)
152                                              |(choice)
153                                              |(config)
154                                              |(menuconfig)
155                                              |(help)
156                                              |(if)
157                                              |(source)
158                                          )
159                                       ''', re.X)
160
161        # closing menu items which decrease the indentation
162        self.re_decrease_level = re.compile(r'''^\s*
163                                          (
164                                               (endmenu)
165                                              |(endchoice)
166                                              |(endif)
167                                          )
168                                       ''', re.X)
169
170        # matching beginning of the closing menuitems
171        self.pair_dic = {'endmenu': 'menu',
172                         'endchoice': 'choice',
173                         'endif': 'if',
174                         }
175
176        # regex for config names
177        self.re_name = re.compile(r'''^
178                                       (
179                                            (?:config)
180                                           |(?:menuconfig)
181                                           |(?:choice)
182
183                                       )\s+
184                                       (\w+)
185                                      ''', re.X)
186
187        # regex for new prefix stack
188        self.re_new_stack = re.compile(r'''^
189                                            (
190                                                 (?:menu(?!config))
191                                                |(?:mainmenu)
192                                                |(?:choice)
193
194                                            )
195                                            ''', re.X)
196
197    def __exit__(self, type, value, traceback):
198        super(IndentAndNameChecker, self).__exit__(type, value, traceback)
199        if len(self.prefix_stack) > 0:
200            self.check_common_prefix('', 'EOF')
201        if len(self.prefix_stack) != 0:
202            if self.debug:
203                print(self.prefix_stack)
204            raise RuntimeError("Prefix stack should be empty. Perhaps a menu/choice hasn't been closed")
205
206    def del_from_level_stack(self, count):
207        """ delete count items from the end of the level_stack """
208        if count > 0:
209            # del self.level_stack[-0:] would delete everything and we expect not to delete anything for count=0
210            del self.level_stack[-count:]
211
212    def update_level_for_inc_pattern(self, new_item):
213        if self.debug:
214            print('level+', new_item, ': ', self.level_stack, end=' -> ')
215        # "config" and "menuconfig" don't have a closing pair. So if new_item is an item which need to be indented
216        # outside the last "config" or "menuconfig" then we need to find to a parent where it belongs
217        if new_item in ['config', 'menuconfig', 'menu', 'choice', 'if', 'source']:
218            # item is not belonging to a previous "config" or "menuconfig" so need to indent to parent
219            for i, item in enumerate(reversed(self.level_stack)):
220                if item in ['menu', 'mainmenu', 'choice', 'if']:
221                    # delete items ("config", "menuconfig", "help") until the appropriate parent
222                    self.del_from_level_stack(i)
223                    break
224            else:
225                # delete everything when configs are at top level without a parent menu, mainmenu...
226                self.del_from_level_stack(len(self.level_stack))
227
228        self.level_stack.append(new_item)
229        if self.debug:
230            print(self.level_stack)
231        # The new indent is for the next line. Use the old one for the current line:
232        return len(self.level_stack) - 1
233
234    def update_level_for_dec_pattern(self, new_item):
235        if self.debug:
236            print('level-', new_item, ': ', self.level_stack, end=' -> ')
237        target = self.pair_dic[new_item]
238        for i, item in enumerate(reversed(self.level_stack)):
239            # find the matching beginning for the closing item in reverse-order search
240            # Note: "menuconfig", "config" and "help" don't have closing pairs and they are also on the stack. Now they
241            # will be deleted together with the "menu" or "choice" we are closing.
242            if item == target:
243                i += 1  # delete also the matching beginning
244                if self.debug:
245                    print('delete ', i, end=' -> ')
246                self.del_from_level_stack(i)
247                break
248        if self.debug:
249            print(self.level_stack)
250        return len(self.level_stack)
251
252    def check_name_and_update_prefix(self, line, line_number):
253        m = self.re_name.search(line)
254        if m:
255            name = m.group(2)
256            name_length = len(name)
257
258            if name_length > CONFIG_NAME_MAX_LENGTH:
259                raise InputError(self.path_in_idf, line_number,
260                                 '{} is {} characters long and it should be {} at most'
261                                 ''.format(name, name_length, CONFIG_NAME_MAX_LENGTH),
262                                 line + '\n')  # no suggested correction for this
263            if len(self.prefix_stack) == 0:
264                self.prefix_stack.append(name)
265            elif self.prefix_stack[-1] is None:
266                self.prefix_stack[-1] = name
267            else:
268                # this has nothing common with paths but the algorithm can be used for this also
269                self.prefix_stack[-1] = os.path.commonprefix([self.prefix_stack[-1], name])
270            if self.debug:
271                print('prefix+', self.prefix_stack)
272        m = self.re_new_stack.search(line)
273        if m:
274            self.prefix_stack.append(None)
275            if self.debug:
276                print('prefix+', self.prefix_stack)
277
278    def check_common_prefix(self, line, line_number):
279        common_prefix = self.prefix_stack.pop()
280        if self.debug:
281            print('prefix-', self.prefix_stack)
282        if common_prefix is None:
283            return
284        common_prefix_len = len(common_prefix)
285        if common_prefix_len < self.min_prefix_length:
286            raise InputError(self.path_in_idf, line_number,
287                             'The common prefix for the config names of the menu ending at this line is "{}".\n'
288                             '\tAll config names in this menu should start with the same prefix of {} characters '
289                             'or more.'.format(common_prefix, self.min_prefix_length),
290                             line)   # no suggested correction for this
291        if len(self.prefix_stack) > 0:
292            parent_prefix = self.prefix_stack[-1]
293            if parent_prefix is None:
294                # propagate to parent level where it will influence the prefix checking with the rest which might
295                # follow later on that level
296                self.prefix_stack[-1] = common_prefix
297            else:
298                if len(self.level_stack) > 0 and self.level_stack[-1] in ['mainmenu', 'menu']:
299                    # the prefix from menu is not required to propagate to the children
300                    return
301                if not common_prefix.startswith(parent_prefix):
302                    raise InputError(self.path_in_idf, line_number,
303                                     'Common prefix "{}" should start with {}'
304                                     ''.format(common_prefix, parent_prefix),
305                                     line)   # no suggested correction for this
306
307    def process_line(self, line, line_number):
308        stripped_line = line.strip()
309        if len(stripped_line) == 0:
310            self.force_next_indent = 0
311            return
312        current_level = len(self.level_stack)
313        m = re.search(r'\S', line)  # indent found as the first non-space character
314        if m:
315            current_indent = m.start()
316        else:
317            current_indent = 0
318
319        if current_level > 0 and self.level_stack[-1] == 'help':
320            if current_indent >= current_level * SPACES_PER_INDENT:
321                # this line belongs to 'help'
322                self.force_next_indent = 0
323                return
324
325        if self.force_next_indent > 0:
326            if current_indent != self.force_next_indent:
327                raise InputError(self.path_in_idf, line_number,
328                                 'Indentation consists of {} spaces instead of {}'.format(current_indent,
329                                                                                          self.force_next_indent),
330                                 (' ' * self.force_next_indent) + line.lstrip())
331            else:
332                if not stripped_line.endswith('\\'):
333                    self.force_next_indent = 0
334                return
335
336        elif stripped_line.endswith('\\') and stripped_line.startswith(('config', 'menuconfig', 'choice')):
337            raise InputError(self.path_in_idf, line_number,
338                             'Line-wrap with backslash is not supported here',
339                             line)  # no suggestion for this
340
341        self.check_name_and_update_prefix(stripped_line, line_number)
342
343        m = self.re_increase_level.search(line)
344        if m:
345            current_level = self.update_level_for_inc_pattern(m.group(1))
346        else:
347            m = self.re_decrease_level.search(line)
348            if m:
349                new_item = m.group(1)
350                current_level = self.update_level_for_dec_pattern(new_item)
351                if new_item not in ['endif']:
352                    # endif doesn't require to check the prefix because the items inside if/endif belong to the
353                    # same prefix level
354                    self.check_common_prefix(line, line_number)
355
356        expected_indent = current_level * SPACES_PER_INDENT
357
358        if stripped_line.endswith('\\'):
359            self.force_next_indent = expected_indent + SPACES_PER_INDENT
360        else:
361            self.force_next_indent = 0
362
363        if current_indent != expected_indent:
364            raise InputError(self.path_in_idf, line_number,
365                             'Indentation consists of {} spaces instead of {}'.format(current_indent, expected_indent),
366                             (' ' * expected_indent) + line.lstrip())
367
368
369def valid_directory(path):
370    if not os.path.isdir(path):
371        raise argparse.ArgumentTypeError('{} is not a valid directory!'.format(path))
372    return path
373
374
375def validate_kconfig_file(kconfig_full_path, verbose=False):  # type: (str, bool) -> bool
376    suggestions_full_path = kconfig_full_path + OUTPUT_SUFFIX
377    fail = False
378
379    with open(kconfig_full_path, 'r', encoding='utf-8') as f, \
380            open(suggestions_full_path, 'w', encoding='utf-8', newline='\n') as f_o, \
381            LineRuleChecker(kconfig_full_path) as line_checker, \
382            SourceChecker(kconfig_full_path) as source_checker, \
383            IndentAndNameChecker(kconfig_full_path, debug=verbose) as indent_and_name_checker:
384        try:
385            for line_number, line in enumerate(f, start=1):
386                try:
387                    for checker in [line_checker, indent_and_name_checker, source_checker]:
388                        checker.process_line(line, line_number)
389                    # The line is correct therefore we echo it to the output file
390                    f_o.write(line)
391                except InputError as e:
392                    print(e)
393                    fail = True
394                    f_o.write(e.suggested_line)
395        except UnicodeDecodeError:
396            raise ValueError('The encoding of {} is not Unicode.'.format(kconfig_full_path))
397
398    if fail:
399        print('\t{} has been saved with suggestions for resolving the issues.\n'
400              '\tPlease note that the suggestions can be wrong and '
401              'you might need to re-run the checker several times '
402              'for solving all issues'.format(suggestions_full_path))
403        print('\tPlease fix the errors and run {} for checking the correctness of '
404              'Kconfig files.'.format(os.path.abspath(__file__)))
405        return False
406    else:
407        print('{}: OK'.format(kconfig_full_path))
408        try:
409            os.remove(suggestions_full_path)
410        except Exception:
411            # not a serious error is when the file cannot be deleted
412            print('{} cannot be deleted!'.format(suggestions_full_path))
413        finally:
414            return True
415
416
417def main():
418    parser = argparse.ArgumentParser(description='Kconfig style checker')
419    parser.add_argument('files', nargs='*',
420                        help='Kconfig files')
421    parser.add_argument('--verbose', '-v', action='store_true',
422                        help='Print more information (useful for debugging)')
423    parser.add_argument('--includes', '-d', nargs='*',
424                        help='Extra paths for recursively searching Kconfig files. (for example $IDF_PATH)',
425                        type=valid_directory)
426    parser.add_argument('--exclude-submodules', action='store_true',
427                        help='Exclude submodules')
428    args = parser.parse_args()
429
430    success_counter = 0
431    failure_counter = 0
432    ignore_counter = 0
433
434    ignore_dirs = IGNORE_DIRS
435    if args.exclude_submodules:
436        ignore_dirs = ignore_dirs + tuple(get_submodule_dirs(full_path=True))
437
438    files = [os.path.abspath(file_path) for file_path in args.files]
439
440    if args.includes:
441        for directory in args.includes:
442            for root, dirnames, filenames in os.walk(directory):
443                for filename in filenames:
444                    full_path = os.path.join(root, filename)
445                    if re.search(RE_KCONFIG, filename):
446                        files.append(full_path)
447                    elif re.search(RE_KCONFIG, filename, re.IGNORECASE):
448                        # On Windows Kconfig files are working with different cases!
449                        print('{}: Incorrect filename. The case should be "Kconfig"!'.format(full_path))
450                        failure_counter += 1
451
452    for full_path in files:
453        if full_path.startswith(ignore_dirs):
454            print('{}: Ignored'.format(full_path))
455            ignore_counter += 1
456            continue
457        is_valid = validate_kconfig_file(full_path, args.verbose)
458        if is_valid:
459            success_counter += 1
460        else:
461            failure_counter += 1
462
463    if ignore_counter > 0:
464        print('{} files have been ignored.'.format(ignore_counter))
465    if success_counter > 0:
466        print('{} files have been successfully checked.'.format(success_counter))
467    if failure_counter > 0:
468        print('{} files have errors. Please take a look at the log.'.format(failure_counter))
469        return 1
470
471    if not files:
472        print('WARNING: no files specified. Please specify files or use '
473              '"--includes" to search Kconfig files recursively')
474    return 0
475
476
477if __name__ == '__main__':
478    sys.exit(main())
479