1#
2# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD
3# SPDX-License-Identifier: Apache-2.0
4#
5import abc
6import os
7import re
8from collections import namedtuple
9from enum import Enum
10
11from entity import Entity
12from pyparsing import (Combine, Forward, Group, Keyword, Literal, OneOrMore, Optional, Or, ParseFatalException,
13                       Suppress, Word, ZeroOrMore, alphanums, alphas, delimitedList, indentedBlock, nums,
14                       originalTextFor, restOfLine)
15from sdkconfig import SDKConfig
16
17
18class FragmentFile():
19    """
20    Processes a fragment file and stores all parsed fragments. For
21    more information on how this class interacts with classes for the different fragment types,
22    see description of Fragment.
23    """
24
25    def __init__(self, fragment_file, sdkconfig):
26        try:
27            fragment_file = open(fragment_file, 'r')
28        except TypeError:
29            pass
30
31        path = os.path.realpath(fragment_file.name)
32
33        indent_stack = [1]
34
35        class parse_ctx:
36            fragment = None  # current fragment
37            key = ''  # current key
38            keys = list()  # list of keys parsed
39            key_grammar = None  # current key grammar
40
41            @staticmethod
42            def reset():
43                parse_ctx.fragment_instance = None
44                parse_ctx.key = ''
45                parse_ctx.keys = list()
46                parse_ctx.key_grammar = None
47
48        def fragment_type_parse_action(toks):
49            parse_ctx.reset()
50            parse_ctx.fragment = FRAGMENT_TYPES[toks[0]]()  # create instance of the fragment
51            return None
52
53        def expand_conditionals(toks, stmts):
54            try:
55                stmt = toks['value']
56                stmts.append(stmt)
57            except KeyError:
58                try:
59                    conditions = toks['conditional']
60                    for condition in conditions:
61                        try:
62                            _toks = condition[1]
63                            _cond = condition[0]
64                            if sdkconfig.evaluate_expression(_cond):
65                                expand_conditionals(_toks, stmts)
66                                break
67                        except IndexError:
68                            expand_conditionals(condition[0], stmts)
69                except KeyError:
70                    for tok in toks:
71                        expand_conditionals(tok, stmts)
72
73        def key_body_parsed(pstr, loc, toks):
74            stmts = list()
75            expand_conditionals(toks, stmts)
76
77            if parse_ctx.key_grammar.min and len(stmts) < parse_ctx.key_grammar.min:
78                raise ParseFatalException(pstr, loc, "fragment requires at least %d values for key '%s'" %
79                                          (parse_ctx.key_grammar.min, parse_ctx.key))
80
81            if parse_ctx.key_grammar.max and len(stmts) > parse_ctx.key_grammar.max:
82                raise ParseFatalException(pstr, loc, "fragment requires at most %d values for key '%s'" %
83                                          (parse_ctx.key_grammar.max, parse_ctx.key))
84
85            try:
86                parse_ctx.fragment.set_key_value(parse_ctx.key, stmts)
87            except Exception as e:
88                raise ParseFatalException(pstr, loc, "unable to add key '%s'; %s" % (parse_ctx.key, str(e)))
89            return None
90
91        key = Word(alphanums + '_') + Suppress(':')
92        key_stmt = Forward()
93
94        condition_block = indentedBlock(key_stmt, indent_stack)
95        key_stmts = OneOrMore(condition_block)
96        key_body = Suppress(key) + key_stmts
97        key_body.setParseAction(key_body_parsed)
98
99        condition = originalTextFor(SDKConfig.get_expression_grammar()).setResultsName('condition')
100        if_condition = Group(Suppress('if') + condition + Suppress(':') + condition_block)
101        elif_condition = Group(Suppress('elif') + condition + Suppress(':') + condition_block)
102        else_condition = Group(Suppress('else') + Suppress(':') + condition_block)
103        conditional = (if_condition + Optional(OneOrMore(elif_condition)) + Optional(else_condition)).setResultsName('conditional')
104
105        def key_parse_action(pstr, loc, toks):
106            key = toks[0]
107
108            if key in parse_ctx.keys:
109                raise ParseFatalException(pstr, loc, "duplicate key '%s' value definition" % parse_ctx.key)
110
111            parse_ctx.key = key
112            parse_ctx.keys.append(key)
113
114            try:
115                parse_ctx.key_grammar = parse_ctx.fragment.get_key_grammars()[key]
116                key_grammar = parse_ctx.key_grammar.grammar
117            except KeyError:
118                raise ParseFatalException(pstr, loc, "key '%s' is not supported by fragment" % key)
119            except Exception as e:
120                raise ParseFatalException(pstr, loc, "unable to parse key '%s'; %s" % (key, str(e)))
121
122            key_stmt << (conditional | Group(key_grammar).setResultsName('value'))
123
124            return None
125
126        def name_parse_action(pstr, loc, toks):
127            parse_ctx.fragment.name = toks[0]
128
129        key.setParseAction(key_parse_action)
130
131        ftype = Word(alphas).setParseAction(fragment_type_parse_action)
132        fid = Suppress(':') + Word(alphanums + '_.').setResultsName('name')
133        fid.setParseAction(name_parse_action)
134        header = Suppress('[') + ftype + fid + Suppress(']')
135
136        def fragment_parse_action(pstr, loc, toks):
137            key_grammars = parse_ctx.fragment.get_key_grammars()
138            required_keys = set([k for (k,v) in key_grammars.items() if v.required])
139            present_keys = required_keys.intersection(set(parse_ctx.keys))
140            if present_keys != required_keys:
141                raise ParseFatalException(pstr, loc, 'required keys %s for fragment not found' %
142                                          list(required_keys - present_keys))
143            return parse_ctx.fragment
144
145        fragment_stmt = Forward()
146        fragment_block = indentedBlock(fragment_stmt, indent_stack)
147
148        fragment_if_condition = Group(Suppress('if') + condition + Suppress(':') + fragment_block)
149        fragment_elif_condition = Group(Suppress('elif') + condition + Suppress(':') + fragment_block)
150        fragment_else_condition = Group(Suppress('else') + Suppress(':') + fragment_block)
151        fragment_conditional = (fragment_if_condition + Optional(OneOrMore(fragment_elif_condition)) +
152                                Optional(fragment_else_condition)).setResultsName('conditional')
153
154        fragment = (header + OneOrMore(indentedBlock(key_body, indent_stack, False))).setResultsName('value')
155        fragment.setParseAction(fragment_parse_action)
156        fragment.ignore('#' + restOfLine)
157
158        deprecated_mapping = DeprecatedMapping.get_fragment_grammar(sdkconfig, fragment_file.name).setResultsName('value')
159
160        fragment_stmt << (Group(deprecated_mapping) | Group(fragment) | Group(fragment_conditional))
161
162        def fragment_stmt_parsed(pstr, loc, toks):
163            stmts = list()
164            expand_conditionals(toks, stmts)
165            return stmts
166
167        parser = ZeroOrMore(fragment_stmt)
168        parser.setParseAction(fragment_stmt_parsed)
169
170        self.fragments = parser.parseFile(fragment_file, parseAll=True)
171
172        for fragment in self.fragments:
173            fragment.path = path
174
175
176class Fragment():
177    """
178    Base class for a fragment that can be parsed from a fragment file. All fragments
179    share the common grammar:
180
181    [type:name]
182    key1:value1
183    key2:value2
184    ...
185
186    Supporting a new fragment type means deriving a concrete class which specifies
187    key-value pairs that the fragment supports and what to do with the parsed key-value pairs.
188
189    The new fragment must also be appended to FRAGMENT_TYPES, specifying the
190    keyword for the type and the derived class.
191
192    The key of the key-value pair is a simple keyword string. Other parameters
193    that describe the key-value pair is specified in Fragment.KeyValue:
194        1. grammar - pyparsing grammar to parse the value of key-value pair
195        2. min - the minimum number of value in the key entry, None means no minimum
196        3. max - the maximum number of value in the key entry, None means no maximum
197        4. required - if the key-value pair is required in the fragment
198
199    Setting min=max=1 means that the key has a single value.
200
201    FragmentFile provides conditional expression evaluation, enforcing
202    the parameters for Fragment.Keyvalue.
203    """
204    __metaclass__ = abc.ABCMeta
205
206    KeyValue = namedtuple('KeyValue', 'grammar min max required')
207
208    IDENTIFIER = Word(alphas + '_', alphanums + '_')
209    ENTITY = Word(alphanums + '.-_$+')
210
211    @abc.abstractmethod
212    def set_key_value(self, key, parse_results):
213        pass
214
215    @abc.abstractmethod
216    def get_key_grammars(self):
217        pass
218
219
220class Sections(Fragment):
221    """
222    Fragment which contains list of input sections.
223
224    [sections:<name>]
225    entries:
226        .section1
227        .section2
228        ...
229    """
230
231    # Unless quoted, symbol names start with a letter, underscore, or point
232    # and may include any letters, underscores, digits, points, and hyphens.
233    GNU_LD_SYMBOLS = Word(alphas + '_.', alphanums + '._-')
234
235    entries_grammar = Combine(GNU_LD_SYMBOLS + Optional('+'))
236
237    grammars = {
238        'entries': Fragment.KeyValue(entries_grammar.setResultsName('section'), 1, None, True)
239    }
240
241    """
242    Utility function that returns a list of sections given a sections fragment entry,
243    with the '+' notation and symbol concatenation handled automatically.
244    """
245    @staticmethod
246    def get_section_data_from_entry(sections_entry, symbol=None):
247        if not symbol:
248            sections = list()
249            sections.append(sections_entry.replace('+', ''))
250            sections.append(sections_entry.replace('+', '.*'))
251            return sections
252        else:
253            if sections_entry.endswith('+'):
254                section = sections_entry.replace('+', '.*')
255                expansion = section.replace('.*', '.' + symbol)
256                return (section, expansion)
257            else:
258                return (sections_entry, None)
259
260    def set_key_value(self, key, parse_results):
261        if key == 'entries':
262            self.entries = set()
263            for result in parse_results:
264                self.entries.add(result['section'])
265
266    def get_key_grammars(self):
267        return self.__class__.grammars
268
269
270class Scheme(Fragment):
271    """
272    Fragment which defines where the input sections defined in a Sections fragment
273    is going to end up, the target. The targets are markers in a linker script template
274    (see LinkerScript in linker_script.py).
275
276    [scheme:<name>]
277    entries:
278        sections1 -> target1
279        ...
280    """
281
282    grammars = {
283        'entries': Fragment.KeyValue(Fragment.IDENTIFIER.setResultsName('sections') + Suppress('->') +
284                                     Fragment.IDENTIFIER.setResultsName('target'), 1, None, True)
285    }
286
287    def set_key_value(self, key, parse_results):
288        if key == 'entries':
289            self.entries = set()
290            for result in parse_results:
291                self.entries.add((result['sections'], result['target']))
292
293    def get_key_grammars(self):
294        return self.__class__.grammars
295
296
297class Mapping(Fragment):
298    """
299    Fragment which attaches a scheme to entities (see Entity in entity.py), specifying where the input
300    sections of the entity will end up.
301
302    [mapping:<name>]
303    archive: lib1.a
304    entries:
305        obj1:symbol1 (scheme1); section1 -> target1 KEEP SURROUND(sym1) ...
306        obj2 (scheme2)
307        ...
308
309    Ultimately, an `entity (scheme)` entry generates an
310    input section description (see https://sourceware.org/binutils/docs/ld/Input-Section.html)
311    in the output linker script. It is possible to attach 'flags' to the
312    `entity (scheme)` to generate different output commands or to
313    emit additional keywords in the generated input section description. The
314    input section description, as well as other output commands, is defined in
315    output_commands.py.
316    """
317
318    class Flag():
319        PRE_POST = (Optional(Suppress(',') + Suppress('pre').setParseAction(lambda: True).setResultsName('pre')) +
320                    Optional(Suppress(',') + Suppress('post').setParseAction(lambda: True).setResultsName('post')))
321
322    class Surround(Flag):
323        def __init__(self, symbol):
324            self.symbol = symbol
325            self.pre = True
326            self.post = True
327
328        @staticmethod
329        def get_grammar():
330            # SURROUND(symbol)
331            #
332            # '__symbol_start', '__symbol_end' is generated before and after
333            # the corresponding input section description, respectively.
334            grammar = (Keyword('SURROUND').suppress() +
335                       Suppress('(') +
336                       Fragment.IDENTIFIER.setResultsName('symbol') +
337                       Suppress(')'))
338
339            grammar.setParseAction(lambda tok: Mapping.Surround(tok.symbol))
340            return grammar
341
342        def __eq__(self, other):
343            return (isinstance(other, Mapping.Surround) and
344                    self.symbol == other.symbol)
345
346    class Align(Flag):
347
348        def __init__(self, alignment, pre=True, post=False):
349            self.alignment = alignment
350            self.pre = pre
351            self.post = post
352
353        @staticmethod
354        def get_grammar():
355            # ALIGN(alignment, [, pre, post]).
356            #
357            # Generates alignment command before and/or after the corresponding
358            # input section description, depending whether pre, post or
359            # both are specified.
360            grammar = (Keyword('ALIGN').suppress() +
361                       Suppress('(') +
362                       Word(nums).setResultsName('alignment') +
363                       Mapping.Flag.PRE_POST +
364                       Suppress(')'))
365
366            def on_parse(tok):
367                alignment = int(tok.alignment)
368                if tok.pre == '' and tok.post == '':
369                    res = Mapping.Align(alignment)
370                elif tok.pre != '' and tok.post == '':
371                    res = Mapping.Align(alignment, tok.pre)
372                elif tok.pre == '' and tok.post != '':
373                    res = Mapping.Align(alignment, False, tok.post)
374                else:
375                    res = Mapping.Align(alignment, tok.pre, tok.post)
376                return res
377
378            grammar.setParseAction(on_parse)
379            return grammar
380
381        def __eq__(self, other):
382            return (isinstance(other, Mapping.Align) and
383                    self.alignment == other.alignment and
384                    self.pre == other.pre and
385                    self.post == other.post)
386
387    class Keep(Flag):
388
389        def __init__(self):
390            pass
391
392        @staticmethod
393        def get_grammar():
394            # KEEP()
395            #
396            # Surrounds input section description with KEEP command.
397            grammar = Keyword('KEEP()').setParseAction(Mapping.Keep)
398            return grammar
399
400        def __eq__(self, other):
401            return isinstance(other, Mapping.Keep)
402
403    class Sort(Flag):
404        class Type(Enum):
405            NAME = 0
406            ALIGNMENT = 1
407            INIT_PRIORITY = 2
408
409        def __init__(self, first, second=None):
410            self.first = first
411            self.second = second
412
413        @staticmethod
414        def get_grammar():
415            # SORT([sort_by_first, sort_by_second])
416            #
417            # where sort_by_first, sort_by_second = {name, alignment, init_priority}
418            #
419            # Emits SORT_BY_NAME, SORT_BY_ALIGNMENT or SORT_BY_INIT_PRIORITY
420            # depending on arguments. Nested sort follows linker script rules.
421            keywords = Keyword('name') | Keyword('alignment') | Keyword('init_priority')
422            grammar = (Keyword('SORT').suppress() + Suppress('(') +
423                       keywords.setResultsName('first') +
424                       Optional(Suppress(',') + keywords.setResultsName('second')) + Suppress(')'))
425
426            grammar.setParseAction(lambda tok: Mapping.Sort(tok.first, tok.second if tok.second != '' else None))
427            return grammar
428
429        def __eq__(self, other):
430            return (isinstance(other, Mapping.Sort) and
431                    self.first == other.first and
432                    self.second == other.second)
433
434    def __init__(self):
435        Fragment.__init__(self)
436        self.entries = set()
437        # k = (obj, symbol, scheme)
438        # v = list((section, target), Mapping.Flag))
439        self.flags = dict()
440        self.deprecated = False
441
442    def set_key_value(self, key, parse_results):
443        if key == 'archive':
444            self.archive = parse_results[0]['archive']
445        elif key == 'entries':
446            for result in parse_results:
447                obj = None
448                symbol = None
449                scheme = None
450
451                obj = result['object']
452
453                try:
454                    symbol = result['symbol']
455                except KeyError:
456                    pass
457
458                scheme = result['scheme']
459
460                mapping = (obj, symbol, scheme)
461                self.entries.add(mapping)
462
463                try:
464                    parsed_flags = result['sections_target_flags']
465                except KeyError:
466                    parsed_flags = []
467
468                if parsed_flags:
469                    entry_flags = []
470                    for pf in parsed_flags:
471                        entry_flags.append((pf.sections, pf.target, list(pf.flags)))
472
473                    try:
474                        existing_flags = self.flags[mapping]
475                    except KeyError:
476                        existing_flags = list()
477                        self.flags[mapping] = existing_flags
478
479                    existing_flags.extend(entry_flags)
480
481    def get_key_grammars(self):
482        # There are three possible patterns for mapping entries:
483        #       obj:symbol (scheme)
484        #       obj (scheme)
485        #       * (scheme)
486        # Flags can be specified for section->target in the scheme specified, ex:
487        #       obj (scheme); section->target SURROUND(symbol), section2->target2 ALIGN(4)
488        obj = Fragment.ENTITY.setResultsName('object')
489        symbol = Suppress(':') + Fragment.IDENTIFIER.setResultsName('symbol')
490        scheme = Suppress('(') + Fragment.IDENTIFIER.setResultsName('scheme') + Suppress(')')
491
492        # The flags are specified for section->target in the scheme specified
493        sections_target = Scheme.grammars['entries'].grammar
494
495        flag = Or([f.get_grammar() for f in [Mapping.Keep, Mapping.Align, Mapping.Surround, Mapping.Sort]])
496
497        section_target_flags = Group(sections_target + Group(OneOrMore(flag)).setResultsName('flags'))
498
499        pattern1 = obj + symbol
500        pattern2 = obj
501        pattern3 = Literal(Entity.ALL).setResultsName('object')
502
503        entry = ((pattern1 | pattern2 | pattern3) + scheme +
504                 Optional(Suppress(';') + delimitedList(section_target_flags).setResultsName('sections_target_flags')))
505
506        grammars = {
507            'archive': Fragment.KeyValue(Or([Fragment.ENTITY, Word(Entity.ALL)]).setResultsName('archive'), 1, 1, True),
508            'entries': Fragment.KeyValue(entry, 0, None, True)
509        }
510
511        return grammars
512
513
514class DeprecatedMapping():
515    """
516    Mapping fragment with old grammar in versions older than ESP-IDF v4.0. Does not conform to
517    requirements of the Fragment class and thus is limited when it comes to conditional expression
518    evaluation.
519    """
520
521    # Name of the default condition entry
522    DEFAULT_CONDITION = 'default'
523
524    @staticmethod
525    def get_fragment_grammar(sdkconfig, fragment_file):
526
527        # Match header [mapping]
528        header = Suppress('[') + Suppress('mapping') + Suppress(']')
529
530        # There are three possible patterns for mapping entries:
531        #       obj:symbol (scheme)
532        #       obj (scheme)
533        #       * (scheme)
534        obj = Fragment.ENTITY.setResultsName('object')
535        symbol = Suppress(':') + Fragment.IDENTIFIER.setResultsName('symbol')
536        scheme = Suppress('(') + Fragment.IDENTIFIER.setResultsName('scheme') + Suppress(')')
537
538        pattern1 = Group(obj + symbol + scheme)
539        pattern2 = Group(obj + scheme)
540        pattern3 = Group(Literal(Entity.ALL).setResultsName('object') + scheme)
541
542        mapping_entry = pattern1 | pattern2 | pattern3
543
544        # To simplify parsing, classify groups of condition-mapping entry into two types: normal and default
545        # A normal grouping is one with a non-default condition. The default grouping is one which contains the
546        # default condition
547        mapping_entries = Group(ZeroOrMore(mapping_entry)).setResultsName('mappings')
548
549        normal_condition = Suppress(':') + originalTextFor(SDKConfig.get_expression_grammar())
550        default_condition = Optional(Suppress(':') + Literal(DeprecatedMapping.DEFAULT_CONDITION))
551
552        normal_group = Group(normal_condition.setResultsName('condition') + mapping_entries)
553        default_group = Group(default_condition + mapping_entries).setResultsName('default_group')
554
555        normal_groups = Group(ZeroOrMore(normal_group)).setResultsName('normal_groups')
556
557        # Any mapping fragment definition can have zero or more normal group and only one default group as a last entry.
558        archive = Suppress('archive') + Suppress(':') + Fragment.ENTITY.setResultsName('archive')
559        entries = Suppress('entries') + Suppress(':') + (normal_groups + default_group).setResultsName('entries')
560
561        mapping = Group(header + archive + entries)
562        mapping.ignore('#' + restOfLine)
563
564        def parsed_deprecated_mapping(pstr, loc, toks):
565            fragment = Mapping()
566            fragment.archive = toks[0].archive
567            fragment.name = re.sub(r'[^0-9a-zA-Z]+', '_', fragment.archive)
568            fragment.deprecated = True
569
570            fragment.entries = set()
571            condition_true = False
572            for entries in toks[0].entries[0]:
573                condition  = next(iter(entries.condition.asList())).strip()
574                condition_val = sdkconfig.evaluate_expression(condition)
575
576                if condition_val:
577                    for entry in entries[1]:
578                        fragment.entries.add((entry.object, None if entry.symbol == '' else entry.symbol, entry.scheme))
579                    condition_true = True
580                    break
581
582            if not fragment.entries and not condition_true:
583                try:
584                    entries = toks[0].entries[1][1]
585                except IndexError:
586                    entries = toks[0].entries[1][0]
587                for entry in entries:
588                    fragment.entries.add((entry.object, None if entry.symbol == '' else entry.symbol, entry.scheme))
589
590            if not fragment.entries:
591                fragment.entries.add(('*', None, 'default'))
592
593            dep_warning = str(ParseFatalException(pstr, loc,
594                              'Warning: Deprecated old-style mapping fragment parsed in file %s.' % fragment_file))
595
596            print(dep_warning)
597            return fragment
598
599        mapping.setParseAction(parsed_deprecated_mapping)
600        return mapping
601
602
603FRAGMENT_TYPES = {
604    'sections': Sections,
605    'scheme': Scheme,
606    'mapping': Mapping
607}
608