1 //
2 // Copyright (c) 2010-2018 Antmicro
3 // Copyright (c) 2011-2015 Realtime Embedded
4 //
5 // This file is licensed under the MIT License.
6 // Full license text is available in 'licenses/MIT.txt'.
7 //
8 using System;
9 using System.Collections.Generic;
10 using System.Text.RegularExpressions;
11 using Antmicro.Renode.Exceptions;
12 
13 namespace Antmicro.Renode.UserInterface.Tokenizer
14 {
15 
16     public class Tokenizer
17     {
CreateTokenizer()18         public static Tokenizer CreateTokenizer()
19         {
20             var tokenizer = new Tokenizer();
21             // comment
22             tokenizer.AddToken(new Regex(@"^\#.*"), x => new CommentToken(x));
23 
24             tokenizer.AddToken(new Regex(@"^:.*"), x => new CommentToken(x));
25 
26             // execution
27             tokenizer.AddToken(new Regex(@"^`.*?`"), x => new ExecutionToken(x));
28 
29             // variable
30             tokenizer.AddToken(new Regex(@"^\$([0-9]|(?i:[a-z])|_|\.)+"), x => new VariableToken(x));
31 
32             // left brace
33             tokenizer.AddToken(new Regex(@"^\["), x => new LeftBraceToken(x));
34 
35             // right brace
36             tokenizer.AddToken(new Regex(@"^\]"), x => new RightBraceToken(x));
37 
38             //multiline string
39             tokenizer.AddToken(new Regex(@"^"""""".+?""""""", RegexOptions.Singleline), x => new MultilineStringToken(x));
40 
41             //multiline terminator
42             tokenizer.AddToken(new Regex(@"^"""""""), x => new MultilineStringTerminatorToken(x));
43 
44             // string
45             tokenizer.AddToken(new Regex(@"^'[^'\\]*(?:\\.[^'\\]*)*'"), x => new StringToken(x));
46             tokenizer.AddToken(new Regex(@"^""[^""\\]*(?:\\.[^""\\]*)*"""), x => new StringToken(x));
47 
48             // absolute range
49             tokenizer.AddToken(new Regex(@"^<\s*((0x([0-9]|(?i:[a-f]))+)|([+-]?\d+))\s*,\s*((0x([0-9]|(?i:[a-f]))+)|([+-]?\d+))\s*>"), x => new AbsoluteRangeToken(x));
50 
51             // relative range
52             tokenizer.AddToken(new Regex(@"^<\s*((0x([0-9]|(?i:[a-f]))+)|([+-]?\d+))\s+((0x([0-9]|(?i:[a-f]))+)|(\+?\d+))\s*>"), x => new RelativeRangeToken(x));
53 
54             // path
55             tokenizer.AddToken(new Regex(@"^\@(?:(?!;)((\\ )|\S))+"), x => new PathToken(x));
56 
57             // hex number
58             tokenizer.AddToken(new Regex(@"^0x([0-9]|(?i:[a-f]))+"), x => new HexToken(x));
59 
60             // float number
61             tokenizer.AddToken(new Regex(@"^[+-]?((\d+\.(\d*)?))"), x => new FloatToken(x));
62 
63             // integer
64             tokenizer.AddToken(new Regex(@"^[+-]?\d+"), x => new DecimalIntegerToken(x));
65 
66             // boolean ignore case
67             tokenizer.AddToken(new Regex(@"^(?i)(true|false)"), x => new BooleanToken(x));
68 
69             // "="
70             tokenizer.AddToken(new Regex(@"^="), x => new EqualityToken(x));
71 
72             // "?="
73             tokenizer.AddToken(new Regex(@"^\?="), x => new ConditionalEqualityToken(x));
74 
75             // ";" or new line
76             tokenizer.AddToken(new Regex(@"^\;"), x => new CommandSplit(x));
77 
78             // literal
79             tokenizer.AddToken(new Regex(@"^[\w\.\-\?]+"), x => new LiteralToken(x));
80 
81             // whitespace
82             tokenizer.AddToken(new Regex(@"^\s+"), x => null);
83             return tokenizer;
84         }
85 
Tokenizer()86         private Tokenizer()
87         {
88             tokens = new List<InternalToken>();
89         }
90 
Tokenize(string input)91         public TokenizationResult Tokenize(string input)
92         {
93             var producedTokens = new List<Token>();
94             RecoverableException exception = null;
95             while(input.Length > 0)
96             {
97                 var success = false;
98                 foreach(var proposition in tokens)
99                 {
100                     var regex = proposition.ApplicabilityCondition;
101                     var match = regex.Match(input);
102                     if(!match.Success)
103                     {
104                         continue;
105                     }
106                     success = true;
107                     Token producedToken;
108                     try
109                     {
110                         producedToken = proposition.Factory(input.Substring(0, match.Length));
111                     }
112                     catch(RecoverableException e)
113                     {
114                         success = false;
115                         exception = exception ?? e;
116                         break;
117                     }
118                     input = input.Substring(match.Length);
119                     if(producedToken != null)
120                     {
121                         producedTokens.Add(producedToken);
122                     }
123                     break;
124                 }
125                 if(!success)
126                 {
127                     break;
128                 }
129             }
130             return new TokenizationResult(input.Length, producedTokens, exception);
131         }
132 
AddToken(Regex applicabilityCondition, Func<string, Token> factory)133         public void AddToken(Regex applicabilityCondition, Func<string, Token> factory)
134         {
135             tokens.Add(new InternalToken(applicabilityCondition, factory));
136         }
137 
138         private readonly List<InternalToken> tokens;
139 
140         private class InternalToken
141         {
InternalToken(Regex applicabilityCondition, Func<string, Token> factory)142             public InternalToken(Regex applicabilityCondition, Func<string, Token> factory)
143             {
144                 this.ApplicabilityCondition = applicabilityCondition;
145                 this.Factory = factory;
146             }
147 
148             public Regex ApplicabilityCondition { get; private set; }
149             public Func<string, Token> Factory { get; private set; }
150         }
151     }
152 }
153