1 //
2 // Copyright (c) 2010-2023 Antmicro
3 // Copyright (c) 2011-2015 Realtime Embedded
4 //
5 // This file is licensed under the MIT License.
6 // Full license text is available in 'licenses/MIT.txt'.
7 //
8 using System;
9 using NUnit.Framework;
10 using Antmicro.Renode.UserInterface;
11 using Antmicro.Renode.UserInterface.Tokenizer;
12 using System.Linq;
13 using Antmicro.Renode.Exceptions;
14 using Antmicro.Renode.Core;
15 
16 using Range = Antmicro.Renode.Core.Range;
17 
18 namespace Antmicro.Renode.MonitorTests
19 {
20     [TestFixture]
21     public class TokenizerTests
22     {
23         [Test]
CommentTest()24         public void CommentTest()
25         {
26             var result = tokenizer.Tokenize("#something");
27             AssertTokenizationTypes(result, typeof(CommentToken));
28             AssertTokenizationValues(result, "something");
29 
30             result = tokenizer.Tokenize("emu[\"SomeIndexWith#Hash\"]");
31             AssertTokenizationTypes(result, typeof(LiteralToken), typeof(LeftBraceToken), typeof(StringToken), typeof(RightBraceToken));
32             AssertTokenizationValues(result, "emu", "[", "SomeIndexWith#Hash", "]");
33 
34             result = tokenizer.Tokenize("emu[\"SomeIndexWithoutHash\"#Comment]");
35             AssertTokenizationTypes(result, typeof(LiteralToken), typeof(LeftBraceToken), typeof(StringToken), typeof(CommentToken));
36             AssertTokenizationValues(result, "emu", "[", "SomeIndexWithoutHash", "Comment]");
37         }
38 
39         [Test]
ExecutionTest()40         public void ExecutionTest()
41         {
42             var result = tokenizer.Tokenize("`something`");
43             AssertTokenizationTypes(result, typeof(ExecutionToken));
44             AssertTokenizationValues(result, "something");
45         }
46 
47         [Test]
VariableTest()48         public void VariableTest()
49         {
50             var result = tokenizer.Tokenize("$name $_name $NaMe $.name $123name123");
51             AssertTokenizationTypes(result, typeof(VariableToken), typeof(VariableToken), typeof(VariableToken), typeof(VariableToken), typeof(VariableToken));
52             AssertTokenizationValues(result, "name", "_name", "NaMe", ".name", "123name123");
53 
54             result = tokenizer.Tokenize("$variable=\"value\"");
55             AssertTokenizationTypes(result, typeof(VariableToken), typeof(EqualityToken), typeof(StringToken));
56             AssertTokenizationValues(result, "variable", "=", "value");
57 
58             result = tokenizer.Tokenize("$variable?=\"value\"");
59             AssertTokenizationTypes(result, typeof(VariableToken), typeof(ConditionalEqualityToken), typeof(StringToken));
60             AssertTokenizationValues(result, "variable", "?=", "value");
61         }
62 
63         [Test]
IndexTest()64         public void IndexTest()
65         {
66             var result = tokenizer.Tokenize("emu[\"SomeIndex\"]");
67             AssertTokenizationTypes(result, typeof(LiteralToken), typeof(LeftBraceToken), typeof(StringToken), typeof(RightBraceToken));
68             AssertTokenizationValues(result, "emu", "[", "SomeIndex", "]");
69 
70             result = tokenizer.Tokenize("emu[15]");
71             AssertTokenizationTypes(result, typeof(LiteralToken), typeof(LeftBraceToken), typeof(DecimalIntegerToken), typeof(RightBraceToken));
72             AssertTokenizationValues(result, "emu", "[", 15, "]");
73         }
74 
75         [Test]
StringTest()76         public void StringTest()
77         {
78             var result = tokenizer.Tokenize("'string1' \"string2\"");
79             AssertTokenizationTypes(result, typeof(StringToken), typeof(StringToken));
80             AssertTokenizationValues(result, "string1", "string2");
81 
82             result = tokenizer.Tokenize("'string1\" 'string2\"");
83             AssertTokenizationResult(result, 1, null, typeof(StringToken), typeof(LiteralToken));
84             AssertTokenizationValues(result, "string1\" ", "string2");
85         }
86 
87         [Test]
UnbalancedStringTest()88         public void UnbalancedStringTest()
89         {
90             var result = tokenizer.Tokenize("\"test\\\"       \\\"         \\\"       test \" 'test\\'        \\'  \\'   test '");
91             AssertTokenizationValues(result, "test\"       \"         \"       test ", "test'        '  '   test ");
92             AssertTokenizationTypes(result, typeof(StringToken), typeof(StringToken));
93         }
94 
95         [Test]
RangeTest()96         public void RangeTest()
97         {
98             var result = tokenizer.Tokenize("<    \t0x123abDE  \t, \t\t  0xabcdef0 \t\t   >");
99             var expectedValue = new Range(0x123abde, 0xabcdef0 - 0x123abde + 1);
100             AssertTokenizationTypes(result, typeof(AbsoluteRangeToken));
101             AssertTokenizationValues(result, expectedValue);
102 
103             result = tokenizer.Tokenize("<0xdefg, 0xefgh>");
104             AssertTokenizationResult(result, 16);
105         }
106 
107         [Test]
RelativeRangeTest()108         public void RelativeRangeTest()
109         {
110             var result = tokenizer.Tokenize("<0x6 0x2>");
111             var expectedValue = new Range(0x6, 0x2);
112             AssertTokenizationTypes(result, typeof(RelativeRangeToken));
113             AssertTokenizationValues(result, expectedValue);
114         }
115 
116         [Test]
SimplePathTest()117         public void SimplePathTest()
118         {
119             var result = tokenizer.Tokenize("@Some\\path\\to\\File");
120             AssertTokenizationTypes(result, typeof(PathToken));
121             AssertTokenizationValues(result, "Some\\path\\to\\File");
122 
123             result = tokenizer.Tokenize("@Some\\path\\to\\Directory\\");
124             AssertTokenizationTypes(result, typeof(PathToken));
125             AssertTokenizationValues(result, "Some\\path\\to\\Directory\\");
126         }
127 
128         [Test]
EscapedPathTest()129         public void EscapedPathTest()
130         {
131             var result = tokenizer.Tokenize("@Some\\path\\to\\file\\ with\\ Spaces");
132             AssertTokenizationTypes(result, typeof(PathToken));
133             AssertTokenizationValues(result, "Some\\path\\to\\file with Spaces");
134 
135             result = tokenizer.Tokenize("@Some\\path\\to\\directory\\ with\\ Spaces\\");
136             AssertTokenizationTypes(result, typeof(PathToken));
137             AssertTokenizationValues(result, "Some\\path\\to\\directory with Spaces\\");
138         }
139 
140         [Test]
MultilineTest()141         public void MultilineTest()
142         {
143             var result = tokenizer.Tokenize("\"\"\"");
144             AssertTokenizationTypes(result, typeof(MultilineStringTerminatorToken));
145             AssertTokenizationValues(result, "\"");
146 
147             result = tokenizer.Tokenize("\"\"\"SomeMultiline\r\nString with many #tokens [\"inside\"] with\r\nnumbers 123 0x23 and\r\n stuff\"\"\"");
148             AssertTokenizationTypes(result, typeof(MultilineStringToken));
149             AssertTokenizationValues(result, "SomeMultiline\r\nString with many #tokens [\"inside\"] with\r\nnumbers 123 0x23 and\r\n stuff");
150         }
151 
152         [Test]
BooleanTest()153         public void BooleanTest()
154         {
155             var result = tokenizer.Tokenize("true");
156             AssertTokenizationTypes(result, typeof(BooleanToken));
157             AssertTokenizationValues(result, true);
158 
159             result = tokenizer.Tokenize("TrUe");
160             AssertTokenizationTypes(result, typeof(BooleanToken));
161             AssertTokenizationValues(result, true);
162 
163             result = tokenizer.Tokenize("FalSE");
164             AssertTokenizationTypes(result, typeof(BooleanToken));
165             AssertTokenizationValues(result, false);
166 
167             result = tokenizer.Tokenize("false");
168             AssertTokenizationTypes(result, typeof(BooleanToken));
169             AssertTokenizationValues(result, false);
170         }
171 
172         [Test]
IntegerTest()173         public void IntegerTest()
174         {
175             var result = tokenizer.Tokenize("123465 -213245 +132432");
176             AssertTokenizationTypes(result, typeof(DecimalIntegerToken), typeof(DecimalIntegerToken), typeof(DecimalIntegerToken));
177             AssertTokenizationValues(result, 123465, -213245, 132432);
178         }
179 
180         [Test]
FloatTest()181         public void FloatTest()
182         {
183             var result = tokenizer.Tokenize("145.5 -0.43 +45.");
184             AssertTokenizationTypes(result, typeof(FloatToken), typeof(FloatToken), typeof(FloatToken));
185             AssertTokenizationValues(result, 145.5f, -0.43f, 45.0f);
186         }
187 
188         [Test]
HexadecimalTest()189         public void HexadecimalTest()
190         {
191             var result = tokenizer.Tokenize("0xabcdef 0x123469 0xABCDEF 0x123AbC");
192             AssertTokenizationTypes(result, typeof(HexToken), typeof(HexToken), typeof(HexToken), typeof(HexToken));
193             AssertTokenizationValues(result, 0xabcdef, 0x123469, 0xabcdef, 0x123abc);
194 
195             result = tokenizer.Tokenize("0xgfd 123bcd");
196             AssertTokenizationTypes(result, typeof(DecimalIntegerToken), typeof(LiteralToken), typeof(DecimalIntegerToken), typeof(LiteralToken));
197             AssertTokenizationValues(result, 0, "xgfd", 123, "bcd");
198         }
199 
LiteralTest()200         public void LiteralTest()
201         {
202             var result = tokenizer.Tokenize(".Some.Literal-With?Extra:SignsIn.It:");
203             AssertTokenizationTypes(result, typeof(LiteralToken));
204             AssertTokenizationValues(result, ".Some.Literal-With?Extra:SignsIn.It:");
205         }
206 
207         [SetUp]
TestSetUp()208         public void TestSetUp()
209         {
210             tokenizer = Tokenizer.CreateTokenizer();
211         }
212 
AssertTokenizationResult(TokenizationResult result, int unmatchedCharacters, Type exception = null, params Type[] types)213         private static void AssertTokenizationResult(TokenizationResult result, int unmatchedCharacters, Type exception = null, params Type[] types)
214         {
215             if(exception != null)
216             {
217                 Assert.AreEqual(result.Exception.GetType(), exception);
218             }
219             else
220             {
221                 Assert.IsNull(result.Exception);
222             }
223             Assert.IsTrue(result.UnmatchedCharactersLeft == unmatchedCharacters);
224             Assert.IsNotNull(result.Tokens);
225             var tokens = result.Tokens.ToArray();
226             Assert.AreEqual(tokens.Length, types.Length);
227             for(var i = 0; i < tokens.Length; ++i)
228             {
229                 Assert.AreSame(tokens[i].GetType(), types[i]);
230             }
231         }
232 
AssertTokenizationTypes(TokenizationResult result, params Type[] types)233         private static void AssertTokenizationTypes(TokenizationResult result, params Type[] types)
234         {
235             Assert.IsNull(result.Exception);
236             Assert.IsTrue(result.UnmatchedCharactersLeft == 0);
237             Assert.IsNotNull(result.Tokens);
238             var tokens = result.Tokens.ToArray();
239             Assert.AreEqual(tokens.Length, types.Length);
240             for(var i = 0; i < tokens.Length; ++i)
241             {
242                 Assert.AreSame(tokens[i].GetType(), types[i]);
243             }
244         }
245 
AssertTokenizationValues(TokenizationResult result, params object[] values)246         private static void AssertTokenizationValues(TokenizationResult result, params object[] values)
247         {
248             var tokens = result.Tokens.ToArray();
249             Assert.AreEqual(tokens.Length, values.Length);
250             CollectionAssert.AreEqual(values, tokens.Select(x => x.GetObjectValue()));
251         }
252 
253         private Tokenizer tokenizer;
254     }
255 }
256 
257