1# Process the test results
2# Test status (like passed, or failed with error code)
3
4import argparse
5import re
6import TestScripts.NewParser as parse
7import TestScripts.CodeGen
8from collections import deque
9import os.path
10import csv
11import TestScripts.ParseTrace
12import colorama
13from colorama import init,Fore, Back, Style
14import sys
15
16resultStatus=0
17
18init()
19
20
21def errorStr(id):
22  if id == 1:
23     return("UNKNOWN_ERROR")
24  if id == 2:
25     return("Equality error")
26  if id == 3:
27     return("Absolute difference error")
28  if id == 4:
29     return("Relative difference error")
30  if id == 5:
31     return("SNR error")
32  if id == 6:
33     return("Different length error")
34  if id == 7:
35     return("Assertion error")
36  if id == 8:
37     return("Memory allocation error")
38  if id == 9:
39     return("Empty pattern error")
40  if id == 10:
41     return("Buffer tail corrupted")
42  if id == 11:
43     return("Close float error")
44
45  return("Unknown error %d" % id)
46
47
48def findItem(root,path):
49        """ Find a node in a tree
50
51        Args:
52          path (list) : A list of node ID
53            This list is describing a path in the tree.
54            By starting from the root and following this path,
55            we can find the node in the tree.
56        Raises:
57          Nothing
58        Returns:
59          TreeItem : A node
60        """
61        # The list is converted into a queue.
62        q = deque(path)
63        q.popleft()
64        c = root
65        while q:
66            n = q.popleft()
67            # We get the children based on its ID and continue
68            c = c[n-1]
69        return(c)
70
71def joinit(iterable, delimiter):
72    # Intersperse a delimiter between element of a list
73    it = iter(iterable)
74    yield next(it)
75    for x in it:
76        yield delimiter
77        yield x
78
79# Return test result as a text tree
80class TextFormatter:
81      def start(self):
82          None
83
84      def printGroup(self,elem,theId):
85        if elem is None:
86           elem = root
87        message=elem.data["message"]
88        if not elem.data["deprecated"]:
89           kind = "Suite"
90           ident = " " * elem.ident
91           if elem.kind == TestScripts.Parser.TreeElem.GROUP:
92              kind = "Group"
93           #print(elem.path)
94           print(Style.BRIGHT + ("%s%s : %s (%d)" % (ident,kind,message,theId)) + Style.RESET_ALL)
95
96      def printTest(self,elem, theId, theError,errorDetail,theLine,passed,cycles,params):
97          message=elem.data["message"]
98          func=elem.data["class"]
99          if not elem.data["deprecated"]:
100             kind = "Test"
101             ident = " " * elem.ident
102             p=Fore.RED + "FAILED" + Style.RESET_ALL
103             if passed == 1:
104                p= Fore.GREEN + "PASSED" + Style.RESET_ALL
105             if cycles > 0:
106                print("%s%s %s(%s - %d)%s : %s (cycles = %d)" % (ident,message,Style.BRIGHT,func,theId,Style.RESET_ALL,p,cycles))
107             else:
108                print("%s%s %s(%s - %d)%s : %s" % (ident,message,Style.BRIGHT,func,theId,Style.RESET_ALL,p))
109             if params:
110                print("%s %s" % (ident,params))
111             if passed != 1:
112                print(Fore.RED + ("%s %s at line %d" % (ident, errorStr(theError), theLine)) + Style.RESET_ALL)
113                if (len(errorDetail)>0):
114                   print(Fore.RED + ident + " " + errorDetail + Style.RESET_ALL)
115
116      def pop(self):
117          None
118
119      def end(self):
120        None
121
122# Return test result as a text tree
123class HTMLFormatter:
124      def __init__(self,append=False):
125        self.nb=1
126        self.suite=False
127        self.append = append
128
129      def start(self):
130          if not self.append:
131             print("<html><head><title>Test Results</title></head><body>")
132
133      def printGroup(self,elem,theId):
134        if elem is None:
135           elem = root
136        message=elem.data["message"]
137        if not elem.data["deprecated"]:
138           kind = "Suite"
139           ident = " " * elem.ident
140           if elem.kind == TestScripts.Parser.TreeElem.GROUP:
141              kind = "Group"
142           if kind == "Group":
143              if not self.append:
144                 print("<h%d> %s (%d) </h%d>" % (self.nb,message,theId,self.nb))
145           else:
146              if not self.append:
147                 print("<h%d> %s (%d) </h%d>" % (self.nb,message,theId,self.nb))
148              else:
149                 print("<h%d> %s (%d) </h%d>" % (3,message,theId,self.nb))
150              self.suite=True
151              print("<table style=\"width:100%\">")
152              print("<tr>")
153              print("<td>Name</td>")
154              print("<td>ID</td>")
155              print("<td>Status</td>")
156              print("<td>Params</td>")
157              print("<td>Cycles</td>")
158              print("</tr>")
159           self.nb = self.nb + 1
160
161      def printTest(self,elem, theId, theError,errorDetail,theLine,passed,cycles,params):
162          message=elem.data["message"]
163          if not elem.data["deprecated"]:
164             kind = "Test"
165             ident = " " * elem.ident
166             p="<font color=\"red\">FAILED</font>"
167             if passed == 1:
168                p= "<font color=\"green\">PASSED</font>"
169             # Green status is not displayed when
170             # generating the full summary in append mode
171             # In summary mode, only errors are displayed
172             if passed != 1 or not self.append:
173                print("<tr>")
174                print("<td><pre>%s</pre></td>" % (message,))
175                print("<td>%d</td>" % theId)
176                print("<td>%s</td>" % p)
177                if params:
178                   print("<td>%s</td>\n" % (params))
179                else:
180                   print("<td></td>\n")
181                if (cycles > 0):
182                   print("<td>%d</td>" % cycles)
183                else:
184                   print("<td>NA</td>")
185                print("</tr>")
186
187             if passed != 1:
188
189                print("<tr><td colspan=4><font color=\"red\">%s at line %d</font></td></tr>" % (errorStr(theError), theLine))
190                if (len(errorDetail)>0):
191                   print("<tr><td colspan=4><font color=\"red\">" + errorDetail + "</font></td></tr>")
192
193      def pop(self):
194          if self.suite:
195            print("</table>")
196          self.nb = self.nb - 1
197          self.suite=False
198
199      def end(self):
200        if not self.append:
201           print("</body></html>")
202
203# Return test result as a CSV
204class CSVFormatter:
205
206      def __init__(self):
207        self.name=[]
208        self._start=True
209
210      def start(self):
211          print("CATEGORY,NAME,ID,STATUS,CYCLES,PARAMS")
212
213      def printGroup(self,elem,theId):
214        if elem is None:
215           elem = root
216        # Remove Root from category name in CSV file.
217        if not self._start:
218           self.name.append(elem.data["class"])
219        else:
220            self._start=False
221        message=elem.data["message"]
222        if not elem.data["deprecated"]:
223           kind = "Suite"
224           ident = " " * elem.ident
225           if elem.kind == TestScripts.Parser.TreeElem.GROUP:
226              kind = "Group"
227
228      def printTest(self,elem, theId, theError, errorDetail,theLine,passed,cycles,params):
229          message=elem.data["message"]
230          if not elem.data["deprecated"]:
231             kind = "Test"
232             name=elem.data["class"]
233             category= "".join(list(joinit(self.name,":")))
234             print("%s,%s,%d,%d,%d,\"%s\"" % (category,name,theId,passed,cycles,params))
235
236      def pop(self):
237         if self.name:
238            self.name.pop()
239
240      def end(self):
241        None
242
243class MathematicaFormatter:
244
245      def __init__(self):
246        self._hasContent=[False]
247        self._toPop=[]
248
249      def start(self):
250          None
251
252      def printGroup(self,elem,theId):
253        if self._hasContent[len(self._hasContent)-1]:
254           print(",",end="")
255
256        print("<|")
257        self._hasContent[len(self._hasContent)-1] = True
258        self._hasContent.append(False)
259        if elem is None:
260           elem = root
261        message=elem.data["message"]
262        if not elem.data["deprecated"]:
263
264           kind = "Suite"
265           ident = " " * elem.ident
266           if elem.kind == TestScripts.Parser.TreeElem.GROUP:
267              kind = "Group"
268           print("\"%s\" ->" % (message))
269           #if kind == "Suite":
270           print("{",end="")
271           self._toPop.append("}")
272           #else:
273           #   self._toPop.append("")
274
275      def printTest(self,elem, theId, theError,errorDetail,theLine,passed,cycles,params):
276          message=elem.data["message"]
277          if not elem.data["deprecated"]:
278             kind = "Test"
279             ident = " " * elem.ident
280             p="FAILED"
281             if passed == 1:
282                p="PASSED"
283             parameters=""
284             if params:
285                parameters = "%s" % params
286             if self._hasContent[len(self._hasContent)-1]:
287               print(",",end="")
288             print("<|\"NAME\" -> \"%s\",\"ID\" -> %d,\"STATUS\" -> \"%s\",\"CYCLES\" -> %d,\"PARAMS\" -> \"%s\"|>" % (message,theId,p,cycles,parameters))
289             self._hasContent[len(self._hasContent)-1] = True
290             #if passed != 1:
291             #   print("%s Error = %d at line %d" % (ident, theError, theLine))
292
293      def pop(self):
294          print(self._toPop.pop(),end="")
295          print("|>")
296          self._hasContent.pop()
297
298      def end(self):
299        None
300
301NORMAL = 1
302INTEST = 2
303TESTPARAM = 3
304ERRORDESC = 4
305
306def createMissingDir(destPath):
307  theDir=os.path.normpath(os.path.dirname(destPath))
308  if not os.path.exists(theDir):
309      os.makedirs(theDir)
310
311def correctPath(path):
312  while (path[0]=="/") or (path[0] == "\\"):
313      path = path[1:]
314  return(path)
315
316def extractDataFiles(results,outputDir):
317  infile = False
318  f = None
319  for l in results:
320      if re.match(r'^.*D:[ ].*$',l):
321          if infile:
322            if re.match(r'^.*D:[ ]END$',l):
323               infile = False
324               if f:
325                 f.close()
326            else:
327              if f:
328                m = re.match(r'^.*D:[ ](.*)$',l)
329                data = m.group(1)
330                f.write(data)
331                f.write("\n")
332
333          else:
334            m = re.match(r'^.*D:[ ](.*)$',l)
335            path = str(m.group(1))
336            infile = True
337            destPath = os.path.join(outputDir,correctPath(path))
338            createMissingDir(destPath)
339            f = open(destPath,"w")
340
341
342
343def writeBenchmark(elem,benchFile,theId,theError,passed,cycles,params,config):
344  if benchFile:
345    testname=elem.data["class"]
346    #category= elem.categoryDesc()
347    name=elem.data["message"]
348    category=elem.getSuiteMessage()
349    old=""
350    if "testData" in elem.data:
351      if "oldID" in elem.data["testData"]:
352         old=elem.data["testData"]["oldID"]
353    benchFile.write("\"%s\",\"%s\",\"%s\",%d,\"%s\",%s,%d,%s\n" % (category,testname,name,theId,old,params,cycles,config))
354
355def getCyclesFromTrace(trace):
356  if not trace:
357    return(0)
358  else:
359    return(TestScripts.ParseTrace.getCycles(trace))
360
361def analyseResult(resultPath,root,results,embedded,benchmark,trace,formatter):
362    global resultStatus
363    calibration = 0
364    if trace:
365      # First cycle in the trace is the calibration data
366      # The noramlisation factor must be coherent with the C code one.
367      calibration = int(getCyclesFromTrace(trace) / 20)
368    formatter.start()
369    path = []
370    state = NORMAL
371    prefix=""
372    elem=None
373    theId=None
374    theError=None
375    errorDetail=""
376    theLine=None
377    passed=0
378    cycles=None
379    benchFile = None
380    config=""
381    if embedded:
382       prefix = ".*[S]+:[ ]"
383
384    # Parse the result file.
385    # NORMAL mode is when we are parsing suite or group.
386    # Otherwise we are parsing a test and we need to analyse the
387    # test result.
388    # TESTPARAM is used to read parameters of the test.
389    # Format of output is:
390    #node ident : s id or g id or t or u
391    #test status : id error linenb status Y or N (Y when passing)
392    #param for this test b x,x,x,x or b alone if not param
393    #node end : p
394    # In FPGA mode:
395    #Prefix S:[ ] before driver dump
396    # D:[ ] before data dump (output patterns)
397
398    for l in results:
399        l = l.strip()
400        if not re.match(r'^.*D:[ ].*$',l):
401           if state == NORMAL:
402              if len(l) > 0:
403                 # Line starting with g or s is a suite or group.
404                 # In FPGA mode, those line are prefixed with 'S: '
405                 # and data file with 'D: '
406                 if re.match(r'^%s[gs][ ]+[0-9]+.*$' % prefix,l):
407                    # Extract the test id
408                    theId=re.sub(r'^%s[gs][ ]+([0-9]+).*$' % prefix,r'\1',l)
409                    theId=int(theId)
410                    path.append(theId)
411                    # From a list of id, find the TreeElem in the Parsed tree
412                    # to know what is the node.
413                    elem = findItem(root,path)
414                    # Display formatted output for this node
415                    if elem.params:
416                       #print(elem.params.full)
417                       benchPath = os.path.join(benchmark,elem.fullPath(),"fullBenchmark.csv")
418                       createMissingDir(benchPath)
419                       if benchFile:
420                          printf("ERROR BENCH FILE %s ALREADY OPEN" % benchPath)
421                          benchFile.close()
422                          benchFile=None
423                       benchFile=open(benchPath,"w")
424                       header = "".join(list(joinit(elem.params.full,",")))
425                       # A test and a benchmark are different
426                       # so we don't dump a status and error
427                       # A status and error in a benchmark would
428                       # impact the cycles since the test
429                       # would be taken into account in the measurement
430                       # So benchmark are always passing and contain no test
431                       #benchFile.write("ID,%s,PASSED,ERROR,CYCLES\n" % header)
432                       csvheaders = ""
433
434                       with open(os.path.join(resultPath,'currentConfig.csv'), 'r') as f:
435                          reader = csv.reader(f)
436                          csvheaders = next(reader, None)
437                          configList = list(reader)
438                          #print(configList)
439                          config = "".join(list(joinit(configList[0],",")))
440                          configHeaders = "".join(list(joinit(csvheaders,",")))
441                       benchFile.write("CATEGORY,TESTNAME,NAME,ID,OLDID,%s,CYCLES,%s\n" % (header,configHeaders))
442
443                    formatter.printGroup(elem,theId)
444
445                 # If we have detected a test, we switch to test mode
446                 if re.match(r'^%s[t][ ]*$' % prefix,l):
447                    state = INTEST
448
449
450                 # Pop
451                 # End of suite or group
452                 if re.match(r'^%sp.*$' % prefix,l):
453                   if benchFile:
454                      benchFile.close()
455                      benchFile=None
456                   path.pop()
457                   formatter.pop()
458           elif state == INTEST:
459             if len(l) > 0:
460               # In test mode, we are looking for test status.
461               # A line starting with S
462               # (There may be empty lines or line for data files)
463               passRe = r'^%s([0-9]+)[ ]+([0-9]+)[ ]+([0-9]+)[ ]+([t0-9]+)[ ]+([YN]).*$'  % prefix
464               if re.match(passRe,l):
465                    # If we have found a test status then we will start again
466                    # in normal mode after this.
467
468                    m = re.match(passRe,l)
469
470                    # Extract test ID, test error code, line number and status
471                    theId=m.group(1)
472                    theId=int(theId)
473
474                    theError=m.group(2)
475                    theError=int(theError)
476
477                    theLine=m.group(3)
478                    theLine=int(theLine)
479
480                    maybeCycles = m.group(4)
481                    if maybeCycles == "t":
482                       cycles = getCyclesFromTrace(trace) - calibration
483                    else:
484                       cycles = int(maybeCycles)
485
486                    status=m.group(5)
487                    passed=0
488
489                    # Convert status to number as used by formatter.
490                    if status=="Y":
491                       passed = 1
492                    if status=="N":
493                       passed = 0
494                    # Compute path to this node
495                    newPath=path.copy()
496                    newPath.append(theId)
497                    # Find the node in the Tree
498                    elem = findItem(root,newPath)
499
500
501                    state = ERRORDESC
502               else:
503                 if re.match(r'^%sp.*$' % prefix,l):
504                   if benchFile:
505                      benchFile.close()
506                      benchFile=None
507                   path.pop()
508                   formatter.pop()
509                 if re.match(r'^%s[t][ ]*$' % prefix,l):
510                    state = INTEST
511                 else:
512                    state = NORMAL
513           elif state == ERRORDESC:
514                    if len(l) > 0:
515                       if re.match(r'^.*E:.*$',l):
516                          if re.match(r'^.*E:[ ].*$',l):
517                             m = re.match(r'^.*E:[ ](.*)$',l)
518                             errorDetail = m.group(1)
519                          else:
520                             errorDetail = ""
521                          state = TESTPARAM
522           else:
523             if len(l) > 0:
524                state = INTEST
525                params=""
526                if re.match(r'^.*b[ ]+([0-9,]+)$',l):
527                   m=re.match(r'^.*b[ ]+([0-9,]+)$',l)
528                   params=m.group(1).strip()
529                   # Format the node
530                   #print(elem.fullPath())
531                   #createMissingDir(destPath)
532                   writeBenchmark(elem,benchFile,theId,theError,passed,cycles,params,config)
533                else:
534                   params=""
535                   writeBenchmark(elem,benchFile,theId,theError,passed,cycles,params,config)
536                   # Format the node
537                if not passed:
538                   resultStatus=1
539                formatter.printTest(elem,theId,theError,errorDetail,theLine,passed,cycles,params)
540
541
542    formatter.end()
543
544
545def analyze(root,results,args,trace):
546  # currentConfig.csv should be in the same place
547  resultPath=os.path.dirname(args.r)
548
549  if args.c:
550     analyseResult(resultPath,root,results,args.e,args.b,trace,CSVFormatter())
551  elif args.html:
552     analyseResult(resultPath,root,results,args.e,args.b,trace,HTMLFormatter())
553  elif args.ahtml:
554     analyseResult(resultPath,root,results,args.e,args.b,trace,HTMLFormatter(append=True))
555  elif args.m:
556     analyseResult(resultPath,root,results,args.e,args.b,trace,MathematicaFormatter())
557  else:
558     print("")
559     print(Fore.RED + "The cycles displayed by this script must not be trusted." + Style.RESET_ALL)
560     print(Fore.RED + "They are just an indication. The timing code has not yet been validated." + Style.RESET_ALL)
561     print("")
562
563     analyseResult(resultPath,root,results,args.e,args.b,trace,TextFormatter())
564
565parser = argparse.ArgumentParser(description='Parse test description')
566
567parser.add_argument('-f', nargs='?',type = str, default="Output.pickle", help="Test description file path")
568# Where the result file can be found
569parser.add_argument('-r', nargs='?',type = str, default=None, help="Result file path")
570parser.add_argument('-c', action='store_true', help="CSV output")
571parser.add_argument('-html', action='store_true', help="HTML output")
572parser.add_argument('-ahtml', action='store_true', help="Partial HTML output")
573
574parser.add_argument('-e', action='store_true', help="Embedded test")
575# -o needed when -e is true to know where to extract the output files
576parser.add_argument('-o', nargs='?',type = str, default="Output", help="Output dir path")
577
578parser.add_argument('-b', nargs='?',type = str, default="FullBenchmark", help="Full Benchmark dir path")
579parser.add_argument('-m', action='store_true', help="Mathematica output")
580parser.add_argument('-t', nargs='?',type = str, default=None, help="External trace file")
581
582args = parser.parse_args()
583
584
585
586
587if args.f is not None:
588    #p = parse.Parser()
589    # Parse the test description file
590    #root = p.parse(args.f)
591    root=parse.loadRoot(args.f)
592    if args.t:
593       with open(args.t,"r") as trace:
594         with open(args.r,"r") as results:
595             analyze(root,results,args,iter(trace))
596    else:
597       with open(args.r,"r") as results:
598           analyze(root,results,args,None)
599    if args.e:
600       # In FPGA mode, extract output files from stdout (result file)
601       with open(args.r,"r") as results:
602          extractDataFiles(results,args.o)
603
604    sys.exit(resultStatus)
605
606else:
607    parser.print_help()