1import argparse
2import sqlite3
3import re
4import pandas as pd
5import numpy as np
6from TestScripts.doc.Structure import *
7from TestScripts.doc.Format import *
8import os.path
9import yaml
10
11refCoreName=""
12
13
14# Command to get last runid
15lastID="""SELECT runid FROM RUN ORDER BY runid DESC LIMIT 1
16"""
17
18# Command to get last runid and date
19lastIDAndDate="""SELECT date FROM RUN WHERE runid=?
20"""
21
22# Command to get last runid
23runIDDetails="""SELECT distinct core FROM %s
24INNER JOIN CORE USING(coreid)
25WHERE %s
26"""
27
28def joinit(iterable, delimiter):
29    # Intersperse a delimiter between element of a list
30    it = iter(iterable)
31    yield next(it)
32    for x in it:
33        yield delimiter
34        yield x
35
36
37def getLastRunID():
38  r=c.execute(lastID)
39  return(int(r.fetchone()[0]))
40
41def getrunIDDate(forID):
42  r=c.execute(lastIDAndDate,(forID,))
43  return(r.fetchone()[0])
44
45
46
47
48parser = argparse.ArgumentParser(description='Generate summary benchmarks')
49
50parser.add_argument('-b', nargs='?',type = str, default="bench.db", help="Database")
51parser.add_argument('-o', nargs='?',type = str, default="full.md", help="Full summary")
52parser.add_argument('-r', action='store_true', help="Regression database")
53parser.add_argument('-t', nargs='?',type = str, default="md", help="type md or html")
54parser.add_argument('-byc', action='store_true', help="Result oganized by Compiler")
55parser.add_argument('-g', action='store_true', help="Include graphs in regression report")
56
57parser.add_argument('-details', action='store_true', help="Details about runids")
58parser.add_argument('-lastid', action='store_true', help="Get last ID")
59parser.add_argument('-comments', nargs='?',type = str, default="comments", help="Comment folder")
60parser.add_argument('-byd', action='store_true', help="Result oganized by datatype")
61parser.add_argument('-ratio', action='store_true', help="Compute ratios for regression by core instead of cycles")
62parser.add_argument('-ref', nargs='?',type = str, default="M55", help="Reference COREDEF for ratio in db")
63parser.add_argument('-clampval', nargs='?',type = float, default=8.0, help="Clamp for ratio")
64parser.add_argument('-clamp', action='store_true', help="Clamp enabled for ratio")
65parser.add_argument('-cores', nargs='?',type = str, help="Cores to keep")
66parser.add_argument('-toc', nargs='?',type = str, help="Yaml for the table of contents")
67
68# For runid or runid range
69parser.add_argument('others', nargs=argparse.REMAINDER,help="Run ID")
70
71args = parser.parse_args()
72
73c = sqlite3.connect(args.b)
74
75coreidSQL="select distinct coreid from CORE where coredef==?"
76
77def getCoreID(corename):
78  r=c.execute(coreidSQL,(corename,))
79  t=r.fetchone()
80  if t is None:
81    print("Unrecognized reference core \"%s\"" % corename)
82    quit()
83  return(t[0])
84
85def parseSelector(o,field="runid"):
86  vals=[]
87  runidCMD=[]
88  # parameters are not allowed in VIEWs
89  runidVIEWcmd=[]
90  for t in o:
91     if re.search(r'-',t):
92       bounds=[int(x) for x in t.split("-")]
93       vals += bounds
94       runidCMD += ["(%s >= ? AND %s <= ?)" % (field,field)]
95       x=(field,bounds[0],field,bounds[1])
96       runidVIEWcmd += ["(%s >= %d AND %s <= %d)" % x]
97     else:
98      theid=int(t)
99      runidCMD += ["%s == ?" % field]
100      runidVIEWcmd += ["%s == %d" % (field,theid)]
101      vals.append(theid)
102
103  runidval = tuple(vals)
104  runidCMD = "(" + "".join(joinit(runidCMD," OR ")) + ")"
105  runidVIEWcmd = "(" + "".join(joinit(runidVIEWcmd," OR ")) + ")"
106  return(runidval,runidCMD,runidVIEWcmd)
107
108if args.others:
109
110   runidval,runidCMD,runidVIEWcmd = parseSelector(args.others)
111
112else:
113   theid=getLastRunID()
114   print("Last run ID = %d\n" % theid)
115   runidval=(theid,)
116   runidCMD = "runid = ?"
117   runidVIEWcmd="(runid = %d)" % theid
118
119# None means all
120coreidval = []
121coreidCMD = []
122keepCoreIds=None
123if args.cores:
124   cores=args.cores.split(",")
125   coreids = [str(getCoreID(x.strip())) for x in cores]
126   keepCoreIds = coreids.copy()
127   if args.ref:
128      coreids.append(str(getCoreID(args.ref.strip())))
129   #print(coreids)
130   coreidval,coreidCMD, coreidVIEWcmd = parseSelector(coreids,field="coreid")
131   runidval += coreidval
132   runidCMD += " AND %s" % coreidCMD
133   runidVIEWcmd += " AND %s" % coreidVIEWcmd
134
135
136# We extract data only from data tables
137# Those tables below are used for descriptions
138REMOVETABLES=['TESTNAME','TESTDATE','RUN','CORE', 'PLATFORM', 'COMPILERKIND', 'COMPILER', 'TYPE', 'CATEGORY', 'CONFIG']
139
140# This is assuming the database is generated by the regression script
141# So platform is the same for all benchmarks.
142# Category and type is coming from the test name in the yaml
143# So no need to add this information here
144# Name is removed here because it is added at the beginning
145REMOVECOLUMNS=['runid','name','type','platform','category','coredef','OPTIMIZED','HARDFP','FASTMATH','NEON','HELIUM','UNROLL','ROUNDING','DATE','compilerkindid','date','categoryid', 'ID', 'platformid', 'coreid', 'compilerid', 'typeid']
146
147REMOVECOLUMNSFORHISTORY=['Regression','MAXREGCOEF','name','type','platform','category','coredef','OPTIMIZED','HARDFP','FASTMATH','NEON','HELIUM','UNROLL','ROUNDING','DATE','compilerkindid','date','categoryid', 'ID', 'platformid', 'coreid', 'compilerid', 'typeid']
148
149# Get existing benchmark tables
150def getBenchTables():
151    r=c.execute("SELECT name FROM sqlite_master WHERE type='table'")
152    benchtables=[]
153    for table in r:
154        if not table[0] in REMOVETABLES:
155          benchtables.append(table[0])
156    return(benchtables)
157
158# get existing types in a table
159def getExistingTypes(benchTable):
160    r=c.execute("select distinct typeid from %s WHERE %s order by typeid desc " % (benchTable,runidCMD),runidval).fetchall()
161    result=[x[0] for x in r]
162    return(result)
163
164# Get existing cores in a table
165def getAllExistingCores(benchTable):
166    r=c.execute("select distinct coreid from %s WHERE %s order by coreid desc " % (benchTable,runidCMD),runidval).fetchall()
167    result=[x[0] for x in r]
168    return(result)
169
170def getrunIDDetails():
171  tables=getBenchTables()
172  r=[]
173  for table in tables:
174      r += [x[0] for x in c.execute(runIDDetails % (table,runidCMD),runidval).fetchall()]
175  r=list(set(r))
176  print(r)
177
178if args.lastid:
179   quit()
180
181if args.details:
182   getrunIDDetails()
183   quit()
184
185
186# Get compilers from specific type and table
187allCompilers="""select distinct compilerid from %s WHERE typeid=?"""
188
189# Get compilers from specific type and table
190allCompilerForCore="""select distinct compilerid from %s WHERE coreid=?"""
191
192# Get compilers from specific type and table
193allCores="""select distinct coreid from %s WHERE typeid=? AND (%s)"""
194
195
196compilerDesc="""select compiler,version from COMPILER
197  INNER JOIN COMPILERKIND USING(compilerkindid) WHERE compilerid=?"""
198
199coreDesc="""select core from CORE WHERE coreid=?"""
200
201# Get existing compiler in a table for a specific type
202# (In case report is structured by types)
203def getExistingCompiler(benchTable,typeid):
204    r=c.execute(allCompilers % benchTable,(typeid,)).fetchall()
205    return([x[0] for x in r])
206
207
208# Get existing compiler in a table for a specific core
209# (In case report is structured by core)
210def getExistingCompilerForCore(benchTable,coreid):
211    r=c.execute(allCompilerForCore % benchTable,(coreid,)).fetchall()
212    return([x[0] for x in r])
213
214def getExistingCores(benchTable,typeid):
215    vals = (typeid,) + runidval
216    r=c.execute(allCores % (benchTable,runidCMD),vals).fetchall()
217    return([x[0] for x in r])
218
219
220
221def getCoreDesc(coreid):
222    r=c.execute(coreDesc,(coreid,)).fetchone()
223    return(r)
224
225def getCompilerDesc(compilerid):
226    r=c.execute(compilerDesc,(compilerid,)).fetchone()
227    return(r)
228
229# Get type name from type id
230def getTypeName(typeid):
231    r=c.execute("select type from TYPE where typeid=?",(typeid,)).fetchone()
232    return(r[0])
233
234# Diff of 2 lists
235def diff(first, second):
236        second = set(second)
237        return [item for item in first if item not in second]
238
239
240# Command to get data for specific compiler
241# and type
242benchCmdForCoreCompiler="""select %s from %s
243  INNER JOIN CATEGORY USING(categoryid)
244  INNER JOIN PLATFORM USING(platformid)
245  INNER JOIN CORE USING(coreid)
246  INNER JOIN COMPILER USING(compilerid)
247  INNER JOIN COMPILERKIND USING(compilerkindid)
248  INNER JOIN TYPE USING(typeid)
249  INNER JOIN TESTNAME USING(testnameid)
250  WHERE coreid=? AND compilerid = ? AND (%s)
251  """
252
253# Command to get data for specific core
254# and type
255historyCmd="""select %s from %s
256  INNER JOIN CATEGORY USING(categoryid)
257  INNER JOIN PLATFORM USING(platformid)
258  INNER JOIN CORE USING(coreid)
259  INNER JOIN COMPILER USING(compilerid)
260  INNER JOIN COMPILERKIND USING(compilerkindid)
261  INNER JOIN TYPE USING(typeid)
262  INNER JOIN TESTNAME USING(testnameid)
263  WHERE compilerid=? AND coreid=? AND typeid = ? AND ID = ? AND runid > (? - 10)
264  """
265
266compilersForHistory="""select distinct compilerid,compiler,version  from %s
267  INNER JOIN COMPILER USING(compilerid)
268  INNER JOIN COMPILERKIND USING(compilerkindid)
269  WHERE coreid=? AND typeid = ? AND ID = ? AND runid > (? - 10)
270  """
271
272# Command to get data for specific core
273# and type
274benchCmdForCore="""select %s from %s
275  INNER JOIN CATEGORY USING(categoryid)
276  INNER JOIN PLATFORM USING(platformid)
277  INNER JOIN CORE USING(coreid)
278  INNER JOIN COMPILER USING(compilerid)
279  INNER JOIN COMPILERKIND USING(compilerkindid)
280  INNER JOIN TYPE USING(typeid)
281  INNER JOIN TESTNAME USING(testnameid)
282  WHERE coreid=? AND typeid = ? AND (%s)
283  """
284
285coresForHistory="""select distinct coreid,core from %s
286  INNER JOIN CORE USING(coreid)
287  WHERE compilerid=? AND typeid = ? AND ID = ? AND runid > (? - 10)
288  """
289
290# Command to get data for specific compiler
291# and type
292benchCmdForCompiler="""select %s from %s
293  INNER JOIN CATEGORY USING(categoryid)
294  INNER JOIN PLATFORM USING(platformid)
295  INNER JOIN CORE USING(coreid)
296  INNER JOIN COMPILER USING(compilerid)
297  INNER JOIN COMPILERKIND USING(compilerkindid)
298  INNER JOIN TYPE USING(typeid)
299  INNER JOIN TESTNAME USING(testnameid)
300  WHERE compilerid=? AND typeid = ? AND (%s)
301  """
302
303# Command to get test names for specific compiler
304# and type
305benchNamesForCore="""select distinct name from %s
306  INNER JOIN COMPILER USING(compilerid)
307  INNER JOIN COMPILERKIND USING(compilerkindid)
308  INNER JOIN TYPE USING(typeid)
309  INNER JOIN TESTNAME USING(testnameid)
310  WHERE coreid=? AND typeid = ? AND (%s)
311  """
312
313# Command to get test names for specific core
314# and compiler
315benchNamesForCoreCompiler="""select distinct name from %s
316  INNER JOIN COMPILER USING(compilerid)
317  INNER JOIN COMPILERKIND USING(compilerkindid)
318  INNER JOIN TYPE USING(typeid)
319  INNER JOIN TESTNAME USING(testnameid)
320  WHERE coreid=? AND compilerid = ? AND (%s)
321  """
322
323# Command to get test names for specific compiler
324# and type
325benchNamesForCompiler="""select distinct name from %s
326  INNER JOIN COMPILER USING(compilerid)
327  INNER JOIN COMPILERKIND USING(compilerkindid)
328  INNER JOIN TYPE USING(typeid)
329  INNER JOIN TESTNAME USING(testnameid)
330  WHERE compilerid=? AND typeid = ? AND (%s)
331  """
332
333# Command to get columns for specific table
334benchCmdColumns="""select * from %s
335  INNER JOIN CATEGORY USING(categoryid)
336  INNER JOIN PLATFORM USING(platformid)
337  INNER JOIN CORE USING(coreid)
338  INNER JOIN COMPILER USING(compilerid)
339  INNER JOIN COMPILERKIND USING(compilerkindid)
340  INNER JOIN TESTNAME USING(testnameid)
341  INNER JOIN TYPE USING(typeid)
342  """
343
344def joinit(iterable, delimiter):
345    it = iter(iterable)
346    yield next(it)
347    for x in it:
348        yield delimiter
349        yield x
350
351# Is not a column name finishing by id
352# (often primary key for thetable)
353def isNotIDColumn(col):
354    if re.match(r'^.*id$',col):
355        return(False)
356    else:
357        return(True)
358
359# Get test names
360# for specific core and compiler (for the data)
361def getTestNamesForCoreCompiler(benchTable,compilerid,core):
362    vals=(core,compilerid) + runidval
363    result=c.execute(benchNamesForCoreCompiler % (benchTable,runidCMD),vals).fetchall()
364    names=[x[0] for x in list(result)]
365    return(names)
366
367# Get test names
368# for specific typeid and core (for the data)
369def getTestNamesForCore(benchTable,core,typeid):
370    vals=(core,typeid) + runidval
371    result=c.execute(benchNamesForCore % (benchTable,runidCMD),vals).fetchall()
372    names=[x[0] for x in list(result)]
373    return(names)
374
375# Get test names
376# for specific typeid and compiler (for the data)
377def getTestNamesForCompiler(benchTable,comp,typeid):
378    vals=(comp,typeid) + runidval
379    result=c.execute(benchNamesForCompiler % (benchTable,runidCMD),vals).fetchall()
380    names=[x[0] for x in list(result)]
381    return(names)
382
383# Command to get data for specific core
384# and type
385nbElemsInBenchAndTypeAndCoreCmd="""select count(*) from %s
386  WHERE coreid=? AND typeid = ? AND (%s)
387  """
388
389# Command to get data for specific compiler
390# and type
391nbElemsInBenchAndTypeAndCompilerCmd="""select count(*) from %s
392  WHERE compilerid=? AND typeid = ? AND (%s)
393  """
394
395# Command to get data for specific compiler
396# and type
397nbElemsInBenchAndCoreAndCompilerCmd="""select count(*) from %s
398  WHERE compilerid=? AND coreid = ? AND (%s)
399  """
400
401nbElemsInBenchAndTypeCmd="""select count(*) from %s
402  WHERE typeid = ? AND (%s)
403  """
404nbElemsInBenchAndCoreCmd="""select count(*) from %s
405  WHERE coreid = ? AND (%s)
406  """
407
408nbElemsInBenchCmd="""select count(*) from %s
409  WHERE %s
410  """
411
412categoryName="""select distinct category from %s
413  INNER JOIN CATEGORY USING(categoryid)
414  WHERE %s
415  """
416
417def getCategoryName(benchTable):
418  result=c.execute(categoryName % (benchTable,runidCMD),runidval).fetchone()
419  return(result[0])
420
421# Get nb elems in a table
422def getNbElemsInBenchAndTypeAndCoreCmd(benchTable,coreid,typeid):
423    vals=(coreid,typeid) + runidval
424    result=c.execute(nbElemsInBenchAndTypeAndCoreCmd % (benchTable,runidCMD),vals).fetchone()
425    return(result[0])
426
427# Get nb elems in a table
428def getNbElemsInBenchAndTypeAndCompilerCmd(benchTable,comp,typeid):
429    vals=(comp,typeid) + runidval
430    result=c.execute(nbElemsInBenchAndTypeAndCompilerCmd % (benchTable,runidCMD),vals).fetchone()
431    return(result[0])
432
433# Get nb elems in a table
434def getNbElemsInBenchAndCoreAndCompilerCmd(benchTable,comp,coreid):
435    vals=(comp,coreid) + runidval
436    result=c.execute(nbElemsInBenchAndCoreAndCompilerCmd % (benchTable,runidCMD),vals).fetchone()
437    return(result[0])
438
439def getNbElemsInBenchAndTypeCmd(benchTable,typeid):
440    vals=(typeid,) + runidval
441    result=c.execute(nbElemsInBenchAndTypeCmd % (benchTable,runidCMD),vals).fetchone()
442    return(result[0])
443
444def getNbElemsInBenchAndCoreCmd(benchTable,coreid):
445    vals=(coreid,) + runidval
446    result=c.execute(nbElemsInBenchAndCoreCmd % (benchTable,runidCMD),vals).fetchone()
447    return(result[0])
448
449def getNbElemsInBenchCmd(benchTable):
450    result=c.execute(nbElemsInBenchCmd % (benchTable,runidCMD),runidval).fetchone()
451    return(result[0])
452
453# Get names of columns and data for a table
454# for specific typeid and coreid (for the data)
455def getColNamesAndHistory(benchTable,compiler,core,typeid,testid):
456    cursor=c.cursor()
457    result=cursor.execute(benchCmdColumns % (benchTable))
458    cols= [member[0] for member in cursor.description]
459    keepCols = ['name'] + [c for c in diff(cols , REMOVECOLUMNSFORHISTORY) if isNotIDColumn(c)]
460    keepColsStr = "".join(joinit(keepCols,","))
461    vals=(compiler,core,typeid,testid,runid)
462    result=cursor.execute(historyCmd % (keepColsStr,benchTable),vals)
463    vals =np.array([list(x) for x in list(result)])
464    return(keepCols,vals)
465
466# Get names of columns and data for a table
467# for specific typeid and coreid (for the data)
468def getColNamesAndDataForCore(benchTable,core,typeid):
469    cursor=c.cursor()
470    result=cursor.execute(benchCmdColumns % (benchTable))
471    cols= [member[0] for member in cursor.description]
472    keepCols = ['name'] + [c for c in diff(cols , REMOVECOLUMNS) if isNotIDColumn(c)]
473    keepColsStr = "".join(joinit(keepCols,","))
474    vals=(core,typeid) + runidval
475    result=cursor.execute(benchCmdForCore % (keepColsStr,benchTable,runidCMD),vals)
476    vals =np.array([list(x) for x in list(result)])
477    return(keepCols,vals)
478
479# Get names of columns and data for a table
480# for specific coreid and compilerid (for the data)
481def getColNamesAndDataForCoreCompiler(benchTable,compilerid,core):
482    cursor=c.cursor()
483    result=cursor.execute(benchCmdColumns % (benchTable))
484    cols= [member[0] for member in cursor.description]
485    keepCols = ['name','type'] + [c for c in diff(cols , REMOVECOLUMNS) if isNotIDColumn(c)]
486    keepColsStr = "".join(joinit(keepCols,","))
487    vals=(core,compilerid) + runidval
488    result=cursor.execute(benchCmdForCoreCompiler % (keepColsStr,benchTable,runidCMD),vals)
489    vals =np.array([list(x) for x in list(result)])
490    return(keepCols,vals)
491
492# Get names of columns and data for a table
493# for specific typeid and compiler (for the data)
494def getColNamesAndDataForCompiler(benchTable,comp,typeid):
495    cursor=c.cursor()
496    result=cursor.execute(benchCmdColumns % (benchTable))
497    cols= [member[0] for member in cursor.description]
498    keepCols = ['name'] + [c for c in diff(cols , REMOVECOLUMNS) if isNotIDColumn(c)]
499    keepColsStr = "".join(joinit(keepCols,","))
500    vals=(comp,typeid) + runidval
501    result=cursor.execute(benchCmdForCompiler % (keepColsStr,benchTable,runidCMD),vals)
502    vals =np.array([list(x) for x in list(result)])
503    return(keepCols,vals)
504
505def formatFloat(s):
506  result=[]
507  for t in s:
508    if type(t) is float:
509      result.append(("%.3f" % t))
510    else:
511      result.append(t)
512  return(result)
513
514PARAMS=["NB","NumTaps", "NBA", "NBB", "Factor", "NumStages","VECDIM","NBR","NBC","NBI","IFFT", "BITREV"]
515
516def regressionTableFor(byname,name,section,ref,toSort,indexCols,field):
517    data=ref.pivot_table(index=indexCols, columns=byname,
518    values=[field], aggfunc='first',fill_value="NA")
519
520    data=data.sort_values(toSort)
521
522    if args.byc:
523      cores = [(c[1] + ":" + c[2]) for c in list(data.columns)]
524    else:
525      cores = [c[1] for c in list(data.columns)]
526    columns = diff(indexCols,['name'])
527
528
529    dataTable=Table(columns,cores)
530    section.addContent(dataTable)
531
532    dataForFunc=data.loc[name]
533
534    if type(dataForFunc) is pd.DataFrame:
535       bars={'cols':columns,'cores':cores,'data':[]}
536       for row in dataForFunc.itertuples():
537           row=list(row)
538           if type(row[0]) is int:
539              row=[row[0]] + row[1:]
540           else:
541              row=list(row[0]) + row[1:]
542           if field=="MAXREGCOEF":
543              newrow = row
544              newrow[len(columns):] = formatFloat(row[len(columns):])
545              row=newrow
546           dataTable.addRow(row)
547           bars['data'].append(row)
548       return(bars)
549    else:
550       if field=="MAXREGCOEF":
551              dataForFunc=formatFloat(dataForFunc)
552       dataTable.addRow(dataForFunc)
553       return(list(zip(cores,dataForFunc)))
554
555def formatColumnName(c):
556  return("".join(joinit(c,":")))
557
558def getCoresForHistory(benchTable,compilerid,typeid,testid,runid):
559    vals=(compilerid,typeid,testid,runid)
560    result=c.execute(coresForHistory % benchTable,vals).fetchall()
561    ids=[(x[0],x[1]) for x in list(result)]
562    return(ids)
563
564def getCompilerForHistory(benchTable,coreid,typeid,testid,runid):
565    vals=(coreid,typeid,testid,runid)
566    result=c.execute(compilersForHistory % benchTable,vals).fetchall()
567    ids=[(x[0],x[1],x[2]) for x in list(result)]
568    return(ids)
569
570def getHistory(desc,testid,indexCols):
571    benchName,sectionID,typeid,runid = desc
572
573    #print(benchName)
574    #print(sectionID)
575    #print(typeid)
576    #print(testid)
577    columns = diff(indexCols,['name'])
578    #print(columns)
579    if args.byc:
580       coreid=sectionID
581       compilerids=getCompilerForHistory(benchName,coreid,typeid,testid,runid)
582       series={}
583       for compilerid,compilername,version in compilerids:
584          result=getColNamesAndHistory(benchName,compilerid,coreid,typeid,testid)
585          #print("%s:%s" % (compilername,version))
586          maxpos = result[0].index('MAX')
587          lrunid = result[0].index('runid')
588          r=[[int(x[lrunid]),int(x[maxpos])] for x in result[1:][0]]
589          series[corename]=r
590       hist=History(series,runid)
591       return(hist)
592    else:
593       compilerid=sectionID
594       coreids = getCoresForHistory(benchName,compilerid,typeid,testid,runid)
595       series={}
596       for coreid,corename in coreids:
597          result=getColNamesAndHistory(benchName,compilerid,coreid,typeid,testid)
598          #print(corename)
599          maxpos = result[0].index('MAX')
600          corepos = result[0].index('core')
601          lrunid = result[0].index('runid')
602          r=[[int(x[lrunid]),int(x[maxpos])] for x in result[1:][0]]
603          series[corename]=r
604       hist=History(series,runid)
605       return(hist)
606
607def convertRowToInt(r):
608  result=[]
609  for e in r:
610    if type(e) is float:
611      result.append(int(e))
612    else:
613      result.append(e)
614
615  return(result)
616
617
618def addSectionComment(section):
619  if os.path.exists(args.comments):
620     fileName=re.sub(r'[() :]','',section.name)
621     path=os.path.join(args.comments,fileName+".html")
622     para=""
623     if os.path.exists(path):
624        commentSection = Section("Comments")
625        section.addSection(commentSection)
626        with open(path,"r") as r:
627            for l in r:
628                if l.strip():
629                  para += l
630                else:
631                  commentSection.addContent(Text(para))
632                  para=""
633        if para:
634           commentSection.addContent(Text(para))
635
636
637def formatTableBy(desc,byname,section,typeSection,testNames,cols,vals):
638    if vals.size != 0:
639       ref=pd.DataFrame(vals,columns=cols)
640       toSort=["name"]
641
642       for param in PARAMS:
643          if param in ref.columns:
644             ref[param]=pd.to_numeric(ref[param])
645             toSort.append(param)
646       if args.r:
647         #  Regression table
648         ref['MAX']=pd.to_numeric(ref['MAX'])
649         ref['MAXREGCOEF']=pd.to_numeric(ref['MAXREGCOEF'])
650
651         indexCols=diff(cols,byname + ['Regression','MAXREGCOEF','MAX'] + section)
652         valList = ['Regression']
653
654       else:
655         ref['CYCLES']=pd.to_numeric(ref['CYCLES']).round(decimals=0)
656
657         indexCols=diff(cols,byname + ['CYCLES'] + section)
658         valList = ['CYCLES']
659
660
661       for name in testNames:
662           if args.r:
663              testSection = Section(name)
664              testSection.setTest()
665              typeSection.addSection(testSection)
666              addSectionComment(testSection)
667
668              maxCyclesSection = Section("Max cycles")
669              testSection.addSection(maxCyclesSection)
670              theCycles=regressionTableFor(byname,name,maxCyclesSection,ref,toSort,indexCols,'MAX')
671              if args.g:
672                 if type(theCycles) is dict:
673                   nbParams=len(theCycles['cols'])
674                   for bar in theCycles['data']:
675                       params=bar[0:nbParams]
676                       values=bar[nbParams:]
677                       title=[("%s=%s" % x) for x in list(zip(theCycles['cols'],params))]
678                       title="".join(joinit(title," "))
679                       sec=Section(title)
680                       maxCyclesSection.addSection(sec)
681                       values=list(zip(theCycles['cores'],values))
682                       barChart=BarChart(values)
683                       sec.addContent(barChart)
684                 else:
685                    #print(theCycles)
686                    sec=Section("Graph")
687                    maxCyclesSection.addSection(sec)
688                    barChart=BarChart(theCycles)
689                    sec.addContent(barChart)
690
691                 #history=getHistory(desc,testid,indexCols)
692                 #testSection.addContent(history)
693
694              regressionSection = Section("Regression")
695              testSection.addSection(regressionSection)
696              regressionTableFor(byname,name,regressionSection,ref,toSort,indexCols,'Regression')
697
698
699              maxRegCoefSection = Section("Max Reg Coef")
700              testSection.addSection(maxRegCoefSection)
701              regressionTableFor(byname,name,maxRegCoefSection,ref,toSort,indexCols,'MAXREGCOEF')
702
703           else:
704              data=ref.pivot_table(index=indexCols, columns=byname,
705              values=valList, aggfunc='first',fill_value="NA")
706
707              data=data.sort_values(toSort)
708
709              #print(list(data.columns))
710
711              testSection = Section(name)
712              testSection.setTest()
713              typeSection.addSection(testSection)
714              addSectionComment(testSection)
715
716              dataForFunc=data.loc[name]
717              dataForFunc = dataForFunc.dropna(axis=1)
718
719              columnsID = [formatColumnName(c[1:]) for c in list(dataForFunc.columns)]
720              columns = diff(indexCols,['name'])
721
722              dataTable=Table(columns,columnsID)
723              testSection.addContent(dataTable)
724
725
726              if type(dataForFunc) is pd.DataFrame:
727                 for row in dataForFunc.itertuples():
728                     if type(row[0]) is int:
729                        row=list([row[0]] + list(row[1:]))
730                     else:
731                        row=list(row[0]) + list(row[1:])
732                     dataTable.addRow(convertRowToInt(row))
733              else:
734                 dataTable.addRow(convertRowToInt(dataForFunc))
735
736referenceCoreID = None
737
738
739
740
741
742refCore="""CREATE TEMP VIEW if not exists refCore AS
743select * from %s where (coreid = %s) and (typeid =  %s)
744 and %s
745 and compilerid = %s"""
746
747
748otherCore="""CREATE TEMP VIEW if not exists otherCore AS
749select * from %s where (typeid =  %s)
750 and %s
751 and compilerid = %s"""
752
753refCoreAllTypes="""CREATE TEMP VIEW if not exists refCore AS
754select * from %s where (coreid = %s)
755 and %s
756 and compilerid = %s"""
757
758otherCoreAllTypes="""CREATE TEMP VIEW if not exists otherCore AS
759select * from %s where (coreid = %s)
760 and %s
761 and compilerid = %s"""
762
763
764ratioSQL="""select name,otherCore.compilerid as compilerid,CORE.core as core,%s(CAST(otherCore.MAX as FLOAT) / CAST(refCore.MAX AS FLOAT)) AS RATIO
765 from otherCore
766 INNER JOIN refCore ON (refCore.categoryid = otherCore.categoryid
767 AND refCore.testnameid = otherCore.testnameid
768 AND refCore.typeid = otherCore.typeid
769 AND refCore.runid = otherCore.runid
770 AND refCore.compilerid = otherCore.compilerid
771 )
772 INNER JOIN TESTNAME ON TESTNAME.testnameid = refCore.testnameid
773 INNER JOIN CORE USING(coreid)
774 %s"""
775
776ratioSQLAllTypes="""select name,otherCore.compilerid as compilerid,TYPE.type as type,%s(CAST(otherCore.MAX as FLOAT) / CAST(refCore.MAX AS FLOAT)) AS RATIO
777 from otherCore
778 INNER JOIN refCore ON (refCore.categoryid = otherCore.categoryid
779 AND refCore.testnameid = otherCore.testnameid
780 AND refCore.typeid = otherCore.typeid
781 AND refCore.runid = otherCore.runid
782 AND refCore.compilerid = otherCore.compilerid
783 )
784 INNER JOIN TESTNAME ON TESTNAME.testnameid = refCore.testnameid
785 INNER JOIN TYPE USING(typeid)
786 %s"""
787
788ratioTestNamesSQL="""select distinct TESTNAME.name
789 from otherCore
790 INNER JOIN refCore ON (refCore.categoryid = otherCore.categoryid
791 AND refCore.testnameid = otherCore.testnameid
792 AND refCore.typeid = otherCore.typeid
793 AND refCore.runid = otherCore.runid
794 AND refCore.compilerid = otherCore.compilerid
795 )
796 INNER JOIN TESTNAME ON TESTNAME.testnameid = refCore.testnameid
797 INNER JOIN CORE USING(coreid)
798 %s"""
799
800
801dropViewsRef="""drop view refCore"""
802
803dropViewsOther="""drop view otherCore"""
804
805def getTableParams(benchTable):
806  cursor=c.cursor()
807  result=cursor.execute("select * from %s limit 1" % (benchTable))
808  cols= [member[0] for member in cursor.description]
809  params=[]
810  for x in cols:
811    if x in PARAMS:
812      params.append(x)
813  return(params)
814
815def computeRatio(benchName,viewParams,refMkViewCmd,otherMkViewCmd,byd):
816  params = getTableParams(benchName)
817  cols=["ratio"]
818  paramscmd=""
819  paramscols=""
820  paramsnames = ["refCore.%s as %s" % (x,x) for x in params]
821  paramseq = ["refCore.%s = otherCore.%s" % (x,x) for x in params]
822  if len(params) > 0:
823     cols = ["%s" % x for x in params]
824     cols.append("ratio")
825     paramscols= ("".join(joinit(paramsnames," , ")) + ",")
826     paramscmd = "WHERE " + "".join(joinit(paramseq," AND "))
827  if byd:
828    ratioCmd = ratioSQLAllTypes  % (paramscols,paramscmd)
829  else:
830    ratioCmd = ratioSQL  % (paramscols,paramscmd)
831  ratioTestNames = ratioTestNamesSQL  % (paramscmd)
832
833  #print(refMkViewCmd)
834  #print(otherMkViewCmd)
835  #
836  #print(ratioCmd)
837  #
838  #print(dropViewsRef)
839  #print(dropViewsOther)
840  #quit()
841
842  c.execute(refMkViewCmd)
843  c.execute(otherMkViewCmd)
844
845  ratio=c.execute(ratioCmd).fetchall()
846  testNames=c.execute(ratioTestNames).fetchall()
847  testNames=[x[0] for x in testNames]
848
849  c.execute(dropViewsRef)
850  c.execute(dropViewsOther)
851
852  #print(ratio)
853  #quit()
854  if byd:
855     return(['name','compilerid','type'] + cols,params,ratio,testNames)
856  else:
857     return(['name','compilerid','core'] + cols,params,ratio,testNames)
858
859# Compute for all core for a given type
860def computeRatioTable(benchName,referenceCore,typeID,compiler):
861  viewParams = (benchName,referenceCore,typeID,runidVIEWcmd,compiler)
862  refMkViewCmd = refCore % viewParams
863  otherParams = (benchName,typeID,runidVIEWcmd,compiler)
864  otherMkViewCmd = otherCore % otherParams
865  #print(refMkViewCmd)
866  #print(otherMkViewCmd)
867  return(computeRatio(benchName,viewParams,refMkViewCmd,otherMkViewCmd,False))
868
869
870def computeRatioTableForCore(benchName,referenceCore,otherCoreID,compiler):
871  viewParams = (benchName,referenceCore,runidVIEWcmd,compiler)
872  refMkViewCmd = refCoreAllTypes % viewParams
873  otherParams = (benchName,otherCoreID,runidVIEWcmd,compiler)
874  otherMkViewCmd = otherCoreAllTypes % otherParams
875  #print(refMkViewCmd)
876  #print(otherMkViewCmd)
877  return(computeRatio(benchName,viewParams,refMkViewCmd,otherMkViewCmd,True))
878
879def formatPerfRatio(s):
880  result=[]
881  for t in s:
882    if type(t) is float:
883      if args.clamp:
884         if t > args.clampval:
885            t = args.clampval
886      if t < 0.0:
887        result.append("NA")
888      else:
889        result.append(("%.3f" % t))
890    else:
891      result.append(s)
892
893  return(result)
894
895
896
897def addRatioTable(cols,params,data,section,testNames,byd):
898  ref=pd.DataFrame(data,columns=cols)
899  toSort=["name"] + params
900  for param in PARAMS:
901    if param in ref.columns:
902        ref[param]=pd.to_numeric(ref[param])
903
904  #print(testNames)
905  for name in testNames:
906      testSection = Section(name)
907      testSection.setTest()
908      section.addSection(testSection)
909      addSectionComment(testSection)
910
911
912      ratioSection = Section("Ratios")
913      testSection.addSection(ratioSection)
914
915      #print(toSort)
916      #print(ref)
917
918      if byd:
919        data=ref.pivot_table(index=toSort, columns=['type'],
920           values=["ratio"], aggfunc='first',fill_value=-1.0)
921      else:
922         data=ref.pivot_table(index=toSort, columns=['core'],
923           values=["ratio"], aggfunc='first')
924
925      data=data.sort_values(toSort)
926
927      #print(data)
928      dataForFunc=data.loc[name]
929
930      cores = [c[1] for c in list(data.columns)]
931
932      dataTable=Table(params,cores)
933
934      if args.g:
935         if type(dataForFunc) is not pd.DataFrame:
936            sec=Section("Graph")
937            testSection.addSection(sec)
938
939            barChart=BarChart(zip(cores,dataForFunc))
940            sec.addContent(barChart)
941
942      ratioSection.addContent(Text("A bigger ratio means the reference core \"%s\" is better" % refCoreName))
943
944      ratioSection.addContent(dataTable)
945
946      if type(dataForFunc) is pd.DataFrame:
947         for row in dataForFunc.itertuples():
948           row=list(row)
949           if type(row[0]) is int:
950              row=[row[0]] + formatPerfRatio(row[1:])
951           else:
952              row=list(row[0]) + formatPerfRatio(row[1:])
953           dataTable.addRow(row)
954      else:
955         row=list(dataForFunc)
956         dataTable.addRow(formatPerfRatio(row))
957
958
959
960# Add a report for each table
961def addReportFor(document,benchName):
962    nbElems = getNbElemsInBenchCmd(benchName)
963    if nbElems > 0:
964       categoryName = getCategoryName(benchName)
965       benchSection = Section(categoryName)
966       document.addSection(benchSection)
967       print("Process %s\n" % benchName)
968       if args.byd:
969          allCores=getAllExistingCores(benchName)
970          if args.ratio:
971             if keepCoreIds:
972                allCores=keepCoreIds
973             if ("%d" % referenceCoreID) in allCores:
974                allCores.remove("%d" % referenceCoreID)
975          for aCoreID in allCores:
976            nbElems = getNbElemsInBenchAndCoreCmd(benchName,aCoreID)
977            if nbElems > 0:
978               coreName=getCoreDesc(aCoreID)
979               coreSection = Section("%s" % coreName)
980               benchSection.addSection(coreSection)
981               allCompilers = getExistingCompilerForCore(benchName,aCoreID)
982               for compiler in allCompilers:
983                 #print(compiler)
984                 if args.ratio:
985                    cols,params,ratios,testNames=computeRatioTableForCore(benchName,referenceCoreID,aCoreID,compiler)
986                    #print(cols)
987                    #print(ratios)
988                    #print(" ")
989                    if len(ratios)>0:
990                       compilerName,version=getCompilerDesc(compiler)
991                       compilerSection = Section("%s (%s)" % (compilerName,version))
992                       coreSection.addSection(compilerSection)
993                       addRatioTable(cols,params,ratios,compilerSection,testNames,True)
994
995                 else:
996                    nbElems = getNbElemsInBenchAndCoreAndCompilerCmd(benchName,compiler,aCoreID)
997
998                    # Print test results for table, type, compiler
999                    if nbElems > 0:
1000                       compilerName,version=getCompilerDesc(compiler)
1001                       compilerSection = Section("%s (%s)" % (compilerName,version))
1002                       coreSection.addSection(compilerSection)
1003                       cols,vals=getColNamesAndDataForCoreCompiler(benchName,compiler,aCoreID)
1004                       desc=(benchName,compiler,aCoreID)
1005                       names=getTestNamesForCoreCompiler(benchName,compiler,aCoreID)
1006
1007                       formatTableBy(desc,['type'],['core','version','compiler'],compilerSection,names,cols,vals)
1008
1009       else:
1010          allTypes = getExistingTypes(benchName)
1011          # Add report for each type
1012          for aTypeID in allTypes:
1013              nbElems = getNbElemsInBenchAndTypeCmd(benchName,aTypeID)
1014              if nbElems > 0:
1015                 typeName = getTypeName(aTypeID)
1016                 typeSection = Section(typeName)
1017                 benchSection.addSection(typeSection)
1018                 if args.byc:
1019                   ## Add report for each core
1020                   allCores = getExistingCores(benchName,aTypeID)
1021                   for core in allCores:
1022                       #print(core)
1023                       nbElems = getNbElemsInBenchAndTypeAndCoreCmd(benchName,core,aTypeID)
1024                       # Print test results for table, type, compiler
1025                       if nbElems > 0:
1026                          coreName=getCoreDesc(core)
1027                          coreSection = Section("%s" % coreName)
1028                          typeSection.addSection(coreSection)
1029                          cols,vals=getColNamesAndDataForCore(benchName,core,aTypeID)
1030                          desc=(benchName,core,aTypeID)
1031                          names=getTestNamesForCore(benchName,core,aTypeID)
1032                          formatTableBy(desc,['compiler','version'],['core'],coreSection,names,cols,vals)
1033                 elif args.ratio:
1034                    allCompilers = getExistingCompiler(benchName,aTypeID)
1035                    for compiler in allCompilers:
1036                       cols,params,ratios,testNames=computeRatioTable(benchName,referenceCoreID,aTypeID,compiler)
1037                       #print(cols)
1038                       #print(ratios)
1039                       #print(" ")
1040                       if len(ratios)>0:
1041                          compilerName,version=getCompilerDesc(compiler)
1042                          compilerSection = Section("%s (%s)" % (compilerName,version))
1043                          typeSection.addSection(compilerSection)
1044                          addRatioTable(cols,params,ratios,compilerSection,testNames,False)
1045
1046                 else:
1047                   ## Add report for each compiler
1048                   allCompilers = getExistingCompiler(benchName,aTypeID)
1049                   for compiler in allCompilers:
1050                       #print(compiler)
1051                       nbElems = getNbElemsInBenchAndTypeAndCompilerCmd(benchName,compiler,aTypeID)
1052                       # Print test results for table, type, compiler
1053                       if nbElems > 0:
1054                          compilerName,version=getCompilerDesc(compiler)
1055                          compilerSection = Section("%s (%s)" % (compilerName,version))
1056                          typeSection.addSection(compilerSection)
1057                          cols,vals=getColNamesAndDataForCompiler(benchName,compiler,aTypeID)
1058                          desc=(benchName,compiler,aTypeID)
1059                          names=getTestNamesForCompiler(benchName,compiler,aTypeID)
1060                          formatTableBy(desc,['core'],['version','compiler'],compilerSection,names,cols,vals)
1061
1062
1063
1064
1065toc=[Hierarchy("BasicMathsBenchmarks"),
1066Hierarchy("ComplexMathsBenchmarks"),
1067Hierarchy("FastMath"),
1068Hierarchy("Filters",
1069  [Hierarchy("FIR"),
1070   Hierarchy("BIQUAD"),
1071   Hierarchy("DECIM"),
1072   Hierarchy("MISC")]),
1073
1074Hierarchy("Support Functions",
1075  [Hierarchy("Support"),
1076   Hierarchy("SupportBar")]),
1077
1078Hierarchy("Matrix Operations"    ,
1079  [Hierarchy("Binary"),
1080   Hierarchy("Unary")]),
1081Hierarchy("Transform"),
1082Hierarchy("Stats"),
1083Hierarchy("Classical ML",[
1084  Hierarchy("Bayes"),
1085  Hierarchy("SVM"),
1086  Hierarchy("Distance"),
1087  Hierarchy("KalmanBenchmarks")
1088  ]),
1089
1090]
1091
1092processed=[]
1093
1094
1095
1096def addComments(document):
1097  if os.path.exists(args.comments):
1098     section=Section("Measurement Context")
1099     path=os.path.join(args.comments,"comments.txt")
1100
1101     document.addSection(section)
1102     para=""
1103     with open(path,"r") as r:
1104         for l in r:
1105             if l.strip():
1106               para += l
1107             else:
1108               section.addContent(Text(para))
1109               para=""
1110     if para:
1111        section.addContent(Text(para))
1112
1113     if args.ratio:
1114        section.addContent(Text("Reference core for the ratio is %s" % refCoreName))
1115        section.addContent(Text("A bigger ratio means the reference code is better"))
1116
1117
1118def processToc(d):
1119    result=[]
1120    for k in d:
1121      if d[k] is not None:
1122         result.append(Hierarchy(k,processToc(d[k])))
1123      else:
1124         result.append(Hierarchy(k))
1125    return(result)
1126
1127
1128
1129def createDoc(document,sections,benchtables):
1130    global processed,referenceCoreID
1131
1132    for s in sections:
1133        if s.name in benchtables:
1134           addReportFor(document,s.name)
1135           processed.append(s.name)
1136        else:
1137           section=Section(s.name)
1138           document.addSection(section)
1139           createDoc(section,s.sections,benchtables)
1140
1141try:
1142      benchtables=getBenchTables()
1143      document = Document(None)
1144
1145      if args.ratio:
1146         referenceCoreID= getCoreID(args.ref)
1147         refCoreName=getCoreDesc(referenceCoreID)
1148
1149      addComments(document)
1150
1151      if args.toc:
1152        with open(args.toc,"r") as f:
1153             config=yaml.safe_load(f)
1154             toc = processToc(config['root'])
1155             #print(toc)
1156             #quit()
1157
1158      createDoc(document,toc,benchtables)
1159
1160      misc=Section("Miscellaneous")
1161      document.addSection(misc)
1162      remaining=diff(benchtables,processed)
1163      for bench in remaining:
1164          addReportFor(misc,bench)
1165
1166      #for bench in benchtables:
1167      #    addReportFor(document,bench)
1168      with open(args.o,"w") as output:
1169          if args.t=="md":
1170             document.accept(Markdown(output))
1171          if args.t=="html":
1172             reorder=NORMALFORMAT
1173             if args.byc:
1174               reorder=BYCFORMAT
1175             if args.byd:
1176               reorder=BYDFORMAT
1177             document.accept(HTML(output,args.r,args.ratio,reorder))
1178
1179finally:
1180     c.close()
1181
1182
1183
1184
1185