import argparse import sqlite3 import re import pandas as pd import numpy as np from TestScripts.doc.Structure import * from TestScripts.doc.Format import * import os.path import yaml refCoreName="" # Command to get last runid lastID="""SELECT runid FROM RUN ORDER BY runid DESC LIMIT 1 """ # Command to get last runid and date lastIDAndDate="""SELECT date FROM RUN WHERE runid=? """ # Command to get last runid runIDDetails="""SELECT distinct core FROM %s INNER JOIN CORE USING(coreid) WHERE %s """ def joinit(iterable, delimiter): # Intersperse a delimiter between element of a list it = iter(iterable) yield next(it) for x in it: yield delimiter yield x def getLastRunID(): r=c.execute(lastID) return(int(r.fetchone()[0])) def getrunIDDate(forID): r=c.execute(lastIDAndDate,(forID,)) return(r.fetchone()[0]) parser = argparse.ArgumentParser(description='Generate summary benchmarks') parser.add_argument('-b', nargs='?',type = str, default="bench.db", help="Database") parser.add_argument('-o', nargs='?',type = str, default="full.md", help="Full summary") parser.add_argument('-r', action='store_true', help="Regression database") parser.add_argument('-t', nargs='?',type = str, default="md", help="type md or html") parser.add_argument('-byc', action='store_true', help="Result oganized by Compiler") parser.add_argument('-g', action='store_true', help="Include graphs in regression report") parser.add_argument('-details', action='store_true', help="Details about runids") parser.add_argument('-lastid', action='store_true', help="Get last ID") parser.add_argument('-comments', nargs='?',type = str, default="comments", help="Comment folder") parser.add_argument('-byd', action='store_true', help="Result oganized by datatype") parser.add_argument('-ratio', action='store_true', help="Compute ratios for regression by core instead of cycles") parser.add_argument('-ref', nargs='?',type = str, default="M55", help="Reference COREDEF for ratio in db") parser.add_argument('-clampval', nargs='?',type = float, default=8.0, help="Clamp for ratio") parser.add_argument('-clamp', action='store_true', help="Clamp enabled for ratio") parser.add_argument('-cores', nargs='?',type = str, help="Cores to keep") parser.add_argument('-toc', nargs='?',type = str, help="Yaml for the table of contents") # For runid or runid range parser.add_argument('others', nargs=argparse.REMAINDER,help="Run ID") args = parser.parse_args() c = sqlite3.connect(args.b) coreidSQL="select distinct coreid from CORE where coredef==?" def getCoreID(corename): r=c.execute(coreidSQL,(corename,)) t=r.fetchone() if t is None: print("Unrecognized reference core \"%s\"" % corename) quit() return(t[0]) def parseSelector(o,field="runid"): vals=[] runidCMD=[] # parameters are not allowed in VIEWs runidVIEWcmd=[] for t in o: if re.search(r'-',t): bounds=[int(x) for x in t.split("-")] vals += bounds runidCMD += ["(%s >= ? AND %s <= ?)" % (field,field)] x=(field,bounds[0],field,bounds[1]) runidVIEWcmd += ["(%s >= %d AND %s <= %d)" % x] else: theid=int(t) runidCMD += ["%s == ?" % field] runidVIEWcmd += ["%s == %d" % (field,theid)] vals.append(theid) runidval = tuple(vals) runidCMD = "(" + "".join(joinit(runidCMD," OR ")) + ")" runidVIEWcmd = "(" + "".join(joinit(runidVIEWcmd," OR ")) + ")" return(runidval,runidCMD,runidVIEWcmd) if args.others: runidval,runidCMD,runidVIEWcmd = parseSelector(args.others) else: theid=getLastRunID() print("Last run ID = %d\n" % theid) runidval=(theid,) runidCMD = "runid = ?" runidVIEWcmd="(runid = %d)" % theid # None means all coreidval = [] coreidCMD = [] keepCoreIds=None if args.cores: cores=args.cores.split(",") coreids = [str(getCoreID(x.strip())) for x in cores] keepCoreIds = coreids.copy() if args.ref: coreids.append(str(getCoreID(args.ref.strip()))) #print(coreids) coreidval,coreidCMD, coreidVIEWcmd = parseSelector(coreids,field="coreid") runidval += coreidval runidCMD += " AND %s" % coreidCMD runidVIEWcmd += " AND %s" % coreidVIEWcmd # We extract data only from data tables # Those tables below are used for descriptions REMOVETABLES=['TESTNAME','TESTDATE','RUN','CORE', 'PLATFORM', 'COMPILERKIND', 'COMPILER', 'TYPE', 'CATEGORY', 'CONFIG'] # This is assuming the database is generated by the regression script # So platform is the same for all benchmarks. # Category and type is coming from the test name in the yaml # So no need to add this information here # Name is removed here because it is added at the beginning REMOVECOLUMNS=['runid','name','type','platform','category','coredef','OPTIMIZED','HARDFP','FASTMATH','NEON','HELIUM','UNROLL','ROUNDING','DATE','compilerkindid','date','categoryid', 'ID', 'platformid', 'coreid', 'compilerid', 'typeid'] REMOVECOLUMNSFORHISTORY=['Regression','MAXREGCOEF','name','type','platform','category','coredef','OPTIMIZED','HARDFP','FASTMATH','NEON','HELIUM','UNROLL','ROUNDING','DATE','compilerkindid','date','categoryid', 'ID', 'platformid', 'coreid', 'compilerid', 'typeid'] # Get existing benchmark tables def getBenchTables(): r=c.execute("SELECT name FROM sqlite_master WHERE type='table'") benchtables=[] for table in r: if not table[0] in REMOVETABLES: benchtables.append(table[0]) return(benchtables) # get existing types in a table def getExistingTypes(benchTable): r=c.execute("select distinct typeid from %s WHERE %s order by typeid desc " % (benchTable,runidCMD),runidval).fetchall() result=[x[0] for x in r] return(result) # Get existing cores in a table def getAllExistingCores(benchTable): r=c.execute("select distinct coreid from %s WHERE %s order by coreid desc " % (benchTable,runidCMD),runidval).fetchall() result=[x[0] for x in r] return(result) def getrunIDDetails(): tables=getBenchTables() r=[] for table in tables: r += [x[0] for x in c.execute(runIDDetails % (table,runidCMD),runidval).fetchall()] r=list(set(r)) print(r) if args.lastid: quit() if args.details: getrunIDDetails() quit() # Get compilers from specific type and table allCompilers="""select distinct compilerid from %s WHERE typeid=?""" # Get compilers from specific type and table allCompilerForCore="""select distinct compilerid from %s WHERE coreid=?""" # Get compilers from specific type and table allCores="""select distinct coreid from %s WHERE typeid=? AND (%s)""" compilerDesc="""select compiler,version from COMPILER INNER JOIN COMPILERKIND USING(compilerkindid) WHERE compilerid=?""" coreDesc="""select core from CORE WHERE coreid=?""" # Get existing compiler in a table for a specific type # (In case report is structured by types) def getExistingCompiler(benchTable,typeid): r=c.execute(allCompilers % benchTable,(typeid,)).fetchall() return([x[0] for x in r]) # Get existing compiler in a table for a specific core # (In case report is structured by core) def getExistingCompilerForCore(benchTable,coreid): r=c.execute(allCompilerForCore % benchTable,(coreid,)).fetchall() return([x[0] for x in r]) def getExistingCores(benchTable,typeid): vals = (typeid,) + runidval r=c.execute(allCores % (benchTable,runidCMD),vals).fetchall() return([x[0] for x in r]) def getCoreDesc(coreid): r=c.execute(coreDesc,(coreid,)).fetchone() return(r) def getCompilerDesc(compilerid): r=c.execute(compilerDesc,(compilerid,)).fetchone() return(r) # Get type name from type id def getTypeName(typeid): r=c.execute("select type from TYPE where typeid=?",(typeid,)).fetchone() return(r[0]) # Diff of 2 lists def diff(first, second): second = set(second) return [item for item in first if item not in second] # Command to get data for specific compiler # and type benchCmdForCoreCompiler="""select %s from %s INNER JOIN CATEGORY USING(categoryid) INNER JOIN PLATFORM USING(platformid) INNER JOIN CORE USING(coreid) INNER JOIN COMPILER USING(compilerid) INNER JOIN COMPILERKIND USING(compilerkindid) INNER JOIN TYPE USING(typeid) INNER JOIN TESTNAME USING(testnameid) WHERE coreid=? AND compilerid = ? AND (%s) """ # Command to get data for specific core # and type historyCmd="""select %s from %s INNER JOIN CATEGORY USING(categoryid) INNER JOIN PLATFORM USING(platformid) INNER JOIN CORE USING(coreid) INNER JOIN COMPILER USING(compilerid) INNER JOIN COMPILERKIND USING(compilerkindid) INNER JOIN TYPE USING(typeid) INNER JOIN TESTNAME USING(testnameid) WHERE compilerid=? AND coreid=? AND typeid = ? AND ID = ? AND runid > (? - 10) """ compilersForHistory="""select distinct compilerid,compiler,version from %s INNER JOIN COMPILER USING(compilerid) INNER JOIN COMPILERKIND USING(compilerkindid) WHERE coreid=? AND typeid = ? AND ID = ? AND runid > (? - 10) """ # Command to get data for specific core # and type benchCmdForCore="""select %s from %s INNER JOIN CATEGORY USING(categoryid) INNER JOIN PLATFORM USING(platformid) INNER JOIN CORE USING(coreid) INNER JOIN COMPILER USING(compilerid) INNER JOIN COMPILERKIND USING(compilerkindid) INNER JOIN TYPE USING(typeid) INNER JOIN TESTNAME USING(testnameid) WHERE coreid=? AND typeid = ? AND (%s) """ coresForHistory="""select distinct coreid,core from %s INNER JOIN CORE USING(coreid) WHERE compilerid=? AND typeid = ? AND ID = ? AND runid > (? - 10) """ # Command to get data for specific compiler # and type benchCmdForCompiler="""select %s from %s INNER JOIN CATEGORY USING(categoryid) INNER JOIN PLATFORM USING(platformid) INNER JOIN CORE USING(coreid) INNER JOIN COMPILER USING(compilerid) INNER JOIN COMPILERKIND USING(compilerkindid) INNER JOIN TYPE USING(typeid) INNER JOIN TESTNAME USING(testnameid) WHERE compilerid=? AND typeid = ? AND (%s) """ # Command to get test names for specific compiler # and type benchNamesForCore="""select distinct name from %s INNER JOIN COMPILER USING(compilerid) INNER JOIN COMPILERKIND USING(compilerkindid) INNER JOIN TYPE USING(typeid) INNER JOIN TESTNAME USING(testnameid) WHERE coreid=? AND typeid = ? AND (%s) """ # Command to get test names for specific core # and compiler benchNamesForCoreCompiler="""select distinct name from %s INNER JOIN COMPILER USING(compilerid) INNER JOIN COMPILERKIND USING(compilerkindid) INNER JOIN TYPE USING(typeid) INNER JOIN TESTNAME USING(testnameid) WHERE coreid=? AND compilerid = ? AND (%s) """ # Command to get test names for specific compiler # and type benchNamesForCompiler="""select distinct name from %s INNER JOIN COMPILER USING(compilerid) INNER JOIN COMPILERKIND USING(compilerkindid) INNER JOIN TYPE USING(typeid) INNER JOIN TESTNAME USING(testnameid) WHERE compilerid=? AND typeid = ? AND (%s) """ # Command to get columns for specific table benchCmdColumns="""select * from %s INNER JOIN CATEGORY USING(categoryid) INNER JOIN PLATFORM USING(platformid) INNER JOIN CORE USING(coreid) INNER JOIN COMPILER USING(compilerid) INNER JOIN COMPILERKIND USING(compilerkindid) INNER JOIN TESTNAME USING(testnameid) INNER JOIN TYPE USING(typeid) """ def joinit(iterable, delimiter): it = iter(iterable) yield next(it) for x in it: yield delimiter yield x # Is not a column name finishing by id # (often primary key for thetable) def isNotIDColumn(col): if re.match(r'^.*id$',col): return(False) else: return(True) # Get test names # for specific core and compiler (for the data) def getTestNamesForCoreCompiler(benchTable,compilerid,core): vals=(core,compilerid) + runidval result=c.execute(benchNamesForCoreCompiler % (benchTable,runidCMD),vals).fetchall() names=[x[0] for x in list(result)] return(names) # Get test names # for specific typeid and core (for the data) def getTestNamesForCore(benchTable,core,typeid): vals=(core,typeid) + runidval result=c.execute(benchNamesForCore % (benchTable,runidCMD),vals).fetchall() names=[x[0] for x in list(result)] return(names) # Get test names # for specific typeid and compiler (for the data) def getTestNamesForCompiler(benchTable,comp,typeid): vals=(comp,typeid) + runidval result=c.execute(benchNamesForCompiler % (benchTable,runidCMD),vals).fetchall() names=[x[0] for x in list(result)] return(names) # Command to get data for specific core # and type nbElemsInBenchAndTypeAndCoreCmd="""select count(*) from %s WHERE coreid=? AND typeid = ? AND (%s) """ # Command to get data for specific compiler # and type nbElemsInBenchAndTypeAndCompilerCmd="""select count(*) from %s WHERE compilerid=? AND typeid = ? AND (%s) """ # Command to get data for specific compiler # and type nbElemsInBenchAndCoreAndCompilerCmd="""select count(*) from %s WHERE compilerid=? AND coreid = ? AND (%s) """ nbElemsInBenchAndTypeCmd="""select count(*) from %s WHERE typeid = ? AND (%s) """ nbElemsInBenchAndCoreCmd="""select count(*) from %s WHERE coreid = ? AND (%s) """ nbElemsInBenchCmd="""select count(*) from %s WHERE %s """ categoryName="""select distinct category from %s INNER JOIN CATEGORY USING(categoryid) WHERE %s """ def getCategoryName(benchTable): result=c.execute(categoryName % (benchTable,runidCMD),runidval).fetchone() return(result[0]) # Get nb elems in a table def getNbElemsInBenchAndTypeAndCoreCmd(benchTable,coreid,typeid): vals=(coreid,typeid) + runidval result=c.execute(nbElemsInBenchAndTypeAndCoreCmd % (benchTable,runidCMD),vals).fetchone() return(result[0]) # Get nb elems in a table def getNbElemsInBenchAndTypeAndCompilerCmd(benchTable,comp,typeid): vals=(comp,typeid) + runidval result=c.execute(nbElemsInBenchAndTypeAndCompilerCmd % (benchTable,runidCMD),vals).fetchone() return(result[0]) # Get nb elems in a table def getNbElemsInBenchAndCoreAndCompilerCmd(benchTable,comp,coreid): vals=(comp,coreid) + runidval result=c.execute(nbElemsInBenchAndCoreAndCompilerCmd % (benchTable,runidCMD),vals).fetchone() return(result[0]) def getNbElemsInBenchAndTypeCmd(benchTable,typeid): vals=(typeid,) + runidval result=c.execute(nbElemsInBenchAndTypeCmd % (benchTable,runidCMD),vals).fetchone() return(result[0]) def getNbElemsInBenchAndCoreCmd(benchTable,coreid): vals=(coreid,) + runidval result=c.execute(nbElemsInBenchAndCoreCmd % (benchTable,runidCMD),vals).fetchone() return(result[0]) def getNbElemsInBenchCmd(benchTable): result=c.execute(nbElemsInBenchCmd % (benchTable,runidCMD),runidval).fetchone() return(result[0]) # Get names of columns and data for a table # for specific typeid and coreid (for the data) def getColNamesAndHistory(benchTable,compiler,core,typeid,testid): cursor=c.cursor() result=cursor.execute(benchCmdColumns % (benchTable)) cols= [member[0] for member in cursor.description] keepCols = ['name'] + [c for c in diff(cols , REMOVECOLUMNSFORHISTORY) if isNotIDColumn(c)] keepColsStr = "".join(joinit(keepCols,",")) vals=(compiler,core,typeid,testid,runid) result=cursor.execute(historyCmd % (keepColsStr,benchTable),vals) vals =np.array([list(x) for x in list(result)]) return(keepCols,vals) # Get names of columns and data for a table # for specific typeid and coreid (for the data) def getColNamesAndDataForCore(benchTable,core,typeid): cursor=c.cursor() result=cursor.execute(benchCmdColumns % (benchTable)) cols= [member[0] for member in cursor.description] keepCols = ['name'] + [c for c in diff(cols , REMOVECOLUMNS) if isNotIDColumn(c)] keepColsStr = "".join(joinit(keepCols,",")) vals=(core,typeid) + runidval result=cursor.execute(benchCmdForCore % (keepColsStr,benchTable,runidCMD),vals) vals =np.array([list(x) for x in list(result)]) return(keepCols,vals) # Get names of columns and data for a table # for specific coreid and compilerid (for the data) def getColNamesAndDataForCoreCompiler(benchTable,compilerid,core): cursor=c.cursor() result=cursor.execute(benchCmdColumns % (benchTable)) cols= [member[0] for member in cursor.description] keepCols = ['name','type'] + [c for c in diff(cols , REMOVECOLUMNS) if isNotIDColumn(c)] keepColsStr = "".join(joinit(keepCols,",")) vals=(core,compilerid) + runidval result=cursor.execute(benchCmdForCoreCompiler % (keepColsStr,benchTable,runidCMD),vals) vals =np.array([list(x) for x in list(result)]) return(keepCols,vals) # Get names of columns and data for a table # for specific typeid and compiler (for the data) def getColNamesAndDataForCompiler(benchTable,comp,typeid): cursor=c.cursor() result=cursor.execute(benchCmdColumns % (benchTable)) cols= [member[0] for member in cursor.description] keepCols = ['name'] + [c for c in diff(cols , REMOVECOLUMNS) if isNotIDColumn(c)] keepColsStr = "".join(joinit(keepCols,",")) vals=(comp,typeid) + runidval result=cursor.execute(benchCmdForCompiler % (keepColsStr,benchTable,runidCMD),vals) vals =np.array([list(x) for x in list(result)]) return(keepCols,vals) def formatFloat(s): result=[] for t in s: if type(t) is float: result.append(("%.3f" % t)) else: result.append(t) return(result) PARAMS=["NB","NumTaps", "NBA", "NBB", "Factor", "NumStages","VECDIM","NBR","NBC","NBI","IFFT", "BITREV"] def regressionTableFor(byname,name,section,ref,toSort,indexCols,field): data=ref.pivot_table(index=indexCols, columns=byname, values=[field], aggfunc='first',fill_value="NA") data=data.sort_values(toSort) if args.byc: cores = [(c[1] + ":" + c[2]) for c in list(data.columns)] else: cores = [c[1] for c in list(data.columns)] columns = diff(indexCols,['name']) dataTable=Table(columns,cores) section.addContent(dataTable) dataForFunc=data.loc[name] if type(dataForFunc) is pd.DataFrame: bars={'cols':columns,'cores':cores,'data':[]} for row in dataForFunc.itertuples(): row=list(row) if type(row[0]) is int: row=[row[0]] + row[1:] else: row=list(row[0]) + row[1:] if field=="MAXREGCOEF": newrow = row newrow[len(columns):] = formatFloat(row[len(columns):]) row=newrow dataTable.addRow(row) bars['data'].append(row) return(bars) else: if field=="MAXREGCOEF": dataForFunc=formatFloat(dataForFunc) dataTable.addRow(dataForFunc) return(list(zip(cores,dataForFunc))) def formatColumnName(c): return("".join(joinit(c,":"))) def getCoresForHistory(benchTable,compilerid,typeid,testid,runid): vals=(compilerid,typeid,testid,runid) result=c.execute(coresForHistory % benchTable,vals).fetchall() ids=[(x[0],x[1]) for x in list(result)] return(ids) def getCompilerForHistory(benchTable,coreid,typeid,testid,runid): vals=(coreid,typeid,testid,runid) result=c.execute(compilersForHistory % benchTable,vals).fetchall() ids=[(x[0],x[1],x[2]) for x in list(result)] return(ids) def getHistory(desc,testid,indexCols): benchName,sectionID,typeid,runid = desc #print(benchName) #print(sectionID) #print(typeid) #print(testid) columns = diff(indexCols,['name']) #print(columns) if args.byc: coreid=sectionID compilerids=getCompilerForHistory(benchName,coreid,typeid,testid,runid) series={} for compilerid,compilername,version in compilerids: result=getColNamesAndHistory(benchName,compilerid,coreid,typeid,testid) #print("%s:%s" % (compilername,version)) maxpos = result[0].index('MAX') lrunid = result[0].index('runid') r=[[int(x[lrunid]),int(x[maxpos])] for x in result[1:][0]] series[corename]=r hist=History(series,runid) return(hist) else: compilerid=sectionID coreids = getCoresForHistory(benchName,compilerid,typeid,testid,runid) series={} for coreid,corename in coreids: result=getColNamesAndHistory(benchName,compilerid,coreid,typeid,testid) #print(corename) maxpos = result[0].index('MAX') corepos = result[0].index('core') lrunid = result[0].index('runid') r=[[int(x[lrunid]),int(x[maxpos])] for x in result[1:][0]] series[corename]=r hist=History(series,runid) return(hist) def convertRowToInt(r): result=[] for e in r: if type(e) is float: result.append(int(e)) else: result.append(e) return(result) def addSectionComment(section): if os.path.exists(args.comments): fileName=re.sub(r'[() :]','',section.name) path=os.path.join(args.comments,fileName+".html") para="" if os.path.exists(path): commentSection = Section("Comments") section.addSection(commentSection) with open(path,"r") as r: for l in r: if l.strip(): para += l else: commentSection.addContent(Text(para)) para="" if para: commentSection.addContent(Text(para)) def formatTableBy(desc,byname,section,typeSection,testNames,cols,vals): if vals.size != 0: ref=pd.DataFrame(vals,columns=cols) toSort=["name"] for param in PARAMS: if param in ref.columns: ref[param]=pd.to_numeric(ref[param]) toSort.append(param) if args.r: # Regression table ref['MAX']=pd.to_numeric(ref['MAX']) ref['MAXREGCOEF']=pd.to_numeric(ref['MAXREGCOEF']) indexCols=diff(cols,byname + ['Regression','MAXREGCOEF','MAX'] + section) valList = ['Regression'] else: ref['CYCLES']=pd.to_numeric(ref['CYCLES']).round(decimals=0) indexCols=diff(cols,byname + ['CYCLES'] + section) valList = ['CYCLES'] for name in testNames: if args.r: testSection = Section(name) testSection.setTest() typeSection.addSection(testSection) addSectionComment(testSection) maxCyclesSection = Section("Max cycles") testSection.addSection(maxCyclesSection) theCycles=regressionTableFor(byname,name,maxCyclesSection,ref,toSort,indexCols,'MAX') if args.g: if type(theCycles) is dict: nbParams=len(theCycles['cols']) for bar in theCycles['data']: params=bar[0:nbParams] values=bar[nbParams:] title=[("%s=%s" % x) for x in list(zip(theCycles['cols'],params))] title="".join(joinit(title," ")) sec=Section(title) maxCyclesSection.addSection(sec) values=list(zip(theCycles['cores'],values)) barChart=BarChart(values) sec.addContent(barChart) else: #print(theCycles) sec=Section("Graph") maxCyclesSection.addSection(sec) barChart=BarChart(theCycles) sec.addContent(barChart) #history=getHistory(desc,testid,indexCols) #testSection.addContent(history) regressionSection = Section("Regression") testSection.addSection(regressionSection) regressionTableFor(byname,name,regressionSection,ref,toSort,indexCols,'Regression') maxRegCoefSection = Section("Max Reg Coef") testSection.addSection(maxRegCoefSection) regressionTableFor(byname,name,maxRegCoefSection,ref,toSort,indexCols,'MAXREGCOEF') else: data=ref.pivot_table(index=indexCols, columns=byname, values=valList, aggfunc='first',fill_value="NA") data=data.sort_values(toSort) #print(list(data.columns)) testSection = Section(name) testSection.setTest() typeSection.addSection(testSection) addSectionComment(testSection) dataForFunc=data.loc[name] dataForFunc = dataForFunc.dropna(axis=1) columnsID = [formatColumnName(c[1:]) for c in list(dataForFunc.columns)] columns = diff(indexCols,['name']) dataTable=Table(columns,columnsID) testSection.addContent(dataTable) if type(dataForFunc) is pd.DataFrame: for row in dataForFunc.itertuples(): if type(row[0]) is int: row=list([row[0]] + list(row[1:])) else: row=list(row[0]) + list(row[1:]) dataTable.addRow(convertRowToInt(row)) else: dataTable.addRow(convertRowToInt(dataForFunc)) referenceCoreID = None refCore="""CREATE TEMP VIEW if not exists refCore AS select * from %s where (coreid = %s) and (typeid = %s) and %s and compilerid = %s""" otherCore="""CREATE TEMP VIEW if not exists otherCore AS select * from %s where (typeid = %s) and %s and compilerid = %s""" refCoreAllTypes="""CREATE TEMP VIEW if not exists refCore AS select * from %s where (coreid = %s) and %s and compilerid = %s""" otherCoreAllTypes="""CREATE TEMP VIEW if not exists otherCore AS select * from %s where (coreid = %s) and %s and compilerid = %s""" ratioSQL="""select name,otherCore.compilerid as compilerid,CORE.core as core,%s(CAST(otherCore.MAX as FLOAT) / CAST(refCore.MAX AS FLOAT)) AS RATIO from otherCore INNER JOIN refCore ON (refCore.categoryid = otherCore.categoryid AND refCore.testnameid = otherCore.testnameid AND refCore.typeid = otherCore.typeid AND refCore.runid = otherCore.runid AND refCore.compilerid = otherCore.compilerid ) INNER JOIN TESTNAME ON TESTNAME.testnameid = refCore.testnameid INNER JOIN CORE USING(coreid) %s""" ratioSQLAllTypes="""select name,otherCore.compilerid as compilerid,TYPE.type as type,%s(CAST(otherCore.MAX as FLOAT) / CAST(refCore.MAX AS FLOAT)) AS RATIO from otherCore INNER JOIN refCore ON (refCore.categoryid = otherCore.categoryid AND refCore.testnameid = otherCore.testnameid AND refCore.typeid = otherCore.typeid AND refCore.runid = otherCore.runid AND refCore.compilerid = otherCore.compilerid ) INNER JOIN TESTNAME ON TESTNAME.testnameid = refCore.testnameid INNER JOIN TYPE USING(typeid) %s""" ratioTestNamesSQL="""select distinct TESTNAME.name from otherCore INNER JOIN refCore ON (refCore.categoryid = otherCore.categoryid AND refCore.testnameid = otherCore.testnameid AND refCore.typeid = otherCore.typeid AND refCore.runid = otherCore.runid AND refCore.compilerid = otherCore.compilerid ) INNER JOIN TESTNAME ON TESTNAME.testnameid = refCore.testnameid INNER JOIN CORE USING(coreid) %s""" dropViewsRef="""drop view refCore""" dropViewsOther="""drop view otherCore""" def getTableParams(benchTable): cursor=c.cursor() result=cursor.execute("select * from %s limit 1" % (benchTable)) cols= [member[0] for member in cursor.description] params=[] for x in cols: if x in PARAMS: params.append(x) return(params) def computeRatio(benchName,viewParams,refMkViewCmd,otherMkViewCmd,byd): params = getTableParams(benchName) cols=["ratio"] paramscmd="" paramscols="" paramsnames = ["refCore.%s as %s" % (x,x) for x in params] paramseq = ["refCore.%s = otherCore.%s" % (x,x) for x in params] if len(params) > 0: cols = ["%s" % x for x in params] cols.append("ratio") paramscols= ("".join(joinit(paramsnames," , ")) + ",") paramscmd = "WHERE " + "".join(joinit(paramseq," AND ")) if byd: ratioCmd = ratioSQLAllTypes % (paramscols,paramscmd) else: ratioCmd = ratioSQL % (paramscols,paramscmd) ratioTestNames = ratioTestNamesSQL % (paramscmd) #print(refMkViewCmd) #print(otherMkViewCmd) # #print(ratioCmd) # #print(dropViewsRef) #print(dropViewsOther) #quit() c.execute(refMkViewCmd) c.execute(otherMkViewCmd) ratio=c.execute(ratioCmd).fetchall() testNames=c.execute(ratioTestNames).fetchall() testNames=[x[0] for x in testNames] c.execute(dropViewsRef) c.execute(dropViewsOther) #print(ratio) #quit() if byd: return(['name','compilerid','type'] + cols,params,ratio,testNames) else: return(['name','compilerid','core'] + cols,params,ratio,testNames) # Compute for all core for a given type def computeRatioTable(benchName,referenceCore,typeID,compiler): viewParams = (benchName,referenceCore,typeID,runidVIEWcmd,compiler) refMkViewCmd = refCore % viewParams otherParams = (benchName,typeID,runidVIEWcmd,compiler) otherMkViewCmd = otherCore % otherParams #print(refMkViewCmd) #print(otherMkViewCmd) return(computeRatio(benchName,viewParams,refMkViewCmd,otherMkViewCmd,False)) def computeRatioTableForCore(benchName,referenceCore,otherCoreID,compiler): viewParams = (benchName,referenceCore,runidVIEWcmd,compiler) refMkViewCmd = refCoreAllTypes % viewParams otherParams = (benchName,otherCoreID,runidVIEWcmd,compiler) otherMkViewCmd = otherCoreAllTypes % otherParams #print(refMkViewCmd) #print(otherMkViewCmd) return(computeRatio(benchName,viewParams,refMkViewCmd,otherMkViewCmd,True)) def formatPerfRatio(s): result=[] for t in s: if type(t) is float: if args.clamp: if t > args.clampval: t = args.clampval if t < 0.0: result.append("NA") else: result.append(("%.3f" % t)) else: result.append(s) return(result) def addRatioTable(cols,params,data,section,testNames,byd): ref=pd.DataFrame(data,columns=cols) toSort=["name"] + params for param in PARAMS: if param in ref.columns: ref[param]=pd.to_numeric(ref[param]) #print(testNames) for name in testNames: testSection = Section(name) testSection.setTest() section.addSection(testSection) addSectionComment(testSection) ratioSection = Section("Ratios") testSection.addSection(ratioSection) #print(toSort) #print(ref) if byd: data=ref.pivot_table(index=toSort, columns=['type'], values=["ratio"], aggfunc='first',fill_value=-1.0) else: data=ref.pivot_table(index=toSort, columns=['core'], values=["ratio"], aggfunc='first') data=data.sort_values(toSort) #print(data) dataForFunc=data.loc[name] cores = [c[1] for c in list(data.columns)] dataTable=Table(params,cores) if args.g: if type(dataForFunc) is not pd.DataFrame: sec=Section("Graph") testSection.addSection(sec) barChart=BarChart(zip(cores,dataForFunc)) sec.addContent(barChart) ratioSection.addContent(Text("A bigger ratio means the reference core \"%s\" is better" % refCoreName)) ratioSection.addContent(dataTable) if type(dataForFunc) is pd.DataFrame: for row in dataForFunc.itertuples(): row=list(row) if type(row[0]) is int: row=[row[0]] + formatPerfRatio(row[1:]) else: row=list(row[0]) + formatPerfRatio(row[1:]) dataTable.addRow(row) else: row=list(dataForFunc) dataTable.addRow(formatPerfRatio(row)) # Add a report for each table def addReportFor(document,benchName): nbElems = getNbElemsInBenchCmd(benchName) if nbElems > 0: categoryName = getCategoryName(benchName) benchSection = Section(categoryName) document.addSection(benchSection) print("Process %s\n" % benchName) if args.byd: allCores=getAllExistingCores(benchName) if args.ratio: if keepCoreIds: allCores=keepCoreIds if ("%d" % referenceCoreID) in allCores: allCores.remove("%d" % referenceCoreID) for aCoreID in allCores: nbElems = getNbElemsInBenchAndCoreCmd(benchName,aCoreID) if nbElems > 0: coreName=getCoreDesc(aCoreID) coreSection = Section("%s" % coreName) benchSection.addSection(coreSection) allCompilers = getExistingCompilerForCore(benchName,aCoreID) for compiler in allCompilers: #print(compiler) if args.ratio: cols,params,ratios,testNames=computeRatioTableForCore(benchName,referenceCoreID,aCoreID,compiler) #print(cols) #print(ratios) #print(" ") if len(ratios)>0: compilerName,version=getCompilerDesc(compiler) compilerSection = Section("%s (%s)" % (compilerName,version)) coreSection.addSection(compilerSection) addRatioTable(cols,params,ratios,compilerSection,testNames,True) else: nbElems = getNbElemsInBenchAndCoreAndCompilerCmd(benchName,compiler,aCoreID) # Print test results for table, type, compiler if nbElems > 0: compilerName,version=getCompilerDesc(compiler) compilerSection = Section("%s (%s)" % (compilerName,version)) coreSection.addSection(compilerSection) cols,vals=getColNamesAndDataForCoreCompiler(benchName,compiler,aCoreID) desc=(benchName,compiler,aCoreID) names=getTestNamesForCoreCompiler(benchName,compiler,aCoreID) formatTableBy(desc,['type'],['core','version','compiler'],compilerSection,names,cols,vals) else: allTypes = getExistingTypes(benchName) # Add report for each type for aTypeID in allTypes: nbElems = getNbElemsInBenchAndTypeCmd(benchName,aTypeID) if nbElems > 0: typeName = getTypeName(aTypeID) typeSection = Section(typeName) benchSection.addSection(typeSection) if args.byc: ## Add report for each core allCores = getExistingCores(benchName,aTypeID) for core in allCores: #print(core) nbElems = getNbElemsInBenchAndTypeAndCoreCmd(benchName,core,aTypeID) # Print test results for table, type, compiler if nbElems > 0: coreName=getCoreDesc(core) coreSection = Section("%s" % coreName) typeSection.addSection(coreSection) cols,vals=getColNamesAndDataForCore(benchName,core,aTypeID) desc=(benchName,core,aTypeID) names=getTestNamesForCore(benchName,core,aTypeID) formatTableBy(desc,['compiler','version'],['core'],coreSection,names,cols,vals) elif args.ratio: allCompilers = getExistingCompiler(benchName,aTypeID) for compiler in allCompilers: cols,params,ratios,testNames=computeRatioTable(benchName,referenceCoreID,aTypeID,compiler) #print(cols) #print(ratios) #print(" ") if len(ratios)>0: compilerName,version=getCompilerDesc(compiler) compilerSection = Section("%s (%s)" % (compilerName,version)) typeSection.addSection(compilerSection) addRatioTable(cols,params,ratios,compilerSection,testNames,False) else: ## Add report for each compiler allCompilers = getExistingCompiler(benchName,aTypeID) for compiler in allCompilers: #print(compiler) nbElems = getNbElemsInBenchAndTypeAndCompilerCmd(benchName,compiler,aTypeID) # Print test results for table, type, compiler if nbElems > 0: compilerName,version=getCompilerDesc(compiler) compilerSection = Section("%s (%s)" % (compilerName,version)) typeSection.addSection(compilerSection) cols,vals=getColNamesAndDataForCompiler(benchName,compiler,aTypeID) desc=(benchName,compiler,aTypeID) names=getTestNamesForCompiler(benchName,compiler,aTypeID) formatTableBy(desc,['core'],['version','compiler'],compilerSection,names,cols,vals) toc=[Hierarchy("BasicMathsBenchmarks"), Hierarchy("ComplexMathsBenchmarks"), Hierarchy("FastMath"), Hierarchy("Filters", [Hierarchy("FIR"), Hierarchy("BIQUAD"), Hierarchy("DECIM"), Hierarchy("MISC")]), Hierarchy("Support Functions", [Hierarchy("Support"), Hierarchy("SupportBar")]), Hierarchy("Matrix Operations" , [Hierarchy("Binary"), Hierarchy("Unary")]), Hierarchy("Transform"), Hierarchy("Stats"), Hierarchy("Classical ML",[ Hierarchy("Bayes"), Hierarchy("SVM"), Hierarchy("Distance"), Hierarchy("KalmanBenchmarks") ]), ] processed=[] def addComments(document): if os.path.exists(args.comments): section=Section("Measurement Context") path=os.path.join(args.comments,"comments.txt") document.addSection(section) para="" with open(path,"r") as r: for l in r: if l.strip(): para += l else: section.addContent(Text(para)) para="" if para: section.addContent(Text(para)) if args.ratio: section.addContent(Text("Reference core for the ratio is %s" % refCoreName)) section.addContent(Text("A bigger ratio means the reference code is better")) def processToc(d): result=[] for k in d: if d[k] is not None: result.append(Hierarchy(k,processToc(d[k]))) else: result.append(Hierarchy(k)) return(result) def createDoc(document,sections,benchtables): global processed,referenceCoreID for s in sections: if s.name in benchtables: addReportFor(document,s.name) processed.append(s.name) else: section=Section(s.name) document.addSection(section) createDoc(section,s.sections,benchtables) try: benchtables=getBenchTables() document = Document(None) if args.ratio: referenceCoreID= getCoreID(args.ref) refCoreName=getCoreDesc(referenceCoreID) addComments(document) if args.toc: with open(args.toc,"r") as f: config=yaml.safe_load(f) toc = processToc(config['root']) #print(toc) #quit() createDoc(document,toc,benchtables) misc=Section("Miscellaneous") document.addSection(misc) remaining=diff(benchtables,processed) for bench in remaining: addReportFor(misc,bench) #for bench in benchtables: # addReportFor(document,bench) with open(args.o,"w") as output: if args.t=="md": document.accept(Markdown(output)) if args.t=="html": reorder=NORMALFORMAT if args.byc: reorder=BYCFORMAT if args.byd: reorder=BYDFORMAT document.accept(HTML(output,args.r,args.ratio,reorder)) finally: c.close()