1# Process the test results 2# Test status (like passed, or failed with error code) 3 4import argparse 5import re 6import TestScripts.NewParser as parse 7import TestScripts.CodeGen 8from collections import deque 9import os.path 10import numpy as np 11import pandas as pd 12import statsmodels.api as sm 13import statsmodels.formula.api as smf 14import csv 15import TestScripts.Deprecate as d 16import sqlite3 17import datetime, time 18import re 19 20# For sql table creation 21MKSTRFIELD=[] 22MKBOOLFIELD=['HARDFP', 'FASTMATH', 'NEON', 'HELIUM','UNROLL', 'ROUNDING','OPTIMIZED'] 23MKINTFIELD=['ID', 'CYCLES'] 24MKDATEFIELD=[] 25MKKEYFIELD=['DATE','NAME','CATEGORY', 'PLATFORM', 'CORE', 'COMPILER','TYPE',"RUN"] 26MKKEYFIELDID={'CATEGORY':'categoryid', 27 'NAME':'testnameid', 28 'DATE':'testdateid', 29 'PLATFORM':'platformid', 30 'CORE':'coreid', 31 'COMPILER':'compilerid', 32 'TYPE':'typeid', 33 'RUN':'runid'} 34 35# For csv table value extraction 36VALSTRFIELD=['TESTNAME','VERSION'] 37VALBOOLFIELD=['HARDFP', 'FASTMATH', 'NEON', 'HELIUM','UNROLL', 'ROUNDING','OPTIMIZED'] 38VALINTFIELD=['ID', 'CYCLES'] 39VALDATEFIELD=[] 40# Some of those fields may be created by the parsing of other fields 41VALKEYFIELD=['DATE','NAME','CATEGORY', 'PLATFORM', 'CORE', 'COMPILER','TYPE'] 42 43def joinit(iterable, delimiter): 44 it = iter(iterable) 45 yield next(it) 46 for x in it: 47 yield delimiter 48 yield x 49 50def tableExists(c,tableName): 51 req=(tableName,) 52 r=c.execute("SELECT name FROM sqlite_master WHERE type='table' AND name=?",req) 53 return(r.fetchone() != None) 54 55def diff(first, second): 56 second = set(second) 57 return [item for item in first if item not in second] 58 59def getColumns(elem,full): 60 colsToKeep=[] 61 cols = list(full.columns) 62 params = list(elem.params.full) 63 common = diff(cols + ["TYPE","RUN"] , ['OLDID'] + params) 64 65 for field in common: 66 if field in MKSTRFIELD: 67 colsToKeep.append(field) 68 if field in MKINTFIELD: 69 colsToKeep.append(field) 70 if field in MKKEYFIELD: 71 colsToKeep.append(field) 72 if field in MKDATEFIELD: 73 colsToKeep.append(field) 74 if field in MKBOOLFIELD: 75 colsToKeep.append(field) 76 return(colsToKeep) 77 78def createTableIfMissing(conn,elem,tableName,full): 79 if not tableExists(conn,tableName): 80 sql = "CREATE TABLE %s (" % tableName 81 cols = list(full.columns) 82 params = list(elem.params.full) 83 common = diff(cols + ["TYPE","RUN"] , ['OLDID'] + params) 84 85 sql += "%sid INTEGER PRIMARY KEY" % (tableName) 86 start = "," 87 88 for field in params: 89 sql += " %s\n %s INTEGER" % (start,field) 90 start = "," 91 92 for field in common: 93 if field in MKSTRFIELD: 94 sql += "%s\n %s TEXT" % (start,field) 95 if field in MKINTFIELD: 96 sql += "%s\n %s INTEGER" % (start,field) 97 if field in MKKEYFIELD: 98 sql += "%s\n %s INTEGER" % (start,MKKEYFIELDID[field]) 99 if field in MKDATEFIELD: 100 sql += "%s\n %s TEXT" % (start,field) 101 if field in MKBOOLFIELD: 102 sql += "%s\n %s INTEGER" % (start,field) 103 start = "," 104 # Create foreign keys 105 sql += "%sFOREIGN KEY(typeid) REFERENCES TYPE(typeid)," % start 106 sql += "FOREIGN KEY(categoryid) REFERENCES CATEGORY(categoryid)," 107 sql += "FOREIGN KEY(testnameid) REFERENCES TESTNAME(testnameid)," 108 sql += "FOREIGN KEY(testdateid) REFERENCES TESTDATE(testdateid)," 109 sql += "FOREIGN KEY(platformid) REFERENCES PLATFORM(platformid)," 110 sql += "FOREIGN KEY(coreid) REFERENCES CORE(coreid)," 111 sql += "FOREIGN KEY(compilerid) REFERENCES COMPILER(compilerid)" 112 sql += "FOREIGN KEY(runid) REFERENCES RUN(runid)" 113 sql += " )" 114 conn.execute(sql) 115 116# Find the key or add it in a table 117def findInTable(conn,table,keystr,strv,key): 118 #print(sql) 119 r = conn.execute("select %s from %s where %s=?" % (key,table,keystr),(strv,)) 120 result=r.fetchone() 121 if result != None: 122 return(result[0]) 123 else: 124 conn.execute("INSERT INTO %s(%s) VALUES(?)" % (table,keystr),(strv,)) 125 conn.commit() 126 r = conn.execute("select %s from %s where %s=?" % (key,table,keystr),(strv,)) 127 result=r.fetchone() 128 if result != None: 129 #print(result) 130 return(result[0]) 131 else: 132 return(None) 133 134def findInCompilerTable(conn,kind,version): 135 #print(sql) 136 r = conn.execute("select compilerid from COMPILER where compilerkindid=? AND version=?" , (kind,version)) 137 result=r.fetchone() 138 if result != None: 139 return(result[0]) 140 else: 141 fullDate = datetime.datetime.now() 142 dateid = findInTable(conn,"TESTDATE","date",str(fullDate),"testdateid") 143 conn.execute("INSERT INTO COMPILER(compilerkindid,version,testdateid) VALUES(?,?,?)" ,(kind,version,dateid)) 144 conn.commit() 145 r = conn.execute("select compilerid from COMPILER where compilerkindid=? AND version=? AND testdateid=?" , (kind,version,dateid)) 146 result=r.fetchone() 147 if result != None: 148 #print(result) 149 return(result[0]) 150 else: 151 return(None) 152 153 154def addRows(conn,elem,tableName,full,runid=0): 155 # List of columns we have in DB which is 156 # different from the columns in the table 157 compilerid = 0 158 platformid = 0 159 coreid = 0 160 keep = getColumns(elem,full) 161 cols = list(full.columns) 162 params = list(elem.params.full) 163 common = diff(["TYPE"] + cols , ['OLDID'] + params) 164 colNameList = [] 165 for c in params + keep: 166 if c in MKKEYFIELD: 167 colNameList.append(MKKEYFIELDID[c]) 168 else: 169 colNameList.append(c) 170 colNames = "".join(joinit(colNameList,",")) 171 #print(colNameList) 172 #print(colNames) 173 #print(full) 174 for index, row in full.iterrows(): 175 sql = "INSERT INTO %s(%s) VALUES(" % (tableName,colNames) 176 keys = {} 177 178 # Get data from columns 179 for field in common: 180 if field in VALSTRFIELD: 181 keys[field]=row[field] 182 if field == "NAME": 183 name = row[field] 184 if field == "TESTNAME": 185 testname = row[field] 186 if re.match(r'^.*_f64',testname): 187 keys["TYPE"] = "f64" 188 if re.match(r'^.*_f32',testname): 189 keys["TYPE"] = "f32" 190 if re.match(r'^.*_f16',testname): 191 keys["TYPE"] = "f16" 192 if re.match(r'^.*_q31',testname): 193 keys["TYPE"] = "q31" 194 if re.match(r'^.*_q15',testname): 195 keys["TYPE"] = "q15" 196 if re.match(r'^.*_q7',testname): 197 keys["TYPE"] = "q7" 198 199 if re.match(r'^.*_s8',testname): 200 keys["TYPE"] = "s8" 201 if re.match(r'^.*_u8',testname): 202 keys["TYPE"] = "u8" 203 if re.match(r'^.*_s16',testname): 204 keys["TYPE"] = "s16" 205 if re.match(r'^.*_u16',testname): 206 keys["TYPE"] = "u16" 207 if re.match(r'^.*_s32',testname): 208 keys["TYPE"] = "s32" 209 if re.match(r'^.*_u32',testname): 210 keys["TYPE"] = "u32" 211 if re.match(r'^.*_s64',testname): 212 keys["TYPE"] = "s64" 213 if re.match(r'^.*_u64',testname): 214 keys["TYPE"] = "u64" 215 216 if field in VALINTFIELD: 217 keys[field]=row[field] 218 if field in VALDATEFIELD: 219 keys[field]=row[field] 220 if field in VALBOOLFIELD: 221 keys[field]=row[field] 222 223 224 keys['RUN']=runid 225 # Get foreign keys and create missing data 226 for field in common: 227 if field in VALKEYFIELD: 228 if field == "CATEGORY": 229 # Remove type extension to get category name so that 230 # all types are maped to same category which will 231 # help for post processing. 232 testField=re.sub(r'^(.*)[:]([^:]+)(F16|F32|F64|Q31|Q15|Q7)$',r'\1',row[field]) 233 val = findInTable(conn,"CATEGORY","category",testField,"categoryid") 234 keys[field]=val 235 if field == "NAME": 236 val = findInTable(conn,"TESTNAME","name",row[field],"testnameid") 237 keys[field]=val 238 if field == "DATE": 239 val = findInTable(conn,"TESTDATE","date",str(row[field]),"testdateid") 240 keys[field]=val 241 if field == "CORE": 242 val = findInTable(conn,"CORE","coredef",row[field],"coreid") 243 keys[field]=val 244 coreid = val 245 if field == "PLATFORM": 246 val = findInTable(conn,"PLATFORM","platform",row[field],"platformid") 247 keys[field]=val 248 platformid = val 249 if field == "TYPE": 250 val = findInTable(conn,"TYPE","type",keys["TYPE"],"typeid") 251 keys[field]=val 252 if field == "COMPILER": 253 compilerkind = findInTable(conn,"COMPILERKIND","compiler",row[field],"compilerkindid") 254 compiler = findInCompilerTable(conn,compilerkind,keys["VERSION"]) 255 keys[field]=compiler 256 compilerid = compiler 257 258 # Generate sql command 259 start = "" 260 for field in params: 261 sql += " %s\n %d" % (start,row[field]) 262 start = "," 263 264 for field in keep: 265 if field in MKSTRFIELD or field in MKDATEFIELD: 266 sql += " %s\n \"%s\"" % (start,keys[field]) 267 elif field in keep: 268 sql += " %s\n %d" % (start,keys[field]) 269 start = "," 270 271 sql += " )" 272 #print(sql) 273 conn.execute(sql) 274 conn.commit() 275 return({'compilerid':compilerid,'platformid':platformid,'coreid':coreid}) 276 277def addConfig(conn,config,fullDate): 278 dateid = findInTable(conn,"TESTDATE","date",str(fullDate),"testdateid") 279 conn.execute("INSERT INTO CONFIG(compilerid,platformid,coreid,testdateid) VALUES(?,?,?,?)" ,(config['compilerid'],config['platformid'],config['coreid'],dateid)) 280 conn.commit() 281 282def getGroup(a): 283 return(re.sub(r'^(.+)(F64|F32|F16|Q31|Q15|Q7|U32|U16|U8|S32|S16|S8)$',r'\1',a)) 284 285def addOneBenchmark(elem,fullPath,db,group,runid): 286 if os.path.isfile(fullPath): 287 full=pd.read_csv(fullPath,dtype={'OLDID': str} ,keep_default_na = False) 288 fullDate = datetime.datetime.now() 289 full['DATE'] = fullDate 290 if group: 291 tableName = getGroup(group) 292 else: 293 tableName = getGroup(elem.data["class"]) 294 conn = sqlite3.connect(db) 295 createTableIfMissing(conn,elem,tableName,full) 296 config = addRows(conn,elem,tableName,full,runid) 297 addConfig(conn,config,fullDate) 298 conn.close() 299 300 301def addToDB(benchmark,dbpath,elem,group,runid): 302 if not elem.data["deprecated"]: 303 if elem.params: 304 benchPath = os.path.join(benchmark,elem.fullPath(),"fullBenchmark.csv") 305 print("Processing %s" % benchPath) 306 addOneBenchmark(elem,benchPath,dbpath,group,runid) 307 308 for c in elem.children: 309 addToDB(benchmark,dbpath,c,group,runid) 310 311 312 313parser = argparse.ArgumentParser(description='Generate summary benchmarks') 314 315parser.add_argument('-f', nargs='?',type = str, default="Output.pickle", help="Pickle") 316parser.add_argument('-b', nargs='?',type = str, default="FullBenchmark", help="Full Benchmark dir path") 317#parser.add_argument('-e', action='store_true', help="Embedded test") 318parser.add_argument('-o', nargs='?',type = str, default="bench.db", help="Benchmark database") 319parser.add_argument('-r', nargs='?',type = int, default=0, help="Run ID") 320 321parser.add_argument('others', nargs=argparse.REMAINDER, help="Suite class") 322 323args = parser.parse_args() 324 325if args.f is not None: 326 #p = parse.Parser() 327 # Parse the test description file 328 #root = p.parse(args.f) 329 root=parse.loadRoot(args.f) 330 d.deprecate(root,args.others) 331 if args.others: 332 group=args.others[0] 333 else: 334 group=None 335 addToDB(args.b,args.o,root,group,args.r) 336 337else: 338 parser.print_help()
