diff options
Diffstat (limited to 'tests/test_pretty_print.py')
| -rwxr-xr-x | tests/test_pretty_print.py | 484 |
1 files changed, 484 insertions, 0 deletions
diff --git a/tests/test_pretty_print.py b/tests/test_pretty_print.py new file mode 100755 index 0000000..fa8b390 --- /dev/null +++ b/tests/test_pretty_print.py @@ -0,0 +1,484 @@ +#!/usr/bin/python + +# Copyright (C) 2011 Don Bright <hugh.m.bright@gmail.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +# +# This program 'pretty prints' the ctest output, namely +# files from builddir/Testing/Temporary. +# html & wiki output are produced in Testing/Temporary/sysid_report +# +# experimental wiki uploading is available by running +# +# python test_pretty_print.py --upload +# + +# Design philosophy +# +# 1. parse the data (images, logs) into easy-to-use data structures +# 2. wikifiy the data +# 3. save the wikified data to disk + +# todo +# do something if tests for GL extensions for OpenCSG fail (test fail, no image production) +# copy all images, sysinfo.txt to bundle for html/upload (images +# can be altered by subsequent runs) +# figure out hwo to make the thing run after the test +# figure out how CTEST treats the logfiles. +# why is hash differing +# instead of having special '-info' prerun, put it as yet-another-test +# and parse the log +# fix windows so that it won't keep asking 'this program crashed' over and over. +# (you can set this in the registry to never happen, but itd be better if the program +# itself was able to disable that temporarily in it's own process) + +import string,sys,re,os,hashlib,subprocess,textwrap,time + +def tryread(filename): + data = None + try: + f = open(filename,'rb') + data = f.read() + f.close() + except: + print 'couldn\'t open ',filename + return data + +def trysave(filename,data): + try: + if not os.path.isdir(os.path.dirname(filename)): + #print 'creating',os.path.dirname(filename) + os.mkdir(os.path.dirname(filename)) + f=open(filename,'wb') + f.write(data) + f.close() + except: + print 'problem writing to',filename + return None + return True + +def ezsearch(pattern,str): + x = re.search(pattern,str,re.DOTALL|re.MULTILINE) + if x and len(x.groups())>0: return x.group(1).strip() + return '' + +def read_gitinfo(): + # won't work if run from outside of branch. + data = subprocess.Popen(['git','remote','-v'],stdout=subprocess.PIPE).stdout.read() + origin = ezsearch('^origin *?(.*?)\(fetch.*?$',data) + upstream = ezsearch('^upstream *?(.*?)\(fetch.*?$',data) + data = subprocess.Popen(['git','branch'],stdout=subprocess.PIPE).stdout.read() + branch = ezsearch('^\*(.*?)$',data) + out = 'Git branch: ' + branch + ' from origin ' + origin + '\n' + out += 'Git upstream: ' + upstream + '\n' + return out + +def read_sysinfo(filename): + data = tryread(filename) + if not data: return 'sysinfo: unknown' + + machine = ezsearch('Machine:(.*?)\n',data) + machine = machine.replace(' ','-').replace('/','-') + + osinfo = ezsearch('OS info:(.*?)\n',data) + osplain = osinfo.split(' ')[0].strip().replace('/','-') + if 'windows' in osinfo.lower(): osplain = 'win' + + renderer = ezsearch('GL Renderer:(.*?)\n',data) + tmp = renderer.split(' ') + tmp = string.join(tmp[0:3],'-') + tmp = tmp.split('/')[0] + renderer = tmp + + data += read_gitinfo() + + data += 'Image comparison: ImageMagick' + + data = data.strip() + + # create 4 letter hash and stick on end of sysid + nondate_data = re.sub("\n.*?ompile date.*?\n","\n",data).strip() + hexhash = hashlib.md5() + hexhash.update(nondate_data) + hexhash = hexhash.hexdigest()[-4:].upper() + hash = '' + for c in hexhash: hash += chr(ord(c)+97-48) + + sysid = osplain + '_' + machine + '_' + renderer + '_' + hash + sysid = sysid.lower() + + return data, sysid + +class Test: + def __init__(self,fullname,time,passed,output,type,actualfile,expectedfile,scadfile,log): + self.fullname,self.time,self.passed,self.output = \ + fullname, time, passed, output + self.type, self.actualfile, self.expectedfile, self.scadfile = \ + type, actualfile, expectedfile, scadfile + self.fulltestlog = log + + def __str__(self): + x = 'fullname: ' + self.fullname + x+= '\nactualfile: ' + self.actualfile + x+= '\nexpectedfile: ' + self.expectedfile + x+= '\ntesttime: ' + self.time + x+= '\ntesttype: ' + self.type + x+= '\npassed: ' + str(self.passed) + x+= '\nscadfile: ' + self.scadfile + x+= '\noutput bytes: ' + str(len(self.output)) + x+= '\ntestlog bytes: ' + str(len(self.fulltestlog)) + x+= '\n' + return x + +def parsetest(teststring): + patterns = ["Test:(.*?)\n", # fullname + "Test time =(.*?) sec\n", + "Test time.*?Test (Passed)", # pass/fail + "Output:(.*?)<end of output>", + 'Command:.*?-s" "(.*?)"', # type + "actual .*?:(.*?)\n", + "expected .*?:(.*?)\n", + 'Command:.*?(testdata.*?)"' # scadfile + ] + hits = map( lambda pattern: ezsearch(pattern,teststring), patterns ) + test = Test(hits[0],hits[1],hits[2]=='Passed',hits[3],hits[4],hits[5],hits[6],hits[7],teststring) + test.actualfile_data = tryread(test.actualfile) + test.expectedfile_data = tryread(test.expectedfile) + return test + +def parselog(data): + startdate = ezsearch('Start testing: (.*?)\n',data) + enddate = ezsearch('End testing: (.*?)\n',data) + pattern = '([0-9]*/[0-9]* Testing:.*?time elapsed.*?\n)' + test_chunks = re.findall(pattern,data,re.S) + tests = map( parsetest, test_chunks ) + tests = sorted(tests, key = lambda t:t.passed) + return startdate, tests, enddate + +def load_makefiles(builddir): + filelist = [] + for root, dirs, files in os.walk(builddir): + for fname in files: filelist += [ os.path.join(root, fname) ] + files = filter(lambda x: 'build.make' in os.path.basename(x), filelist) + files += filter(lambda x: 'flags.make' in os.path.basename(x), filelist) + files = filter(lambda x: 'esting' not in x and 'emporary' not in x, files) + result = {} + for fname in files: + result[fname.replace(builddir,'')] = open(fname,'rb').read() + return result + +def wikify_filename(fname, wiki_rootpath, sysid): + wikifname = fname.replace('/','_').replace('\\','_').strip('.') + return wiki_rootpath + '_' + sysid + '_' + wikifname + +def towiki(wiki_rootpath, startdate, tests, enddate, sysinfo, sysid, makefiles): + + wiki_template = """ +<h3>[[WIKI_ROOTPATH]] test run report</h3> + +'''Sysid''': SYSID + +'''Result summary''': NUMPASSED / NUMTESTS tests passed ( PERCENTPASSED % ) <br> + +'''System info''': +<pre> +SYSINFO +</pre> + +start time: STARTDATE <br> +end time : ENDDATE <br> + +'''Image tests''' + +<REPEAT1> +{| border=1 cellspacing=0 cellpadding=1 +|- +| colspan=2 | FTESTNAME +|- +| Expected image || Actual image +|- +| [[File:EXPECTEDFILE|250px]] || ACTUALFILE_WIKI +|} + +<pre> +TESTLOG +</pre> + + + +</REPEAT1> + + +'''Text tests''' + +<REPEAT2> +{|border=1 cellspacing=0 cellpadding=1 +|- +| FTESTNAME +|} + +<pre> +TESTLOG +</pre> + + +</REPEAT2> + +'''build.make and flags.make''' +<REPEAT3> +*[[MAKEFILE_NAME]] +</REPEAT3> +""" + txtpages = {} + imgs = {} + passed_tests = filter(lambda x: x.passed, tests) + failed_tests = filter(lambda x: not x.passed, tests) + percent = str(int(100.0*len(passed_tests) / len(tests))) + s = wiki_template + repeat1 = ezsearch('(<REPEAT1>.*?</REPEAT1>)',s) + repeat2 = ezsearch('(<REPEAT2>.*?</REPEAT2>)',s) + repeat3 = ezsearch('(<REPEAT3>.*?</REPEAT3>)',s) + dic = { 'STARTDATE': startdate, 'ENDDATE': enddate, 'WIKI_ROOTPATH': wiki_rootpath, + 'SYSINFO': sysinfo, 'SYSID':sysid, + 'NUMTESTS':len(tests), 'NUMPASSED':len(passed_tests), 'PERCENTPASSED':percent } + for key in dic.keys(): + s = s.replace(key,str(dic[key])) + for t in tests: + if t.type=='txt': + newchunk = re.sub('FTESTNAME',t.fullname,repeat2) + newchunk = newchunk.replace('TESTLOG',t.fulltestlog) + s = s.replace(repeat2, newchunk+repeat2) + elif t.type=='png': + tmp = t.actualfile.replace(builddir,'') + wikiname_a = wikify_filename(tmp,wiki_rootpath,sysid) + tmp = t.expectedfile.replace(os.path.dirname(builddir),'') + wikiname_e = wikify_filename(tmp,wiki_rootpath,sysid) + imgs[wikiname_e] = t.expectedfile_data + if t.actualfile: + actualfile_wiki = '[[File:'+wikiname_a+'|250px]]' + imgs[wikiname_a] = t.actualfile_data + else: + actualfile_wiki = 'No image generated.' + newchunk = re.sub('FTESTNAME',t.fullname,repeat1) + newchunk = newchunk.replace('ACTUALFILE_WIKI',actualfile_wiki) + newchunk = newchunk.replace('EXPECTEDFILE',wikiname_e) + newchunk = newchunk.replace('TESTLOG',t.fulltestlog) + s = s.replace(repeat1, newchunk+repeat1) + + makefiles_wikinames = {} + for mf in sorted(makefiles.keys()): + tmp = mf.replace('CMakeFiles','').replace('.dir','') + wikiname = wikify_filename(tmp,wiki_rootpath,sysid) + newchunk = re.sub('MAKEFILE_NAME',wikiname,repeat3) + s = s.replace(repeat3, newchunk+repeat3) + makefiles_wikinames[mf] = wikiname + + s = s.replace(repeat1,'') + s = s.replace(repeat2,'') + s = s.replace(repeat3,'') + s = re.sub('<REPEAT.*?>\n','',s) + s = re.sub('</REPEAT.*?>','',s) + + mainpage_wikiname = wiki_rootpath + '_' + sysid + '_test_report' + txtpages[ mainpage_wikiname ] = s + for mf in sorted(makefiles.keys()): + txtpages[ makefiles_wikinames[ mf ] ] = '\n*Subreport from [['+mainpage_wikiname+']]\n\n\n<pre>\n'+makefiles[mf]+'\n</pre>' + + return imgs, txtpages + +def tohtml(wiki_rootpath, startdate, tests, enddate, sysinfo, sysid, makefiles): + # kludge. assume wiki stuff has alreayd run and dumped files properly + head = '<html><head><title>'+wiki_rootpath+' test run for '+sysid +'</title></head><body>' + tail = '</body></html>' + + passed_tests = filter(lambda x: x.passed, tests) + failed_tests = filter(lambda x: not x.passed, tests) + percent = str(int(100.0*len(passed_tests) / len(tests))) + + s='' + + s+= '\n<pre>' + s+= '\nSYSINFO\n'+ sysinfo + s+= '\n</pre><p>' + + s+= '\n<pre>' + s+= '\nSTARTDATE: '+ startdate + s+= '\nENDDATE: '+ enddate + s+= '\nWIKI_ROOTPATH: '+ wiki_rootpath + s+= '\nSYSID: '+sysid + s+= '\nNUMTESTS: '+str(len(tests)) + s+= '\nNUMPASSED: '+str(len(passed_tests)) + s+= '\nPERCENTPASSED: '+ percent + s+= '\n</pre>' + + for t in tests: + if t.type=='txt': + s+='\n<pre>'+t.fullname+'</pre>\n' + s+='<p><pre>'+t.fulltestlog+'</pre>\n\n' + elif t.type=='png': + tmp = t.actualfile.replace(builddir,'') + wikiname_a = wikify_filename(tmp,wiki_rootpath,sysid) + tmp = t.expectedfile.replace(os.path.dirname(builddir),'') + wikiname_e = wikify_filename(tmp,wiki_rootpath,sysid) + s+='<table>' + s+='\n<tr><td colspan=2>'+t.fullname + s+='\n<tr><td>Expected<td>Actual' + s+='\n<tr><td><img src='+wikiname_e+' width=250/>' + s+='\n <td><img src='+wikiname_a+' width=250/>' + s+='\n</table>' + s+='\n<pre>' + s+=t.fulltestlog + s+='\n</pre>' + + s+='\n\n<p>\n\n' + makefiles_wikinames = {} + for mf in sorted(makefiles.keys()): + tmp = mf.replace('CMakeFiles','').replace('.dir','') + wikiname = wikify_filename(tmp,wiki_rootpath,sysid) + s += '\n<a href='+wikiname+'>'+wikiname+'</a><br>' + s+='\n' + + return head + s + tail + +def wiki_login(wikiurl,api_php_path,botname,botpass): + site = mwclient.Site(wikiurl,api_php_path) + site.login(botname,botpass) + return site + +def wiki_upload(wikiurl,api_php_path,botname,botpass,filedata,wikipgname): + counter = 0 + done = False + descrip = 'test' + time.sleep(1) + while not done: + try: + print 'login',botname,'to',wikiurl + site = wiki_login(wikiurl,api_php_path,botname,botpass) + print 'uploading...', + if wikipgname.endswith('png'): + site.upload(filedata,wikipgname,descrip,ignore=True) + else: + page = site.Pages[wikipgname] + text = page.edit() + page.save(filedata) + done = True + print 'transfer ok' + except Exception, e: + print 'Error:', type(e),e + counter += 1 + if counter>maxretry: + print 'giving up. please try a different wiki site' + done = True + else: + print 'wiki',wikiurl,'down. retrying in 15 seconds' + time.sleep(15) + +def upload(wikiurl,api_php_path='/',wiki_rootpath='test', sysid='null', botname='cakebaby',botpass='anniew',wikidir='.',dryrun=True): + wetrun = not dryrun + if dryrun: print 'dry run' + try: + global mwclient + import mwclient + except: + print 'please download mwclient 0.6.5 and unpack here:', os.getcwd() + sys.exit() + + if wetrun: site = wiki_login(wikiurl,api_php_path,botname,botpass) + + wikifiles = os.listdir(wikidir) + testreport_page = filter( lambda x: 'test_report' in x, wikifiles ) + if (len(testreport_page)>1): + print 'multiple test reports found, please clean dir',wikidir + sys.exit() + + rootpage = testreport_page[0] + print 'add',rootpage,' to main report page ',wiki_rootpath + if wetrun: + page = site.Pages[wiki_rootpath] + text = page.edit() + if not '[['+rootpage+']]' in text: + page.save(text +'\n*[['+rootpage+']]\n') + + wikifiles = os.listdir(wikidir) + wikifiles = filter(lambda x: not x.endswith('html'), wikifiles) + + print 'upload wiki pages:' + for wikiname in wikifiles: + filename = os.path.join(wikidir,wikiname) + filedata = tryread(filename) + print 'upload',len(filedata),'bytes from',wikiname + if wetrun: + wiki_upload(wikiurl,api_php_path,botname,botpass,filedata,wikiname) + +def findlogfile(builddir): + logpath = os.path.join(builddir,'Testing','Temporary') + logfilename = os.path.join(logpath,'LastTest.log.tmp') + if not os.path.isfile(logfilename): + logfilename = os.path.join(logpath,'LastTest.log') + if not os.path.isfile(logfilename): + print 'cant find and/or open logfile',logfilename + sys.exit() + return logpath, logfilename + +def main(): + dry = False + if verbose: print 'running test_pretty_print' + if '--dryrun' in sys.argv: dry=True + suffix = ezsearch('--suffix=(.*?) ',string.join(sys.argv)+' ') + builddir = ezsearch('--builddir=(.*?) ',string.join(sys.argv)+' ') + if builddir=='': builddir=os.getcwd() + if verbose: print 'build dir set to', builddir + + sysinfo, sysid = read_sysinfo(os.path.join(builddir,'sysinfo.txt')) + makefiles = load_makefiles(builddir) + logpath, logfilename = findlogfile(builddir) + testlog = tryread(logfilename) + startdate, tests, enddate = parselog(testlog) + if verbose: + print 'found sysinfo.txt,', + print 'found', len(makefiles),'makefiles,', + print 'found', len(tests),'test results' + + imgs, txtpages = towiki(wiki_rootpath, startdate, tests, enddate, sysinfo, sysid, makefiles) + + wikidir = os.path.join(logpath,sysid+'_report') + if verbose: print 'erasing files in',wikidir + try: map(lambda x:os.remove(os.path.join(wikidir,x)), os.listdir(wikidir)) + except: pass + print 'writing',len(imgs),'images and',len(txtpages),'text pages to:\n', ' .'+wikidir.replace(os.getcwd(),'') + for pgname in sorted(imgs): trysave( os.path.join(wikidir,pgname), imgs[pgname]) + for pgname in sorted(txtpages): trysave( os.path.join(wikidir,pgname), txtpages[pgname]) + + htmldata = tohtml(wiki_rootpath, startdate, tests, enddate, sysinfo, sysid, makefiles) + trysave( os.path.join(wikidir,'index.html'), htmldata ) + + if '--upload' in sys.argv: + upload(wikisite,wiki_api_path,wiki_rootpath,sysid,'openscadbot', + 'tobdacsnepo',wikidir,dryrun=dry) + print 'upload attempt complete' + + if verbose: print 'test_pretty_print complete' + +#wikisite = 'cakebaby.referata.com' +#wiki_api_path = '' +wikisite = 'cakebaby.wikia.com' +wiki_api_path = '/' +wiki_rootpath = 'OpenSCAD' +builddir = os.getcwd() # os.getcwd()+'/build' +verbose = False +maxretry = 10 + +main() |