simplify test_pretty_print to work better with multiple page uploads

1 files changed, 114 insertions, 103 deletions

diff --git a/tests/test_pretty_print.py b/tests/test_pretty_print.py
index 82ece3d..3f5041d 100755
--- a/tests/test_pretty_print.py
+++ b/tests/test_pretty_print.py
@@ -22,18 +22,19 @@ import string,sys,re,os,hashlib,subprocess,textwrap
 def tryread(filename):
 	data = None
 	try:
-		print 'reading', filename
-		f = open(filename)
+		f = open(filename,'rb')
 		data = f.read()
 		f.close()
 	except:
 		print 'couldn\'t open ',filename
 	return data
 
-def trysave(data,filename):
+def trysave(filename,data):
 	try:
-		f=open(filename,'w')
-		print 'writing',len(data),'bytes to',filename
+		if not os.path.isdir(os.path.dirname(filename)):
+			print 'creating',os.path.dirname(filename)
+			os.mkdir(os.path.dirname(filename))
+		f=open(filename,'wb')
 		f.write(data)
 		f.close()
 	except:
@@ -125,6 +126,8 @@ def parsetest(teststring):
 		]
 	hits = map( lambda pattern: ezsearch(pattern,teststring), patterns )
 	test = Test(hits[0],hits[1],hits[2]=='Passed',hits[3],hits[4],hits[5],hits[6],hits[7],teststring)
+	test.actualfile_data = tryread(test.actualfile)
+	test.expectedfile_data = tryread(test.expectedfile)
 	return test
 
 def parselog(data):
@@ -133,35 +136,26 @@ def parselog(data):
 	pattern = '([0-9]*/[0-9]* Testing:.*?time elapsed.*?\n)'
 	test_chunks = re.findall(pattern,data,re.S)
 	tests = map( parsetest, test_chunks )
-	print 'found', len(tests),'test results'
+	tests = sorted(tests, key = lambda t:t.passed)
 	return startdate, tests, enddate
 
-def wikify_filename(testname,filename,sysid):
-	# translate from local system to wiki style filename.
-	result = wiki_rootpath+'_'+testname+'_'
-	expected = ezsearch('(expected....$)',filename)
-	if expected!='': result += expected
-	actual = ezsearch(os.sep+'.*?-output.*?(actual.*)',filename)
-	if actual!='': 
-		result += sysid+'_'+actual
-	return result.replace('/','_').replace('\\','_')
-
-def parsemakes_towiki(wiki_rootpath, sysid):
+def load_makefiles(builddir):
 	filelist = []
-	for root, dirs, files in os.walk('.'):
+	for root, dirs, files in os.walk(builddir):
 		for fname in files: filelist += [ os.path.join(root, fname) ]
 	files  = filter(lambda x: 'build.make' in os.path.basename(x), filelist)
 	files += filter(lambda x: 'flags.make' in os.path.basename(x), filelist)
-	manifest = {}
+	files = filter(lambda x: 'esting' not in x and 'emporary' not in x, files)
+	result = {}
 	for fname in files:
-		wikifname = fname.replace('/','_').replace('\\','_').strip('.')
-		wikifname = wikifname.replace('CMakeFiles','').replace('.dir','')
-		manifest[fname] = wiki_rootpath + '_' + sysid + '_' + wikifname
-	wikicode = "\n'''build.make and flags.make'''\n\n\n"
-	for key in sorted(manifest.keys()): wikicode += '*[[' + manifest[key] + ']]\n'
-	return manifest, wikicode
+		result[fname.replace(builddir,'')] = open(fname,'rb').read()
+	return result
+
+def wikify_filename(fname, wiki_rootpath, sysid):
+	wikifname = fname.replace('/','_').replace('\\','_').strip('.')
+	return wiki_rootpath + '_' + sysid + '_' + wikifname
 
-def towiki(wiki_rootpath, startdate, tests, enddate, sysinfo, sysid):
+def towiki(wiki_rootpath, startdate, tests, enddate, sysinfo, sysid, makefiles):
 
 	wiki_template = """
 <h3>[[WIKI_ROOTPATH]] test run report</h3>
@@ -209,8 +203,13 @@ TESTLOG
 </REPEAT2>
 |}
 
-
+'''build.make and flags.make'''
+<REPEAT3>
+*[[MAKEFILE_NAME]]
+</REPEAT3>
 """
+	txtpages = {}
+	imgs = {}
 	passed_tests = filter(lambda x: x.passed, tests)
 	failed_tests = filter(lambda x: not x.passed, tests)
 	percent = str(int(100.0*len(passed_tests) / len(tests)))
@@ -218,6 +217,7 @@ TESTLOG
 	s = wiki_template
 	repeat1 = ezsearch('(<REPEAT1>.*?</REPEAT1>)',s)
 	repeat2 = ezsearch('(<REPEAT2>.*?</REPEAT2>)',s)
+	repeat3 = ezsearch('(<REPEAT3>.*?</REPEAT3>)',s)
 	dic = { 'STARTDATE': startdate, 'ENDDATE': enddate, 'WIKI_ROOTPATH': wiki_rootpath,
 		'SYSINFO': sysinfo, 'SYSID':sysid, 
 		'NUMTESTS':len(tests), 'NUMPASSED':len(passed_tests), 'PERCENTPASSED':percent }
@@ -231,24 +231,43 @@ TESTLOG
 			newchunk = re.sub('FTESTNAME',t.fullname,repeat2)
 			s = s.replace(repeat2, newchunk+repeat2)
 		elif t.type=='png':
-			manifest[t.actualfile] = wikify_filename(t.fullname,t.actualfile,sysid)
-			manifest[t.expectedfile] = wikify_filename(t.fullname,t.expectedfile,sysid)
+			tmp = t.actualfile.replace(builddir,'')
+			wikiname_a = wikify_filename(tmp,wiki_rootpath,sysid)
+			# erase /home/whatever/openscad/tests/regression
+			tmp = t.expectedfile.replace(os.path.dirname(builddir),'')
+			wikiname_e = wikify_filename(tmp,wiki_rootpath,sysid)
+			imgs[wikiname_e] = t.expectedfile_data
 			if t.actualfile: 
-				actualfile_wiki = '[[File:'+manifest[t.actualfile]+'|250px]]'
+				actualfile_wiki = '[[File:'+wikiname_a+'|250px]]'
+				imgs[wikiname_a] = t.actualfile_data
 			else:
 				actualfile_wiki = 'No image generated.'
 			newchunk = re.sub('FTESTNAME',t.fullname,repeat1)
 			newchunk = newchunk.replace('ACTUALFILE_WIKI',actualfile_wiki)
-			newchunk = newchunk.replace('EXPECTEDFILE',manifest[t.expectedfile])
+			newchunk = newchunk.replace('EXPECTEDFILE',wikiname_e)
 			newchunk = newchunk.replace('TESTLOG',t.fulltestlog)
 			s = s.replace(repeat1, newchunk+repeat1)
 
+	makefiles_wikinames = {}
+	for mf in sorted(makefiles.keys()):
+		tmp = mf.replace('CMakeFiles','').replace('.dir','')
+		wikiname = wikify_filename(tmp,wiki_rootpath,sysid)
+		newchunk = re.sub('MAKEFILE_NAME',wikiname,repeat3)
+		s = s.replace(repeat3, newchunk+repeat3)
+		makefiles_wikinames[mf] = wikiname
+
 	s = s.replace(repeat1,'')
 	s = s.replace(repeat2,'')
+	s = s.replace(repeat3,'')
 	s = re.sub('<REPEAT.*?>\n','',s)
 	s = re.sub('</REPEAT.*?>','',s)
-	return manifest, s
 
+	mainpage_wikiname = wiki_rootpath + '_' + sysid + '_test_report'
+	txtpages[ mainpage_wikiname ] = s
+	for mf in sorted(makefiles.keys()):
+		txtpages[ makefiles_wikinames[ mf ] ] = '<pre>\n'+makefiles[mf]+'\n</pre>'
+
+	return imgs, txtpages
 
 def wikitohtml(wiki_rootpath, sysid, wikidata, manifest):
 	head = '<html><head><title>'+wiki_rootpath+' test run for '+sysid +'</title></head><body>'
@@ -266,97 +285,89 @@ def wikitohtml(wiki_rootpath, sysid, wikidata, manifest):
 	x=re.sub("\[\[(.*?)\]\]","\\1",x)
 	return head + x + '</body></html>'
 
-def upload(wikiurl,api_php_path,wikidata,manifest,wiki_rootpath,sysid,botname,botpass,dryrun=True,forceupload=False):
+def upload(wikiurl,api_php_path='/',wiki_rootpath='test', sysid='null', botname='cakebaby',botpass='anniew',wikidir='.',dryrun=True):
+	wetrun = not dryrun
 	if dryrun: print 'dry run'
 	try:
 		import mwclient
 	except:
-		print 'please download mwclient and unpack here:', os.cwd()
+		print 'please download mwclient and unpack here:', os.getcwd()
+		sys.exit()
 	print 'opening site:',wikiurl
-	if not dryrun:
-		if not api_php_path == '':
-			site = mwclient.Site(wikiurl,api_php_path)
-		else:
-			site = mwclient.Site(wikiurl)
+	if wetrun:
+		site = mwclient.Site(wikiurl,api_php_path)
 		
 	print 'bot login:', botname
-	if not dryrun: site.login(botname,botpass)
-
-	print 'edit page:',wiki_rootpath
-	rootpage = wiki_rootpath + sysid
-	if not dryrun: 
+	if wetrun: site.login(botname,botpass)
+
+	wikifiles = os.listdir(wikidir)
+	testreport_page = filter( lambda x: 'test_report' in x, wikifiles )
+	if (len(testreport_page)>1): 
+		print 'multiple test reports found, please clean dir',wikidir
+		sys.exit()
+	rootpage = testreport_page[0]
+	print 'add',rootpage,' to main report page ',wiki_rootpath
+	if wetrun:
 		page = site.Pages[wiki_rootpath]
 		text = page.edit()
 		if not '[['+rootpage+']]' in text:
 			page.save(text +'\n*[['+rootpage+']]\n')
 
-	print 'upload wiki page:',rootpage
-	if not dryrun: 
-		page = site.Pages[rootpage]
-		text = page.edit()
-		page.save(wikidata)
-
-	print 'upload images:'
-	imagekeys = filter(lambda x: x.endswith('.png'), manifest.keys())
-	for localfile in sorted(imagekeys):
-		if localfile:
-			localf = open(localfile,'rb')
-			wikifile = manifest[localfile]
-			skip=False
-			if 'expected.png' in wikifile.lower():
-				if not dryrun: 
-					image = site.Images[wikifile]
-					if image.exists and forceupload==False:
-						print 'skipping',wikifile, '(already on wiki)'
-						skip=True
-			if not skip:
-				print 'uploading',wikifile,'...'
-				if not dryrun:
-					site.upload(localf,wikifile,wiki_rootpath + ' test', ignore=True)
-
-	print 'upload makefiles:'
-	makekeys = filter(lambda x: x.endswith('.make'), manifest.keys())	
-	for localfile in sorted(makekeys):
-		if localfile:
-			localf = open(localfile,'rb')
-			wikifile = manifest[localfile]
-			print 'uploading',wikifile,'...'
-			if not dryrun:
-				page = site.Pages[wikifile]
+	wikifiles = os.listdir(wikidir)
+	print 'upload wiki pages:'
+	for wikiname in wikifiles:
+		filename = os.path.join(wikidir,wikiname)
+		filedata = tryread(filename)
+		print 'upload',len(filedata),'bytes from',wikiname,'...',
+		sys.stdout.flush()
+		if wikiname.endswith('.png'):
+			localf = open(filename,'rb') # mwclient needs file handle
+			descrip = wiki_rootpath + ' test'
+			if wetrun:
+				site.upload(localf,wikiname,descrip,ignore=True)
+			print 'image uploaded'
+		else: # textpage
+			if wetrun:
+				page = site.Pages[wikiname]
 				text = page.edit()
-				page.save('<pre>\n'+localf.read()+'\n</pre>')
+				page.save(filedata)
+			print 'text page uploaded'
+
+def findlogfile(builddir):
+	logpath = os.path.join(builddir,'Testing','Temporary')
+	logfilename = os.path.join(logpath,'LastTest.log')
+	return logpath, logfilename
+
+def main():
+	dry = False
+	if '--dryrun' in sys.argv: dry=True
+	print 'build dir set to', builddir
+
+	sysinfo, sysid = read_sysinfo(os.path.join(builddir,'sysinfo.txt'))
+	makefiles = load_makefiles(builddir)
+	logpath, logfilename = findlogfile(builddir)
+	testlog = tryread(logfilename)
+	startdate, tests, enddate = parselog(testlog)
+	print 'found sysinfo.txt'
+	print 'found', len(makefiles),'makefiles'
+	print 'found', len(tests),'test results'
 
+	imgs, txtpages = towiki(wiki_rootpath, startdate, tests, enddate, sysinfo, sysid, makefiles)
 
+	wikidir = os.path.join(logpath,'wiki')
+	print 'writing',len(imgs),'images and',len(txtpages),'wiki pages to:\n ', wikidir
+	for k in sorted(imgs): trysave( os.path.join(wikidir,k), imgs[k])
+	for k in sorted(txtpages): trysave( os.path.join(wikidir,k), txtpages[k])
 
+	if '--upload' in sys.argv:
+		upload(wikisite,wiki_api_path,wiki_rootpath,sysid,'openscadbot',
+			'tobdacsnepo',wikidir,dryrun=dry)
+	
 #wikisite = 'cakebaby.referata.com'
 #wiki_api_path = ''
 wikisite = 'cakebaby.wikia.com'
 wiki_api_path = '/'
 wiki_rootpath = 'OpenSCAD'
-builddir = os.getcwd()
-logpath = os.path.join(builddir,'Testing','Temporary')
-logfilename = os.path.join(logpath,'LastTest.log')
-
-def main():
-	testlog = tryread(logfilename)
-	startdate, tests, enddate = parselog(testlog)
-	tests = sorted(tests, key = lambda t:t.passed)
-	sysinfo, sysid = read_sysinfo('sysinfo.txt')
-	if '--forceupload' in sys.argv: forceupl=True
-	else: forceupl=False
-	if '--dryrun' in sys.argv: dry=True
-	else: dry=False
-
-	manifest_makes, wiki_makes = parsemakes_towiki(wiki_rootpath, sysid)
-	manifest, wikidata = towiki(wiki_rootpath, startdate, tests, enddate, sysinfo, sysid)
-	manifest.update(manifest_makes)
-	wikidata += wiki_makes
-	trysave(wikidata, os.path.join(logpath,sysid+'.wiki'))
-	htmldata = wikitohtml(wiki_rootpath, sysid, wikidata, manifest)
-	trysave(htmldata, os.path.join(logpath,sysid+'.html'))
-	if '--upload' in sys.argv:
-		upload(wikisite,wiki_api_path,wikidata,manifest,wiki_rootpath,sysid,'openscadbot','tobdacsnepo',dryrun=dry,forceupload=forceupl)
-
+builddir = os.getcwd() # os.getcwd()+'/build'
 
 main()
-