1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
|
#!/usr/bin/env python
#
# Regression test driver for cmd-line tools
#
# Usage: test_cmdline_tool.py [<options>] <tool> <arguments>
#
# If the -g option is given or the TEST_GENERATE environment variable is set to 1,
# *-expected.<suffix> files will be generated instead of running the tests.
#
# Any generated output is written to the file `basename <argument`-actual.<suffix>
# Any warning or errors are written to stderr.
#
# Returns 0 on passed test
# 1 on error
# 2 on invalid cmd-line options
#
# Author: Marius Kintel <marius@kintel.net>
#
import sys
import os
import glob
import subprocess
import re
import getopt
import shutil
import platform
def initialize_environment():
if not options.generate: options.generate = bool(os.getenv("TEST_GENERATE"))
return True
def init_expected_filename(testname, cmd):
global expecteddir, expectedfilename
expecteddir = os.path.join(options.regressiondir, os.path.split(cmd)[1])
expectedfilename = os.path.join(expecteddir, testname + "-expected." + options.suffix)
expectedfilename = os.path.normpath( expectedfilename )
def verify_test(testname, cmd):
global expectedfilename
if not options.generate:
if not os.path.isfile(expectedfilename):
print >> sys.stderr, "Error: test '%s' is missing expected output in %s" % (testname, expectedfilename)
return False
return True
def execute_and_redirect(cmd, params, outfile):
retval = -1
try:
proc = subprocess.Popen([cmd] + params, stdout=outfile)
out = proc.communicate()[0]
retval = proc.wait()
except:
print >> sys.stderr, "Error running subprocess: ", sys.exc_info()[1]
print >> sys.stderr, " cmd:", cmd
print >> sys.stderr, " params:", params
print >> sys.stderr, " outfile:", outfile
if outfile == subprocess.PIPE: return (retval, out)
else: return retval
def get_normalized_text(filename):
text = open(filename).read()
return text.strip("\r\n").replace("\r\n", "\n") + "\n"
def compare_text(expected, actual):
return get_normalized_text(expected) == get_normalized_text(actual)
def compare_default(resultfilename):
print >> sys.stderr, 'diff text compare: '
print >> sys.stderr, ' expected textfile: ', expectedfilename
print >> sys.stderr, ' actual textfile: ', resultfilename
if not compare_text(expectedfilename, resultfilename):
execute_and_redirect("diff", [expectedfilename, resultfilename], sys.stderr)
return False
return True
def compare_png(resultfilename):
#args = [expectedfilename, resultfilename, "-alpha", "Off", "-compose", "difference", "-composite", "-threshold", "10%", "-blur", "2", "-threshold", "30%", "-format", "%[fx:w*h*mean]", "info:"]
#args = [expectedfilename, resultfilename, "-alpha", "Off", "-compose", "difference", "-composite", "-threshold", "10%", "-morphology", "Erode", "Square", "-format", "%[fx:w*h*mean]", "info:"]
# 'morphology' is only available in newer versions of ImageMagick.
# http://www.imagemagick.org/Usage/morphology/#alturnative
args = [expectedfilename, resultfilename, "-alpha", "Off",
"-compose", "difference", "-composite", "-threshold", "10%",
#"-morphology", "Erode", "Square",
"-gaussian-blur","3x65535", "-threshold","99.999%",
"-format", "%[fx:w*h*mean]", "info:"]
print >> sys.stderr, 'ImageMagick image comparison: convert ', ' '.join(args[2:])
print >> sys.stderr, ' expected image: ', expectedfilename
if not resultfilename:
print >> sys.stderr, "Error: OpenSCAD did not generate an image to test"
return False
print >> sys.stderr, ' actual image: ', resultfilename
(retval, output) = execute_and_redirect(options.convert_exec, args, subprocess.PIPE)
if retval == 0:
pixelerr = int(float(output.strip()))
if pixelerr < 32: return True
else: print >> sys.stderr, pixelerr, ' pixel errors'
return False
def compare_with_expected(resultfilename):
if not options.generate:
if "compare_" + options.suffix in globals(): return globals()["compare_" + options.suffix](resultfilename)
else: return compare_default(resultfilename)
return True
def run_test(testname, cmd, args):
cmdname = os.path.split(options.cmd)[1]
outputdir = os.path.join(os.getcwd(), cmdname + "-output")
actualfilename = os.path.join(outputdir, testname + "-actual." + options.suffix)
actualfilename = os.path.normpath(actualfilename)
if options.generate:
if not os.path.exists(expecteddir): os.makedirs(expecteddir)
outputname = expectedfilename
else:
if not os.path.exists(outputdir): os.makedirs(outputdir)
outputname = actualfilename
outputname = os.path.normpath( outputname )
outfile = open(outputname, "wb")
try:
proc = subprocess.Popen([cmd] + args + [outputname], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
errtext = proc.communicate()[1]
if errtext != None and len(errtext) > 0:
print >> sys.stderr, "Error output: " + errtext
outfile.close()
if proc.returncode != 0:
print >> sys.stderr, "Error: %s failed with return code %d" % (cmdname, proc.returncode)
return None
return outputname
except OSError, err:
print >> sys.stderr, "Error: %s \"%s\"" % (err.strerror, cmd)
return None
class Options:
def __init__(self):
self.__dict__['options'] = {}
def __setattr__(self, name, value):
self.options[name] = value
def __getattr__(self, name):
return self.options[name]
def usage():
print >> sys.stderr, "Usage: " + sys.argv[0] + " [<options>] <cmdline-tool> <argument>"
print >> sys.stderr, "Options:"
print >> sys.stderr, " -g, --generate Generate expected output for the given tests"
print >> sys.stderr, " -s, --suffix=<suffix> Write -expected and -actual files with the given suffix instead of .txt"
print >> sys.stderr, " -t, --test=<name> Specify test name instead of deducting it from the argument"
print >> sys.stderr, " -c, --convexec=<name> Path to ImageMagick 'convert' executable"
if __name__ == '__main__':
# Handle command-line arguments
try:
opts, args = getopt.getopt(sys.argv[1:], "gs:c:t:", ["generate", "convexec=", "suffix=", "test="])
except getopt.GetoptError, err:
usage()
sys.exit(2)
global options
options = Options()
options.regressiondir = os.path.join(os.path.split(sys.argv[0])[0], "regression")
options.generate = False
options.suffix = "txt"
for o, a in opts:
if o in ("-g", "--generate"): options.generate = True
elif o in ("-s", "--suffix"):
if a[0] == '.': options.suffix = a[1:]
else: options.suffix = a
elif o in ("-t", "--test"):
options.testname = a
elif o in ("-c", "--convexec"):
options.convert_exec = os.path.normpath( a )
# <cmdline-tool> and <argument>
if len(args) < 2:
usage()
sys.exit(2)
options.cmd = args[0]
# If only one test file, we can usually deduct the test name from the file
if len(args) == 2:
basename = os.path.splitext(args[1])[0]
path, options.testname = os.path.split(basename)
if not hasattr(options, "testname"):
print >> sys.stderr, "Test name cannot be deducted from arguments. Specify test name using the -t option"
sys.exit(2)
# Initialize and verify run-time environment
if not initialize_environment(): sys.exit(1)
init_expected_filename(options.testname, options.cmd)
# Verify test environment
verification = verify_test(options.testname, options.cmd)
resultfile = run_test(options.testname, options.cmd, args[1:])
if not verification or not compare_with_expected(resultfile): exit(1)
|
