1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
|
#!/usr/bin/env python
#
# Regression test driver for cmd-line tools
#
# Usage: test_cmdline_tool.py [<options>] <tool> <arguments>
#
# If the -g option is given or the TEST_GENERATE environment variable is set to 1,
# *-expected.<suffix> files will be generated instead of running the tests.
#
# Any generated output is written to the file `basename <argument`-actual.<suffix>
# Any warning or errors are written to stderr.
#
# Returns 0 on passed test
# 1 on error
# 2 on invalid cmd-line options
#
# Author: Marius Kintel <marius@kintel.net>
#
import sys
import os
import glob
import subprocess
import re
import getopt
def initialize_environment():
if not options.generate: options.generate = bool(os.getenv("TEST_GENERATE"))
return True
def init_expected_filename(testname, cmd):
global expecteddir, expectedfilename
expecteddir = os.path.join(options.regressiondir, os.path.split(cmd)[1])
expectedfilename = os.path.join(expecteddir, testname + "-expected" + options.suffix)
def verify_test(testname, cmd):
global expectedfilename
if not options.generate:
if not os.path.isfile(expectedfilename):
print >> sys.stderr, "Error: test '%s' is missing expected output in %s" % (testname, expectedfilename)
return False
return True
def execute_and_redirect(cmd, params, outfile):
proc = subprocess.Popen([cmd] + params, stdout=outfile)
retval = proc.wait()
return retval
def get_normalized_text(filename):
text = open(filename).read()
return text.strip("\r\n").replace("\r\n", "\n") + "\n"
def compare_text(expected, actual):
return get_normalized_text(expected) == get_normalized_text(actual)
def compare_with_expected(resultfilename):
if not options.generate:
if not compare_text(expectedfilename, resultfilename):
execute_and_redirect("diff", [expectedfilename, resultfilename], sys.stderr)
return False
return True
def run_test(testname, cmd, args):
cmdname = os.path.split(options.cmd)[1]
outputdir = os.path.join(os.getcwd(), cmdname + "-output")
actualfilename = os.path.join(outputdir, testname + "-actual" + options.suffix)
if options.generate:
if not os.path.exists(expecteddir): os.makedirs(expecteddir)
outputname = expectedfilename
else:
if not os.path.exists(outputdir): os.makedirs(outputdir)
outputname = actualfilename
outfile = open(outputname, "wb")
try:
proc = subprocess.Popen([cmd] + args, stdout=outfile, stderr=subprocess.PIPE)
errtext = proc.communicate()[1]
if errtext != None and len(errtext) > 0:
print >> sys.stderr, "Error output: " + errtext
outfile.close()
if proc.returncode != 0:
print >> sys.stderr, "Error: %s failed with return code %d" % (cmdname, proc.returncode)
return None
return outputname
except OSError, err:
print >> sys.stderr, "Error: %s \"%s\"" % (err.strerror, cmd)
return None
class Options:
def __init__(self):
self.__dict__['options'] = {}
def __setattr__(self, name, value):
self.options[name] = value
def __getattr__(self, name):
return self.options[name]
def usage():
print >> sys.stderr, "Usage: " + sys.argv[0] + " [<options>] <cmdline-tool> <argument>"
print >> sys.stderr, "Options:"
print >> sys.stderr, " -g, --generate Generate expected output for the given tests"
print >> sys.stderr, " -s, --suffix=<suffix> Write -expected and -actual files with the given suffix instead of .txt"
print >> sys.stderr, " -t, --test=<name> Specify test name instead of deducting it from the argument"
if __name__ == '__main__':
# Handle command-line arguments
try:
opts, args = getopt.getopt(sys.argv[1:], "gs:t:", ["generate", "suffix=", "test="])
except getopt.GetoptError, err:
usage()
sys.exit(2)
global options
options = Options()
options.regressiondir = os.path.join(os.path.split(sys.argv[0])[0], "regression")
options.generate = False
options.suffix = ".txt"
for o, a in opts:
if o in ("-g", "--generate"): options.generate = True
elif o in ("-s", "--suffix"):
if a[0] == '.': options.suffix = ""
else: options.suffix = "."
options.suffix += a
elif o in ("-t", "--test"):
options.testname = a
# <cmdline-tool> and <argument>
if len(args) < 2:
usage()
sys.exit(2)
options.cmd = args[0]
# If only one test file, we can usually deduct the test name from the file
if len(args) == 2:
basename = os.path.splitext(args[1])[0]
path, options.testname = os.path.split(basename)
if not hasattr(options, "testname"):
print >> sys.stderr, "Test name cannot be deducted from arguments. Specify test name using the -t option"
sys.exit(2)
# Initialize and verify run-time environment
if not initialize_environment(): sys.exit(1)
init_expected_filename(options.testname, options.cmd)
# Verify test environment
verification = verify_test(options.testname, options.cmd)
resultfile = run_test(options.testname, options.cmd, args[1:])
if not verification or not compare_with_expected(resultfile): exit(1)
|