Commit | Line | Data |
---|---|---|
31690700 JM |
1 | #!/usr/bin/env python |
2 | ||
3 | import os, sys, re | |
7907cd90 | 4 | import argparse, time |
b3c30da9 | 5 | import signal, atexit |
31690700 | 6 | |
7907cd90 JM |
7 | from subprocess import Popen, STDOUT, PIPE |
8 | from select import select | |
31690700 | 9 | |
b3c30da9 JM |
10 | # Pseudo-TTY and terminal manipulation |
11 | import pty, array, fcntl, termios | |
12 | ||
8e4628da | 13 | IS_PY_3 = sys.version_info[0] == 3 |
14 | ||
31690700 JM |
15 | # TODO: do we need to support '\n' too |
16 | sep = "\r\n" | |
7907cd90 | 17 | #sep = "\n" |
31690700 JM |
18 | rundir = None |
19 | ||
20 | parser = argparse.ArgumentParser( | |
21 | description="Run a test file against a Mal implementation") | |
22 | parser.add_argument('--rundir', | |
23 | help="change to the directory before running tests") | |
24 | parser.add_argument('--start-timeout', default=10, type=int, | |
25 | help="default timeout for initial prompt") | |
26 | parser.add_argument('--test-timeout', default=20, type=int, | |
27 | help="default timeout for each individual test action") | |
cc021efe JM |
28 | parser.add_argument('--pre-eval', default=None, type=str, |
29 | help="Mal code to evaluate prior to running the test") | |
ab01be18 JM |
30 | parser.add_argument('--no-pty', action='store_true', |
31 | help="Use direct pipes instead of pseudo-tty") | |
5abaa3dc JM |
32 | parser.add_argument('--log-file', type=str, |
33 | help="Write all test interaction the named file") | |
98af2ae3 JM |
34 | parser.add_argument('--soft', action='store_true', |
35 | help="Report but do not fail tests after ';>>> soft=True'") | |
31690700 JM |
36 | |
37 | parser.add_argument('test_file', type=argparse.FileType('r'), | |
38 | help="a test file formatted as with mal test data") | |
39 | parser.add_argument('mal_cmd', nargs="*", | |
40 | help="Mal implementation command line. Use '--' to " | |
41 | "specify a Mal command line with dashed options.") | |
42 | ||
7907cd90 | 43 | class Runner(): |
5abaa3dc | 44 | def __init__(self, args, no_pty=False, log_file=None): |
612bfe4a | 45 | #print "args: %s" % repr(args) |
ab01be18 | 46 | self.no_pty = no_pty |
612bfe4a | 47 | |
5abaa3dc JM |
48 | if log_file: self.logf = open(log_file, "a") |
49 | else: self.logf = None | |
50 | ||
612bfe4a JM |
51 | # Cleanup child process on exit |
52 | atexit.register(self.cleanup) | |
53 | ||
8caf6211 JM |
54 | self.p = None |
55 | env = os.environ | |
56 | env['TERM'] = 'dumb' | |
92dcc815 | 57 | env['INPUTRC'] = '/dev/null' |
82acd3de | 58 | env['PERL_RL'] = 'false' |
ab01be18 | 59 | if no_pty: |
612bfe4a JM |
60 | self.p = Popen(args, bufsize=0, |
61 | stdin=PIPE, stdout=PIPE, stderr=STDOUT, | |
8caf6211 JM |
62 | preexec_fn=os.setsid, |
63 | env=env) | |
7907cd90 JM |
64 | self.stdin = self.p.stdin |
65 | self.stdout = self.p.stdout | |
66 | else: | |
67 | # provide tty to get 'interactive' readline to work | |
68 | master, slave = pty.openpty() | |
b3c30da9 JM |
69 | |
70 | # Set terminal size large so that readline will not send | |
71 | # ANSI/VT escape codes when the lines are long. | |
72 | buf = array.array('h', [100, 200, 0, 0]) | |
73 | fcntl.ioctl(master, termios.TIOCSWINSZ, buf, True) | |
74 | ||
612bfe4a JM |
75 | self.p = Popen(args, bufsize=0, |
76 | stdin=slave, stdout=slave, stderr=STDOUT, | |
8caf6211 JM |
77 | preexec_fn=os.setsid, |
78 | env=env) | |
b020aa3e JM |
79 | # Now close slave so that we will get an exception from |
80 | # read when the child exits early | |
81 | # http://stackoverflow.com/questions/11165521 | |
82 | os.close(slave) | |
7907cd90 JM |
83 | self.stdin = os.fdopen(master, 'r+b', 0) |
84 | self.stdout = self.stdin | |
85 | ||
86 | #print "started" | |
87 | self.buf = "" | |
88 | self.last_prompt = "" | |
89 | ||
90 | def read_to_prompt(self, prompts, timeout): | |
91 | end_time = time.time() + timeout | |
92 | while time.time() < end_time: | |
96deb6a9 JM |
93 | [outs,_,_] = select([self.stdout], [], [], 1) |
94 | if self.stdout in outs: | |
95 | new_data = self.stdout.read(1) | |
8e4628da | 96 | new_data = new_data.decode("utf-8") if IS_PY_3 else new_data |
b020aa3e | 97 | #print "new_data: '%s'" % new_data |
5abaa3dc | 98 | self.log(new_data) |
ab01be18 | 99 | if self.no_pty: |
9a383535 JM |
100 | self.buf += new_data.replace("\n", "\r\n") |
101 | else: | |
102 | self.buf += new_data | |
7907cd90 JM |
103 | for prompt in prompts: |
104 | regexp = re.compile(prompt) | |
105 | match = regexp.search(self.buf) | |
106 | if match: | |
107 | end = match.end() | |
108 | buf = self.buf[0:end-len(prompt)] | |
109 | self.buf = self.buf[end:] | |
110 | self.last_prompt = prompt | |
111 | return buf | |
112 | return None | |
113 | ||
5abaa3dc JM |
114 | def log(self, data): |
115 | if self.logf: | |
116 | self.logf.write(data) | |
117 | self.logf.flush() | |
118 | ||
119 | ||
96deb6a9 | 120 | def writeline(self, str): |
8e4628da | 121 | def _to_bytes(s): |
122 | return bytes(s, "utf-8") if IS_PY_3 else s | |
123 | ||
124 | self.stdin.write(_to_bytes(str + "\n")) | |
7907cd90 | 125 | |
f6c83b2b | 126 | def cleanup(self): |
612bfe4a | 127 | #print "cleaning up" |
f6c83b2b | 128 | if self.p: |
10034e82 JM |
129 | try: |
130 | os.killpg(self.p.pid, signal.SIGTERM) | |
131 | except OSError: | |
132 | pass | |
f6c83b2b JM |
133 | self.p = None |
134 | ||
98af2ae3 JM |
135 | class TestReader: |
136 | def __init__(self, test_file): | |
137 | self.line_num = 0 | |
138 | self.data = test_file.read().split('\n') | |
139 | self.soft = False | |
140 | ||
141 | def next(self): | |
142 | self.form = None | |
143 | self.out = "" | |
144 | self.ret = None | |
145 | ||
146 | while self.data: | |
147 | self.line_num += 1 | |
148 | line = self.data.pop(0) | |
149 | if re.match(r"^\s*$", line): # blank line | |
150 | continue | |
151 | elif line[0:3] == ";;;": # ignore comment | |
152 | continue | |
153 | elif line[0:2] == ";;": # output comment | |
154 | print(line[3:]) | |
155 | continue | |
156 | elif line[0:5] == ";>>> ": # settings/commands | |
157 | settings = {} | |
158 | exec(line[5:], {}, settings) | |
159 | if 'soft' in settings: self.soft = True | |
160 | continue | |
161 | elif line[0:1] == ";": # unexpected comment | |
162 | print("Test data error at line %d:\n%s" % (self.line_num, line)) | |
163 | return None | |
164 | self.form = line # the line is a form to send | |
165 | ||
166 | # Now find the output and return value | |
167 | while self.data: | |
168 | line = self.data[0] | |
169 | if line[0:3] == ";=>": | |
170 | self.ret = line[3:].replace('\\r', '\r').replace('\\n', '\n') | |
171 | self.line_num += 1 | |
172 | self.data.pop(0) | |
173 | break | |
174 | elif line[0:2] == "; ": | |
175 | self.out = self.out + line[2:] + sep | |
176 | self.line_num += 1 | |
177 | self.data.pop(0) | |
178 | else: | |
179 | self.ret = "*" | |
180 | break | |
181 | if self.ret: break | |
182 | ||
183 | return self.form | |
184 | ||
f6c83b2b | 185 | |
31690700 | 186 | args = parser.parse_args(sys.argv[1:]) |
31690700 JM |
187 | |
188 | if args.rundir: os.chdir(args.rundir) | |
189 | ||
5abaa3dc | 190 | r = Runner(args.mal_cmd, no_pty=args.no_pty, log_file=args.log_file) |
98af2ae3 | 191 | t = TestReader(args.test_file) |
53beaa0a | 192 | |
31690700 | 193 | |
98af2ae3 | 194 | def assert_prompt(runner, prompts, timeout): |
cc021efe | 195 | # Wait for the initial prompt |
98af2ae3 | 196 | header = runner.read_to_prompt(prompts, timeout=timeout) |
7907cd90 JM |
197 | if not header == None: |
198 | if header: | |
8e4628da | 199 | print("Started with:\n%s" % header) |
7907cd90 | 200 | else: |
98af2ae3 | 201 | print("Did not one of following prompt(s): %s" % repr(prompts)) |
8e4628da | 202 | print(" Got : %s" % repr(r.buf)) |
cc021efe JM |
203 | sys.exit(1) |
204 | ||
31690700 JM |
205 | |
206 | # Wait for the initial prompt | |
98af2ae3 | 207 | assert_prompt(r, ['user> ', 'mal-user> '], args.start_timeout) |
cc021efe JM |
208 | |
209 | # Send the pre-eval code if any | |
210 | if args.pre_eval: | |
211 | sys.stdout.write("RUNNING pre-eval: %s" % args.pre_eval) | |
7907cd90 | 212 | p.write(args.pre_eval) |
cc021efe | 213 | assert_prompt(args.test_timeout) |
31690700 JM |
214 | |
215 | fail_cnt = 0 | |
98af2ae3 | 216 | soft_fail_cnt = 0 |
31690700 | 217 | |
98af2ae3 JM |
218 | while t.next(): |
219 | sys.stdout.write("TEST: %s -> [%s,%s]" % (t.form, repr(t.out), t.ret)) | |
31690700 | 220 | sys.stdout.flush() |
d2f0f672 JM |
221 | |
222 | # The repeated form is to get around an occasional OS X issue | |
223 | # where the form is repeated. | |
224 | # https://github.com/kanaka/mal/issues/30 | |
98af2ae3 JM |
225 | expected = ["%s%s%s%s" % (t.form, sep, t.out, t.ret), |
226 | "%s%s%s%s%s%s" % (t.form, sep, t.form, sep, t.out, t.ret)] | |
31690700 | 227 | |
98af2ae3 | 228 | r.writeline(t.form) |
31690700 | 229 | try: |
7907cd90 JM |
230 | res = r.read_to_prompt(['\r\nuser> ', '\nuser> ', |
231 | '\r\nmal-user> ', '\nmal-user> '], | |
232 | timeout=args.test_timeout) | |
31690700 | 233 | #print "%s,%s,%s" % (idx, repr(p.before), repr(p.after)) |
98af2ae3 | 234 | if t.ret == "*" or res in expected: |
8e4628da | 235 | print(" -> SUCCESS") |
31690700 | 236 | else: |
98af2ae3 JM |
237 | if args.soft and t.soft: |
238 | print(" -> SOFT FAIL (line %d):" % t.line_num) | |
239 | soft_fail_cnt += 1 | |
240 | else: | |
241 | print(" -> FAIL (line %d):" % t.line_num) | |
242 | fail_cnt += 1 | |
8e4628da | 243 | print(" Expected : %s" % repr(expected)) |
244 | print(" Got : %s" % repr(res)) | |
7907cd90 | 245 | except: |
10034e82 | 246 | _, exc, _ = sys.exc_info() |
b020aa3e JM |
247 | print("\nException: %s" % repr(exc)) |
248 | print("Output before exception:\n%s" % r.buf) | |
31690700 JM |
249 | sys.exit(1) |
250 | ||
98af2ae3 JM |
251 | if soft_fail_cnt > 0: |
252 | print("SOFT FAILURES: %d" % soft_fail_cnt) | |
31690700 | 253 | if fail_cnt > 0: |
8e4628da | 254 | print("FAILURES: %d" % fail_cnt) |
31690700 JM |
255 | sys.exit(2) |
256 | sys.exit(0) |