Commit | Line | Data |
---|---|---|
31690700 JM |
1 | #!/usr/bin/env python |
2 | ||
97e27599 | 3 | from __future__ import print_function |
31690700 | 4 | import os, sys, re |
7907cd90 | 5 | import argparse, time |
b3c30da9 | 6 | import signal, atexit |
31690700 | 7 | |
7907cd90 JM |
8 | from subprocess import Popen, STDOUT, PIPE |
9 | from select import select | |
31690700 | 10 | |
b3c30da9 JM |
11 | # Pseudo-TTY and terminal manipulation |
12 | import pty, array, fcntl, termios | |
13 | ||
8e4628da | 14 | IS_PY_3 = sys.version_info[0] == 3 |
15 | ||
97e27599 JM |
16 | debug_file = None |
17 | log_file = None | |
18 | ||
19 | def debug(data): | |
20 | if debug_file: | |
21 | debug_file.write(data) | |
22 | debug_file.flush() | |
23 | ||
24 | def log(data, end='\n'): | |
25 | if log_file: | |
26 | log_file.write(data + end) | |
27 | log_file.flush() | |
28 | print(data, end=end) | |
29 | sys.stdout.flush() | |
30 | ||
31690700 | 31 | # TODO: do we need to support '\n' too |
c89b2a6f | 32 | sep = "\r\n" |
31690700 JM |
33 | rundir = None |
34 | ||
35 | parser = argparse.ArgumentParser( | |
36 | description="Run a test file against a Mal implementation") | |
37 | parser.add_argument('--rundir', | |
38 | help="change to the directory before running tests") | |
39 | parser.add_argument('--start-timeout', default=10, type=int, | |
40 | help="default timeout for initial prompt") | |
41 | parser.add_argument('--test-timeout', default=20, type=int, | |
42 | help="default timeout for each individual test action") | |
cc021efe JM |
43 | parser.add_argument('--pre-eval', default=None, type=str, |
44 | help="Mal code to evaluate prior to running the test") | |
ab01be18 JM |
45 | parser.add_argument('--no-pty', action='store_true', |
46 | help="Use direct pipes instead of pseudo-tty") | |
5abaa3dc | 47 | parser.add_argument('--log-file', type=str, |
97e27599 JM |
48 | help="Write messages to the named file in addition the screen") |
49 | parser.add_argument('--debug-file', type=str, | |
5abaa3dc | 50 | help="Write all test interaction the named file") |
f3ea3be3 | 51 | parser.add_argument('--hard', action='store_true', |
108d07a2 | 52 | help="Turn soft tests (soft, deferrable, optional) into hard failures") |
31690700 | 53 | |
a1eb30fc JM |
54 | # Control whether deferrable and optional tests are executed |
55 | parser.add_argument('--deferrable', dest='deferrable', action='store_true', | |
56 | help="Enable deferrable tests that follow a ';>>> deferrable=True'") | |
57 | parser.add_argument('--no-deferrable', dest='deferrable', action='store_false', | |
58 | help="Disable deferrable tests that follow a ';>>> deferrable=True'") | |
59 | parser.set_defaults(deferrable=True) | |
46e25689 JM |
60 | parser.add_argument('--optional', dest='optional', action='store_true', |
61 | help="Enable optional tests that follow a ';>>> optional=True'") | |
62 | parser.add_argument('--no-optional', dest='optional', action='store_false', | |
63 | help="Disable optional tests that follow a ';>>> optional=True'") | |
64 | parser.set_defaults(optional=True) | |
65 | ||
18616b10 | 66 | parser.add_argument('test_file', type=str, |
31690700 JM |
67 | help="a test file formatted as with mal test data") |
68 | parser.add_argument('mal_cmd', nargs="*", | |
69 | help="Mal implementation command line. Use '--' to " | |
70 | "specify a Mal command line with dashed options.") | |
9d4138ed AMP |
71 | parser.add_argument('--crlf', dest='crlf', action='store_true', |
72 | help="Write \\r\\n instead of \\n to the input") | |
31690700 | 73 | |
ea125102 JM |
74 | import errno |
75 | def list_fds(): | |
76 | """List process currently open FDs and their target """ | |
77 | if sys.platform != 'linux2': | |
78 | raise NotImplementedError('Unsupported platform: %s' % sys.platform) | |
79 | ||
80 | ret = {} | |
81 | base = '/proc/self/fd' | |
82 | for num in os.listdir(base): | |
83 | path = None | |
84 | try: | |
85 | path = os.readlink(os.path.join(base, num)) | |
86 | except OSError as err: | |
87 | # Last FD is always the "listdir" one (which may be closed) | |
88 | if err.errno != errno.ENOENT: | |
89 | raise | |
90 | ret[int(num)] = path | |
91 | ||
92 | return ret | |
93 | ||
7907cd90 | 94 | class Runner(): |
9d4138ed | 95 | def __init__(self, args, no_pty=False, line_break="\n"): |
612bfe4a | 96 | #print "args: %s" % repr(args) |
ab01be18 | 97 | self.no_pty = no_pty |
612bfe4a JM |
98 | |
99 | # Cleanup child process on exit | |
100 | atexit.register(self.cleanup) | |
101 | ||
8caf6211 JM |
102 | self.p = None |
103 | env = os.environ | |
104 | env['TERM'] = 'dumb' | |
92dcc815 | 105 | env['INPUTRC'] = '/dev/null' |
82acd3de | 106 | env['PERL_RL'] = 'false' |
77489b59 | 107 | #print("FDS before: %s" % list_fds()) |
ab01be18 | 108 | if no_pty: |
612bfe4a JM |
109 | self.p = Popen(args, bufsize=0, |
110 | stdin=PIPE, stdout=PIPE, stderr=STDOUT, | |
8caf6211 | 111 | preexec_fn=os.setsid, |
77489b59 JM |
112 | env=env) |
113 | #env=env, close_fds=True) | |
7907cd90 JM |
114 | self.stdin = self.p.stdin |
115 | self.stdout = self.p.stdout | |
116 | else: | |
117 | # provide tty to get 'interactive' readline to work | |
118 | master, slave = pty.openpty() | |
b3c30da9 JM |
119 | |
120 | # Set terminal size large so that readline will not send | |
121 | # ANSI/VT escape codes when the lines are long. | |
122 | buf = array.array('h', [100, 200, 0, 0]) | |
123 | fcntl.ioctl(master, termios.TIOCSWINSZ, buf, True) | |
124 | ||
612bfe4a JM |
125 | self.p = Popen(args, bufsize=0, |
126 | stdin=slave, stdout=slave, stderr=STDOUT, | |
8caf6211 | 127 | preexec_fn=os.setsid, |
77489b59 JM |
128 | env=env) |
129 | #env=env, close_fds=True) | |
b020aa3e JM |
130 | # Now close slave so that we will get an exception from |
131 | # read when the child exits early | |
132 | # http://stackoverflow.com/questions/11165521 | |
133 | os.close(slave) | |
7907cd90 JM |
134 | self.stdin = os.fdopen(master, 'r+b', 0) |
135 | self.stdout = self.stdin | |
136 | ||
ea125102 JM |
137 | print("FDS after: %s" % list_fds()) |
138 | ||
7907cd90 JM |
139 | #print "started" |
140 | self.buf = "" | |
141 | self.last_prompt = "" | |
142 | ||
9d4138ed AMP |
143 | self.line_break = line_break |
144 | ||
7907cd90 JM |
145 | def read_to_prompt(self, prompts, timeout): |
146 | end_time = time.time() + timeout | |
147 | while time.time() < end_time: | |
96deb6a9 JM |
148 | [outs,_,_] = select([self.stdout], [], [], 1) |
149 | if self.stdout in outs: | |
150 | new_data = self.stdout.read(1) | |
8e4628da | 151 | new_data = new_data.decode("utf-8") if IS_PY_3 else new_data |
97e27599 JM |
152 | #print("new_data: '%s'" % new_data) |
153 | debug(new_data) | |
e17aef04 | 154 | # Perform newline cleanup |
ab01be18 | 155 | if self.no_pty: |
9a383535 JM |
156 | self.buf += new_data.replace("\n", "\r\n") |
157 | else: | |
158 | self.buf += new_data | |
0e508fa5 | 159 | self.buf = self.buf.replace("\r\r", "\r") |
9a66ffcd JM |
160 | # Remove ANSI codes generally |
161 | #ansi_escape = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]') | |
162 | # Remove rustyline ANSI CSI codes: | |
163 | # - [6C - CR + cursor forward | |
164 | # - [6K - CR + erase in line | |
165 | ansi_escape = re.compile(r'\r\x1B\[[0-9]*[CK]') | |
166 | self.buf = ansi_escape.sub('', self.buf) | |
7907cd90 JM |
167 | for prompt in prompts: |
168 | regexp = re.compile(prompt) | |
169 | match = regexp.search(self.buf) | |
170 | if match: | |
171 | end = match.end() | |
96c09dcd | 172 | buf = self.buf[0:match.start()] |
7907cd90 JM |
173 | self.buf = self.buf[end:] |
174 | self.last_prompt = prompt | |
18616b10 | 175 | return buf.replace("^M", "\r") |
7907cd90 JM |
176 | return None |
177 | ||
96deb6a9 | 178 | def writeline(self, str): |
8e4628da | 179 | def _to_bytes(s): |
180 | return bytes(s, "utf-8") if IS_PY_3 else s | |
181 | ||
9d4138ed | 182 | self.stdin.write(_to_bytes(str.replace('\r', '\x16\r') + self.line_break)) |
7907cd90 | 183 | |
f6c83b2b | 184 | def cleanup(self): |
612bfe4a | 185 | #print "cleaning up" |
f6c83b2b | 186 | if self.p: |
10034e82 JM |
187 | try: |
188 | os.killpg(self.p.pid, signal.SIGTERM) | |
189 | except OSError: | |
190 | pass | |
f6c83b2b JM |
191 | self.p = None |
192 | ||
98af2ae3 | 193 | class TestReader: |
46e25689 | 194 | def __init__(self, test_file): |
98af2ae3 | 195 | self.line_num = 0 |
18616b10 JM |
196 | f = open(test_file, newline='') if IS_PY_3 else open(test_file) |
197 | self.data = f.read().split('\n') | |
98af2ae3 | 198 | self.soft = False |
a1eb30fc | 199 | self.deferrable = False |
46e25689 | 200 | self.optional = False |
98af2ae3 JM |
201 | |
202 | def next(self): | |
46e25689 | 203 | self.msg = None |
98af2ae3 JM |
204 | self.form = None |
205 | self.out = "" | |
206 | self.ret = None | |
207 | ||
208 | while self.data: | |
209 | self.line_num += 1 | |
210 | line = self.data.pop(0) | |
211 | if re.match(r"^\s*$", line): # blank line | |
212 | continue | |
213 | elif line[0:3] == ";;;": # ignore comment | |
214 | continue | |
215 | elif line[0:2] == ";;": # output comment | |
46e25689 JM |
216 | self.msg = line[3:] |
217 | return True | |
98af2ae3 JM |
218 | elif line[0:5] == ";>>> ": # settings/commands |
219 | settings = {} | |
220 | exec(line[5:], {}, settings) | |
46e25689 JM |
221 | if 'soft' in settings: |
222 | self.soft = settings['soft'] | |
a1eb30fc JM |
223 | if 'deferrable' in settings and settings['deferrable']: |
224 | self.deferrable = "\nSkipping deferrable and optional tests" | |
46e25689 JM |
225 | return True |
226 | if 'optional' in settings and settings['optional']: | |
227 | self.optional = "\nSkipping optional tests" | |
228 | return True | |
98af2ae3 JM |
229 | continue |
230 | elif line[0:1] == ";": # unexpected comment | |
18f0ec21 | 231 | raise Exception("Test data error at line %d:\n%s" % (self.line_num, line)) |
98af2ae3 JM |
232 | self.form = line # the line is a form to send |
233 | ||
234 | # Now find the output and return value | |
235 | while self.data: | |
236 | line = self.data[0] | |
237 | if line[0:3] == ";=>": | |
8d78bc26 | 238 | self.ret = line[3:] |
98af2ae3 JM |
239 | self.line_num += 1 |
240 | self.data.pop(0) | |
241 | break | |
f6f5d4f2 | 242 | elif line[0:2] == ";/": |
98af2ae3 JM |
243 | self.out = self.out + line[2:] + sep |
244 | self.line_num += 1 | |
245 | self.data.pop(0) | |
246 | else: | |
f6f5d4f2 | 247 | self.ret = "" |
98af2ae3 | 248 | break |
f6f5d4f2 | 249 | if self.ret != None: break |
98af2ae3 | 250 | |
f6f5d4f2 JM |
251 | if self.out[-2:] == sep and not self.ret: |
252 | # If there is no return value, output should not end in | |
253 | # separator | |
254 | self.out = self.out[0:-2] | |
98af2ae3 JM |
255 | return self.form |
256 | ||
31690700 | 257 | args = parser.parse_args(sys.argv[1:]) |
406761e7 JM |
258 | # Workaround argparse issue with two '--' on command line |
259 | if sys.argv.count('--') > 0: | |
260 | args.mal_cmd = sys.argv[sys.argv.index('--')+1:] | |
31690700 JM |
261 | |
262 | if args.rundir: os.chdir(args.rundir) | |
263 | ||
97e27599 JM |
264 | if args.log_file: log_file = open(args.log_file, "a") |
265 | if args.debug_file: debug_file = open(args.debug_file, "a") | |
266 | ||
9d4138ed | 267 | r = Runner(args.mal_cmd, no_pty=args.no_pty, line_break="\r\n" if args.crlf else "\n") |
98af2ae3 | 268 | t = TestReader(args.test_file) |
53beaa0a | 269 | |
31690700 | 270 | |
98af2ae3 | 271 | def assert_prompt(runner, prompts, timeout): |
cc021efe | 272 | # Wait for the initial prompt |
98af2ae3 | 273 | header = runner.read_to_prompt(prompts, timeout=timeout) |
7907cd90 JM |
274 | if not header == None: |
275 | if header: | |
97e27599 | 276 | log("Started with:\n%s" % header) |
7907cd90 | 277 | else: |
0821b2d4 | 278 | log("Did not receive one of following prompt(s): %s" % repr(prompts)) |
97e27599 | 279 | log(" Got : %s" % repr(r.buf)) |
cc021efe JM |
280 | sys.exit(1) |
281 | ||
31690700 JM |
282 | |
283 | # Wait for the initial prompt | |
16d5b0c3 | 284 | try: |
96c09dcd | 285 | assert_prompt(r, ['[^\s()<>]+> '], args.start_timeout) |
16d5b0c3 JM |
286 | except: |
287 | _, exc, _ = sys.exc_info() | |
288 | log("\nException: %s" % repr(exc)) | |
289 | log("Output before exception:\n%s" % r.buf) | |
290 | sys.exit(1) | |
cc021efe JM |
291 | |
292 | # Send the pre-eval code if any | |
293 | if args.pre_eval: | |
294 | sys.stdout.write("RUNNING pre-eval: %s" % args.pre_eval) | |
c2b6285e NB |
295 | r.writeline(args.pre_eval) |
296 | assert_prompt(r, ['[^\s()<>]+> '], args.test_timeout) | |
31690700 | 297 | |
97e27599 JM |
298 | test_cnt = 0 |
299 | pass_cnt = 0 | |
31690700 | 300 | fail_cnt = 0 |
98af2ae3 | 301 | soft_fail_cnt = 0 |
00724049 | 302 | failures = [] |
31690700 | 303 | |
98af2ae3 | 304 | while t.next(): |
a1eb30fc JM |
305 | if args.deferrable == False and t.deferrable: |
306 | log(t.deferrable) | |
46e25689 JM |
307 | break |
308 | ||
309 | if args.optional == False and t.optional: | |
310 | log(t.optional) | |
311 | break | |
312 | ||
313 | if t.msg != None: | |
314 | log(t.msg) | |
315 | continue | |
316 | ||
317 | if t.form == None: continue | |
318 | ||
18616b10 | 319 | log("TEST: %s -> [%s,%s]" % (repr(t.form), repr(t.out), t.ret), end='') |
d2f0f672 JM |
320 | |
321 | # The repeated form is to get around an occasional OS X issue | |
322 | # where the form is repeated. | |
323 | # https://github.com/kanaka/mal/issues/30 | |
f6f5d4f2 JM |
324 | expects = ["%s%s%s%s" % (re.escape(t.form), sep, |
325 | t.out, re.escape(t.ret)), | |
326 | "%s%s%s%s%s%s" % (re.escape(t.form), sep, | |
327 | re.escape(t.form), sep, | |
328 | t.out, re.escape(t.ret))] | |
31690700 | 329 | |
98af2ae3 | 330 | r.writeline(t.form) |
31690700 | 331 | try: |
97e27599 | 332 | test_cnt += 1 |
96c09dcd | 333 | res = r.read_to_prompt(['\r\n[^\s()<>]+> ', '\n[^\s()<>]+> '], |
7907cd90 | 334 | timeout=args.test_timeout) |
31690700 | 335 | #print "%s,%s,%s" % (idx, repr(p.before), repr(p.after)) |
f6f5d4f2 JM |
336 | if (t.ret == "" and t.out == ""): |
337 | log(" -> SUCCESS (result ignored)") | |
338 | pass_cnt += 1 | |
339 | elif (re.search(expects[0], res, re.S) or | |
340 | re.search(expects[1], res, re.S)): | |
97e27599 JM |
341 | log(" -> SUCCESS") |
342 | pass_cnt += 1 | |
31690700 | 343 | else: |
f3ea3be3 | 344 | if t.soft and not args.hard: |
97e27599 | 345 | log(" -> SOFT FAIL (line %d):" % t.line_num) |
98af2ae3 | 346 | soft_fail_cnt += 1 |
00724049 | 347 | fail_type = "SOFT " |
98af2ae3 | 348 | else: |
97e27599 | 349 | log(" -> FAIL (line %d):" % t.line_num) |
98af2ae3 | 350 | fail_cnt += 1 |
00724049 | 351 | fail_type = "" |
f6f5d4f2 | 352 | log(" Expected : %s" % repr(expects[0])) |
97e27599 | 353 | log(" Got : %s" % repr(res)) |
00724049 DM |
354 | failed_test = """%sFAILED TEST (line %d): %s -> [%s,%s]: |
355 | Expected : %s | |
f6f5d4f2 JM |
356 | Got : %s""" % (fail_type, t.line_num, t.form, repr(t.out), |
357 | t.ret, repr(expects[0]), repr(res)) | |
00724049 | 358 | failures.append(failed_test) |
7907cd90 | 359 | except: |
10034e82 | 360 | _, exc, _ = sys.exc_info() |
97e27599 JM |
361 | log("\nException: %s" % repr(exc)) |
362 | log("Output before exception:\n%s" % r.buf) | |
31690700 JM |
363 | sys.exit(1) |
364 | ||
00724049 DM |
365 | if len(failures) > 0: |
366 | log("\nFAILURES:") | |
367 | for f in failures: | |
368 | log(f) | |
369 | ||
370 | results = """ | |
371 | TEST RESULTS (for %s): | |
97e27599 JM |
372 | %3d: soft failing tests |
373 | %3d: failing tests | |
374 | %3d: passing tests | |
375 | %3d: total tests | |
18616b10 | 376 | """ % (args.test_file, soft_fail_cnt, fail_cnt, |
97e27599 JM |
377 | pass_cnt, test_cnt) |
378 | log(results) | |
379 | ||
380 | debug("\n") # add some separate to debug log | |
381 | ||
31690700 | 382 | if fail_cnt > 0: |
97e27599 | 383 | sys.exit(1) |
31690700 | 384 | sys.exit(0) |