Commit | Line | Data |
---|---|---|
31690700 JM |
1 | #!/usr/bin/env python |
2 | ||
3 | import os, sys, re | |
7907cd90 | 4 | import argparse, time |
b3c30da9 | 5 | import signal, atexit |
31690700 | 6 | |
7907cd90 JM |
7 | from subprocess import Popen, STDOUT, PIPE |
8 | from select import select | |
31690700 | 9 | |
b3c30da9 JM |
10 | # Pseudo-TTY and terminal manipulation |
11 | import pty, array, fcntl, termios | |
12 | ||
8e4628da | 13 | IS_PY_3 = sys.version_info[0] == 3 |
14 | ||
31690700 JM |
15 | # TODO: do we need to support '\n' too |
16 | sep = "\r\n" | |
7907cd90 | 17 | #sep = "\n" |
31690700 JM |
18 | rundir = None |
19 | ||
20 | parser = argparse.ArgumentParser( | |
21 | description="Run a test file against a Mal implementation") | |
22 | parser.add_argument('--rundir', | |
23 | help="change to the directory before running tests") | |
24 | parser.add_argument('--start-timeout', default=10, type=int, | |
25 | help="default timeout for initial prompt") | |
26 | parser.add_argument('--test-timeout', default=20, type=int, | |
27 | help="default timeout for each individual test action") | |
cc021efe JM |
28 | parser.add_argument('--pre-eval', default=None, type=str, |
29 | help="Mal code to evaluate prior to running the test") | |
ab01be18 JM |
30 | parser.add_argument('--no-pty', action='store_true', |
31 | help="Use direct pipes instead of pseudo-tty") | |
5abaa3dc JM |
32 | parser.add_argument('--log-file', type=str, |
33 | help="Write all test interaction the named file") | |
31690700 JM |
34 | |
35 | parser.add_argument('test_file', type=argparse.FileType('r'), | |
36 | help="a test file formatted as with mal test data") | |
37 | parser.add_argument('mal_cmd', nargs="*", | |
38 | help="Mal implementation command line. Use '--' to " | |
39 | "specify a Mal command line with dashed options.") | |
40 | ||
7907cd90 | 41 | class Runner(): |
5abaa3dc | 42 | def __init__(self, args, no_pty=False, log_file=None): |
612bfe4a | 43 | #print "args: %s" % repr(args) |
ab01be18 | 44 | self.no_pty = no_pty |
612bfe4a | 45 | |
5abaa3dc JM |
46 | if log_file: self.logf = open(log_file, "a") |
47 | else: self.logf = None | |
48 | ||
612bfe4a JM |
49 | # Cleanup child process on exit |
50 | atexit.register(self.cleanup) | |
51 | ||
8caf6211 JM |
52 | self.p = None |
53 | env = os.environ | |
54 | env['TERM'] = 'dumb' | |
92dcc815 | 55 | env['INPUTRC'] = '/dev/null' |
82acd3de | 56 | env['PERL_RL'] = 'false' |
ab01be18 | 57 | if no_pty: |
612bfe4a JM |
58 | self.p = Popen(args, bufsize=0, |
59 | stdin=PIPE, stdout=PIPE, stderr=STDOUT, | |
8caf6211 JM |
60 | preexec_fn=os.setsid, |
61 | env=env) | |
7907cd90 JM |
62 | self.stdin = self.p.stdin |
63 | self.stdout = self.p.stdout | |
64 | else: | |
65 | # provide tty to get 'interactive' readline to work | |
66 | master, slave = pty.openpty() | |
b3c30da9 JM |
67 | |
68 | # Set terminal size large so that readline will not send | |
69 | # ANSI/VT escape codes when the lines are long. | |
70 | buf = array.array('h', [100, 200, 0, 0]) | |
71 | fcntl.ioctl(master, termios.TIOCSWINSZ, buf, True) | |
72 | ||
612bfe4a JM |
73 | self.p = Popen(args, bufsize=0, |
74 | stdin=slave, stdout=slave, stderr=STDOUT, | |
8caf6211 JM |
75 | preexec_fn=os.setsid, |
76 | env=env) | |
b020aa3e JM |
77 | # Now close slave so that we will get an exception from |
78 | # read when the child exits early | |
79 | # http://stackoverflow.com/questions/11165521 | |
80 | os.close(slave) | |
7907cd90 JM |
81 | self.stdin = os.fdopen(master, 'r+b', 0) |
82 | self.stdout = self.stdin | |
83 | ||
84 | #print "started" | |
85 | self.buf = "" | |
86 | self.last_prompt = "" | |
87 | ||
88 | def read_to_prompt(self, prompts, timeout): | |
89 | end_time = time.time() + timeout | |
90 | while time.time() < end_time: | |
96deb6a9 JM |
91 | [outs,_,_] = select([self.stdout], [], [], 1) |
92 | if self.stdout in outs: | |
93 | new_data = self.stdout.read(1) | |
8e4628da | 94 | new_data = new_data.decode("utf-8") if IS_PY_3 else new_data |
b020aa3e | 95 | #print "new_data: '%s'" % new_data |
5abaa3dc | 96 | self.log(new_data) |
ab01be18 | 97 | if self.no_pty: |
9a383535 JM |
98 | self.buf += new_data.replace("\n", "\r\n") |
99 | else: | |
100 | self.buf += new_data | |
7907cd90 JM |
101 | for prompt in prompts: |
102 | regexp = re.compile(prompt) | |
103 | match = regexp.search(self.buf) | |
104 | if match: | |
105 | end = match.end() | |
106 | buf = self.buf[0:end-len(prompt)] | |
107 | self.buf = self.buf[end:] | |
108 | self.last_prompt = prompt | |
109 | return buf | |
110 | return None | |
111 | ||
5abaa3dc JM |
112 | def log(self, data): |
113 | if self.logf: | |
114 | self.logf.write(data) | |
115 | self.logf.flush() | |
116 | ||
117 | ||
96deb6a9 | 118 | def writeline(self, str): |
8e4628da | 119 | def _to_bytes(s): |
120 | return bytes(s, "utf-8") if IS_PY_3 else s | |
121 | ||
122 | self.stdin.write(_to_bytes(str + "\n")) | |
7907cd90 | 123 | |
f6c83b2b | 124 | def cleanup(self): |
612bfe4a | 125 | #print "cleaning up" |
f6c83b2b | 126 | if self.p: |
10034e82 JM |
127 | try: |
128 | os.killpg(self.p.pid, signal.SIGTERM) | |
129 | except OSError: | |
130 | pass | |
f6c83b2b JM |
131 | self.p = None |
132 | ||
133 | ||
31690700 JM |
134 | args = parser.parse_args(sys.argv[1:]) |
135 | test_data = args.test_file.read().split('\n') | |
136 | ||
137 | if args.rundir: os.chdir(args.rundir) | |
138 | ||
5abaa3dc | 139 | r = Runner(args.mal_cmd, no_pty=args.no_pty, log_file=args.log_file) |
53beaa0a | 140 | |
31690700 JM |
141 | |
142 | test_idx = 0 | |
143 | def read_test(data): | |
144 | global test_idx | |
145 | form, output, ret = None, "", None | |
146 | while data: | |
147 | test_idx += 1 | |
148 | line = data.pop(0) | |
149 | if re.match(r"^\s*$", line): # blank line | |
150 | continue | |
151 | elif line[0:3] == ";;;": # ignore comment | |
152 | continue | |
153 | elif line[0:2] == ";;": # output comment | |
8e4628da | 154 | print(line[3:]) |
31690700 JM |
155 | continue |
156 | elif line[0:2] == ";": # unexpected comment | |
8e4628da | 157 | print("Test data error at line %d:\n%s" % (test_idx, line)) |
31690700 JM |
158 | return None, None, None, test_idx |
159 | form = line # the line is a form to send | |
160 | ||
161 | # Now find the output and return value | |
162 | while data: | |
163 | line = data[0] | |
164 | if line[0:3] == ";=>": | |
165 | ret = line[3:].replace('\\r', '\r').replace('\\n', '\n') | |
166 | test_idx += 1 | |
167 | data.pop(0) | |
168 | break | |
169 | elif line[0:2] == "; ": | |
170 | output = output + line[2:] + sep | |
171 | test_idx += 1 | |
172 | data.pop(0) | |
173 | else: | |
174 | ret = "*" | |
175 | break | |
176 | if ret: break | |
177 | ||
178 | return form, output, ret, test_idx | |
179 | ||
cc021efe JM |
180 | def assert_prompt(timeout): |
181 | # Wait for the initial prompt | |
7907cd90 JM |
182 | header = r.read_to_prompt(['user> ', 'mal-user> '], timeout=timeout) |
183 | if not header == None: | |
184 | if header: | |
8e4628da | 185 | print("Started with:\n%s" % header) |
7907cd90 | 186 | else: |
8e4628da | 187 | print("Did not get 'user> ' or 'mal-user> ' prompt") |
188 | print(" Got : %s" % repr(r.buf)) | |
cc021efe JM |
189 | sys.exit(1) |
190 | ||
31690700 JM |
191 | |
192 | # Wait for the initial prompt | |
cc021efe JM |
193 | assert_prompt(args.start_timeout) |
194 | ||
195 | # Send the pre-eval code if any | |
196 | if args.pre_eval: | |
197 | sys.stdout.write("RUNNING pre-eval: %s" % args.pre_eval) | |
7907cd90 | 198 | p.write(args.pre_eval) |
cc021efe | 199 | assert_prompt(args.test_timeout) |
31690700 JM |
200 | |
201 | fail_cnt = 0 | |
202 | ||
203 | while test_data: | |
204 | form, out, ret, line_num = read_test(test_data) | |
205 | if form == None: | |
206 | break | |
207 | sys.stdout.write("TEST: %s -> [%s,%s]" % (form, repr(out), repr(ret))) | |
208 | sys.stdout.flush() | |
d2f0f672 JM |
209 | |
210 | # The repeated form is to get around an occasional OS X issue | |
211 | # where the form is repeated. | |
212 | # https://github.com/kanaka/mal/issues/30 | |
213 | expected = ["%s%s%s%s" % (form, sep, out, ret), | |
214 | "%s%s%s%s%s%s" % (form, sep, form, sep, out, ret)] | |
31690700 | 215 | |
96deb6a9 | 216 | r.writeline(form) |
31690700 | 217 | try: |
7907cd90 JM |
218 | res = r.read_to_prompt(['\r\nuser> ', '\nuser> ', |
219 | '\r\nmal-user> ', '\nmal-user> '], | |
220 | timeout=args.test_timeout) | |
31690700 | 221 | #print "%s,%s,%s" % (idx, repr(p.before), repr(p.after)) |
d2f0f672 | 222 | if ret == "*" or res in expected: |
8e4628da | 223 | print(" -> SUCCESS") |
31690700 | 224 | else: |
8e4628da | 225 | print(" -> FAIL (line %d):" % line_num) |
226 | print(" Expected : %s" % repr(expected)) | |
227 | print(" Got : %s" % repr(res)) | |
31690700 | 228 | fail_cnt += 1 |
7907cd90 | 229 | except: |
10034e82 | 230 | _, exc, _ = sys.exc_info() |
b020aa3e JM |
231 | print("\nException: %s" % repr(exc)) |
232 | print("Output before exception:\n%s" % r.buf) | |
31690700 JM |
233 | sys.exit(1) |
234 | ||
235 | if fail_cnt > 0: | |
8e4628da | 236 | print("FAILURES: %d" % fail_cnt) |
31690700 JM |
237 | sys.exit(2) |
238 | sys.exit(0) |