Commit | Line | Data |
---|---|---|
31690700 JM |
1 | #!/usr/bin/env python |
2 | ||
3 | import os, sys, re | |
7907cd90 | 4 | import argparse, time |
31690700 | 5 | |
7907cd90 JM |
6 | import pty |
7 | from subprocess import Popen, STDOUT, PIPE | |
8 | from select import select | |
31690700 JM |
9 | |
10 | # TODO: do we need to support '\n' too | |
11 | sep = "\r\n" | |
7907cd90 | 12 | #sep = "\n" |
31690700 JM |
13 | rundir = None |
14 | ||
15 | parser = argparse.ArgumentParser( | |
16 | description="Run a test file against a Mal implementation") | |
17 | parser.add_argument('--rundir', | |
18 | help="change to the directory before running tests") | |
19 | parser.add_argument('--start-timeout', default=10, type=int, | |
20 | help="default timeout for initial prompt") | |
21 | parser.add_argument('--test-timeout', default=20, type=int, | |
22 | help="default timeout for each individual test action") | |
cc021efe JM |
23 | parser.add_argument('--pre-eval', default=None, type=str, |
24 | help="Mal code to evaluate prior to running the test") | |
53beaa0a JM |
25 | parser.add_argument('--redirect', action='store_true', |
26 | help="Run implementation in bash and redirect output to /dev/null") | |
31690700 JM |
27 | |
28 | parser.add_argument('test_file', type=argparse.FileType('r'), | |
29 | help="a test file formatted as with mal test data") | |
30 | parser.add_argument('mal_cmd', nargs="*", | |
31 | help="Mal implementation command line. Use '--' to " | |
32 | "specify a Mal command line with dashed options.") | |
33 | ||
7907cd90 JM |
34 | class Runner(): |
35 | def __init__(self, args, redirect=False): | |
36 | print "args: %s" % repr(args) | |
37 | if redirect: | |
38 | print "using redirect" | |
39 | self.p = Popen(args, bufsize=0, stdin=PIPE, stdout=PIPE, stderr=STDOUT) | |
40 | self.stdin = self.p.stdin | |
41 | self.stdout = self.p.stdout | |
42 | else: | |
43 | # provide tty to get 'interactive' readline to work | |
44 | master, slave = pty.openpty() | |
45 | self.p = Popen(args, bufsize=0, stdin=slave, stdout=slave, stderr=STDOUT) | |
46 | self.stdin = os.fdopen(master, 'r+b', 0) | |
47 | self.stdout = self.stdin | |
48 | ||
49 | #print "started" | |
50 | self.buf = "" | |
51 | self.last_prompt = "" | |
52 | ||
53 | def read_to_prompt(self, prompts, timeout): | |
54 | end_time = time.time() + timeout | |
55 | while time.time() < end_time: | |
56 | [outs,_,_] = select([self.stdin], [], [], 1) | |
57 | if self.stdin in outs: | |
58 | new_data = self.stdin.read(1) | |
59 | #print "new_data: '%s'" % new_data | |
60 | self.buf += new_data | |
61 | for prompt in prompts: | |
62 | regexp = re.compile(prompt) | |
63 | match = regexp.search(self.buf) | |
64 | if match: | |
65 | end = match.end() | |
66 | buf = self.buf[0:end-len(prompt)] | |
67 | self.buf = self.buf[end:] | |
68 | self.last_prompt = prompt | |
69 | return buf | |
70 | return None | |
71 | ||
72 | def write(self, str): | |
73 | self.stdout.write(str) | |
74 | ||
31690700 JM |
75 | args = parser.parse_args(sys.argv[1:]) |
76 | test_data = args.test_file.read().split('\n') | |
77 | ||
78 | if args.rundir: os.chdir(args.rundir) | |
79 | ||
7907cd90 | 80 | r = Runner(args.mal_cmd, redirect=args.redirect) |
53beaa0a | 81 | |
31690700 JM |
82 | |
83 | test_idx = 0 | |
84 | def read_test(data): | |
85 | global test_idx | |
86 | form, output, ret = None, "", None | |
87 | while data: | |
88 | test_idx += 1 | |
89 | line = data.pop(0) | |
90 | if re.match(r"^\s*$", line): # blank line | |
91 | continue | |
92 | elif line[0:3] == ";;;": # ignore comment | |
93 | continue | |
94 | elif line[0:2] == ";;": # output comment | |
95 | print line[3:] | |
96 | continue | |
97 | elif line[0:2] == ";": # unexpected comment | |
98 | print "Test data error at line %d:\n%s" % (test_idx, line) | |
99 | return None, None, None, test_idx | |
100 | form = line # the line is a form to send | |
101 | ||
102 | # Now find the output and return value | |
103 | while data: | |
104 | line = data[0] | |
105 | if line[0:3] == ";=>": | |
106 | ret = line[3:].replace('\\r', '\r').replace('\\n', '\n') | |
107 | test_idx += 1 | |
108 | data.pop(0) | |
109 | break | |
110 | elif line[0:2] == "; ": | |
111 | output = output + line[2:] + sep | |
112 | test_idx += 1 | |
113 | data.pop(0) | |
114 | else: | |
115 | ret = "*" | |
116 | break | |
117 | if ret: break | |
118 | ||
119 | return form, output, ret, test_idx | |
120 | ||
cc021efe JM |
121 | def assert_prompt(timeout): |
122 | # Wait for the initial prompt | |
7907cd90 JM |
123 | header = r.read_to_prompt(['user> ', 'mal-user> '], timeout=timeout) |
124 | if not header == None: | |
125 | if header: | |
126 | print "Started with:\n%s" % header | |
127 | else: | |
cc021efe | 128 | print "Did not get 'user> ' or 'mal-user> ' prompt" |
7907cd90 | 129 | print " Got : %s" % repr(r.buf) |
cc021efe JM |
130 | sys.exit(1) |
131 | ||
31690700 JM |
132 | |
133 | # Wait for the initial prompt | |
cc021efe JM |
134 | assert_prompt(args.start_timeout) |
135 | ||
136 | # Send the pre-eval code if any | |
137 | if args.pre_eval: | |
138 | sys.stdout.write("RUNNING pre-eval: %s" % args.pre_eval) | |
7907cd90 | 139 | p.write(args.pre_eval) |
cc021efe | 140 | assert_prompt(args.test_timeout) |
31690700 JM |
141 | |
142 | fail_cnt = 0 | |
143 | ||
144 | while test_data: | |
145 | form, out, ret, line_num = read_test(test_data) | |
146 | if form == None: | |
147 | break | |
148 | sys.stdout.write("TEST: %s -> [%s,%s]" % (form, repr(out), repr(ret))) | |
149 | sys.stdout.flush() | |
150 | expected = "%s%s%s%s" % (form, sep, out, ret) | |
151 | ||
7907cd90 | 152 | r.write(form + "\n") |
31690700 | 153 | try: |
7907cd90 JM |
154 | res = r.read_to_prompt(['\r\nuser> ', '\nuser> ', |
155 | '\r\nmal-user> ', '\nmal-user> '], | |
156 | timeout=args.test_timeout) | |
31690700 | 157 | #print "%s,%s,%s" % (idx, repr(p.before), repr(p.after)) |
7907cd90 | 158 | if ret == "*" or res == expected: |
31690700 JM |
159 | print " -> SUCCESS" |
160 | else: | |
161 | print " -> FAIL (line %d):" % line_num | |
162 | print " Expected : %s" % repr(expected) | |
7907cd90 | 163 | print " Got : %s" % repr(res) |
31690700 | 164 | fail_cnt += 1 |
7907cd90 JM |
165 | except: |
166 | print "Got Exception" | |
31690700 JM |
167 | sys.exit(1) |
168 | ||
169 | if fail_cnt > 0: | |
170 | print "FAILURES: %d" % fail_cnt | |
171 | sys.exit(2) | |
172 | sys.exit(0) |