Commit | Line | Data |
---|---|---|
31690700 JM |
1 | #!/usr/bin/env python |
2 | ||
3 | import os, sys, re | |
4 | import argparse | |
5 | ||
6 | # http://pexpect.sourceforge.net/pexpect.html | |
7 | from pexpect import spawn, EOF, TIMEOUT | |
8 | ||
9 | # TODO: do we need to support '\n' too | |
10 | sep = "\r\n" | |
11 | rundir = None | |
12 | ||
13 | parser = argparse.ArgumentParser( | |
14 | description="Run a test file against a Mal implementation") | |
15 | parser.add_argument('--rundir', | |
16 | help="change to the directory before running tests") | |
17 | parser.add_argument('--start-timeout', default=10, type=int, | |
18 | help="default timeout for initial prompt") | |
19 | parser.add_argument('--test-timeout', default=20, type=int, | |
20 | help="default timeout for each individual test action") | |
cc021efe JM |
21 | parser.add_argument('--pre-eval', default=None, type=str, |
22 | help="Mal code to evaluate prior to running the test") | |
53beaa0a JM |
23 | parser.add_argument('--redirect', action='store_true', |
24 | help="Run implementation in bash and redirect output to /dev/null") | |
31690700 JM |
25 | |
26 | parser.add_argument('test_file', type=argparse.FileType('r'), | |
27 | help="a test file formatted as with mal test data") | |
28 | parser.add_argument('mal_cmd', nargs="*", | |
29 | help="Mal implementation command line. Use '--' to " | |
30 | "specify a Mal command line with dashed options.") | |
31 | ||
32 | args = parser.parse_args(sys.argv[1:]) | |
33 | test_data = args.test_file.read().split('\n') | |
34 | ||
35 | if args.rundir: os.chdir(args.rundir) | |
36 | ||
53beaa0a JM |
37 | if args.redirect: |
38 | # Redirect to try and force raw mode (no ASCII codes) | |
39 | p = spawn('/bin/bash -c "' + " ".join(args.mal_cmd) + ' |tee /dev/null"') | |
40 | else: | |
41 | p = spawn(args.mal_cmd[0], args.mal_cmd[1:]) | |
42 | ||
31690700 JM |
43 | |
44 | test_idx = 0 | |
45 | def read_test(data): | |
46 | global test_idx | |
47 | form, output, ret = None, "", None | |
48 | while data: | |
49 | test_idx += 1 | |
50 | line = data.pop(0) | |
51 | if re.match(r"^\s*$", line): # blank line | |
52 | continue | |
53 | elif line[0:3] == ";;;": # ignore comment | |
54 | continue | |
55 | elif line[0:2] == ";;": # output comment | |
56 | print line[3:] | |
57 | continue | |
58 | elif line[0:2] == ";": # unexpected comment | |
59 | print "Test data error at line %d:\n%s" % (test_idx, line) | |
60 | return None, None, None, test_idx | |
61 | form = line # the line is a form to send | |
62 | ||
63 | # Now find the output and return value | |
64 | while data: | |
65 | line = data[0] | |
66 | if line[0:3] == ";=>": | |
67 | ret = line[3:].replace('\\r', '\r').replace('\\n', '\n') | |
68 | test_idx += 1 | |
69 | data.pop(0) | |
70 | break | |
71 | elif line[0:2] == "; ": | |
72 | output = output + line[2:] + sep | |
73 | test_idx += 1 | |
74 | data.pop(0) | |
75 | else: | |
76 | ret = "*" | |
77 | break | |
78 | if ret: break | |
79 | ||
80 | return form, output, ret, test_idx | |
81 | ||
cc021efe JM |
82 | def assert_prompt(timeout): |
83 | # Wait for the initial prompt | |
84 | idx = p.expect(['user> ', 'mal-user> ', EOF, TIMEOUT], | |
85 | timeout=timeout) | |
86 | if idx not in [0,1]: | |
87 | print "Did not get 'user> ' or 'mal-user> ' prompt" | |
88 | print " Got : %s" % repr(p.before) | |
89 | sys.exit(1) | |
90 | ||
31690700 JM |
91 | |
92 | # Wait for the initial prompt | |
cc021efe JM |
93 | assert_prompt(args.start_timeout) |
94 | ||
95 | # Send the pre-eval code if any | |
96 | if args.pre_eval: | |
97 | sys.stdout.write("RUNNING pre-eval: %s" % args.pre_eval) | |
98 | p.sendline(args.pre_eval) | |
99 | assert_prompt(args.test_timeout) | |
31690700 JM |
100 | |
101 | fail_cnt = 0 | |
102 | ||
103 | while test_data: | |
104 | form, out, ret, line_num = read_test(test_data) | |
105 | if form == None: | |
106 | break | |
107 | sys.stdout.write("TEST: %s -> [%s,%s]" % (form, repr(out), repr(ret))) | |
108 | sys.stdout.flush() | |
109 | expected = "%s%s%s%s" % (form, sep, out, ret) | |
110 | ||
111 | p.sendline(form) | |
112 | try: | |
113 | idx = p.expect(['\r\nuser> ', '\nuser> ', | |
114 | '\r\nmal-user> ', '\nmal-user> '], | |
115 | timeout=args.test_timeout) | |
116 | #print "%s,%s,%s" % (idx, repr(p.before), repr(p.after)) | |
117 | if ret == "*" or p.before == expected: | |
118 | print " -> SUCCESS" | |
119 | else: | |
120 | print " -> FAIL (line %d):" % line_num | |
121 | print " Expected : %s" % repr(expected) | |
122 | print " Got : %s" % repr(p.before) | |
123 | fail_cnt += 1 | |
124 | except EOF: | |
125 | print "Got EOF" | |
126 | sys.exit(1) | |
127 | except TIMEOUT: | |
128 | print "Got TIMEOUT, received: %s" % repr(p.before) | |
129 | sys.exit(1) | |
130 | ||
131 | if fail_cnt > 0: | |
132 | print "FAILURES: %d" % fail_cnt | |
133 | sys.exit(2) | |
134 | sys.exit(0) |