3 from __future__
import print_function
8 from subprocess
import Popen
, STDOUT
, PIPE
9 from select
import select
11 # Pseudo-TTY and terminal manipulation
12 import pty
, array
, fcntl
, termios
14 IS_PY_3
= sys
.version_info
[0] == 3
21 debug_file
.write(data
)
24 def log(data
, end
='\n'):
26 log_file
.write(data
+ end
)
31 # TODO: do we need to support '\n' too
36 parser
= argparse
.ArgumentParser(
37 description
="Run a test file against a Mal implementation")
38 parser
.add_argument('--rundir',
39 help="change to the directory before running tests")
40 parser
.add_argument('--start-timeout', default
=10, type=int,
41 help="default timeout for initial prompt")
42 parser
.add_argument('--test-timeout', default
=20, type=int,
43 help="default timeout for each individual test action")
44 parser
.add_argument('--pre-eval', default
=None, type=str,
45 help="Mal code to evaluate prior to running the test")
46 parser
.add_argument('--no-pty', action
='store_true',
47 help="Use direct pipes instead of pseudo-tty")
48 parser
.add_argument('--log-file', type=str,
49 help="Write messages to the named file in addition the screen")
50 parser
.add_argument('--debug-file', type=str,
51 help="Write all test interaction the named file")
52 parser
.add_argument('--hard', action
='store_true',
53 help="Turn soft tests following a ';>>> soft=True' into hard failures")
55 parser
.add_argument('test_file', type=argparse
.FileType('r'),
56 help="a test file formatted as with mal test data")
57 parser
.add_argument('mal_cmd', nargs
="*",
58 help="Mal implementation command line. Use '--' to "
59 "specify a Mal command line with dashed options.")
62 def __init__(self
, args
, no_pty
=False):
63 #print "args: %s" % repr(args)
66 # Cleanup child process on exit
67 atexit
.register(self
.cleanup
)
72 env
['INPUTRC'] = '/dev/null'
73 env
['PERL_RL'] = 'false'
75 self
.p
= Popen(args
, bufsize
=0,
76 stdin
=PIPE
, stdout
=PIPE
, stderr
=STDOUT
,
79 self
.stdin
= self
.p
.stdin
80 self
.stdout
= self
.p
.stdout
82 # provide tty to get 'interactive' readline to work
83 master
, slave
= pty
.openpty()
85 # Set terminal size large so that readline will not send
86 # ANSI/VT escape codes when the lines are long.
87 buf
= array
.array('h', [100, 200, 0, 0])
88 fcntl
.ioctl(master
, termios
.TIOCSWINSZ
, buf
, True)
90 self
.p
= Popen(args
, bufsize
=0,
91 stdin
=slave
, stdout
=slave
, stderr
=STDOUT
,
94 # Now close slave so that we will get an exception from
95 # read when the child exits early
96 # http://stackoverflow.com/questions/11165521
98 self
.stdin
= os
.fdopen(master
, 'r+b', 0)
99 self
.stdout
= self
.stdin
103 self
.last_prompt
= ""
105 def read_to_prompt(self
, prompts
, timeout
):
106 end_time
= time
.time() + timeout
107 while time
.time() < end_time
:
108 [outs
,_
,_
] = select([self
.stdout
], [], [], 1)
109 if self
.stdout
in outs
:
110 new_data
= self
.stdout
.read(1)
111 new_data
= new_data
.decode("utf-8") if IS_PY_3
else new_data
112 #print("new_data: '%s'" % new_data)
115 self
.buf
+= new_data
.replace("\n", "\r\n")
118 for prompt
in prompts
:
119 regexp
= re
.compile(prompt
)
120 match
= regexp
.search(self
.buf
)
123 buf
= self
.buf
[0:end
-len(prompt
)]
124 self
.buf
= self
.buf
[end
:]
125 self
.last_prompt
= prompt
129 def writeline(self
, str):
131 return bytes(s
, "utf-8") if IS_PY_3
else s
133 self
.stdin
.write(_to_bytes(str + "\n"))
139 os
.killpg(self
.p
.pid
, signal
.SIGTERM
)
145 def __init__(self
, test_file
, print=print):
147 self
.data
= test_file
.read().split('\n')
158 line
= self
.data
.pop(0)
159 if re
.match(r
"^\s*$", line
): # blank line
161 elif line
[0:3] == ";;;": # ignore comment
163 elif line
[0:2] == ";;": # output comment
166 elif line
[0:5] == ";>>> ": # settings/commands
168 exec(line
[5:], {}, settings
)
169 if 'soft' in settings
: self
.soft
= True
171 elif line
[0:1] == ";": # unexpected comment
172 log("Test data error at line %d:\n%s" % (self
.line_num
, line
))
174 self
.form
= line
# the line is a form to send
176 # Now find the output and return value
179 if line
[0:3] == ";=>":
184 elif line
[0:2] == "; ":
185 self
.out
= self
.out
+ line
[2:] + sep
195 args
= parser
.parse_args(sys
.argv
[1:])
196 # Workaround argparse issue with two '--' on command line
197 if sys
.argv
.count('--') > 0:
198 args
.mal_cmd
= sys
.argv
[sys
.argv
.index('--')+1:]
200 if args
.rundir
: os
.chdir(args
.rundir
)
202 if args
.log_file
: log_file
= open(args
.log_file
, "a")
203 if args
.debug_file
: debug_file
= open(args
.debug_file
, "a")
205 r
= Runner(args
.mal_cmd
, no_pty
=args
.no_pty
)
206 t
= TestReader(args
.test_file
)
209 def assert_prompt(runner
, prompts
, timeout
):
210 # Wait for the initial prompt
211 header
= runner
.read_to_prompt(prompts
, timeout
=timeout
)
212 if not header
== None:
214 log("Started with:\n%s" % header
)
216 log("Did not one of following prompt(s): %s" % repr(prompts
))
217 log(" Got : %s" % repr(r
.buf
))
221 # Wait for the initial prompt
222 assert_prompt(r
, ['user> ', 'mal-user> '], args
.start_timeout
)
224 # Send the pre-eval code if any
226 sys
.stdout
.write("RUNNING pre-eval: %s" % args
.pre_eval
)
227 p
.write(args
.pre_eval
)
228 assert_prompt(args
.test_timeout
)
236 log("TEST: %s -> [%s,%s]" % (t
.form
, repr(t
.out
), t
.ret
), end
='')
238 # The repeated form is to get around an occasional OS X issue
239 # where the form is repeated.
240 # https://github.com/kanaka/mal/issues/30
241 expected
= ["%s%s%s%s" % (t
.form
, sep
, t
.out
, t
.ret
),
242 "%s%s%s%s%s%s" % (t
.form
, sep
, t
.form
, sep
, t
.out
, t
.ret
)]
247 res
= r
.read_to_prompt(['\r\nuser> ', '\nuser> ',
248 '\r\nmal-user> ', '\nmal-user> '],
249 timeout
=args
.test_timeout
)
250 #print "%s,%s,%s" % (idx, repr(p.before), repr(p.after))
251 if t
.ret
== "*" or res
in expected
:
255 if t
.soft
and not args
.hard
:
256 log(" -> SOFT FAIL (line %d):" % t
.line_num
)
259 log(" -> FAIL (line %d):" % t
.line_num
)
261 log(" Expected : %s" % repr(expected
[0]))
262 log(" Got : %s" % repr(res
))
264 _
, exc
, _
= sys
.exc_info()
265 log("\nException: %s" % repr(exc
))
266 log("Output before exception:\n%s" % r
.buf
)
269 results
= """TEST RESULTS (for %s):
270 %3d: soft failing tests
274 """ % (args
.test_file
.name
, soft_fail_cnt
, fail_cnt
,
278 debug("\n") # add some separate to debug log