runtest.py: process output/errors as regex match.

Update output test data to be regex compatible.

diff --git a/runtest.py b/runtest.py
index bd1d24b..6e2a495 100755 (executable)
--- a/runtest.py
+++ b/runtest.py
@@ -215,15 +215,19 @@ class TestReader:
                     self.line_num += 1
                     self.data.pop(0)
                     break
-                elif line[0:2] == "; ":
+                elif line[0:2] == ";/":
                     self.out = self.out + line[2:] + sep
                     self.line_num += 1
                     self.data.pop(0)
                 else:
-                    self.ret = "*"
+                    self.ret = ""
                     break
-            if self.ret: break
+            if self.ret != None: break
 
+        if self.out[-2:] == sep and not self.ret:
+            # If there is no return value, output should not end in
+            # separator
+            self.out = self.out[0:-2]
         return self.form
 
 args = parser.parse_args(sys.argv[1:])
@@ -293,8 +297,11 @@ while t.next():
     # The repeated form is to get around an occasional OS X issue
     # where the form is repeated.
     # https://github.com/kanaka/mal/issues/30
-    expected = ["%s%s%s%s" % (t.form, sep, t.out, t.ret),
-                "%s%s%s%s%s%s" % (t.form, sep, t.form, sep, t.out, t.ret)]
+    expects = ["%s%s%s%s" % (re.escape(t.form), sep,
+                              t.out, re.escape(t.ret)),
+               "%s%s%s%s%s%s" % (re.escape(t.form), sep,
+                                  re.escape(t.form), sep,
+                                  t.out, re.escape(t.ret))]
 
     r.writeline(t.form)
     try:
@@ -302,7 +309,11 @@ while t.next():
         res = r.read_to_prompt(['\r\n[^\s()<>]+> ', '\n[^\s()<>]+> '],
                                 timeout=args.test_timeout)
         #print "%s,%s,%s" % (idx, repr(p.before), repr(p.after))
-        if t.ret == "*" or res in expected:
+        if (t.ret == "" and t.out == ""):
+            log(" -> SUCCESS (result ignored)")
+            pass_cnt += 1
+        elif (re.search(expects[0], res, re.S) or
+                re.search(expects[1], res, re.S)):
             log(" -> SUCCESS")
             pass_cnt += 1
         else:
@@ -314,11 +325,12 @@ while t.next():
                 log(" -> FAIL (line %d):" % t.line_num)
                 fail_cnt += 1
                 fail_type = ""
-            log("    Expected : %s" % repr(expected[0]))
+            log("    Expected : %s" % repr(expects[0]))
             log("    Got      : %s" % repr(res))
             failed_test = """%sFAILED TEST (line %d): %s -> [%s,%s]:
     Expected : %s
-    Got      : %s""" % (fail_type, t.line_num, t.form, repr(t.out), t.ret, repr(expected[0]), repr(res))
+    Got      : %s""" % (fail_type, t.line_num, t.form, repr(t.out),
+                        t.ret, repr(expects[0]), repr(res))
             failures.append(failed_test)
     except:
         _, exc, _ = sys.exc_info()

diff --git a/tests/step1_read_print.mal b/tests/step1_read_print.mal
index a4d40a0..266b016 100644 (file)
--- a/tests/step1_read_print.mal
+++ b/tests/step1_read_print.mal
@@ -75,15 +75,18 @@ false
 ;=>""
 
 ;; Testing reader errors
-;;; TODO: fix these so they fail correctly
 (1 2
-; expected ')', got EOF
+;/.*(EOF|end of input|unbalanced).*
 [1 2
-; expected ']', got EOF
+;/.*(EOF|end of input|unbalanced).*
+
+;;; These should throw some error with no return value
 "abc
-; expected '"', got EOF
+;/.+
 (1 "abc
-; expected ')', got EOF
+;/.+
+(1 "abc"
+;/.+
 
 ;; Testing read of quoting
 '1

diff --git a/tests/step2_eval.mal b/tests/step2_eval.mal
index c92fa84..16a3589 100644 (file)
--- a/tests/step2_eval.mal
+++ b/tests/step2_eval.mal
@@ -20,8 +20,9 @@
 (/ (- (+ 515 (* -87 311)) 296) 27)
 ;=>-994
 
+;;; This should throw an error with no return value
 (abc 1 2 3)
-; .*\'abc\' not found.*
+;/.+
 
 ;; Testing empty list
 ()

diff --git a/tests/step3_env.mal b/tests/step3_env.mal
index ebd2e8a..1539e56 100644 (file)
--- a/tests/step3_env.mal
+++ b/tests/step3_env.mal
@@ -31,7 +31,7 @@ MYNUM
 
 ;; Check env lookup non-fatal error
 (abc 1 2 3)
-; .*\'abc\' not found.*
+;/.*\'?abc\'? not found.*
 ;; Check that error aborts def!
 (def! w 123)
 (def! w (abc))

diff --git a/tests/step4_if_fn_do.mal b/tests/step4_if_fn_do.mal
index 1117818..991b44b 100644 (file)
--- a/tests/step4_if_fn_do.mal
+++ b/tests/step4_if_fn_do.mal
@@ -173,14 +173,14 @@
 
 ;; Testing do form
 (do (prn "prn output1"))
-; "prn output1"
+;/"prn output1"
 ;=>nil
 (do (prn "prn output2") 7)
-; "prn output2"
+;/"prn output2"
 ;=>7
 (do (prn "prn output1") (prn "prn output2") (+ 1 2))
-; "prn output1"
-; "prn output2"
+;/"prn output1"
+;/"prn output2"
 ;=>3
 
 (do (def! a 6) 7 (+ a 8))
@@ -341,69 +341,69 @@ a
 
 ;; Testing prn
 (prn)
-; 
+;/
 ;=>nil
 
 (prn "")
-; ""
+;/""
 ;=>nil
 
 (prn "abc")
-; "abc"
+;/"abc"
 ;=>nil
 
 (prn "abc  def" "ghi jkl")
-; "abc  def" "ghi jkl"
+;/"abc  def" "ghi jkl"
 
 (prn "\"")
-; "\""
+;/"\\""
 ;=>nil
 
 (prn "abc\ndef\nghi")
-; "abc\ndef\nghi"
+;/"abc\\ndef\\nghi"
 ;=>nil
 
 (prn "abc\\def\\ghi")
-; "abc\\def\\ghi"
+;/"abc\\\\def\\\\ghi"
 nil
 
 (prn (list 1 2 "abc" "\"") "def")
-; (1 2 "abc" "\"") "def"
+;/\(1 2 "abc" "\\""\) "def"
 ;=>nil
 
 
 ;; Testing println
 (println)
-; 
+;/
 ;=>nil
 
 (println "")
-; 
+;/
 ;=>nil
 
 (println "abc")
-; abc
+;/abc
 ;=>nil
 
 (println "abc  def" "ghi jkl")
-; abc  def ghi jkl
+;/abc  def ghi jkl
 
 (println "\"")
-; "
+;/"
 ;=>nil
 
 (println "abc\ndef\nghi")
-; abc
-; def
-; ghi
+;/abc
+;/def
+;/ghi
 ;=>nil
 
 (println "abc\\def\\ghi")
-; abc\def\ghi
+;/abc\\def\\ghi
 ;=>nil
 
 (println (list 1 2 "abc" "\"") "def")
-; (1 2 abc ") def
+;/\(1 2 abc "\) def
 ;=>nil
 
 ;>>> optional=True

diff --git a/tests/step6_file.mal b/tests/step6_file.mal
index c024e0d..486725e 100644 (file)
--- a/tests/step6_file.mal
+++ b/tests/step6_file.mal
@@ -99,7 +99,7 @@
 
 ;; Testing comments in a file
 (load-file "../tests/incB.mal")
-; "incB.mal finished"
+;/"incB.mal finished"
 ;=>"incB.mal return string"
 (inc4 7)
 ;=>11

diff --git a/tests/step9_try.mal b/tests/step9_try.mal
index d50250e..5108914 100644 (file)
--- a/tests/step9_try.mal
+++ b/tests/step9_try.mal
@@ -5,11 +5,11 @@
 ;=>123
 
 (try* (abc 1 2) (catch* exc (prn "exc is:" exc)))
-; "exc is:" "'abc' not found"
+;/"exc is:" "'abc' not found"
 ;=>nil
 
 (try* (throw "my exception") (catch* exc (do (prn "exc:" exc) 7)))
-; "exc:" "my exception"
+;/"exc:" "my exception"
 ;=>7
 
 ;;; Test that throw is a function:
@@ -48,10 +48,10 @@
 (apply + 4 (list 5))
 ;=>9
 (apply prn (list 1 2 "3" (list)))
-; 1 2 "3" ()
+;/1 2 "3" \(\)
 ;=>nil
 (apply prn 1 2 (list "3" (list)))
-; 1 2 "3" ()
+;/1 2 "3" \(\)
 ;=>nil
 (apply list (list))
 ;=>()
@@ -124,7 +124,7 @@
 (apply + 4 [5])
 ;=>9
 (apply prn 1 2 ["3" 4])
-; 1 2 "3" 4
+;/1 2 "3" 4
 ;=>nil
 (apply list [])
 ;=>()
@@ -307,7 +307,7 @@
 ;;
 ;; Testing throwing non-strings
 (try* (throw (list 1 2 3)) (catch* exc (do (prn "err:" exc) 7)))
-; "err:" (1 2 3)
+;/"err:" \(1 2 3\)
 ;=>7
 
 ;;