nim: fix tokenizer endless loop (and out-of-memory) on bad input

The input string `(prn "abc` caused the nim implementaion (from step1
onwards) to enter an endless loop in tokenize (which eventually caused
out-of-memory).

The fix verfies that indeed the regex matches a non-empty substring
before adding that substring as a token.

diff --git a/nim/reader.nim b/nim/reader.nim
index 323b3d0..4df8c9c 100644 (file)
--- a/nim/reader.nim
+++ b/nim/reader.nim
@@ -28,7 +28,7 @@ proc tokenize(str: string): seq[string] =
   while pos < str.len:
     var matches: array[2, string]
     var len = str.findBounds(tokenRE, matches, pos)
-    if len.first != -1 and len.last != -1:
+    if len.first != -1 and len.last != -1 and len.last >= len.first:
       pos = len.last + 1
       if matches[0][0] != ';':
         result.add matches[0]

diff --git a/tests/step1_read_print.mal b/tests/step1_read_print.mal
index 985b5fb..06e05ed 100644 (file)
--- a/tests/step1_read_print.mal
+++ b/tests/step1_read_print.mal
@@ -87,6 +87,8 @@ abc-def
 ; expected ']', got EOF
 "abc
 ; expected '"', got EOF
+(1 "abc
+; expected ')', got EOF
 
 ;;
 ;; -------- Optional Functionality --------