Commit | Line | Data |
---|---|---|
31690700 JM |
1 | # |
2 | # Command line settings | |
3 | # | |
4 | ||
5 | MAL_IMPL = js | |
6 | ||
a05f7822 | 7 | PYTHON = python |
47699629 | 8 | USE_MATLAB = |
32d0a1cf JM |
9 | # python, js, cpp, or neko are currently supported |
10 | HAXE_MODE = neko | |
a05f7822 | 11 | |
337c8031 JM |
12 | # Extra options to pass to runtest.py |
13 | TEST_OPTS = | |
14 | ||
17180e85 JM |
15 | # Test with previous test files not just the test files for the |
16 | # current step. Step 0 and 1 tests are special and not included in | |
17 | # later steps. | |
18 | REGRESS= | |
19 | ||
337c8031 JM |
20 | # Extra implementation specific options to pass to runtest.py |
21 | mal_TEST_OPTS = --start-timeout 60 --test-timeout 120 | |
22 | ||
31690700 JM |
23 | # |
24 | # Settings | |
25 | # | |
26 | ||
6c4c14bd JM |
27 | IMPLS = awk bash c d clojure coffee cpp crystal cs erlang elixir es6 \ |
28 | factor forth fsharp go groovy guile haskell haxe java julia \ | |
29 | js kotlin lua make mal ocaml matlab miniMAL nim perl php ps \ | |
30 | python r racket rpython ruby rust scala swift tcl vb vimscript | |
31690700 JM |
31 | |
32 | step0 = step0_repl | |
33 | step1 = step1_read_print | |
34 | step2 = step2_eval | |
35 | step3 = step3_env | |
36 | step4 = step4_if_fn_do | |
37 | step5 = step5_tco | |
38 | step6 = step6_file | |
39 | step7 = step7_quote | |
40 | step8 = step8_macros | |
01c97316 | 41 | step9 = step9_try |
90f618cb | 42 | stepA = stepA_mal |
31690700 | 43 | |
17180e85 JM |
44 | regress_step0 = step0 |
45 | regress_step1 = step1 | |
46 | regress_step2 = step2 | |
47 | regress_step3 = $(regress_step2) step3 | |
48 | regress_step4 = $(regress_step3) step4 | |
49 | regress_step5 = $(regress_step4) step5 | |
50 | regress_step6 = $(regress_step5) step6 | |
51 | regress_step7 = $(regress_step6) step7 | |
52 | regress_step8 = $(regress_step7) step8 | |
53 | regress_step9 = $(regress_step8) step9 | |
54 | regress_stepA = $(regress_step9) stepA | |
55 | ||
56 | STEP5_EXCLUDES += awk # completes at 10,000 | |
57 | STEP5_EXCLUDES += bash # no stack exhaustion or completion | |
58 | STEP5_EXCLUDES += c # segfault | |
59 | STEP5_EXCLUDES += cpp # completes at 10,000 | |
60 | STEP5_EXCLUDES += cs # fatal stack overflow fault | |
61 | STEP5_EXCLUDES += d # completes at 10,000, fatal stack overflow at 1,000,000 | |
62 | STEP5_EXCLUDES += erlang # erlang is TCO, test passes | |
63 | STEP5_EXCLUDES += elixir # elixir is TCO, test passes | |
64 | STEP5_EXCLUDES += fsharp # completes at 10,000, fatal stack overflow at 100,000 | |
65 | STEP5_EXCLUDES += haskell # test completes | |
66 | STEP5_EXCLUDES += make # no TCO capability/step | |
67 | STEP5_EXCLUDES += mal # no TCO capability/step | |
47699629 | 68 | STEP5_EXCLUDES += matlab # too slow to complete 10,000 |
17180e85 JM |
69 | STEP5_EXCLUDES += miniMAL # strange error with runtest.py |
70 | STEP5_EXCLUDES += nim # test completes, even at 100,000 | |
71 | STEP5_EXCLUDES += go # test completes, even at 100,000 | |
72 | STEP5_EXCLUDES += php # test completes, even at 100,000 | |
73 | STEP5_EXCLUDES += racket # test completes | |
74 | STEP5_EXCLUDES += ruby # test completes, even at 100,000 | |
75 | STEP5_EXCLUDES += rust # no catching stack overflows | |
76 | STEP5_EXCLUDES += ocaml # test completes, even at 1,000,000 | |
77 | STEP5_EXCLUDES += vb # completes at 10,000 | |
78 | STEP5_EXCLUDES += crystal # test completes, even at 1,000,000 | |
79 | ||
80 | PERF_EXCLUDES = mal # TODO: fix this | |
31690700 JM |
81 | |
82 | # | |
83 | # Utility functions | |
84 | # | |
85 | ||
47699629 JM |
86 | MATLAB = matlab -nodisplay -nosplash -nodesktop -nojvm -r |
87 | OCTAVE = octave --no-gui -q --traditional --eval | |
88 | matlab_args = $(subst $(SPACE),$(COMMA),$(foreach x,$(strip $(1)),'$(x)')) | |
89 | matlab_cmd = $(if $(strip $(USE_MATLAB)),$(MATLAB),$(OCTAVE)) | |
90 | ||
32d0a1cf JM |
91 | haxe_STEP_TO_PROG_neko = haxe/$($(1)).n |
92 | haxe_STEP_TO_PROG_python = haxe/$($(1)).py | |
93 | haxe_STEP_TO_PROG_cpp = haxe/cpp/$($(1)) | |
94 | haxe_STEP_TO_PROG_js = haxe/$($(1)).js | |
95 | ||
96 | haxe_RUNSTEP_neko = neko ../$(2) $(3) | |
97 | haxe_RUNSTEP_python = python3 ../$(2) $(3) | |
98 | haxe_RUNSTEP_cpp = ../$(2) $(3) | |
99 | haxe_RUNSTEP_js = node ../$(2) $(3) | |
100 | ||
17180e85 JM |
101 | # Return list of test files for a given step. If REGRESS is set then |
102 | # test files will include step 2 tests through tests for the step | |
103 | # being tested. | |
104 | STEP_TEST_FILES = $(strip $(wildcard \ | |
105 | $(foreach s,$(if $(strip $(REGRESS)),$(regress_$(2)),$(2)),\ | |
106 | $(1)/tests/$($(s)).mal tests/$($(s)).mal))) | |
31690700 | 107 | |
17180e85 | 108 | # Map of step (e.g. "step8") to executable file for that step |
3b7ef8c7 | 109 | awk_STEP_TO_PROG = awk/$($(1)).awk |
db4c329a JM |
110 | bash_STEP_TO_PROG = bash/$($(1)).sh |
111 | c_STEP_TO_PROG = c/$($(1)) | |
f82cb965 | 112 | d_STEP_TO_PROG = d/$($(1)) |
31690700 | 113 | clojure_STEP_TO_PROG = clojure/src/$($(1)).clj |
891c3f3b | 114 | coffee_STEP_TO_PROG = coffee/$($(1)).coffee |
9ddaa0b9 | 115 | cpp_STEP_TO_PROG = cpp/$($(1)) |
86fe6314 | 116 | crystal_STEP_TO_PROG = crystal/$($(1)) |
db4c329a | 117 | cs_STEP_TO_PROG = cs/$($(1)).exe |
51c2c1fe | 118 | elixir_STEP_TO_PROG = elixir/lib/mix/tasks/$($(1)).ex |
2cc3804b | 119 | erlang_STEP_TO_PROG = erlang/$($(1)) |
4eb71990 | 120 | es6_STEP_TO_PROG = es6/build/$($(1)).js |
199b1ce7 | 121 | factor_STEP_TO_PROG = factor/$($(1))/$($(1)).factor |
59038a10 | 122 | forth_STEP_TO_PROG = forth/$($(1)).fs |
1c358979 | 123 | fsharp_STEP_TO_PROG = fsharp/$($(1)).exe |
69b4abd6 | 124 | go_STEP_TO_PROG = go/$($(1)) |
a9cd6543 | 125 | groovy_STEP_TO_PROG = groovy/$($(1)).groovy |
db4c329a | 126 | java_STEP_TO_PROG = java/src/main/java/mal/$($(1)).java |
b76aa73b | 127 | haskell_STEP_TO_PROG = haskell/$($(1)) |
32d0a1cf | 128 | haxe_STEP_TO_PROG = $(haxe_STEP_TO_PROG_$(HAXE_MODE)) |
a23e0cdb | 129 | julia_STEP_TO_PROG = julia/$($(1)).jl |
db4c329a | 130 | js_STEP_TO_PROG = js/$($(1)).js |
53c2ea70 | 131 | kotlin_STEP_TO_PROG = kotlin/$($(1)).jar |
9d42904e | 132 | lua_STEP_TO_PROG = lua/$($(1)).lua |
db4c329a JM |
133 | make_STEP_TO_PROG = make/$($(1)).mk |
134 | mal_STEP_TO_PROG = mal/$($(1)).mal | |
59d10e1b | 135 | ocaml_STEP_TO_PROG = ocaml/$($(1)) |
9a54ea18 | 136 | matlab_STEP_TO_PROG = matlab/$($(1)).m |
c1fe72ae | 137 | miniMAL_STEP_TO_PROG = miniMAL/$($(1)).json |
dbac60df | 138 | nim_STEP_TO_PROG = nim/$($(1)) |
b5dedee0 | 139 | perl_STEP_TO_PROG = perl/$($(1)).pl |
db4c329a JM |
140 | php_STEP_TO_PROG = php/$($(1)).php |
141 | ps_STEP_TO_PROG = ps/$($(1)).ps | |
142 | python_STEP_TO_PROG = python/$($(1)).py | |
4d1456b9 | 143 | r_STEP_TO_PROG = r/$($(1)).r |
f5223195 | 144 | racket_STEP_TO_PROG = racket/$($(1)).rkt |
80320efc | 145 | rpython_STEP_TO_PROG = rpython/$($(1)) |
db4c329a | 146 | ruby_STEP_TO_PROG = ruby/$($(1)).rb |
434516e0 | 147 | rust_STEP_TO_PROG = rust/target/release/$($(1)) |
821930db | 148 | scala_STEP_TO_PROG = scala/$($(1)).scala |
2539e6af | 149 | swift_STEP_TO_PROG = swift/$($(1)) |
54d9903c | 150 | tcl_STEP_TO_PROG = tcl/$($(1)).tcl |
ee7cd585 | 151 | vb_STEP_TO_PROG = vb/$($(1)).exe |
50a964ce | 152 | vimscript_STEP_TO_PROG = vimscript/$($(1)).vim |
5eb1f5cb | 153 | guile_STEP_TO_PROG = guile/$($(1)).scm |
db4c329a | 154 | |
17180e85 | 155 | |
c4033aab JM |
156 | # Needed some argument munging |
157 | COMMA = , | |
158 | noop = | |
159 | SPACE = $(noop) $(noop) | |
d1596ac2 | 160 | export FACTOR_ROOTS := . |
db4c329a | 161 | |
17180e85 JM |
162 | # Macro for running a step: |
163 | # $(1): step (e.g. "stepA") | |
164 | # $(2): program for step (e.g. result of *_STEP_TO_PROG | |
165 | # $(3): program arguments | |
3b7ef8c7 | 166 | awk_RUNSTEP = awk -O -f ../$(2) $(3) |
db4c329a JM |
167 | bash_RUNSTEP = bash ../$(2) $(3) |
168 | c_RUNSTEP = ../$(2) $(3) | |
f82cb965 | 169 | d_RUNSTEP = ../$(2) $(3) |
db4c329a | 170 | clojure_RUNSTEP = lein with-profile +$(1) trampoline run $(3) |
891c3f3b | 171 | coffee_RUNSTEP = coffee ../$(2) $(3) |
9ddaa0b9 | 172 | cpp_RUNSTEP = ../$(2) $(3) |
86fe6314 | 173 | crystal_RUNSTEP = ../$(2) $(3) |
db4c329a | 174 | cs_RUNSTEP = mono ../$(2) --raw $(3) |
df2ca97b | 175 | elixir_RUNSTEP = mix $(notdir $(basename $(2))) $(3) |
2cc3804b | 176 | erlang_RUNSTEP = ../$(2) $(3) |
4eb71990 | 177 | es6_RUNSTEP = node ../$(2) $(3) |
389c9b8a | 178 | factor_RUNSTEP = factor ../$(2) $(3) |
59038a10 | 179 | forth_RUNSTEP = gforth ../$(2) $(3) |
1c358979 | 180 | fsharp_RUNSTEP = mono ../$(2) --raw $(3) |
69b4abd6 | 181 | go_RUNSTEP = ../$(2) $(3) |
a9cd6543 | 182 | groovy_RUNSTEP = groovy ../$(2) $(3) |
b76aa73b | 183 | haskell_RUNSTEP = ../$(2) $(3) |
6c4c14bd | 184 | haxe_RUNSTEP = python3 ../$(2) $(3) |
32d0a1cf | 185 | haxe_RUNSTEP = $(haxe_RUNSTEP_$(HAXE_MODE)) |
798206ac | 186 | java_RUNSTEP = mvn -quiet exec:java -Dexec.mainClass="mal.$($(1))" $(if $(3), -Dexec.args="$(3)",) |
a23e0cdb | 187 | julia_RUNSTEP = ../$(2) $(3) |
db4c329a | 188 | js_RUNSTEP = node ../$(2) $(3) |
53c2ea70 | 189 | kotlin_RUNSTEP = java -jar ../$(2) $(3) |
798206ac | 190 | lua_RUNSTEP = ../$(2) $(3) |
db4c329a | 191 | make_RUNSTEP = make -f ../$(2) $(3) |
922d4c3b | 192 | mal_RUNSTEP = $(call $(MAL_IMPL)_RUNSTEP,stepA,$(call $(MAL_IMPL)_STEP_TO_PROG,stepA),../$(2),") #" |
59d10e1b | 193 | ocaml_RUNSTEP = ../$(2) $(3) |
47699629 | 194 | matlab_RUNSTEP = $(matlab_cmd) "$($(1))($(call matlab_args,$(3)));quit;" |
c1fe72ae | 195 | miniMAL_RUNSTEP = miniMAL ../$(2) $(3) |
dbac60df | 196 | nim_RUNSTEP = ../$(2) $(3) |
82acd3de | 197 | perl_RUNSTEP = perl ../$(2) $(3) |
db4c329a | 198 | php_RUNSTEP = php ../$(2) $(3) |
406761e7 | 199 | ps_RUNSTEP = gs -q -I./ -dNODISPLAY -- ../$(2) $(3) |
db4c329a | 200 | python_RUNSTEP = $(PYTHON) ../$(2) $(3) |
4d1456b9 | 201 | r_RUNSTEP = Rscript ../$(2) $(3) |
f5223195 | 202 | racket_RUNSTEP = ../$(2) $(3) |
80320efc | 203 | rpython_RUNSTEP = ../$(2) $(3) |
db4c329a | 204 | ruby_RUNSTEP = ruby ../$(2) $(3) |
abdd56eb | 205 | rust_RUNSTEP = ../$(2) $(3) |
821930db | 206 | scala_RUNSTEP = sbt 'run-main $($(1))$(if $(3), $(3),)' |
2539e6af | 207 | swift_RUNSTEP = ../$(2) $(3) |
576ef370 | 208 | tcl_RUNSTEP = tclsh ../$(2) --raw $(3) |
ee7cd585 | 209 | vb_RUNSTEP = mono ../$(2) --raw $(3) |
50a964ce | 210 | vimscript_RUNSTEP = ./run_vimscript.sh ../$(2) $(3) |
5eb1f5cb | 211 | # needs TERM=dumb to work with readline |
337c8031 | 212 | guile_RUNSTEP = guile --no-auto-compile -L ../guile ../$(2) $(3) |
db4c329a | 213 | |
31690700 | 214 | |
50a964ce DM |
215 | vimscript_TEST_OPTS = --test-timeout 30 |
216 | ifeq ($(MAL_IMPL),vimscript) | |
217 | mal_TEST_OPTS = --start-timeout 60 --test-timeout 180 | |
218 | endif | |
31690700 JM |
219 | |
220 | # Derived lists | |
221 | STEPS = $(sort $(filter step%,$(.VARIABLES))) | |
8569b2af JM |
222 | DO_IMPLS = $(filter-out $(SKIP_IMPLS),$(IMPLS)) |
223 | IMPL_TESTS = $(foreach impl,$(DO_IMPLS),test^$(impl)) | |
31690700 | 224 | STEP_TESTS = $(foreach step,$(STEPS),test^$(step)) |
17180e85 | 225 | ALL_TESTS = $(filter-out $(foreach impl,$(STEP5_EXCLUDES),test^$(impl)^step5),\ |
31690700 | 226 | $(strip $(sort \ |
8569b2af | 227 | $(foreach impl,$(DO_IMPLS),\ |
31690700 JM |
228 | $(foreach step,$(STEPS),test^$(impl)^$(step)))))) |
229 | ||
8569b2af JM |
230 | IMPL_STATS = $(foreach impl,$(DO_IMPLS),stats^$(impl)) |
231 | IMPL_STATS_LISP = $(foreach impl,$(DO_IMPLS),stats-lisp^$(impl)) | |
31690700 | 232 | |
cf1d3eae JM |
233 | DOCKER_BUILD = $(foreach impl,$(DO_IMPLS),docker-build^$(impl)) |
234 | ||
17180e85 | 235 | IMPL_PERF = $(foreach impl,$(filter-out $(PERF_EXCLUDES),$(DO_IMPLS)),perf^$(impl)) |
db4c329a | 236 | |
854cf2a6 DM |
237 | IMPL_REPL = $(foreach impl,$(DO_IMPLS),repl^$(impl)) |
238 | ALL_REPL = $(strip $(sort \ | |
239 | $(foreach impl,$(DO_IMPLS),\ | |
240 | $(foreach step,$(STEPS),repl^$(impl)^$(step))))) | |
241 | ||
31690700 JM |
242 | # |
243 | # Build rules | |
244 | # | |
245 | ||
4fc7a281 | 246 | # Build a program in an implementation directory |
f045aba1 JM |
247 | # Make sure we always try and build first because the dependencies are |
248 | # encoded in the implementation Makefile not here | |
249 | .PHONY: $(foreach i,$(DO_IMPLS),$(foreach s,$(STEPS),$(call $(i)_STEP_TO_PROG,$(s)))) | |
4fc7a281 | 250 | $(foreach i,$(DO_IMPLS),$(foreach s,$(STEPS),$(call $(i)_STEP_TO_PROG,$(s)))): |
f045aba1 JM |
251 | $(foreach impl,$(word 1,$(subst /, ,$(@))),\ |
252 | $(MAKE) -C $(impl) $(subst $(impl)/,,$(@))) | |
9ddaa0b9 | 253 | |
31690700 JM |
254 | # Allow test, test^STEP, test^IMPL, and test^IMPL^STEP |
255 | .SECONDEXPANSION: | |
256 | $(IMPL_TESTS): $$(filter $$@^%,$$(ALL_TESTS)) | |
257 | ||
258 | .SECONDEXPANSION: | |
259 | $(STEP_TESTS): $$(foreach step,$$(subst test^,,$$@),$$(filter %^$$(step),$$(ALL_TESTS))) | |
260 | ||
261 | .SECONDEXPANSION: | |
262 | $(ALL_TESTS): $$(call $$(word 2,$$(subst ^, ,$$(@)))_STEP_TO_PROG,$$(word 3,$$(subst ^, ,$$(@)))) | |
263 | @$(foreach impl,$(word 2,$(subst ^, ,$(@))),\ | |
264 | $(foreach step,$(word 3,$(subst ^, ,$(@))),\ | |
265 | cd $(if $(filter mal,$(impl)),$(MAL_IMPL),$(impl)); \ | |
266 | $(foreach test,$(call STEP_TEST_FILES,$(impl),$(step)),\ | |
267 | echo '----------------------------------------------'; \ | |
268 | echo 'Testing $@, step file: $+, test file: $(test)'; \ | |
fc4c7889 JM |
269 | echo 'Running: ../runtest.py $(TEST_OPTS) $(call $(impl)_TEST_OPTS) ../$(test) -- $(call $(impl)_RUNSTEP,$(step),$(+))'; \ |
270 | ../runtest.py $(TEST_OPTS) $(call $(impl)_TEST_OPTS) ../$(test) -- $(call $(impl)_RUNSTEP,$(step),$(+));))) | |
31690700 JM |
271 | |
272 | test: $(ALL_TESTS) | |
273 | tests: $(ALL_TESTS) | |
274 | ||
275 | ||
276 | # Stats rules | |
277 | ||
db4c329a JM |
278 | stats: $(IMPL_STATS) |
279 | stats-lisp: $(IMPL_STATS_LISP) | |
280 | ||
31690700 JM |
281 | .SECONDEXPANSION: |
282 | $(IMPL_STATS): | |
283 | @echo "----------------------------------------------"; \ | |
284 | $(foreach impl,$(word 2,$(subst ^, ,$(@))),\ | |
285 | echo "Stats for $(impl):"; \ | |
286 | $(MAKE) --no-print-directory -C $(impl) stats) | |
287 | ||
712af9ef JM |
288 | .SECONDEXPANSION: |
289 | $(IMPL_STATS_LISP): | |
290 | @echo "----------------------------------------------"; \ | |
291 | $(foreach impl,$(word 2,$(subst ^, ,$(@))),\ | |
292 | echo "Stats (lisp only) for $(impl):"; \ | |
293 | $(MAKE) --no-print-directory -C $(impl) stats-lisp) | |
294 | ||
cf1d3eae JM |
295 | # Docker build rules |
296 | ||
297 | docker-build: $(DOCKER_BUILD) | |
298 | ||
299 | .SECONDEXPANSION: | |
300 | $(DOCKER_BUILD): | |
301 | echo "----------------------------------------------"; \ | |
302 | $(foreach impl,$(word 2,$(subst ^, ,$(@))),\ | |
f7824465 JM |
303 | echo "Running: docker build -t kanaka/mal-test-$(impl) .:"; \ |
304 | cd $(impl) && docker build -t kanaka/mal-test-$(impl) .) | |
db4c329a JM |
305 | |
306 | # Performance test rules | |
307 | ||
308 | perf: $(IMPL_PERF) | |
309 | ||
310 | .SECONDEXPANSION: | |
311 | $(IMPL_PERF): | |
312 | @echo "----------------------------------------------"; \ | |
313 | $(foreach impl,$(word 2,$(subst ^, ,$(@))),\ | |
314 | cd $(if $(filter mal,$(impl)),$(MAL_IMPL),$(impl)); \ | |
315 | echo "Performance test for $(impl):"; \ | |
316 | echo 'Running: $(call $(impl)_RUNSTEP,stepA,$(call $(impl)_STEP_TO_PROG,stepA),../tests/perf1.mal)'; \ | |
317 | $(call $(impl)_RUNSTEP,stepA,$(call $(impl)_STEP_TO_PROG,stepA),../tests/perf1.mal); \ | |
318 | echo 'Running: $(call $(impl)_RUNSTEP,stepA,$(call $(impl)_STEP_TO_PROG,stepA),../tests/perf2.mal)'; \ | |
699f0ad2 JM |
319 | $(call $(impl)_RUNSTEP,stepA,$(call $(impl)_STEP_TO_PROG,stepA),../tests/perf2.mal); \ |
320 | echo 'Running: $(call $(impl)_RUNSTEP,stepA,$(call $(impl)_STEP_TO_PROG,stepA),../tests/perf3.mal)'; \ | |
321 | $(call $(impl)_RUNSTEP,stepA,$(call $(impl)_STEP_TO_PROG,stepA),../tests/perf3.mal)) | |
854cf2a6 DM |
322 | |
323 | # REPL invocation rules | |
324 | # Allow repl^IMPL^STEP and repl^IMPL (which starts REPL of stepA) | |
325 | ||
326 | .SECONDEXPANSION: | |
327 | $(IMPL_REPL): $$@^stepA | |
328 | ||
329 | .SECONDEXPANSION: | |
330 | $(ALL_REPL): $$(call $$(word 2,$$(subst ^, ,$$(@)))_STEP_TO_PROG,$$(word 3,$$(subst ^, ,$$(@)))) | |
331 | @$(foreach impl,$(word 2,$(subst ^, ,$(@))),\ | |
332 | $(foreach step,$(word 3,$(subst ^, ,$(@))),\ | |
333 | cd $(if $(filter mal,$(impl)),$(MAL_IMPL),$(impl)); \ | |
334 | echo 'REPL implementation $(impl), step file: $+'; \ | |
335 | echo 'Running: $(call $(impl)_RUNSTEP,$(step),$(+))'; \ | |
336 | $(call $(impl)_RUNSTEP,$(step),$(+));)) |