8 @echo
'make "IMPL" # build all steps of IMPL'
9 @echo
'make "IMPL^STEP" # build STEP of IMPL'
11 @echo
'make "test" # test all implementations'
12 @echo
'make "test^IMPL" # test all steps of IMPL'
13 @echo
'make "test^STEP" # test STEP for all implementations'
14 @echo
'make "test^IMPL^STEP" # test STEP of IMPL'
16 @echo
'make "perf" # run microbenchmarks for all implementations'
17 @echo
'make "perf^IMPL" # run microbenchmarks for IMPL'
19 @echo
'make "repl^IMPL" # run stepA of IMPL'
20 @echo
'make "repl^IMPL^STEP" # test STEP of IMPL'
22 @echo
'make "clean" # run 'make
clean' for all implementations'
23 @echo
'make "clean^IMPL" # run 'make
clean' for IMPL'
25 @echo
'make "stats" # run 'make stats
' for all implementations'
26 @echo
'make "stats-lisp" # run 'make stats-lisp
' for all implementations'
27 @echo
'make "stats^IMPL" # run 'make stats
' for IMPL'
28 @echo
'make "stats-lisp^IMPL" # run 'make stats-lisp
' for IMPL'
30 @echo
'Options/Settings:'
32 @echo
'make MAL_IMPL=IMPL "test^mal..." # use IMPL for self-host tests'
33 @echo
'make REGRESS=1 "test..." # test with previous step tests too'
34 @echo
'make DOCKERIZE=1 ... # to dockerize above rules/targets'
38 @echo
'make "docker-build^IMPL" # build docker image for IMPL'
42 # Command line settings
49 # python, js, cpp, or neko are currently supported
52 # Extra options to pass to runtest.py
55 # Test with previous test files not just the test files for the
56 # current step. Step 0 and 1 tests are special and not included in
63 # Extra implementation specific options to pass to runtest.py
64 logo_TEST_OPTS
= --start-timeout
60 --test-timeout
120
65 mal_TEST_OPTS
= --start-timeout
60 --test-timeout
120
66 miniMAL_TEST_OPTS
= --start-timeout
60 --test-timeout
120
67 plpgsql_TEST_OPTS
= --start-timeout
60 --test-timeout
180
68 plsql_TEST_OPTS
= --start-timeout
120 --test-timeout
120
69 perl6_TEST_OPTS
= --test-timeout
=60
73 # Run target/rule within docker image for the implementation
80 IMPLS
= ada awk bash basic c d chuck clojure coffee clisp
cpp crystal cs erlang elisp \
81 elixir es6 factor forth fsharp go groovy guile haskell haxe \
82 io java julia js kotlin logo lua make mal ocaml matlab miniMAL \
83 nim objc objpascal perl perl6 php plpgsql plsql powershell ps \
84 python r racket rpython ruby rust scala swift swift3 tcl vb vhdl \
88 step1
= step1_read_print
91 step4
= step4_if_fn_do
100 regress_step1
= step1
101 regress_step2
= step2
102 regress_step3
= $(regress_step2
) step3
103 regress_step4
= $(regress_step3
) step4
104 regress_step5
= $(regress_step4
) step5
105 regress_step6
= $(regress_step5
) step6
106 regress_step7
= $(regress_step6
) step7
107 regress_step8
= $(regress_step7
) step8
108 regress_step9
= $(regress_step8
) step9
109 regress_stepA
= $(regress_step9
) stepA
111 test_EXCLUDES
+= test^bash^step5
# never completes at 10,000
112 test_EXCLUDES
+= test^basic^step5
# too slow, and limited to ints of 2^16
113 test_EXCLUDES
+= test^logo^step5
# too slow for 10,000
114 test_EXCLUDES
+= test^make^step5
# no TCO capability (iteration or recursion)
115 test_EXCLUDES
+= test^mal^step5
# host impl dependent
116 test_EXCLUDES
+= test^matlab^step5
# never completes at 10,000
117 test_EXCLUDES
+= test^plpgsql^step5
# too slow for 10,000
118 test_EXCLUDES
+= test^plsql^step5
# too slow for 10,000
119 test_EXCLUDES
+= test^powershell^step5
# too slow for 10,000
121 perf_EXCLUDES
= mal
# TODO: fix this
124 # TODO: still need to implement dist
125 dist_EXCLUDES
+= guile io julia matlab swift
131 haxe_STEP_TO_PROG_neko
= haxe
/$($(1)).n
132 haxe_STEP_TO_PROG_python
= haxe
/$($(1)).py
133 haxe_STEP_TO_PROG_cpp
= haxe
/cpp/$($(1))
134 haxe_STEP_TO_PROG_js
= haxe
/$($(1)).js
136 opt_DEFERRABLE
= $(if
$(strip $(DEFERRABLE
)),$(if
$(filter t true T True TRUE
1 y yes Yes YES
,$(DEFERRABLE
)),--deferrable
,--no-deferrable
),--no-deferrable
)
137 opt_OPTIONAL
= $(if
$(strip $(OPTIONAL
)),$(if
$(filter t true T True TRUE
1 y yes Yes YES
,$(OPTIONAL
)),--optional
,--no-optional
),--no-optional
)
139 # Return list of test files for a given step. If REGRESS is set then
140 # test files will include step 2 tests through tests for the step
142 STEP_TEST_FILES
= $(strip $(wildcard \
143 $(foreach s
,$(if
$(strip $(REGRESS
)),$(regress_
$(2)),$(2)),\
144 $(1)/tests
/$($(s
)).mal tests
/$($(s
)).mal
)))
146 # Map of step (e.g. "step8") to executable file for that step
147 ada_STEP_TO_PROG
= ada
/$($(1))
148 awk_STEP_TO_PROG
= awk
/$($(1)).awk
149 bash_STEP_TO_PROG
= bash
/$($(1)).sh
150 basic_STEP_TO_PROG
= basic
/$($(1)).bas
151 c_STEP_TO_PROG
= c
/$($(1))
152 d_STEP_TO_PROG
= d
/$($(1))
153 chuck_STEP_TO_PROG
= chuck
/$($(1)).ck
154 clojure_STEP_TO_PROG
= clojure
/target
/$($(1)).jar
155 coffee_STEP_TO_PROG
= coffee
/$($(1)).coffee
156 clisp_STEP_TO_PROG
= clisp
/$($(1)).fas
157 cpp_STEP_TO_PROG
= cpp/$($(1))
158 crystal_STEP_TO_PROG
= crystal
/$($(1))
159 cs_STEP_TO_PROG
= cs
/$($(1)).exe
160 elisp_STEP_TO_PROG
= elisp
/$($(1)).el
161 elixir_STEP_TO_PROG
= elixir
/lib
/mix
/tasks
/$($(1)).ex
162 erlang_STEP_TO_PROG
= erlang
/$($(1))
163 es6_STEP_TO_PROG
= es6
/build
/$($(1)).js
164 factor_STEP_TO_PROG
= factor
/$($(1))/$($(1)).factor
165 forth_STEP_TO_PROG
= forth
/$($(1)).fs
166 fsharp_STEP_TO_PROG
= fsharp
/$($(1)).exe
167 go_STEP_TO_PROG
= go
/$($(1))
168 groovy_STEP_TO_PROG
= groovy
/$($(1)).groovy
169 java_STEP_TO_PROG
= java
/target
/classes
/mal
/$($(1)).class
170 haskell_STEP_TO_PROG
= haskell
/$($(1))
171 haxe_STEP_TO_PROG
= $(haxe_STEP_TO_PROG_
$(HAXE_MODE
))
172 io_STEP_TO_PROG
= io
/$($(1)).io
173 julia_STEP_TO_PROG
= julia
/$($(1)).jl
174 js_STEP_TO_PROG
= js
/$($(1)).js
175 kotlin_STEP_TO_PROG
= kotlin
/$($(1)).jar
176 lua_STEP_TO_PROG
= lua
/$($(1)).lua
177 make_STEP_TO_PROG
= make
/$($(1)).mk
178 mal_STEP_TO_PROG
= mal
/$($(1)).mal
179 ocaml_STEP_TO_PROG
= ocaml
/$($(1))
180 matlab_STEP_TO_PROG
= matlab
/$($(1)).m
181 miniMAL_STEP_TO_PROG
= miniMAL
/$($(1)).json
182 nim_STEP_TO_PROG
= nim
/$($(1))
183 objc_STEP_TO_PROG
= objc
/$($(1))
184 objpascal_STEP_TO_PROG
= objpascal
/$($(1))
185 perl_STEP_TO_PROG
= perl
/$($(1)).pl
186 perl6_STEP_TO_PROG
= perl6
/$($(1)).pl
187 php_STEP_TO_PROG
= php
/$($(1)).php
188 plpgsql_STEP_TO_PROG
= plpgsql
/$($(1)).sql
189 plsql_STEP_TO_PROG
= plsql
/$($(1)).sql
190 powershell_STEP_TO_PROG
= powershell
/$($(1)).ps1
191 ps_STEP_TO_PROG
= ps
/$($(1)).ps
192 python_STEP_TO_PROG
= python
/$($(1)).py
193 r_STEP_TO_PROG
= r
/$($(1)).r
194 racket_STEP_TO_PROG
= racket
/$($(1)).rkt
195 rpython_STEP_TO_PROG
= rpython
/$($(1))
196 ruby_STEP_TO_PROG
= ruby
/$($(1)).rb
197 rust_STEP_TO_PROG
= rust
/target
/release
/$($(1))
198 scala_STEP_TO_PROG
= scala
/target
/scala-2.11
/classes
/$($(1)).class
199 swift_STEP_TO_PROG
= swift
/$($(1))
200 swift3_STEP_TO_PROG
= swift3
/$($(1))
201 tcl_STEP_TO_PROG
= tcl
/$($(1)).tcl
202 vb_STEP_TO_PROG
= vb
/$($(1)).exe
203 vhdl_STEP_TO_PROG
= vhdl
/$($(1))
204 vimscript_STEP_TO_PROG
= vimscript
/$($(1)).vim
205 guile_STEP_TO_PROG
= guile
/$($(1)).scm
208 # Needed some argument munging
211 SPACE
= $(noop
) $(noop
)
212 export FACTOR_ROOTS
:= .
214 # DOCKERIZE utility functions
215 lc
= $(subst A
,a
,$(subst B
,b
,$(subst C
,c
,$(subst D
,d
,$(subst E
,e
,$(subst F
,f
,$(subst G
,g
,$(subst H
,h
,$(subst I
,i
,$(subst J
,j
,$(subst K
,k
,$(subst L
,l
,$(subst M
,m
,$(subst N
,n
,$(subst O
,o
,$(subst P
,p
,$(subst Q
,q
,$(subst R
,r
,$(subst S
,s
,$(subst T
,t
,$(subst U
,u
,$(subst V
,v
,$(subst W
,w
,$(subst X
,x
,$(subst Y
,y
,$(subst Z
,z
,$1))))))))))))))))))))))))))
216 impl_to_image
= kanaka
/mal-test-
$(call lc
,$(1))
218 actual_impl
= $(if
$(filter mal
,$(1)),$(MAL_IMPL
),$(1))
221 # Returns nothing if DOCKERIZE is not set, otherwise returns the
222 # docker prefix necessary to run make within the docker environment
224 get_build_prefix
= $(if
$(strip $(DOCKERIZE
)),docker run
-it
--rm -u
$(shell id
-u
) -v
$(dir $(abspath
$(lastword
$(MAKEFILE_LIST
)))):/mal
-w
/mal
/$(1) $(if
$(filter factor
,$(1)),-e FACTOR_ROOTS
=$(FACTOR_ROOTS
),) $(call impl_to_image
,$(1)) ,)
226 # Takes impl and step arguments
227 # Returns a command prefix (docker command and environment variables)
228 # necessary to launch the given impl and step
229 get_run_prefix
= $(strip $(if
$(strip $(DOCKERIZE
)),\
230 docker run
-e STEP
=$($2) -e MAL_IMPL
=$(MAL_IMPL
) \
231 -it
--rm -u
$(shell id
-u
) \
232 -v
$(dir $(abspath
$(lastword
$(MAKEFILE_LIST
)))):/mal \
233 -w
/mal
/$(call actual_impl
,$(1)) \
234 $(if
$(filter haxe
,$(1)),-e HAXE_MODE
=$(HAXE_MODE
),) \
235 $(if
$(filter factor
,$(1)),-e FACTOR_ROOTS
=$(FACTOR_ROOTS
),) \
236 $(foreach env
,$(3),-e
$(env
)) \
237 $(call impl_to_image
,$(call actual_impl
,$(1))) \
239 env STEP
=$($2) MAL_IMPL
=$(MAL_IMPL
) \
240 $(if
$(filter haxe
,$(1)),HAXE_MODE
=$(HAXE_MODE
),) \
241 $(if
$(filter factor
,$(1)),FACTOR_ROOTS
=$(FACTOR_ROOTS
),) \
244 # Takes impl and step
245 # Returns the runtest command prefix (with runtest options) for testing the given step
246 get_runtest_cmd
= $(call get_run_prefix
,$(1),$(2),$(if
$(filter cs fsharp tcl vb
,$(1)),RAW
=1,)) \
247 ..
/runtest.py
$(opt_DEFERRABLE
) $(opt_OPTIONAL
) $(call
$(1)_TEST_OPTS
) $(TEST_OPTS
)
249 # Takes impl and step
250 # Returns the runtest command prefix (with runtest options) for testing the given step
251 get_argvtest_cmd
= $(call get_run_prefix
,$(1),$(2)) ..
/run_argv_test.sh
253 vimscript_TEST_OPTS
= --test-timeout
30
254 ifeq ($(MAL_IMPL
),vimscript
)
255 mal_TEST_OPTS
= --start-timeout
60 --test-timeout
180
256 else ifeq ($(MAL_IMPL
),powershell
)
257 mal_TEST_OPTS
= --start-timeout
60 --test-timeout
180
261 STEPS
= $(sort $(filter step
%,$(.VARIABLES
)))
262 DO_IMPLS
= $(filter-out $(SKIP_IMPLS
),$(IMPLS
))
263 IMPL_TESTS
= $(foreach impl
,$(DO_IMPLS
),test^
$(impl
))
264 STEP_TESTS
= $(foreach step
,$(STEPS
),test^
$(step
))
265 ALL_TESTS
= $(filter-out $(test_EXCLUDES
),\
267 $(foreach impl
,$(DO_IMPLS
),\
268 $(foreach step
,$(STEPS
),test^
$(impl
)^
$(step
))))))
270 DOCKER_BUILD
= $(foreach impl
,$(DO_IMPLS
),docker-build^
$(impl
))
272 IMPL_PERF
= $(foreach impl
,$(filter-out $(perf_EXCLUDES
),$(DO_IMPLS
)),perf^
$(impl
))
274 IMPL_REPL
= $(foreach impl
,$(DO_IMPLS
),repl^
$(impl
))
275 ALL_REPL
= $(strip $(sort \
276 $(foreach impl
,$(DO_IMPLS
),\
277 $(foreach step
,$(STEPS
),repl^
$(impl
)^
$(step
)))))
283 # Build a program in an implementation directory
284 # Make sure we always try and build first because the dependencies are
285 # encoded in the implementation Makefile not here
286 .PHONY
: $(foreach i
,$(DO_IMPLS
),$(foreach s
,$(STEPS
),$(call
$(i
)_STEP_TO_PROG
,$(s
))))
287 $(foreach i
,$(DO_IMPLS
),$(foreach s
,$(STEPS
),$(call
$(i
)_STEP_TO_PROG
,$(s
)))):
288 $(foreach impl
,$(word 1,$(subst /, ,$(@
))),\
290 $(call get_build_prefix
,$(impl
))$(MAKE
) $(patsubst $(impl
)/%,%,$(@
)), \
291 $(MAKE
) -C
$(impl
) $(subst $(impl
)/,,$(@
))))
293 # Allow IMPL, and IMPL^STEP
295 $(DO_IMPLS
): $$(foreach s
,$$(STEPS
),$$(call
$$(@
)_STEP_TO_PROG
,$$(s
)))
298 $(foreach i
,$(DO_IMPLS
),$(foreach s
,$(STEPS
),$(i
)^
$(s
))): $$(call
$$(word 1,$$(subst ^
, ,$$(@
)))_STEP_TO_PROG
,$$(word 2,$$(subst ^
, ,$$(@
))))
306 $(ALL_TESTS
): $$(call
$$(word 2,$$(subst ^
, ,$$(@
)))_STEP_TO_PROG
,$$(word 3,$$(subst ^
, ,$$(@
))))
307 @
$(foreach impl
,$(word 2,$(subst ^
, ,$(@
))),\
308 $(foreach step
,$(word 3,$(subst ^
, ,$(@
))),\
309 cd
$(if
$(filter mal
,$(impl
)),$(MAL_IMPL
),$(impl
)) && \
310 $(foreach test,$(call STEP_TEST_FILES
,$(impl
),$(step
)),\
311 echo
'----------------------------------------------' && \
312 echo
'Testing $@; step file: $+, test file: $(test)' && \
313 echo
'Running: $(call get_runtest_cmd,$(impl),$(step)) ../$(test) -- ../$(impl)/run' && \
314 $(call get_runtest_cmd
,$(impl
),$(step
)) ..
/$(test) -- ..
/$(impl
)/run
&& \
315 $(if
$(filter tests
/step6_file.mal
,$(test)),\
316 echo
'----------------------------------------------' && \
317 echo
'Testing ARGV of $@; step file: $+' && \
318 echo
'Running: $(call get_argvtest_cmd,$(impl),$(step)) ../$(impl)/run ' && \
319 $(call get_argvtest_cmd
,$(impl
),$(step
)) ..
/$(impl
)/run
&& ,\
323 # Allow test, tests, test^STEP, test^IMPL, and test^IMPL^STEP
328 $(IMPL_TESTS
): $$(filter $$@^
%,$$(ALL_TESTS
))
331 $(STEP_TESTS
): $$(foreach step
,$$(subst test^
,,$$@
),$$(filter %^
$$(step
),$$(ALL_TESTS
)))
342 @echo
"----------------------------------------------"; \
343 $(foreach impl
,$(word 2,$(subst ^
, ,$(@
))),\
344 echo
"Running: make -C $(impl) dist"; \
345 $(MAKE
) --no-print-directory
-C
$(impl
) dist)
352 docker-build
: $(DOCKER_BUILD
)
356 echo
"----------------------------------------------"; \
357 $(foreach impl
,$(word 2,$(subst ^
, ,$(@
))),\
358 echo
"Running: docker build -t $(call impl_to_image,$(impl)) .:"; \
359 cd
$(impl
) && docker build
-t
$(call impl_to_image
,$(impl
)) .
)
363 # Performance test rules
370 @echo
"----------------------------------------------"; \
371 $(foreach impl
,$(word 2,$(subst ^
, ,$(@
))),\
372 cd
$(if
$(filter mal
,$(impl
)),$(MAL_IMPL
),$(impl
)); \
373 echo
"Performance test for $(impl):"; \
374 echo
'Running: $(call get_run_prefix,$(impl),stepA) ../$(impl)/run ../tests/perf1.mal'; \
375 $(call get_run_prefix
,$(impl
),stepA
) ..
/$(impl
)/run ..
/tests
/perf1.mal
; \
376 echo
'Running: $(call get_run_prefix,$(impl),stepA) ../$(impl)/run ../tests/perf2.mal'; \
377 $(call get_run_prefix
,$(impl
),stepA
) ..
/$(impl
)/run ..
/tests
/perf2.mal
; \
378 echo
'Running: $(call get_run_prefix,$(impl),stepA) ../$(impl)/run ../tests/perf3.mal'; \
379 $(call get_run_prefix
,$(impl
),stepA
) ..
/$(impl
)/run ..
/tests
/perf3.mal
)
383 # REPL invocation rules
387 $(ALL_REPL
): $$(call
$$(word 2,$$(subst ^
, ,$$(@
)))_STEP_TO_PROG
,$$(word 3,$$(subst ^
, ,$$(@
))))
388 @
$(foreach impl
,$(word 2,$(subst ^
, ,$(@
))),\
389 $(foreach step
,$(word 3,$(subst ^
, ,$(@
))),\
390 cd
$(if
$(filter mal
,$(impl
)),$(MAL_IMPL
),$(impl
)); \
391 echo
'REPL implementation $(impl), step file: $+'; \
392 echo
'Running: $(call get_run_prefix,$(impl),$(step)) ../$(impl)/run'; \
393 $(call get_run_prefix
,$(impl
),$(step
)) ..
/$(impl
)/run
;))
395 # Allow repl^IMPL^STEP and repl^IMPL (which starts REPL of stepA)
397 $(IMPL_REPL
): $$@^stepA
407 # Recursive rules (call make FOO in each subdirectory)
410 define recur_template
415 @echo
"----------------------------------------------"; \
416 $$(foreach impl
,$$(word 2,$$(subst ^
, ,$$(@
))),\
417 $$(if
$$(DOCKERIZE
), \
418 echo
"Running: $$(call get_build_prefix,$$(impl))$$(MAKE) --no-print-directory $(1)"; \
419 $$(call get_build_prefix
,$$(impl
))$$(MAKE
) --no-print-directory
$(1), \
420 echo
"Running: $$(MAKE) --no-print-directory -C $$(impl) $(1)"; \
421 $$(MAKE
) --no-print-directory
-C
$$(impl
) $(1)))
424 recur_impls_
= $(filter-out $(foreach impl
,$($(1)_EXCLUDES
),$(1)^
$(impl
)),$(foreach impl
,$(IMPLS
),$(1)^
$(impl
)))
427 $(eval
$(call recur_template
,clean,$(call recur_impls_
,clean)))
430 $(eval
$(call recur_template
,stats
,$(call recur_impls_
,stats
)))
431 $(eval
$(call recur_template
,stats-lisp
,$(call recur_impls_
,stats-lisp
)))
434 $(eval
$(call recur_template
,dist,$(call recur_impls_
,dist)))