8 @echo
'make "IMPL" # build all steps of IMPL'
9 @echo
'make "IMPL^STEP" # build STEP of IMPL'
11 @echo
'make "test" # test all implementations'
12 @echo
'make "test^IMPL" # test all steps of IMPL'
13 @echo
'make "test^STEP" # test STEP for all implementations'
14 @echo
'make "test^IMPL^STEP" # test STEP of IMPL'
16 @echo
'make "perf" # run microbenchmarks for all implementations'
17 @echo
'make "perf^IMPL" # run microbenchmarks for IMPL'
19 @echo
'make "repl^IMPL" # run stepA of IMPL'
20 @echo
'make "repl^IMPL^STEP" # test STEP of IMPL'
22 @echo
'make "clean" # run 'make
clean' for all implementations'
23 @echo
'make "clean^IMPL" # run 'make
clean' for IMPL'
25 @echo
'make "stats" # run 'make stats
' for all implementations'
26 @echo
'make "stats-lisp" # run 'make stats-lisp
' for all implementations'
27 @echo
'make "stats^IMPL" # run 'make stats
' for IMPL'
28 @echo
'make "stats-lisp^IMPL" # run 'make stats-lisp
' for IMPL'
30 @echo
'Options/Settings:'
32 @echo
'make MAL_IMPL=IMPL "test^mal..." # use IMPL for self-host tests'
33 @echo
'make REGRESS=1 "test..." # test with previous step tests too'
34 @echo
'make DOCKERIZE=1 ... # to dockerize above rules/targets'
38 @echo
'make "docker-build^IMPL" # build docker image for IMPL'
42 # Command line settings
49 # python, js, cpp, or neko are currently supported
51 # clj or cljs are currently supported (Clojure vs ClojureScript/lumo)
54 # Extra options to pass to runtest.py
57 # Test with previous test files not just the test files for the
58 # current step. Step 0 and 1 tests are special and not included in
65 # Extra implementation specific options to pass to runtest.py
66 logo_TEST_OPTS
= --start-timeout
60 --test-timeout
120
67 mal_TEST_OPTS
= --start-timeout
60 --test-timeout
120
68 miniMAL_TEST_OPTS
= --start-timeout
60 --test-timeout
120
69 plpgsql_TEST_OPTS
= --start-timeout
60 --test-timeout
180
70 plsql_TEST_OPTS
= --start-timeout
120 --test-timeout
120
71 perl6_TEST_OPTS
= --test-timeout
=60
73 # Run target/rule within docker image for the implementation
80 IMPLS
= ada awk bash basic c d chuck clojure coffee common-lisp
cpp crystal cs dart \
81 erlang elisp elixir es6 factor forth fsharp go groovy gst guile haskell \
82 haxe io java julia js kotlin logo lua make mal ocaml matlab miniMAL \
83 nim objc objpascal perl perl6 php pil plpgsql plsql powershell ps \
84 python r racket rexx rpython ruby rust scala scm skew swift swift3 tcl \
85 ts vb vhdl vimscript livescript elm
90 step1
= step1_read_print
93 step4
= step4_if_fn_do
101 argv_STEP
= step6_file
104 regress_step0
= step0
105 regress_step1
= step1
106 regress_step2
= step2
107 regress_step3
= $(regress_step2
) step3
108 regress_step4
= $(regress_step3
) step4
109 regress_step5
= $(regress_step4
) step5
110 regress_step6
= $(regress_step5
) step6
111 regress_step7
= $(regress_step6
) step7
112 regress_step8
= $(regress_step7
) step8
113 regress_step9
= $(regress_step8
) step9
114 regress_stepA
= $(regress_step9
) stepA
116 test_EXCLUDES
+= test^bash^step5
# never completes at 10,000
117 test_EXCLUDES
+= test^basic^step5
# too slow, and limited to ints of 2^16
118 test_EXCLUDES
+= test^logo^step5
# too slow for 10,000
119 test_EXCLUDES
+= test^make^step5
# no TCO capability (iteration or recursion)
120 test_EXCLUDES
+= test^mal^step5
# host impl dependent
121 test_EXCLUDES
+= test^matlab^step5
# never completes at 10,000
122 test_EXCLUDES
+= test^plpgsql^step5
# too slow for 10,000
123 test_EXCLUDES
+= test^plsql^step5
# too slow for 10,000
124 test_EXCLUDES
+= test^powershell^step5
# too slow for 10,000
126 perf_EXCLUDES
= mal
# TODO: fix this
129 # TODO: still need to implement dist
130 dist_EXCLUDES
+= guile io julia matlab swift
136 haxe_STEP_TO_PROG_neko
= haxe
/$($(1)).n
137 haxe_STEP_TO_PROG_python
= haxe
/$($(1)).py
138 haxe_STEP_TO_PROG_cpp
= haxe
/cpp/$($(1))
139 haxe_STEP_TO_PROG_js
= haxe
/$($(1)).js
141 clojure_STEP_TO_PROG_clj
= clojure
/target
/$($(1)).jar
142 clojure_STEP_TO_PROG_cljs
= clojure
/src
/mal
/$($(1)).cljc
144 opt_DEFERRABLE
= $(if
$(strip $(DEFERRABLE
)),$(if
$(filter t true T True TRUE
1 y yes Yes YES
,$(DEFERRABLE
)),--deferrable
,--no-deferrable
),--no-deferrable
)
145 opt_OPTIONAL
= $(if
$(strip $(OPTIONAL
)),$(if
$(filter t true T True TRUE
1 y yes Yes YES
,$(OPTIONAL
)),--optional
,--no-optional
),--no-optional
)
147 # Return list of test files for a given step. If REGRESS is set then
148 # test files will include step 2 tests through tests for the step
150 STEP_TEST_FILES
= $(strip $(wildcard \
151 $(foreach s
,$(if
$(strip $(REGRESS
)),$(regress_
$(2)),$(2)),\
152 $(1)/tests
/$($(s
))$(EXTENSION
) tests
/$($(s
))$(EXTENSION
))))
154 # Map of step (e.g. "step8") to executable file for that step
155 ada_STEP_TO_PROG
= ada
/$($(1))
156 awk_STEP_TO_PROG
= awk
/$($(1)).awk
157 bash_STEP_TO_PROG
= bash
/$($(1)).sh
158 basic_STEP_TO_PROG
= basic
/$($(1)).bas
159 c_STEP_TO_PROG
= c
/$($(1))
160 d_STEP_TO_PROG
= d
/$($(1))
161 chuck_STEP_TO_PROG
= chuck
/$($(1)).ck
162 clojure_STEP_TO_PROG
= $(clojure_STEP_TO_PROG_
$(CLJ_MODE
))
163 coffee_STEP_TO_PROG
= coffee
/$($(1)).coffee
164 common-lisp_STEP_TO_PROG
= common-lisp
/$($(1))
165 cpp_STEP_TO_PROG
= cpp/$($(1))
166 crystal_STEP_TO_PROG
= crystal
/$($(1))
167 cs_STEP_TO_PROG
= cs
/$($(1)).exe
168 dart_STEP_TO_PROG
= dart
/$($(1)).dart
169 elisp_STEP_TO_PROG
= elisp
/$($(1)).el
170 elixir_STEP_TO_PROG
= elixir
/lib
/mix
/tasks
/$($(1)).ex
171 erlang_STEP_TO_PROG
= erlang
/$($(1))
172 es6_STEP_TO_PROG
= es6
/build
/$($(1)).js
173 factor_STEP_TO_PROG
= factor
/$($(1))/$($(1)).factor
174 forth_STEP_TO_PROG
= forth
/$($(1)).fs
175 fsharp_STEP_TO_PROG
= fsharp
/$($(1)).exe
176 go_STEP_TO_PROG
= go
/$($(1))
177 groovy_STEP_TO_PROG
= groovy
/$($(1)).groovy
178 gst_STEP_TO_PROG
= gst
/$($(1)).st
179 java_STEP_TO_PROG
= java
/target
/classes
/mal
/$($(1)).class
180 haskell_STEP_TO_PROG
= haskell
/$($(1))
181 haxe_STEP_TO_PROG
= $(haxe_STEP_TO_PROG_
$(HAXE_MODE
))
182 io_STEP_TO_PROG
= io
/$($(1)).io
183 julia_STEP_TO_PROG
= julia
/$($(1)).jl
184 js_STEP_TO_PROG
= js
/$($(1)).js
185 kotlin_STEP_TO_PROG
= kotlin
/$($(1)).jar
186 lua_STEP_TO_PROG
= lua
/$($(1)).lua
187 make_STEP_TO_PROG
= make
/$($(1)).mk
188 mal_STEP_TO_PROG
= mal
/$($(1)).mal
189 ocaml_STEP_TO_PROG
= ocaml
/$($(1))
190 matlab_STEP_TO_PROG
= matlab
/$($(1)).m
191 miniMAL_STEP_TO_PROG
= miniMAL
/$($(1)).json
192 nim_STEP_TO_PROG
= nim
/$($(1))
193 objc_STEP_TO_PROG
= objc
/$($(1))
194 objpascal_STEP_TO_PROG
= objpascal
/$($(1))
195 perl_STEP_TO_PROG
= perl
/$($(1)).pl
196 perl6_STEP_TO_PROG
= perl6
/$($(1)).pl
197 php_STEP_TO_PROG
= php
/$($(1)).php
198 pil_STEP_TO_PROG
= pil
/$($(1)).l
199 plpgsql_STEP_TO_PROG
= plpgsql
/$($(1)).sql
200 plsql_STEP_TO_PROG
= plsql
/$($(1)).sql
201 powershell_STEP_TO_PROG
= powershell
/$($(1)).ps1
202 ps_STEP_TO_PROG
= ps
/$($(1)).ps
203 python_STEP_TO_PROG
= python
/$($(1)).py
204 r_STEP_TO_PROG
= r
/$($(1)).r
205 racket_STEP_TO_PROG
= racket
/$($(1)).rkt
206 rexx_STEP_TO_PROG
= rexx
/$($(1)).rexx
207 rpython_STEP_TO_PROG
= rpython
/$($(1))
208 ruby_STEP_TO_PROG
= ruby
/$($(1)).rb
209 rust_STEP_TO_PROG
= rust
/target
/release
/$($(1))
210 scala_STEP_TO_PROG
= scala
/target
/scala-2.11
/classes
/$($(1)).class
211 scm_STEP_TO_PROG
= scm
/$($(1)).scm
212 skew_STEP_TO_PROG
= skew
/$($(1)).js
213 swift_STEP_TO_PROG
= swift
/$($(1))
214 swift3_STEP_TO_PROG
= swift3
/$($(1))
215 tcl_STEP_TO_PROG
= tcl
/$($(1)).tcl
216 ts_STEP_TO_PROG
= ts
/$($(1)).js
217 vb_STEP_TO_PROG
= vb
/$($(1)).exe
218 vhdl_STEP_TO_PROG
= vhdl
/$($(1))
219 vimscript_STEP_TO_PROG
= vimscript
/$($(1)).vim
220 guile_STEP_TO_PROG
= guile
/$($(1)).scm
221 livescript_STEP_TO_PROG
= livescript
/$($(1)).js
222 elm_STEP_TO_PROG
= elm
/$($(1)).js
225 # Needed some argument munging
228 SPACE
= $(noop
) $(noop
)
229 export FACTOR_ROOTS
:= .
231 # DOCKERIZE utility functions
232 lc
= $(subst A
,a
,$(subst B
,b
,$(subst C
,c
,$(subst D
,d
,$(subst E
,e
,$(subst F
,f
,$(subst G
,g
,$(subst H
,h
,$(subst I
,i
,$(subst J
,j
,$(subst K
,k
,$(subst L
,l
,$(subst M
,m
,$(subst N
,n
,$(subst O
,o
,$(subst P
,p
,$(subst Q
,q
,$(subst R
,r
,$(subst S
,s
,$(subst T
,t
,$(subst U
,u
,$(subst V
,v
,$(subst W
,w
,$(subst X
,x
,$(subst Y
,y
,$(subst Z
,z
,$1))))))))))))))))))))))))))
233 impl_to_image
= kanaka
/mal-test-
$(call lc
,$(1))
235 actual_impl
= $(if
$(filter mal
,$(1)),$(MAL_IMPL
),$(1))
238 # Returns nothing if DOCKERIZE is not set, otherwise returns the
239 # docker prefix necessary to run make within the docker environment
241 get_build_prefix
= $(strip $(if
$(strip $(DOCKERIZE
)),\
243 -it
--rm -u
$(shell id
-u
) \
244 -v
$(dir $(abspath
$(lastword
$(MAKEFILE_LIST
)))):/mal \
246 $(if
$(filter clojure
,$(1)),-e CLJ_MODE
=$(CLJ_MODE
),) \
247 $(if
$(filter factor
,$(1)),-e FACTOR_ROOTS
=$(FACTOR_ROOTS
),) \
248 $(call impl_to_image
,$(1)) \
251 # Takes impl and step arguments
252 # Returns a command prefix (docker command and environment variables)
253 # necessary to launch the given impl and step
254 get_run_prefix
= $(strip $(if
$(strip $(DOCKERIZE
)),\
255 docker run
-e STEP
=$($2) -e MAL_IMPL
=$(MAL_IMPL
) \
256 -it
--rm -u
$(shell id
-u
) \
257 -v
$(dir $(abspath
$(lastword
$(MAKEFILE_LIST
)))):/mal \
258 -w
/mal
/$(call actual_impl
,$(1)) \
259 $(if
$(filter clojure
,$(1)),-e CLJ_MODE
=$(CLJ_MODE
),) \
260 $(if
$(filter haxe
,$(1)),-e HAXE_MODE
=$(HAXE_MODE
),) \
261 $(if
$(filter factor
,$(1)),-e FACTOR_ROOTS
=$(FACTOR_ROOTS
),) \
262 $(foreach env
,$(3),-e
$(env
)) \
263 $(call impl_to_image
,$(call actual_impl
,$(1))) \
265 env STEP
=$($2) MAL_IMPL
=$(MAL_IMPL
) \
266 $(if
$(filter haxe
,$(1)),HAXE_MODE
=$(HAXE_MODE
),) \
267 $(if
$(filter factor
,$(1)),FACTOR_ROOTS
=$(FACTOR_ROOTS
),) \
270 # Takes impl and step
271 # Returns the runtest command prefix (with runtest options) for testing the given step
272 get_runtest_cmd
= $(call get_run_prefix
,$(1),$(2),$(if
$(filter cs fsharp tcl vb
,$(1)),RAW
=1,)) \
273 ..
/runtest.py
$(opt_DEFERRABLE
) $(opt_OPTIONAL
) $(call
$(1)_TEST_OPTS
) $(TEST_OPTS
)
275 # Takes impl and step
276 # Returns the runtest command prefix (with runtest options) for testing the given step
277 get_argvtest_cmd
= $(call get_run_prefix
,$(1),$(2)) ..
/run_argv_test.sh
279 vimscript_TEST_OPTS
= --test-timeout
30
280 ifeq ($(MAL_IMPL
),vimscript
)
281 mal_TEST_OPTS
= --start-timeout
60 --test-timeout
180
282 else ifeq ($(MAL_IMPL
),powershell
)
283 mal_TEST_OPTS
= --start-timeout
60 --test-timeout
180
287 STEPS
= $(sort $(filter step
%,$(.VARIABLES
)))
288 DO_IMPLS
= $(filter-out $(SKIP_IMPLS
),$(IMPLS
))
289 IMPL_TESTS
= $(foreach impl
,$(DO_IMPLS
),test^
$(impl
))
290 STEP_TESTS
= $(foreach step
,$(STEPS
),test^
$(step
))
291 ALL_TESTS
= $(filter-out $(test_EXCLUDES
),\
293 $(foreach impl
,$(DO_IMPLS
),\
294 $(foreach step
,$(STEPS
),test^
$(impl
)^
$(step
))))))
296 DOCKER_BUILD
= $(foreach impl
,$(DO_IMPLS
),docker-build^
$(impl
))
298 IMPL_PERF
= $(foreach impl
,$(filter-out $(perf_EXCLUDES
),$(DO_IMPLS
)),perf^
$(impl
))
300 IMPL_REPL
= $(foreach impl
,$(DO_IMPLS
),repl^
$(impl
))
301 ALL_REPL
= $(strip $(sort \
302 $(foreach impl
,$(DO_IMPLS
),\
303 $(foreach step
,$(STEPS
),repl^
$(impl
)^
$(step
)))))
309 # Build a program in an implementation directory
310 # Make sure we always try and build first because the dependencies are
311 # encoded in the implementation Makefile not here
312 .PHONY
: $(foreach i
,$(DO_IMPLS
),$(foreach s
,$(STEPS
),$(call
$(i
)_STEP_TO_PROG
,$(s
))))
313 $(foreach i
,$(DO_IMPLS
),$(foreach s
,$(STEPS
),$(call
$(i
)_STEP_TO_PROG
,$(s
)))):
314 $(foreach impl
,$(word 1,$(subst /, ,$(@
))),\
316 $(call get_build_prefix
,$(impl
)) $(MAKE
) $(patsubst $(impl
)/%,%,$(@
)), \
317 $(MAKE
) -C
$(impl
) $(subst $(impl
)/,,$(@
))))
319 # Allow IMPL, and IMPL^STEP
321 $(DO_IMPLS
): $$(foreach s
,$$(STEPS
),$$(call
$$(@
)_STEP_TO_PROG
,$$(s
)))
324 $(foreach i
,$(DO_IMPLS
),$(foreach s
,$(STEPS
),$(i
)^
$(s
))): $$(call
$$(word 1,$$(subst ^
, ,$$(@
)))_STEP_TO_PROG
,$$(word 2,$$(subst ^
, ,$$(@
))))
332 $(ALL_TESTS
): $$(call
$$(word 2,$$(subst ^
, ,$$(@
)))_STEP_TO_PROG
,$$(word 3,$$(subst ^
, ,$$(@
))))
333 @
$(foreach impl
,$(word 2,$(subst ^
, ,$(@
))),\
334 $(foreach step
,$(word 3,$(subst ^
, ,$(@
))),\
335 cd
$(if
$(filter mal
,$(impl
)),$(MAL_IMPL
),$(impl
)) && \
336 $(foreach test,$(call STEP_TEST_FILES
,$(impl
),$(step
)),\
337 echo
'----------------------------------------------' && \
338 echo
'Testing $@; step file: $+, test file: $(test)' && \
339 echo
'Running: $(call get_runtest_cmd,$(impl),$(step)) ../$(test) -- ../$(impl)/run' && \
340 $(call get_runtest_cmd
,$(impl
),$(step
)) ..
/$(test) -- ..
/$(impl
)/run
&& \
341 $(if
$(filter tests
/$(argv_STEP
)$(EXTENSION
),$(test)),\
342 echo
'----------------------------------------------' && \
343 echo
'Testing ARGV of $@; step file: $+' && \
344 echo
'Running: $(call get_argvtest_cmd,$(impl),$(step)) ../$(impl)/run ' && \
345 $(call get_argvtest_cmd
,$(impl
),$(step
)) ..
/$(impl
)/run
&& ,\
349 # Allow test, tests, test^STEP, test^IMPL, and test^IMPL^STEP
354 $(IMPL_TESTS
): $$(filter $$@^
%,$$(ALL_TESTS
))
357 $(STEP_TESTS
): $$(foreach step
,$$(subst test^
,,$$@
),$$(filter %^
$$(step
),$$(ALL_TESTS
)))
368 @echo
"----------------------------------------------"; \
369 $(foreach impl
,$(word 2,$(subst ^
, ,$(@
))),\
370 echo
"Running: make -C $(impl) dist"; \
371 $(MAKE
) --no-print-directory
-C
$(impl
) dist)
378 docker-build
: $(DOCKER_BUILD
)
382 echo
"----------------------------------------------"; \
383 $(foreach impl
,$(word 2,$(subst ^
, ,$(@
))),\
384 echo
"Running: docker build -t $(call impl_to_image,$(impl)) .:"; \
385 cd
$(impl
) && docker build
-t
$(call impl_to_image
,$(impl
)) .
)
389 # Performance test rules
396 @echo
"----------------------------------------------"; \
397 $(foreach impl
,$(word 2,$(subst ^
, ,$(@
))),\
398 cd
$(if
$(filter mal
,$(impl
)),$(MAL_IMPL
),$(impl
)); \
399 echo
"Performance test for $(impl):"; \
400 echo
'Running: $(call get_run_prefix,$(impl),stepA) ../$(impl)/run ../tests/perf1.mal'; \
401 $(call get_run_prefix
,$(impl
),stepA
) ..
/$(impl
)/run ..
/tests
/perf1.mal
; \
402 echo
'Running: $(call get_run_prefix,$(impl),stepA) ../$(impl)/run ../tests/perf2.mal'; \
403 $(call get_run_prefix
,$(impl
),stepA
) ..
/$(impl
)/run ..
/tests
/perf2.mal
; \
404 echo
'Running: $(call get_run_prefix,$(impl),stepA) ../$(impl)/run ../tests/perf3.mal'; \
405 $(call get_run_prefix
,$(impl
),stepA
) ..
/$(impl
)/run ..
/tests
/perf3.mal
)
409 # REPL invocation rules
413 $(ALL_REPL
): $$(call
$$(word 2,$$(subst ^
, ,$$(@
)))_STEP_TO_PROG
,$$(word 3,$$(subst ^
, ,$$(@
))))
414 @
$(foreach impl
,$(word 2,$(subst ^
, ,$(@
))),\
415 $(foreach step
,$(word 3,$(subst ^
, ,$(@
))),\
416 cd
$(if
$(filter mal
,$(impl
)),$(MAL_IMPL
),$(impl
)); \
417 echo
'REPL implementation $(impl), step file: $+'; \
418 echo
'Running: $(call get_run_prefix,$(impl),$(step)) ../$(impl)/run'; \
419 $(call get_run_prefix
,$(impl
),$(step
)) ..
/$(impl
)/run
;))
421 # Allow repl^IMPL^STEP and repl^IMPL (which starts REPL of stepA)
423 $(IMPL_REPL
): $$@^stepA
433 # Recursive rules (call make FOO in each subdirectory)
436 define recur_template
441 @echo
"----------------------------------------------"; \
442 $$(foreach impl
,$$(word 2,$$(subst ^
, ,$$(@
))),\
443 $$(if
$$(DOCKERIZE
), \
444 echo
"Running: $$(call get_build_prefix,$$(impl))$$(MAKE) --no-print-directory $(1)"; \
445 $$(call get_build_prefix
,$$(impl
))$$(MAKE
) --no-print-directory
$(1), \
446 echo
"Running: $$(MAKE) --no-print-directory -C $$(impl) $(1)"; \
447 $$(MAKE
) --no-print-directory
-C
$$(impl
) $(1)))
450 recur_impls_
= $(filter-out $(foreach impl
,$($(1)_EXCLUDES
),$(1)^
$(impl
)),$(foreach impl
,$(IMPLS
),$(1)^
$(impl
)))
453 $(eval
$(call recur_template
,clean,$(call recur_impls_
,clean)))
456 $(eval
$(call recur_template
,stats
,$(call recur_impls_
,stats
)))
457 $(eval
$(call recur_template
,stats-lisp
,$(call recur_impls_
,stats-lisp
)))
460 $(eval
$(call recur_template
,dist,$(call recur_impls_
,dist)))