8 @echo
'make "IMPL" # build all steps of IMPL'
9 @echo
'make "IMPL^STEP" # build STEP of IMPL'
11 @echo
'make "test" # test all implementations'
12 @echo
'make "test^IMPL" # test all steps of IMPL'
13 @echo
'make "test^STEP" # test STEP for all implementations'
14 @echo
'make "test^IMPL^STEP" # test STEP of IMPL'
16 @echo
'make "perf" # run microbenchmarks for all implementations'
17 @echo
'make "perf^IMPL" # run microbenchmarks for IMPL'
19 @echo
'make "repl^IMPL" # run stepA of IMPL'
20 @echo
'make "repl^IMPL^STEP" # test STEP of IMPL'
22 @echo
'make "clean" # run 'make
clean' for all implementations'
23 @echo
'make "clean^IMPL" # run 'make
clean' for IMPL'
25 @echo
'make "stats" # run 'make stats
' for all implementations'
26 @echo
'make "stats-lisp" # run 'make stats-lisp
' for all implementations'
27 @echo
'make "stats^IMPL" # run 'make stats
' for IMPL'
28 @echo
'make "stats-lisp^IMPL" # run 'make stats-lisp
' for IMPL'
30 @echo
'Options/Settings:'
32 @echo
'make MAL_IMPL=IMPL "test^mal..." # use IMPL for self-host tests'
33 @echo
'make REGRESS=1 "test..." # test with previous step tests too'
34 @echo
'make DOCKERIZE=1 ... # to dockerize above rules/targets'
38 @echo
'make "docker-build^IMPL" # build docker image for IMPL'
42 # Command line settings
49 # python, js, cpp, or neko are currently supported
51 # clj or cljs are currently supported (Clojure vs ClojureScript/lumo)
54 # Extra options to pass to runtest.py
57 # Test with previous test files not just the test files for the
58 # current step. Step 0 and 1 tests are special and not included in
65 # Extra implementation specific options to pass to runtest.py
66 logo_TEST_OPTS
= --start-timeout
60 --test-timeout
120
67 mal_TEST_OPTS
= --start-timeout
60 --test-timeout
120
68 miniMAL_TEST_OPTS
= --start-timeout
60 --test-timeout
120
69 plpgsql_TEST_OPTS
= --start-timeout
60 --test-timeout
180
70 plsql_TEST_OPTS
= --start-timeout
120 --test-timeout
120
71 perl6_TEST_OPTS
= --test-timeout
=60
73 # Run target/rule within docker image for the implementation
80 IMPLS
= ada awk bash basic c d chuck clojure coffee common-lisp
cpp crystal cs dart \
81 erlang elisp elixir es6 factor forth fsharp go groovy guile haskell \
82 haxe io java julia js kotlin logo lua make mal ocaml matlab miniMAL \
83 nim objc objpascal perl perl6 php pil plpgsql plsql powershell ps \
84 python r racket rpython ruby rust scala skew swift swift3 tcl ts vb vhdl \
85 vimscript livescript elm
90 step1
= step1_read_print
93 step4
= step4_if_fn_do
101 argv_STEP
= step6_file
104 regress_step0
= step0
105 regress_step1
= step1
106 regress_step2
= step2
107 regress_step3
= $(regress_step2
) step3
108 regress_step4
= $(regress_step3
) step4
109 regress_step5
= $(regress_step4
) step5
110 regress_step6
= $(regress_step5
) step6
111 regress_step7
= $(regress_step6
) step7
112 regress_step8
= $(regress_step7
) step8
113 regress_step9
= $(regress_step8
) step9
114 regress_stepA
= $(regress_step9
) stepA
116 test_EXCLUDES
+= test^bash^step5
# never completes at 10,000
117 test_EXCLUDES
+= test^basic^step5
# too slow, and limited to ints of 2^16
118 test_EXCLUDES
+= test^logo^step5
# too slow for 10,000
119 test_EXCLUDES
+= test^make^step5
# no TCO capability (iteration or recursion)
120 test_EXCLUDES
+= test^mal^step5
# host impl dependent
121 test_EXCLUDES
+= test^matlab^step5
# never completes at 10,000
122 test_EXCLUDES
+= test^plpgsql^step5
# too slow for 10,000
123 test_EXCLUDES
+= test^plsql^step5
# too slow for 10,000
124 test_EXCLUDES
+= test^powershell^step5
# too slow for 10,000
126 perf_EXCLUDES
= mal
# TODO: fix this
129 # TODO: still need to implement dist
130 dist_EXCLUDES
+= guile io julia matlab swift
136 haxe_STEP_TO_PROG_neko
= haxe
/$($(1)).n
137 haxe_STEP_TO_PROG_python
= haxe
/$($(1)).py
138 haxe_STEP_TO_PROG_cpp
= haxe
/cpp/$($(1))
139 haxe_STEP_TO_PROG_js
= haxe
/$($(1)).js
141 clojure_STEP_TO_PROG_clj
= clojure
/target
/$($(1)).jar
142 clojure_STEP_TO_PROG_cljs
= clojure
/src
/mal
/$($(1)).cljc
144 opt_DEFERRABLE
= $(if
$(strip $(DEFERRABLE
)),$(if
$(filter t true T True TRUE
1 y yes Yes YES
,$(DEFERRABLE
)),--deferrable
,--no-deferrable
),--no-deferrable
)
145 opt_OPTIONAL
= $(if
$(strip $(OPTIONAL
)),$(if
$(filter t true T True TRUE
1 y yes Yes YES
,$(OPTIONAL
)),--optional
,--no-optional
),--no-optional
)
147 # Return list of test files for a given step. If REGRESS is set then
148 # test files will include step 2 tests through tests for the step
150 STEP_TEST_FILES
= $(strip $(wildcard \
151 $(foreach s
,$(if
$(strip $(REGRESS
)),$(regress_
$(2)),$(2)),\
152 $(1)/tests
/$($(s
))$(EXTENSION
) tests
/$($(s
))$(EXTENSION
))))
154 # Map of step (e.g. "step8") to executable file for that step
155 ada_STEP_TO_PROG
= ada
/$($(1))
156 awk_STEP_TO_PROG
= awk
/$($(1)).awk
157 bash_STEP_TO_PROG
= bash
/$($(1)).sh
158 basic_STEP_TO_PROG
= basic
/$($(1)).bas
159 c_STEP_TO_PROG
= c
/$($(1))
160 d_STEP_TO_PROG
= d
/$($(1))
161 chuck_STEP_TO_PROG
= chuck
/$($(1)).ck
162 clojure_STEP_TO_PROG
= $(clojure_STEP_TO_PROG_
$(CLJ_MODE
))
163 coffee_STEP_TO_PROG
= coffee
/$($(1)).coffee
164 common-lisp_STEP_TO_PROG
= common-lisp
/$($(1))
165 cpp_STEP_TO_PROG
= cpp/$($(1))
166 crystal_STEP_TO_PROG
= crystal
/$($(1))
167 cs_STEP_TO_PROG
= cs
/$($(1)).exe
168 dart_STEP_TO_PROG
= dart
/$($(1)).dart
169 elisp_STEP_TO_PROG
= elisp
/$($(1)).el
170 elixir_STEP_TO_PROG
= elixir
/lib
/mix
/tasks
/$($(1)).ex
171 erlang_STEP_TO_PROG
= erlang
/$($(1))
172 es6_STEP_TO_PROG
= es6
/build
/$($(1)).js
173 factor_STEP_TO_PROG
= factor
/$($(1))/$($(1)).factor
174 forth_STEP_TO_PROG
= forth
/$($(1)).fs
175 fsharp_STEP_TO_PROG
= fsharp
/$($(1)).exe
176 go_STEP_TO_PROG
= go
/$($(1))
177 groovy_STEP_TO_PROG
= groovy
/$($(1)).groovy
178 java_STEP_TO_PROG
= java
/target
/classes
/mal
/$($(1)).class
179 haskell_STEP_TO_PROG
= haskell
/$($(1))
180 haxe_STEP_TO_PROG
= $(haxe_STEP_TO_PROG_
$(HAXE_MODE
))
181 io_STEP_TO_PROG
= io
/$($(1)).io
182 julia_STEP_TO_PROG
= julia
/$($(1)).jl
183 js_STEP_TO_PROG
= js
/$($(1)).js
184 kotlin_STEP_TO_PROG
= kotlin
/$($(1)).jar
185 lua_STEP_TO_PROG
= lua
/$($(1)).lua
186 make_STEP_TO_PROG
= make
/$($(1)).mk
187 mal_STEP_TO_PROG
= mal
/$($(1)).mal
188 ocaml_STEP_TO_PROG
= ocaml
/$($(1))
189 matlab_STEP_TO_PROG
= matlab
/$($(1)).m
190 miniMAL_STEP_TO_PROG
= miniMAL
/$($(1)).json
191 nim_STEP_TO_PROG
= nim
/$($(1))
192 objc_STEP_TO_PROG
= objc
/$($(1))
193 objpascal_STEP_TO_PROG
= objpascal
/$($(1))
194 perl_STEP_TO_PROG
= perl
/$($(1)).pl
195 perl6_STEP_TO_PROG
= perl6
/$($(1)).pl
196 php_STEP_TO_PROG
= php
/$($(1)).php
197 pil_STEP_TO_PROG
= pil
/$($(1)).l
198 plpgsql_STEP_TO_PROG
= plpgsql
/$($(1)).sql
199 plsql_STEP_TO_PROG
= plsql
/$($(1)).sql
200 powershell_STEP_TO_PROG
= powershell
/$($(1)).ps1
201 ps_STEP_TO_PROG
= ps
/$($(1)).ps
202 python_STEP_TO_PROG
= python
/$($(1)).py
203 r_STEP_TO_PROG
= r
/$($(1)).r
204 racket_STEP_TO_PROG
= racket
/$($(1)).rkt
205 rpython_STEP_TO_PROG
= rpython
/$($(1))
206 ruby_STEP_TO_PROG
= ruby
/$($(1)).rb
207 rust_STEP_TO_PROG
= rust
/target
/release
/$($(1))
208 scala_STEP_TO_PROG
= scala
/target
/scala-2.11
/classes
/$($(1)).class
209 skew_STEP_TO_PROG
= skew
/$($(1)).js
210 swift_STEP_TO_PROG
= swift
/$($(1))
211 swift3_STEP_TO_PROG
= swift3
/$($(1))
212 tcl_STEP_TO_PROG
= tcl
/$($(1)).tcl
213 ts_STEP_TO_PROG
= ts
/$($(1)).js
214 vb_STEP_TO_PROG
= vb
/$($(1)).exe
215 vhdl_STEP_TO_PROG
= vhdl
/$($(1))
216 vimscript_STEP_TO_PROG
= vimscript
/$($(1)).vim
217 guile_STEP_TO_PROG
= guile
/$($(1)).scm
218 livescript_STEP_TO_PROG
= livescript
/$($(1)).js
219 elm_STEP_TO_PROG
= elm
/$($(1)).js
222 # Needed some argument munging
225 SPACE
= $(noop
) $(noop
)
226 export FACTOR_ROOTS
:= .
228 # DOCKERIZE utility functions
229 lc
= $(subst A
,a
,$(subst B
,b
,$(subst C
,c
,$(subst D
,d
,$(subst E
,e
,$(subst F
,f
,$(subst G
,g
,$(subst H
,h
,$(subst I
,i
,$(subst J
,j
,$(subst K
,k
,$(subst L
,l
,$(subst M
,m
,$(subst N
,n
,$(subst O
,o
,$(subst P
,p
,$(subst Q
,q
,$(subst R
,r
,$(subst S
,s
,$(subst T
,t
,$(subst U
,u
,$(subst V
,v
,$(subst W
,w
,$(subst X
,x
,$(subst Y
,y
,$(subst Z
,z
,$1))))))))))))))))))))))))))
230 impl_to_image
= kanaka
/mal-test-
$(call lc
,$(1))
232 actual_impl
= $(if
$(filter mal
,$(1)),$(MAL_IMPL
),$(1))
235 # Returns nothing if DOCKERIZE is not set, otherwise returns the
236 # docker prefix necessary to run make within the docker environment
238 get_build_prefix
= $(if
$(strip $(DOCKERIZE
)),docker run
-it
--rm -u
$(shell id
-u
) -v
$(dir $(abspath
$(lastword
$(MAKEFILE_LIST
)))):/mal
-w
/mal
/$(1) $(if
$(filter factor
,$(1)),-e FACTOR_ROOTS
=$(FACTOR_ROOTS
),) $(call impl_to_image
,$(1)) ,)
240 # Takes impl and step arguments
241 # Returns a command prefix (docker command and environment variables)
242 # necessary to launch the given impl and step
243 get_run_prefix
= $(strip $(if
$(strip $(DOCKERIZE
)),\
244 docker run
-e STEP
=$($2) -e MAL_IMPL
=$(MAL_IMPL
) \
245 -it
--rm -u
$(shell id
-u
) \
246 -v
$(dir $(abspath
$(lastword
$(MAKEFILE_LIST
)))):/mal \
247 -w
/mal
/$(call actual_impl
,$(1)) \
248 $(if
$(filter clojure
,$(1)),-e CLJ_MODE
=$(CLJ_MODE
),) \
249 $(if
$(filter haxe
,$(1)),-e HAXE_MODE
=$(HAXE_MODE
),) \
250 $(if
$(filter factor
,$(1)),-e FACTOR_ROOTS
=$(FACTOR_ROOTS
),) \
251 $(foreach env
,$(3),-e
$(env
)) \
252 $(call impl_to_image
,$(call actual_impl
,$(1))) \
254 env STEP
=$($2) MAL_IMPL
=$(MAL_IMPL
) \
255 $(if
$(filter haxe
,$(1)),HAXE_MODE
=$(HAXE_MODE
),) \
256 $(if
$(filter factor
,$(1)),FACTOR_ROOTS
=$(FACTOR_ROOTS
),) \
259 # Takes impl and step
260 # Returns the runtest command prefix (with runtest options) for testing the given step
261 get_runtest_cmd
= $(call get_run_prefix
,$(1),$(2),$(if
$(filter cs fsharp tcl vb
,$(1)),RAW
=1,)) \
262 ..
/runtest.py
$(opt_DEFERRABLE
) $(opt_OPTIONAL
) $(call
$(1)_TEST_OPTS
) $(TEST_OPTS
)
264 # Takes impl and step
265 # Returns the runtest command prefix (with runtest options) for testing the given step
266 get_argvtest_cmd
= $(call get_run_prefix
,$(1),$(2)) ..
/run_argv_test.sh
268 vimscript_TEST_OPTS
= --test-timeout
30
269 ifeq ($(MAL_IMPL
),vimscript
)
270 mal_TEST_OPTS
= --start-timeout
60 --test-timeout
180
271 else ifeq ($(MAL_IMPL
),powershell
)
272 mal_TEST_OPTS
= --start-timeout
60 --test-timeout
180
276 STEPS
= $(sort $(filter step
%,$(.VARIABLES
)))
277 DO_IMPLS
= $(filter-out $(SKIP_IMPLS
),$(IMPLS
))
278 IMPL_TESTS
= $(foreach impl
,$(DO_IMPLS
),test^
$(impl
))
279 STEP_TESTS
= $(foreach step
,$(STEPS
),test^
$(step
))
280 ALL_TESTS
= $(filter-out $(test_EXCLUDES
),\
282 $(foreach impl
,$(DO_IMPLS
),\
283 $(foreach step
,$(STEPS
),test^
$(impl
)^
$(step
))))))
285 DOCKER_BUILD
= $(foreach impl
,$(DO_IMPLS
),docker-build^
$(impl
))
287 IMPL_PERF
= $(foreach impl
,$(filter-out $(perf_EXCLUDES
),$(DO_IMPLS
)),perf^
$(impl
))
289 IMPL_REPL
= $(foreach impl
,$(DO_IMPLS
),repl^
$(impl
))
290 ALL_REPL
= $(strip $(sort \
291 $(foreach impl
,$(DO_IMPLS
),\
292 $(foreach step
,$(STEPS
),repl^
$(impl
)^
$(step
)))))
298 # Build a program in an implementation directory
299 # Make sure we always try and build first because the dependencies are
300 # encoded in the implementation Makefile not here
301 .PHONY
: $(foreach i
,$(DO_IMPLS
),$(foreach s
,$(STEPS
),$(call
$(i
)_STEP_TO_PROG
,$(s
))))
302 $(foreach i
,$(DO_IMPLS
),$(foreach s
,$(STEPS
),$(call
$(i
)_STEP_TO_PROG
,$(s
)))):
303 $(foreach impl
,$(word 1,$(subst /, ,$(@
))),\
305 $(call get_build_prefix
,$(impl
))$(MAKE
) $(patsubst $(impl
)/%,%,$(@
)), \
306 $(MAKE
) -C
$(impl
) $(subst $(impl
)/,,$(@
))))
308 # Allow IMPL, and IMPL^STEP
310 $(DO_IMPLS
): $$(foreach s
,$$(STEPS
),$$(call
$$(@
)_STEP_TO_PROG
,$$(s
)))
313 $(foreach i
,$(DO_IMPLS
),$(foreach s
,$(STEPS
),$(i
)^
$(s
))): $$(call
$$(word 1,$$(subst ^
, ,$$(@
)))_STEP_TO_PROG
,$$(word 2,$$(subst ^
, ,$$(@
))))
321 $(ALL_TESTS
): $$(call
$$(word 2,$$(subst ^
, ,$$(@
)))_STEP_TO_PROG
,$$(word 3,$$(subst ^
, ,$$(@
))))
322 @
$(foreach impl
,$(word 2,$(subst ^
, ,$(@
))),\
323 $(foreach step
,$(word 3,$(subst ^
, ,$(@
))),\
324 cd
$(if
$(filter mal
,$(impl
)),$(MAL_IMPL
),$(impl
)) && \
325 $(foreach test,$(call STEP_TEST_FILES
,$(impl
),$(step
)),\
326 echo
'----------------------------------------------' && \
327 echo
'Testing $@; step file: $+, test file: $(test)' && \
328 echo
'Running: $(call get_runtest_cmd,$(impl),$(step)) ../$(test) -- ../$(impl)/run' && \
329 $(call get_runtest_cmd
,$(impl
),$(step
)) ..
/$(test) -- ..
/$(impl
)/run
&& \
330 $(if
$(filter tests
/$(argv_STEP
)$(EXTENSION
),$(test)),\
331 echo
'----------------------------------------------' && \
332 echo
'Testing ARGV of $@; step file: $+' && \
333 echo
'Running: $(call get_argvtest_cmd,$(impl),$(step)) ../$(impl)/run ' && \
334 $(call get_argvtest_cmd
,$(impl
),$(step
)) ..
/$(impl
)/run
&& ,\
338 # Allow test, tests, test^STEP, test^IMPL, and test^IMPL^STEP
343 $(IMPL_TESTS
): $$(filter $$@^
%,$$(ALL_TESTS
))
346 $(STEP_TESTS
): $$(foreach step
,$$(subst test^
,,$$@
),$$(filter %^
$$(step
),$$(ALL_TESTS
)))
357 @echo
"----------------------------------------------"; \
358 $(foreach impl
,$(word 2,$(subst ^
, ,$(@
))),\
359 echo
"Running: make -C $(impl) dist"; \
360 $(MAKE
) --no-print-directory
-C
$(impl
) dist)
367 docker-build
: $(DOCKER_BUILD
)
371 echo
"----------------------------------------------"; \
372 $(foreach impl
,$(word 2,$(subst ^
, ,$(@
))),\
373 echo
"Running: docker build -t $(call impl_to_image,$(impl)) .:"; \
374 cd
$(impl
) && docker build
-t
$(call impl_to_image
,$(impl
)) .
)
378 # Performance test rules
385 @echo
"----------------------------------------------"; \
386 $(foreach impl
,$(word 2,$(subst ^
, ,$(@
))),\
387 cd
$(if
$(filter mal
,$(impl
)),$(MAL_IMPL
),$(impl
)); \
388 echo
"Performance test for $(impl):"; \
389 echo
'Running: $(call get_run_prefix,$(impl),stepA) ../$(impl)/run ../tests/perf1.mal'; \
390 $(call get_run_prefix
,$(impl
),stepA
) ..
/$(impl
)/run ..
/tests
/perf1.mal
; \
391 echo
'Running: $(call get_run_prefix,$(impl),stepA) ../$(impl)/run ../tests/perf2.mal'; \
392 $(call get_run_prefix
,$(impl
),stepA
) ..
/$(impl
)/run ..
/tests
/perf2.mal
; \
393 echo
'Running: $(call get_run_prefix,$(impl),stepA) ../$(impl)/run ../tests/perf3.mal'; \
394 $(call get_run_prefix
,$(impl
),stepA
) ..
/$(impl
)/run ..
/tests
/perf3.mal
)
398 # REPL invocation rules
402 $(ALL_REPL
): $$(call
$$(word 2,$$(subst ^
, ,$$(@
)))_STEP_TO_PROG
,$$(word 3,$$(subst ^
, ,$$(@
))))
403 @
$(foreach impl
,$(word 2,$(subst ^
, ,$(@
))),\
404 $(foreach step
,$(word 3,$(subst ^
, ,$(@
))),\
405 cd
$(if
$(filter mal
,$(impl
)),$(MAL_IMPL
),$(impl
)); \
406 echo
'REPL implementation $(impl), step file: $+'; \
407 echo
'Running: $(call get_run_prefix,$(impl),$(step)) ../$(impl)/run'; \
408 $(call get_run_prefix
,$(impl
),$(step
)) ..
/$(impl
)/run
;))
410 # Allow repl^IMPL^STEP and repl^IMPL (which starts REPL of stepA)
412 $(IMPL_REPL
): $$@^stepA
422 # Recursive rules (call make FOO in each subdirectory)
425 define recur_template
430 @echo
"----------------------------------------------"; \
431 $$(foreach impl
,$$(word 2,$$(subst ^
, ,$$(@
))),\
432 $$(if
$$(DOCKERIZE
), \
433 echo
"Running: $$(call get_build_prefix,$$(impl))$$(MAKE) --no-print-directory $(1)"; \
434 $$(call get_build_prefix
,$$(impl
))$$(MAKE
) --no-print-directory
$(1), \
435 echo
"Running: $$(MAKE) --no-print-directory -C $$(impl) $(1)"; \
436 $$(MAKE
) --no-print-directory
-C
$$(impl
) $(1)))
439 recur_impls_
= $(filter-out $(foreach impl
,$($(1)_EXCLUDES
),$(1)^
$(impl
)),$(foreach impl
,$(IMPLS
),$(1)^
$(impl
)))
442 $(eval
$(call recur_template
,clean,$(call recur_impls_
,clean)))
445 $(eval
$(call recur_template
,stats
,$(call recur_impls_
,stats
)))
446 $(eval
$(call recur_template
,stats-lisp
,$(call recur_impls_
,stats-lisp
)))
449 $(eval
$(call recur_template
,dist,$(call recur_impls_
,dist)))