Commit | Line | Data |
---|---|---|
15c9d04e | 1 | ;;; ert.el --- Emacs Lisp Regression Testing -*- lexical-binding: t -*- |
d221e780 | 2 | |
ba318903 | 3 | ;; Copyright (C) 2007-2008, 2010-2014 Free Software Foundation, Inc. |
d221e780 CO |
4 | |
5 | ;; Author: Christian Ohler <ohler@gnu.org> | |
6 | ;; Keywords: lisp, tools | |
7 | ||
8 | ;; This file is part of GNU Emacs. | |
9 | ||
74bfe42a GM |
10 | ;; GNU Emacs is free software: you can redistribute it and/or modify |
11 | ;; it under the terms of the GNU General Public License as published by | |
12 | ;; the Free Software Foundation, either version 3 of the License, or | |
13 | ;; (at your option) any later version. | |
14 | ||
15 | ;; GNU Emacs is distributed in the hope that it will be useful, | |
16 | ;; but WITHOUT ANY WARRANTY; without even the implied warranty of | |
17 | ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
18 | ;; GNU General Public License for more details. | |
19 | ||
d221e780 | 20 | ;; You should have received a copy of the GNU General Public License |
74bfe42a | 21 | ;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>. |
d221e780 CO |
22 | |
23 | ;;; Commentary: | |
24 | ||
25 | ;; ERT is a tool for automated testing in Emacs Lisp. Its main | |
26 | ;; features are facilities for defining and running test cases and | |
27 | ;; reporting the results as well as for debugging test failures | |
28 | ;; interactively. | |
29 | ;; | |
30 | ;; The main entry points are `ert-deftest', which is similar to | |
31 | ;; `defun' but defines a test, and `ert-run-tests-interactively', | |
32 | ;; which runs tests and offers an interactive interface for inspecting | |
33 | ;; results and debugging. There is also | |
34 | ;; `ert-run-tests-batch-and-exit' for non-interactive use. | |
35 | ;; | |
36 | ;; The body of `ert-deftest' forms resembles a function body, but the | |
50b5b857 MA |
37 | ;; additional operators `should', `should-not', `should-error' and |
38 | ;; `skip-unless' are available. `should' is similar to cl's `assert', | |
39 | ;; but signals a different error when its condition is violated that | |
40 | ;; is caught and processed by ERT. In addition, it analyzes its | |
41 | ;; argument form and records information that helps debugging | |
42 | ;; (`assert' tries to do something similar when its second argument | |
43 | ;; SHOW-ARGS is true, but `should' is more sophisticated). For | |
44 | ;; information on `should-not' and `should-error', see their | |
45 | ;; docstrings. `skip-unless' skips the test immediately without | |
46 | ;; processing further, this is useful for checking the test | |
47 | ;; environment (like availability of features, external binaries, etc). | |
d221e780 CO |
48 | ;; |
49 | ;; See ERT's info manual as well as the docstrings for more details. | |
50 | ;; To compile the manual, run `makeinfo ert.texinfo' in the ERT | |
51 | ;; directory, then C-u M-x info ert.info in Emacs to view it. | |
52 | ;; | |
53 | ;; To see some examples of tests written in ERT, see its self-tests in | |
54 | ;; ert-tests.el. Some of these are tricky due to the bootstrapping | |
55 | ;; problem of writing tests for a testing tool, others test simple | |
56 | ;; functions and are straightforward. | |
57 | ||
58 | ;;; Code: | |
59 | ||
a19b3c2d | 60 | (require 'cl-lib) |
d221e780 CO |
61 | (require 'button) |
62 | (require 'debug) | |
63 | (require 'easymenu) | |
64 | (require 'ewoc) | |
65 | (require 'find-func) | |
66 | (require 'help) | |
67 | ||
68 | ||
69 | ;;; UI customization options. | |
70 | ||
71 | (defgroup ert () | |
72 | "ERT, the Emacs Lisp regression testing tool." | |
73 | :prefix "ert-" | |
74 | :group 'lisp) | |
75 | ||
76 | (defface ert-test-result-expected '((((class color) (background light)) | |
77 | :background "green1") | |
78 | (((class color) (background dark)) | |
79 | :background "green3")) | |
80 | "Face used for expected results in the ERT results buffer." | |
81 | :group 'ert) | |
82 | ||
83 | (defface ert-test-result-unexpected '((((class color) (background light)) | |
84 | :background "red1") | |
85 | (((class color) (background dark)) | |
86 | :background "red3")) | |
87 | "Face used for unexpected results in the ERT results buffer." | |
88 | :group 'ert) | |
89 | ||
90 | ||
91 | ;;; Copies/reimplementations of cl functions. | |
92 | ||
d221e780 CO |
93 | (defun ert-equal-including-properties (a b) |
94 | "Return t if A and B have similar structure and contents. | |
95 | ||
96 | This is like `equal-including-properties' except that it compares | |
97 | the property values of text properties structurally (by | |
98 | recursing) rather than with `eq'. Perhaps this is what | |
99 | `equal-including-properties' should do in the first place; see | |
100 | Emacs bug 6581 at URL `http://debbugs.gnu.org/cgi/bugreport.cgi?bug=6581'." | |
101 | ;; This implementation is inefficient. Rather than making it | |
102 | ;; efficient, let's hope bug 6581 gets fixed so that we can delete | |
103 | ;; it altogether. | |
de69c0a8 | 104 | (not (ert--explain-equal-including-properties a b))) |
d221e780 CO |
105 | |
106 | ||
107 | ;;; Defining and locating tests. | |
108 | ||
109 | ;; The data structure that represents a test case. | |
15c9d04e | 110 | (cl-defstruct ert-test |
d221e780 CO |
111 | (name nil) |
112 | (documentation nil) | |
15c9d04e | 113 | (body (cl-assert nil)) |
d221e780 CO |
114 | (most-recent-result nil) |
115 | (expected-result-type ':passed) | |
116 | (tags '())) | |
117 | ||
118 | (defun ert-test-boundp (symbol) | |
119 | "Return non-nil if SYMBOL names a test." | |
120 | (and (get symbol 'ert--test) t)) | |
121 | ||
122 | (defun ert-get-test (symbol) | |
123 | "If SYMBOL names a test, return that. Signal an error otherwise." | |
124 | (unless (ert-test-boundp symbol) (error "No test named `%S'" symbol)) | |
125 | (get symbol 'ert--test)) | |
126 | ||
127 | (defun ert-set-test (symbol definition) | |
128 | "Make SYMBOL name the test DEFINITION, and return DEFINITION." | |
129 | (when (eq symbol 'nil) | |
130 | ;; We disallow nil since `ert-test-at-point' and related functions | |
131 | ;; want to return a test name, but also need an out-of-band value | |
132 | ;; on failure. Nil is the most natural out-of-band value; using 0 | |
8350f087 | 133 | ;; or "" or signaling an error would be too awkward. |
d221e780 CO |
134 | ;; |
135 | ;; Note that nil is still a valid value for the `name' slot in | |
136 | ;; ert-test objects. It designates an anonymous test. | |
137 | (error "Attempt to define a test named nil")) | |
138 | (put symbol 'ert--test definition) | |
139 | definition) | |
140 | ||
141 | (defun ert-make-test-unbound (symbol) | |
142 | "Make SYMBOL name no test. Return SYMBOL." | |
a19b3c2d | 143 | (cl-remprop symbol 'ert--test) |
d221e780 CO |
144 | symbol) |
145 | ||
146 | (defun ert--parse-keys-and-body (keys-and-body) | |
147 | "Split KEYS-AND-BODY into keyword-and-value pairs and the remaining body. | |
148 | ||
149 | KEYS-AND-BODY should have the form of a property list, with the | |
150 | exception that only keywords are permitted as keys and that the | |
151 | tail -- the body -- is a list of forms that does not start with a | |
152 | keyword. | |
153 | ||
154 | Returns a two-element list containing the keys-and-values plist | |
155 | and the body." | |
156 | (let ((extracted-key-accu '()) | |
157 | (remaining keys-and-body)) | |
15c9d04e | 158 | (while (keywordp (car-safe remaining)) |
d221e780 CO |
159 | (let ((keyword (pop remaining))) |
160 | (unless (consp remaining) | |
161 | (error "Value expected after keyword %S in %S" | |
162 | keyword keys-and-body)) | |
163 | (when (assoc keyword extracted-key-accu) | |
164 | (warn "Keyword %S appears more than once in %S" keyword | |
165 | keys-and-body)) | |
166 | (push (cons keyword (pop remaining)) extracted-key-accu))) | |
167 | (setq extracted-key-accu (nreverse extracted-key-accu)) | |
15c9d04e SM |
168 | (list (cl-loop for (key . value) in extracted-key-accu |
169 | collect key | |
170 | collect value) | |
d221e780 CO |
171 | remaining))) |
172 | ||
173 | ;;;###autoload | |
15c9d04e | 174 | (cl-defmacro ert-deftest (name () &body docstring-keys-and-body) |
d221e780 CO |
175 | "Define NAME (a symbol) as a test. |
176 | ||
177 | BODY is evaluated as a `progn' when the test is run. It should | |
178 | signal a condition on failure or just return if the test passes. | |
179 | ||
50b5b857 MA |
180 | `should', `should-not', `should-error' and `skip-unless' are |
181 | useful for assertions in BODY. | |
d221e780 CO |
182 | |
183 | Use `ert' to run tests interactively. | |
184 | ||
185 | Tests that are expected to fail can be marked as such | |
186 | using :expected-result. See `ert-test-result-type-p' for a | |
187 | description of valid values for RESULT-TYPE. | |
188 | ||
189 | \(fn NAME () [DOCSTRING] [:expected-result RESULT-TYPE] \ | |
190 | \[:tags '(TAG...)] BODY...)" | |
191 | (declare (debug (&define :name test | |
192 | name sexp [&optional stringp] | |
193 | [&rest keywordp sexp] def-body)) | |
194 | (doc-string 3) | |
195 | (indent 2)) | |
196 | (let ((documentation nil) | |
197 | (documentation-supplied-p nil)) | |
15c9d04e | 198 | (when (stringp (car docstring-keys-and-body)) |
d221e780 CO |
199 | (setq documentation (pop docstring-keys-and-body) |
200 | documentation-supplied-p t)) | |
15c9d04e SM |
201 | (cl-destructuring-bind |
202 | ((&key (expected-result nil expected-result-supplied-p) | |
203 | (tags nil tags-supplied-p)) | |
204 | body) | |
d221e780 | 205 | (ert--parse-keys-and-body docstring-keys-and-body) |
50b5b857 | 206 | `(cl-macrolet ((skip-unless (form) `(ert--skip-unless ,form))) |
d221e780 CO |
207 | (ert-set-test ',name |
208 | (make-ert-test | |
209 | :name ',name | |
210 | ,@(when documentation-supplied-p | |
211 | `(:documentation ,documentation)) | |
212 | ,@(when expected-result-supplied-p | |
213 | `(:expected-result-type ,expected-result)) | |
214 | ,@(when tags-supplied-p | |
215 | `(:tags ,tags)) | |
216 | :body (lambda () ,@body))) | |
217 | ;; This hack allows `symbol-file' to associate `ert-deftest' | |
218 | ;; forms with files, and therefore enables `find-function' to | |
219 | ;; work with tests. However, it leads to warnings in | |
220 | ;; `unload-feature', which doesn't know how to undefine tests | |
221 | ;; and has no mechanism for extension. | |
222 | (push '(ert-deftest . ,name) current-load-list) | |
223 | ',name)))) | |
224 | ||
225 | ;; We use these `put' forms in addition to the (declare (indent)) in | |
226 | ;; the defmacro form since the `declare' alone does not lead to | |
227 | ;; correct indentation before the .el/.elc file is loaded. | |
228 | ;; Autoloading these `put' forms solves this. | |
229 | ;;;###autoload | |
230 | (progn | |
231 | ;; TODO(ohler): Figure out what these mean and make sure they are correct. | |
232 | (put 'ert-deftest 'lisp-indent-function 2) | |
233 | (put 'ert-info 'lisp-indent-function 1)) | |
234 | ||
235 | (defvar ert--find-test-regexp | |
236 | (concat "^\\s-*(ert-deftest" | |
237 | find-function-space-re | |
238 | "%s\\(\\s-\\|$\\)") | |
239 | "The regexp the `find-function' mechanisms use for finding test definitions.") | |
240 | ||
241 | ||
54bd972f | 242 | (define-error 'ert-test-failed "Test failed") |
50b5b857 | 243 | (define-error 'ert-test-skipped "Test skipped") |
d221e780 CO |
244 | |
245 | (defun ert-pass () | |
246 | "Terminate the current test and mark it passed. Does not return." | |
247 | (throw 'ert--pass nil)) | |
248 | ||
249 | (defun ert-fail (data) | |
250 | "Terminate the current test and mark it failed. Does not return. | |
251 | DATA is displayed to the user and should state the reason of the failure." | |
252 | (signal 'ert-test-failed (list data))) | |
253 | ||
50b5b857 MA |
254 | (defun ert-skip (data) |
255 | "Terminate the current test and mark it skipped. Does not return. | |
256 | DATA is displayed to the user and should state the reason for skipping." | |
257 | (signal 'ert-test-skipped (list data))) | |
258 | ||
d221e780 CO |
259 | |
260 | ;;; The `should' macros. | |
261 | ||
262 | (defvar ert--should-execution-observer nil) | |
263 | ||
264 | (defun ert--signal-should-execution (form-description) | |
265 | "Tell the current `should' form observer (if any) about FORM-DESCRIPTION." | |
266 | (when ert--should-execution-observer | |
267 | (funcall ert--should-execution-observer form-description))) | |
268 | ||
269 | (defun ert--special-operator-p (thing) | |
270 | "Return non-nil if THING is a symbol naming a special operator." | |
271 | (and (symbolp thing) | |
272 | (let ((definition (indirect-function thing t))) | |
273 | (and (subrp definition) | |
274 | (eql (cdr (subr-arity definition)) 'unevalled))))) | |
275 | ||
276 | (defun ert--expand-should-1 (whole form inner-expander) | |
277 | "Helper function for the `should' macro and its variants." | |
278 | (let ((form | |
bc715d67 SM |
279 | (macroexpand form (cond |
280 | ((boundp 'macroexpand-all-environment) | |
281 | macroexpand-all-environment) | |
282 | ((boundp 'cl-macro-environment) | |
283 | cl-macro-environment))))) | |
d221e780 CO |
284 | (cond |
285 | ((or (atom form) (ert--special-operator-p (car form))) | |
a19b3c2d GM |
286 | (let ((value (cl-gensym "value-"))) |
287 | `(let ((,value (cl-gensym "ert-form-evaluation-aborted-"))) | |
d221e780 CO |
288 | ,(funcall inner-expander |
289 | `(setq ,value ,form) | |
290 | `(list ',whole :form ',form :value ,value) | |
291 | value) | |
292 | ,value))) | |
293 | (t | |
294 | (let ((fn-name (car form)) | |
295 | (arg-forms (cdr form))) | |
15c9d04e SM |
296 | (cl-assert (or (symbolp fn-name) |
297 | (and (consp fn-name) | |
298 | (eql (car fn-name) 'lambda) | |
299 | (listp (cdr fn-name))))) | |
a19b3c2d GM |
300 | (let ((fn (cl-gensym "fn-")) |
301 | (args (cl-gensym "args-")) | |
302 | (value (cl-gensym "value-")) | |
303 | (default-value (cl-gensym "ert-form-evaluation-aborted-"))) | |
d221e780 CO |
304 | `(let ((,fn (function ,fn-name)) |
305 | (,args (list ,@arg-forms))) | |
306 | (let ((,value ',default-value)) | |
307 | ,(funcall inner-expander | |
308 | `(setq ,value (apply ,fn ,args)) | |
309 | `(nconc (list ',whole) | |
310 | (list :form `(,,fn ,@,args)) | |
311 | (unless (eql ,value ',default-value) | |
312 | (list :value ,value)) | |
313 | (let ((-explainer- | |
314 | (and (symbolp ',fn-name) | |
315 | (get ',fn-name 'ert-explainer)))) | |
316 | (when -explainer- | |
317 | (list :explanation | |
318 | (apply -explainer- ,args))))) | |
319 | value) | |
320 | ,value)))))))) | |
321 | ||
322 | (defun ert--expand-should (whole form inner-expander) | |
323 | "Helper function for the `should' macro and its variants. | |
324 | ||
325 | Analyzes FORM and returns an expression that has the same | |
326 | semantics under evaluation but records additional debugging | |
327 | information. | |
328 | ||
329 | INNER-EXPANDER should be a function and is called with two | |
330 | arguments: INNER-FORM and FORM-DESCRIPTION-FORM, where INNER-FORM | |
331 | is an expression equivalent to FORM, and FORM-DESCRIPTION-FORM is | |
332 | an expression that returns a description of FORM. INNER-EXPANDER | |
333 | should return code that calls INNER-FORM and performs the checks | |
8350f087 | 334 | and error signaling specific to the particular variant of |
d221e780 CO |
335 | `should'. The code that INNER-EXPANDER returns must not call |
336 | FORM-DESCRIPTION-FORM before it has called INNER-FORM." | |
15c9d04e SM |
337 | (ert--expand-should-1 |
338 | whole form | |
339 | (lambda (inner-form form-description-form value-var) | |
a19b3c2d | 340 | (let ((form-description (cl-gensym "form-description-"))) |
15c9d04e SM |
341 | `(let (,form-description) |
342 | ,(funcall inner-expander | |
343 | `(unwind-protect | |
344 | ,inner-form | |
345 | (setq ,form-description ,form-description-form) | |
346 | (ert--signal-should-execution ,form-description)) | |
347 | `,form-description | |
348 | value-var)))))) | |
349 | ||
350 | (cl-defmacro should (form) | |
d221e780 CO |
351 | "Evaluate FORM. If it returns nil, abort the current test as failed. |
352 | ||
353 | Returns the value of FORM." | |
285c8184 | 354 | (declare (debug t)) |
d221e780 | 355 | (ert--expand-should `(should ,form) form |
15c9d04e | 356 | (lambda (inner-form form-description-form _value-var) |
d221e780 CO |
357 | `(unless ,inner-form |
358 | (ert-fail ,form-description-form))))) | |
359 | ||
15c9d04e | 360 | (cl-defmacro should-not (form) |
d221e780 CO |
361 | "Evaluate FORM. If it returns non-nil, abort the current test as failed. |
362 | ||
363 | Returns nil." | |
285c8184 | 364 | (declare (debug t)) |
d221e780 | 365 | (ert--expand-should `(should-not ,form) form |
15c9d04e | 366 | (lambda (inner-form form-description-form _value-var) |
d221e780 CO |
367 | `(unless (not ,inner-form) |
368 | (ert-fail ,form-description-form))))) | |
369 | ||
370 | (defun ert--should-error-handle-error (form-description-fn | |
371 | condition type exclude-subtypes) | |
372 | "Helper function for `should-error'. | |
373 | ||
374 | Determines whether CONDITION matches TYPE and EXCLUDE-SUBTYPES, | |
375 | and aborts the current test as failed if it doesn't." | |
8350f087 | 376 | (let ((signaled-conditions (get (car condition) 'error-conditions)) |
15c9d04e | 377 | (handled-conditions (cl-etypecase type |
d221e780 CO |
378 | (list type) |
379 | (symbol (list type))))) | |
15c9d04e | 380 | (cl-assert signaled-conditions) |
a19b3c2d | 381 | (unless (cl-intersection signaled-conditions handled-conditions) |
d221e780 CO |
382 | (ert-fail (append |
383 | (funcall form-description-fn) | |
384 | (list | |
385 | :condition condition | |
8350f087 | 386 | :fail-reason (concat "the error signaled did not" |
d221e780 CO |
387 | " have the expected type"))))) |
388 | (when exclude-subtypes | |
389 | (unless (member (car condition) handled-conditions) | |
390 | (ert-fail (append | |
391 | (funcall form-description-fn) | |
392 | (list | |
393 | :condition condition | |
8350f087 | 394 | :fail-reason (concat "the error signaled was a subtype" |
d221e780 CO |
395 | " of the expected type")))))))) |
396 | ||
397 | ;; FIXME: The expansion will evaluate the keyword args (if any) in | |
398 | ;; nonstandard order. | |
15c9d04e | 399 | (cl-defmacro should-error (form &rest keys &key type exclude-subtypes) |
d221e780 CO |
400 | "Evaluate FORM and check that it signals an error. |
401 | ||
8350f087 | 402 | The error signaled needs to match TYPE. TYPE should be a list |
d221e780 CO |
403 | of condition names. (It can also be a non-nil symbol, which is |
404 | equivalent to a singleton list containing that symbol.) If | |
405 | EXCLUDE-SUBTYPES is nil, the error matches TYPE if one of its | |
406 | condition names is an element of TYPE. If EXCLUDE-SUBTYPES is | |
407 | non-nil, the error matches TYPE if it is an element of TYPE. | |
408 | ||
409 | If the error matches, returns (ERROR-SYMBOL . DATA) from the | |
8350f087 | 410 | error. If not, or if no error was signaled, abort the test as |
d221e780 | 411 | failed." |
e3e7b504 | 412 | (declare (debug t)) |
d221e780 CO |
413 | (unless type (setq type ''error)) |
414 | (ert--expand-should | |
415 | `(should-error ,form ,@keys) | |
416 | form | |
417 | (lambda (inner-form form-description-form value-var) | |
a19b3c2d GM |
418 | (let ((errorp (cl-gensym "errorp")) |
419 | (form-description-fn (cl-gensym "form-description-fn-"))) | |
d221e780 CO |
420 | `(let ((,errorp nil) |
421 | (,form-description-fn (lambda () ,form-description-form))) | |
422 | (condition-case -condition- | |
423 | ,inner-form | |
424 | ;; We can't use ,type here because we want to evaluate it. | |
425 | (error | |
426 | (setq ,errorp t) | |
427 | (ert--should-error-handle-error ,form-description-fn | |
428 | -condition- | |
429 | ,type ,exclude-subtypes) | |
430 | (setq ,value-var -condition-))) | |
431 | (unless ,errorp | |
432 | (ert-fail (append | |
433 | (funcall ,form-description-fn) | |
434 | (list | |
435 | :fail-reason "did not signal an error"))))))))) | |
436 | ||
50b5b857 MA |
437 | (cl-defmacro ert--skip-unless (form) |
438 | "Evaluate FORM. If it returns nil, skip the current test. | |
a67c4ae0 | 439 | Errors during evaluation are caught and handled like nil." |
50b5b857 MA |
440 | (declare (debug t)) |
441 | (ert--expand-should `(skip-unless ,form) form | |
442 | (lambda (inner-form form-description-form _value-var) | |
443 | `(unless (ignore-errors ,inner-form) | |
444 | (ert-skip ,form-description-form))))) | |
445 | ||
d221e780 CO |
446 | |
447 | ;;; Explanation of `should' failures. | |
448 | ||
449 | ;; TODO(ohler): Rework explanations so that they are displayed in a | |
450 | ;; similar way to `ert-info' messages; in particular, allow text | |
451 | ;; buttons in explanations that give more detail or open an ediff | |
452 | ;; buffer. Perhaps explanations should be reported through `ert-info' | |
453 | ;; rather than as part of the condition. | |
454 | ||
455 | (defun ert--proper-list-p (x) | |
456 | "Return non-nil if X is a proper list, nil otherwise." | |
15c9d04e | 457 | (cl-loop |
d221e780 CO |
458 | for firstp = t then nil |
459 | for fast = x then (cddr fast) | |
460 | for slow = x then (cdr slow) do | |
15c9d04e SM |
461 | (when (null fast) (cl-return t)) |
462 | (when (not (consp fast)) (cl-return nil)) | |
463 | (when (null (cdr fast)) (cl-return t)) | |
464 | (when (not (consp (cdr fast))) (cl-return nil)) | |
465 | (when (and (not firstp) (eq fast slow)) (cl-return nil)))) | |
d221e780 CO |
466 | |
467 | (defun ert--explain-format-atom (x) | |
de69c0a8 | 468 | "Format the atom X for `ert--explain-equal'." |
15c9d04e | 469 | (cl-typecase x |
84a06b50 GM |
470 | (character (list x (format "#x%x" x) (format "?%c" x))) |
471 | (fixnum (list x (format "#x%x" x))) | |
d221e780 CO |
472 | (t x))) |
473 | ||
de69c0a8 | 474 | (defun ert--explain-equal-rec (a b) |
c235b555 | 475 | "Return a programmer-readable explanation of why A and B are not `equal'. |
de69c0a8 | 476 | Returns nil if they are." |
d221e780 CO |
477 | (if (not (equal (type-of a) (type-of b))) |
478 | `(different-types ,a ,b) | |
15c9d04e | 479 | (cl-etypecase a |
d221e780 CO |
480 | (cons |
481 | (let ((a-proper-p (ert--proper-list-p a)) | |
482 | (b-proper-p (ert--proper-list-p b))) | |
483 | (if (not (eql (not a-proper-p) (not b-proper-p))) | |
484 | `(one-list-proper-one-improper ,a ,b) | |
485 | (if a-proper-p | |
486 | (if (not (equal (length a) (length b))) | |
487 | `(proper-lists-of-different-length ,(length a) ,(length b) | |
488 | ,a ,b | |
489 | first-mismatch-at | |
a19b3c2d | 490 | ,(cl-mismatch a b :test 'equal)) |
15c9d04e SM |
491 | (cl-loop for i from 0 |
492 | for ai in a | |
493 | for bi in b | |
494 | for xi = (ert--explain-equal-rec ai bi) | |
495 | do (when xi (cl-return `(list-elt ,i ,xi))) | |
496 | finally (cl-assert (equal a b) t))) | |
de69c0a8 | 497 | (let ((car-x (ert--explain-equal-rec (car a) (car b)))) |
d221e780 CO |
498 | (if car-x |
499 | `(car ,car-x) | |
de69c0a8 | 500 | (let ((cdr-x (ert--explain-equal-rec (cdr a) (cdr b)))) |
d221e780 CO |
501 | (if cdr-x |
502 | `(cdr ,cdr-x) | |
15c9d04e | 503 | (cl-assert (equal a b) t) |
d221e780 CO |
504 | nil)))))))) |
505 | (array (if (not (equal (length a) (length b))) | |
506 | `(arrays-of-different-length ,(length a) ,(length b) | |
507 | ,a ,b | |
508 | ,@(unless (char-table-p a) | |
509 | `(first-mismatch-at | |
a19b3c2d | 510 | ,(cl-mismatch a b :test 'equal)))) |
15c9d04e SM |
511 | (cl-loop for i from 0 |
512 | for ai across a | |
513 | for bi across b | |
514 | for xi = (ert--explain-equal-rec ai bi) | |
515 | do (when xi (cl-return `(array-elt ,i ,xi))) | |
516 | finally (cl-assert (equal a b) t)))) | |
d221e780 CO |
517 | (atom (if (not (equal a b)) |
518 | (if (and (symbolp a) (symbolp b) (string= a b)) | |
519 | `(different-symbols-with-the-same-name ,a ,b) | |
520 | `(different-atoms ,(ert--explain-format-atom a) | |
521 | ,(ert--explain-format-atom b))) | |
522 | nil))))) | |
de69c0a8 CO |
523 | |
524 | (defun ert--explain-equal (a b) | |
525 | "Explainer function for `equal'." | |
526 | ;; Do a quick comparison in C to avoid running our expensive | |
527 | ;; comparison when possible. | |
528 | (if (equal a b) | |
529 | nil | |
530 | (ert--explain-equal-rec a b))) | |
531 | (put 'equal 'ert-explainer 'ert--explain-equal) | |
d221e780 CO |
532 | |
533 | (defun ert--significant-plist-keys (plist) | |
534 | "Return the keys of PLIST that have non-null values, in order." | |
15c9d04e SM |
535 | (cl-assert (zerop (mod (length plist) 2)) t) |
536 | (cl-loop for (key value . rest) on plist by #'cddr | |
537 | unless (or (null value) (memq key accu)) collect key into accu | |
538 | finally (cl-return accu))) | |
d221e780 CO |
539 | |
540 | (defun ert--plist-difference-explanation (a b) | |
541 | "Return a programmer-readable explanation of why A and B are different plists. | |
542 | ||
543 | Returns nil if they are equivalent, i.e., have the same value for | |
544 | each key, where absent values are treated as nil. The order of | |
545 | key/value pairs in each list does not matter." | |
15c9d04e SM |
546 | (cl-assert (zerop (mod (length a) 2)) t) |
547 | (cl-assert (zerop (mod (length b) 2)) t) | |
d221e780 CO |
548 | ;; Normalizing the plists would be another way to do this but it |
549 | ;; requires a total ordering on all lisp objects (since any object | |
550 | ;; is valid as a text property key). Perhaps defining such an | |
551 | ;; ordering is useful in other contexts, too, but it's a lot of | |
552 | ;; work, so let's punt on it for now. | |
553 | (let* ((keys-a (ert--significant-plist-keys a)) | |
554 | (keys-b (ert--significant-plist-keys b)) | |
a19b3c2d GM |
555 | (keys-in-a-not-in-b (cl-set-difference keys-a keys-b :test 'eq)) |
556 | (keys-in-b-not-in-a (cl-set-difference keys-b keys-a :test 'eq))) | |
15c9d04e SM |
557 | (cl-flet ((explain-with-key (key) |
558 | (let ((value-a (plist-get a key)) | |
559 | (value-b (plist-get b key))) | |
560 | (cl-assert (not (equal value-a value-b)) t) | |
561 | `(different-properties-for-key | |
562 | ,key ,(ert--explain-equal-including-properties value-a | |
563 | value-b))))) | |
d221e780 | 564 | (cond (keys-in-a-not-in-b |
15c9d04e | 565 | (explain-with-key (car keys-in-a-not-in-b))) |
d221e780 | 566 | (keys-in-b-not-in-a |
15c9d04e | 567 | (explain-with-key (car keys-in-b-not-in-a))) |
d221e780 | 568 | (t |
15c9d04e SM |
569 | (cl-loop for key in keys-a |
570 | when (not (equal (plist-get a key) (plist-get b key))) | |
571 | return (explain-with-key key))))))) | |
d221e780 CO |
572 | |
573 | (defun ert--abbreviate-string (s len suffixp) | |
574 | "Shorten string S to at most LEN chars. | |
575 | ||
576 | If SUFFIXP is non-nil, returns a suffix of S, otherwise a prefix." | |
577 | (let ((n (length s))) | |
578 | (cond ((< n len) | |
579 | s) | |
580 | (suffixp | |
581 | (substring s (- n len))) | |
582 | (t | |
583 | (substring s 0 len))))) | |
584 | ||
de69c0a8 CO |
585 | ;; TODO(ohler): Once bug 6581 is fixed, rename this to |
586 | ;; `ert--explain-equal-including-properties-rec' and add a fast-path | |
587 | ;; wrapper like `ert--explain-equal'. | |
588 | (defun ert--explain-equal-including-properties (a b) | |
d221e780 CO |
589 | "Explainer function for `ert-equal-including-properties'. |
590 | ||
591 | Returns a programmer-readable explanation of why A and B are not | |
592 | `ert-equal-including-properties', or nil if they are." | |
593 | (if (not (equal a b)) | |
de69c0a8 | 594 | (ert--explain-equal a b) |
15c9d04e SM |
595 | (cl-assert (stringp a) t) |
596 | (cl-assert (stringp b) t) | |
597 | (cl-assert (eql (length a) (length b)) t) | |
598 | (cl-loop for i from 0 to (length a) | |
599 | for props-a = (text-properties-at i a) | |
600 | for props-b = (text-properties-at i b) | |
601 | for difference = (ert--plist-difference-explanation | |
602 | props-a props-b) | |
603 | do (when difference | |
604 | (cl-return `(char ,i ,(substring-no-properties a i (1+ i)) | |
605 | ,difference | |
606 | context-before | |
607 | ,(ert--abbreviate-string | |
608 | (substring-no-properties a 0 i) | |
609 | 10 t) | |
610 | context-after | |
611 | ,(ert--abbreviate-string | |
612 | (substring-no-properties a (1+ i)) | |
613 | 10 nil)))) | |
614 | ;; TODO(ohler): Get `equal-including-properties' fixed in | |
615 | ;; Emacs, delete `ert-equal-including-properties', and | |
616 | ;; re-enable this assertion. | |
617 | ;;finally (cl-assert (equal-including-properties a b) t) | |
618 | ))) | |
d221e780 CO |
619 | (put 'ert-equal-including-properties |
620 | 'ert-explainer | |
de69c0a8 | 621 | 'ert--explain-equal-including-properties) |
d221e780 CO |
622 | |
623 | ||
624 | ;;; Implementation of `ert-info'. | |
625 | ||
626 | ;; TODO(ohler): The name `info' clashes with | |
627 | ;; `ert--test-execution-info'. One or both should be renamed. | |
628 | (defvar ert--infos '() | |
629 | "The stack of `ert-info' infos that currently apply. | |
630 | ||
631 | Bound dynamically. This is a list of (PREFIX . MESSAGE) pairs.") | |
632 | ||
15c9d04e SM |
633 | (cl-defmacro ert-info ((message-form &key ((:prefix prefix-form) "Info: ")) |
634 | &body body) | |
d221e780 CO |
635 | "Evaluate MESSAGE-FORM and BODY, and report the message if BODY fails. |
636 | ||
637 | To be used within ERT tests. MESSAGE-FORM should evaluate to a | |
638 | string that will be displayed together with the test result if | |
639 | the test fails. PREFIX-FORM should evaluate to a string as well | |
640 | and is displayed in front of the value of MESSAGE-FORM." | |
641 | (declare (debug ((form &rest [sexp form]) body)) | |
642 | (indent 1)) | |
643 | `(let ((ert--infos (cons (cons ,prefix-form ,message-form) ert--infos))) | |
644 | ,@body)) | |
645 | ||
646 | ||
647 | ||
648 | ;;; Facilities for running a single test. | |
649 | ||
650 | (defvar ert-debug-on-error nil | |
651 | "Non-nil means enter debugger when a test fails or terminates with an error.") | |
652 | ||
653 | ;; The data structures that represent the result of running a test. | |
15c9d04e | 654 | (cl-defstruct ert-test-result |
d221e780 CO |
655 | (messages nil) |
656 | (should-forms nil) | |
657 | ) | |
15c9d04e SM |
658 | (cl-defstruct (ert-test-passed (:include ert-test-result))) |
659 | (cl-defstruct (ert-test-result-with-condition (:include ert-test-result)) | |
660 | (condition (cl-assert nil)) | |
661 | (backtrace (cl-assert nil)) | |
662 | (infos (cl-assert nil))) | |
663 | (cl-defstruct (ert-test-quit (:include ert-test-result-with-condition))) | |
664 | (cl-defstruct (ert-test-failed (:include ert-test-result-with-condition))) | |
50b5b857 | 665 | (cl-defstruct (ert-test-skipped (:include ert-test-result-with-condition))) |
15c9d04e SM |
666 | (cl-defstruct (ert-test-aborted-with-non-local-exit |
667 | (:include ert-test-result))) | |
d221e780 CO |
668 | |
669 | ||
670 | (defun ert--record-backtrace () | |
671 | "Record the current backtrace (as a list) and return it." | |
672 | ;; Since the backtrace is stored in the result object, result | |
673 | ;; objects must only be printed with appropriate limits | |
674 | ;; (`print-level' and `print-length') in place. For interactive | |
675 | ;; use, the cost of ensuring this possibly outweighs the advantage | |
676 | ;; of storing the backtrace for | |
677 | ;; `ert-results-pop-to-backtrace-for-test-at-point' given that we | |
678 | ;; already have `ert-results-rerun-test-debugging-errors-at-point'. | |
679 | ;; For batch use, however, printing the backtrace may be useful. | |
15c9d04e | 680 | (cl-loop |
d221e780 CO |
681 | ;; 6 is the number of frames our own debugger adds (when |
682 | ;; compiled; more when interpreted). FIXME: Need to describe a | |
683 | ;; procedure for determining this constant. | |
684 | for i from 6 | |
685 | for frame = (backtrace-frame i) | |
686 | while frame | |
687 | collect frame)) | |
688 | ||
689 | (defun ert--print-backtrace (backtrace) | |
690 | "Format the backtrace BACKTRACE to the current buffer." | |
691 | ;; This is essentially a reimplementation of Fbacktrace | |
692 | ;; (src/eval.c), but for a saved backtrace, not the current one. | |
693 | (let ((print-escape-newlines t) | |
694 | (print-level 8) | |
695 | (print-length 50)) | |
696 | (dolist (frame backtrace) | |
15c9d04e | 697 | (cl-ecase (car frame) |
d221e780 CO |
698 | ((nil) |
699 | ;; Special operator. | |
15c9d04e | 700 | (cl-destructuring-bind (special-operator &rest arg-forms) |
d221e780 CO |
701 | (cdr frame) |
702 | (insert | |
15c9d04e | 703 | (format " %S\n" (cons special-operator arg-forms))))) |
d221e780 CO |
704 | ((t) |
705 | ;; Function call. | |
15c9d04e | 706 | (cl-destructuring-bind (fn &rest args) (cdr frame) |
d221e780 | 707 | (insert (format " %S(" fn)) |
15c9d04e SM |
708 | (cl-loop for firstp = t then nil |
709 | for arg in args do | |
710 | (unless firstp | |
711 | (insert " ")) | |
712 | (insert (format "%S" arg))) | |
d221e780 CO |
713 | (insert ")\n"))))))) |
714 | ||
715 | ;; A container for the state of the execution of a single test and | |
716 | ;; environment data needed during its execution. | |
15c9d04e SM |
717 | (cl-defstruct ert--test-execution-info |
718 | (test (cl-assert nil)) | |
719 | (result (cl-assert nil)) | |
d221e780 CO |
720 | ;; A thunk that may be called when RESULT has been set to its final |
721 | ;; value and test execution should be terminated. Should not | |
722 | ;; return. | |
15c9d04e | 723 | (exit-continuation (cl-assert nil)) |
d221e780 CO |
724 | ;; The binding of `debugger' outside of the execution of the test. |
725 | next-debugger | |
726 | ;; The binding of `ert-debug-on-error' that is in effect for the | |
727 | ;; execution of the current test. We store it to avoid being | |
728 | ;; affected by any new bindings the test itself may establish. (I | |
729 | ;; don't remember whether this feature is important.) | |
730 | ert-debug-on-error) | |
731 | ||
15c9d04e | 732 | (defun ert--run-test-debugger (info args) |
d221e780 CO |
733 | "During a test run, `debugger' is bound to a closure that calls this function. |
734 | ||
735 | This function records failures and errors and either terminates | |
736 | the test silently or calls the interactive debugger, as | |
737 | appropriate. | |
738 | ||
739 | INFO is the ert--test-execution-info corresponding to this test | |
15c9d04e SM |
740 | run. ARGS are the arguments to `debugger'." |
741 | (cl-destructuring-bind (first-debugger-arg &rest more-debugger-args) | |
742 | args | |
743 | (cl-ecase first-debugger-arg | |
d221e780 | 744 | ((lambda debug t exit nil) |
15c9d04e | 745 | (apply (ert--test-execution-info-next-debugger info) args)) |
d221e780 | 746 | (error |
15c9d04e SM |
747 | (let* ((condition (car more-debugger-args)) |
748 | (type (cl-case (car condition) | |
d221e780 | 749 | ((quit) 'quit) |
50b5b857 | 750 | ((ert-test-skipped) 'skipped) |
d221e780 CO |
751 | (otherwise 'failed))) |
752 | (backtrace (ert--record-backtrace)) | |
753 | (infos (reverse ert--infos))) | |
754 | (setf (ert--test-execution-info-result info) | |
15c9d04e | 755 | (cl-ecase type |
d221e780 CO |
756 | (quit |
757 | (make-ert-test-quit :condition condition | |
758 | :backtrace backtrace | |
759 | :infos infos)) | |
50b5b857 MA |
760 | (skipped |
761 | (make-ert-test-skipped :condition condition | |
762 | :backtrace backtrace | |
763 | :infos infos)) | |
d221e780 CO |
764 | (failed |
765 | (make-ert-test-failed :condition condition | |
766 | :backtrace backtrace | |
767 | :infos infos)))) | |
44e97401 | 768 | ;; Work around Emacs's heuristic (in eval.c) for detecting |
d221e780 | 769 | ;; errors in the debugger. |
15c9d04e | 770 | (cl-incf num-nonmacro-input-events) |
d221e780 CO |
771 | ;; FIXME: We should probably implement more fine-grained |
772 | ;; control a la non-t `debug-on-error' here. | |
773 | (cond | |
774 | ((ert--test-execution-info-ert-debug-on-error info) | |
15c9d04e | 775 | (apply (ert--test-execution-info-next-debugger info) args)) |
d221e780 CO |
776 | (t)) |
777 | (funcall (ert--test-execution-info-exit-continuation info))))))) | |
778 | ||
15c9d04e SM |
779 | (defun ert--run-test-internal (test-execution-info) |
780 | "Low-level function to run a test according to TEST-EXECUTION-INFO. | |
d221e780 CO |
781 | |
782 | This mainly sets up debugger-related bindings." | |
15c9d04e SM |
783 | (setf (ert--test-execution-info-next-debugger test-execution-info) debugger |
784 | (ert--test-execution-info-ert-debug-on-error test-execution-info) | |
785 | ert-debug-on-error) | |
786 | (catch 'ert--pass | |
787 | ;; For now, each test gets its own temp buffer and its own | |
788 | ;; window excursion, just to be safe. If this turns out to be | |
789 | ;; too expensive, we can remove it. | |
790 | (with-temp-buffer | |
791 | (save-window-excursion | |
792 | (let ((debugger (lambda (&rest args) | |
793 | (ert--run-test-debugger test-execution-info | |
794 | args))) | |
795 | (debug-on-error t) | |
796 | (debug-on-quit t) | |
797 | ;; FIXME: Do we need to store the old binding of this | |
798 | ;; and consider it in `ert--run-test-debugger'? | |
799 | (debug-ignored-errors nil) | |
800 | (ert--infos '())) | |
801 | (funcall (ert-test-body (ert--test-execution-info-test | |
802 | test-execution-info)))))) | |
803 | (ert-pass)) | |
804 | (setf (ert--test-execution-info-result test-execution-info) | |
805 | (make-ert-test-passed)) | |
d221e780 CO |
806 | nil) |
807 | ||
808 | (defun ert--force-message-log-buffer-truncation () | |
809 | "Immediately truncate *Messages* buffer according to `message-log-max'. | |
810 | ||
811 | This can be useful after reducing the value of `message-log-max'." | |
90582f05 | 812 | (with-current-buffer (messages-buffer) |
d221e780 CO |
813 | ;; This is a reimplementation of this part of message_dolog() in xdisp.c: |
814 | ;; if (NATNUMP (Vmessage_log_max)) | |
815 | ;; { | |
816 | ;; scan_newline (Z, Z_BYTE, BEG, BEG_BYTE, | |
817 | ;; -XFASTINT (Vmessage_log_max) - 1, 0); | |
818 | ;; del_range_both (BEG, BEG_BYTE, PT, PT_BYTE, 0); | |
819 | ;; } | |
820 | (when (and (integerp message-log-max) (>= message-log-max 0)) | |
821 | (let ((begin (point-min)) | |
822 | (end (save-excursion | |
823 | (goto-char (point-max)) | |
824 | (forward-line (- message-log-max)) | |
90582f05 GM |
825 | (point))) |
826 | (inhibit-read-only t)) | |
d221e780 CO |
827 | (delete-region begin end))))) |
828 | ||
829 | (defvar ert--running-tests nil | |
830 | "List of tests that are currently in execution. | |
831 | ||
832 | This list is empty while no test is running, has one element | |
833 | while a test is running, two elements while a test run from | |
834 | inside a test is running, etc. The list is in order of nesting, | |
835 | innermost test first. | |
836 | ||
837 | The elements are of type `ert-test'.") | |
838 | ||
839 | (defun ert-run-test (ert-test) | |
840 | "Run ERT-TEST. | |
841 | ||
842 | Returns the result and stores it in ERT-TEST's `most-recent-result' slot." | |
843 | (setf (ert-test-most-recent-result ert-test) nil) | |
15c9d04e SM |
844 | (cl-block error |
845 | (let ((begin-marker | |
90582f05 | 846 | (with-current-buffer (messages-buffer) |
30818a23 | 847 | (point-max-marker)))) |
d221e780 | 848 | (unwind-protect |
15c9d04e SM |
849 | (let ((info (make-ert--test-execution-info |
850 | :test ert-test | |
851 | :result | |
852 | (make-ert-test-aborted-with-non-local-exit) | |
853 | :exit-continuation (lambda () | |
854 | (cl-return-from error nil)))) | |
855 | (should-form-accu (list))) | |
d221e780 CO |
856 | (unwind-protect |
857 | (let ((ert--should-execution-observer | |
858 | (lambda (form-description) | |
859 | (push form-description should-form-accu))) | |
860 | (message-log-max t) | |
861 | (ert--running-tests (cons ert-test ert--running-tests))) | |
862 | (ert--run-test-internal info)) | |
863 | (let ((result (ert--test-execution-info-result info))) | |
864 | (setf (ert-test-result-messages result) | |
90582f05 | 865 | (with-current-buffer (messages-buffer) |
d221e780 CO |
866 | (buffer-substring begin-marker (point-max)))) |
867 | (ert--force-message-log-buffer-truncation) | |
868 | (setq should-form-accu (nreverse should-form-accu)) | |
869 | (setf (ert-test-result-should-forms result) | |
870 | should-form-accu) | |
871 | (setf (ert-test-most-recent-result ert-test) result)))) | |
872 | (set-marker begin-marker nil)))) | |
873 | (ert-test-most-recent-result ert-test)) | |
874 | ||
875 | (defun ert-running-test () | |
876 | "Return the top-level test currently executing." | |
877 | (car (last ert--running-tests))) | |
878 | ||
879 | ||
880 | ;;; Test selectors. | |
881 | ||
882 | (defun ert-test-result-type-p (result result-type) | |
883 | "Return non-nil if RESULT matches type RESULT-TYPE. | |
884 | ||
885 | Valid result types: | |
886 | ||
887 | nil -- Never matches. | |
888 | t -- Always matches. | |
50b5b857 | 889 | :failed, :passed, :skipped -- Matches corresponding results. |
d221e780 CO |
890 | \(and TYPES...\) -- Matches if all TYPES match. |
891 | \(or TYPES...\) -- Matches if some TYPES match. | |
892 | \(not TYPE\) -- Matches if TYPE does not match. | |
893 | \(satisfies PREDICATE\) -- Matches if PREDICATE returns true when called with | |
894 | RESULT." | |
895 | ;; It would be easy to add `member' and `eql' types etc., but I | |
896 | ;; haven't bothered yet. | |
15c9d04e | 897 | (cl-etypecase result-type |
d221e780 CO |
898 | ((member nil) nil) |
899 | ((member t) t) | |
900 | ((member :failed) (ert-test-failed-p result)) | |
901 | ((member :passed) (ert-test-passed-p result)) | |
50b5b857 | 902 | ((member :skipped) (ert-test-skipped-p result)) |
d221e780 | 903 | (cons |
15c9d04e SM |
904 | (cl-destructuring-bind (operator &rest operands) result-type |
905 | (cl-ecase operator | |
d221e780 | 906 | (and |
15c9d04e | 907 | (cl-case (length operands) |
d221e780 CO |
908 | (0 t) |
909 | (t | |
15c9d04e SM |
910 | (and (ert-test-result-type-p result (car operands)) |
911 | (ert-test-result-type-p result `(and ,@(cdr operands))))))) | |
d221e780 | 912 | (or |
15c9d04e | 913 | (cl-case (length operands) |
d221e780 CO |
914 | (0 nil) |
915 | (t | |
15c9d04e SM |
916 | (or (ert-test-result-type-p result (car operands)) |
917 | (ert-test-result-type-p result `(or ,@(cdr operands))))))) | |
d221e780 | 918 | (not |
15c9d04e SM |
919 | (cl-assert (eql (length operands) 1)) |
920 | (not (ert-test-result-type-p result (car operands)))) | |
d221e780 | 921 | (satisfies |
15c9d04e SM |
922 | (cl-assert (eql (length operands) 1)) |
923 | (funcall (car operands) result))))))) | |
d221e780 CO |
924 | |
925 | (defun ert-test-result-expected-p (test result) | |
926 | "Return non-nil if TEST's expected result type matches RESULT." | |
50b5b857 MA |
927 | (or |
928 | (ert-test-result-type-p result :skipped) | |
929 | (ert-test-result-type-p result (ert-test-expected-result-type test)))) | |
d221e780 CO |
930 | |
931 | (defun ert-select-tests (selector universe) | |
c235b555 | 932 | "Return a list of tests that match SELECTOR. |
d221e780 | 933 | |
c235b555 GM |
934 | UNIVERSE specifies the set of tests to select from; it should be a list |
935 | of tests, or t, which refers to all tests named by symbols in `obarray'. | |
d221e780 | 936 | |
c235b555 | 937 | Valid SELECTORs: |
d221e780 | 938 | |
c235b555 GM |
939 | nil -- Selects the empty set. |
940 | t -- Selects UNIVERSE. | |
d221e780 | 941 | :new -- Selects all tests that have not been run yet. |
c235b555 | 942 | :failed, :passed -- Select tests according to their most recent result. |
d221e780 | 943 | :expected, :unexpected -- Select tests according to their most recent result. |
c235b555 GM |
944 | a string -- A regular expression selecting all tests with matching names. |
945 | a test -- (i.e., an object of the ert-test data-type) Selects that test. | |
d221e780 | 946 | a symbol -- Selects the test that the symbol names, errors if none. |
c235b555 GM |
947 | \(member TESTS...) -- Selects the elements of TESTS, a list of tests |
948 | or symbols naming tests. | |
d221e780 | 949 | \(eql TEST\) -- Selects TEST, a test or a symbol naming a test. |
c235b555 GM |
950 | \(and SELECTORS...) -- Selects the tests that match all SELECTORS. |
951 | \(or SELECTORS...) -- Selects the tests that match any of the SELECTORS. | |
952 | \(not SELECTOR) -- Selects all tests that do not match SELECTOR. | |
d221e780 | 953 | \(tag TAG) -- Selects all tests that have TAG on their tags list. |
c235b555 GM |
954 | A tag is an arbitrary label you can apply when you define a test. |
955 | \(satisfies PREDICATE) -- Selects all tests that satisfy PREDICATE. | |
956 | PREDICATE is a function that takes an ert-test object as argument, | |
957 | and returns non-nil if it is selected. | |
d221e780 CO |
958 | |
959 | Only selectors that require a superset of tests, such | |
960 | as (satisfies ...), strings, :new, etc. make use of UNIVERSE. | |
c235b555 | 961 | Selectors that do not, such as (member ...), just return the |
d221e780 CO |
962 | set implied by them without checking whether it is really |
963 | contained in UNIVERSE." | |
964 | ;; This code needs to match the etypecase in | |
965 | ;; `ert-insert-human-readable-selector'. | |
15c9d04e | 966 | (cl-etypecase selector |
d221e780 | 967 | ((member nil) nil) |
15c9d04e | 968 | ((member t) (cl-etypecase universe |
d221e780 CO |
969 | (list universe) |
970 | ((member t) (ert-select-tests "" universe)))) | |
971 | ((member :new) (ert-select-tests | |
972 | `(satisfies ,(lambda (test) | |
973 | (null (ert-test-most-recent-result test)))) | |
974 | universe)) | |
975 | ((member :failed) (ert-select-tests | |
976 | `(satisfies ,(lambda (test) | |
977 | (ert-test-result-type-p | |
978 | (ert-test-most-recent-result test) | |
979 | ':failed))) | |
980 | universe)) | |
981 | ((member :passed) (ert-select-tests | |
982 | `(satisfies ,(lambda (test) | |
983 | (ert-test-result-type-p | |
984 | (ert-test-most-recent-result test) | |
985 | ':passed))) | |
986 | universe)) | |
987 | ((member :expected) (ert-select-tests | |
988 | `(satisfies | |
989 | ,(lambda (test) | |
990 | (ert-test-result-expected-p | |
991 | test | |
992 | (ert-test-most-recent-result test)))) | |
993 | universe)) | |
994 | ((member :unexpected) (ert-select-tests `(not :expected) universe)) | |
995 | (string | |
15c9d04e | 996 | (cl-etypecase universe |
d221e780 CO |
997 | ((member t) (mapcar #'ert-get-test |
998 | (apropos-internal selector #'ert-test-boundp))) | |
a19b3c2d | 999 | (list (cl-remove-if-not (lambda (test) |
d221e780 CO |
1000 | (and (ert-test-name test) |
1001 | (string-match selector | |
049638ad GM |
1002 | (symbol-name |
1003 | (ert-test-name test))))) | |
d221e780 CO |
1004 | universe)))) |
1005 | (ert-test (list selector)) | |
1006 | (symbol | |
15c9d04e | 1007 | (cl-assert (ert-test-boundp selector)) |
d221e780 CO |
1008 | (list (ert-get-test selector))) |
1009 | (cons | |
15c9d04e SM |
1010 | (cl-destructuring-bind (operator &rest operands) selector |
1011 | (cl-ecase operator | |
d221e780 CO |
1012 | (member |
1013 | (mapcar (lambda (purported-test) | |
15c9d04e SM |
1014 | (cl-etypecase purported-test |
1015 | (symbol (cl-assert (ert-test-boundp purported-test)) | |
d221e780 CO |
1016 | (ert-get-test purported-test)) |
1017 | (ert-test purported-test))) | |
1018 | operands)) | |
1019 | (eql | |
15c9d04e | 1020 | (cl-assert (eql (length operands) 1)) |
d221e780 CO |
1021 | (ert-select-tests `(member ,@operands) universe)) |
1022 | (and | |
1023 | ;; Do these definitions of AND, NOT and OR satisfy de | |
1024 | ;; Morgan's laws? Should they? | |
15c9d04e | 1025 | (cl-case (length operands) |
d221e780 | 1026 | (0 (ert-select-tests 't universe)) |
15c9d04e SM |
1027 | (t (ert-select-tests `(and ,@(cdr operands)) |
1028 | (ert-select-tests (car operands) | |
d221e780 CO |
1029 | universe))))) |
1030 | (not | |
15c9d04e | 1031 | (cl-assert (eql (length operands) 1)) |
d221e780 | 1032 | (let ((all-tests (ert-select-tests 't universe))) |
a19b3c2d | 1033 | (cl-set-difference all-tests |
15c9d04e | 1034 | (ert-select-tests (car operands) |
d221e780 CO |
1035 | all-tests)))) |
1036 | (or | |
15c9d04e | 1037 | (cl-case (length operands) |
d221e780 | 1038 | (0 (ert-select-tests 'nil universe)) |
a19b3c2d | 1039 | (t (cl-union (ert-select-tests (car operands) universe) |
15c9d04e | 1040 | (ert-select-tests `(or ,@(cdr operands)) |
d221e780 CO |
1041 | universe))))) |
1042 | (tag | |
15c9d04e SM |
1043 | (cl-assert (eql (length operands) 1)) |
1044 | (let ((tag (car operands))) | |
d221e780 CO |
1045 | (ert-select-tests `(satisfies |
1046 | ,(lambda (test) | |
1047 | (member tag (ert-test-tags test)))) | |
1048 | universe))) | |
1049 | (satisfies | |
15c9d04e | 1050 | (cl-assert (eql (length operands) 1)) |
a19b3c2d | 1051 | (cl-remove-if-not (car operands) |
d221e780 CO |
1052 | (ert-select-tests 't universe)))))))) |
1053 | ||
1054 | (defun ert--insert-human-readable-selector (selector) | |
1055 | "Insert a human-readable presentation of SELECTOR into the current buffer." | |
1056 | ;; This is needed to avoid printing the (huge) contents of the | |
1057 | ;; `backtrace' slot of the result objects in the | |
1058 | ;; `most-recent-result' slots of test case objects in (eql ...) or | |
1059 | ;; (member ...) selectors. | |
15c9d04e SM |
1060 | (cl-labels ((rec (selector) |
1061 | ;; This code needs to match the etypecase in | |
1062 | ;; `ert-select-tests'. | |
1063 | (cl-etypecase selector | |
1064 | ((or (member nil t | |
1065 | :new :failed :passed | |
1066 | :expected :unexpected) | |
1067 | string | |
1068 | symbol) | |
1069 | selector) | |
1070 | (ert-test | |
1071 | (if (ert-test-name selector) | |
1072 | (make-symbol (format "<%S>" (ert-test-name selector))) | |
1073 | (make-symbol "<unnamed test>"))) | |
1074 | (cons | |
1075 | (cl-destructuring-bind (operator &rest operands) selector | |
1076 | (cl-ecase operator | |
1077 | ((member eql and not or) | |
1078 | `(,operator ,@(mapcar #'rec operands))) | |
1079 | ((member tag satisfies) | |
1080 | selector))))))) | |
d221e780 CO |
1081 | (insert (format "%S" (rec selector))))) |
1082 | ||
1083 | ||
1084 | ;;; Facilities for running a whole set of tests. | |
1085 | ||
1086 | ;; The data structure that contains the set of tests being executed | |
1087 | ;; during one particular test run, their results, the state of the | |
1088 | ;; execution, and some statistics. | |
1089 | ;; | |
1090 | ;; The data about results and expected results of tests may seem | |
1091 | ;; redundant here, since the test objects also carry such information. | |
1092 | ;; However, the information in the test objects may be more recent, it | |
1093 | ;; may correspond to a different test run. We need the information | |
1094 | ;; that corresponds to this run in order to be able to update the | |
1095 | ;; statistics correctly when a test is re-run interactively and has a | |
1096 | ;; different result than before. | |
15c9d04e SM |
1097 | (cl-defstruct ert--stats |
1098 | (selector (cl-assert nil)) | |
d221e780 | 1099 | ;; The tests, in order. |
15c9d04e | 1100 | (tests (cl-assert nil) :type vector) |
d221e780 CO |
1101 | ;; A map of test names (or the test objects themselves for unnamed |
1102 | ;; tests) to indices into the `tests' vector. | |
15c9d04e | 1103 | (test-map (cl-assert nil) :type hash-table) |
d221e780 | 1104 | ;; The results of the tests during this run, in order. |
15c9d04e | 1105 | (test-results (cl-assert nil) :type vector) |
d221e780 CO |
1106 | ;; The start times of the tests, in order, as reported by |
1107 | ;; `current-time'. | |
15c9d04e | 1108 | (test-start-times (cl-assert nil) :type vector) |
d221e780 CO |
1109 | ;; The end times of the tests, in order, as reported by |
1110 | ;; `current-time'. | |
15c9d04e | 1111 | (test-end-times (cl-assert nil) :type vector) |
d221e780 CO |
1112 | (passed-expected 0) |
1113 | (passed-unexpected 0) | |
1114 | (failed-expected 0) | |
1115 | (failed-unexpected 0) | |
50b5b857 | 1116 | (skipped 0) |
d221e780 CO |
1117 | (start-time nil) |
1118 | (end-time nil) | |
1119 | (aborted-p nil) | |
1120 | (current-test nil) | |
1121 | ;; The time at or after which the next redisplay should occur, as a | |
1122 | ;; float. | |
1123 | (next-redisplay 0.0)) | |
1124 | ||
1125 | (defun ert-stats-completed-expected (stats) | |
1126 | "Return the number of tests in STATS that had expected results." | |
1127 | (+ (ert--stats-passed-expected stats) | |
1128 | (ert--stats-failed-expected stats))) | |
1129 | ||
1130 | (defun ert-stats-completed-unexpected (stats) | |
1131 | "Return the number of tests in STATS that had unexpected results." | |
1132 | (+ (ert--stats-passed-unexpected stats) | |
1133 | (ert--stats-failed-unexpected stats))) | |
1134 | ||
50b5b857 MA |
1135 | (defun ert-stats-skipped (stats) |
1136 | "Number of tests in STATS that have skipped." | |
1137 | (ert--stats-skipped stats)) | |
1138 | ||
d221e780 CO |
1139 | (defun ert-stats-completed (stats) |
1140 | "Number of tests in STATS that have run so far." | |
1141 | (+ (ert-stats-completed-expected stats) | |
50b5b857 MA |
1142 | (ert-stats-completed-unexpected stats) |
1143 | (ert-stats-skipped stats))) | |
d221e780 CO |
1144 | |
1145 | (defun ert-stats-total (stats) | |
1146 | "Number of tests in STATS, regardless of whether they have run yet." | |
1147 | (length (ert--stats-tests stats))) | |
1148 | ||
1149 | ;; The stats object of the current run, dynamically bound. This is | |
1150 | ;; used for the mode line progress indicator. | |
1151 | (defvar ert--current-run-stats nil) | |
1152 | ||
1153 | (defun ert--stats-test-key (test) | |
1154 | "Return the key used for TEST in the test map of ert--stats objects. | |
1155 | ||
1156 | Returns the name of TEST if it has one, or TEST itself otherwise." | |
1157 | (or (ert-test-name test) test)) | |
1158 | ||
1159 | (defun ert--stats-set-test-and-result (stats pos test result) | |
1160 | "Change STATS by replacing the test at position POS with TEST and RESULT. | |
1161 | ||
1162 | Also changes the counters in STATS to match." | |
1163 | (let* ((tests (ert--stats-tests stats)) | |
1164 | (results (ert--stats-test-results stats)) | |
1165 | (old-test (aref tests pos)) | |
1166 | (map (ert--stats-test-map stats))) | |
15c9d04e SM |
1167 | (cl-flet ((update (d) |
1168 | (if (ert-test-result-expected-p (aref tests pos) | |
1169 | (aref results pos)) | |
1170 | (cl-etypecase (aref results pos) | |
1171 | (ert-test-passed | |
1172 | (cl-incf (ert--stats-passed-expected stats) d)) | |
1173 | (ert-test-failed | |
1174 | (cl-incf (ert--stats-failed-expected stats) d)) | |
50b5b857 MA |
1175 | (ert-test-skipped |
1176 | (cl-incf (ert--stats-skipped stats) d)) | |
15c9d04e SM |
1177 | (null) |
1178 | (ert-test-aborted-with-non-local-exit) | |
1179 | (ert-test-quit)) | |
1180 | (cl-etypecase (aref results pos) | |
1181 | (ert-test-passed | |
1182 | (cl-incf (ert--stats-passed-unexpected stats) d)) | |
1183 | (ert-test-failed | |
1184 | (cl-incf (ert--stats-failed-unexpected stats) d)) | |
50b5b857 MA |
1185 | (ert-test-skipped |
1186 | (cl-incf (ert--stats-skipped stats) d)) | |
15c9d04e SM |
1187 | (null) |
1188 | (ert-test-aborted-with-non-local-exit) | |
1189 | (ert-test-quit))))) | |
d221e780 CO |
1190 | ;; Adjust counters to remove the result that is currently in stats. |
1191 | (update -1) | |
1192 | ;; Put new test and result into stats. | |
1193 | (setf (aref tests pos) test | |
1194 | (aref results pos) result) | |
1195 | (remhash (ert--stats-test-key old-test) map) | |
1196 | (setf (gethash (ert--stats-test-key test) map) pos) | |
1197 | ;; Adjust counters to match new result. | |
1198 | (update +1) | |
1199 | nil))) | |
1200 | ||
1201 | (defun ert--make-stats (tests selector) | |
1202 | "Create a new `ert--stats' object for running TESTS. | |
1203 | ||
1204 | SELECTOR is the selector that was used to select TESTS." | |
a19b3c2d | 1205 | (setq tests (cl-coerce tests 'vector)) |
d221e780 | 1206 | (let ((map (make-hash-table :size (length tests)))) |
15c9d04e SM |
1207 | (cl-loop for i from 0 |
1208 | for test across tests | |
1209 | for key = (ert--stats-test-key test) do | |
1210 | (cl-assert (not (gethash key map))) | |
1211 | (setf (gethash key map) i)) | |
d221e780 CO |
1212 | (make-ert--stats :selector selector |
1213 | :tests tests | |
1214 | :test-map map | |
1215 | :test-results (make-vector (length tests) nil) | |
1216 | :test-start-times (make-vector (length tests) nil) | |
1217 | :test-end-times (make-vector (length tests) nil)))) | |
1218 | ||
1219 | (defun ert-run-or-rerun-test (stats test listener) | |
1220 | ;; checkdoc-order: nil | |
1221 | "Run the single test TEST and record the result using STATS and LISTENER." | |
1222 | (let ((ert--current-run-stats stats) | |
1223 | (pos (ert--stats-test-pos stats test))) | |
1224 | (ert--stats-set-test-and-result stats pos test nil) | |
1225 | ;; Call listener after setting/before resetting | |
1226 | ;; (ert--stats-current-test stats); the listener might refresh the | |
1227 | ;; mode line display, and if the value is not set yet/any more | |
1228 | ;; during this refresh, the mode line will flicker unnecessarily. | |
1229 | (setf (ert--stats-current-test stats) test) | |
1230 | (funcall listener 'test-started stats test) | |
1231 | (setf (ert-test-most-recent-result test) nil) | |
1232 | (setf (aref (ert--stats-test-start-times stats) pos) (current-time)) | |
1233 | (unwind-protect | |
1234 | (ert-run-test test) | |
1235 | (setf (aref (ert--stats-test-end-times stats) pos) (current-time)) | |
1236 | (let ((result (ert-test-most-recent-result test))) | |
1237 | (ert--stats-set-test-and-result stats pos test result) | |
1238 | (funcall listener 'test-ended stats test result)) | |
1239 | (setf (ert--stats-current-test stats) nil)))) | |
1240 | ||
1241 | (defun ert-run-tests (selector listener) | |
1242 | "Run the tests specified by SELECTOR, sending progress updates to LISTENER." | |
1243 | (let* ((tests (ert-select-tests selector t)) | |
1244 | (stats (ert--make-stats tests selector))) | |
1245 | (setf (ert--stats-start-time stats) (current-time)) | |
1246 | (funcall listener 'run-started stats) | |
1247 | (let ((abortedp t)) | |
1248 | (unwind-protect | |
1249 | (let ((ert--current-run-stats stats)) | |
1250 | (force-mode-line-update) | |
1251 | (unwind-protect | |
1252 | (progn | |
15c9d04e SM |
1253 | (cl-loop for test in tests do |
1254 | (ert-run-or-rerun-test stats test listener)) | |
d221e780 CO |
1255 | (setq abortedp nil)) |
1256 | (setf (ert--stats-aborted-p stats) abortedp) | |
1257 | (setf (ert--stats-end-time stats) (current-time)) | |
1258 | (funcall listener 'run-ended stats abortedp))) | |
1259 | (force-mode-line-update)) | |
1260 | stats))) | |
1261 | ||
1262 | (defun ert--stats-test-pos (stats test) | |
1263 | ;; checkdoc-order: nil | |
1264 | "Return the position (index) of TEST in the run represented by STATS." | |
1265 | (gethash (ert--stats-test-key test) (ert--stats-test-map stats))) | |
1266 | ||
1267 | ||
1268 | ;;; Formatting functions shared across UIs. | |
1269 | ||
1270 | (defun ert--format-time-iso8601 (time) | |
1271 | "Format TIME in the variant of ISO 8601 used for timestamps in ERT." | |
1272 | (format-time-string "%Y-%m-%d %T%z" time)) | |
1273 | ||
1274 | (defun ert-char-for-test-result (result expectedp) | |
1275 | "Return a character that represents the test result RESULT. | |
1276 | ||
1277 | EXPECTEDP specifies whether the result was expected." | |
15c9d04e | 1278 | (let ((s (cl-etypecase result |
d221e780 CO |
1279 | (ert-test-passed ".P") |
1280 | (ert-test-failed "fF") | |
50b5b857 | 1281 | (ert-test-skipped "sS") |
d221e780 | 1282 | (null "--") |
7c0d1441 CO |
1283 | (ert-test-aborted-with-non-local-exit "aA") |
1284 | (ert-test-quit "qQ")))) | |
d221e780 CO |
1285 | (elt s (if expectedp 0 1)))) |
1286 | ||
1287 | (defun ert-string-for-test-result (result expectedp) | |
1288 | "Return a string that represents the test result RESULT. | |
1289 | ||
1290 | EXPECTEDP specifies whether the result was expected." | |
15c9d04e | 1291 | (let ((s (cl-etypecase result |
d221e780 CO |
1292 | (ert-test-passed '("passed" "PASSED")) |
1293 | (ert-test-failed '("failed" "FAILED")) | |
50b5b857 | 1294 | (ert-test-skipped '("skipped" "SKIPPED")) |
d221e780 | 1295 | (null '("unknown" "UNKNOWN")) |
7c0d1441 CO |
1296 | (ert-test-aborted-with-non-local-exit '("aborted" "ABORTED")) |
1297 | (ert-test-quit '("quit" "QUIT"))))) | |
d221e780 CO |
1298 | (elt s (if expectedp 0 1)))) |
1299 | ||
1300 | (defun ert--pp-with-indentation-and-newline (object) | |
1301 | "Pretty-print OBJECT, indenting it to the current column of point. | |
1302 | Ensures a final newline is inserted." | |
1303 | (let ((begin (point))) | |
1304 | (pp object (current-buffer)) | |
1305 | (unless (bolp) (insert "\n")) | |
1306 | (save-excursion | |
1307 | (goto-char begin) | |
1308 | (indent-sexp)))) | |
1309 | ||
1310 | (defun ert--insert-infos (result) | |
1311 | "Insert `ert-info' infos from RESULT into current buffer. | |
1312 | ||
1313 | RESULT must be an `ert-test-result-with-condition'." | |
15c9d04e | 1314 | (cl-check-type result ert-test-result-with-condition) |
d221e780 | 1315 | (dolist (info (ert-test-result-with-condition-infos result)) |
15c9d04e | 1316 | (cl-destructuring-bind (prefix . message) info |
d221e780 CO |
1317 | (let ((begin (point)) |
1318 | (indentation (make-string (+ (length prefix) 4) ?\s)) | |
1319 | (end nil)) | |
1320 | (unwind-protect | |
1321 | (progn | |
1322 | (insert message "\n") | |
9c3883b4 | 1323 | (setq end (point-marker)) |
d221e780 CO |
1324 | (goto-char begin) |
1325 | (insert " " prefix) | |
1326 | (forward-line 1) | |
1327 | (while (< (point) end) | |
1328 | (insert indentation) | |
1329 | (forward-line 1))) | |
1330 | (when end (set-marker end nil))))))) | |
1331 | ||
1332 | ||
1333 | ;;; Running tests in batch mode. | |
1334 | ||
1335 | (defvar ert-batch-backtrace-right-margin 70 | |
fb7ada5f | 1336 | "The maximum line length for printing backtraces in `ert-run-tests-batch'.") |
d221e780 CO |
1337 | |
1338 | ;;;###autoload | |
1339 | (defun ert-run-tests-batch (&optional selector) | |
1340 | "Run the tests specified by SELECTOR, printing results to the terminal. | |
1341 | ||
1342 | SELECTOR works as described in `ert-select-tests', except if | |
1343 | SELECTOR is nil, in which case all tests rather than none will be | |
1344 | run; this makes the command line \"emacs -batch -l my-tests.el -f | |
1345 | ert-run-tests-batch-and-exit\" useful. | |
1346 | ||
1347 | Returns the stats object." | |
1348 | (unless selector (setq selector 't)) | |
1349 | (ert-run-tests | |
1350 | selector | |
1351 | (lambda (event-type &rest event-args) | |
15c9d04e | 1352 | (cl-ecase event-type |
d221e780 | 1353 | (run-started |
15c9d04e | 1354 | (cl-destructuring-bind (stats) event-args |
d221e780 CO |
1355 | (message "Running %s tests (%s)" |
1356 | (length (ert--stats-tests stats)) | |
1357 | (ert--format-time-iso8601 (ert--stats-start-time stats))))) | |
1358 | (run-ended | |
15c9d04e | 1359 | (cl-destructuring-bind (stats abortedp) event-args |
d221e780 | 1360 | (let ((unexpected (ert-stats-completed-unexpected stats)) |
50b5b857 MA |
1361 | (skipped (ert-stats-skipped stats)) |
1362 | (expected-failures (ert--stats-failed-expected stats))) | |
1363 | (message "\n%sRan %s tests, %s results as expected%s%s (%s)%s\n" | |
d221e780 CO |
1364 | (if (not abortedp) |
1365 | "" | |
1366 | "Aborted: ") | |
1367 | (ert-stats-total stats) | |
1368 | (ert-stats-completed-expected stats) | |
1369 | (if (zerop unexpected) | |
1370 | "" | |
1371 | (format ", %s unexpected" unexpected)) | |
50b5b857 MA |
1372 | (if (zerop skipped) |
1373 | "" | |
1374 | (format ", %s skipped" skipped)) | |
d221e780 CO |
1375 | (ert--format-time-iso8601 (ert--stats-end-time stats)) |
1376 | (if (zerop expected-failures) | |
1377 | "" | |
1378 | (format "\n%s expected failures" expected-failures))) | |
1379 | (unless (zerop unexpected) | |
1380 | (message "%s unexpected results:" unexpected) | |
15c9d04e SM |
1381 | (cl-loop for test across (ert--stats-tests stats) |
1382 | for result = (ert-test-most-recent-result test) do | |
1383 | (when (not (ert-test-result-expected-p test result)) | |
1384 | (message "%9s %S" | |
1385 | (ert-string-for-test-result result nil) | |
1386 | (ert-test-name test)))) | |
50b5b857 MA |
1387 | (message "%s" "")) |
1388 | (unless (zerop skipped) | |
1389 | (message "%s skipped results:" skipped) | |
1390 | (cl-loop for test across (ert--stats-tests stats) | |
1391 | for result = (ert-test-most-recent-result test) do | |
1392 | (when (ert-test-result-type-p result :skipped) | |
1393 | (message "%9s %S" | |
1394 | (ert-string-for-test-result result nil) | |
1395 | (ert-test-name test)))) | |
d221e780 CO |
1396 | (message "%s" ""))))) |
1397 | (test-started | |
1398 | ) | |
1399 | (test-ended | |
15c9d04e | 1400 | (cl-destructuring-bind (stats test result) event-args |
d221e780 | 1401 | (unless (ert-test-result-expected-p test result) |
15c9d04e | 1402 | (cl-etypecase result |
d221e780 CO |
1403 | (ert-test-passed |
1404 | (message "Test %S passed unexpectedly" (ert-test-name test))) | |
1405 | (ert-test-result-with-condition | |
1406 | (message "Test %S backtrace:" (ert-test-name test)) | |
1407 | (with-temp-buffer | |
1408 | (ert--print-backtrace (ert-test-result-with-condition-backtrace | |
1409 | result)) | |
1410 | (goto-char (point-min)) | |
1411 | (while (not (eobp)) | |
1412 | (let ((start (point)) | |
1413 | (end (progn (end-of-line) (point)))) | |
1414 | (setq end (min end | |
1415 | (+ start ert-batch-backtrace-right-margin))) | |
1416 | (message "%s" (buffer-substring-no-properties | |
1417 | start end))) | |
1418 | (forward-line 1))) | |
1419 | (with-temp-buffer | |
1420 | (ert--insert-infos result) | |
1421 | (insert " ") | |
1422 | (let ((print-escape-newlines t) | |
1423 | (print-level 5) | |
1424 | (print-length 10)) | |
7d476bde CO |
1425 | (ert--pp-with-indentation-and-newline |
1426 | (ert-test-result-with-condition-condition result))) | |
d221e780 | 1427 | (goto-char (1- (point-max))) |
15c9d04e | 1428 | (cl-assert (looking-at "\n")) |
d221e780 CO |
1429 | (delete-char 1) |
1430 | (message "Test %S condition:" (ert-test-name test)) | |
1431 | (message "%s" (buffer-string)))) | |
1432 | (ert-test-aborted-with-non-local-exit | |
1433 | (message "Test %S aborted with non-local exit" | |
7c0d1441 CO |
1434 | (ert-test-name test))) |
1435 | (ert-test-quit | |
1436 | (message "Quit during %S" (ert-test-name test))))) | |
d221e780 CO |
1437 | (let* ((max (prin1-to-string (length (ert--stats-tests stats)))) |
1438 | (format-string (concat "%9s %" | |
1439 | (prin1-to-string (length max)) | |
1440 | "s/" max " %S"))) | |
1441 | (message format-string | |
1442 | (ert-string-for-test-result result | |
1443 | (ert-test-result-expected-p | |
1444 | test result)) | |
1445 | (1+ (ert--stats-test-pos stats test)) | |
1446 | (ert-test-name test))))))))) | |
1447 | ||
1448 | ;;;###autoload | |
1449 | (defun ert-run-tests-batch-and-exit (&optional selector) | |
1450 | "Like `ert-run-tests-batch', but exits Emacs when done. | |
1451 | ||
1452 | The exit status will be 0 if all test results were as expected, 1 | |
1453 | on unexpected results, or 2 if the tool detected an error outside | |
1454 | of the tests (e.g. invalid SELECTOR or bug in the code that runs | |
1455 | the tests)." | |
1456 | (unwind-protect | |
1457 | (let ((stats (ert-run-tests-batch selector))) | |
1458 | (kill-emacs (if (zerop (ert-stats-completed-unexpected stats)) 0 1))) | |
1459 | (unwind-protect | |
1460 | (progn | |
1461 | (message "Error running tests") | |
1462 | (backtrace)) | |
1463 | (kill-emacs 2)))) | |
1464 | ||
1465 | ||
5a8816f3 GM |
1466 | (defun ert-summarize-tests-batch-and-exit () |
1467 | "Summarize the results of testing. | |
1468 | Expects to be called in batch mode, with logfiles as command-line arguments. | |
1469 | The logfiles should have the `ert-run-tests-batch' format. When finished, | |
1470 | this exits Emacs, with status as per `ert-run-tests-batch-and-exit'." | |
1471 | (or noninteractive | |
1472 | (user-error "This function is only for use in batch mode")) | |
1473 | (let ((nlogs (length command-line-args-left)) | |
1474 | (ntests 0) (nrun 0) (nexpected 0) (nunexpected 0) (nskipped 0) | |
1475 | nnotrun logfile notests badtests unexpected) | |
1476 | (with-temp-buffer | |
1477 | (while (setq logfile (pop command-line-args-left)) | |
1478 | (erase-buffer) | |
1479 | (insert-file-contents logfile) | |
1480 | (if (not (re-search-forward "^Running \\([0-9]+\\) tests" nil t)) | |
1481 | (push logfile notests) | |
1482 | (setq ntests (+ ntests (string-to-number (match-string 1)))) | |
1483 | (if (not (re-search-forward "^\\(Aborted: \\)?\ | |
1484 | Ran \\([0-9]+\\) tests, \\([0-9]+\\) results as expected\ | |
1485 | \\(?:, \\([0-9]+\\) unexpected\\)?\ | |
1486 | \\(?:, \\([0-9]+\\) skipped\\)?" nil t)) | |
1487 | (push logfile badtests) | |
1488 | (if (match-string 1) (push logfile badtests)) | |
1489 | (setq nrun (+ nrun (string-to-number (match-string 2))) | |
1490 | nexpected (+ nexpected (string-to-number (match-string 3)))) | |
1491 | (when (match-string 4) | |
1492 | (push logfile unexpected) | |
1493 | (setq nunexpected (+ nunexpected | |
1494 | (string-to-number (match-string 4))))) | |
1495 | (if (match-string 5) | |
1496 | (setq nskipped (+ nskipped | |
1497 | (string-to-number (match-string 5))))))))) | |
1498 | (setq nnotrun (- ntests nrun)) | |
1499 | (message "\nSUMMARY OF TEST RESULTS") | |
1500 | (message "-----------------------") | |
1501 | (message "Files examined: %d" nlogs) | |
1502 | (message "Ran %d tests%s, %d results as expected%s%s" | |
1503 | nrun | |
1504 | (if (zerop nnotrun) "" (format ", %d failed to run" nnotrun)) | |
1505 | nexpected | |
1506 | (if (zerop nunexpected) | |
1507 | "" | |
1508 | (format ", %d unexpected" nunexpected)) | |
1509 | (if (zerop nskipped) | |
1510 | "" | |
1511 | (format ", %d skipped" nskipped))) | |
1512 | (when notests | |
1513 | (message "%d files did not contain any tests:" (length notests)) | |
1514 | (mapc (lambda (l) (message " %s" l)) notests)) | |
1515 | (when badtests | |
1516 | (message "%d files did not finish:" (length badtests)) | |
1517 | (mapc (lambda (l) (message " %s" l)) badtests)) | |
1518 | (when unexpected | |
1519 | (message "%d files contained unexpected results:" (length unexpected)) | |
1520 | (mapc (lambda (l) (message " %s" l)) unexpected)) | |
1521 | (kill-emacs (cond ((or notests badtests (not (zerop nnotrun))) 2) | |
1522 | (unexpected 1) | |
1523 | (t 0))))) | |
1524 | ||
d221e780 CO |
1525 | ;;; Utility functions for load/unload actions. |
1526 | ||
1527 | (defun ert--activate-font-lock-keywords () | |
1528 | "Activate font-lock keywords for some of ERT's symbols." | |
1529 | (font-lock-add-keywords | |
1530 | nil | |
08e41897 | 1531 | '(("(\\(\\<ert-deftest\\)\\>\\s *\\(\\(?:\\sw\\|\\s_\\)+\\)?" |
d221e780 CO |
1532 | (1 font-lock-keyword-face nil t) |
1533 | (2 font-lock-function-name-face nil t))))) | |
1534 | ||
15c9d04e | 1535 | (cl-defun ert--remove-from-list (list-var element &key key test) |
d221e780 CO |
1536 | "Remove ELEMENT from the value of LIST-VAR if present. |
1537 | ||
1538 | This can be used as an inverse of `add-to-list'." | |
1539 | (unless key (setq key #'identity)) | |
1540 | (unless test (setq test #'equal)) | |
1541 | (setf (symbol-value list-var) | |
a19b3c2d GM |
1542 | (cl-remove element |
1543 | (symbol-value list-var) | |
1544 | :key key | |
1545 | :test test))) | |
d221e780 CO |
1546 | |
1547 | ||
1548 | ;;; Some basic interactive functions. | |
1549 | ||
1550 | (defun ert-read-test-name (prompt &optional default history | |
1551 | add-default-to-prompt) | |
1552 | "Read the name of a test and return it as a symbol. | |
1553 | ||
1554 | Prompt with PROMPT. If DEFAULT is a valid test name, use it as a | |
1555 | default. HISTORY is the history to use; see `completing-read'. | |
1556 | If ADD-DEFAULT-TO-PROMPT is non-nil, PROMPT will be modified to | |
1557 | include the default, if any. | |
1558 | ||
1559 | Signals an error if no test name was read." | |
15c9d04e | 1560 | (cl-etypecase default |
d221e780 CO |
1561 | (string (let ((symbol (intern-soft default))) |
1562 | (unless (and symbol (ert-test-boundp symbol)) | |
1563 | (setq default nil)))) | |
1564 | (symbol (setq default | |
1565 | (if (ert-test-boundp default) | |
1566 | (symbol-name default) | |
1567 | nil))) | |
1568 | (ert-test (setq default (ert-test-name default)))) | |
1569 | (when add-default-to-prompt | |
1570 | (setq prompt (if (null default) | |
1571 | (format "%s: " prompt) | |
1572 | (format "%s (default %s): " prompt default)))) | |
1573 | (let ((input (completing-read prompt obarray #'ert-test-boundp | |
1574 | t nil history default nil))) | |
1575 | ;; completing-read returns an empty string if default was nil and | |
1576 | ;; the user just hit enter. | |
1577 | (let ((sym (intern-soft input))) | |
1578 | (if (ert-test-boundp sym) | |
1579 | sym | |
1580 | (error "Input does not name a test"))))) | |
1581 | ||
1582 | (defun ert-read-test-name-at-point (prompt) | |
1583 | "Read the name of a test and return it as a symbol. | |
1584 | As a default, use the symbol at point, or the test at point if in | |
1585 | the ERT results buffer. Prompt with PROMPT, augmented with the | |
1586 | default (if any)." | |
1587 | (ert-read-test-name prompt (ert-test-at-point) nil t)) | |
1588 | ||
1589 | (defun ert-find-test-other-window (test-name) | |
1590 | "Find, in another window, the definition of TEST-NAME." | |
1591 | (interactive (list (ert-read-test-name-at-point "Find test definition: "))) | |
1592 | (find-function-do-it test-name 'ert-deftest 'switch-to-buffer-other-window)) | |
1593 | ||
1594 | (defun ert-delete-test (test-name) | |
1595 | "Make the test TEST-NAME unbound. | |
1596 | ||
1597 | Nothing more than an interactive interface to `ert-make-test-unbound'." | |
1598 | (interactive (list (ert-read-test-name-at-point "Delete test"))) | |
1599 | (ert-make-test-unbound test-name)) | |
1600 | ||
1601 | (defun ert-delete-all-tests () | |
1602 | "Make all symbols in `obarray' name no test." | |
1603 | (interactive) | |
7d476bde | 1604 | (when (called-interactively-p 'any) |
d221e780 CO |
1605 | (unless (y-or-n-p "Delete all tests? ") |
1606 | (error "Aborted"))) | |
1607 | ;; We can't use `ert-select-tests' here since that gives us only | |
1608 | ;; test objects, and going from them back to the test name symbols | |
1609 | ;; can fail if the `ert-test' defstruct has been redefined. | |
1610 | (mapc #'ert-make-test-unbound (apropos-internal "" #'ert-test-boundp)) | |
1611 | t) | |
1612 | ||
1613 | ||
1614 | ;;; Display of test progress and results. | |
1615 | ||
1616 | ;; An entry in the results buffer ewoc. There is one entry per test. | |
15c9d04e SM |
1617 | (cl-defstruct ert--ewoc-entry |
1618 | (test (cl-assert nil)) | |
d221e780 CO |
1619 | ;; If the result of this test was expected, its ewoc entry is hidden |
1620 | ;; initially. | |
15c9d04e | 1621 | (hidden-p (cl-assert nil)) |
d221e780 CO |
1622 | ;; An ewoc entry may be collapsed to hide details such as the error |
1623 | ;; condition. | |
1624 | ;; | |
1625 | ;; I'm not sure the ability to expand and collapse entries is still | |
1626 | ;; a useful feature. | |
1627 | (expanded-p t) | |
1628 | ;; By default, the ewoc entry presents the error condition with | |
1629 | ;; certain limits on how much to print (`print-level', | |
1630 | ;; `print-length'). The user can interactively switch to a set of | |
1631 | ;; higher limits. | |
1632 | (extended-printer-limits-p nil)) | |
1633 | ||
1634 | ;; Variables local to the results buffer. | |
1635 | ||
1636 | ;; The ewoc. | |
1637 | (defvar ert--results-ewoc) | |
1638 | ;; The stats object. | |
1639 | (defvar ert--results-stats) | |
1640 | ;; A string with one character per test. Each character represents | |
1641 | ;; the result of the corresponding test. The string is displayed near | |
1642 | ;; the top of the buffer and serves as a progress bar. | |
1643 | (defvar ert--results-progress-bar-string) | |
1644 | ;; The position where the progress bar button begins. | |
1645 | (defvar ert--results-progress-bar-button-begin) | |
1646 | ;; The test result listener that updates the buffer when tests are run. | |
1647 | (defvar ert--results-listener) | |
1648 | ||
1649 | (defun ert-insert-test-name-button (test-name) | |
1650 | "Insert a button that links to TEST-NAME." | |
1651 | (insert-text-button (format "%S" test-name) | |
1652 | :type 'ert--test-name-button | |
1653 | 'ert-test-name test-name)) | |
1654 | ||
1655 | (defun ert--results-format-expected-unexpected (expected unexpected) | |
1656 | "Return a string indicating EXPECTED expected results, UNEXPECTED unexpected." | |
1657 | (if (zerop unexpected) | |
1658 | (format "%s" expected) | |
1659 | (format "%s (%s unexpected)" (+ expected unexpected) unexpected))) | |
1660 | ||
1661 | (defun ert--results-update-ewoc-hf (ewoc stats) | |
1662 | "Update the header and footer of EWOC to show certain information from STATS. | |
1663 | ||
1664 | Also sets `ert--results-progress-bar-button-begin'." | |
1665 | (let ((run-count (ert-stats-completed stats)) | |
1666 | (results-buffer (current-buffer)) | |
1667 | ;; Need to save buffer-local value. | |
1668 | (font-lock font-lock-mode)) | |
1669 | (ewoc-set-hf | |
1670 | ewoc | |
1671 | ;; header | |
1672 | (with-temp-buffer | |
1673 | (insert "Selector: ") | |
1674 | (ert--insert-human-readable-selector (ert--stats-selector stats)) | |
1675 | (insert "\n") | |
1676 | (insert | |
50b5b857 MA |
1677 | (format (concat "Passed: %s\n" |
1678 | "Failed: %s\n" | |
1679 | "Skipped: %s\n" | |
1680 | "Total: %s/%s\n\n") | |
d221e780 CO |
1681 | (ert--results-format-expected-unexpected |
1682 | (ert--stats-passed-expected stats) | |
1683 | (ert--stats-passed-unexpected stats)) | |
1684 | (ert--results-format-expected-unexpected | |
1685 | (ert--stats-failed-expected stats) | |
1686 | (ert--stats-failed-unexpected stats)) | |
50b5b857 | 1687 | (ert-stats-skipped stats) |
d221e780 CO |
1688 | run-count |
1689 | (ert-stats-total stats))) | |
1690 | (insert | |
1691 | (format "Started at: %s\n" | |
1692 | (ert--format-time-iso8601 (ert--stats-start-time stats)))) | |
1693 | ;; FIXME: This is ugly. Need to properly define invariants of | |
1694 | ;; the `stats' data structure. | |
1695 | (let ((state (cond ((ert--stats-aborted-p stats) 'aborted) | |
1696 | ((ert--stats-current-test stats) 'running) | |
1697 | ((ert--stats-end-time stats) 'finished) | |
1698 | (t 'preparing)))) | |
15c9d04e | 1699 | (cl-ecase state |
d221e780 CO |
1700 | (preparing |
1701 | (insert "")) | |
1702 | (aborted | |
1703 | (cond ((ert--stats-current-test stats) | |
1704 | (insert "Aborted during test: ") | |
1705 | (ert-insert-test-name-button | |
1706 | (ert-test-name (ert--stats-current-test stats)))) | |
1707 | (t | |
1708 | (insert "Aborted.")))) | |
1709 | (running | |
15c9d04e | 1710 | (cl-assert (ert--stats-current-test stats)) |
d221e780 CO |
1711 | (insert "Running test: ") |
1712 | (ert-insert-test-name-button (ert-test-name | |
1713 | (ert--stats-current-test stats)))) | |
1714 | (finished | |
15c9d04e | 1715 | (cl-assert (not (ert--stats-current-test stats))) |
d221e780 CO |
1716 | (insert "Finished."))) |
1717 | (insert "\n") | |
1718 | (if (ert--stats-end-time stats) | |
1719 | (insert | |
1720 | (format "%s%s\n" | |
1721 | (if (ert--stats-aborted-p stats) | |
1722 | "Aborted at: " | |
1723 | "Finished at: ") | |
1724 | (ert--format-time-iso8601 (ert--stats-end-time stats)))) | |
1725 | (insert "\n")) | |
1726 | (insert "\n")) | |
1727 | (let ((progress-bar-string (with-current-buffer results-buffer | |
1728 | ert--results-progress-bar-string))) | |
1729 | (let ((progress-bar-button-begin | |
1730 | (insert-text-button progress-bar-string | |
1731 | :type 'ert--results-progress-bar-button | |
1732 | 'face (or (and font-lock | |
1733 | (ert-face-for-stats stats)) | |
1734 | 'button)))) | |
1735 | ;; The header gets copied verbatim to the results buffer, | |
1736 | ;; and all positions remain the same, so | |
1737 | ;; `progress-bar-button-begin' will be the right position | |
1738 | ;; even in the results buffer. | |
1739 | (with-current-buffer results-buffer | |
1740 | (set (make-local-variable 'ert--results-progress-bar-button-begin) | |
1741 | progress-bar-button-begin)))) | |
1742 | (insert "\n\n") | |
1743 | (buffer-string)) | |
1744 | ;; footer | |
1745 | ;; | |
1746 | ;; We actually want an empty footer, but that would trigger a bug | |
1747 | ;; in ewoc, sometimes clearing the entire buffer. (It's possible | |
1748 | ;; that this bug has been fixed since this has been tested; we | |
1749 | ;; should test it again.) | |
1750 | "\n"))) | |
1751 | ||
1752 | ||
1753 | (defvar ert-test-run-redisplay-interval-secs .1 | |
1754 | "How many seconds ERT should wait between redisplays while running tests. | |
1755 | ||
1756 | While running tests, ERT shows the current progress, and this variable | |
1757 | determines how frequently the progress display is updated.") | |
1758 | ||
1759 | (defun ert--results-update-stats-display (ewoc stats) | |
1760 | "Update EWOC and the mode line to show data from STATS." | |
1761 | ;; TODO(ohler): investigate using `make-progress-reporter'. | |
1762 | (ert--results-update-ewoc-hf ewoc stats) | |
1763 | (force-mode-line-update) | |
1764 | (redisplay t) | |
1765 | (setf (ert--stats-next-redisplay stats) | |
1766 | (+ (float-time) ert-test-run-redisplay-interval-secs))) | |
1767 | ||
1768 | (defun ert--results-update-stats-display-maybe (ewoc stats) | |
1769 | "Call `ert--results-update-stats-display' if not called recently. | |
1770 | ||
1771 | EWOC and STATS are arguments for `ert--results-update-stats-display'." | |
1772 | (when (>= (float-time) (ert--stats-next-redisplay stats)) | |
1773 | (ert--results-update-stats-display ewoc stats))) | |
1774 | ||
1775 | (defun ert--tests-running-mode-line-indicator () | |
1776 | "Return a string for the mode line that shows the test run progress." | |
1777 | (let* ((stats ert--current-run-stats) | |
1778 | (tests-total (ert-stats-total stats)) | |
1779 | (tests-completed (ert-stats-completed stats))) | |
1780 | (if (>= tests-completed tests-total) | |
1781 | (format " ERT(%s/%s,finished)" tests-completed tests-total) | |
1782 | (format " ERT(%s/%s):%s" | |
1783 | (1+ tests-completed) | |
1784 | tests-total | |
1785 | (if (null (ert--stats-current-test stats)) | |
1786 | "?" | |
1787 | (format "%S" | |
1788 | (ert-test-name (ert--stats-current-test stats)))))))) | |
1789 | ||
1790 | (defun ert--make-xrefs-region (begin end) | |
1791 | "Attach cross-references to function names between BEGIN and END. | |
1792 | ||
1793 | BEGIN and END specify a region in the current buffer." | |
1794 | (save-excursion | |
1795 | (save-restriction | |
7d476bde | 1796 | (narrow-to-region begin end) |
d221e780 CO |
1797 | ;; Inhibit optimization in `debugger-make-xrefs' that would |
1798 | ;; sometimes insert unrelated backtrace info into our buffer. | |
1799 | (let ((debugger-previous-backtrace nil)) | |
1800 | (debugger-make-xrefs))))) | |
1801 | ||
1802 | (defun ert--string-first-line (s) | |
1803 | "Return the first line of S, or S if it contains no newlines. | |
1804 | ||
1805 | The return value does not include the line terminator." | |
a19b3c2d | 1806 | (substring s 0 (cl-position ?\n s))) |
d221e780 CO |
1807 | |
1808 | (defun ert-face-for-test-result (expectedp) | |
1809 | "Return a face that shows whether a test result was expected or unexpected. | |
1810 | ||
1811 | If EXPECTEDP is nil, returns the face for unexpected results; if | |
1812 | non-nil, returns the face for expected results.." | |
1813 | (if expectedp 'ert-test-result-expected 'ert-test-result-unexpected)) | |
1814 | ||
1815 | (defun ert-face-for-stats (stats) | |
1816 | "Return a face that represents STATS." | |
1817 | (cond ((ert--stats-aborted-p stats) 'nil) | |
15c9d04e | 1818 | ((cl-plusp (ert-stats-completed-unexpected stats)) |
d221e780 CO |
1819 | (ert-face-for-test-result nil)) |
1820 | ((eql (ert-stats-completed-expected stats) (ert-stats-total stats)) | |
1821 | (ert-face-for-test-result t)) | |
1822 | (t 'nil))) | |
1823 | ||
1824 | (defun ert--print-test-for-ewoc (entry) | |
1825 | "The ewoc print function for ewoc test entries. ENTRY is the entry to print." | |
1826 | (let* ((test (ert--ewoc-entry-test entry)) | |
1827 | (stats ert--results-stats) | |
1828 | (result (let ((pos (ert--stats-test-pos stats test))) | |
15c9d04e | 1829 | (cl-assert pos) |
d221e780 CO |
1830 | (aref (ert--stats-test-results stats) pos))) |
1831 | (hiddenp (ert--ewoc-entry-hidden-p entry)) | |
1832 | (expandedp (ert--ewoc-entry-expanded-p entry)) | |
1833 | (extended-printer-limits-p (ert--ewoc-entry-extended-printer-limits-p | |
1834 | entry))) | |
1835 | (cond (hiddenp) | |
1836 | (t | |
1837 | (let ((expectedp (ert-test-result-expected-p test result))) | |
1838 | (insert-text-button (format "%c" (ert-char-for-test-result | |
1839 | result expectedp)) | |
1840 | :type 'ert--results-expand-collapse-button | |
1841 | 'face (or (and font-lock-mode | |
1842 | (ert-face-for-test-result | |
1843 | expectedp)) | |
1844 | 'button))) | |
1845 | (insert " ") | |
1846 | (ert-insert-test-name-button (ert-test-name test)) | |
1847 | (insert "\n") | |
1848 | (when (and expandedp (not (eql result 'nil))) | |
1849 | (when (ert-test-documentation test) | |
1850 | (insert " " | |
1851 | (propertize | |
1852 | (ert--string-first-line (ert-test-documentation test)) | |
1853 | 'font-lock-face 'font-lock-doc-face) | |
1854 | "\n")) | |
15c9d04e | 1855 | (cl-etypecase result |
d221e780 CO |
1856 | (ert-test-passed |
1857 | (if (ert-test-result-expected-p test result) | |
1858 | (insert " passed\n") | |
1859 | (insert " passed unexpectedly\n")) | |
1860 | (insert "")) | |
1861 | (ert-test-result-with-condition | |
1862 | (ert--insert-infos result) | |
1863 | (let ((print-escape-newlines t) | |
1864 | (print-level (if extended-printer-limits-p 12 6)) | |
1865 | (print-length (if extended-printer-limits-p 100 10))) | |
1866 | (insert " ") | |
1867 | (let ((begin (point))) | |
1868 | (ert--pp-with-indentation-and-newline | |
1869 | (ert-test-result-with-condition-condition result)) | |
1870 | (ert--make-xrefs-region begin (point))))) | |
1871 | (ert-test-aborted-with-non-local-exit | |
7c0d1441 CO |
1872 | (insert " aborted\n")) |
1873 | (ert-test-quit | |
1874 | (insert " quit\n"))) | |
d221e780 CO |
1875 | (insert "\n"))))) |
1876 | nil) | |
1877 | ||
1878 | (defun ert--results-font-lock-function (enabledp) | |
1879 | "Redraw the ERT results buffer after font-lock-mode was switched on or off. | |
1880 | ||
1881 | ENABLEDP is true if font-lock-mode is switched on, false | |
1882 | otherwise." | |
1883 | (ert--results-update-ewoc-hf ert--results-ewoc ert--results-stats) | |
1884 | (ewoc-refresh ert--results-ewoc) | |
1885 | (font-lock-default-function enabledp)) | |
1886 | ||
1887 | (defun ert--setup-results-buffer (stats listener buffer-name) | |
1888 | "Set up a test results buffer. | |
1889 | ||
1890 | STATS is the stats object; LISTENER is the results listener; | |
1891 | BUFFER-NAME, if non-nil, is the buffer name to use." | |
1892 | (unless buffer-name (setq buffer-name "*ert*")) | |
1893 | (let ((buffer (get-buffer-create buffer-name))) | |
1894 | (with-current-buffer buffer | |
d221e780 CO |
1895 | (let ((inhibit-read-only t)) |
1896 | (buffer-disable-undo) | |
1897 | (erase-buffer) | |
5da16a86 | 1898 | (ert-results-mode) |
d221e780 CO |
1899 | ;; Erase buffer again in case switching out of the previous |
1900 | ;; mode inserted anything. (This happens e.g. when switching | |
1901 | ;; from ert-results-mode to ert-results-mode when | |
1902 | ;; font-lock-mode turns itself off in change-major-mode-hook.) | |
1903 | (erase-buffer) | |
1904 | (set (make-local-variable 'font-lock-function) | |
1905 | 'ert--results-font-lock-function) | |
1906 | (let ((ewoc (ewoc-create 'ert--print-test-for-ewoc nil nil t))) | |
1907 | (set (make-local-variable 'ert--results-ewoc) ewoc) | |
1908 | (set (make-local-variable 'ert--results-stats) stats) | |
1909 | (set (make-local-variable 'ert--results-progress-bar-string) | |
1910 | (make-string (ert-stats-total stats) | |
1911 | (ert-char-for-test-result nil t))) | |
1912 | (set (make-local-variable 'ert--results-listener) listener) | |
15c9d04e SM |
1913 | (cl-loop for test across (ert--stats-tests stats) do |
1914 | (ewoc-enter-last ewoc | |
1915 | (make-ert--ewoc-entry :test test | |
1916 | :hidden-p t))) | |
d221e780 | 1917 | (ert--results-update-ewoc-hf ert--results-ewoc ert--results-stats) |
5da16a86 CO |
1918 | (goto-char (1- (point-max))) |
1919 | buffer))))) | |
d221e780 CO |
1920 | |
1921 | ||
1922 | (defvar ert--selector-history nil | |
1923 | "List of recent test selectors read from terminal.") | |
1924 | ||
1925 | ;; Should OUTPUT-BUFFER-NAME and MESSAGE-FN really be arguments here? | |
1926 | ;; They are needed only for our automated self-tests at the moment. | |
1927 | ;; Or should there be some other mechanism? | |
1928 | ;;;###autoload | |
1929 | (defun ert-run-tests-interactively (selector | |
1930 | &optional output-buffer-name message-fn) | |
1931 | "Run the tests specified by SELECTOR and display the results in a buffer. | |
1932 | ||
1933 | SELECTOR works as described in `ert-select-tests'. | |
1934 | OUTPUT-BUFFER-NAME and MESSAGE-FN should normally be nil; they | |
1935 | are used for automated self-tests and specify which buffer to use | |
1936 | and how to display message." | |
1937 | (interactive | |
1938 | (list (let ((default (if ert--selector-history | |
1939 | ;; Can't use `first' here as this form is | |
1940 | ;; not compiled, and `first' is not | |
1941 | ;; defined without cl. | |
1942 | (car ert--selector-history) | |
1943 | "t"))) | |
40ff7f86 DG |
1944 | (read |
1945 | (completing-read (if (null default) | |
1946 | "Run tests: " | |
1947 | (format "Run tests (default %s): " default)) | |
1948 | obarray #'ert-test-boundp nil nil | |
1949 | 'ert--selector-history default nil))) | |
d221e780 CO |
1950 | nil)) |
1951 | (unless message-fn (setq message-fn 'message)) | |
15c9d04e SM |
1952 | (let ((output-buffer-name output-buffer-name) |
1953 | buffer | |
1954 | listener | |
1955 | (message-fn message-fn)) | |
d221e780 CO |
1956 | (setq listener |
1957 | (lambda (event-type &rest event-args) | |
15c9d04e | 1958 | (cl-ecase event-type |
d221e780 | 1959 | (run-started |
15c9d04e | 1960 | (cl-destructuring-bind (stats) event-args |
d221e780 CO |
1961 | (setq buffer (ert--setup-results-buffer stats |
1962 | listener | |
1963 | output-buffer-name)) | |
1964 | (pop-to-buffer buffer))) | |
1965 | (run-ended | |
15c9d04e | 1966 | (cl-destructuring-bind (stats abortedp) event-args |
d221e780 | 1967 | (funcall message-fn |
50b5b857 | 1968 | "%sRan %s tests, %s results were as expected%s%s" |
d221e780 CO |
1969 | (if (not abortedp) |
1970 | "" | |
1971 | "Aborted: ") | |
1972 | (ert-stats-total stats) | |
1973 | (ert-stats-completed-expected stats) | |
1974 | (let ((unexpected | |
1975 | (ert-stats-completed-unexpected stats))) | |
1976 | (if (zerop unexpected) | |
1977 | "" | |
50b5b857 MA |
1978 | (format ", %s unexpected" unexpected))) |
1979 | (let ((skipped | |
1980 | (ert-stats-skipped stats))) | |
1981 | (if (zerop skipped) | |
1982 | "" | |
1983 | (format ", %s skipped" skipped)))) | |
d221e780 CO |
1984 | (ert--results-update-stats-display (with-current-buffer buffer |
1985 | ert--results-ewoc) | |
1986 | stats))) | |
1987 | (test-started | |
15c9d04e | 1988 | (cl-destructuring-bind (stats test) event-args |
d221e780 CO |
1989 | (with-current-buffer buffer |
1990 | (let* ((ewoc ert--results-ewoc) | |
1991 | (pos (ert--stats-test-pos stats test)) | |
1992 | (node (ewoc-nth ewoc pos))) | |
15c9d04e | 1993 | (cl-assert node) |
d221e780 CO |
1994 | (setf (ert--ewoc-entry-test (ewoc-data node)) test) |
1995 | (aset ert--results-progress-bar-string pos | |
1996 | (ert-char-for-test-result nil t)) | |
1997 | (ert--results-update-stats-display-maybe ewoc stats) | |
1998 | (ewoc-invalidate ewoc node))))) | |
1999 | (test-ended | |
15c9d04e | 2000 | (cl-destructuring-bind (stats test result) event-args |
d221e780 CO |
2001 | (with-current-buffer buffer |
2002 | (let* ((ewoc ert--results-ewoc) | |
2003 | (pos (ert--stats-test-pos stats test)) | |
2004 | (node (ewoc-nth ewoc pos))) | |
2005 | (when (ert--ewoc-entry-hidden-p (ewoc-data node)) | |
2006 | (setf (ert--ewoc-entry-hidden-p (ewoc-data node)) | |
2007 | (ert-test-result-expected-p test result))) | |
2008 | (aset ert--results-progress-bar-string pos | |
2009 | (ert-char-for-test-result result | |
2010 | (ert-test-result-expected-p | |
2011 | test result))) | |
2012 | (ert--results-update-stats-display-maybe ewoc stats) | |
2013 | (ewoc-invalidate ewoc node)))))))) | |
2014 | (ert-run-tests | |
2015 | selector | |
2016 | listener))) | |
2017 | ;;;###autoload | |
2018 | (defalias 'ert 'ert-run-tests-interactively) | |
2019 | ||
2020 | ||
2021 | ;;; Simple view mode for auxiliary information like stack traces or | |
2022 | ;;; messages. Mainly binds "q" for quit. | |
2023 | ||
abef340a | 2024 | (define-derived-mode ert-simple-view-mode special-mode "ERT-View" |
d221e780 CO |
2025 | "Major mode for viewing auxiliary information in ERT.") |
2026 | ||
d221e780 CO |
2027 | ;;; Commands and button actions for the results buffer. |
2028 | ||
abef340a | 2029 | (define-derived-mode ert-results-mode special-mode "ERT-Results" |
d221e780 CO |
2030 | "Major mode for viewing results of ERT test runs.") |
2031 | ||
15c9d04e SM |
2032 | (cl-loop for (key binding) in |
2033 | '( ;; Stuff that's not in the menu. | |
2034 | ("\t" forward-button) | |
2035 | ([backtab] backward-button) | |
2036 | ("j" ert-results-jump-between-summary-and-result) | |
2037 | ("L" ert-results-toggle-printer-limits-for-test-at-point) | |
2038 | ("n" ert-results-next-test) | |
2039 | ("p" ert-results-previous-test) | |
2040 | ;; Stuff that is in the menu. | |
2041 | ("R" ert-results-rerun-all-tests) | |
2042 | ("r" ert-results-rerun-test-at-point) | |
2043 | ("d" ert-results-rerun-test-at-point-debugging-errors) | |
2044 | ("." ert-results-find-test-at-point-other-window) | |
2045 | ("b" ert-results-pop-to-backtrace-for-test-at-point) | |
2046 | ("m" ert-results-pop-to-messages-for-test-at-point) | |
2047 | ("l" ert-results-pop-to-should-forms-for-test-at-point) | |
2048 | ("h" ert-results-describe-test-at-point) | |
2049 | ("D" ert-delete-test) | |
2050 | ("T" ert-results-pop-to-timings) | |
2051 | ) | |
2052 | do | |
2053 | (define-key ert-results-mode-map key binding)) | |
d221e780 CO |
2054 | |
2055 | (easy-menu-define ert-results-mode-menu ert-results-mode-map | |
2056 | "Menu for `ert-results-mode'." | |
2057 | '("ERT Results" | |
2058 | ["Re-run all tests" ert-results-rerun-all-tests] | |
2059 | "--" | |
2060 | ["Re-run test" ert-results-rerun-test-at-point] | |
2061 | ["Debug test" ert-results-rerun-test-at-point-debugging-errors] | |
2062 | ["Show test definition" ert-results-find-test-at-point-other-window] | |
2063 | "--" | |
2064 | ["Show backtrace" ert-results-pop-to-backtrace-for-test-at-point] | |
2065 | ["Show messages" ert-results-pop-to-messages-for-test-at-point] | |
2066 | ["Show `should' forms" ert-results-pop-to-should-forms-for-test-at-point] | |
2067 | ["Describe test" ert-results-describe-test-at-point] | |
2068 | "--" | |
2069 | ["Delete test" ert-delete-test] | |
2070 | "--" | |
2071 | ["Show execution time of each test" ert-results-pop-to-timings] | |
2072 | )) | |
2073 | ||
2074 | (define-button-type 'ert--results-progress-bar-button | |
2075 | 'action #'ert--results-progress-bar-button-action | |
2076 | 'help-echo "mouse-2, RET: Reveal test result") | |
2077 | ||
2078 | (define-button-type 'ert--test-name-button | |
2079 | 'action #'ert--test-name-button-action | |
2080 | 'help-echo "mouse-2, RET: Find test definition") | |
2081 | ||
2082 | (define-button-type 'ert--results-expand-collapse-button | |
2083 | 'action #'ert--results-expand-collapse-button-action | |
2084 | 'help-echo "mouse-2, RET: Expand/collapse test result") | |
2085 | ||
2086 | (defun ert--results-test-node-or-null-at-point () | |
2087 | "If point is on a valid ewoc node, return it; return nil otherwise. | |
2088 | ||
2089 | To be used in the ERT results buffer." | |
2090 | (let* ((ewoc ert--results-ewoc) | |
2091 | (node (ewoc-locate ewoc))) | |
2092 | ;; `ewoc-locate' will return an arbitrary node when point is on | |
2093 | ;; header or footer, or when all nodes are invisible. So we need | |
2094 | ;; to validate its return value here. | |
2095 | ;; | |
2096 | ;; Update: I'm seeing nil being returned in some cases now, | |
2097 | ;; perhaps this has been changed? | |
2098 | (if (and node | |
2099 | (>= (point) (ewoc-location node)) | |
2100 | (not (ert--ewoc-entry-hidden-p (ewoc-data node)))) | |
2101 | node | |
2102 | nil))) | |
2103 | ||
2104 | (defun ert--results-test-node-at-point () | |
2105 | "If point is on a valid ewoc node, return it; signal an error otherwise. | |
2106 | ||
2107 | To be used in the ERT results buffer." | |
2108 | (or (ert--results-test-node-or-null-at-point) | |
2109 | (error "No test at point"))) | |
2110 | ||
2111 | (defun ert-results-next-test () | |
2112 | "Move point to the next test. | |
2113 | ||
2114 | To be used in the ERT results buffer." | |
2115 | (interactive) | |
2116 | (ert--results-move (ewoc-locate ert--results-ewoc) 'ewoc-next | |
2117 | "No tests below")) | |
2118 | ||
2119 | (defun ert-results-previous-test () | |
2120 | "Move point to the previous test. | |
2121 | ||
2122 | To be used in the ERT results buffer." | |
2123 | (interactive) | |
2124 | (ert--results-move (ewoc-locate ert--results-ewoc) 'ewoc-prev | |
2125 | "No tests above")) | |
2126 | ||
2127 | (defun ert--results-move (node ewoc-fn error-message) | |
2128 | "Move point from NODE to the previous or next node. | |
2129 | ||
2130 | EWOC-FN specifies the direction and should be either `ewoc-prev' | |
2131 | or `ewoc-next'. If there are no more nodes in that direction, an | |
8350f087 | 2132 | error is signaled with the message ERROR-MESSAGE." |
15c9d04e | 2133 | (cl-loop |
d221e780 CO |
2134 | (setq node (funcall ewoc-fn ert--results-ewoc node)) |
2135 | (when (null node) | |
2136 | (error "%s" error-message)) | |
2137 | (unless (ert--ewoc-entry-hidden-p (ewoc-data node)) | |
2138 | (goto-char (ewoc-location node)) | |
15c9d04e | 2139 | (cl-return)))) |
d221e780 | 2140 | |
15c9d04e | 2141 | (defun ert--results-expand-collapse-button-action (_button) |
d221e780 CO |
2142 | "Expand or collapse the test node BUTTON belongs to." |
2143 | (let* ((ewoc ert--results-ewoc) | |
2144 | (node (save-excursion | |
2145 | (goto-char (ert--button-action-position)) | |
2146 | (ert--results-test-node-at-point))) | |
2147 | (entry (ewoc-data node))) | |
2148 | (setf (ert--ewoc-entry-expanded-p entry) | |
2149 | (not (ert--ewoc-entry-expanded-p entry))) | |
2150 | (ewoc-invalidate ewoc node))) | |
2151 | ||
2152 | (defun ert-results-find-test-at-point-other-window () | |
2153 | "Find the definition of the test at point in another window. | |
2154 | ||
2155 | To be used in the ERT results buffer." | |
2156 | (interactive) | |
2157 | (let ((name (ert-test-at-point))) | |
2158 | (unless name | |
2159 | (error "No test at point")) | |
2160 | (ert-find-test-other-window name))) | |
2161 | ||
2162 | (defun ert--test-name-button-action (button) | |
2163 | "Find the definition of the test BUTTON belongs to, in another window." | |
2164 | (let ((name (button-get button 'ert-test-name))) | |
2165 | (ert-find-test-other-window name))) | |
2166 | ||
2167 | (defun ert--ewoc-position (ewoc node) | |
2168 | ;; checkdoc-order: nil | |
2169 | "Return the position of NODE in EWOC, or nil if NODE is not in EWOC." | |
15c9d04e SM |
2170 | (cl-loop for i from 0 |
2171 | for node-here = (ewoc-nth ewoc 0) then (ewoc-next ewoc node-here) | |
2172 | do (when (eql node node-here) | |
2173 | (cl-return i)) | |
2174 | finally (cl-return nil))) | |
d221e780 CO |
2175 | |
2176 | (defun ert-results-jump-between-summary-and-result () | |
2177 | "Jump back and forth between the test run summary and individual test results. | |
2178 | ||
2179 | From an ewoc node, jumps to the character that represents the | |
2180 | same test in the progress bar, and vice versa. | |
2181 | ||
2182 | To be used in the ERT results buffer." | |
2183 | ;; Maybe this command isn't actually needed much, but if it is, it | |
2184 | ;; seems like an indication that the UI design is not optimal. If | |
2185 | ;; jumping back and forth between a summary at the top of the buffer | |
2186 | ;; and the error log in the remainder of the buffer is useful, then | |
2187 | ;; the summary apparently needs to be easily accessible from the | |
2188 | ;; error log, and perhaps it would be better to have it in a | |
2189 | ;; separate buffer to keep it visible. | |
2190 | (interactive) | |
2191 | (let ((ewoc ert--results-ewoc) | |
2192 | (progress-bar-begin ert--results-progress-bar-button-begin)) | |
2193 | (cond ((ert--results-test-node-or-null-at-point) | |
2194 | (let* ((node (ert--results-test-node-at-point)) | |
2195 | (pos (ert--ewoc-position ewoc node))) | |
2196 | (goto-char (+ progress-bar-begin pos)))) | |
2197 | ((and (<= progress-bar-begin (point)) | |
2198 | (< (point) (button-end (button-at progress-bar-begin)))) | |
2199 | (let* ((node (ewoc-nth ewoc (- (point) progress-bar-begin))) | |
2200 | (entry (ewoc-data node))) | |
2201 | (when (ert--ewoc-entry-hidden-p entry) | |
2202 | (setf (ert--ewoc-entry-hidden-p entry) nil) | |
2203 | (ewoc-invalidate ewoc node)) | |
2204 | (ewoc-goto-node ewoc node))) | |
2205 | (t | |
2206 | (goto-char progress-bar-begin))))) | |
2207 | ||
2208 | (defun ert-test-at-point () | |
2209 | "Return the name of the test at point as a symbol, or nil if none." | |
2210 | (or (and (eql major-mode 'ert-results-mode) | |
2211 | (let ((test (ert--results-test-at-point-no-redefinition))) | |
2212 | (and test (ert-test-name test)))) | |
2213 | (let* ((thing (thing-at-point 'symbol)) | |
2214 | (sym (intern-soft thing))) | |
2215 | (and (ert-test-boundp sym) | |
2216 | sym)))) | |
2217 | ||
2218 | (defun ert--results-test-at-point-no-redefinition () | |
2219 | "Return the test at point, or nil. | |
2220 | ||
2221 | To be used in the ERT results buffer." | |
15c9d04e | 2222 | (cl-assert (eql major-mode 'ert-results-mode)) |
d221e780 CO |
2223 | (if (ert--results-test-node-or-null-at-point) |
2224 | (let* ((node (ert--results-test-node-at-point)) | |
2225 | (test (ert--ewoc-entry-test (ewoc-data node)))) | |
2226 | test) | |
2227 | (let ((progress-bar-begin ert--results-progress-bar-button-begin)) | |
2228 | (when (and (<= progress-bar-begin (point)) | |
2229 | (< (point) (button-end (button-at progress-bar-begin)))) | |
2230 | (let* ((test-index (- (point) progress-bar-begin)) | |
2231 | (test (aref (ert--stats-tests ert--results-stats) | |
2232 | test-index))) | |
2233 | test))))) | |
2234 | ||
2235 | (defun ert--results-test-at-point-allow-redefinition () | |
2236 | "Look up the test at point, and check whether it has been redefined. | |
2237 | ||
2238 | To be used in the ERT results buffer. | |
2239 | ||
2240 | Returns a list of two elements: the test (or nil) and a symbol | |
2241 | specifying whether the test has been redefined. | |
2242 | ||
2243 | If a new test has been defined with the same name as the test at | |
2244 | point, replaces the test at point with the new test, and returns | |
2245 | the new test and the symbol `redefined'. | |
2246 | ||
2247 | If the test has been deleted, returns the old test and the symbol | |
2248 | `deleted'. | |
2249 | ||
2250 | If the test is still current, returns the test and the symbol nil. | |
2251 | ||
2252 | If there is no test at point, returns a list with two nils." | |
2253 | (let ((test (ert--results-test-at-point-no-redefinition))) | |
2254 | (cond ((null test) | |
2255 | `(nil nil)) | |
2256 | ((null (ert-test-name test)) | |
2257 | `(,test nil)) | |
2258 | (t | |
2259 | (let* ((name (ert-test-name test)) | |
2260 | (new-test (and (ert-test-boundp name) | |
2261 | (ert-get-test name)))) | |
2262 | (cond ((eql test new-test) | |
2263 | `(,test nil)) | |
2264 | ((null new-test) | |
2265 | `(,test deleted)) | |
2266 | (t | |
2267 | (ert--results-update-after-test-redefinition | |
2268 | (ert--stats-test-pos ert--results-stats test) | |
2269 | new-test) | |
2270 | `(,new-test redefined)))))))) | |
2271 | ||
2272 | (defun ert--results-update-after-test-redefinition (pos new-test) | |
2273 | "Update results buffer after the test at pos POS has been redefined. | |
2274 | ||
2275 | Also updates the stats object. NEW-TEST is the new test | |
2276 | definition." | |
2277 | (let* ((stats ert--results-stats) | |
2278 | (ewoc ert--results-ewoc) | |
2279 | (node (ewoc-nth ewoc pos)) | |
2280 | (entry (ewoc-data node))) | |
2281 | (ert--stats-set-test-and-result stats pos new-test nil) | |
2282 | (setf (ert--ewoc-entry-test entry) new-test | |
2283 | (aref ert--results-progress-bar-string pos) (ert-char-for-test-result | |
2284 | nil t)) | |
2285 | (ewoc-invalidate ewoc node)) | |
2286 | nil) | |
2287 | ||
2288 | (defun ert--button-action-position () | |
2289 | "The buffer position where the last button action was triggered." | |
2290 | (cond ((integerp last-command-event) | |
2291 | (point)) | |
2292 | ((eventp last-command-event) | |
2293 | (posn-point (event-start last-command-event))) | |
15c9d04e | 2294 | (t (cl-assert nil)))) |
d221e780 | 2295 | |
15c9d04e | 2296 | (defun ert--results-progress-bar-button-action (_button) |
d221e780 CO |
2297 | "Jump to details for the test represented by the character clicked in BUTTON." |
2298 | (goto-char (ert--button-action-position)) | |
2299 | (ert-results-jump-between-summary-and-result)) | |
2300 | ||
2301 | (defun ert-results-rerun-all-tests () | |
2302 | "Re-run all tests, using the same selector. | |
2303 | ||
2304 | To be used in the ERT results buffer." | |
2305 | (interactive) | |
15c9d04e | 2306 | (cl-assert (eql major-mode 'ert-results-mode)) |
d221e780 CO |
2307 | (let ((selector (ert--stats-selector ert--results-stats))) |
2308 | (ert-run-tests-interactively selector (buffer-name)))) | |
2309 | ||
2310 | (defun ert-results-rerun-test-at-point () | |
2311 | "Re-run the test at point. | |
2312 | ||
2313 | To be used in the ERT results buffer." | |
2314 | (interactive) | |
15c9d04e | 2315 | (cl-destructuring-bind (test redefinition-state) |
d221e780 CO |
2316 | (ert--results-test-at-point-allow-redefinition) |
2317 | (when (null test) | |
2318 | (error "No test at point")) | |
2319 | (let* ((stats ert--results-stats) | |
2320 | (progress-message (format "Running %stest %S" | |
15c9d04e | 2321 | (cl-ecase redefinition-state |
d221e780 CO |
2322 | ((nil) "") |
2323 | (redefined "new definition of ") | |
2324 | (deleted "deleted ")) | |
2325 | (ert-test-name test)))) | |
2326 | ;; Need to save and restore point manually here: When point is on | |
2327 | ;; the first visible ewoc entry while the header is updated, point | |
2328 | ;; moves to the top of the buffer. This is undesirable, and a | |
2329 | ;; simple `save-excursion' doesn't prevent it. | |
2330 | (let ((point (point))) | |
2331 | (unwind-protect | |
2332 | (unwind-protect | |
2333 | (progn | |
2334 | (message "%s..." progress-message) | |
2335 | (ert-run-or-rerun-test stats test | |
2336 | ert--results-listener)) | |
2337 | (ert--results-update-stats-display ert--results-ewoc stats) | |
2338 | (message "%s...%s" | |
2339 | progress-message | |
2340 | (let ((result (ert-test-most-recent-result test))) | |
2341 | (ert-string-for-test-result | |
2342 | result (ert-test-result-expected-p test result))))) | |
2343 | (goto-char point)))))) | |
2344 | ||
2345 | (defun ert-results-rerun-test-at-point-debugging-errors () | |
2346 | "Re-run the test at point with `ert-debug-on-error' bound to t. | |
2347 | ||
2348 | To be used in the ERT results buffer." | |
2349 | (interactive) | |
2350 | (let ((ert-debug-on-error t)) | |
2351 | (ert-results-rerun-test-at-point))) | |
2352 | ||
2353 | (defun ert-results-pop-to-backtrace-for-test-at-point () | |
2354 | "Display the backtrace for the test at point. | |
2355 | ||
2356 | To be used in the ERT results buffer." | |
2357 | (interactive) | |
2358 | (let* ((test (ert--results-test-at-point-no-redefinition)) | |
2359 | (stats ert--results-stats) | |
2360 | (pos (ert--stats-test-pos stats test)) | |
2361 | (result (aref (ert--stats-test-results stats) pos))) | |
15c9d04e | 2362 | (cl-etypecase result |
d221e780 CO |
2363 | (ert-test-passed (error "Test passed, no backtrace available")) |
2364 | (ert-test-result-with-condition | |
2365 | (let ((backtrace (ert-test-result-with-condition-backtrace result)) | |
2366 | (buffer (get-buffer-create "*ERT Backtrace*"))) | |
2367 | (pop-to-buffer buffer) | |
d221e780 CO |
2368 | (let ((inhibit-read-only t)) |
2369 | (buffer-disable-undo) | |
2370 | (erase-buffer) | |
5da16a86 | 2371 | (ert-simple-view-mode) |
d221e780 CO |
2372 | ;; Use unibyte because `debugger-setup-buffer' also does so. |
2373 | (set-buffer-multibyte nil) | |
2374 | (setq truncate-lines t) | |
2375 | (ert--print-backtrace backtrace) | |
2376 | (debugger-make-xrefs) | |
2377 | (goto-char (point-min)) | |
2378 | (insert "Backtrace for test `") | |
2379 | (ert-insert-test-name-button (ert-test-name test)) | |
5da16a86 | 2380 | (insert "':\n"))))))) |
d221e780 CO |
2381 | |
2382 | (defun ert-results-pop-to-messages-for-test-at-point () | |
2383 | "Display the part of the *Messages* buffer generated during the test at point. | |
2384 | ||
2385 | To be used in the ERT results buffer." | |
2386 | (interactive) | |
2387 | (let* ((test (ert--results-test-at-point-no-redefinition)) | |
2388 | (stats ert--results-stats) | |
2389 | (pos (ert--stats-test-pos stats test)) | |
2390 | (result (aref (ert--stats-test-results stats) pos))) | |
2391 | (let ((buffer (get-buffer-create "*ERT Messages*"))) | |
2392 | (pop-to-buffer buffer) | |
d221e780 CO |
2393 | (let ((inhibit-read-only t)) |
2394 | (buffer-disable-undo) | |
2395 | (erase-buffer) | |
5da16a86 | 2396 | (ert-simple-view-mode) |
d221e780 CO |
2397 | (insert (ert-test-result-messages result)) |
2398 | (goto-char (point-min)) | |
2399 | (insert "Messages for test `") | |
2400 | (ert-insert-test-name-button (ert-test-name test)) | |
5da16a86 | 2401 | (insert "':\n"))))) |
d221e780 CO |
2402 | |
2403 | (defun ert-results-pop-to-should-forms-for-test-at-point () | |
2404 | "Display the list of `should' forms executed during the test at point. | |
2405 | ||
2406 | To be used in the ERT results buffer." | |
2407 | (interactive) | |
2408 | (let* ((test (ert--results-test-at-point-no-redefinition)) | |
2409 | (stats ert--results-stats) | |
2410 | (pos (ert--stats-test-pos stats test)) | |
2411 | (result (aref (ert--stats-test-results stats) pos))) | |
2412 | (let ((buffer (get-buffer-create "*ERT list of should forms*"))) | |
2413 | (pop-to-buffer buffer) | |
d221e780 CO |
2414 | (let ((inhibit-read-only t)) |
2415 | (buffer-disable-undo) | |
2416 | (erase-buffer) | |
5da16a86 | 2417 | (ert-simple-view-mode) |
d221e780 CO |
2418 | (if (null (ert-test-result-should-forms result)) |
2419 | (insert "\n(No should forms during this test.)\n") | |
15c9d04e SM |
2420 | (cl-loop for form-description |
2421 | in (ert-test-result-should-forms result) | |
2422 | for i from 1 do | |
2423 | (insert "\n") | |
2424 | (insert (format "%s: " i)) | |
2425 | (let ((begin (point))) | |
2426 | (ert--pp-with-indentation-and-newline form-description) | |
2427 | (ert--make-xrefs-region begin (point))))) | |
d221e780 CO |
2428 | (goto-char (point-min)) |
2429 | (insert "`should' forms executed during test `") | |
2430 | (ert-insert-test-name-button (ert-test-name test)) | |
2431 | (insert "':\n") | |
2432 | (insert "\n") | |
2433 | (insert (concat "(Values are shallow copies and may have " | |
2434 | "looked different during the test if they\n" | |
2435 | "have been modified destructively.)\n")) | |
5da16a86 | 2436 | (forward-line 1))))) |
d221e780 CO |
2437 | |
2438 | (defun ert-results-toggle-printer-limits-for-test-at-point () | |
2439 | "Toggle how much of the condition to print for the test at point. | |
2440 | ||
2441 | To be used in the ERT results buffer." | |
2442 | (interactive) | |
2443 | (let* ((ewoc ert--results-ewoc) | |
2444 | (node (ert--results-test-node-at-point)) | |
2445 | (entry (ewoc-data node))) | |
2446 | (setf (ert--ewoc-entry-extended-printer-limits-p entry) | |
2447 | (not (ert--ewoc-entry-extended-printer-limits-p entry))) | |
2448 | (ewoc-invalidate ewoc node))) | |
2449 | ||
2450 | (defun ert-results-pop-to-timings () | |
2451 | "Display test timings for the last run. | |
2452 | ||
2453 | To be used in the ERT results buffer." | |
2454 | (interactive) | |
2455 | (let* ((stats ert--results-stats) | |
d221e780 | 2456 | (buffer (get-buffer-create "*ERT timings*")) |
15c9d04e SM |
2457 | (data (cl-loop for test across (ert--stats-tests stats) |
2458 | for start-time across (ert--stats-test-start-times | |
2459 | stats) | |
2460 | for end-time across (ert--stats-test-end-times stats) | |
2461 | collect (list test | |
2462 | (float-time (subtract-time | |
2463 | end-time start-time)))))) | |
d221e780 | 2464 | (setq data (sort data (lambda (a b) |
15c9d04e | 2465 | (> (cl-second a) (cl-second b))))) |
d221e780 | 2466 | (pop-to-buffer buffer) |
d221e780 CO |
2467 | (let ((inhibit-read-only t)) |
2468 | (buffer-disable-undo) | |
2469 | (erase-buffer) | |
5da16a86 | 2470 | (ert-simple-view-mode) |
d221e780 CO |
2471 | (if (null data) |
2472 | (insert "(No data)\n") | |
2473 | (insert (format "%-3s %8s %8s\n" "" "time" "cumul")) | |
15c9d04e SM |
2474 | (cl-loop for (test time) in data |
2475 | for cumul-time = time then (+ cumul-time time) | |
2476 | for i from 1 do | |
2477 | (progn | |
2478 | (insert (format "%3s: %8.3f %8.3f " i time cumul-time)) | |
2479 | (ert-insert-test-name-button (ert-test-name test)) | |
2480 | (insert "\n")))) | |
d221e780 CO |
2481 | (goto-char (point-min)) |
2482 | (insert "Tests by run time (seconds):\n\n") | |
5da16a86 | 2483 | (forward-line 1)))) |
d221e780 CO |
2484 | |
2485 | ;;;###autoload | |
2486 | (defun ert-describe-test (test-or-test-name) | |
2487 | "Display the documentation for TEST-OR-TEST-NAME (a symbol or ert-test)." | |
2488 | (interactive (list (ert-read-test-name-at-point "Describe test"))) | |
2489 | (when (< emacs-major-version 24) | |
2490 | (error "Requires Emacs 24")) | |
2491 | (let (test-name | |
2492 | test-definition) | |
15c9d04e | 2493 | (cl-etypecase test-or-test-name |
d221e780 CO |
2494 | (symbol (setq test-name test-or-test-name |
2495 | test-definition (ert-get-test test-or-test-name))) | |
2496 | (ert-test (setq test-name (ert-test-name test-or-test-name) | |
2497 | test-definition test-or-test-name))) | |
2498 | (help-setup-xref (list #'ert-describe-test test-or-test-name) | |
2499 | (called-interactively-p 'interactive)) | |
2500 | (save-excursion | |
2501 | (with-help-window (help-buffer) | |
2502 | (with-current-buffer (help-buffer) | |
2503 | (insert (if test-name (format "%S" test-name) "<anonymous test>")) | |
2504 | (insert " is a test") | |
2505 | (let ((file-name (and test-name | |
2506 | (symbol-file test-name 'ert-deftest)))) | |
2507 | (when file-name | |
2508 | (insert " defined in `" (file-name-nondirectory file-name) "'") | |
2509 | (save-excursion | |
2510 | (re-search-backward "`\\([^`']+\\)'" nil t) | |
2511 | (help-xref-button 1 'help-function-def test-name file-name))) | |
2512 | (insert ".") | |
2513 | (fill-region-as-paragraph (point-min) (point)) | |
2514 | (insert "\n\n") | |
2515 | (unless (and (ert-test-boundp test-name) | |
2516 | (eql (ert-get-test test-name) test-definition)) | |
2517 | (let ((begin (point))) | |
2518 | (insert "Note: This test has been redefined or deleted, " | |
2519 | "this documentation refers to an old definition.") | |
2520 | (fill-region-as-paragraph begin (point))) | |
2521 | (insert "\n\n")) | |
2522 | (insert (or (ert-test-documentation test-definition) | |
2523 | "It is not documented.") | |
2524 | "\n"))))))) | |
2525 | ||
2526 | (defun ert-results-describe-test-at-point () | |
2527 | "Display the documentation of the test at point. | |
2528 | ||
2529 | To be used in the ERT results buffer." | |
2530 | (interactive) | |
2531 | (ert-describe-test (ert--results-test-at-point-no-redefinition))) | |
2532 | ||
2533 | ||
2534 | ;;; Actions on load/unload. | |
2535 | ||
2536 | (add-to-list 'find-function-regexp-alist '(ert-deftest . ert--find-test-regexp)) | |
2537 | (add-to-list 'minor-mode-alist '(ert--current-run-stats | |
2538 | (:eval | |
2539 | (ert--tests-running-mode-line-indicator)))) | |
2540 | (add-to-list 'emacs-lisp-mode-hook 'ert--activate-font-lock-keywords) | |
2541 | ||
2542 | (defun ert--unload-function () | |
2543 | "Unload function to undo the side-effects of loading ert.el." | |
2544 | (ert--remove-from-list 'find-function-regexp-alist 'ert-deftest :key #'car) | |
2545 | (ert--remove-from-list 'minor-mode-alist 'ert--current-run-stats :key #'car) | |
2546 | (ert--remove-from-list 'emacs-lisp-mode-hook | |
2547 | 'ert--activate-font-lock-keywords) | |
2548 | nil) | |
2549 | ||
2550 | (defvar ert-unload-hook '()) | |
2551 | (add-hook 'ert-unload-hook 'ert--unload-function) | |
2552 | ||
2553 | ||
2554 | (provide 'ert) | |
2555 | ||
2556 | ;;; ert.el ends here |