2 * Run a set of tests, reporting results.
6 * runtests [-b <build-dir>] [-s <source-dir>] <test-list>
7 * runtests -o [-b <build-dir>] [-s <source-dir>] <test>
9 * In the first case, expects a list of executables located in the given file,
10 * one line per executable. For each one, runs it as part of a test suite,
11 * reporting results. Test output should start with a line containing the
12 * number of tests (numbered from 1 to this number), optionally preceded by
13 * "1..", although that line may be given anywhere in the output. Each
14 * additional line should be in the following format:
19 * not ok <number> # todo
21 * where <number> is the number of the test. An optional comment is permitted
22 * after the number if preceded by whitespace. ok indicates success, not ok
23 * indicates failure. "# skip" and "# todo" are a special cases of a comment,
24 * and must start with exactly that formatting. They indicate the test was
25 * skipped for some reason (maybe because it doesn't apply to this platform)
26 * or is testing something known to currently fail. The text following either
27 * "# skip" or "# todo" and whitespace is the reason.
29 * As a special case, the first line of the output may be in the form:
31 * 1..0 # skip some reason
33 * which indicates that this entire test case should be skipped and gives a
36 * Any other lines are ignored, although for compliance with the TAP protocol
37 * all lines other than the ones in the above format should be sent to
38 * standard error rather than standard output and start with #.
40 * This is a subset of TAP as documented in Test::Harness::TAP or
41 * TAP::Parser::Grammar, which comes with Perl.
43 * If the -o option is given, instead run a single test and display all of its
44 * output. This is intended for use with failing tests so that the person
45 * running the test suite can get more details about what failed.
47 * If built with the C preprocessor symbols SOURCE and BUILD defined, C TAP
48 * Harness will export those values in the environment so that tests can find
49 * the source and build directory and will look for tests under both
50 * directories. These paths can also be set with the -b and -s command-line
51 * options, which will override anything set at build time.
53 * Any bug reports, bug fixes, and improvements are very much welcome and
54 * should be sent to the e-mail address below. This program is part of C TAP
55 * Harness <http://www.eyrie.org/~eagle/software/c-tap-harness/>.
57 * Copyright 2000, 2001, 2004, 2006, 2007, 2008, 2009, 2010, 2011
58 * Russ Allbery <rra@stanford.edu>
60 * Permission is hereby granted, free of charge, to any person obtaining a
61 * copy of this software and associated documentation files (the "Software"),
62 * to deal in the Software without restriction, including without limitation
63 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
64 * and/or sell copies of the Software, and to permit persons to whom the
65 * Software is furnished to do so, subject to the following conditions:
67 * The above copyright notice and this permission notice shall be included in
68 * all copies or substantial portions of the Software.
70 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
71 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
72 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
73 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
74 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
75 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
76 * DEALINGS IN THE SOFTWARE.
79 /* Required for fdopen(), getopt(), and putenv(). */
80 #if defined(__STRICT_ANSI__) || defined(PEDANTIC)
81 # ifndef _XOPEN_SOURCE
82 # define _XOPEN_SOURCE 500
96 #include <sys/types.h>
101 /* sys/time.h must be included before sys/resource.h on some platforms. */
102 #include <sys/resource.h>
104 /* AIX doesn't have WCOREDUMP. */
106 # define WCOREDUMP(status) ((unsigned)(status) & 0x80)
110 * The source and build versions of the tests directory. This is used to set
111 * the SOURCE and BUILD environment variables and find test programs, if set.
112 * Normally, this should be set as part of the build process to the test
113 * subdirectories of $(abs_top_srcdir) and $(abs_top_builddir) respectively.
122 /* Test status codes. */
130 /* Indicates the state of our plan. */
132 PLAN_INIT
, /* Nothing seen yet. */
133 PLAN_FIRST
, /* Plan seen before any tests. */
134 PLAN_PENDING
, /* Test seen and no plan yet. */
135 PLAN_FINAL
/* Plan seen after some tests. */
138 /* Error exit statuses for test processes. */
139 #define CHILDERR_DUP 100 /* Couldn't redirect stderr or stdout. */
140 #define CHILDERR_EXEC 101 /* Couldn't exec child process. */
141 #define CHILDERR_STDERR 102 /* Couldn't open stderr file. */
143 /* Structure to hold data for a set of tests. */
145 char *file
; /* The file name of the test. */
146 char *path
; /* The path to the test program. */
147 enum plan_status plan
; /* The status of our plan. */
148 unsigned long count
; /* Expected count of tests. */
149 unsigned long current
; /* The last seen test number. */
150 unsigned int length
; /* The length of the last status message. */
151 unsigned long passed
; /* Count of passing tests. */
152 unsigned long failed
; /* Count of failing lists. */
153 unsigned long skipped
; /* Count of skipped tests (passed). */
154 unsigned long allocated
; /* The size of the results table. */
155 enum test_status
*results
; /* Table of results by test number. */
156 unsigned int aborted
; /* Whether the set as aborted. */
157 int reported
; /* Whether the results were reported. */
158 int status
; /* The exit status of the test. */
159 unsigned int all_skipped
; /* Whether all tests were skipped. */
160 char *reason
; /* Why all tests were skipped. */
163 /* Structure to hold a linked list of test sets. */
166 struct testlist
*next
;
170 * Usage message. Should be used as a printf format with two arguments: the
171 * path to runtests, given twice.
173 static const char usage_message
[] = "\
174 Usage: %s [-b <build-dir>] [-s <source-dir>] <test-list>\n\
175 %s -o [-b <build-dir>] [-s <source-dir>] <test>\n\
178 -b <build-dir> Set the build directory to <build-dir>\n\
179 -o Run a single test rather than a list of tests\n\
180 -s <source-dir> Set the source directory to <source-dir>\n\
182 runtests normally runs each test listed in a file whose path is given as\n\
183 its command-line argument. With the -o option, it instead runs a single\n\
184 test and shows its complete output.\n";
187 * Header used for test output. %s is replaced by the file name of the list
190 static const char banner
[] = "\n\
191 Running all tests listed in %s. If any tests fail, run the failing\n\
192 test program with runtests -o to see more details.\n\n";
194 /* Header for reports of failed tests. */
195 static const char header
[] = "\n\
196 Failed Set Fail/Total (%) Skip Stat Failing Tests\n\
197 -------------------------- -------------- ---- ---- ------------------------";
199 /* Include the file name and line number in malloc failures. */
200 #define xmalloc(size) x_malloc((size), __FILE__, __LINE__)
201 #define xrealloc(p, size) x_realloc((p), (size), __FILE__, __LINE__)
202 #define xstrdup(p) x_strdup((p), __FILE__, __LINE__)
206 * Report a fatal error, including the results of strerror, and exit.
209 sysdie(const char *format
, ...)
216 fprintf(stderr
, "runtests: ");
217 va_start(args
, format
);
218 vfprintf(stderr
, format
, args
);
220 fprintf(stderr
, ": %s\n", strerror(oerrno
));
226 * Allocate memory, reporting a fatal error and exiting on failure.
229 x_malloc(size_t size
, const char *file
, int line
)
235 sysdie("failed to malloc %lu bytes at %s line %d",
236 (unsigned long) size
, file
, line
);
242 * Reallocate memory, reporting a fatal error and exiting on failure.
245 x_realloc(void *p
, size_t size
, const char *file
, int line
)
247 p
= realloc(p
, size
);
249 sysdie("failed to realloc %lu bytes at %s line %d",
250 (unsigned long) size
, file
, line
);
256 * Copy a string, reporting a fatal error and exiting on failure.
259 x_strdup(const char *s
, const char *file
, int line
)
267 sysdie("failed to strdup %lu bytes at %s line %d",
268 (unsigned long) len
, file
, line
);
275 * Given a struct timeval, return the number of seconds it represents as a
276 * double. Use difftime() to convert a time_t to a double.
279 tv_seconds(const struct timeval
*tv
)
281 return difftime(tv
->tv_sec
, 0) + tv
->tv_usec
* 1e-6;
286 * Given two struct timevals, return the difference in seconds.
289 tv_diff(const struct timeval
*tv1
, const struct timeval
*tv0
)
291 return tv_seconds(tv1
) - tv_seconds(tv0
);
296 * Given two struct timevals, return the sum in seconds as a double.
299 tv_sum(const struct timeval
*tv1
, const struct timeval
*tv2
)
301 return tv_seconds(tv1
) + tv_seconds(tv2
);
306 * Given a pointer to a string, skip any leading whitespace and return a
307 * pointer to the first non-whitespace character.
310 skip_whitespace(const char *p
)
312 while (isspace((unsigned char)(*p
)))
319 * Start a program, connecting its stdout to a pipe on our end and its stderr
320 * to /dev/null, and storing the file descriptor to read from in the two
321 * argument. Returns the PID of the new process. Errors are fatal.
324 test_start(const char *path
, int *fd
)
329 if (pipe(fds
) == -1) {
332 sysdie("can't create pipe");
335 if (child
== (pid_t
) -1) {
338 sysdie("can't fork");
339 } else if (child
== 0) {
340 /* In child. Set up our stdout and stderr. */
341 errfd
= open("/dev/null", O_WRONLY
);
343 _exit(CHILDERR_STDERR
);
344 if (dup2(errfd
, 2) == -1)
347 if (dup2(fds
[1], 1) == -1)
350 /* Now, exec our process. */
351 if (execl(path
, path
, (char *) 0) == -1)
352 _exit(CHILDERR_EXEC
);
354 /* In parent. Close the extra file descriptor. */
363 * Back up over the output saying what test we were executing.
366 test_backspace(struct testset
*ts
)
370 if (!isatty(STDOUT_FILENO
))
372 for (i
= 0; i
< ts
->length
; i
++)
374 for (i
= 0; i
< ts
->length
; i
++)
376 for (i
= 0; i
< ts
->length
; i
++)
383 * Read the plan line of test output, which should contain the range of test
384 * numbers. We may initialize the testset structure here if we haven't yet
385 * seen a test. Return true if initialization succeeded and the test should
386 * continue, false otherwise.
389 test_plan(const char *line
, struct testset
*ts
)
395 * Accept a plan without the leading 1.. for compatibility with older
396 * versions of runtests. This will only be allowed if we've not yet seen
399 line
= skip_whitespace(line
);
400 if (strncmp(line
, "1..", 3) == 0)
404 * Get the count, check it for validity, and initialize the struct. If we
405 * have something of the form "1..0 # skip foo", the whole file was
406 * skipped; record that. If we do skip the whole file, zero out all of
407 * our statistics, since they're no longer relevant. strtol is called
408 * with a second argument to advance the line pointer past the count to
409 * make it simpler to detect the # skip case.
411 n
= strtol(line
, (char **) &line
, 10);
413 line
= skip_whitespace(line
);
415 line
= skip_whitespace(line
+ 1);
416 if (strncasecmp(line
, "skip", 4) == 0) {
417 line
= skip_whitespace(line
+ 4);
419 ts
->reason
= xstrdup(line
);
420 ts
->reason
[strlen(ts
->reason
) - 1] = '\0';
433 puts("ABORTED (invalid test count)");
438 if (ts
->plan
== PLAN_INIT
&& ts
->allocated
== 0) {
441 ts
->plan
= PLAN_FIRST
;
442 ts
->results
= xmalloc(ts
->count
* sizeof(enum test_status
));
443 for (i
= 0; i
< ts
->count
; i
++)
444 ts
->results
[i
] = TEST_INVALID
;
445 } else if (ts
->plan
== PLAN_PENDING
) {
446 if ((unsigned long) n
< ts
->count
) {
447 printf("ABORTED (invalid test number %lu)\n", ts
->count
);
453 if ((unsigned long) n
> ts
->allocated
) {
454 ts
->results
= xrealloc(ts
->results
, n
* sizeof(enum test_status
));
455 for (i
= ts
->allocated
; i
< ts
->count
; i
++)
456 ts
->results
[i
] = TEST_INVALID
;
459 ts
->plan
= PLAN_FINAL
;
466 * Given a single line of output from a test, parse it and return the success
467 * status of that test. Anything printed to stdout not matching the form
468 * /^(not )?ok \d+/ is ignored. Sets ts->current to the test number that just
472 test_checkline(const char *line
, struct testset
*ts
)
474 enum test_status status
= TEST_PASS
;
478 unsigned long i
, current
;
481 /* Before anything, check for a test abort. */
482 bail
= strstr(line
, "Bail out!");
484 bail
= skip_whitespace(bail
+ strlen("Bail out!"));
488 length
= strlen(bail
);
489 if (bail
[length
- 1] == '\n')
492 printf("ABORTED (%.*s)\n", (int) length
, bail
);
500 * If the given line isn't newline-terminated, it was too big for an
501 * fgets(), which means ignore it.
503 if (line
[strlen(line
) - 1] != '\n')
506 /* If the line begins with a hash mark, ignore it. */
510 /* If we haven't yet seen a plan, look for one. */
511 if (ts
->plan
== PLAN_INIT
&& isdigit((unsigned char)(*line
))) {
512 if (!test_plan(line
, ts
))
514 } else if (strncmp(line
, "1..", 3) == 0) {
515 if (ts
->plan
== PLAN_PENDING
) {
516 if (!test_plan(line
, ts
))
519 puts("ABORTED (multiple plans)");
526 /* Parse the line, ignoring something we can't parse. */
527 if (strncmp(line
, "not ", 4) == 0) {
531 if (strncmp(line
, "ok", 2) != 0)
533 line
= skip_whitespace(line
+ 2);
535 number
= strtol(line
, &end
, 10);
536 if (errno
!= 0 || end
== line
)
537 number
= ts
->current
+ 1;
539 if (number
<= 0 || (current
> ts
->count
&& ts
->plan
== PLAN_FIRST
)) {
541 printf("ABORTED (invalid test number %lu)\n", current
);
547 /* We have a valid test result. Tweak the results array if needed. */
548 if (ts
->plan
== PLAN_INIT
|| ts
->plan
== PLAN_PENDING
) {
549 ts
->plan
= PLAN_PENDING
;
550 if (current
> ts
->count
)
552 if (current
> ts
->allocated
) {
555 n
= (ts
->allocated
== 0) ? 32 : ts
->allocated
* 2;
558 ts
->results
= xrealloc(ts
->results
, n
* sizeof(enum test_status
));
559 for (i
= ts
->allocated
; i
< n
; i
++)
560 ts
->results
[i
] = TEST_INVALID
;
566 * Handle directives. We should probably do something more interesting
567 * with unexpected passes of todo tests.
569 while (isdigit((unsigned char)(*line
)))
571 line
= skip_whitespace(line
);
573 line
= skip_whitespace(line
+ 1);
574 if (strncasecmp(line
, "skip", 4) == 0)
576 if (strncasecmp(line
, "todo", 4) == 0)
577 status
= (status
== TEST_FAIL
) ? TEST_SKIP
: TEST_FAIL
;
580 /* Make sure that the test number is in range and not a duplicate. */
581 if (ts
->results
[current
- 1] != TEST_INVALID
) {
583 printf("ABORTED (duplicate test number %lu)\n", current
);
589 /* Good results. Increment our various counters. */
591 case TEST_PASS
: ts
->passed
++; break;
592 case TEST_FAIL
: ts
->failed
++; break;
593 case TEST_SKIP
: ts
->skipped
++; break;
594 case TEST_INVALID
: break;
596 ts
->current
= current
;
597 ts
->results
[current
- 1] = status
;
599 if (isatty(STDOUT_FILENO
)) {
600 outlen
= printf("%lu/%lu", current
, ts
->count
);
601 ts
->length
= (outlen
>= 0) ? outlen
: 0;
608 * Print out a range of test numbers, returning the number of characters it
609 * took up. Takes the first number, the last number, the number of characters
610 * already printed on the line, and the limit of number of characters the line
611 * can hold. Add a comma and a space before the range if chars indicates that
612 * something has already been printed on the line, and print ... instead if
613 * chars plus the space needed would go over the limit (use a limit of 0 to
617 test_print_range(unsigned long first
, unsigned long last
, unsigned int chars
,
620 unsigned int needed
= 0;
623 for (n
= first
; n
> 0; n
/= 10)
626 for (n
= last
; n
> 0; n
/= 10)
632 if (limit
> 0 && chars
+ needed
> limit
) {
634 if (chars
<= limit
) {
646 printf("%lu-", first
);
654 * Summarize a single test set. The second argument is 0 if the set exited
655 * cleanly, a positive integer representing the exit status if it exited
656 * with a non-zero status, and a negative integer representing the signal
657 * that terminated it if it was killed by a signal.
660 test_summarize(struct testset
*ts
, int status
)
663 unsigned long missing
= 0;
664 unsigned long failed
= 0;
665 unsigned long first
= 0;
666 unsigned long last
= 0;
669 fputs("ABORTED", stdout
);
671 printf(" (passed %lu/%lu)", ts
->passed
, ts
->count
- ts
->skipped
);
673 for (i
= 0; i
< ts
->count
; i
++) {
674 if (ts
->results
[i
] == TEST_INVALID
) {
676 fputs("MISSED ", stdout
);
677 if (first
&& i
== last
)
681 test_print_range(first
, last
, missing
- 1, 0);
689 test_print_range(first
, last
, missing
- 1, 0);
692 for (i
= 0; i
< ts
->count
; i
++) {
693 if (ts
->results
[i
] == TEST_FAIL
) {
694 if (missing
&& !failed
)
697 fputs("FAILED ", stdout
);
698 if (first
&& i
== last
)
702 test_print_range(first
, last
, failed
- 1, 0);
710 test_print_range(first
, last
, failed
- 1, 0);
711 if (!missing
&& !failed
) {
712 fputs(!status
? "ok" : "dubious", stdout
);
713 if (ts
->skipped
> 0) {
714 if (ts
->skipped
== 1)
715 printf(" (skipped %lu test)", ts
->skipped
);
717 printf(" (skipped %lu tests)", ts
->skipped
);
722 printf(" (exit status %d)", status
);
724 printf(" (killed by signal %d%s)", -status
,
725 WCOREDUMP(ts
->status
) ? ", core dumped" : "");
731 * Given a test set, analyze the results, classify the exit status, handle a
732 * few special error messages, and then pass it along to test_summarize() for
733 * the regular output. Returns true if the test set ran successfully and all
734 * tests passed or were skipped, false otherwise.
737 test_analyze(struct testset
*ts
)
741 if (ts
->all_skipped
) {
742 if (ts
->reason
== NULL
)
745 printf("skipped (%s)\n", ts
->reason
);
747 } else if (WIFEXITED(ts
->status
) && WEXITSTATUS(ts
->status
) != 0) {
748 switch (WEXITSTATUS(ts
->status
)) {
751 puts("ABORTED (can't dup file descriptors)");
755 puts("ABORTED (execution failed -- not found?)");
757 case CHILDERR_STDERR
:
759 puts("ABORTED (can't open /dev/null)");
762 test_summarize(ts
, WEXITSTATUS(ts
->status
));
766 } else if (WIFSIGNALED(ts
->status
)) {
767 test_summarize(ts
, -WTERMSIG(ts
->status
));
769 } else if (ts
->plan
!= PLAN_FIRST
&& ts
->plan
!= PLAN_FINAL
) {
770 puts("ABORTED (no valid test plan)");
774 test_summarize(ts
, 0);
775 return (ts
->failed
== 0);
781 * Runs a single test set, accumulating and then reporting the results.
782 * Returns true if the test set was successfully run and all tests passed,
786 test_run(struct testset
*ts
)
788 pid_t testpid
, child
;
794 /* Run the test program. */
795 testpid
= test_start(ts
->path
, &outfd
);
796 output
= fdopen(outfd
, "r");
800 sysdie("fdopen failed");
803 /* Pass each line of output to test_checkline(). */
804 while (!ts
->aborted
&& fgets(buffer
, sizeof(buffer
), output
))
805 test_checkline(buffer
, ts
);
806 if (ferror(output
) || ts
->plan
== PLAN_INIT
)
811 * Consume the rest of the test output, close the output descriptor,
812 * retrieve the exit status, and pass that information to test_analyze()
813 * for eventual output.
815 while (fgets(buffer
, sizeof(buffer
), output
))
818 child
= waitpid(testpid
, &ts
->status
, 0);
819 if (child
== (pid_t
) -1) {
824 sysdie("waitpid for %u failed", (unsigned int) testpid
);
828 status
= test_analyze(ts
);
830 /* Convert missing tests to failed tests. */
831 for (i
= 0; i
< ts
->count
; i
++) {
832 if (ts
->results
[i
] == TEST_INVALID
) {
834 ts
->results
[i
] = TEST_FAIL
;
842 /* Summarize a list of test failures. */
844 test_fail_summary(const struct testlist
*fails
)
848 unsigned long i
, first
, last
, total
;
852 /* Failed Set Fail/Total (%) Skip Stat Failing (25)
853 -------------------------- -------------- ---- ---- -------------- */
854 for (; fails
; fails
= fails
->next
) {
856 total
= ts
->count
- ts
->skipped
;
857 printf("%-26.26s %4lu/%-4lu %3.0f%% %4lu ", ts
->file
, ts
->failed
,
858 total
, total
? (ts
->failed
* 100.0) / total
: 0,
860 if (WIFEXITED(ts
->status
))
861 printf("%4d ", WEXITSTATUS(ts
->status
));
871 for (i
= 0; i
< ts
->count
; i
++) {
872 if (ts
->results
[i
] == TEST_FAIL
) {
873 if (first
!= 0 && i
== last
)
877 chars
+= test_print_range(first
, last
, chars
, 19);
884 test_print_range(first
, last
, chars
, 19);
889 if (ts
->reason
!= NULL
)
897 * Given the name of a test, a pointer to the testset struct, and the source
898 * and build directories, find the test. We try first relative to the current
899 * directory, then in the build directory (if not NULL), then in the source
900 * directory. In each of those directories, we first try a "-t" extension and
901 * then a ".t" extension. When we find an executable program, we fill in the
902 * path member of the testset struct. If none of those paths are executable,
903 * just fill in the name of the test with "-t" appended.
905 * The caller is responsible for freeing the path member of the testset
909 find_test(const char *name
, struct testset
*ts
, const char *source
,
913 const char *bases
[4];
921 for (i
= 0; i
< 3; i
++) {
922 if (bases
[i
] == NULL
)
924 path
= xmalloc(strlen(bases
[i
]) + strlen(name
) + 4);
925 sprintf(path
, "%s/%s-t", bases
[i
], name
);
926 if (access(path
, X_OK
) != 0)
927 path
[strlen(path
) - 2] = '.';
928 if (access(path
, X_OK
) == 0)
934 path
= xmalloc(strlen(name
) + 3);
935 sprintf(path
, "%s-t", name
);
942 * Run a batch of tests from a given file listing each test on a line by
943 * itself. Takes two additional parameters: the root of the source directory
944 * and the root of the build directory. Test programs will be first searched
945 * for in the current directory, then the build directory, then the source
946 * directory. The file must be rewindable. Returns true iff all tests
950 test_batch(const char *testlist
, const char *source
, const char *build
)
953 unsigned int length
, i
;
954 unsigned int longest
= 0;
957 struct testset ts
, *tmp
;
958 struct timeval start
, end
;
960 struct testlist
*failhead
= NULL
;
961 struct testlist
*failtail
= NULL
;
962 struct testlist
*next
;
963 unsigned long total
= 0;
964 unsigned long passed
= 0;
965 unsigned long skipped
= 0;
966 unsigned long failed
= 0;
967 unsigned long aborted
= 0;
970 * Open our file of tests to run and scan it, checking for lines that
971 * are too long and searching for the longest line.
973 tests
= fopen(testlist
, "r");
975 sysdie("can't open %s", testlist
);
977 while (fgets(buffer
, sizeof(buffer
), tests
)) {
979 length
= strlen(buffer
) - 1;
980 if (buffer
[length
] != '\n') {
981 fprintf(stderr
, "%s:%u: line too long\n", testlist
, line
);
984 if (length
> longest
)
987 if (fseek(tests
, 0, SEEK_SET
) == -1)
988 sysdie("can't rewind %s", testlist
);
991 * Add two to longest and round up to the nearest tab stop. This is how
992 * wide the column for printing the current test name will be.
996 longest
+= 8 - (longest
% 8);
998 /* Start the wall clock timer. */
999 gettimeofday(&start
, NULL
);
1002 * Now, plow through our tests again, running each one. Check line
1003 * length again out of paranoia.
1006 while (fgets(buffer
, sizeof(buffer
), tests
)) {
1008 length
= strlen(buffer
) - 1;
1009 if (buffer
[length
] != '\n') {
1010 fprintf(stderr
, "%s:%u: line too long\n", testlist
, line
);
1013 buffer
[length
] = '\0';
1014 fputs(buffer
, stdout
);
1015 for (i
= length
; i
< longest
; i
++)
1017 if (isatty(STDOUT_FILENO
))
1019 memset(&ts
, 0, sizeof(ts
));
1020 ts
.plan
= PLAN_INIT
;
1021 ts
.file
= xstrdup(buffer
);
1022 find_test(buffer
, &ts
, source
, build
);
1024 if (test_run(&ts
)) {
1028 if (ts
.reason
!= NULL
)
1031 tmp
= xmalloc(sizeof(struct testset
));
1032 memcpy(tmp
, &ts
, sizeof(struct testset
));
1034 failhead
= xmalloc(sizeof(struct testset
));
1036 failhead
->next
= NULL
;
1037 failtail
= failhead
;
1039 failtail
->next
= xmalloc(sizeof(struct testset
));
1040 failtail
= failtail
->next
;
1042 failtail
->next
= NULL
;
1045 aborted
+= ts
.aborted
;
1046 total
+= ts
.count
+ ts
.all_skipped
;
1047 passed
+= ts
.passed
;
1048 skipped
+= ts
.skipped
+ ts
.all_skipped
;
1049 failed
+= ts
.failed
;
1054 /* Stop the timer and get our child resource statistics. */
1055 gettimeofday(&end
, NULL
);
1056 getrusage(RUSAGE_CHILDREN
, &stats
);
1058 /* Print out our final results. */
1059 if (failhead
!= NULL
) {
1060 test_fail_summary(failhead
);
1061 while (failhead
!= NULL
) {
1062 next
= failhead
->next
;
1070 printf("Aborted %lu test set", aborted
);
1072 printf("Aborted %lu test sets", aborted
);
1073 printf(", passed %lu/%lu tests", passed
, total
);
1075 else if (failed
== 0)
1076 fputs("All tests successful", stdout
);
1078 printf("Failed %lu/%lu tests, %.2f%% okay", failed
, total
,
1079 (total
- failed
) * 100.0 / total
);
1082 printf(", %lu test skipped", skipped
);
1084 printf(", %lu tests skipped", skipped
);
1087 printf("Files=%u, Tests=%lu", line
, total
);
1088 printf(", %.2f seconds", tv_diff(&end
, &start
));
1089 printf(" (%.2f usr + %.2f sys = %.2f CPU)\n",
1090 tv_seconds(&stats
.ru_utime
), tv_seconds(&stats
.ru_stime
),
1091 tv_sum(&stats
.ru_utime
, &stats
.ru_stime
));
1092 return (failed
== 0 && aborted
== 0);
1097 * Run a single test case. This involves just running the test program after
1098 * having done the environment setup and finding the test program.
1101 test_single(const char *program
, const char *source
, const char *build
)
1105 memset(&ts
, 0, sizeof(ts
));
1106 find_test(program
, &ts
, source
, build
);
1107 if (execl(ts
.path
, ts
.path
, (char *) 0) == -1)
1108 sysdie("cannot exec %s", ts
.path
);
1113 * Main routine. Set the SOURCE and BUILD environment variables and then,
1114 * given a file listing tests, run each test listed.
1117 main(int argc
, char *argv
[])
1122 char *source_env
= NULL
;
1123 char *build_env
= NULL
;
1125 const char *source
= SOURCE
;
1126 const char *build
= BUILD
;
1128 while ((option
= getopt(argc
, argv
, "b:hos:")) != EOF
) {
1134 printf(usage_message
, argv
[0], argv
[0]);
1147 if (argc
- optind
!= 1) {
1148 fprintf(stderr
, usage_message
, argv
[0], argv
[0]);
1154 if (source
!= NULL
) {
1155 source_env
= xmalloc(strlen("SOURCE=") + strlen(source
) + 1);
1156 sprintf(source_env
, "SOURCE=%s", source
);
1157 if (putenv(source_env
) != 0)
1158 sysdie("cannot set SOURCE in the environment");
1160 if (build
!= NULL
) {
1161 build_env
= xmalloc(strlen("BUILD=") + strlen(build
) + 1);
1162 sprintf(build_env
, "BUILD=%s", build
);
1163 if (putenv(build_env
) != 0)
1164 sysdie("cannot set BUILD in the environment");
1168 test_single(argv
[0], source
, build
);
1170 list
= strrchr(argv
[0], '/');
1175 printf(banner
, list
);
1176 status
= test_batch(argv
[0], source
, build
) ? 0 : 1;
1179 /* For valgrind cleanliness. */
1180 if (source_env
!= NULL
) {
1181 putenv((char *) "SOURCE=");
1184 if (build_env
!= NULL
) {
1185 putenv((char *) "BUILD=");