Sat Jul 4 22:01:10 2020 UTC ()
make(1): add post-processing for tests

By defining SED_CMDS.${test}, a test can do additional post-processing
on its output, for example to normalize paths or error messages.

The Makefile for the tests now follows the common style to have all
relevant definitions at the top of the file, and all the implementation
details at the bottom.

By using the += instead of backslashed lists, it becomes easier to
comment out a single test, either temporarily or permanently (such as
the POSIX tests).

The :C modifier has been replaced with the simpler :S, some :S modifiers
have been replaced with simpler := modifiers.

The postprocessing commands have been extracted from the actual command.
This makes editing and commenting them easier.


(rillig)
diff -r1.60 -r1.61 src/usr.bin/make/unit-tests/Makefile
diff -r1.2 -r1.3 src/usr.bin/make/unit-tests/varshell.exp
cvs diff -r1.60 -r1.61 src/usr.bin/make/unit-tests/Makefile (expand / switch to unified diff)

--- src/usr.bin/make/unit-tests/Makefile 2020/07/04 21:04:25 1.60
+++ src/usr.bin/make/unit-tests/Makefile 2020/07/04 22:01:10 1.61
@@ -1,154 +1,163 @@ @@ -1,154 +1,163 @@
1# $NetBSD: Makefile,v 1.60 2020/07/04 21:04:25 rillig Exp $ 1# $NetBSD: Makefile,v 1.61 2020/07/04 22:01:10 rillig Exp $
2# 2#
3# Unit tests for make(1) 3# Unit tests for make(1)
4# 4#
5# The main targets are: 5# The main targets are:
6# 6#
7# all: 7# all:
8# run all the tests 8# run all the tests
9# test: 9# test:
10# run 'all', and compare to expected results 10# run 'all', and compare to expected results
11# accept: 11# accept:
12# move generated output to expected results 12# move generated output to expected results
13# 13#
 14# Settable variables
 15#
 16# TEST_MAKE
 17# The make program to be tested.
 18#
14# 19#
15# Adding a test case 20# Adding a test case
16# 21#
17# Each feature should get its own set of tests in its own suitably 22# Each feature should get its own set of tests in its own suitably
18# named makefile (*.mk), with its own set of expected results (*.exp), 23# named makefile (*.mk), with its own set of expected results (*.exp),
19# and it should be added to the TESTNAMES list. 24# and it should be added to the TESTS list.
20# 25#
21# Any added files must also be added to src/distrib/sets/lists/tests/mi. 26# Any added files must also be added to src/distrib/sets/lists/tests/mi.
22# Makefiles that are not added to TESTNAMES must be ignored in 27# Makefiles that are not added to TESTS must be ignored in
23# src/tests/usr.bin/make/t_make.sh (example: include-sub). 28# src/tests/usr.bin/make/t_make.sh (example: include-sub).
24# 29#
25 30
 31# Each test is in a sub-makefile.
 32# Keep the list sorted.
 33TESTS+= comment
 34TESTS+= cond-late
 35TESTS+= cond-short
 36TESTS+= cond1
 37TESTS+= cond2
 38TESTS+= dollar
 39TESTS+= doterror
 40TESTS+= dotwait
 41TESTS+= error
 42TESTS+= # escape # broken by referting POSIX changes
 43TESTS+= export
 44TESTS+= export-all
 45TESTS+= export-env
 46TESTS+= forloop
 47TESTS+= forsubst
 48TESTS+= hash
 49TESTS+= # impsrc # broken by referting POSIX changes
 50TESTS+= include-main
 51TESTS+= misc
 52TESTS+= moderrs
 53TESTS+= modmatch
 54TESTS+= modmisc
 55TESTS+= modorder
 56TESTS+= modts
 57TESTS+= modword
 58TESTS+= order
 59TESTS+= # phony-end # broken by referting POSIX changes
 60TESTS+= posix
 61TESTS+= # posix1 # broken by referting POSIX changes
 62TESTS+= qequals
 63TESTS+= # suffixes # broken by referting POSIX changes
 64TESTS+= sunshcmd
 65TESTS+= sysv
 66TESTS+= ternary
 67TESTS+= unexport
 68TESTS+= unexport-env
 69TESTS+= varcmd
 70TESTS+= varmisc
 71TESTS+= varmod-edge
 72TESTS+= varquote
 73TESTS+= varshell
 74
 75# Override make flags for certain tests; default is -k.
 76FLAGS.doterror= # none
 77FLAGS.order= -j1
 78
 79# Some tests need extra post-processing.
 80SED_CMDS.varshell+= -e 's,^[a-z]*sh: ,,'
 81SED_CMDS.varshell+= -e '/command/s,No such.*,not found,'
 82
 83# End of the configuration section.
 84
26.MAIN: all 85.MAIN: all
27 86
28UNIT_TESTS:= ${.PARSEDIR} 87UNIT_TESTS:= ${.PARSEDIR}
29.PATH: ${UNIT_TESTS} 88.PATH: ${UNIT_TESTS}
30 89
31# Each test is in a sub-makefile. 90OUTFILES= ${TESTS:=.out}
32# Keep the list sorted. 
33TESTNAMES= \ 
34 comment \ 
35 cond-late \ 
36 cond-short \ 
37 cond1 \ 
38 cond2 \ 
39 dollar \ 
40 doterror \ 
41 dotwait \ 
42 error \ 
43 export \ 
44 export-all \ 
45 export-env \ 
46 forloop \ 
47 forsubst \ 
48 hash \ 
49 include-main \ 
50 misc \ 
51 moderrs \ 
52 modmatch \ 
53 modmisc \ 
54 modorder \ 
55 modts \ 
56 modword \ 
57 order \ 
58 posix \ 
59 qequals \ 
60 sunshcmd \ 
61 sysv \ 
62 ternary \ 
63 unexport \ 
64 unexport-env \ 
65 varcmd \ 
66 varmisc \ 
67 varmod-edge \ 
68 varquote \ 
69 varshell 
70 
71# these tests were broken by referting POSIX chanegs 
72STRICT_POSIX_TESTS = \ 
73 escape \ 
74 impsrc \ 
75 phony-end \ 
76 posix1 \ 
77 suffixes 
78 
79# Override make flags for certain tests 
80flags.doterror= 
81flags.order=-j1 
82 
83OUTFILES= ${TESTNAMES:S/$/.out/} 
84 91
85all: ${OUTFILES} 92all: ${OUTFILES}
86 93
87CLEANFILES += *.rawout *.out *.status *.tmp *.core *.tmp 94CLEANFILES+= *.rawout *.out *.status *.tmp *.core *.tmp
88CLEANFILES += obj*.[och] lib*.a # posix1.mk 95CLEANFILES+= obj*.[och] lib*.a # posix1.mk
89CLEANFILES += issue* .[ab]* # suffixes.mk 96CLEANFILES+= issue* .[ab]* # suffixes.mk
90CLEANRECURSIVE += dir dummy # posix1.mk 97CLEANRECURSIVE+= dir dummy # posix1.mk
91 98
92clean: 99clean:
93 rm -f ${CLEANFILES} 100 rm -f ${CLEANFILES}
94.if !empty(CLEANRECURSIVE) 101.if !empty(CLEANRECURSIVE)
95 rm -rf ${CLEANRECURSIVE} 102 rm -rf ${CLEANRECURSIVE}
96.endif 103.endif
97 104
98TEST_MAKE?= ${.MAKE} 105TEST_MAKE?= ${.MAKE}
99TOOL_SED?= sed 106TOOL_SED?= sed
100 107
101# ensure consistent results from sort(1) 108# ensure consistent results from sort(1)
102LC_ALL= C 109LC_ALL= C
103LANG= C 110LANG= C
104.export LANG LC_ALL 111.export LANG LC_ALL
105 112
106# the tests are actually done with sub-makes. 113# the tests are actually done with sub-makes.
107.SUFFIXES: .mk .rawout .out 114.SUFFIXES: .mk .rawout .out
108.mk.rawout: 115.mk.rawout:
109 @echo ${TEST_MAKE} ${flags.${.TARGET:R}:U-k} -f ${.IMPSRC} 116 @echo ${TEST_MAKE} ${FLAGS.${.TARGET:R}:U-k} -f ${.IMPSRC}
110 -@cd ${.OBJDIR} && \ 117 -@cd ${.OBJDIR} && \
111 { ${TEST_MAKE} ${flags.${.TARGET:R}:U-k} -f ${.IMPSRC} \ 118 { ${TEST_MAKE} ${FLAGS.${.TARGET:R}:U-k} -f ${.IMPSRC} \
112 2>&1 ; echo $$? >${.TARGET:R}.status ; } > ${.TARGET}.tmp 119 2>&1 ; echo $$? >${.TARGET:R}.status ; } > ${.TARGET}.tmp
113 @mv ${.TARGET}.tmp ${.TARGET} 120 @mv ${.TARGET}.tmp ${.TARGET}
114 121
 122_SED_CMDS+= -e 's,^${TEST_MAKE:T:S,.,\\.,g}[][0-9]*:,make:,'
 123_SED_CMDS+= -e 's,${TEST_MAKE:S,.,\\.,g},make,'
 124_SED_CMDS+= -e '/stopped/s, /.*, unit-tests,'
 125_SED_CMDS+= -e 's,${.CURDIR:S,.,\\.,g}/,,g'
 126_SED_CMDS+= -e 's,${UNIT_TESTS:S,.,\\.,g}/,,g'
 127
115# We always pretend .MAKE was called 'make'  128# We always pretend .MAKE was called 'make'
116# and strip ${.CURDIR}/ from the output 129# and strip ${.CURDIR}/ from the output
117# and replace anything after 'stopped in' with unit-tests 130# and replace anything after 'stopped in' with unit-tests
118# so the results can be compared. 131# so the results can be compared.
119.rawout.out: 132.rawout.out:
120 @echo postprocess ${.TARGET} 133 @echo postprocess ${.TARGET}
121 @${TOOL_SED} -e 's,^${TEST_MAKE:T:C/\./\\\./g}[][0-9]*:,make:,' \ 134 @${TOOL_SED} ${_SED_CMDS} ${SED_CMDS.${.TARGET:R}} \
122 -e 's,${TEST_MAKE:C/\./\\\./g},make,' \ 
123 -e '/stopped/s, /.*, unit-tests,' \ 
124 -e 's,${.CURDIR:C/\./\\\./g}/,,g' \ 
125 -e 's,${UNIT_TESTS:C/\./\\\./g}/,,g' \ 
126 < ${.IMPSRC} > ${.TARGET}.tmp 135 < ${.IMPSRC} > ${.TARGET}.tmp
127 @echo "exit status `cat ${.TARGET:R}.status`" >> ${.TARGET}.tmp 136 @echo "exit status `cat ${.TARGET:R}.status`" >> ${.TARGET}.tmp
128 @mv ${.TARGET}.tmp ${.TARGET} 137 @mv ${.TARGET}.tmp ${.TARGET}
129 138
130# Compare all output files 139# Compare all output files
131test: ${OUTFILES} .PHONY 140test: ${OUTFILES} .PHONY
132 @failed= ; \ 141 @failed= ; \
133 for test in ${TESTNAMES}; do \ 142 for test in ${TESTS}; do \
134 diff -u ${UNIT_TESTS}/$${test}.exp $${test}.out \ 143 diff -u ${UNIT_TESTS}/$${test}.exp $${test}.out \
135 || failed="$${failed}$${failed:+ }$${test}" ; \ 144 || failed="$${failed}$${failed:+ }$${test}" ; \
136 done ; \ 145 done ; \
137 if [ -n "$${failed}" ]; then \ 146 if [ -n "$${failed}" ]; then \
138 echo "Failed tests: $${failed}" ; false ; \ 147 echo "Failed tests: $${failed}" ; false ; \
139 else \ 148 else \
140 echo "All tests passed" ; \ 149 echo "All tests passed" ; \
141 fi 150 fi
142 151
143accept: 152accept:
144 @for test in ${TESTNAMES}; do \ 153 @for test in ${TESTS}; do \
145 cmp -s ${UNIT_TESTS}/$${test}.exp $${test}.out \ 154 cmp -s ${UNIT_TESTS}/$${test}.exp $${test}.out \
146 || { echo "Replacing $${test}.exp" ; \ 155 || { echo "Replacing $${test}.exp" ; \
147 cp $${test}.out ${UNIT_TESTS}/$${test}.exp ; } \ 156 cp $${test}.out ${UNIT_TESTS}/$${test}.exp ; } \
148 done 157 done
149 158
150.if exists(${TEST_MAKE}) 159.if exists(${TEST_MAKE})
151${TESTNAMES:S/$/.rawout/}: ${TEST_MAKE} 160${TESTS:=.rawout}: ${TEST_MAKE}
152.endif 161.endif
153 162
154.-include <bsd.obj.mk> 163.-include <bsd.obj.mk>
cvs diff -r1.2 -r1.3 src/usr.bin/make/unit-tests/Attic/varshell.exp (expand / switch to unified diff)

--- src/usr.bin/make/unit-tests/Attic/varshell.exp 2015/04/10 20:41:59 1.2
+++ src/usr.bin/make/unit-tests/Attic/varshell.exp 2020/07/04 22:01:10 1.3
@@ -1,12 +1,12 @@ @@ -1,12 +1,12 @@
1sh: /bin/no/such/command: not found 1/bin/no/such/command: not found
2make: "varshell.mk" line 5: warning: "/bin/no/such/command" returned non-zero status 2make: "varshell.mk" line 5: warning: "/bin/no/such/command" returned non-zero status
3make: "varshell.mk" line 6: warning: "kill -14 $$" exited on a signal 3make: "varshell.mk" line 6: warning: "kill -14 $$" exited on a signal
4make: "varshell.mk" line 7: warning: "false" returned non-zero status 4make: "varshell.mk" line 7: warning: "false" returned non-zero status
5make: "varshell.mk" line 8: warning: "echo "output before the error"; false" returned non-zero status 5make: "varshell.mk" line 8: warning: "echo "output before the error"; false" returned non-zero status
6EXEC_FAILED='' 6EXEC_FAILED=''
7TERMINATED_BY_SIGNAL='' 7TERMINATED_BY_SIGNAL=''
8ERROR_NO_OUTPUT='' 8ERROR_NO_OUTPUT=''
9ERROR_WITH_OUTPUT='output before the error' 9ERROR_WITH_OUTPUT='output before the error'
10NO_ERROR_NO_OUTPUT='' 10NO_ERROR_NO_OUTPUT=''
11NO_ERROR_WITH_OUTPUT='this is good' 11NO_ERROR_WITH_OUTPUT='this is good'
12exit status 0 12exit status 0