Thu Mar 19 16:57:35 2020 UTC ()
mk/subst.mk: add SUBST_NOOP_OK to catch typos and outdated definitions

In a bulk build with very strict settings (WARNING_MSG fails, as well as
no-op substitutions), it became clear that nearly all of the cases where
SUBST didn't replace anything were bugs in the package definition.

Most of them were just outdated, which is no surprise given that some
packages are already over 20 years old.

For backwards compatibility, SUBST_NOOP_OK defaults to "yes" right now.
After correcting the affected packages, the default will change to "no".


(rillig)
diff -r1.62 -r1.63 pkgsrc/mk/subst.mk

cvs diff -r1.62 -r1.63 pkgsrc/mk/subst.mk (expand / switch to unified diff)

--- pkgsrc/mk/subst.mk 2019/11/22 18:04:49 1.62
+++ pkgsrc/mk/subst.mk 2020/03/19 16:57:35 1.63
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1# $NetBSD: subst.mk,v 1.62 2019/11/22 18:04:49 minskim Exp $ 1# $NetBSD: subst.mk,v 1.63 2020/03/19 16:57:35 rillig Exp $
2# 2#
3# The subst framework replaces text in one or more files in the WRKSRC 3# The subst framework replaces text in one or more files in the WRKSRC
4# directory. Packages can define several ``classes'' of replacements. 4# directory. Packages can define several ``classes'' of replacements.
5# Each such class defines: 5# Each such class defines:
6# 6#
7# - in which stage of the build process the replacement happens 7# - in which stage of the build process the replacement happens
8# - which files are affected by the replacement 8# - which files are affected by the replacement
9# - which text or pattern is replaced by which replacement text 9# - which text or pattern is replaced by which replacement text
10# 10#
11# A typical example is: 11# A typical example is:
12# 12#
13# SUBST_CLASSES+= prefix 13# SUBST_CLASSES+= prefix
14# SUBST_STAGE.prefix= pre-configure 14# SUBST_STAGE.prefix= pre-configure
@@ -22,129 +22,152 @@ @@ -22,129 +22,152 @@
22# sure to append them (+=) instead of overriding them (=). 22# sure to append them (+=) instead of overriding them (=).
23# 23#
24# SUBST_STAGE.<class> 24# SUBST_STAGE.<class>
25# "stage" at which we do the text replacement. Should be one of 25# "stage" at which we do the text replacement. Should be one of
26# {pre,do,post}-{extract,patch,configure,build,install}. 26# {pre,do,post}-{extract,patch,configure,build,install}.
27# 27#
28# SUBST_MESSAGE.<class> 28# SUBST_MESSAGE.<class>
29# The message to display before doing the substitution. 29# The message to display before doing the substitution.
30# 30#
31# SUBST_FILES.<class> 31# SUBST_FILES.<class>
32# A list of file patterns on which to run the substitution; 32# A list of file patterns on which to run the substitution;
33# the filenames are either absolute or relative to ${WRKSRC}. 33# the filenames are either absolute or relative to ${WRKSRC}.
34# 34#
 35# Starting with 2020Q1, it is an error if any of these patterns
 36# has no effect at all, to catch typos and outdated definitions.
 37# To prevent this, see SUBST_NOOP_OK.<class> below.
 38#
35# SUBST_SED.<class> 39# SUBST_SED.<class>
36# List of sed(1) arguments to run on the specified files. 40# List of sed(1) arguments to run on the specified files.
37# Multiple commands can be specified using the -e option of sed. 41# Multiple commands can be specified using the -e option of sed.
38# Do not use non-standard sed options (e.g. -E). 42# Do not use non-standard sed options (e.g. -E).
39# 43#
40# SUBST_VARS.<class> 44# SUBST_VARS.<class>
41# List of variables that are substituted whenever they appear in 45# List of variables that are substituted whenever they appear in
42# the form @VARNAME@. This is basically a shortcut for 46# the form @VARNAME@. This is basically a shortcut for
43# 47#
44# -e 's,@VARNAME@,${VARNAME},g' 48# -e 's,@VARNAME@,${VARNAME},g'
45# 49#
46# that even works when ${VARNAME} contains arbitrary characters. 50# that even works when ${VARNAME} contains arbitrary characters.
47# Both SUBST_SED and SUBST_VARS can be used in a single class. 51# SUBST_SED and SUBST_VARS can combined freely.
48# 52#
49# SUBST_FILTER_CMD.<class> 53# SUBST_FILTER_CMD.<class>
50# Filter used to perform the actual substitution on the specified 54# Filter used to perform the actual substitution on the specified
51# files. Defaults to ${SED} ${SUBST_SED.<class>}. 55# files. Defaults to ${SED} ${SUBST_SED.<class>}.
52# 56#
53# SUBST_SKIP_TEXT_CHECK.<class> 57# SUBST_SKIP_TEXT_CHECK.<class>
54# By default, each file is checked whether it really is a text file 58# By default, each file is checked whether it really is a text file
55# before any substitutions are done to it. Since that test is not 59# before any substitutions are done to it. Since that test is not
56# perfect, it can be disabled by setting this variable to "yes". 60# perfect, it can be disabled by setting this variable to "yes".
57# 61#
58# SUBST_SHOW_DIFF.<class> 62# SUBST_SHOW_DIFF.<class>
59# During development of a package, this can be set to "yes" to see 63# During development of a package, this can be set to "yes" to see
60# the actual changes as a unified diff. 64# the actual changes as a unified diff.
61# 65#
 66# SUBST_NOOP_OK.<class>
 67# Whether to fail when a SUBST_FILES pattern has no effect.
 68# In most cases, "yes" is appropriate, to catch typos and outdated
 69# definitions.
 70#
 71# Default: no (up to 2019Q4), yes (starting with 2020Q1)
 72#
62# See also: 73# See also:
63# PLIST_SUBST 74# PLIST_SUBST
64# 75#
65# Keywords: subst 76# Keywords: subst
66# 77#
67 78
68_VARGROUPS+= subst 79_VARGROUPS+= subst
69_PKG_VARS.subst= SUBST_CLASSES 80_PKG_VARS.subst= SUBST_CLASSES
70.for c in ${SUBST_CLASSES} 81.for c in ${SUBST_CLASSES}
71. for pv in SUBST_STAGE SUBST_MESSAGE SUBST_FILES SUBST_SED SUBST_VARS \ 82. for pv in SUBST_STAGE SUBST_MESSAGE SUBST_FILES SUBST_SED SUBST_VARS \
72 SUBST_FILTER_CMD SUBST_SKIP_TEXT_CHECK 83 SUBST_FILTER_CMD SUBST_SKIP_TEXT_CHECK SUBST_NOOP_OK
73_PKG_VARS.subst+= ${pv}.${c} 84_PKG_VARS.subst+= ${pv}.${c}
74. endfor 85. endfor
75.endfor 86.endfor
76_DEF_VARS.subst= ECHO_SUBST_MSG 87_DEF_VARS.subst= ECHO_SUBST_MSG
77_USE_VARS.subst= WRKDIR WRKSRC 88_USE_VARS.subst= WRKDIR WRKSRC
78_IGN_VARS.subst= _SUBST_IS_TEXT_FILE_CMD 89_IGN_VARS.subst= _SUBST_IS_TEXT_FILE_CMD
79_SORTED_VARS.subst= SUBST_CLASSES SUBST_FILES.* SUBST_VARS.* 90_SORTED_VARS.subst= SUBST_CLASSES SUBST_FILES.* SUBST_VARS.*
80_LISTED_VARS.subst= SUBST_SED.* SUBST_FILTER_CMD.* 91_LISTED_VARS.subst= SUBST_SED.* SUBST_FILTER_CMD.*
81 92
82ECHO_SUBST_MSG?= ${STEP_MSG} 93ECHO_SUBST_MSG?= ${STEP_MSG}
83 94
84# _SUBST_IS_TEXT_FILE_CMD returns 0 if $$file is a text file. 95# _SUBST_IS_TEXT_FILE_CMD returns 0 if $$file is a text file.
85_SUBST_IS_TEXT_FILE_CMD?= \ 96_SUBST_IS_TEXT_FILE_CMD?= \
86 [ -z "`LC_ALL=C ${TR} -cd '\\0' < "$$file" | ${TR} '\\0' 'x'`" ] 97 [ -z "`LC_ALL=C ${TR} -cd '\\0' < "$$file" | ${TR} '\\0' 'x'`" ]
87 98
 99.if ${SUBST_CLASSES:U:O} != ${SUBST_CLASSES:U:O:u}
 100PKG_FAIL_REASON+= "[subst.mk] duplicate SUBST class: ${SUBST_CLASSES:O:u}"
 101.endif
 102
88.for _class_ in ${SUBST_CLASSES} 103.for _class_ in ${SUBST_CLASSES}
89_SUBST_COOKIE.${_class_}= ${WRKDIR}/.subst_${_class_}_done 104_SUBST_COOKIE.${_class_}= ${WRKDIR}/.subst_${_class_}_done
90 105
91SUBST_FILTER_CMD.${_class_}?= LC_ALL=C ${SED} ${SUBST_SED.${_class_}} 106SUBST_FILTER_CMD.${_class_}?= LC_ALL=C ${SED} ${SUBST_SED.${_class_}}
92SUBST_VARS.${_class_}?= # none 107SUBST_VARS.${_class_}?= # none
93SUBST_MESSAGE.${_class_}?= Substituting "${_class_}" in ${SUBST_FILES.${_class_}} 108SUBST_MESSAGE.${_class_}?= Substituting "${_class_}" in ${SUBST_FILES.${_class_}}
94. for v in ${SUBST_VARS.${_class_}} 109. for v in ${SUBST_VARS.${_class_}}
95SUBST_FILTER_CMD.${_class_}+= -e s,@${v:C|[^A-Za-z0-9_]|\\\\&|gW:Q}@,${${v}:S|\\|\\\\|gW:S|,|\\,|gW:S|&|\\\&|gW:S|${.newline}|\\${.newline}|gW:Q},g 110SUBST_FILTER_CMD.${_class_}+= -e s,@${v:C|[^A-Za-z0-9_]|\\\\&|gW:Q}@,${${v}:S|\\|\\\\|gW:S|,|\\,|gW:S|&|\\\&|gW:S|${.newline}|\\${.newline}|gW:Q},g
96. endfor 111. endfor
97. if !empty(SUBST_SHOW_DIFF.${_class_}:Uno:M[Yy][Ee][Ss]) 112. if !empty(SUBST_SHOW_DIFF.${_class_}:Uno:M[Yy][Ee][Ss])
98_SUBST_KEEP.${_class_}?= ${DIFF} -u "$$file" "$$tmpfile" || true 113_SUBST_KEEP.${_class_}?= ${DIFF} -u "$$file" "$$tmpfile" || true
99. endif 114. endif
100_SUBST_KEEP.${_class_}?= ${DO_NADA} 115_SUBST_KEEP.${_class_}?= ${DO_NADA}
101SUBST_SKIP_TEXT_CHECK.${_class_}?= no 116SUBST_SKIP_TEXT_CHECK.${_class_}?= no
 117SUBST_NOOP_OK.${_class_}?= yes # TODO: change to no after 2020Q1
102 118
103.if !empty(SUBST_SKIP_TEXT_CHECK.${_class_}:M[Yy][Ee][Ss]) 119.if !empty(SUBST_SKIP_TEXT_CHECK.${_class_}:M[Yy][Ee][Ss])
104_SUBST_IS_TEXT_FILE_CMD.${_class_}= ${TRUE} 120_SUBST_IS_TEXT_FILE_CMD.${_class_}= ${TRUE}
105.else 121.else
106_SUBST_IS_TEXT_FILE_CMD.${_class_}= ${_SUBST_IS_TEXT_FILE_CMD} 122_SUBST_IS_TEXT_FILE_CMD.${_class_}= ${_SUBST_IS_TEXT_FILE_CMD}
107.endif 123.endif
108 124
109. if defined(SUBST_STAGE.${_class_}) 125. if defined(SUBST_STAGE.${_class_})
110${SUBST_STAGE.${_class_}}: subst-${_class_} 126${SUBST_STAGE.${_class_}}: subst-${_class_}
111. else 127. else
112# SUBST_STAGE.* does not need to be defined. 128# SUBST_STAGE.* does not need to be defined.
113#PKG_FAIL_REASON+= "SUBST_STAGE missing for ${_class_}." 129#PKG_FAIL_REASON+= "SUBST_STAGE missing for ${_class_}."
114. endif 130. endif
115 131
116.PHONY: subst-${_class_} 132.PHONY: subst-${_class_}
117subst-${_class_}: ${_SUBST_COOKIE.${_class_}} 133subst-${_class_}: ${_SUBST_COOKIE.${_class_}}
118 134
119${_SUBST_COOKIE.${_class_}}: 135${_SUBST_COOKIE.${_class_}}:
120. if !empty(SUBST_MESSAGE.${_class_}) 136. if !empty(SUBST_MESSAGE.${_class_})
121 ${RUN} ${ECHO_SUBST_MSG} ${SUBST_MESSAGE.${_class_}:Q} 137 ${RUN} ${ECHO_SUBST_MSG} ${SUBST_MESSAGE.${_class_}:Q}
122. endif 138. endif
 139.for pattern in ${SUBST_FILES.${_class_}}
123 ${RUN} cd ${WRKSRC:Q}; \ 140 ${RUN} cd ${WRKSRC:Q}; \
124 files=${SUBST_FILES.${_class_}:Q}; \ 141 changed=no; \
125 for file in $$files; do \ 142 for file in ${pattern}; do \
126 case $$file in /*) ;; *) file="./$$file";; esac; \ 143 case $$file in /*) ;; *) file="./$$file";; esac; \
127 tmpfile="$$file.subst.sav"; \ 144 tmpfile="$$file.subst.sav"; \
128 if [ ! -f "$$file" ]; then \ 145 if [ ! -f "$$file" ]; then \
129 ${WARNING_MSG} "[subst.mk:${_class_}] Ignoring non-existent file \"$$file\"."; \ 146 ${WARNING_MSG} "[subst.mk:${_class_}] Ignoring non-existent file \"$$file\"."; \
130 elif ${_SUBST_IS_TEXT_FILE_CMD.${_class_}}; then \ 147 elif ${_SUBST_IS_TEXT_FILE_CMD.${_class_}}; then \
131 ${SUBST_FILTER_CMD.${_class_}} \ 148 ${SUBST_FILTER_CMD.${_class_}} \
132 < "$$file" \ 149 < "$$file" \
133 > "$$tmpfile"; \ 150 > "$$tmpfile"; \
134 if ${TEST} -x "$$file"; then \ 151 if ${TEST} -x "$$file"; then \
135 ${CHMOD} +x "$$tmpfile"; \ 152 ${CHMOD} +x "$$tmpfile"; \
136 fi; \ 153 fi; \
137 if ${CMP} -s "$$tmpfile" "$$file"; then \ 154 if ${CMP} -s "$$tmpfile" "$$file"; then \
138 ${INFO_MSG} "[subst.mk:${_class_}] Nothing changed in $$file."; \ 155 ${INFO_MSG} "[subst.mk:${_class_}] Nothing changed in $$file."; \
139 ${RM} -f "$$tmpfile"; \ 156 ${RM} -f "$$tmpfile"; \
140 else \ 157 else \
 158 changed=yes; \
141 ${_SUBST_KEEP.${_class_}}; \ 159 ${_SUBST_KEEP.${_class_}}; \
142 ${MV} -f "$$tmpfile" "$$file"; \ 160 ${MV} -f "$$tmpfile" "$$file"; \
143 ${ECHO} "$$file" >> ${.TARGET}; \ 161 ${ECHO} "$$file" >> ${.TARGET}; \
144 fi; \ 162 fi; \
145 else \ 163 else \
146 ${WARNING_MSG} "[subst.mk:${_class_}] Ignoring non-text file \"$$file\"."; \ 164 ${WARNING_MSG} "[subst.mk:${_class_}] Ignoring non-text file \"$$file\"."; \
147 fi; \ 165 fi; \
148 done 166 done; \
 167 \
 168 if test "$$changed,${SUBST_NOOP_OK.${_class_}:tl}" = no,no; then \
 169 ${FAIL_MSG} "[subst.mk:${_class_}] The pattern "${pattern:Q}" has no effect."; \
 170 fi
 171.endfor
149 ${RUN} ${TOUCH} ${TOUCH_FLAGS} ${.TARGET:Q} 172 ${RUN} ${TOUCH} ${TOUCH_FLAGS} ${.TARGET:Q}
150.endfor 173.endfor