| @@ -1,495 +1,494 @@ | | | @@ -1,495 +1,494 @@ |
1 | #!/bin/sh | | 1 | #!/bin/sh |
2 | # $NetBSD: build,v 1.103 2008/01/04 15:49:07 rillig Exp $ | | 2 | # $NetBSD: build,v 1.103.2.1 2008/06/17 14:58:14 tron Exp $ |
3 | | | 3 | |
4 | # | | 4 | # |
5 | # Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@NetBSD.org> | | 5 | # Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@NetBSD.org> |
6 | # All rights reserved. | | 6 | # All rights reserved. |
7 | # | | 7 | # |
8 | # Redistribution and use in source and binary forms, with or without | | 8 | # Redistribution and use in source and binary forms, with or without |
9 | # modification, are permitted provided that the following conditions | | 9 | # modification, are permitted provided that the following conditions |
10 | # are met: | | 10 | # are met: |
11 | # 1. Redistributions of source code must retain the above copyright | | 11 | # 1. Redistributions of source code must retain the above copyright |
12 | # notice, this list of conditions and the following disclaimer. | | 12 | # notice, this list of conditions and the following disclaimer. |
13 | # 2. Redistributions in binary form must reproduce the above copyright | | 13 | # 2. Redistributions in binary form must reproduce the above copyright |
14 | # notice, this list of conditions and the following disclaimer in the | | 14 | # notice, this list of conditions and the following disclaimer in the |
15 | # documentation and/or other materials provided with the distribution. | | 15 | # documentation and/or other materials provided with the distribution. |
16 | # 3. All advertising materials mentioning features or use of this software | | 16 | # 3. All advertising materials mentioning features or use of this software |
17 | # must display the following acknowledgement: | | 17 | # must display the following acknowledgement: |
18 | # This product includes software developed by Hubert Feyrer for | | 18 | # This product includes software developed by Hubert Feyrer for |
19 | # the NetBSD Project. | | 19 | # the NetBSD Project. |
20 | # | | 20 | # |
21 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR | | 21 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR |
22 | # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES | | 22 | # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
23 | # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. | | 23 | # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. |
24 | # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, | | 24 | # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, |
25 | # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT | | 25 | # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
26 | # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | | 26 | # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
27 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | | 27 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
28 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | | 28 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
29 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF | | 29 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
30 | # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | | 30 | # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
31 | # | | 31 | # |
32 | | | 32 | |
33 | ## | | 33 | ## |
34 | ## Globals | | 34 | ## Globals |
35 | ## | | 35 | ## |
36 | scriptdir=`dirname "$0"` | | 36 | scriptdir=`dirname "$0"` |
37 | scriptdir=`cd "${scriptdir}" && pwd` | | 37 | scriptdir=`cd "${scriptdir}" && pwd` |
38 | | | 38 | |
39 | # | | 39 | # |
40 | # Default values for command line options. | | 40 | # Default values for command line options. |
41 | # | | 41 | # |
42 | resume=no | | 42 | resume=no |
43 | mirror_only=no | | 43 | mirror_only=no |
44 | target=bulk-package | | 44 | target=bulk-package |
45 | makeargs="" | | 45 | makeargs="" |
46 | noemail=no | | 46 | noemail=no |
47 | post_only=no | | 47 | post_only=no |
48 | prepare_only=no | | 48 | prepare_only=no |
49 | | | 49 | |
50 | ## | | 50 | ## |
51 | ## Functions | | 51 | ## Functions |
52 | ## | | 52 | ## |
53 | | | 53 | |
54 | usage () { | | 54 | usage () { |
55 | cat <<EOF | | 55 | cat <<EOF |
56 | usage: $0 [options] | | 56 | usage: $0 [options] |
57 | $0 -h | --help | | 57 | $0 -h | --help |
58 | | | 58 | |
59 | Runs a bulk pkgsrc build. | | 59 | Runs a bulk pkgsrc build. |
60 | | | 60 | |
61 | The following options are supported: | | 61 | The following options are supported: |
62 | | | 62 | |
63 | -c | --config <file> | | 63 | -c | --config <file> |
64 | Load the following configuration file instead of the default | | 64 | Load the following configuration file instead of the default |
65 | one. | | 65 | one. |
66 | | | 66 | |
67 | -e | --no-email | | 67 | -e | --no-email |
68 | Don't send email when the bulk build is finished. | | 68 | Don't send email when the bulk build is finished. |
69 | | | 69 | |
70 | -h | --help | | 70 | -h | --help |
71 | Displays this message. | | 71 | Displays this message. |
72 | | | 72 | |
73 | -m | --mirror_only | | 73 | -m | --mirror_only |
74 | Downloads all distfiles needed for the build but does not run | | 74 | Downloads all distfiles needed for the build but does not run |
75 | the build. IMPORTANT: Note that this will still run all the | | 75 | the build. IMPORTANT: Note that this will still run all the |
76 | pre-build stuff which involves removing all of your installed | | 76 | pre-build stuff which involves removing all of your installed |
77 | packages. | | 77 | packages. |
78 | | | 78 | |
79 | The only difference between this option and a regular bulk build | | 79 | The only difference between this option and a regular bulk build |
80 | is that the packages are not actually built. | | 80 | is that the packages are not actually built. |
81 | | | 81 | |
82 | -p | --post-build | | 82 | -p | --post-build |
83 | Run the post-build processing and generate the report only. | | 83 | Run the post-build processing and generate the report only. |
84 | | | 84 | |
85 | --prepare | | 85 | --prepare |
86 | Only generate the package database that will be used when bulk | | 86 | Only generate the package database that will be used when bulk |
87 | building the packages. | | 87 | building the packages. |
88 | | | 88 | |
89 | -r | --resume | | 89 | -r | --resume |
90 | Resume a previously interrupted bulk build. | | 90 | Resume a previously interrupted bulk build. |
91 | | | 91 | |
92 | The --resume option may be combined with the --mirror_only | | 92 | The --resume option may be combined with the --mirror_only |
93 | option. | | 93 | option. |
94 | | | 94 | |
95 | -s | --specific-pkgs | | 95 | -s | --specific-pkgs |
96 | Sets SPECIFIC_PKGS=1 when building packages. This option is | | 96 | Sets SPECIFIC_PKGS=1 when building packages. This option is |
97 | used for building a subset of pkgsrc. | | 97 | used for building a subset of pkgsrc. |
98 | | | 98 | |
99 | EOF | | 99 | EOF |
100 | } | | 100 | } |
101 | | | 101 | |
102 | # print out error message and exit 1 | | 102 | # print out error message and exit 1 |
103 | die () { | | 103 | die () { |
104 | echo "$0: error:" 1>&2 | | 104 | echo "$0: error:" 1>&2 |
105 | for i in "$@"; do | | 105 | for i in "$@"; do |
106 | echo " $i" 1>&2 | | 106 | echo " $i" 1>&2 |
107 | done | | 107 | done |
108 | exit 1 | | 108 | exit 1 |
109 | } | | 109 | } |
110 | | | 110 | |
111 | # This function can be overridden in the build.conf file to change the | | 111 | # This function can be overridden in the build.conf file to change the |
112 | # output format of the bulk build. It is used in a pipe, so if you want | | 112 | # output format of the bulk build. It is used in a pipe, so if you want |
113 | # the original output, just define post_filter_cmd() { cat; }. | | 113 | # the original output, just define post_filter_cmd() { cat; }. |
114 | # | | 114 | # |
115 | # For more sophisticated output, you may use all the variables that this | | 115 | # For more sophisticated output, you may use all the variables that this |
116 | # example function uses. | | 116 | # example function uses. |
117 | post_filter_cmd() { | | 117 | post_filter_cmd() { |
118 | ${SED} "s;^;`date '+%Y/%m/%d %H:%M:%S'` ${percent} ${pkgdir} @ ${MACHINE_ARCH}> ;g" | | 118 | ${SED} "s;^;`date '+%Y/%m/%d %H:%M:%S'` ${percent} ${pkgdir} @ ${MACHINE_ARCH}> ;g" |
119 | } | | 119 | } |
120 | | | 120 | |
121 | # perform post-processing of the bulk-build results | | 121 | # perform post-processing of the bulk-build results |
122 | do_post_build () { | | 122 | do_post_build () { |
123 | echo "build> Post processing bulk build results..." | | 123 | echo "build> Post processing bulk build results..." |
124 | | | 124 | |
125 | # Re-install BULK_PREREQ as we may need functionality (e.g. SMTP) provided by | | 125 | # Re-install BULK_PREREQ as we may need functionality (e.g. SMTP) provided by |
126 | # them for post-build to run. | | 126 | # them for post-build to run. |
127 | echo "build> Re-installing prerequisite packages specified with BULK_PREREQ..." | | 127 | echo "build> Re-installing prerequisite packages specified with BULK_PREREQ..." |
128 | for pkgdir in $BULK_PREREQ lang/perl5; do | | 128 | for pkgdir in $BULK_PREREQ lang/perl5; do |
129 | echo "build> Installing prerequisite package $pkgdir" | | 129 | echo "build> Installing prerequisite package $pkgdir" |
130 | ( cd "${pkgsrc_dir}/${pkgdir}" \ | | 130 | ( cd "${pkgsrc_dir}/${pkgdir}" \ |
131 | && ${BMAKE} bulk-install | | 131 | && ${BMAKE} bulk-install |
132 | ) || die "Failed to install prerequisite packages." | | 132 | ) || die "Failed to install prerequisite packages." |
133 | done | | 133 | done |
134 | | | 134 | |
135 | # | | 135 | # |
136 | # Generate the post-build report. | | 136 | # Generate the post-build report. |
137 | # | | 137 | # |
138 | echo "build> Generating the bulk build report..." | | 138 | echo "build> Generating the bulk build report..." |
139 | | | 139 | |
140 | bulk_build_id=`cat "${BULK_BUILD_ID_FILE}"` \ | | 140 | bulk_build_id=`cat "${BULK_BUILD_ID_FILE}"` \ |
141 | || die "Could not read the bulk build ID from ${BULK_BUILD_ID_FILE}." | | 141 | || die "Could not read the bulk build ID from ${BULK_BUILD_ID_FILE}." |
142 | | | 142 | |
143 | report_dir="${REPORTS_DIR}/${bulk_build_id}" | | 143 | report_dir="${REPORTS_DIR}/${bulk_build_id}" |
144 | ${MKDIR} "${report_dir}" | | 144 | ${MKDIR} "${report_dir}" |
145 | ( cd "${pkgsrc_dir}" \ | | 145 | ( cd "${pkgsrc_dir}" \ |
146 | && ${PERL5} mk/bulk/post-build \ | | 146 | && ${PERL5} mk/bulk/post-build \ |
147 | > "${report_dir}/${REPORT_TXT_FILE}" | | 147 | > "${report_dir}/${REPORT_TXT_FILE}" |
148 | ) || die "Could not write the results file." | | 148 | ) || die "Could not write the results file." |
149 | } | | 149 | } |
150 | | | 150 | |
151 | # Notify the ADMIN of the finished build. | | 151 | # Notify the ADMIN of the finished build. |
152 | do_email () { | | 152 | do_email () { |
153 | case $noemail in | | 153 | case $noemail in |
154 | no) cat "${report_dir}/${REPORT_TXT_FILE}" \ | | 154 | no) cat "${report_dir}/${REPORT_TXT_FILE}" \ |
155 | | ${MAIL_CMD} -s "pkgsrc ${OPSYS} ${OS_VERSION}/${MACHINE_ARCH} bulk build results ${bulk_build_id}" "$ADMIN" | | 155 | | ${MAIL_CMD} -s "pkgsrc ${OPSYS} ${OS_VERSION}/${MACHINE_ARCH} bulk build results ${bulk_build_id}" "$ADMIN" |
156 | esac | | 156 | esac |
157 | } | | 157 | } |
158 | | | 158 | |
159 | # output final note that we're done | | 159 | # output final note that we're done |
160 | do_done () { | | 160 | do_done () { |
161 | echo "" | | 161 | echo "" |
162 | echo "build> Bulk build ended: `date`" | | 162 | echo "build> Bulk build ended: `date`" |
163 | } | | 163 | } |
164 | | | 164 | |
165 | # set all commonly used variables, prepare files etc. | | 165 | # set all commonly used variables, prepare files etc. |
166 | do_common_setup () { | | 166 | do_common_setup () { |
167 | | | 167 | |
168 | # | | 168 | # |
169 | # Choose an appropriate value for BMAKE depending on the operating | | 169 | # Choose an appropriate value for BMAKE depending on the operating |
170 | # system. | | 170 | # system. |
171 | # | | 171 | # |
172 | opsys=`uname -s` | | 172 | opsys=`uname -s` |
173 | case "$opsys" in | | 173 | case "$opsys" in |
174 | NetBSD) BMAKE=make ;; | | 174 | NetBSD) BMAKE=make ;; |
175 | *) BMAKE=bmake ;; | | 175 | *) BMAKE=bmake ;; |
176 | esac | | 176 | esac |
177 | export BMAKE | | 177 | export BMAKE |
178 | | | 178 | |
179 | # | | 179 | # |
180 | # Set resource limits as high as possible | | 180 | # Set resource limits as high as possible |
181 | # | | 181 | # |
182 | ulimit -S -s `ulimit -H -s` # XXX: why do we need unlimited stack? | | 182 | ulimit -S -s `ulimit -H -s` # XXX: why do we need unlimited stack? |
183 | ulimit -S -d `ulimit -H -d` # some processes grow rather large | | 183 | ulimit -S -d `ulimit -H -d` # some processes grow rather large |
184 | ulimit -S -t `ulimit -H -t` # pkgsrc bulk builds need _much_ time | | 184 | ulimit -S -t `ulimit -H -t` # pkgsrc bulk builds need _much_ time |
185 | | | 185 | |
186 | # | | 186 | # |
187 | # Find the configuration file. | | 187 | # Find the configuration file. |
188 | # | | 188 | # |
189 | BULK_BUILD_CONF="${BULK_BUILD_CONF-${scriptdir}/build.conf}" | | 189 | BULK_BUILD_CONF="${BULK_BUILD_CONF-${scriptdir}/build.conf}" |
190 | case $BULK_BUILD_CONF in | | 190 | case $BULK_BUILD_CONF in |
191 | /*) ;; | | 191 | /*) ;; |
192 | *) BULK_BUILD_CONF="${PWD}/${BULK_BUILD_CONF}" | | 192 | *) BULK_BUILD_CONF="${PWD}/${BULK_BUILD_CONF}" |
193 | esac | | 193 | esac |
194 | | | 194 | |
195 | # | | 195 | # |
196 | # Load the variables from the configuration file. | | 196 | # Load the variables from the configuration file. |
197 | # | | 197 | # |
198 | { test -f "${BULK_BUILD_CONF}" \ | | 198 | { test -f "${BULK_BUILD_CONF}" \ |
199 | && . "${BULK_BUILD_CONF}" \ | | 199 | && . "${BULK_BUILD_CONF}" \ |
200 | && . "${scriptdir}/post-build-conf" \ | | 200 | && . "${scriptdir}/post-build-conf" \ |
201 | && check_config_vars \ | | 201 | && check_config_vars \ |
202 | && export_config_vars | | 202 | && export_config_vars |
203 | } || die "Cannot load config file ${BULK_BUILD_CONF}, aborting." | | 203 | } || die "Cannot load config file ${BULK_BUILD_CONF}, aborting." |
204 | | | 204 | |
205 | # | | 205 | # |
206 | # Set the paths to commonly used directories. | | 206 | # Set the paths to commonly used directories. |
207 | # | | 207 | # |
208 | pkgsrc_dir="${USR_PKGSRC}" | | 208 | pkgsrc_dir="${USR_PKGSRC}" |
209 | lintpkgsrc_dir="${USR_PKGSRC}/pkgtools/lintpkgsrc" | | 209 | lintpkgsrc_dir="${USR_PKGSRC}/pkgtools/lintpkgsrc" |
210 | | | 210 | |
211 | # | | 211 | # |
212 | # Set up variables specific for the bulk build. | | 212 | # Set up variables specific for the bulk build. |
213 | # | | 213 | # |
214 | BATCH="1" | | 214 | BATCH="1" |
215 | DEPENDS_TARGET="bulk-install" | | 215 | DEPENDS_TARGET="bulk-install" |
216 | export BATCH DEPENDS_TARGET | | 216 | export BATCH DEPENDS_TARGET |
217 | | | 217 | |
218 | # | | 218 | # |
219 | # Unset some environment variables that could disturbe the build. | | 219 | # Unset some environment variables that could disturbe the build. |
220 | # | | 220 | # |
221 | unset CDPATH || true # ensure cd does not print new cwd to stdout, which | | 221 | unset CDPATH || true # ensure cd does not print new cwd to stdout, which |
222 | # confuses the printindex script. | | 222 | # confuses the printindex script. |
223 | unset DISPLAY || true # allow sane failure for gimp, xlispstat | | 223 | unset DISPLAY || true # allow sane failure for gimp, xlispstat |
224 | } | | 224 | } |
225 | | | 225 | |
226 | # Check that the package tools are up to date. | | 226 | # Check that the package tools are up to date. |
227 | check_tools () { | | 227 | check_tools () { |
228 | echo "build> Checking if the pkgtools are up-to-date" | | 228 | echo "build> Checking if the pkgtools are up-to-date" |
229 | ( cd "${lintpkgsrc_dir}" \ | | 229 | ( cd "${lintpkgsrc_dir}" \ |
230 | && ${BMAKE} fetch >/dev/null 2>&1 | | 230 | && ${BMAKE} fetch >/dev/null 2>&1 |
231 | ) || { | | 231 | ) || { |
232 | echo "build> Updating pkgtools" | | 232 | echo "build> Updating pkgtools" |
233 | ( cd "${pkgsrc_dir}/pkgtools/pkg_install" \ | | 233 | ( cd "${pkgsrc_dir}/pkgtools/pkg_install" \ |
234 | && ${BMAKE} clean \ | | 234 | && ${BMAKE} clean \ |
235 | && ${BMAKE} install \ | | 235 | && ${BMAKE} update |
236 | && ${BMAKE} clean | | | |
237 | ) || die "Could not update the package tools." | | 236 | ) || die "Could not update the package tools." |
238 | } | | 237 | } |
239 | } | | 238 | } |
240 | | | 239 | |
241 | # Run the pre-build script if necessary. | | 240 | # Run the pre-build script if necessary. |
242 | run_pre_build () { | | 241 | run_pre_build () { |
243 | case $resume in | | 242 | case $resume in |
244 | yes) echo "build> Resuming -- skipping pre-build script";; | | 243 | yes) echo "build> Resuming -- skipping pre-build script";; |
245 | *) # make veryveryclean :) | | 244 | *) # make veryveryclean :) |
246 | ( cd "${pkgsrc_dir}" \ | | 245 | ( cd "${pkgsrc_dir}" \ |
247 | && /bin/sh mk/bulk/pre-build | | 246 | && /bin/sh mk/bulk/pre-build |
248 | ) || die "Error during bulk-build preparations, aborting.";; | | 247 | ) || die "Error during bulk-build preparations, aborting.";; |
249 | esac | | 248 | esac |
250 | } | | 249 | } |
251 | | | 250 | |
252 | # Load pkgsrc variables that affect the build process. | | 251 | # Load pkgsrc variables that affect the build process. |
253 | load_vars () { | | 252 | load_vars () { |
254 | echo "+----------------------------------------+" | | 253 | echo "+----------------------------------------+" |
255 | echo "| Some variables used in the bulk build: |" | | 254 | echo "| Some variables used in the bulk build: |" |
256 | echo "+----------------------------------------+" | | 255 | echo "+----------------------------------------+" |
257 | | | 256 | |
258 | vars=" OPSYS OS_VERSION MACHINE_ARCH | | 257 | vars=" OPSYS OS_VERSION MACHINE_ARCH |
259 | BULK_PREREQ | | 258 | BULK_PREREQ |
260 | BULKFILESDIR | | 259 | BULKFILESDIR |
261 | BULK_DBFILE DEPENDSFILE INDEXFILE ORDERFILE STARTFILE | | 260 | BULK_DBFILE DEPENDSFILE INDEXFILE ORDERFILE STARTFILE |
262 | SUPPORTSFILE BULK_BUILD_ID_FILE BUILDLOG BROKENFILE | | 261 | SUPPORTSFILE BULK_BUILD_ID_FILE BUILDLOG BROKENFILE |
263 | BROKENWRKLOG | | 262 | BROKENWRKLOG |
264 | AWK GREP MAIL_CMD MKDIR PERL5 SED | | 263 | AWK GREP MAIL_CMD MKDIR PERL5 SED |
265 | PKG_DELETE PKG_INFO PKGBASE" | | 264 | PKG_DELETE PKG_INFO PKGBASE" |
266 | | | 265 | |
267 | values=`cd "$lintpkgsrc_dir" && $BMAKE show-vars VARNAMES="$vars" USE_TOOLS="awk grep mail mkdir perl sed"` | | 266 | values=`cd "$lintpkgsrc_dir" && $BMAKE show-vars VARNAMES="$vars" USE_TOOLS="awk grep mail mkdir perl sed"` |
268 | | | 267 | |
269 | for v in $vars; do | | 268 | for v in $vars; do |
270 | eval "read $v" || die "Could not read value for $v" | | 269 | eval "read $v" || die "Could not read value for $v" |
271 | eval "value=\$$v" | | 270 | eval "value=\$$v" |
272 | if [ "$v" != "BULK_PREREQ" ] && [ ! "$value" ]; then | | 271 | if [ "$v" != "BULK_PREREQ" ] && [ ! "$value" ]; then |
273 | die "$v must not be empty." | | 272 | die "$v must not be empty." |
274 | fi | | 273 | fi |
275 | printf "%-15s = %s\\n" "$v" "$value" | | 274 | printf "%-15s = %s\\n" "$v" "$value" |
276 | done <<EOF | | 275 | done <<EOF |
277 | $values | | 276 | $values |
278 | EOF | | 277 | EOF |
279 | echo "------------------------------------------" | | 278 | echo "------------------------------------------" |
280 | [ "$PKGBASE" = "lintpkgsrc" ] \ | | 279 | [ "$PKGBASE" = "lintpkgsrc" ] \ |
281 | || die "Error reading the variables." \ | | 280 | || die "Error reading the variables." \ |
282 | "Try running $0 with another shell." | | 281 | "Try running $0 with another shell." |
283 | | | 282 | |
284 | # Get the location of commonly used files | | 283 | # Get the location of commonly used files |
285 | main_buildlog="${BULKFILESDIR}/${BUILDLOG}" | | 284 | main_buildlog="${BULKFILESDIR}/${BUILDLOG}" |
286 | } | | 285 | } |
287 | | | 286 | |
288 | # Install prerequisite packages. | | 287 | # Install prerequisite packages. |
289 | # | | 288 | # |
290 | # Note: we do this _before_ the depends tree because some packages like | | 289 | # Note: we do this _before_ the depends tree because some packages like |
291 | # xpkgwedge only become DEPENDS if they are installed. | | 290 | # xpkgwedge only become DEPENDS if they are installed. |
292 | install_prereqs () { | | 291 | install_prereqs () { |
293 | echo "build> Installing prerequisite packages specified with BULK_PREREQ..." | | 292 | echo "build> Installing prerequisite packages specified with BULK_PREREQ..." |
294 | for pkgdir in $BULK_PREREQ; do | | 293 | for pkgdir in $BULK_PREREQ; do |
295 | echo "build> Installing prerequisite package $pkgdir" | | 294 | echo "build> Installing prerequisite package $pkgdir" |
296 | ( cd "${pkgsrc_dir}/${pkgdir}" \ | | 295 | ( cd "${pkgsrc_dir}/${pkgdir}" \ |
297 | && ${BMAKE} bulk-install | | 296 | && ${BMAKE} bulk-install |
298 | ) || die "Could not install prerequisite packages." | | 297 | ) || die "Could not install prerequisite packages." |
299 | done | | 298 | done |
300 | } | | 299 | } |
301 | | | 300 | |
302 | # Everything is prepared. We can start building the real packages now. | | 301 | # Everything is prepared. We can start building the real packages now. |
303 | # | | 302 | # |
304 | # Loop over every package in the correct order. Before building | | 303 | # Loop over every package in the correct order. Before building |
305 | # each one, check to see if we've already processed this package | | 304 | # each one, check to see if we've already processed this package |
306 | # before. This could happen if the build got interrupted and we | | 305 | # before. This could happen if the build got interrupted and we |
307 | # started it again with the '-resume' option. This prevents us | | 306 | # started it again with the '-resume' option. This prevents us |
308 | # from having to do a potentially very large number of make's to | | 307 | # from having to do a potentially very large number of make's to |
309 | # get back to where we let off. After we build each package, add | | 308 | # get back to where we let off. After we build each package, add |
310 | # it to the top level buildlog | | 309 | # it to the top level buildlog |
311 | # (usually '.make' or '.make.${MACHINE}'). As a side benefit, this | | 310 | # (usually '.make' or '.make.${MACHINE}'). As a side benefit, this |
312 | # can make a progress-meter very simple to add! | | 311 | # can make a progress-meter very simple to add! |
313 | do_real_bulk_build () { | | 312 | do_real_bulk_build () { |
314 | | | 313 | |
315 | cd "${pkgsrc_dir}" || die "The pkgsrc directory does not exist." | | 314 | cd "${pkgsrc_dir}" || die "The pkgsrc directory does not exist." |
316 | | | 315 | |
317 | echo "build> Starting actual build using the order specified in $ORDERFILE..." | | 316 | echo "build> Starting actual build using the order specified in $ORDERFILE..." |
318 | | | 317 | |
319 | # make sure we have something to grep in in the build loop | | 318 | # make sure we have something to grep in in the build loop |
320 | touch "${main_buildlog}" || die "Cannot write to ${main_buildlog}." | | 319 | touch "${main_buildlog}" || die "Cannot write to ${main_buildlog}." |
321 | | | 320 | |
322 | tot=`${AWK} 'END { print NR }' "${ORDERFILE}"` | | 321 | tot=`${AWK} 'END { print NR }' "${ORDERFILE}"` |
323 | for pkgdir in `cat "${ORDERFILE}"` | | 322 | for pkgdir in `cat "${ORDERFILE}"` |
324 | do | | 323 | do |
325 | if ${GREP} -q "^${pkgdir}\$" "${main_buildlog}"; then | | 324 | if ${GREP} -q "^${pkgdir}\$" "${main_buildlog}"; then |
326 | : "skip this package" | | 325 | : "skip this package" |
327 | else | | 326 | else |
328 | percent=`${AWK} -v tot="${tot}" ' | | 327 | percent=`${AWK} -v tot="${tot}" ' |
329 | END { | | 328 | END { |
330 | printf("%d/%d=%4.1f%%", NR, tot, NR*100/tot); | | 329 | printf("%d/%d=%4.1f%%", NR, tot, NR*100/tot); |
331 | }' "${main_buildlog}"` | | 330 | }' "${main_buildlog}"` |
332 | ( cd "${pkgsrc_dir}/${pkgdir}" \ | | 331 | ( cd "${pkgsrc_dir}/${pkgdir}" \ |
333 | && ${NICE_LEVEL} ${BMAKE} USE_BULK_CACHE=yes "${target}" \ | | 332 | && ${NICE_LEVEL} ${BMAKE} USE_BULK_CACHE=yes "${target}" \ |
334 | $makeargs </dev/null | post_filter_cmd | | 333 | $makeargs </dev/null | post_filter_cmd |
335 | ) || true | | 334 | ) || true |
336 | echo "$pkgdir" >> "${main_buildlog}" | | 335 | echo "$pkgdir" >> "${main_buildlog}" |
337 | fi | | 336 | fi |
338 | done | | 337 | done |
339 | | | 338 | |
340 | echo "build> Build finished." | | 339 | echo "build> Build finished." |
341 | } | | 340 | } |
342 | | | 341 | |
343 | # clean up installed packages left over | | 342 | # clean up installed packages left over |
344 | do_bulk_cleanup () { | | 343 | do_bulk_cleanup () { |
345 | | | 344 | |
346 | echo "build> Removing all installed packages left over from build..." | | 345 | echo "build> Removing all installed packages left over from build..." |
347 | for pkgname in `${PKG_INFO} -e \*` | | 346 | for pkgname in `${PKG_INFO} -e \*` |
348 | do | | 347 | do |
349 | if ${PKG_INFO} -qe "${pkgname}"; then | | 348 | if ${PKG_INFO} -qe "${pkgname}"; then |
350 | pkgdir=`${AWK} '$2 == "'"$pkgname"'" { print $1; }' "$INDEXFILE"` | | 349 | pkgdir=`${AWK} '$2 == "'"$pkgname"'" { print $1; }' "$INDEXFILE"` |
351 | case "${BULK_PREREQ}" in | | 350 | case "${BULK_PREREQ}" in |
352 | *"${pkgdir}"* ) | | 351 | *"${pkgdir}"* ) |
353 | echo "build> Keeping BULK_PREREQ: $pkgname ($pkgdir)" ; | | 352 | echo "build> Keeping BULK_PREREQ: $pkgname ($pkgdir)" ; |
354 | ;; | | 353 | ;; |
355 | * ) | | 354 | * ) |
356 | echo "build> ${PKG_DELETE} -r ${pkgname}" | | 355 | echo "build> ${PKG_DELETE} -r ${pkgname}" |
357 | ${PKG_DELETE} -r "${pkgname}" | | 356 | ${PKG_DELETE} -r "${pkgname}" |
358 | if ${PKG_INFO} -qe "${pkgname}"; then | | 357 | if ${PKG_INFO} -qe "${pkgname}"; then |
359 | echo "build> $pkgname ($pkgdir) did not deinstall nicely. Forcing the deinstall" | | 358 | echo "build> $pkgname ($pkgdir) did not deinstall nicely. Forcing the deinstall" |
360 | ${PKG_DELETE} -f "${pkgname}" || true | | 359 | ${PKG_DELETE} -f "${pkgname}" || true |
361 | fi | | 360 | fi |
362 | ;; | | 361 | ;; |
363 | esac | | 362 | esac |
364 | fi | | 363 | fi |
365 | done | | 364 | done |
366 | } | | 365 | } |
367 | | | 366 | |
368 | # start the full bulk-build | | 367 | # start the full bulk-build |
369 | do_bulk_build () { | | 368 | do_bulk_build () { |
370 | echo "build> Bulk build started: `date`" | | 369 | echo "build> Bulk build started: `date`" |
371 | echo "" | | 370 | echo "" |
372 | | | 371 | |
373 | # this function from post-build-conf | | 372 | # this function from post-build-conf |
374 | show_config_vars | | 373 | show_config_vars |
375 | | | 374 | |
376 | check_tools | | 375 | check_tools |
377 | run_pre_build | | 376 | run_pre_build |
378 | load_vars | | 377 | load_vars |
379 | | | 378 | |
380 | # | | 379 | # |
381 | # Create the directory for the log files if necessary | | 380 | # Create the directory for the log files if necessary |
382 | # | | 381 | # |
383 | if [ "${BULKFILESDIR}" != "${pkgsrc_dir}" ]; then | | 382 | if [ "${BULKFILESDIR}" != "${pkgsrc_dir}" ]; then |
384 | ${MKDIR} "${BULKFILESDIR}" | | 383 | ${MKDIR} "${BULKFILESDIR}" |
385 | fi | | 384 | fi |
386 | | | 385 | |
387 | # | | 386 | # |
388 | # Save the bulk build ID in a file, as it most often contains a time | | 387 | # Save the bulk build ID in a file, as it most often contains a time |
389 | # stamp. | | 388 | # stamp. |
390 | # | | 389 | # |
391 | case $resume in | | 390 | case $resume in |
392 | no) echo "${REPORT_BASEDIR}" > "${BULK_BUILD_ID_FILE}" \ | | 391 | no) echo "${REPORT_BASEDIR}" > "${BULK_BUILD_ID_FILE}" \ |
393 | || die "Could not save the bulk build ID in ${BULK_BUILD_ID_FILE}.";; | | 392 | || die "Could not save the bulk build ID in ${BULK_BUILD_ID_FILE}.";; |
394 | esac | | 393 | esac |
395 | | | 394 | |
396 | install_prereqs | | 395 | install_prereqs |
397 | | | 396 | |
398 | # | | 397 | # |
399 | # Create the bulk cache files. | | 398 | # Create the bulk cache files. |
400 | # | | 399 | # |
401 | if [ "x$resume" != "xyes" ]; then | | 400 | if [ "x$resume" != "xyes" ]; then |
402 | ( cd "${pkgsrc_dir}" \ | | 401 | ( cd "${pkgsrc_dir}" \ |
403 | && env PKGLIST="${PKGLIST-}" ${BMAKE} bulk-cache $makeargs | | 402 | && env PKGLIST="${PKGLIST-}" ${BMAKE} bulk-cache $makeargs |
404 | ) || die "Could not create the bulk build cache." | | 403 | ) || die "Could not create the bulk build cache." |
405 | else | | 404 | else |
406 | if [ ! -f "${ORDERFILE}" ]; then | | 405 | if [ ! -f "${ORDERFILE}" ]; then |
407 | die "The ${ORDERFILE} does not exist." \ | | 406 | die "The ${ORDERFILE} does not exist." \ |
408 | "(You cannot resume a bulk build that has not yet started.)" | | 407 | "(You cannot resume a bulk build that has not yet started.)" |
409 | fi | | 408 | fi |
410 | fi | | 409 | fi |
411 | | | 410 | |
412 | # XXX: This looks like a hack, and indeed, the functions in this file | | 411 | # XXX: This looks like a hack, and indeed, the functions in this file |
413 | # should be reorganized to better reflect the phases of the bulk build. | | 412 | # should be reorganized to better reflect the phases of the bulk build. |
414 | if [ $prepare_only = yes ]; then | | 413 | if [ $prepare_only = yes ]; then |
415 | exit 0 | | 414 | exit 0 |
416 | fi | | 415 | fi |
417 | | | 416 | |
418 | do_real_bulk_build | | 417 | do_real_bulk_build |
419 | do_bulk_cleanup | | 418 | do_bulk_cleanup |
420 | } | | 419 | } |
421 | | | 420 | |
422 | | | 421 | |
423 | ## | | 422 | ## |
424 | ## main | | 423 | ## main |
425 | ## | | 424 | ## |
426 | | | 425 | |
427 | # | | 426 | # |
428 | # Parse the command line. | | 427 | # Parse the command line. |
429 | # | | 428 | # |
430 | while test $# -gt 0; do | | 429 | while test $# -gt 0; do |
431 | case $1 in | | 430 | case $1 in |
432 | -c|--config) | | 431 | -c|--config) |
433 | shift | | 432 | shift |
434 | BULK_BUILD_CONF=$1; shift | | 433 | BULK_BUILD_CONF=$1; shift |
435 | ;; | | 434 | ;; |
436 | -e|--no-email) | | 435 | -e|--no-email) |
437 | noemail=yes | | 436 | noemail=yes |
438 | shift | | 437 | shift |
439 | ;; | | 438 | ;; |
440 | -h|--help) | | 439 | -h|--help) |
441 | usage | | 440 | usage |
442 | exit 0 | | 441 | exit 0 |
443 | ;; | | 442 | ;; |
444 | -m|--mirror_only) | | 443 | -m|--mirror_only) |
445 | mirror_only=yes | | 444 | mirror_only=yes |
446 | target=mirror-distfiles | | 445 | target=mirror-distfiles |
447 | shift | | 446 | shift |
448 | ;; | | 447 | ;; |
449 | -p|--post-build) | | 448 | -p|--post-build) |
450 | post_only=yes | | 449 | post_only=yes |
451 | shift | | 450 | shift |
452 | ;; | | 451 | ;; |
453 | --prepare) | | 452 | --prepare) |
454 | prepare_only=yes | | 453 | prepare_only=yes |
455 | shift | | 454 | shift |
456 | ;; | | 455 | ;; |
457 | -r|--resume|--restart|restart) | | 456 | -r|--resume|--restart|restart) |
458 | resume=yes | | 457 | resume=yes |
459 | shift | | 458 | shift |
460 | ;; | | 459 | ;; |
461 | -s|--specific-pkgs) | | 460 | -s|--specific-pkgs) |
462 | makeargs="$makeargs SPECIFIC_PKGS=1" | | 461 | makeargs="$makeargs SPECIFIC_PKGS=1" |
463 | shift | | 462 | shift |
464 | ;; | | 463 | ;; |
465 | *) | | 464 | *) |
466 | echo "unknown option: $1" 1>&2 | | 465 | echo "unknown option: $1" 1>&2 |
467 | usage 1>&2 | | 466 | usage 1>&2 |
468 | exit 1 | | 467 | exit 1 |
469 | ;; | | 468 | ;; |
470 | esac | | 469 | esac |
471 | done | | 470 | done |
472 | | | 471 | |
473 | do_common_setup | | 472 | do_common_setup |
474 | | | 473 | |
475 | if [ "x$post_only" = "xyes" ]; then | | 474 | if [ "x$post_only" = "xyes" ]; then |
476 | load_vars | | 475 | load_vars |
477 | do_post_build | | 476 | do_post_build |
478 | exit 0 | | 477 | exit 0 |
479 | fi | | 478 | fi |
480 | | | 479 | |
481 | do_bulk_build | | 480 | do_bulk_build |
482 | | | 481 | |
483 | # for now, just quit if we were only mirroring distfiles. At somepoint we | | 482 | # for now, just quit if we were only mirroring distfiles. At somepoint we |
484 | # should teach the post-build script to generate a nice report about how many | | 483 | # should teach the post-build script to generate a nice report about how many |
485 | # distfiles were downloaded, how many had bad checksums, failed master sites, | | 484 | # distfiles were downloaded, how many had bad checksums, failed master sites, |
486 | # network speed, etc. | | 485 | # network speed, etc. |
487 | if [ "x$mirror_only" = "xyes" ]; then | | 486 | if [ "x$mirror_only" = "xyes" ]; then |
488 | echo "build> Bulk mirror of distfiles completed: `date`" | | 487 | echo "build> Bulk mirror of distfiles completed: `date`" |
489 | exit 0 | | 488 | exit 0 |
490 | fi | | 489 | fi |
491 | | | 490 | |
492 | do_post_build | | 491 | do_post_build |
493 | do_email | | 492 | do_email |
494 | do_done | | 493 | do_done |
495 | | | 494 | |