 @@ -1,494 +1,505 @@
 #!/bin/sh
-# $NetBSD: build,v 1.105 2008/06/13 21:52:16 sketch Exp $
+# $NetBSD: build,v 1.105.14.1 2009/07/09 20:10:39 tron Exp $
+#
 # Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@NetBSD.org>
 # All rights reserved.
+#
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 # are met:
 # 1. Redistributions of source code must retain the above copyright
 #    notice, this list of conditions and the following disclaimer.
 # 2. Redistributions in binary form must reproduce the above copyright
 #    notice, this list of conditions and the following disclaimer in the
 #    documentation and/or other materials provided with the distribution.
 # 3. All advertising materials mentioning features or use of this software
 #    must display the following acknowledgement:
 #      This product includes software developed by Hubert Feyrer for
 #	the NetBSD Project.
+#
 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
 ##
 ## Globals
 ##
 scriptdir=`dirname "$0"`
 scriptdir=`cd "${scriptdir}" && pwd`
+#
 # Default values for command line options.
+#
 resume=no
 mirror_only=no
 target=bulk-package
 makeargs=""
 noemail=no
 post_only=no
 prepare_only=no
 ##
 ## Functions
 ##
 usage () {
 	cat <<EOF
 usage: $0 [options]
        $0 -h | --help
 Runs a bulk pkgsrc build.
 The following options are supported:
    -c | --config <file>
 	Load the following configuration file instead of the default
 	one.
    -e | --no-email
 	Don't send email when the bulk build is finished.
    -h | --help
 	Displays this message.
    -m | --mirror_only
 	Downloads all distfiles needed for the build but does not run
 	the build.  IMPORTANT:  Note that this will still run all the
 	pre-build stuff which involves removing all of your installed
 	packages.
 	The only difference between this option and a regular bulk build
 	is that the packages are not actually built.
    -p | --post-build
         Run the post-build processing and generate the report only.
    --prepare
         Only generate the package database that will be used when bulk
         building the packages.
    -r | --resume
 	Resume a previously interrupted bulk build.
 	The --resume option may be combined with the --mirror_only
 	option.
    -s | --specific-pkgs
 	Sets SPECIFIC_PKGS=1 when building packages.  This option is
 	used for building a subset of pkgsrc.
 EOF
+}
 # print out error message and exit 1
 die () {
 	echo "$0: error:" 1>&2
 	for i in "$@"; do
 		echo "    $i" 1>&2
 	done
 	exit 1
+}
 # This function can be overridden in the build.conf file to change the
 # output format of the bulk build. It is used in a pipe, so if you want
 # the original output, just define post_filter_cmd() { cat; }.
+#
 # For more sophisticated output, you may use all the variables that this
 # example function uses.
 post_filter_cmd() {
 	${SED} "s;^;`date '+%Y/%m/%d %H:%M:%S'`  ${percent} ${pkgdir} @ ${MACHINE_ARCH}> ;g"
+}
 # perform post-processing of the bulk-build results
 do_post_build () {
 	echo "build> Post processing bulk build results..."
 	# Re-install BULK_PREREQ as we may need functionality (e.g. SMTP) provided by
 	# them for post-build to run.
 	echo "build> Re-installing prerequisite packages specified with BULK_PREREQ..."
 	for pkgdir in $BULK_PREREQ lang/perl5; do
 		echo "build> Installing prerequisite package $pkgdir"
 		( cd "${pkgsrc_dir}/${pkgdir}" \
 		  && ${BMAKE} bulk-install
 		) || die "Failed to install prerequisite packages."
 	done
+	#
 	# Generate the post-build report.
+	#
 	echo "build> Generating the bulk build report..."
 	bulk_build_id=`cat "${BULK_BUILD_ID_FILE}"` \
 	|| die "Could not read the bulk build ID from ${BULK_BUILD_ID_FILE}."
 	report_dir="${REPORTS_DIR}/${bulk_build_id}"
 	${MKDIR} "${report_dir}"
 	( cd "${pkgsrc_dir}" \
   	  && ${PERL5} mk/bulk/post-build \
      	  > "${report_dir}/${REPORT_TXT_FILE}"
 	) || die "Could not write the results file."
+}
 # Notify the ADMIN of the finished build.
 do_email () {
 	case $noemail in
 	no)	cat "${report_dir}/${REPORT_TXT_FILE}" \
 		| ${MAIL_CMD} -s "pkgsrc ${OPSYS} ${OS_VERSION}/${MACHINE_ARCH} bulk build results ${bulk_build_id}" "$ADMIN"
 	esac
+}
 # output final note that we're done
 do_done () {
 	echo ""
 	echo "build> Bulk build ended: `date`"
+}
 # set all commonly used variables, prepare files etc.
 do_common_setup () {
+	#
 	# Choose an appropriate value for BMAKE depending on the operating
 	# system.
+	#
 	opsys=`uname -s`
 	case "$opsys" in
 		NetBSD)	BMAKE=make ;;
 		*)	BMAKE=bmake ;;
 	esac
 	export BMAKE
+	#
 	# Set resource limits as high as possible
+	#
 	ulimit -S -s `ulimit -H -s` # XXX: why do we need unlimited stack?
 	ulimit -S -d `ulimit -H -d` # some processes grow rather large
 	ulimit -S -t `ulimit -H -t` # pkgsrc bulk builds need _much_ time
+	#
 	# Find the configuration file.
+	#
 	BULK_BUILD_CONF="${BULK_BUILD_CONF-${scriptdir}/build.conf}"
 	case $BULK_BUILD_CONF in
 		/*) ;;
 		 *)	BULK_BUILD_CONF="${PWD}/${BULK_BUILD_CONF}"
 	 esac
+	#
 	# Load the variables from the configuration file.
+	#
 	{ test -f "${BULK_BUILD_CONF}"		\
 	  && . "${BULK_BUILD_CONF}"		\
 	  && . "${scriptdir}/post-build-conf"	\
 	  && check_config_vars 			\
 	  && export_config_vars
 	} || die "Cannot load config file ${BULK_BUILD_CONF}, aborting."
+	#
 	# Set the paths to commonly used directories.
+	#
 	pkgsrc_dir="${USR_PKGSRC}"
 	lintpkgsrc_dir="${USR_PKGSRC}/pkgtools/lintpkgsrc"
+	#
 	# Set up variables specific for the bulk build.
+	#
 	BATCH="1"
 	DEPENDS_TARGET="bulk-install"
 	export BATCH DEPENDS_TARGET
+	#
 	# Unset some environment variables that could disturbe the build.
+	#
 	unset CDPATH || true	# ensure cd does not print new cwd to stdout, which
 				# confuses the printindex script.
 	unset DISPLAY || true	# allow sane failure for gimp, xlispstat
+}
 # Check that the package tools are up to date.
 check_tools () {
 	echo "build> Checking if the pkgtools are up-to-date"
 	( cd "${lintpkgsrc_dir}"				\
   	  && ${BMAKE} fetch >/dev/null 2>&1
 	) || {
 	  echo "build> Updating pkgtools"
 	  ( cd "${pkgsrc_dir}/pkgtools/pkg_install"	\
 	    && ${BMAKE} clean 				\
 	    && ${BMAKE} update
 	  ) || die "Could not update the package tools."
+	}
+}
 # Run the pre-build script if necessary.
 run_pre_build () {
 	case $resume in
 		yes)	echo "build> Resuming -- skipping pre-build script";;
 		*)	# make veryveryclean :)
 			( cd "${pkgsrc_dir}"		\
 	  		  && /bin/sh mk/bulk/pre-build
 			) || die "Error during bulk-build preparations, aborting.";;
 	esac
+}
 # Load pkgsrc variables that affect the build process.
 load_vars () {
 	echo "+----------------------------------------+"
 	echo "| Some variables used in the bulk build: |"
 	echo "+----------------------------------------+"
 	vars="	OPSYS OS_VERSION MACHINE_ARCH
 		BULK_PREREQ
 		BULKFILESDIR
 		BULK_DBFILE DEPENDSFILE INDEXFILE ORDERFILE STARTFILE
 		SUPPORTSFILE BULK_BUILD_ID_FILE BUILDLOG BROKENFILE
 		BROKENWRKLOG
 		PREFIX
 		AWK GREP MAIL_CMD MKDIR PAX PERL5 SED
 		PKG_DELETE PKG_INFO PKGBASE"
 	values=`cd "$lintpkgsrc_dir" && $BMAKE show-vars VARNAMES="$vars" USE_TOOLS="awk grep mail mkdir pax perl sed"`
 	for v in $vars; do
 		eval "read $v" || die "Could not read value for $v"
 		eval "value=\$$v"
 		if [ "$v" != "BULK_PREREQ" ] && [ ! "$value" ]; then
 			die "$v must not be empty."
 		fi
 		printf "%-15s = %s\\n" "$v" "$value"
 	done <<EOF
 $values
 EOF
 	echo "------------------------------------------"
 	[ "$PKGBASE" = "lintpkgsrc" ] \
 	|| die	"Error reading the variables." \
 		"Try running $0 with another shell."
 	# Get the location of commonly used files
 	main_buildlog="${BULKFILESDIR}/${BUILDLOG}"
+}
 # Install prerequisite packages.
+#
 # Note: we do this _before_ the depends tree because some packages like
 # xpkgwedge only become DEPENDS if they are installed.
 install_prereqs () {
 	echo "build> Installing prerequisite packages specified with BULK_PREREQ..."
 	for pkgdir in $BULK_PREREQ; do
 		echo "build> Installing prerequisite package $pkgdir"
 		( cd "${pkgsrc_dir}/${pkgdir}"		\
 	  	  && ${BMAKE} bulk-install
 		) || die "Could not install prerequisite packages."
 	done
+}
 # Everything is prepared. We can start building the real packages now.
+#
 # Loop over every package in the correct order.  Before building
 # each one, check to see if we've already processed this package
 # before.  This could happen if the build got interrupted and we
 # started it again with the '-resume' option.  This prevents us
 # from having to do a potentially very large number of make's to
 # get back to where we let off.  After we build each package, add
 # it to the top level buildlog
 # (usually '.make' or '.make.${MACHINE}').  As a side benefit, this
 # can make a progress-meter very simple to add!
 do_real_bulk_build () {
 	cd "${pkgsrc_dir}" || die "The pkgsrc directory does not exist."
 	echo "build> Starting actual build using the order specified in $ORDERFILE..."
 	# make sure we have something to grep in in the build loop
 	touch "${main_buildlog}" || die "Cannot write to ${main_buildlog}."
 	tot=`${AWK} 'END { print NR }' "${ORDERFILE}"`
 	for pkgdir in `cat "${ORDERFILE}"`
 	do
 		if ${GREP} -q "^${pkgdir}\$" "${main_buildlog}"; then
 			: "skip this package"
 		else
 			percent=`${AWK} -v tot="${tot}" '
 				END {
 					printf("%d/%d=%4.1f%%", NR, tot, NR*100/tot);
 				}' "${main_buildlog}"`
 			( cd "${pkgsrc_dir}/${pkgdir}" \
 			  && ${NICE_LEVEL} ${BMAKE} USE_BULK_CACHE=yes "${target}" \
 				$makeargs </dev/null | post_filter_cmd
 			) || true
 			echo "$pkgdir" >> "${main_buildlog}"
 			check_pkg_dirs
 		fi
 	done
 	echo "build> Build finished."
+}
 check_pkg_dirs () {
 	for d in bin etc include info lib libexec man sbin share; do
 		if [ -f $PREFIX/$d ]; then
 			echo "Removing file, should be dir: $PREFIX/$d" >&2
 			rm -f $PREFIX/$d
 		fi
 	done
+}
 # clean up installed packages left over
 do_bulk_cleanup () {
 	echo "build> Removing all installed packages left over from build..."
 	for pkgname in `${PKG_INFO} -e \*`
 	do
 		if ${PKG_INFO} -qe "${pkgname}"; then
 			pkgdir=`${AWK} '$2 == "'"$pkgname"'" { print $1; }' "$INDEXFILE"`
 			case "${BULK_PREREQ}" in
 				*"${pkgdir}"* )
 					echo "build> Keeping BULK_PREREQ: $pkgname ($pkgdir)" ;
 					;;
 				* )
 					echo "build> ${PKG_DELETE} -r ${pkgname}"
 					${PKG_DELETE} -r "${pkgname}"
 					if ${PKG_INFO} -qe "${pkgname}"; then
 						echo "build> $pkgname ($pkgdir) did not deinstall nicely.  Forcing the deinstall"
 						${PKG_DELETE} -f "${pkgname}" || true
 					fi
 					;;
 			esac
 		fi
 	done
+}
 # start the full bulk-build
 do_bulk_build () {
 	echo "build> Bulk build started: `date`"
 	echo ""
 	# this function from post-build-conf
 	show_config_vars
 	check_tools
 	run_pre_build
 	load_vars
+	#
 	# Create the directory for the log files if necessary
+	#
 	if [ "${BULKFILESDIR}" != "${pkgsrc_dir}" ]; then
 		${MKDIR} "${BULKFILESDIR}"
 	fi
+	#
 	# Save the bulk build ID in a file, as it most often contains a time
 	# stamp.
+	#
 	case $resume in
 	no)	echo "${REPORT_BASEDIR}" > "${BULK_BUILD_ID_FILE}" \
 		|| die "Could not save the bulk build ID in ${BULK_BUILD_ID_FILE}.";;
 	esac
 	install_prereqs
+	#
 	# Create the bulk cache files.
+	#
 	if [ "x$resume" != "xyes" ]; then
 		( cd "${pkgsrc_dir}"				\
 	  	  && env PKGLIST="${PKGLIST-}" ${BMAKE} bulk-cache $makeargs
 		) || die "Could not create the bulk build cache."
 	else
 		if [ ! -f "${ORDERFILE}" ]; then
 			die	"The ${ORDERFILE} does not exist." \
 				"(You cannot resume a bulk build that has not yet started.)"
 		fi
 	fi
 	# XXX: This looks like a hack, and indeed, the functions in this file
 	# should be reorganized to better reflect the phases of the bulk build.
 	if [ $prepare_only = yes ]; then
 		exit 0
 	fi
 	do_real_bulk_build
 	do_bulk_cleanup
+}
 ##
 ## main
 ##
+#
 # Parse the command line.
+#
 while test $# -gt 0; do
 	case $1 in
 	-c|--config)
 		shift
 		BULK_BUILD_CONF=$1; shift
 		;;
 	-e|--no-email)
 		noemail=yes
 		shift
 		;;
 	-h|--help)
 		usage
 		exit 0
 		;;
 	-m|--mirror_only)
 		mirror_only=yes
 		target=mirror-distfiles
 		shift
 		;;
 	-p|--post-build)
 		post_only=yes
 		shift
 		;;
 	--prepare)
 		prepare_only=yes
 		shift
 		;;
 	-r|--resume|--restart|restart)
 		resume=yes
 		shift
 		;;
 	-s|--specific-pkgs)
 		makeargs="$makeargs SPECIFIC_PKGS=1"
 		shift
 		;;
 	*)
 		echo "unknown option:  $1" 1>&2
 		usage 1>&2
 		exit 1
 		;;
 	esac
 done
 do_common_setup
 if [ "x$post_only" = "xyes" ]; then
 	load_vars
 	do_post_build
 	exit 0
 fi
 do_bulk_build
 # for now, just quit if we were only mirroring distfiles.  At somepoint we
 # should teach the post-build script to generate a nice report about how many
 # distfiles were downloaded, how many had bad checksums, failed master sites,
 # network speed, etc.
 if [ "x$mirror_only" = "xyes" ]; then
 	echo "build> Bulk mirror of distfiles completed: `date`"
 	exit 0
 fi
 do_post_build
 do_email
 do_done