| @@ -1,15 +1,15 @@ | | | @@ -1,15 +1,15 @@ |
1 | #!/bin/sh | | 1 | #!/bin/sh |
2 | # $NetBSD: upload,v 1.47 2010/03/21 15:31:42 wiz Exp $ | | 2 | # $NetBSD: upload,v 1.48 2010/04/10 21:44:44 wiz Exp $ |
3 | | | 3 | |
4 | # | | 4 | # |
5 | # Upload non-restricted binary pkgs to ftp server | | 5 | # Upload non-restricted binary pkgs to ftp server |
6 | # | | 6 | # |
7 | | | 7 | |
8 | AWK=${AWK:-/usr/bin/awk} | | 8 | AWK=${AWK:-/usr/bin/awk} |
9 | PKG_ADMIN="pkg_admin" | | 9 | PKG_ADMIN="pkg_admin" |
10 | PKG_INFO="pkg_info" | | 10 | PKG_INFO="pkg_info" |
11 | | | 11 | |
12 | set -eu | | 12 | set -eu |
13 | | | 13 | |
14 | # | | 14 | # |
15 | # Find out where we are | | 15 | # Find out where we are |
| @@ -208,85 +208,68 @@ fi | | | @@ -208,85 +208,68 @@ fi |
208 | # | | 208 | # |
209 | # Some temp files | | 209 | # Some temp files |
210 | # | | 210 | # |
211 | | | 211 | |
212 | umask 022 | | 212 | umask 022 |
213 | TMPDIR="${TMPDIR:-/tmp}" | | 213 | TMPDIR="${TMPDIR:-/tmp}" |
214 | TMP="${TMPDIR}"/pkg_upload.$$ | | 214 | TMP="${TMPDIR}"/pkg_upload.$$ |
215 | (umask 077 && mkdir "${TMP}") \ | | 215 | (umask 077 && mkdir "${TMP}") \ |
216 | || { | | 216 | || { |
217 | echo "upload> ERROR: cannot create temporary directory \"${TMP}\"." 1>&2 | | 217 | echo "upload> ERROR: cannot create temporary directory \"${TMP}\"." 1>&2 |
218 | exit 1 | | 218 | exit 1 |
219 | } | | 219 | } |
220 | | | 220 | |
221 | vulnerable_packages="$TMP/vulnerable_packages" | | | |
222 | restricted_packages="$TMP/restricted_packages" | | 221 | restricted_packages="$TMP/restricted_packages" |
223 | old_packages="$TMP/old_packages" | | 222 | old_packages="$TMP/old_packages" |
224 | good_packages="$TMP/regular_packages" | | 223 | good_packages="$TMP/regular_packages" |
225 | all_good_packages="$TMP/all_regular_packages" | | 224 | all_good_packages="$TMP/all_regular_packages" |
226 | | | 225 | |
227 | upload_general="$TMP"/upload_general | | 226 | upload_general="$TMP"/upload_general |
228 | upload_vulnerable="$TMP"/upload_vulnerable | | | |
229 | | | 227 | |
230 | # May be different than $USR_PKGSRC: | | 228 | # May be different than $USR_PKGSRC: |
231 | echo "upload> Running ${BMAKE} to get the pkgsrc variables" | | 229 | echo "upload> Running ${BMAKE} to get the pkgsrc variables" |
232 | pkgsrcdir=`cd pkgtools/lintpkgsrc ; ${BMAKE} show-var VARNAME=_PKGSRCDIR` | | 230 | pkgsrcdir=`cd pkgtools/lintpkgsrc ; ${BMAKE} show-var VARNAME=_PKGSRCDIR` |
233 | packages=`cd pkgtools/lintpkgsrc ; ${BMAKE} show-var VARNAME=PACKAGES` | | 231 | packages=`cd pkgtools/lintpkgsrc ; ${BMAKE} show-var VARNAME=PACKAGES` |
234 | distdir=`cd pkgtools/lintpkgsrc ; ${BMAKE} show-var VARNAME=DISTDIR` | | 232 | distdir=`cd pkgtools/lintpkgsrc ; ${BMAKE} show-var VARNAME=DISTDIR` |
235 | gzip_cmd=`cd pkgtools/lintpkgsrc; ${BMAKE} show-var VARNAME=GZIP_CMD USE_TOOLS=gzip` | | 233 | gzip_cmd=`cd pkgtools/lintpkgsrc; ${BMAKE} show-var VARNAME=GZIP_CMD USE_TOOLS=gzip` |
236 | pkg_info=`cd pkgtools/lintpkgsrc && ${BMAKE} show-var VARNAME=PKG_INFO` | | 234 | pkg_info=`cd pkgtools/lintpkgsrc && ${BMAKE} show-var VARNAME=PKG_INFO` |
237 | shell=`cd pkgtools/lintpkgsrc && ${BMAKE} show-var VARNAME=TOOLS_PLATFORM.sh` | | 235 | shell=`cd pkgtools/lintpkgsrc && ${BMAKE} show-var VARNAME=TOOLS_PLATFORM.sh` |
238 | | | 236 | |
239 | # Pull in some pkgs needed | | 237 | # Pull in some pkgs needed |
240 | for pkg in ${REQUIRED_PACKAGES}; do | | 238 | for pkg in ${REQUIRED_PACKAGES}; do |
241 | install_required $pkg | | 239 | install_required $pkg |
242 | done | | 240 | done |
243 | | | 241 | |
244 | echo "upload> Making sure vulnerability-list is up-to-date:" | | | |
245 | if [ -z "$UPDATE_VULNERABILITY_LIST" -o "$UPDATE_VULNERABILITY_LIST" = "yes" ] | | | |
246 | then | | | |
247 | _PKGVULNDIR=`audit-packages ${AUDIT_PACKAGES_FLAGS} -Q PKGVULNDIR` | | | |
248 | download-vulnerability-list ${DOWNLOAD_VULNERABILITY_LIST_FLAGS} | | | |
249 | if [ "x${_PKGVULNDIR}" != "x${distdir}" ]; then | | | |
250 | cp ${_PKGVULNDIR}/pkg-vulnerabilities ${distdir} | | | |
251 | fi | | | |
252 | echo " done." | | | |
253 | else | | | |
254 | echo " (skipped)" | | | |
255 | fi | | | |
256 | | | | |
257 | case $LINTPKGSRC_CACHE in | | 242 | case $LINTPKGSRC_CACHE in |
258 | yes|YES) | | 243 | yes|YES) |
259 | lintpkgsrc_cache="-I `cd pkgtools/lintpkgsrc ; ${BMAKE} show-var VARNAME=LINTPKGSRC_DB`" | | 244 | lintpkgsrc_cache="-I `cd pkgtools/lintpkgsrc ; ${BMAKE} show-var VARNAME=LINTPKGSRC_DB`" |
260 | ;; | | 245 | ;; |
261 | *) | | 246 | *) |
262 | lintpkgsrc_cache='' | | 247 | lintpkgsrc_cache='' |
263 | ;; | | 248 | ;; |
264 | esac | | 249 | esac |
265 | | | 250 | |
266 | echo "upload> Checking for out of date packages:" | | 251 | echo "upload> Checking for out of date packages:" |
267 | # -p = report old versions of packages | | 252 | # -p = report old versions of packages |
268 | lintpkgsrc $lintpkgsrc_cache -K $packages -P $pkgsrcdir -p > "${old_packages}.tmp" | | 253 | lintpkgsrc $lintpkgsrc_cache -K $packages -P $pkgsrcdir -p > "${old_packages}.tmp" |
269 | sed 's@'$packages'/@@' < "${old_packages}.tmp" > "$old_packages" | | 254 | sed 's@'$packages'/@@' < "${old_packages}.tmp" > "$old_packages" |
270 | | | 255 | |
271 | RSFLAGS="-vap --progress $RSYNC_OPTS" | | 256 | RSFLAGS="-vap --progress $RSYNC_OPTS" |
272 | | | 257 | |
273 | failed=no | | 258 | failed=no |
274 | cd $packages | | 259 | cd $packages |
275 | | | 260 | |
276 | echo "upload> Checking for restricted and vulnerable packages" | | 261 | echo "upload> Checking for restricted packages" |
277 | (cd All && env PKG_INFO="${pkg_info}" OUTDIR="${TMP}" PKGVULNDIR="${distdir}" \ | | 262 | (cd All && env PKG_INFO="${pkg_info}" OUTDIR="${TMP}" \ |
278 | AUDIT_PACKAGES_FLAGS="${AUDIT_PACKAGES_FLAGS}" \ | | | |
279 | DOWNLOAD_VULNERABILITY_LIST_FLAGS="${DOWNLOAD_VULNERABILITY_LIST_FLAGS}" \ | | | |
280 | ${shell} "${pkgsrcdir}/mk/bulk/sort-packages") | | 263 | ${shell} "${pkgsrcdir}/mk/bulk/sort-packages") |
281 | | | 264 | |
282 | # Add the name of the package file, including all its symlinks to the | | 265 | # Add the name of the package file, including all its symlinks to the |
283 | # list of files to be uploaded. | | 266 | # list of files to be uploaded. |
284 | while read package; do | | 267 | while read package; do |
285 | ls -1 */"$package" | | 268 | ls -1 */"$package" |
286 | done < "$good_packages" > "$all_good_packages" | | 269 | done < "$good_packages" > "$all_good_packages" |
287 | | | 270 | |
288 | if [ "${MKSUMS}" = "yes" -o "${MKSUMS}" = "YES" ]; then | | 271 | if [ "${MKSUMS}" = "yes" -o "${MKSUMS}" = "YES" ]; then |
289 | | | 272 | |
290 | echo "upload> Calculating checksum files..." | | 273 | echo "upload> Calculating checksum files..." |
291 | | | 274 | |
292 | SUMFILES="BSDSUM CKSUM MD5 SHA1 SYSVSUM" | | 275 | SUMFILES="BSDSUM CKSUM MD5 SHA1 SYSVSUM" |
| @@ -298,27 +281,27 @@ if [ "${MKSUMS}" = "yes" -o "${MKSUMS}" | | | @@ -298,27 +281,27 @@ if [ "${MKSUMS}" = "yes" -o "${MKSUMS}" |
298 | for i in ${SUMFILES}; do | | 281 | for i in ${SUMFILES}; do |
299 | echo > $i | | 282 | echo > $i |
300 | echo "This file is signed with ${SIGN_AS}'s PGP key." >> $i | | 283 | echo "This file is signed with ${SIGN_AS}'s PGP key." >> $i |
301 | echo >> $i | | 284 | echo >> $i |
302 | done | | 285 | done |
303 | fi | | 286 | fi |
304 | | | 287 | |
305 | install_required "pkgtools/digest" | | 288 | install_required "pkgtools/digest" |
306 | | | 289 | |
307 | [ -z "${BSDSUM}" ] && BSDSUM="echo" | | 290 | [ -z "${BSDSUM}" ] && BSDSUM="echo" |
308 | [ -z "${CKSUM}" ] && CKSUM="echo" | | 291 | [ -z "${CKSUM}" ] && CKSUM="echo" |
309 | [ -z "${SYSVSUM}" ] && SYSVSUM="echo" | | 292 | [ -z "${SYSVSUM}" ] && SYSVSUM="echo" |
310 | | | 293 | |
311 | for pkg in `cat "${good_packages}" "${vulnerable_packages}"`; do | | 294 | for pkg in `cat "${good_packages}"`; do |
312 | pkg="All/$pkg" | | 295 | pkg="All/$pkg" |
313 | ${BSDSUM} "$pkg" >> BSDSUM | | 296 | ${BSDSUM} "$pkg" >> BSDSUM |
314 | ${CKSUM} "$pkg" >> CKSUM | | 297 | ${CKSUM} "$pkg" >> CKSUM |
315 | ${MD5} "$pkg" >> MD5 | | 298 | ${MD5} "$pkg" >> MD5 |
316 | ${SHA1} "$pkg" >> SHA1 | | 299 | ${SHA1} "$pkg" >> SHA1 |
317 | ${SYSVSUM} "$pkg" >> SYSVSUM | | 300 | ${SYSVSUM} "$pkg" >> SYSVSUM |
318 | done | | 301 | done |
319 | | | 302 | |
320 | [ "${BSDSUM}" = "echo" ] && rm BSDSUM | | 303 | [ "${BSDSUM}" = "echo" ] && rm BSDSUM |
321 | [ "${CKSUM}" = "echo" ] && rm CKSUM | | 304 | [ "${CKSUM}" = "echo" ] && rm CKSUM |
322 | [ "${SYSVSUM}" = "echo" ] && rm SYSVSUM | | 305 | [ "${SYSVSUM}" = "echo" ] && rm SYSVSUM |
323 | | | 306 | |
324 | if [ "${SIGN_AS-}" != "" ]; then | | 307 | if [ "${SIGN_AS-}" != "" ]; then |
| @@ -340,54 +323,34 @@ if [ "${MKSUMMARY-}" = "yes" -o "${MKSUM | | | @@ -340,54 +323,34 @@ if [ "${MKSUMMARY-}" = "yes" -o "${MKSUM |
340 | && ls -t | grep '\.t[gb]z$' | while read n; do pkg_info -X "$n"; done) \ | | 323 | && ls -t | grep '\.t[gb]z$' | while read n; do pkg_info -X "$n"; done) \ |
341 | | ${gzip_cmd} > "${packages}"/All/pkg_summary.gz | | 324 | | ${gzip_cmd} > "${packages}"/All/pkg_summary.gz |
342 | fi | | 325 | fi |
343 | | | 326 | |
344 | cat <<EOF > "$upload_general" | | 327 | cat <<EOF > "$upload_general" |
345 | #! /bin/sh | | 328 | #! /bin/sh |
346 | set -e | | 329 | set -e |
347 | cd "$packages" | | 330 | cd "$packages" |
348 | rsync $RSFLAGS --files-from="${all_good_packages}" --exclude-from="${old_packages}" . "$RSYNC_DST/" | | 331 | rsync $RSFLAGS --files-from="${all_good_packages}" --exclude-from="${old_packages}" . "$RSYNC_DST/" |
349 | EOF | | 332 | EOF |
350 | chmod +x "$upload_general" | | 333 | chmod +x "$upload_general" |
351 | | | 334 | |
352 | if [ "$do_upload" = "yes" ]; then | | 335 | if [ "$do_upload" = "yes" ]; then |
353 | echo "upload> Uploading non-vulnerable packages" | | 336 | echo "upload> Uploading packages" |
354 | ${shell} "$upload_general" \ | | 337 | ${shell} "$upload_general" \ |
355 | || { | | 338 | || { |
356 | echo "upload> ERROR: rsync failed. To retry later, you can run $upload_general" 1>&2 | | 339 | echo "upload> ERROR: rsync failed. To retry later, you can run $upload_general" 1>&2 |
357 | failed=yes | | 340 | failed=yes |
358 | } | | 341 | } |
359 | else | | 342 | else |
360 | echo "upload> Skipping upload of non-vulnerable packages." | | 343 | echo "upload> Skipping upload of packages." |
361 | echo " Run \"$upload_general\" to upload them later." | | 344 | echo " Run \"$upload_general\" to upload them later." |
362 | fi | | 345 | fi |
363 | | | 346 | |
364 | cat <<EOF > "$upload_vulnerable" | | | |
365 | #! /bin/sh | | | |
366 | set -e | | | |
367 | cd "$packages/All" | | | |
368 | rsync $RSFLAGS --files-from="${vulnerable_packages}" --exclude-from="${old_packages}" . "$RSYNC_DST/All/" | | | |
369 | EOF | | | |
370 | chmod +x "$upload_vulnerable" | | | |
371 | | | | |
372 | if [ "$do_upload" = "yes" ]; then | | | |
373 | echo "upload> Uploading vulnerable packages" | | | |
374 | ${shell} "$upload_vulnerable" \ | | | |
375 | || { | | | |
376 | echo "upload> ERROR: rsync failed. To retry later, you can run $upload_vulnerable" 1>&2 | | | |
377 | failed=yes | | | |
378 | } | | | |
379 | else | | | |
380 | echo "upload> Skipping upload of vulnerable packages." | | | |
381 | echo " Run \"$upload_vulnerable\" to upload them later." | | | |
382 | fi | | | |
383 | | | | |
384 | # clean up temp files | | 347 | # clean up temp files |
385 | if [ "$failed,$debug,$do_upload" = "no,no,yes" ]; then | | 348 | if [ "$failed,$debug,$do_upload" = "no,no,yes" ]; then |
386 | rm -fr "$TMP" | | 349 | rm -fr "$TMP" |
387 | else | | 350 | else |
388 | echo "upload> Preserving temporary directory ${TMP}" | | 351 | echo "upload> Preserving temporary directory ${TMP}" |
389 | fi | | 352 | fi |
390 | | | 353 | |
391 | if [ "$failed" = "yes" ]; then | | 354 | if [ "$failed" = "yes" ]; then |
392 | exit 1 | | 355 | exit 1 |
393 | fi | | 356 | fi |