Sun Nov 12 13:34:14 2017 UTC ()
Add initial support for building packages reproducibly

It currently tackles two problems:
- gcc(1) hard-coding full paths in debugging information (with one
  caveat at the moment)
- ar(1) hard-coding user IDs in archive headers

This allows packages built from the same tree and options to produce
identical results bit by bit. This option should be combined with ASLR
and PKGSRC_MKPIE to avoid predictable address offsets for attackers
attempting to exploit security vulnerabilities.

This is still disabled by default, and only supports NetBSD so far.

As discussed on tech-pkg@


(khorben)
diff -r1.2027 -r1.2028 pkgsrc/mk/bsd.pkg.mk
diff -r1.394 -r1.395 pkgsrc/mk/bsd.prefs.mk
diff -r1.186 -r1.187 pkgsrc/mk/compiler/gcc.mk
diff -r1.285 -r1.286 pkgsrc/mk/defaults/mk.conf
diff -r1.56 -r1.57 pkgsrc/mk/platform/NetBSD.mk
diff -r0 -r1.1 pkgsrc/mk/repro/ar
diff -r0 -r1.1 pkgsrc/mk/repro/repro.mk

cvs diff -r1.2027 -r1.2028 pkgsrc/mk/bsd.pkg.mk (expand / switch to unified diff)

--- pkgsrc/mk/bsd.pkg.mk 2017/09/16 09:34:25 1.2027
+++ pkgsrc/mk/bsd.pkg.mk 2017/11/12 13:34:14 1.2028
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1# $NetBSD: bsd.pkg.mk,v 1.2027 2017/09/16 09:34:25 wiz Exp $ 1# $NetBSD: bsd.pkg.mk,v 1.2028 2017/11/12 13:34:14 khorben Exp $
2# 2#
3# This file is in the public domain. 3# This file is in the public domain.
4# 4#
5# Please see the pkgsrc/doc/guide manual for details on the 5# Please see the pkgsrc/doc/guide manual for details on the
6# variables used in this make file template. 6# variables used in this make file template.
7# 7#
8# Default sequence for "all" is: 8# Default sequence for "all" is:
9# 9#
10# bootstrap-depends 10# bootstrap-depends
11# fetch 11# fetch
12# checksum 12# checksum
13# depends 13# depends
14# tools 14# tools
@@ -305,26 +305,30 @@ OVERRIDE_DIRDEPTH?= 2 @@ -305,26 +305,30 @@ OVERRIDE_DIRDEPTH?= 2
305 305
306# Handle alternatives 306# Handle alternatives
307# 307#
308.include "alternatives.mk" 308.include "alternatives.mk"
309 309
310# Handle alternative init systems 310# Handle alternative init systems
311# 311#
312.if ${_USE_NEW_PKGINSTALL:Uno} == "no" 312.if ${_USE_NEW_PKGINSTALL:Uno} == "no"
313.if ${INIT_SYSTEM} == "smf" 313.if ${INIT_SYSTEM} == "smf"
314. include "smf.mk" 314. include "smf.mk"
315.endif 315.endif
316.endif 316.endif
317 317
 318# Handle Reproducible Builds
 319#
 320.include "repro/repro.mk"
 321
318# Define SMART_MESSAGES in /etc/mk.conf for messages giving the tree 322# Define SMART_MESSAGES in /etc/mk.conf for messages giving the tree
319# of dependencies for building, and the current target. 323# of dependencies for building, and the current target.
320_PKGSRC_IN?= ===${SMART_MESSAGES:D> ${.TARGET} [${PKGNAME}${_PKGSRC_DEPS}] ===} 324_PKGSRC_IN?= ===${SMART_MESSAGES:D> ${.TARGET} [${PKGNAME}${_PKGSRC_DEPS}] ===}
321 325
322# Used to print all the '===>' style prompts - override this to turn them off. 326# Used to print all the '===>' style prompts - override this to turn them off.
323ECHO_MSG?= ${ECHO} 327ECHO_MSG?= ${ECHO}
324PHASE_MSG?= ${ECHO_MSG} ${_PKGSRC_IN:Q}\> 328PHASE_MSG?= ${ECHO_MSG} ${_PKGSRC_IN:Q}\>
325STEP_MSG?= ${ECHO_MSG} "=>" 329STEP_MSG?= ${ECHO_MSG} "=>"
326INFO_MSG?= ${ECHO_MSG} "INFO:" 330INFO_MSG?= ${ECHO_MSG} "INFO:"
327WARNING_MSG?= ${ECHO_MSG} 1>&2 "WARNING:" 331WARNING_MSG?= ${ECHO_MSG} 1>&2 "WARNING:"
328ERROR_MSG?= ${ECHO_MSG} 1>&2 "ERROR:" 332ERROR_MSG?= ${ECHO_MSG} 1>&2 "ERROR:"
329FAIL_MSG?= ${FAIL} ${ERROR_MSG} 333FAIL_MSG?= ${FAIL} ${ERROR_MSG}
330 334

cvs diff -r1.394 -r1.395 pkgsrc/mk/bsd.prefs.mk (expand / switch to unified diff)

--- pkgsrc/mk/bsd.prefs.mk 2017/11/03 18:07:40 1.394
+++ pkgsrc/mk/bsd.prefs.mk 2017/11/12 13:34:14 1.395
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1# $NetBSD: bsd.prefs.mk,v 1.394 2017/11/03 18:07:40 bsiegert Exp $ 1# $NetBSD: bsd.prefs.mk,v 1.395 2017/11/12 13:34:14 khorben Exp $
2# 2#
3# This file includes the mk.conf file, which contains the user settings. 3# This file includes the mk.conf file, which contains the user settings.
4# 4#
5# Packages should include this file before any of the .if directives, as 5# Packages should include this file before any of the .if directives, as
6# well as before modifying variables like CFLAGS, LDFLAGS, and so on. 6# well as before modifying variables like CFLAGS, LDFLAGS, and so on.
7# Otherwise the behavior may be unexpected. 7# Otherwise the behavior may be unexpected.
8# 8#
9# When mk.conf is included by this file, the following variables are 9# When mk.conf is included by this file, the following variables are
10# defined: 10# defined:
11# 11#
12# ACCEPTABLE_LICENSES 12# ACCEPTABLE_LICENSES
13# This variable is set to the list of Open Source licenses. See 13# This variable is set to the list of Open Source licenses. See
14# mk/license.mk for details. 14# mk/license.mk for details.
@@ -695,26 +695,32 @@ PREPEND_PATH+= ${LOCALBASE}/bin @@ -695,26 +695,32 @@ PREPEND_PATH+= ${LOCALBASE}/bin
695.if ${_USE_NEW_PKGINSTALL:Uno} == "no" 695.if ${_USE_NEW_PKGINSTALL:Uno} == "no"
696# Support alternative init systems. 696# Support alternative init systems.
697# 697#
698INIT_SYSTEM?= rc.d 698INIT_SYSTEM?= rc.d
699_BUILD_DEFS+= INIT_SYSTEM 699_BUILD_DEFS+= INIT_SYSTEM
700.endif 700.endif
701 701
702_PKGSRC_MKPIE= no 702_PKGSRC_MKPIE= no
703.if (${PKGSRC_MKPIE:tl} == "yes") && \ 703.if (${PKGSRC_MKPIE:tl} == "yes") && \
704 (${_OPSYS_SUPPORTS_MKPIE:Uno} == "yes") 704 (${_OPSYS_SUPPORTS_MKPIE:Uno} == "yes")
705_PKGSRC_MKPIE= yes 705_PKGSRC_MKPIE= yes
706.endif 706.endif
707 707
 708_PKGSRC_MKREPRO= no
 709.if (${PKGSRC_MKREPRO:tl} == "yes") && \
 710 (${_OPSYS_SUPPORTS_MKREPRO:Uno} == "yes")
 711_PKGSRC_MKREPRO= yes
 712.endif
 713
708_PKGSRC_USE_FORTIFY= no 714_PKGSRC_USE_FORTIFY= no
709.if (${PKGSRC_USE_FORTIFY:tl} != "no") && \ 715.if (${PKGSRC_USE_FORTIFY:tl} != "no") && \
710 (${_OPSYS_SUPPORTS_FORTIFY:Uno} == "yes") 716 (${_OPSYS_SUPPORTS_FORTIFY:Uno} == "yes")
711_PKGSRC_USE_FORTIFY= yes 717_PKGSRC_USE_FORTIFY= yes
712.endif 718.endif
713 719
714_PKGSRC_USE_RELRO= no 720_PKGSRC_USE_RELRO= no
715.if (${PKGSRC_USE_RELRO:tl} != "no") && \ 721.if (${PKGSRC_USE_RELRO:tl} != "no") && \
716 (${_OPSYS_SUPPORTS_RELRO:Uno} == "yes") 722 (${_OPSYS_SUPPORTS_RELRO:Uno} == "yes")
717_PKGSRC_USE_RELRO= yes 723_PKGSRC_USE_RELRO= yes
718.endif 724.endif
719 725
720_PKGSRC_USE_SSP= no 726_PKGSRC_USE_SSP= no

cvs diff -r1.186 -r1.187 pkgsrc/mk/compiler/gcc.mk (expand / switch to unified diff)

--- pkgsrc/mk/compiler/gcc.mk 2017/11/07 16:57:58 1.186
+++ pkgsrc/mk/compiler/gcc.mk 2017/11/12 13:34:14 1.187
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1# $NetBSD: gcc.mk,v 1.186 2017/11/07 16:57:58 khorben Exp $ 1# $NetBSD: gcc.mk,v 1.187 2017/11/12 13:34:14 khorben Exp $
2# 2#
3# This is the compiler definition for the GNU Compiler Collection. 3# This is the compiler definition for the GNU Compiler Collection.
4# 4#
5# User-settable variables: 5# User-settable variables:
6# 6#
7# GCCBASE 7# GCCBASE
8# If using a native GCC and the compiler is not in $PATH then 8# If using a native GCC and the compiler is not in $PATH then
9# this should be set to the base installation directory. 9# this should be set to the base installation directory.
10# 10#
11# USE_NATIVE_GCC 11# USE_NATIVE_GCC
12# When set to "yes", the native gcc is used, no matter which 12# When set to "yes", the native gcc is used, no matter which
13# compiler version a package requires. 13# compiler version a package requires.
14# 14#
@@ -337,26 +337,37 @@ _MKPIE_CFLAGS.gcc= -fPIC @@ -337,26 +337,37 @@ _MKPIE_CFLAGS.gcc= -fPIC
337#_MKPIE_CFLAGS.gcc= -fPIE 337#_MKPIE_CFLAGS.gcc= -fPIE
338# XXX for libraries a sink wrapper around gcc is required and used instead 338# XXX for libraries a sink wrapper around gcc is required and used instead
339_MKPIE_LDFLAGS.gcc= -fPIC -pie 339_MKPIE_LDFLAGS.gcc= -fPIC -pie
340.endif 340.endif
341 341
342.if ${_PKGSRC_MKPIE} == "yes" 342.if ${_PKGSRC_MKPIE} == "yes"
343_GCC_CFLAGS+= ${_MKPIE_CFLAGS.gcc} 343_GCC_CFLAGS+= ${_MKPIE_CFLAGS.gcc}
344#_GCC_LDFLAGS+= ${_MKPIE_LDFLAGS.gcc} 344#_GCC_LDFLAGS+= ${_MKPIE_LDFLAGS.gcc}
345CWRAPPERS_APPEND.cc+= ${_MKPIE_CFLAGS.gcc} 345CWRAPPERS_APPEND.cc+= ${_MKPIE_CFLAGS.gcc}
346# this differs for libraries and executables (handled in mk/cwrappers.mk) 346# this differs for libraries and executables (handled in mk/cwrappers.mk)
347# CWRAPPERS_APPEND.ld+= ${_MKPIE_LDFLAGS.gcc} 347# CWRAPPERS_APPEND.ld+= ${_MKPIE_LDFLAGS.gcc}
348.endif 348.endif
349 349
 350.if ${_PKGSRC_MKREPRO} == "yes"
 351.export WRKDIR
 352# XXX the dollar sign should not be expanded by the shell
 353_GCC_CFLAGS+= -fdebug-prefix-map=$$$$WRKDIR/=
 354.endif
 355
 356.if ${_PKGSRC_MKREPRO} == "yes"
 357_GCC_CFLAGS+= ${_MKREPRO_CFLAGS.gcc}
 358CWRAPPERS_APPEND.cc+= ${_MKREPRO_CFLAGS.gcc}
 359.endif
 360
350# The user can choose the level of FORTIFY. 361# The user can choose the level of FORTIFY.
351.if ${PKGSRC_USE_FORTIFY} == "weak" 362.if ${PKGSRC_USE_FORTIFY} == "weak"
352_FORTIFY_CFLAGS= -D_FORTIFY_SOURCE=1 363_FORTIFY_CFLAGS= -D_FORTIFY_SOURCE=1
353.else 364.else
354_FORTIFY_CFLAGS= -D_FORTIFY_SOURCE=2 365_FORTIFY_CFLAGS= -D_FORTIFY_SOURCE=2
355.endif 366.endif
356 367
357.if ${_PKGSRC_USE_FORTIFY} == "yes" 368.if ${_PKGSRC_USE_FORTIFY} == "yes"
358_GCC_CFLAGS+= ${_FORTIFY_CFLAGS} 369_GCC_CFLAGS+= ${_FORTIFY_CFLAGS}
359CWRAPPERS_APPEND.cc+= ${_FORTIFY_CFLAGS} 370CWRAPPERS_APPEND.cc+= ${_FORTIFY_CFLAGS}
360.endif 371.endif
361 372
362# The user can choose the level of RELRO. 373# The user can choose the level of RELRO.

cvs diff -r1.285 -r1.286 pkgsrc/mk/defaults/mk.conf (expand / switch to unified diff)

--- pkgsrc/mk/defaults/mk.conf 2017/10/28 15:56:48 1.285
+++ pkgsrc/mk/defaults/mk.conf 2017/11/12 13:34:14 1.286
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1# $NetBSD: mk.conf,v 1.285 2017/10/28 15:56:48 schmonz Exp $ 1# $NetBSD: mk.conf,v 1.286 2017/11/12 13:34:14 khorben Exp $
2# 2#
3 3
4# This file provides default values for variables that may be overridden 4# This file provides default values for variables that may be overridden
5# in the MAKECONF file, which is /etc/mk.conf by default. 5# in the MAKECONF file, which is /etc/mk.conf by default.
6# 6#
7# Note: This file is included after the MAKECONF file, so you cannot query 7# Note: This file is included after the MAKECONF file, so you cannot query
8# these default values in the MAKECONF using the ".if" and ".for" 8# these default values in the MAKECONF using the ".if" and ".for"
9# preprocessing directives. 9# preprocessing directives.
10 10
11# ************************************************************************ 11# ************************************************************************
12# NOTE TO PEOPLE EDITING THIS FILE - USE LEADING SPACES, NOT LEADING TABS. 12# NOTE TO PEOPLE EDITING THIS FILE - USE LEADING SPACES, NOT LEADING TABS.
13# ************************************************************************ 13# ************************************************************************
14 14
@@ -221,26 +221,37 @@ PKGSRC_RUN_TEST?= no @@ -221,26 +221,37 @@ PKGSRC_RUN_TEST?= no
221# their tests, so need to make sure that there are always enough random 221# their tests, so need to make sure that there are always enough random
222# numbers on your machine. The package security/bitstir may help here. 222# numbers on your machine. The package security/bitstir may help here.
223# 223#
224# Possible: yes, no 224# Possible: yes, no
225# Default: no 225# Default: no
226 226
227PKGSRC_MKPIE?= no 227PKGSRC_MKPIE?= no
228# If no, create regular executables. Otherwise create PIE (Position Independent 228# If no, create regular executables. Otherwise create PIE (Position Independent
229# Executables, on supported platforms). This option is necessary to fully 229# Executables, on supported platforms). This option is necessary to fully
230# leverage ASLR as a mitigation for security vulnerabilities. 230# leverage ASLR as a mitigation for security vulnerabilities.
231# Possible: yes, no 231# Possible: yes, no
232# Default: no 232# Default: no
233 233
 234PKGSRC_MKREPRO?= no
 235# If no, do not alter the build process. Otherwise, try to build reproducibly.
 236# This allows packages built from the same tree and options to produce identical
 237# results bit by bit.
 238# This option should be combined with ASLR and PKGSRC_MKPIE to avoid predictable
 239# address offsets for attackers attempting to exploit security vulnerabilities.
 240# Possible: yes, no
 241# Default: no
 242#
 243# Keywords: reproducible
 244
234PKGSRC_USE_FORTIFY?= strong 245PKGSRC_USE_FORTIFY?= strong
235# Turns on substitute wrappers for commonly used functions that do not bounds 246# Turns on substitute wrappers for commonly used functions that do not bounds
236# checking regularly, but could in some cases. This is effectively in use only 247# checking regularly, but could in some cases. This is effectively in use only
237# when both enabled and supported. 248# when both enabled and supported.
238# Possible values: 249# Possible values:
239# no: Do not pass any flags for FORTIFY 250# no: Do not pass any flags for FORTIFY
240# weak: Pass -D_FORTIFY_SOURCE=1 251# weak: Pass -D_FORTIFY_SOURCE=1
241# strong: Pass -D_FORTIFY_SOURCE=2 252# strong: Pass -D_FORTIFY_SOURCE=2
242# Default: strong 253# Default: strong
243# 254#
244# Keywords: fortify FORTIFY_SOURCE 255# Keywords: fortify FORTIFY_SOURCE
245 256
246PKGSRC_USE_RELRO?= no 257PKGSRC_USE_RELRO?= no

cvs diff -r1.56 -r1.57 pkgsrc/mk/platform/NetBSD.mk (expand / switch to unified diff)

--- pkgsrc/mk/platform/NetBSD.mk 2017/10/03 13:18:00 1.56
+++ pkgsrc/mk/platform/NetBSD.mk 2017/11/12 13:34:14 1.57
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1# $NetBSD: NetBSD.mk,v 1.56 2017/10/03 13:18:00 jperkin Exp $ 1# $NetBSD: NetBSD.mk,v 1.57 2017/11/12 13:34:14 khorben Exp $
2# 2#
3# Variable definitions for the NetBSD operating system. 3# Variable definitions for the NetBSD operating system.
4 4
5# Needed for 1.6.1 and earlier due to rpcgen bugs and paths 5# Needed for 1.6.1 and earlier due to rpcgen bugs and paths
6.if defined(CPP) && ${CPP} == "cpp" 6.if defined(CPP) && ${CPP} == "cpp"
7CPP= /usr/bin/cpp 7CPP= /usr/bin/cpp
8.endif 8.endif
9ECHO_N?= ${ECHO} -n 9ECHO_N?= ${ECHO} -n
10IMAKE_MAKE?= ${MAKE} # program which gets invoked by imake 10IMAKE_MAKE?= ${MAKE} # program which gets invoked by imake
11PKGLOCALEDIR?= share 11PKGLOCALEDIR?= share
12PS?= /bin/ps 12PS?= /bin/ps
13SU?= /usr/bin/su 13SU?= /usr/bin/su
14TYPE?= type # Shell builtin 14TYPE?= type # Shell builtin
@@ -132,26 +132,29 @@ _OPSYS_SUPPORTS_FORTIFY=yes @@ -132,26 +132,29 @@ _OPSYS_SUPPORTS_FORTIFY=yes
132 132
133# Register support for PIE on supported architectures (with GCC) 133# Register support for PIE on supported architectures (with GCC)
134.if (${MACHINE_ARCH} == "i386") || \ 134.if (${MACHINE_ARCH} == "i386") || \
135 (${MACHINE_ARCH} == "x86_64") 135 (${MACHINE_ARCH} == "x86_64")
136_OPSYS_SUPPORTS_MKPIE= yes 136_OPSYS_SUPPORTS_MKPIE= yes
137.endif 137.endif
138 138
139# Register support for RELRO on supported architectures (with GCC) 139# Register support for RELRO on supported architectures (with GCC)
140.if (${MACHINE_ARCH} == "i386") || \ 140.if (${MACHINE_ARCH} == "i386") || \
141 (${MACHINE_ARCH} == "x86_64") 141 (${MACHINE_ARCH} == "x86_64")
142_OPSYS_SUPPORTS_RELRO= yes 142_OPSYS_SUPPORTS_RELRO= yes
143.endif 143.endif
144 144
 145# Register support for REPRO (with GCC)
 146_OPSYS_SUPPORTS_MKREPRO= yes
 147
145# Register support for SSP on most architectures (with GCC) 148# Register support for SSP on most architectures (with GCC)
146.if (${MACHINE_ARCH} != "alpha") && \ 149.if (${MACHINE_ARCH} != "alpha") && \
147 (${MACHINE_ARCH} != "hppa") && \ 150 (${MACHINE_ARCH} != "hppa") && \
148 (${MACHINE_ARCH} != "ia64") && \ 151 (${MACHINE_ARCH} != "ia64") && \
149 (${MACHINE_ARCH} != "mips") 152 (${MACHINE_ARCH} != "mips")
150_OPSYS_SUPPORTS_SSP= yes 153_OPSYS_SUPPORTS_SSP= yes
151.endif 154.endif
152 155
153# Register support for stack check on supported architectures (with GCC) 156# Register support for stack check on supported architectures (with GCC)
154.if (${MACHINE_ARCH} == "i386") || \ 157.if (${MACHINE_ARCH} == "i386") || \
155 (${MACHINE_ARCH} == "x86_64") 158 (${MACHINE_ARCH} == "x86_64")
156_OPSYS_SUPPORTS_STACK_CHECK= yes 159_OPSYS_SUPPORTS_STACK_CHECK= yes
157.endif 160.endif

File Added: pkgsrc/mk/repro/ar
#!/bin/sh

if [ $# -ge 2 ]; then
	args="$1"
	mod="$2"
	shift 2

	exec /usr/bin/ar "$mod$args" "$@"
else
	exec /usr/bin/ar "$@"
fi

File Added: pkgsrc/mk/repro/repro.mk
# $NetBSD: repro.mk,v 1.1 2017/11/12 13:34:14 khorben Exp $
#
# Infrastructure support for PKGSRC_MKREPRO.
#
# Keywords: reproducible
#

.if ${_PKGSRC_MKREPRO} == "yes"

# force ar(1) to be deterministic
TOOLS_CREATE+=	ar
TOOLS_PATH.ar?=	${PKGSRCDIR}/mk/repro/ar
TOOLS_ARGS.ar?=	D

.endif