netpgp: Unbreak with OpenSSL 1.1, mostly by grabbing changes from netbsd src


(nia)

$NetBSD: patch-src_lib_openssl__crypto.c,v 1.1 2020/04/25 12:07:47 nia Exp $

Sync with NetBSD src to fix build with OpenSSL 1.1.

--- src/lib/openssl_crypto.c.orig	2020-04-25 11:54:50.243962468 +0000
+++ src/lib/openssl_crypto.c
@@ -88,18 +88,144 @@ __COPYRIGHT("@(#) Copyright (c) 2009 The
 #include "netpgpdigest.h"
 #include "packet.h"
 
+static void
+takeRSA(const RSA *orsa, pgp_rsa_pubkey_t *pk, pgp_rsa_seckey_t *sk)
+{
+	const BIGNUM *n, *e, *d, *q, *p;
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+	RSA_get0_key(orsa, &n, &e, &d);
+	RSA_get0_factors(orsa, &q, &p);
+#else
+	n = orsa->n;
+	e = orsa->e;
+	d = orsa->d;
+	p = orsa->p;
+	q = orsa->q;
+#endif
+	if (sk) {
+		sk->d = BN_dup(d);
+		sk->p = BN_dup(p);
+		sk->q = BN_dup(q);
+	}
+	if (pk) {
+		pk->n = BN_dup(n);
+		pk->e = BN_dup(e);
+	}
+}
 
-static void 
-test_seckey(const pgp_seckey_t *seckey)
+static RSA *
+makeRSA(const pgp_rsa_pubkey_t *pubkey, const pgp_rsa_seckey_t *seckey)
+{
+	BIGNUM	*n, *e, *d, *p, *q;
+	RSA *orsa;
+
+	orsa = RSA_new();
+	n = BN_dup(pubkey->n);
+	e = BN_dup(pubkey->e);
+
+	if (seckey) {
+		d = BN_dup(seckey->d);
+		p = BN_dup(seckey->p);
+		q = BN_dup(seckey->q);
+	} else {
+		d = p = q = NULL;
+	}
+
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+	RSA_set0_key(orsa, n, e, d);
+	RSA_set0_factors(orsa, p, q);
+#else
+	BN_free(orsa->n);
+	BN_free(orsa->e);
+	orsa->n = n;
+	orsa->e = e;
+	if (d) {
+		BN_free(orsa->d);
+		orsa->d = d;
+	}
+	if (p) {
+		BN_free(orsa->p);
+		orsa->p = p;
+	}
+	if (q) {
+		BN_free(orsa->q);
+		orsa->q = q;
+	}
+#endif
+	return orsa;
+}
+
+static DSA_SIG *
+makeDSA_SIG(const pgp_dsa_sig_t *sig)
+{
+	DSA_SIG        *osig;
+	BIGNUM	       *r, *s;
+
+	osig = DSA_SIG_new();
+	r = BN_dup(sig->r);
+	s = BN_dup(sig->s);
+
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+	DSA_SIG_set0(osig, r, s);
+#else
+	BN_free(osig->r);
+	BN_free(osig->s);
+	osig->r = r;
+	osig->s = s;
+#endif
+
+	return osig;
+}
+
+static DSA *
+makeDSA(const pgp_dsa_pubkey_t *dsa, const pgp_dsa_seckey_t *secdsa)
 {
-	RSA            *test = RSA_new();
+	DSA            *odsa;
+	BIGNUM	       *p, *q, *g, *y, *x;
 
-	test->n = BN_dup(seckey->pubkey.key.rsa.n);
-	test->e = BN_dup(seckey->pubkey.key.rsa.e);
+	odsa = DSA_new();
 
-	test->d = BN_dup(seckey->key.rsa.d);
-	test->p = BN_dup(seckey->key.rsa.p);
-	test->q = BN_dup(seckey->key.rsa.q);
+	p = BN_dup(dsa->p);
+	q = BN_dup(dsa->q);
+	g = BN_dup(dsa->g);
+	y = BN_dup(dsa->y);
+	x = secdsa ? secdsa->x : NULL;
+
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+	DSA_set0_key(odsa, y, x);
+#else
+	BN_free(odsa->p);
+	BN_free(odsa->q);
+	BN_free(odsa->g);
+	BN_free(odsa->pub_key);
+	odsa->p = p;
+	odsa->q = q;
+	odsa->g = g;
+	odsa->pub_key = y;
+	if (x) {
+		BN_free(odsa->priv_key);
+		odsa->priv_key = x;
+	}
+#endif
+	return odsa;
+}
+
+static void
+takeDSA(const DSA *odsa, pgp_dsa_seckey_t *sk)
+{
+	const BIGNUM *x;
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L
+	DSA_get0_key(odsa, NULL, &x);
+#else
+	x = odsa->priv_key;
+#endif
+	sk->x = BN_dup(x);
+}
+
+static void 
+test_seckey(const pgp_seckey_t *seckey)
+{
+	RSA *test = makeRSA(&seckey->pubkey.key.rsa, &seckey->key.rsa);
 
 	if (RSA_check_key(test) != 1) {
 		(void) fprintf(stderr,
@@ -434,25 +560,15 @@ pgp_dsa_verify(const uint8_t *hash, size
 	       const pgp_dsa_pubkey_t *dsa)
 {
 	unsigned	qlen;
-	DSA_SIG        *osig;
-	DSA            *odsa;
+	DSA_SIG        *osig = makeDSA_SIG(sig);
+	DSA	       *odsa = makeDSA(dsa, NULL);
 	int             ret;
 
-	osig = DSA_SIG_new();
-	osig->r = sig->r;
-	osig->s = sig->s;
-
-	odsa = DSA_new();
-	odsa->p = dsa->p;
-	odsa->q = dsa->q;
-	odsa->g = dsa->g;
-	odsa->pub_key = dsa->y;
-
 	if (pgp_get_debug_level(__FILE__)) {
 		hexdump(stderr, "input hash", hash, hash_length);
-		(void) fprintf(stderr, "Q=%d\n", BN_num_bytes(odsa->q));
+		(void) fprintf(stderr, "Q=%d\n", BN_num_bytes(dsa->q));
 	}
-	if ((qlen = (unsigned)BN_num_bytes(odsa->q)) < hash_length) {
+	if ((qlen = (unsigned)BN_num_bytes(dsa->q)) < hash_length) {
 		hash_length = qlen;
 	}
 	ret = DSA_do_verify(hash, (int)hash_length, osig, odsa);
@@ -464,10 +580,7 @@ pgp_dsa_verify(const uint8_t *hash, size
 		return 0;
 	}
 
-	odsa->p = odsa->q = odsa->g = odsa->pub_key = NULL;
 	DSA_free(odsa);
-
-	osig->r = osig->s = NULL;
 	DSA_SIG_free(osig);
 
 	return (unsigned)ret;
@@ -488,19 +601,14 @@ pgp_rsa_public_decrypt(uint8_t *out,
 			size_t length,
 			const pgp_rsa_pubkey_t *pubkey)
 {
-	RSA            *orsa;
-	int             n;
-
-	orsa = RSA_new();
-	orsa->n = pubkey->n;
-	orsa->e = pubkey->e;
+	RSA            *orsa = makeRSA(pubkey, NULL);
+	int             ret;
 
-	n = RSA_public_decrypt((int)length, in, out, orsa, RSA_NO_PADDING);
+	ret = RSA_public_decrypt((int)length, in, out, orsa, RSA_NO_PADDING);
 
-	orsa->n = orsa->e = NULL;
 	RSA_free(orsa);
 
-	return n;
+	return ret;
 }
 
 /**
@@ -520,21 +628,10 @@ pgp_rsa_private_encrypt(uint8_t *out,
 			const pgp_rsa_seckey_t *seckey,
 			const pgp_rsa_pubkey_t *pubkey)
 {
-	RSA            *orsa;
-	int             n;
+	RSA            *orsa = makeRSA(pubkey, seckey);
+	int             ret;
 
-	orsa = RSA_new();
-	orsa->n = BN_dup(pubkey->n);
-	orsa->d = seckey->d;
-	orsa->p = seckey->q;	/* p and q are round the other way in openssl */
-	orsa->q = seckey->p;
-
-	/* debug */
-	orsa->e = BN_dup(pubkey->e);
-	/* If this isn't set, it's very likely that the programmer hasn't */
-	/* decrypted the secret key. RSA_check_key segfaults in that case. */
-	/* Use pgp_decrypt_seckey() to do that. */
-	if (orsa->d == NULL) {
+	if (seckey->d == NULL) {
 		(void) fprintf(stderr, "orsa is not set\n");
 		return 0;
 	}
@@ -544,12 +641,11 @@ pgp_rsa_private_encrypt(uint8_t *out,
 	}
 	/* end debug */
 
-	n = RSA_private_encrypt((int)length, in, out, orsa, RSA_NO_PADDING);
+	ret = RSA_private_encrypt((int)length, in, out, orsa, RSA_NO_PADDING);
 
-	orsa->n = orsa->d = orsa->p = orsa->q = NULL;
 	RSA_free(orsa);
 
-	return n;
+	return ret;
 }
 
 /**
@@ -569,18 +665,10 @@ pgp_rsa_private_decrypt(uint8_t *out,
 			const pgp_rsa_seckey_t *seckey,
 			const pgp_rsa_pubkey_t *pubkey)
 {
-	RSA            *keypair;
+	RSA            *keypair = makeRSA(pubkey, seckey);
 	int             n;
 	char            errbuf[1024];
 
-	keypair = RSA_new();
-	keypair->n = pubkey->n;	/* XXX: do we need n? */
-	keypair->d = seckey->d;
-	keypair->p = seckey->q;
-	keypair->q = seckey->p;
-
-	/* debug */
-	keypair->e = pubkey->e;
 	if (RSA_check_key(keypair) != 1) {
 		(void) fprintf(stderr, "RSA_check_key is not set\n");
 		return 0;
@@ -600,7 +688,6 @@ pgp_rsa_private_decrypt(uint8_t *out,
 		ERR_error_string(err, &errbuf[0]);
 		(void) fprintf(stderr, "openssl error : %s\n", errbuf);
 	}
-	keypair->n = keypair->d = keypair->p = keypair->q = NULL;
 	RSA_free(keypair);
 
 	return n;
@@ -620,15 +707,11 @@ pgp_rsa_public_encrypt(uint8_t *out,
 			size_t length,
 			const pgp_rsa_pubkey_t *pubkey)
 {
-	RSA            *orsa;
+	RSA            *orsa = makeRSA(pubkey, NULL);
 	int             n;
 
 	/* printf("pgp_rsa_public_encrypt: length=%ld\n", length); */
 
-	orsa = RSA_new();
-	orsa->n = pubkey->n;
-	orsa->e = pubkey->e;
-
 	/* printf("len: %ld\n", length); */
 	/* pgp_print_bn("n: ", orsa->n); */
 	/* pgp_print_bn("e: ", orsa->e); */
@@ -640,7 +723,6 @@ pgp_rsa_public_encrypt(uint8_t *out,
 		fd_out = BIO_new_fd(fileno(stderr), BIO_NOCLOSE);
 		ERR_print_errors(fd_out);
 	}
-	orsa->n = orsa->e = NULL;
 	RSA_free(orsa);
 
 	return n;
@@ -656,7 +738,9 @@ void 
 pgp_crypto_finish(void)
 {
 	CRYPTO_cleanup_all_ex_data();
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
 	ERR_remove_state((unsigned long)0);
+#endif
 }
 
 /**
@@ -692,25 +776,33 @@ rsa_generate_keypair(pgp_key_t *keydata,
 	BN_CTX         *ctx;
 	pgp_output_t *output;
 	pgp_memory_t   *mem;
+	BIGNUM *bne;
+	pgp_rsa_pubkey_t *pk;
+	pgp_rsa_seckey_t *sk;
 
 	ctx = BN_CTX_new();
 	pgp_keydata_init(keydata, PGP_PTAG_CT_SECRET_KEY);
 	seckey = pgp_get_writable_seckey(keydata);
+	pk = &seckey->pubkey.key.rsa;
+	sk = &seckey->key.rsa;
 
 	/* generate the key pair */
 
-	rsa = RSA_generate_key(numbits, e, NULL, NULL);
+	bne = BN_new();
+	BN_set_word(bne, e);
+
+	rsa = RSA_new();
+	RSA_generate_key_ex(rsa, numbits, bne, NULL);
+	BN_free(bne);
 
 	/* populate pgp key from ssl key */
+	takeRSA(rsa, pk, sk);
 
 	seckey->pubkey.version = PGP_V4;
 	seckey->pubkey.birthtime = time(NULL);
 	seckey->pubkey.days_valid = 0;
 	seckey->pubkey.alg = PGP_PKA_RSA;
 
-	seckey->pubkey.key.rsa.n = BN_dup(rsa->n);
-	seckey->pubkey.key.rsa.e = BN_dup(rsa->e);
-
 	seckey->s2k_usage = PGP_S2KU_ENCRYPTED_AND_HASHED;
 	seckey->s2k_specifier = PGP_S2KS_SALTED;
 	/* seckey->s2k_specifier=PGP_S2KS_SIMPLE; */
@@ -721,11 +813,8 @@ rsa_generate_keypair(pgp_key_t *keydata,
 	seckey->octetc = 0;
 	seckey->checksum = 0;
 
-	seckey->key.rsa.d = BN_dup(rsa->d);
-	seckey->key.rsa.p = BN_dup(rsa->p);
-	seckey->key.rsa.q = BN_dup(rsa->q);
-	seckey->key.rsa.u = BN_mod_inverse(NULL, rsa->p, rsa->q, ctx);
-	if (seckey->key.rsa.u == NULL) {
+	sk->u = BN_mod_inverse(NULL, sk->p, sk->q, ctx);
+	if (sk->u == NULL) {
 		(void) fprintf(stderr, "seckey->key.rsa.u is NULL\n");
 		return 0;
 	}
@@ -817,18 +906,10 @@ pgp_dsa_sign(uint8_t *hashbuf,
 		const pgp_dsa_pubkey_t *pubdsa)
 {
 	DSA_SIG        *dsasig;
-	DSA            *odsa;
-
-	odsa = DSA_new();
-	odsa->p = pubdsa->p;
-	odsa->q = pubdsa->q;
-	odsa->g = pubdsa->g;
-	odsa->pub_key = pubdsa->y;
-	odsa->priv_key = secdsa->x;
+	DSA            *odsa = makeDSA(pubdsa, secdsa);
 
 	dsasig = DSA_do_sign(hashbuf, (int)hashsize, odsa);
 
-	odsa->p = odsa->q = odsa->g = odsa->pub_key = odsa->priv_key = NULL;
 	DSA_free(odsa);
 
 	return dsasig;
@@ -860,15 +941,12 @@ openssl_read_pem_seckey(const char *f, p
 				rsa = PEM_read_RSAPrivateKey(fp, NULL, NULL, pass);
 			} while (rsa == NULL);
 		}
-		key->key.seckey.key.rsa.d = rsa->d;
-		key->key.seckey.key.rsa.p = rsa->p;
-		key->key.seckey.key.rsa.q = rsa->q;
-		key->key.seckey.key.rsa.d = rsa->d;
+		takeRSA(rsa, NULL, &key->key.seckey.key.rsa);
 	} else if (strcmp(type, "ssh-dss") == 0) {
 		if ((dsa = PEM_read_DSAPrivateKey(fp, NULL, NULL, NULL)) == NULL) {
 			ok = 0;
 		} else {
-			key->key.seckey.key.dsa.x = dsa->priv_key;
+			takeDSA(dsa, &key->key.seckey.key.dsa);
 		}
 	} else {
 		ok = 0;