Porting conserver to OpenSSL 1.1

Hi,

I've been trying to get conserver to work with OpenSSL 1.1, as this will
soon be the only version available in Debian Linux:
https://bugs.debian.org/851085

The attached patch fixes a few trivial compile errors caused by API
changes. With the attached patch applied, the code builds against
Debian sid's libssl-dev (1.1). However, the server rejects all
connections with a "handshake error", and a pretty generic error
message that just means something is wrong with ciphers, certificates,
or something like that. This causes all the tests to fail. The code
doesn't even reach the place the patch changes, so it's unlikely to be
the cause.

Has anyone made any further progress? Does anyone know what the problem
is?

Any help greatly appreciated.

Chris.

---

The errors are:

140691693188864:error:14094410:SSL routines:ssl3_read_bytes:sslv3
  alert handshake failure:../ssl/record/rec_layer_s3.c:1399:SSL alert number 40

error:1417A0C1:SSL routines:tls_post_process_client_hello:no shared cipher

>From 2a3aad60bea93bc849881983b6f5cb930b900334 Mon Sep 17 00:00:00 2001
From: "Chris West (Faux)" <git@goeswhere.com>
Date: Tue, 25 Jul 2017 19:04:22 +0000
Subject: [PATCH] new openssl api for generating DH

---
 conserver-8.2.1/conserver/main.c | 76 +++++++++++++++++++---------------------
 conserver-8.2.1/debian/control   |  2 +-
 2 files changed, 37 insertions(+), 41 deletions(-)

diff --git a/conserver-8.2.1/conserver/main.c b/conserver-8.2.1/conserver/main.c
index cb9af46..c5d9ca7 100644
--- a/conserver-8.2.1/conserver/main.c
+++ b/conserver-8.2.1/conserver/main.c
@@ -92,6 +92,38 @@ DH *dh1024 = (DH *)0;
 DH *dh2048 = (DH *)0;
 DH *dh4096 = (DH *)0;
 
+DH *
+DHFromArray(char *dh_p, size_t dh_p_size, char *dh_g, size_t dh_g_size) {
+    DH *dh;
+    BIGNUM *p, *g;
+
+    p = BN_bin2bn(dh_p, dh_p_size, NULL);
+    if (p == NULL) {
+	BN_free(p);
+	return (NULL);
+    }
+
+    g = BN_bin2bn(dh_g, dh_g_size, NULL);
+    if (g == NULL) {
+	BN_free(g);
+	return (NULL);
+    }
+
+    if ((dh = DH_new()) == NULL) {
+	BN_free(p);
+	BN_free(g);
+	return (NULL);
+    }
+
+    if (!DH_set0_pqg(dh, p, NULL, g)) {
+	BN_free(p);
+	BN_free(g);
+	DH_free(dh);
+	return (NULL);
+    }
+
+    return (dh);
+}
 
 DH *
 GetDH512(void)
@@ -108,17 +140,8 @@ GetDH512(void)
     static unsigned char dh512_g[] = {
 	0x02,
     };
-    DH *dh;
 
-    if ((dh = DH_new()) == NULL)
-	return (NULL);
-    dh->p = BN_bin2bn(dh512_p, sizeof(dh512_p), NULL);
-    dh->g = BN_bin2bn(dh512_g, sizeof(dh512_g), NULL);
-    if ((dh->p == NULL) || (dh->g == NULL)) {
-	DH_free(dh);
-	return (NULL);
-    }
-    return (dh);
+    return DHFromArray(dh512_p, sizeof(dh512_p), dh512_g, sizeof(dh512_g));
 }
 
 DH *
@@ -142,17 +165,8 @@ GetDH1024(void)
     static unsigned char dh1024_g[] = {
 	0x02,
     };
-    DH *dh;
 
-    if ((dh = DH_new()) == NULL)
-	return (NULL);
-    dh->p = BN_bin2bn(dh1024_p, sizeof(dh1024_p), NULL);
-    dh->g = BN_bin2bn(dh1024_g, sizeof(dh1024_g), NULL);
-    if ((dh->p == NULL) || (dh->g == NULL)) {
-	DH_free(dh);
-	return (NULL);
-    }
-    return (dh);
+    return DHFromArray(dh1024_p, sizeof(dh1024_p), dh1024_g, sizeof(dh1024_g));
 }
 
 DH *
@@ -189,17 +203,8 @@ GetDH2048(void)
     static unsigned char dh2048_g[] = {
 	0x02,
     };
-    DH *dh;
 
-    if ((dh = DH_new()) == NULL)
-	return (NULL);
-    dh->p = BN_bin2bn(dh2048_p, sizeof(dh2048_p), NULL);
-    dh->g = BN_bin2bn(dh2048_g, sizeof(dh2048_g), NULL);
-    if ((dh->p == NULL) || (dh->g == NULL)) {
-	DH_free(dh);
-	return (NULL);
-    }
-    return (dh);
+    return DHFromArray(dh2048_p, sizeof(dh2048_p), dh2048_g, sizeof(dh2048_g));
 }
 
 DH *
@@ -262,17 +267,8 @@ GetDH4096(void)
     static unsigned char dh4096_g[] = {
 	0x02,
     };
-    DH *dh;
 
-    if ((dh = DH_new()) == NULL)
-	return (NULL);
-    dh->p = BN_bin2bn(dh4096_p, sizeof(dh4096_p), NULL);
-    dh->g = BN_bin2bn(dh4096_g, sizeof(dh4096_g), NULL);
-    if ((dh->p == NULL) || (dh->g == NULL)) {
-	DH_free(dh);
-	return (NULL);
-    }
-    return (dh);
+    return DHFromArray(dh4096_p, sizeof(dh4096_p), dh4096_g, sizeof(dh4096_g));
 }
 
 DH *
diff --git a/conserver-8.2.1/debian/control b/conserver-8.2.1/debian/control
index 6e78071..59e8e01 100644
--- a/conserver-8.2.1/debian/control
+++ b/conserver-8.2.1/debian/control
@@ -2,7 +2,7 @@ Source: conserver
 Section: non-free/comm
 Priority: optional
 Maintainer: Jörgen Hägg <jh@debian.org>
-Build-Depends: debhelper (>= 7.0.50), po-debconf, libpam0g-dev, libwrap0-dev, libssl1.0-dev
+Build-Depends: debhelper (>= 7.0.50), po-debconf, libpam0g-dev, libwrap0-dev, libssl-dev
 Standards-Version: 3.9.8
 XS-Autobuild: yes
 Homepage: http://www.conserver.com/
-- 
2.13.3