make DTLS work (#1462541)

openssl-1.1.0-dtls-failure.patch

@@ -0,0 +1,100 @@
+From 290cfa823fb3afea5b36ed17113ffb4f675d0165 Mon Sep 17 00:00:00 2001
+From: Matt Caswell <matt@openssl.org>
+Date: Tue, 20 Jun 2017 16:36:30 +0100
+Subject: [PATCH] Fix DTLS failure when used in a build which has SCTP enabled
+
+The value of BIO_CTRL_DGRAM_SET_PEEK_MODE was clashing with the value for
+BIO_CTRL_DGRAM_SCTP_SET_IN_HANDSHAKE. In an SCTP enabled build
+BIO_CTRL_DGRAM_SCTP_SET_IN_HANDSHAKE was used unconditionally with
+the reasoning that it would be ignored if SCTP wasn't in use. Unfortunately
+due to this clash, this wasn't the case. The BIO ended up going into peek
+mode and was continually reading the same data over and over - throwing it
+away as a replay.
+
+Fixes #3723
+---
+ crypto/bio/bss_dgram.c | 9 +++++++++
+ include/openssl/bio.h  | 7 ++++---
+ ssl/statem/statem.c    | 8 ++++----
+ 3 files changed, 17 insertions(+), 7 deletions(-)
+
+diff --git a/crypto/bio/bss_dgram.c b/crypto/bio/bss_dgram.c
+index 7ef4281..4070f20 100644
+--- a/crypto/bio/bss_dgram.c
++++ b/crypto/bio/bss_dgram.c
+@@ -787,6 +787,15 @@ static long dgram_ctrl(BIO *b, int cmd, long num, void *ptr)
+     case BIO_CTRL_DGRAM_GET_MTU_OVERHEAD:
+         ret = dgram_get_mtu_overhead(data);
+         break;
++
++    /*
++     * BIO_CTRL_DGRAM_SCTP_SET_IN_HANDSHAKE is used here for compatibility
++     * reasons. When BIO_CTRL_DGRAM_SET_PEEK_MODE was first defined its value
++     * was incorrectly clashing with BIO_CTRL_DGRAM_SCTP_SET_IN_HANDSHAKE. The
++     * value has been updated to a non-clashing value. However to preserve
++     * binary compatiblity we now respond to both the old value and the new one
++     */
++    case BIO_CTRL_DGRAM_SCTP_SET_IN_HANDSHAKE:
+     case BIO_CTRL_DGRAM_SET_PEEK_MODE:
+         data->peekmode = (unsigned int)num;
+         break;
+diff --git a/include/openssl/bio.h b/include/openssl/bio.h
+index 0955b76..5f8f83a 100644
+--- a/include/openssl/bio.h
++++ b/include/openssl/bio.h
+@@ -126,11 +126,10 @@ extern "C" {
+ 
+ # define BIO_CTRL_DGRAM_GET_MTU_OVERHEAD   49
+ 
+-# define BIO_CTRL_DGRAM_SET_PEEK_MODE      50
+-
++/* Deliberately outside of OPENSSL_NO_SCTP - used in bss_dgram.c */
++#  define BIO_CTRL_DGRAM_SCTP_SET_IN_HANDSHAKE    50
+ # ifndef OPENSSL_NO_SCTP
+ /* SCTP stuff */
+-#  define BIO_CTRL_DGRAM_SCTP_SET_IN_HANDSHAKE    50
+ #  define BIO_CTRL_DGRAM_SCTP_ADD_AUTH_KEY                51
+ #  define BIO_CTRL_DGRAM_SCTP_NEXT_AUTH_KEY               52
+ #  define BIO_CTRL_DGRAM_SCTP_AUTH_CCS_RCVD               53
+@@ -143,6 +142,8 @@ extern "C" {
+ #  define BIO_CTRL_DGRAM_SCTP_SAVE_SHUTDOWN               70
+ # endif
+ 
++# define BIO_CTRL_DGRAM_SET_PEEK_MODE      71
++
+ /* modifiers */
+ # define BIO_FP_READ             0x02
+ # define BIO_FP_WRITE            0x04
+diff --git a/ssl/statem/statem.c b/ssl/statem/statem.c
+index 52beac7..9eab8ce 100644
+--- a/ssl/statem/statem.c
++++ b/ssl/statem/statem.c
+@@ -300,10 +300,10 @@ static int state_machine(SSL *s, int server)
+             return -1;
+     }
+ #ifndef OPENSSL_NO_SCTP
+-    if (SSL_IS_DTLS(s)) {
++    if (SSL_IS_DTLS(s) && BIO_dgram_is_sctp(SSL_get_wbio(s))) {
+         /*
+          * Notify SCTP BIO socket to enter handshake mode and prevent stream
+-         * identifier other than 0. Will be ignored if no SCTP is used.
++         * identifier other than 0.
+          */
+         BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_SCTP_SET_IN_HANDSHAKE,
+                  st->in_handshake, NULL);
+@@ -421,10 +421,10 @@ static int state_machine(SSL *s, int server)
+     st->in_handshake--;
+ 
+ #ifndef OPENSSL_NO_SCTP
+-    if (SSL_IS_DTLS(s)) {
++    if (SSL_IS_DTLS(s) && BIO_dgram_is_sctp(SSL_get_wbio(s))) {
+         /*
+          * Notify SCTP BIO socket to leave handshake mode and allow stream
+-         * identifier other than 0. Will be ignored if no SCTP is used.
++         * identifier other than 0.
+          */
+         BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_SCTP_SET_IN_HANDSHAKE,
+                  st->in_handshake, NULL);
+-- 
+2.9.3
+

openssl.spec

@@ -22,7 +22,7 @@
 Summary: Utilities from the general purpose cryptography library with TLS implementation
 Name: openssl
 Version: 1.1.0f
-Release: 3%{?dist}
+Release: 4%{?dist}
 Epoch: 1
 # We have to remove certain patented algorithms from the openssl source
 # tarball with the hobble-openssl script which is included below.
@@ -63,6 +63,7 @@ Patch44: openssl-1.1.0-bio-fd-preserve-nl.patch
 Patch45: openssl-1.1.0-weak-ciphers.patch
 # Backported fixes including security fixes
 Patch70: openssl-1.1.0-thread-local.patch
+Patch71: openssl-1.1.0-dtls-failure.patch
 
 License: OpenSSL
 Group: System Environment/Libraries
@@ -166,6 +167,7 @@ cp %{SOURCE13} test/
 %patch45 -p1 -b .weak-ciphers
 
 %patch70 -p1 -b .thread-local
+%patch71 -p1 -b .dtls-failure
 
 %build
 # Figure out which flags we want to use.
@@ -434,6 +436,9 @@ export LD_LIBRARY_PATH
 %postun libs -p /sbin/ldconfig
 
 %changelog
+* Fri Jun 23 2017 Tomáš Mráz <tmraz@redhat.com> 1.1.0f-4
+- make DTLS work (#1462541)
+
 * Thu Jun 15 2017 Tomáš Mráz <tmraz@redhat.com> 1.1.0f-3
 - enable 3DES SSL ciphersuites, RC4 is kept disabled (#1453066)