vgauth patch for openssl-1.1.0

open-vm-tools.spec

@@ -49,6 +49,7 @@ Patch2:           vmw-bitmask-gcc6.patch
 Patch3:           hgfs-cache.patch
 Patch4:           udev-rules.patch
 Patch5:           glibc-sysmacros.patch
+Patch6:           openssl-vgauth.patch
 
 BuildRequires:    autoconf
 BuildRequires:    automake
@@ -135,6 +136,7 @@ VMware virtual machines.
 %patch3 -p0
 %patch4 -p0
 %patch5 -p0
+%patch6 -p0
 
 %build
 mkdir -p udev
@@ -313,6 +315,7 @@ fi
 %changelog
 * Thu Feb 16 2017 Ravindra Kumar <ravindrakumar@vmware.com> - 10.0.5-10
 - sysmacros patch for glibc-2.25 (RHBZ#1411807).
+- vgauth patch for openssl-1.1.0.
 
 * Thu Feb 16 2017 Ravindra Kumar <ravindrakumar@vmware.com> - 10.0.5-9
 - udev rules patch for longer SCSI timeouts (RHBZ#1214347).

openssl-vgauth.patch

@@ -0,0 +1,77 @@
+--- vgauth/common/certverify.c.orig	2017-02-16 19:08:36.509896717 -0800
++++ vgauth/common/certverify.c	2017-02-16 19:15:02.716084270 -0800
+@@ -827,11 +827,15 @@
+                           const unsigned char *signature)
+ {
+    VGAuthError err = VGAUTH_E_FAIL;
+-   EVP_MD_CTX mdCtx;
++   EVP_MD_CTX *mdCtx = NULL;
+    const EVP_MD *hashAlg;
+    int ret;
+ 
+-   EVP_MD_CTX_init(&mdCtx);
++   mdCtx = EVP_MD_CTX_new();
++   if (mdCtx == NULL) {
++      g_warning("%s: unable to allocate a message digest.\n", __FUNCTION__);
++      return(VGAUTH_E_OUT_OF_MEMORY);
++   }
+ 
+    switch (hash) {
+    case VGAUTH_HASH_ALG_SHA256:
+@@ -843,7 +847,7 @@
+       goto done;
+    }
+ 
+-   ret = EVP_VerifyInit(&mdCtx, hashAlg);
++   ret = EVP_VerifyInit(mdCtx, hashAlg);
+    if (ret <= 0) {
+       VerifyDumpSSLErrors();
+       g_warning("%s: unable to initialize verificatation context (ret = %d)\n",
+@@ -856,7 +860,7 @@
+     * one shot. We probably should put some upper bound on the size of the
+     * data.
+     */
+-   ret = EVP_VerifyUpdate(&mdCtx, data, dataLen);
++   ret = EVP_VerifyUpdate(mdCtx, data, dataLen);
+    if (ret <= 0) {
+       VerifyDumpSSLErrors();
+       g_warning("%s: unable to update verificatation context (ret = %d)\n",
+@@ -864,7 +868,7 @@
+       goto done;
+    }
+ 
+-   ret = EVP_VerifyFinal(&mdCtx, signature, (unsigned int) signatureLen, publicKey);
++   ret = EVP_VerifyFinal(mdCtx, signature, (unsigned int) signatureLen, publicKey);
+    if (0 == ret) {
+       g_debug("%s: verification failed!\n", __FUNCTION__);
+       err = VGAUTH_E_AUTHENTICATION_DENIED;
+@@ -879,7 +883,7 @@
+    err = VGAUTH_E_OK;
+ 
+ done:
+-   EVP_MD_CTX_cleanup(&mdCtx);
++   EVP_MD_CTX_free(mdCtx);
+ 
+    return err;
+ }
+--- vgauth/common/certverify.h.orig	2017-02-16 19:08:43.843033377 -0800
++++ vgauth/common/certverify.h	2017-02-16 19:22:38.248130476 -0800
+@@ -28,6 +28,18 @@
+ #include <glib.h>
+ #include "VGAuthAuthentication.h"
+ 
++/* new API from OpenSSL 1.1.0
++ *      https://www.openssl.org/docs/manmaster/crypto/EVP_DigestInit.html
++ *
++ * EVP_MD_CTX_create() and EVP_MD_CTX_destroy() were renamed to
++ * EVP_MD_CTX_new() and EVP_MD_CTX_free() in OpenSSL 1.1.
++ */
++#if OPENSSL_VERSION_NUMBER < 0x10100000L
++#define EVP_MD_CTX_new()        EVP_MD_CTX_create()
++#define EVP_MD_CTX_free(x)      EVP_MD_CTX_destroy((x))
++#endif /* OpenSSL version < 1.1.0 */
++
++
+ /*
+  * XXX Do we still need this?  What other algorithms do SAML tokens use?
+  */