Backport upstream generating base64 encoded output for NT gensalt

libxcrypt-4.4.0-use_base64_output_gensalt_nt_rn.patch

@@ -0,0 +1,169 @@
+From fdeddd908b6c659b281bbef7e535f9060b5b6186 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Bj=C3=B6rn=20Esser?= <besser82@fedoraproject.org>
+Date: Wed, 21 Nov 2018 18:59:21 +0100
+Subject: [PATCH] Use base64 encoding for output of gensalt_nt_rn.
+
+---
+ crypt-nthash.c | 89 ++++++++++++++++++++++++++++++++++++++++----------
+ test-gensalt.c | 10 +++---
+ 2 files changed, 76 insertions(+), 23 deletions(-)
+
+diff --git a/crypt-nthash.c b/crypt-nthash.c
+index 83c45ea..2f47476 100644
+--- a/crypt-nthash.c
++++ b/crypt-nthash.c
+@@ -43,6 +43,58 @@
+ 
+ #if INCLUDE_nt
+ 
++static uint8_t *
++encode64_uint32 (uint8_t * dst, ssize_t dstlen,
++                 uint32_t src, uint32_t srcbits)
++{
++  uint32_t bit;
++
++  for (bit = 0; bit < srcbits; bit += 6)
++    {
++      if (dstlen < 1)
++        {
++          errno = ERANGE;
++          return NULL;
++        }
++      *dst++ = ascii64[src & 0x3f];
++      dstlen--;
++      src >>= 6;
++    }
++
++  *dst = '\0';
++  return dst;
++}
++
++static uint8_t *
++encode64 (uint8_t * dst, ssize_t dstlen,
++          const uint8_t * src, size_t srclen)
++{
++  size_t i;
++
++  for (i = 0; i < srclen; )
++    {
++      uint8_t * dnext;
++      uint32_t value = 0, bits = 0;
++      do
++        {
++          value |= (uint32_t) src[i++] << bits;
++          bits += 8;
++        }
++      while (bits < 24 && i < srclen);
++      dnext = encode64_uint32 (dst, dstlen, value, bits);
++      if (!dnext)
++        {
++          errno = ERANGE;
++          return NULL;
++        }
++      dstlen -= (dnext - dst);
++      dst = dnext;
++    }
++
++  *dst = '\0';
++  return dst;
++}
++
+ /*
+  * NT HASH = md4(str2unicode(phrase))
+  */
+@@ -104,48 +156,49 @@ crypt_nt_rn (const char *phrase, size_t ARG_UNUSED (phr_size),
+    SETTING for the crypt function.  */
+ void
+ gensalt_nt_rn (unsigned long count,
+-                   const uint8_t *rbytes,
+-                   size_t nrbytes,
+-                   uint8_t *output,
+-                   size_t o_size)
++               const uint8_t *rbytes,
++               size_t nrbytes,
++               uint8_t *output,
++               size_t o_size)
+ {
+-  static const char *salt = "$3$__not_used__";
++  const char *salt = "$3$__not_used__";
++  const size_t saltlen = strlen (salt);
+   MD4_CTX ctx;
+   unsigned char hashbuf[16];
+-  char hashstr[14 + 1];
+-  unsigned long i;
++  size_t i;
+ 
+   /* Minimal O_SIZE to store the fake salt.
+      At least 1 byte of RBYTES is needed
+      to calculate the MD4 hash used in the
+      fake salt.  */
+-  if ((o_size < 30) || (nrbytes < 1))
++  if ((o_size < saltlen + BASE64_LEN (sizeof (hashbuf)) + 1) ||
++      (nrbytes < 2))
+     {
+       errno = ERANGE;
+       return;
+     }
++
+   if (count != 0)
+     {
+       errno = EINVAL;
+       return;
+     }
+ 
++  XCRYPT_STRCPY_OR_ABORT (output, o_size, salt);
++
+   MD4_Init (&ctx);
+-  for (i = 0; i < 20; i++)
++  for (i = 0; i < saltlen * nrbytes; i++)
+     {
+-      MD4_Update (&ctx, salt, (i % 15) + 1);
++      MD4_Update (&ctx, salt, (i % saltlen) + 1);
+       MD4_Update (&ctx, rbytes, nrbytes);
+-      MD4_Update (&ctx, salt, 15);
+-      MD4_Update (&ctx, salt, 15 - (i % 15));
++      MD4_Update (&ctx, rbytes, nrbytes - (i % nrbytes));
++      MD4_Update (&ctx, salt, saltlen);
++      MD4_Update (&ctx, salt, saltlen - (i % saltlen));
+     }
+   MD4_Final (hashbuf, &ctx);
+ 
+-  for (i = 0; i < 7; i++)
+-    sprintf (&(hashstr[i * 2]), "%02x", hashbuf[i]);
+-  hashstr[14] = '\0';
+-
+-  XCRYPT_STRCPY_OR_ABORT (output, o_size, salt);
+-  XCRYPT_STRCPY_OR_ABORT (output + 15, o_size - 15, hashstr);
++  encode64 (output + saltlen, (ssize_t) (o_size - saltlen),
++            hashbuf, sizeof (hashbuf));
+ }
+ 
+ #endif
+diff --git a/test-gensalt.c b/test-gensalt.c
+index 49873a5..676edf1 100644
+--- a/test-gensalt.c
++++ b/test-gensalt.c
+@@ -67,10 +67,10 @@ static const char *const md5_expected_output[] =
+ #if INCLUDE_nt
+ static const char *const nthash_expected_output[] =
+ {
+-  "$3$__not_used__c809a450df09a3",
+-  "$3$__not_used__30d0d6f834c0c3",
+-  "$3$__not_used__0eeeebb83d6fe4",
+-  "$3$__not_used__1c690d6a9ef88c"
++  "$3$__not_used__oTs8adyvc5SpgTxXhllxC/",
++  "$3$__not_used__CPYy.33449S0xlcz6wv2W/",
++  "$3$__not_used__HUo2S2sloOywHZxmgM.wo/",
++  "$3$__not_used__ZL45/aAV4iGWPnlWZamDX0"
+ };
+ #endif
+ #if INCLUDE_sunmd5
+@@ -330,7 +330,7 @@ static const struct testcase testcases[] =
+   // MD5/BSD doesn't have variable round count.
+ #endif
+ #if INCLUDE_nt
+-  { "$3$",   nthash_expected_output,   29,  0, 0 },
++  { "$3$",   nthash_expected_output,   37,  0, 0 },
+   // NTHASH doesn't have variable round count.
+ #endif
+ #if INCLUDE_sunmd5

libxcrypt.spec

@@ -101,6 +101,7 @@ Source0:        %{url}/archive/v%{version}/%{name}-%{version}.tar.gz
 # Patch 0000 - 2999: Backported patches from upstream.
 Patch0:         %{url}/commit/a60130b2211c6666bee69051b2b76413af2d679f.patch#/%{name}-4.4.0-fix_manual-page-warning_list-type-stack0.patch
 Patch1:         %{url}/commit/664bfda5a51dbaa75904b29a7cd3c51888db6bd9.patch#/%{name}-4.4.0-use_XCRYPT_STRCPY_OR_ABORT_in_crypt_nt_rn.patch
+Patch2:         %{url}/commit/fdeddd908b6c659b281bbef7e535f9060b5b6186.patch#/%{name}-4.4.0-use_base64_output_gensalt_nt_rn.patch
 
 # Patch 3000 - 5999: Backported patches from pull requests.
 
@@ -330,6 +331,7 @@ done
 %changelog
 * Mon Nov 26 2018 Björn Esser <besser82@fedoraproject.org> - 4.4.0-2
 - Backport upstream commit to use a safer strcpy for the NT method
+- Backport upstream generating base64 encoded output for NT gensalt
 
 * Tue Nov 20 2018 Björn Esser <besser82@fedoraproject.org> - 4.4.0-1
 - New upstream release