libxcrypt/libxcrypt-4.4.0-use_base64_output_gensalt_nt_rn.patch

From fdeddd908b6c659b281bbef7e535f9060b5b6186 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Esser?= <besser82@fedoraproject.org>
Date: Wed, 21 Nov 2018 18:59:21 +0100
Subject: [PATCH] Use base64 encoding for output of gensalt_nt_rn.

---
 crypt-nthash.c | 89 ++++++++++++++++++++++++++++++++++++++++----------
 test-gensalt.c | 10 +++---
 2 files changed, 76 insertions(+), 23 deletions(-)

diff --git a/crypt-nthash.c b/crypt-nthash.c
index 83c45ea..2f47476 100644
--- a/crypt-nthash.c
+++ b/crypt-nthash.c
@@ -43,6 +43,58 @@
 
 #if INCLUDE_nt
 
+static uint8_t *
+encode64_uint32 (uint8_t * dst, ssize_t dstlen,
+                 uint32_t src, uint32_t srcbits)
+{
+  uint32_t bit;
+
+  for (bit = 0; bit < srcbits; bit += 6)
+    {
+      if (dstlen < 1)
+        {
+          errno = ERANGE;
+          return NULL;
+        }
+      *dst++ = ascii64[src & 0x3f];
+      dstlen--;
+      src >>= 6;
+    }
+
+  *dst = '\0';
+  return dst;
+}
+
+static uint8_t *
+encode64 (uint8_t * dst, ssize_t dstlen,
+          const uint8_t * src, size_t srclen)
+{
+  size_t i;
+
+  for (i = 0; i < srclen; )
+    {
+      uint8_t * dnext;
+      uint32_t value = 0, bits = 0;
+      do
+        {
+          value |= (uint32_t) src[i++] << bits;
+          bits += 8;
+        }
+      while (bits < 24 && i < srclen);
+      dnext = encode64_uint32 (dst, dstlen, value, bits);
+      if (!dnext)
+        {
+          errno = ERANGE;
+          return NULL;
+        }
+      dstlen -= (dnext - dst);
+      dst = dnext;
+    }
+
+  *dst = '\0';
+  return dst;
+}
+
 /*
  * NT HASH = md4(str2unicode(phrase))
  */
@@ -104,48 +156,49 @@ crypt_nt_rn (const char *phrase, size_t ARG_UNUSED (phr_size),
    SETTING for the crypt function.  */
 void
 gensalt_nt_rn (unsigned long count,
-                   const uint8_t *rbytes,
-                   size_t nrbytes,
-                   uint8_t *output,
-                   size_t o_size)
+               const uint8_t *rbytes,
+               size_t nrbytes,
+               uint8_t *output,
+               size_t o_size)
 {
-  static const char *salt = "$3$__not_used__";
+  const char *salt = "$3$__not_used__";
+  const size_t saltlen = strlen (salt);
   MD4_CTX ctx;
   unsigned char hashbuf[16];
-  char hashstr[14 + 1];
-  unsigned long i;
+  size_t i;
 
   /* Minimal O_SIZE to store the fake salt.
      At least 1 byte of RBYTES is needed
      to calculate the MD4 hash used in the
      fake salt.  */
-  if ((o_size < 30) || (nrbytes < 1))
+  if ((o_size < saltlen + BASE64_LEN (sizeof (hashbuf)) + 1) ||
+      (nrbytes < 2))
     {
       errno = ERANGE;
       return;
     }
+
   if (count != 0)
     {
       errno = EINVAL;
       return;
     }
 
+  XCRYPT_STRCPY_OR_ABORT (output, o_size, salt);
+
   MD4_Init (&ctx);
-  for (i = 0; i < 20; i++)
+  for (i = 0; i < saltlen * nrbytes; i++)
     {
-      MD4_Update (&ctx, salt, (i % 15) + 1);
+      MD4_Update (&ctx, salt, (i % saltlen) + 1);
       MD4_Update (&ctx, rbytes, nrbytes);
-      MD4_Update (&ctx, salt, 15);
-      MD4_Update (&ctx, salt, 15 - (i % 15));
+      MD4_Update (&ctx, rbytes, nrbytes - (i % nrbytes));
+      MD4_Update (&ctx, salt, saltlen);
+      MD4_Update (&ctx, salt, saltlen - (i % saltlen));
     }
   MD4_Final (hashbuf, &ctx);
 
-  for (i = 0; i < 7; i++)
-    sprintf (&(hashstr[i * 2]), "%02x", hashbuf[i]);
-  hashstr[14] = '\0';
-
-  XCRYPT_STRCPY_OR_ABORT (output, o_size, salt);
-  XCRYPT_STRCPY_OR_ABORT (output + 15, o_size - 15, hashstr);
+  encode64 (output + saltlen, (ssize_t) (o_size - saltlen),
+            hashbuf, sizeof (hashbuf));
 }
 
 #endif
diff --git a/test-gensalt.c b/test-gensalt.c
index 49873a5..676edf1 100644
--- a/test-gensalt.c
+++ b/test-gensalt.c
@@ -67,10 +67,10 @@ static const char *const md5_expected_output[] =
 #if INCLUDE_nt
 static const char *const nthash_expected_output[] =
 {
-  "$3$__not_used__c809a450df09a3",
-  "$3$__not_used__30d0d6f834c0c3",
-  "$3$__not_used__0eeeebb83d6fe4",
-  "$3$__not_used__1c690d6a9ef88c"
+  "$3$__not_used__oTs8adyvc5SpgTxXhllxC/",
+  "$3$__not_used__CPYy.33449S0xlcz6wv2W/",
+  "$3$__not_used__HUo2S2sloOywHZxmgM.wo/",
+  "$3$__not_used__ZL45/aAV4iGWPnlWZamDX0"
 };
 #endif
 #if INCLUDE_sunmd5
@@ -330,7 +330,7 @@ static const struct testcase testcases[] =
   // MD5/BSD doesn't have variable round count.
 #endif
 #if INCLUDE_nt
-  { "$3$",   nthash_expected_output,   29,  0, 0 },
+  { "$3$",   nthash_expected_output,   37,  0, 0 },
   // NTHASH doesn't have variable round count.
 #endif
 #if INCLUDE_sunmd5