krb5/Make-kprop-work-for-dump-files-larger-than-4GB.patch

From b778f10e31e566269768cba92a0005f35e134333 Mon Sep 17 00:00:00 2001
From: Julien Rische <jrische@redhat.com>
Date: Wed, 19 Jan 2022 19:46:08 +0100
Subject: [PATCH] Make kprop work for dump files larger than 4GB

If the dump file size does not fit in 32 bits, encode four zero bytes
(forcing an error for unmodified kpropd) followed by the size in the
next 64 bits.

Add a functional test case, but only run it when an environment
variable is set, as processing a 4GB dump file is too
resource-intensive for make check.

[ghudson@mit.edu: edited comments and commit message; eliminated use
of defined constant in some cases; added test case]

ticket: 9053 (new)
---
 src/kprop/kprop.c      | 37 +++++++++++++++++++++----------------
 src/kprop/kprop.h      | 12 ++++++++++++
 src/kprop/kprop_util.c | 42 ++++++++++++++++++++++++++++++++++++++++++
 src/kprop/kpropd.c     | 33 +++++++++++++++++++++------------
 src/tests/t_kprop.py   | 34 ++++++++++++++++++++++++++++++++++
 5 files changed, 130 insertions(+), 28 deletions(-)

diff --git a/src/kprop/kprop.c b/src/kprop/kprop.c
index 0b53aae7e..5adb4d31f 100644
--- a/src/kprop/kprop.c
+++ b/src/kprop/kprop.c
@@ -25,6 +25,7 @@
  */
 
 #include "k5-int.h"
+#include <inttypes.h>
 #include <locale.h>
 #include <sys/file.h>
 #include <signal.h>
@@ -71,11 +72,11 @@ static void open_connection(krb5_context context, char *host, int *fd_out);
 static void kerberos_authenticate(krb5_context context,
                                   krb5_auth_context *auth_context, int fd,
                                   krb5_principal me, krb5_creds **new_creds);
-static int open_database(krb5_context context, char *data_fn, int *size);
+static int open_database(krb5_context context, char *data_fn, off_t *size);
 static void close_database(krb5_context context, int fd);
 static void xmit_database(krb5_context context,
                           krb5_auth_context auth_context, krb5_creds *my_creds,
-                          int fd, int database_fd, int in_database_size);
+                          int fd, int database_fd, off_t in_database_size);
 static void send_error(krb5_context context, krb5_creds *my_creds, int fd,
                        char *err_text, krb5_error_code err_code);
 static void update_last_prop_file(char *hostname, char *file_name);
@@ -90,7 +91,8 @@ static void usage()
 int
 main(int argc, char **argv)
 {
-    int fd, database_fd, database_size;
+    int fd, database_fd;
+    off_t database_size;
     krb5_error_code retval;
     krb5_context context;
     krb5_creds *my_creds;
@@ -339,7 +341,7 @@ kerberos_authenticate(krb5_context context, krb5_auth_context *auth_context,
  * in the size of the database file.
  */
 static int
-open_database(krb5_context context, char *data_fn, int *size)
+open_database(krb5_context context, char *data_fn, off_t *size)
 {
     struct stat stbuf, stbuf_ok;
     char *data_ok_fn;
@@ -413,19 +415,18 @@ close_database(krb5_context context, int fd)
 static void
 xmit_database(krb5_context context, krb5_auth_context auth_context,
               krb5_creds *my_creds, int fd, int database_fd,
-              int in_database_size)
+              off_t in_database_size)
 {
     krb5_int32 n;
     krb5_data inbuf, outbuf;
-    char buf[KPROP_BUFSIZ];
+    char buf[KPROP_BUFSIZ], dbsize_buf[KPROP_DBSIZE_MAX_BUFSIZ];
     krb5_error_code retval;
     krb5_error *error;
-    krb5_ui_4 database_size = in_database_size, send_size, sent_size;
+    uint64_t database_size = in_database_size, send_size, sent_size;
 
     /* Send over the size. */
-    send_size = htonl(database_size);
-    inbuf.data = (char *)&send_size;
-    inbuf.length = sizeof(send_size); /* must be 4, really */
+    inbuf = make_data(dbsize_buf, sizeof(dbsize_buf));
+    encode_database_size(database_size, &inbuf);
     /* KPROP_CKSUMTYPE */
     retval = krb5_mk_safe(context, auth_context, &inbuf, &outbuf, NULL);
     if (retval) {
@@ -460,7 +461,7 @@ xmit_database(krb5_context context, krb5_auth_context auth_context,
         retval = krb5_mk_priv(context, auth_context, &inbuf, &outbuf, NULL);
         if (retval) {
             snprintf(buf, sizeof(buf),
-                     "while encoding database block starting at %d",
+                     "while encoding database block starting at %"PRIu64,
                      sent_size);
             com_err(progname, retval, "%s", buf);
             send_error(context, my_creds, fd, buf, retval);
@@ -471,14 +472,14 @@ xmit_database(krb5_context context, krb5_auth_context auth_context,
         if (retval) {
             krb5_free_data_contents(context, &outbuf);
             com_err(progname, retval,
-                    _("while sending database block starting at %d"),
+                    _("while sending database block starting at %"PRIu64),
                     sent_size);
             exit(1);
         }
         krb5_free_data_contents(context, &outbuf);
         sent_size += n;
         if (debug)
-            printf("%d bytes sent.\n", sent_size);
+            printf("%"PRIu64" bytes sent.\n", sent_size);
     }
     if (sent_size != database_size) {
         com_err(progname, 0, _("Premature EOF found for database file!"));
@@ -533,10 +534,14 @@ xmit_database(krb5_context context, krb5_auth_context auth_context,
         exit(1);
     }
 
-    memcpy(&send_size, outbuf.data, sizeof(send_size));
-    send_size = ntohl(send_size);
+    retval = decode_database_size(&outbuf, &send_size);
+    if (retval) {
+        com_err(progname, retval, _("malformed sent database size message"));
+        exit(1);
+    }
     if (send_size != database_size) {
-        com_err(progname, 0, _("Kpropd sent database size %d, expecting %d"),
+        com_err(progname, 0, _("Kpropd sent database size %"PRIu64
+                               ", expecting %"PRIu64),
                 send_size, database_size);
         exit(1);
     }
diff --git a/src/kprop/kprop.h b/src/kprop/kprop.h
index 75331cc8a..3a319b535 100644
--- a/src/kprop/kprop.h
+++ b/src/kprop/kprop.h
@@ -32,6 +32,7 @@
 #define KPROP_PROT_VERSION "kprop5_01"
 
 #define KPROP_BUFSIZ 32768
+#define KPROP_DBSIZE_MAX_BUFSIZ 12  /* max length of an encoded DB size */
 
 /* pathnames are in osconf.h, included via k5-int.h */
 
@@ -41,3 +42,14 @@ int sockaddr2krbaddr(krb5_context context, int family, struct sockaddr *sa,
 krb5_error_code
 sn2princ_realm(krb5_context context, const char *hostname, const char *sname,
                const char *realm, krb5_principal *princ_out);
+
+/*
+ * Encode size in four bytes (for backward compatibility) if it fits; otherwise
+ * use the larger encoding.  buf must be allocated with at least
+ * KPROP_DBSIZE_MAX_BUFSIZ bytes.
+ */
+void encode_database_size(uint64_t size, krb5_data *buf);
+
+/* Decode a database size.  Return KRB5KRB_ERR_GENERIC if buf has an invalid
+ * length or did not encode a 32-bit size compactly. */
+krb5_error_code decode_database_size(const krb5_data *buf, uint64_t *size_out);
diff --git a/src/kprop/kprop_util.c b/src/kprop/kprop_util.c
index c2b2e8764..c0aa2c89b 100644
--- a/src/kprop/kprop_util.c
+++ b/src/kprop/kprop_util.c
@@ -92,3 +92,45 @@ sn2princ_realm(krb5_context context, const char *hostname, const char *sname,
     *princ_out = princ;
     return 0;
 }
+
+void
+encode_database_size(uint64_t size, krb5_data *buf)
+{
+    assert(buf->length >= 12);
+    if (size > 0 && size <= UINT32_MAX) {
+        /* Encode in 32 bits for backward compatibility. */
+        store_32_be(size, buf->data);
+        buf->length = 4;
+    } else {
+        /* Set the first 32 bits to 0 and encode in the following 64 bits. */
+        store_32_be(0, buf->data);
+        store_64_be(size, buf->data + 4);
+        buf->length = 12;
+    }
+}
+
+krb5_error_code
+decode_database_size(const krb5_data *buf, uint64_t *size_out)
+{
+    uint64_t size;
+
+    if (buf->length == 12) {
+        /* A 12-byte buffer must have the first four bytes zeroed. */
+        if (load_32_be(buf->data) != 0)
+            return KRB5KRB_ERR_GENERIC;
+
+        /* The size is stored in the next 64 bits.  Values from 1..2^32-1 must
+         * be encoded in four bytes. */
+        size = load_64_be(buf->data + 4);
+        if (size > 0 && size <= UINT32_MAX)
+            return KRB5KRB_ERR_GENERIC;
+    } else if (buf->length == 4) {
+        size = load_32_be(buf->data);
+    } else {
+        /* Invalid buffer size. */
+        return KRB5KRB_ERR_GENERIC;
+    }
+
+    *size_out = size;
+    return 0;
+}
diff --git a/src/kprop/kpropd.c b/src/kprop/kpropd.c
index c6b8efc28..dd6c697c0 100644
--- a/src/kprop/kpropd.c
+++ b/src/kprop/kpropd.c
@@ -55,6 +55,7 @@
 #include "com_err.h"
 #include "fake-addrinfo.h"
 
+#include <inttypes.h>
 #include <locale.h>
 #include <ctype.h>
 #include <sys/file.h>
@@ -1358,9 +1359,10 @@ static void
 recv_database(krb5_context context, int fd, int database_fd,
               krb5_data *confmsg)
 {
-    krb5_ui_4 database_size, received_size;
+    uint64_t database_size, received_size;
     int n;
     char buf[1024];
+    char dbsize_buf[KPROP_DBSIZE_MAX_BUFSIZ];
     krb5_data inbuf, outbuf;
     krb5_error_code retval;
 
@@ -1382,10 +1384,17 @@ recv_database(krb5_context context, int fd, int database_fd,
                 _("while decoding database size from client"));
         exit(1);
     }
-    memcpy(&database_size, outbuf.data, sizeof(database_size));
+
+    retval = decode_database_size(&outbuf, &database_size);
+    if (retval) {
+        send_error(context, fd, retval, "malformed database size message");
+        com_err(progname, retval,
+                _("malformed database size message from client"));
+        exit(1);
+    }
+
     krb5_free_data_contents(context, &inbuf);
     krb5_free_data_contents(context, &outbuf);
-    database_size = ntohl(database_size);
 
     /* Initialize the initial vector. */
     retval = krb5_auth_con_initivector(context, auth_context);
@@ -1405,7 +1414,7 @@ recv_database(krb5_context context, int fd, int database_fd,
         retval = krb5_read_message(context, &fd, &inbuf);
         if (retval) {
             snprintf(buf, sizeof(buf),
-                     "while reading database block starting at offset %d",
+                     "while reading database block starting at offset %"PRIu64,
                      received_size);
             com_err(progname, retval, "%s", buf);
             send_error(context, fd, retval, buf);
@@ -1416,8 +1425,8 @@ recv_database(krb5_context context, int fd, int database_fd,
         retval = krb5_rd_priv(context, auth_context, &inbuf, &outbuf, NULL);
         if (retval) {
             snprintf(buf, sizeof(buf),
-                     "while decoding database block starting at offset %d",
-                     received_size);
+                     "while decoding database block starting at offset %"
+                     PRIu64, received_size);
             com_err(progname, retval, "%s", buf);
             send_error(context, fd, retval, buf);
             krb5_free_data_contents(context, &inbuf);
@@ -1427,13 +1436,13 @@ recv_database(krb5_context context, int fd, int database_fd,
         krb5_free_data_contents(context, &inbuf);
         if (n < 0) {
             snprintf(buf, sizeof(buf),
-                     "while writing database block starting at offset %d",
+                     "while writing database block starting at offset %"PRIu64,
                      received_size);
             send_error(context, fd, errno, buf);
         } else if ((unsigned int)n != outbuf.length) {
             snprintf(buf, sizeof(buf),
                      "incomplete write while writing database block starting "
-                     "at \noffset %d (%d written, %d expected)",
+                     "at \noffset %"PRIu64" (%d written, %d expected)",
                      received_size, n, outbuf.length);
             send_error(context, fd, KRB5KRB_ERR_GENERIC, buf);
         }
@@ -1444,7 +1453,8 @@ recv_database(krb5_context context, int fd, int database_fd,
     /* OK, we've seen the entire file.  Did we get too many bytes? */
     if (received_size > database_size) {
         snprintf(buf, sizeof(buf),
-                 "Received %d bytes, expected %d bytes for database file",
+                 "Received %"PRIu64" bytes, expected %"PRIu64
+                 " bytes for database file",
                  received_size, database_size);
         send_error(context, fd, KRB5KRB_ERR_GENERIC, buf);
     }
@@ -1454,9 +1464,8 @@ recv_database(krb5_context context, int fd, int database_fd,
 
     /* Create message acknowledging number of bytes received, but
      * don't send it until kdb5_util returns successfully. */
-    database_size = htonl(database_size);
-    inbuf.data = (char *)&database_size;
-    inbuf.length = sizeof(database_size);
+    inbuf = make_data(dbsize_buf, sizeof(dbsize_buf));
+    encode_database_size(database_size, &inbuf);
     retval = krb5_mk_safe(context,auth_context,&inbuf,confmsg,NULL);
     if (retval) {
         com_err(progname, retval, "while encoding # of received bytes");
diff --git a/src/tests/t_kprop.py b/src/tests/t_kprop.py
index d96f7c560..4421a7c35 100755
--- a/src/tests/t_kprop.py
+++ b/src/tests/t_kprop.py
@@ -87,5 +87,39 @@ realm.run([kdb5_util, 'dump', dumpfile])
 realm.run([kprop, '-f', dumpfile, '-P', str(realm.kprop_port()), hostname])
 check_output(kpropd)
 realm.run([kadminl, 'listprincs'], replica3, expected_msg='wakawaka')
+stop_daemon(kpropd)
+
+# This test is too resource-intensive to be included in "make check"
+# by default, but it can be enabled in the environment to test the
+# propagation of databases large enough to require a 12-byte encoding
+# of the database size.
+if 'KPROP_LARGE_DB_TEST' in os.environ:
+    output('Generating >4GB dumpfile\n')
+    with open(dumpfile, 'w') as f:
+        f.write('kdb5_util load_dump version 6\n')
+        f.write('princ\t38\t15\t3\t1\t0\tK/M@KRBTEST.COM\t64\t86400\t0\t0\t0'
+                '\t0\t0\t0\t8\t2\t0100\t9\t8\t0100010000000000\t2\t28'
+                '\tb93e105164625f6372656174696f6e404b5242544553542e434f4d00'
+                '\t1\t1\t18\t62\t2000408c027c250e8cc3b81476414f2214d57c1ce'
+                '38891e29792e87258247c73547df4d5756266931dd6686b62270e6568'
+                '95a31ec66bfe913b4f15226227\t-1;\n')
+        for i in range(1, 20000000):
+            f.write('princ\t38\t21\t1\t1\t0\tp%08d@KRBTEST.COM' % i)
+            f.write('\t0\t86400\t0\t0\t0\t0\t0\t0\t2\t27'
+                    '\td73e1051757365722f61646d696e404b5242544553542e434f4d00'
+                    '\t1\t1\t17\t46'
+                    '\t10009c8ab7b3f89ccf3ca3ad98352a461b7f4f1b0c49'
+                    '5605117591d9ad52ba4da0adef7a902126973ed2bdc3ffbf\t-1;\n')
+    assert os.path.getsize(dumpfile) > 4 * 1024 * 1024 * 1024
+    with open(dumpfile + '.dump_ok', 'w') as f:
+        f.write('\0')
+    conf_large = {'dbmodules': {'db': {'database_name': '$testdir/db.large'}},
+                  'realms': {'$realm': {'iprop_resync_timeout': '3600'}}}
+    large = realm.special_env('large', True, kdc_conf=conf_large)
+    kpropd = realm.start_kpropd(large, ['-d'])
+    realm.run([kprop, '-f', dumpfile, '-P', str(realm.kprop_port()), hostname])
+    check_output(kpropd)
+    realm.run([kadminl, 'getprinc', 'p19999999'], env=large,
+              expected_msg='Principal: p19999999')
 
 success('kprop tests')
-- 
2.37.3