2107240 - FIPS breaks pdftopdf and bannertopdf

2022-07-14 17:48:07 +02:00 · 2022-07-14 17:48:07 +02:00 · 39298d9f55
commit 39298d9f55
parent 38a91e7bdd
2 changed files with 76 additions and 27 deletions
--- a/qpdf-relax.patch
+++ b/qpdf-relax.patch
@ -1,11 +1,11 @@
-diff -up qpdf-10.6.0/libqpdf/QPDF.cc.relax qpdf-10.6.0/libqpdf/QPDF.cc
--- qpdf-10.6.0/libqpdf/QPDF.cc.relax	2022-02-10 12:27:57.433450324 +0100
-+++ qpdf-10.6.0/libqpdf/QPDF.cc	2022-02-10 12:41:11.071905145 +0100
+diff -up qpdf-10.6.3/libqpdf/QPDF.cc.relax qpdf-10.6.3/libqpdf/QPDF.cc
+--- qpdf-10.6.3/libqpdf/QPDF.cc.relax	2022-03-08 18:32:06.000000000 +0100
+++ qpdf-10.6.3/libqpdf/QPDF.cc	2022-07-14 17:08:35.395724219 +0200
@@ -13,6 +13,10 @@
 #include <memory.h>
 #include <regex>
 
-+#ifdef HAVE_GNUTLS
+#ifdef USE_CRYPTO_GNUTLS
 +# include <gnutls/crypto.h>
 +#endif
 +
@ -16,11 +16,11 @@ diff -up qpdf-10.6.0/libqpdf/QPDF.cc.relax qpdf-10.6.0/libqpdf/QPDF.cc
 {
     FileInputSource* fi = new FileInputSource();
     fi->setFilename(filename);
-+#ifdef HAVE_GNUTLS
+#ifdef USE_CRYPTO_GNUTLS
 +    GNUTLS_FIPS140_SET_LAX_MODE();
 +#endif
     processInputSource(PointerHolder<InputSource>(fi), password);
-+#ifdef HAVE_GNUTLS
+#ifdef USE_CRYPTO_GNUTLS
 +    GNUTLS_FIPS140_SET_STRICT_MODE();
 +#endif
 }
@ -30,19 +30,19 @@ diff -up qpdf-10.6.0/libqpdf/QPDF.cc.relax qpdf-10.6.0/libqpdf/QPDF.cc
 {
     FileInputSource* fi = new FileInputSource();
     fi->setFile(description, filep, close_file);
-+#ifdef HAVE_GNUTLS
+#ifdef USE_CRYPTO_GNUTLS
 +    GNUTLS_FIPS140_SET_LAX_MODE();
 +#endif
     processInputSource(PointerHolder<InputSource>(fi), password);
-+#ifdef HAVE_GNUTLS
+#ifdef USE_CRYPTO_GNUTLS
 +    GNUTLS_FIPS140_SET_STRICT_MODE();
 +#endif
 }
 
 void
-diff -up qpdf-10.6.0/libqpdf/QPDF_encryption.cc.relax qpdf-10.6.0/libqpdf/QPDF_encryption.cc
--- qpdf-10.6.0/libqpdf/QPDF_encryption.cc.relax	2022-02-09 11:47:58.000000000 +0100
-+++ qpdf-10.6.0/libqpdf/QPDF_encryption.cc	2022-02-10 12:39:22.295390998 +0100
+diff -up qpdf-10.6.3/libqpdf/QPDF_encryption.cc.relax qpdf-10.6.3/libqpdf/QPDF_encryption.cc
+--- qpdf-10.6.3/libqpdf/QPDF_encryption.cc.relax	2022-03-08 18:32:06.000000000 +0100
+++ qpdf-10.6.3/libqpdf/QPDF_encryption.cc	2022-07-14 17:10:08.450106652 +0200
@@ -1,6 +1,8 @@
 // This file implements methods from the QPDF class that involve
 // encryption.
@ -56,18 +56,40 @@ diff -up qpdf-10.6.0/libqpdf/QPDF_encryption.cc.relax qpdf-10.6.0/libqpdf/QPDF_e
 #include <assert.h>
 #include <string.h>
 
-+#ifdef HAVE_GNUTLS
+#ifdef USE_CRYPTO_GNUTLS
 +# include <gnutls/crypto.h>
 +#endif
 +
 static unsigned char const padding_string[] = {
     0x28, 0xbf, 0x4e, 0x5e, 0x4e, 0x75, 0x8a, 0x41,
     0x64, 0x00, 0x4e, 0x56, 0xff, 0xfa, 0x01, 0x08,
-@@ -1149,6 +1155,12 @@ QPDF::getKeyForObject(
+@@ -379,10 +385,21 @@ QPDF::compute_data_key(std::string const
+         result += "sAlT";
+     }
+ 
+#ifdef USE_CRYPTO_GNUTLS
+    unsigned oldmode = gnutls_fips140_mode_enabled();
+
+    gnutls_fips140_set_mode(GNUTLS_FIPS140_LAX, GNUTLS_FIPS140_SET_MODE_THREAD);
+#endif
+
+     MD5 md5;
+     md5.encodeDataIncrementally(result.c_str(), result.length());
+     MD5::Digest digest;
+     md5.digest(digest);
+
+#ifdef USE_CRYPTO_GNUTLS
+    gnutls_fips140_set_mode(static_cast<gnutls_fips_mode_t>(oldmode), GNUTLS_FIPS140_SET_MODE_THREAD);
+#endif
+
+     return std::string(reinterpret_cast<char*>(digest),
+                        std::min(result.length(), toS(16)));
+ }
+@@ -1149,6 +1166,12 @@ QPDF::getKeyForObject(
 void
 QPDF::decryptString(std::string& str, int objid, int generation)
 {
-+#ifdef HAVE_GNUTLS
+#ifdef USE_CRYPTO_GNUTLS
 +    unsigned oldmode = gnutls_fips140_mode_enabled();
 +
 +    gnutls_fips140_set_mode(GNUTLS_FIPS140_LAX, GNUTLS_FIPS140_SET_MODE_THREAD);
@ -76,22 +98,22 @@ diff -up qpdf-10.6.0/libqpdf/QPDF_encryption.cc.relax qpdf-10.6.0/libqpdf/QPDF_e
     if (objid == 0)
     {
         return;
-@@ -1229,6 +1241,10 @@ QPDF::decryptString(std::string& str, in
+@@ -1229,6 +1252,10 @@ QPDF::decryptString(std::string& str, in
                       QUtil::int_to_string(objid) + " " +
                       QUtil::int_to_string(generation) + ": " + e.what());
     }
 +
-+#ifdef HAVE_GNUTLS
+#ifdef USE_CRYPTO_GNUTLS
 +    gnutls_fips140_set_mode(static_cast<gnutls_fips_mode_t>(oldmode), GNUTLS_FIPS140_SET_MODE_THREAD);
 +#endif
 }
 
 void
-@@ -1239,6 +1255,12 @@ QPDF::decryptStream(PointerHolder<Encryp
+@@ -1239,6 +1266,12 @@ QPDF::decryptStream(PointerHolder<Encryp
                     QPDFObjectHandle& stream_dict,
                     std::vector<std::shared_ptr<Pipeline>>& heap)
 {
-+#ifdef HAVE_GNUTLS
+#ifdef USE_CRYPTO_GNUTLS
 +    unsigned oldmode = gnutls_fips140_mode_enabled();
 +
 +    gnutls_fips140_set_mode(GNUTLS_FIPS140_LAX, GNUTLS_FIPS140_SET_MODE_THREAD);
@ -100,25 +122,25 @@ diff -up qpdf-10.6.0/libqpdf/QPDF_encryption.cc.relax qpdf-10.6.0/libqpdf/QPDF_e
     std::string type;
     if (stream_dict.getKey("/Type").isName())
     {
-@@ -1361,6 +1383,10 @@ QPDF::decryptStream(PointerHolder<Encryp
+@@ -1361,6 +1394,10 @@ QPDF::decryptStream(PointerHolder<Encryp
     }
     pipeline = new_pipeline.get();
     heap.push_back(new_pipeline);
 +
-+#ifdef HAVE_GNUTLS
+#ifdef USE_CRYPTO_GNUTLS
 +    gnutls_fips140_set_mode(static_cast<gnutls_fips_mode_t>(oldmode), GNUTLS_FIPS140_SET_MODE_THREAD);
 +#endif
 }
 
 void
-diff -up qpdf-10.6.0/libqpdf/QPDFWriter.cc.relax qpdf-10.6.0/libqpdf/QPDFWriter.cc
--- qpdf-10.6.0/libqpdf/QPDFWriter.cc.relax	2022-02-09 11:47:58.000000000 +0100
-+++ qpdf-10.6.0/libqpdf/QPDFWriter.cc	2022-02-10 12:27:57.434450320 +0100
+diff -up qpdf-10.6.3/libqpdf/QPDFWriter.cc.relax qpdf-10.6.3/libqpdf/QPDFWriter.cc
+--- qpdf-10.6.3/libqpdf/QPDFWriter.cc.relax	2022-03-08 18:32:06.000000000 +0100
+++ qpdf-10.6.3/libqpdf/QPDFWriter.cc	2022-07-14 17:12:35.972712932 +0200
@@ -25,6 +25,10 @@
 #include <algorithm>
 #include <stdlib.h>
 
-+#ifdef HAVE_GNUTLS
+#ifdef USE_CRYPTO_GNUTLS
 +#include <gnutls/crypto.h>
 +#endif
 +
@ -130,7 +152,7 @@ diff -up qpdf-10.6.0/libqpdf/QPDFWriter.cc.relax qpdf-10.6.0/libqpdf/QPDFWriter.
 {
     this->m->deterministic_id = val;
 +
-+#ifdef HAVE_GNUTLS
+#ifdef USE_CRYPTO_GNUTLS
 +    if (val)
 +	GNUTLS_FIPS140_SET_LAX_MODE();
 +    else
@ -144,7 +166,7 @@ diff -up qpdf-10.6.0/libqpdf/QPDFWriter.cc.relax qpdf-10.6.0/libqpdf/QPDFWriter.
 {
     this->m->preserve_encryption = val;
 +
-+#ifdef HAVE_GNUTLS
+#ifdef USE_CRYPTO_GNUTLS
 +    if (val)
 +	GNUTLS_FIPS140_SET_STRICT_MODE();
 +    else
@ -153,3 +175,27 @@ diff -up qpdf-10.6.0/libqpdf/QPDFWriter.cc.relax qpdf-10.6.0/libqpdf/QPDFWriter.
 }
 
 void
+@@ -2311,12 +2329,23 @@ QPDFWriter::generateID()
+             }
+         }
+ 
+#ifdef USE_CRYPTO_GNUTLS
+        unsigned oldmode = gnutls_fips140_mode_enabled();
+
+        gnutls_fips140_set_mode(GNUTLS_FIPS140_LAX, GNUTLS_FIPS140_SET_MODE_THREAD);
+#endif
+
+         MD5 m;
+         m.encodeString(seed.c_str());
+         MD5::Digest digest;
+         m.digest(digest);
+         result = std::string(reinterpret_cast<char*>(digest),
+                              sizeof(MD5::Digest));
+
+#ifdef USE_CRYPTO_GNUTLS
+        gnutls_fips140_set_mode(static_cast<gnutls_fips_mode_t>(oldmode), GNUTLS_FIPS140_SET_MODE_THREAD);
+#endif
+
+     }
+ 
+     // If /ID already exists, follow the spec: use the original first
--- a/qpdf.spec
+++ b/qpdf.spec
@ -1,7 +1,7 @@
 Summary: Command-line tools and library for transforming PDF files
 Name:    qpdf
 Version: 10.6.3
-Release: 1%{?dist}
+Release: 2%{?dist}
 # MIT: e.g. libqpdf/sha2.c
 # upstream uses ASL 2.0 now, but he allowed other to distribute qpdf under
 # old license (see README)
@ -154,6 +154,9 @@ make check


 %changelog
+* Thu Jul 14 2022 Zdenek Dohnal <zdohnal@redhat.com> - 10.6.3-2
+- 2107240 - FIPS breaks pdftopdf and bannertopdf
+
 * Fri Mar 18 2022 Zdenek Dohnal <zdohnal@redhat.com> - 10.6.3-1
 - 2063429 - qpdf-10.6.3 is available