qpdf/qpdf-relax.patch

diff -up qpdf-11.3.0/libqpdf/QPDF.cc.relax qpdf-11.3.0/libqpdf/QPDF.cc
--- qpdf-11.3.0/libqpdf/QPDF.cc.relax	2023-02-25 22:24:41.000000000 +0100
+++ qpdf-11.3.0/libqpdf/QPDF.cc	2023-03-02 08:19:28.200881077 +0100
@@ -13,6 +13,10 @@
 #include <string.h>
 #include <vector>
 
+#ifdef USE_CRYPTO_GNUTLS
+# include <gnutls/crypto.h>
+#endif
+
 #include <qpdf/BufferInputSource.hh>
 #include <qpdf/FileInputSource.hh>
 #include <qpdf/OffsetInputSource.hh>
@@ -264,7 +268,13 @@ void
 QPDF::processFile(char const* filename, char const* password)
 {
     FileInputSource* fi = new FileInputSource(filename);
+#ifdef USE_CRYPTO_GNUTLS
+    GNUTLS_FIPS140_SET_LAX_MODE();
+#endif
     processInputSource(std::shared_ptr<InputSource>(fi), password);
+#ifdef USE_CRYPTO_GNUTLS
+    GNUTLS_FIPS140_SET_STRICT_MODE();
+#endif
 }
 
 void
@@ -272,7 +282,13 @@ QPDF::processFile(
     char const* description, FILE* filep, bool close_file, char const* password)
 {
     FileInputSource* fi = new FileInputSource(description, filep, close_file);
+#ifdef USE_CRYPTO_GNUTLS
+    GNUTLS_FIPS140_SET_LAX_MODE();
+#endif
     processInputSource(std::shared_ptr<InputSource>(fi), password);
+#ifdef USE_CRYPTO_GNUTLS
+    GNUTLS_FIPS140_SET_STRICT_MODE();
+#endif
 }
 
 void
diff -up qpdf-11.3.0/libqpdf/QPDF_encryption.cc.relax qpdf-11.3.0/libqpdf/QPDF_encryption.cc
--- qpdf-11.3.0/libqpdf/QPDF_encryption.cc.relax	2023-02-25 22:24:41.000000000 +0100
+++ qpdf-11.3.0/libqpdf/QPDF_encryption.cc	2023-03-02 08:19:28.200881077 +0100
@@ -3,6 +3,8 @@
 
 #include <qpdf/assert_debug.h>
 
+#include <qpdf/qpdf-config.h>
+
 #include <qpdf/QPDF.hh>
 
 #include <qpdf/QPDFExc.hh>
@@ -19,6 +21,10 @@
 #include <algorithm>
 #include <string.h>
 
+#ifdef USE_CRYPTO_GNUTLS
+# include <gnutls/crypto.h>
+#endif
+
 static unsigned char const padding_string[] = {
     0x28, 0xbf, 0x4e, 0x5e, 0x4e, 0x75, 0x8a, 0x41, 0x64, 0x00, 0x4e,
     0x56, 0xff, 0xfa, 0x01, 0x08, 0x2e, 0x2e, 0x00, 0xb6, 0xd0, 0x68,
@@ -367,10 +373,21 @@ QPDF::compute_data_key(
         result += "sAlT";
     }
 
+#ifdef USE_CRYPTO_GNUTLS
+    unsigned oldmode = gnutls_fips140_mode_enabled();
+
+    gnutls_fips140_set_mode(GNUTLS_FIPS140_LAX, GNUTLS_FIPS140_SET_MODE_THREAD);
+#endif
+
     MD5 md5;
     md5.encodeDataIncrementally(result.c_str(), result.length());
     MD5::Digest digest;
     md5.digest(digest);
+
+#ifdef USE_CRYPTO_GNUTLS
+    gnutls_fips140_set_mode(static_cast<gnutls_fips_mode_t>(oldmode), GNUTLS_FIPS140_SET_MODE_THREAD);
+#endif
+
     return std::string(
         reinterpret_cast<char*>(digest), std::min(result.length(), toS(16)));
 }
@@ -1076,6 +1093,12 @@ QPDF::getKeyForObject(
 void
 QPDF::decryptString(std::string& str, QPDFObjGen const& og)
 {
+#ifdef USE_CRYPTO_GNUTLS
+    unsigned oldmode = gnutls_fips140_mode_enabled();
+
+    gnutls_fips140_set_mode(GNUTLS_FIPS140_LAX, GNUTLS_FIPS140_SET_MODE_THREAD);
+#endif
+
     if (!og.isIndirect()) {
         return;
     }
@@ -1142,6 +1165,10 @@ QPDF::decryptString(std::string& str, QP
             "error decrypting string for object " + og.unparse() + ": " +
             e.what());
     }
+
+#ifdef USE_CRYPTO_GNUTLS
+    gnutls_fips140_set_mode(static_cast<gnutls_fips_mode_t>(oldmode), GNUTLS_FIPS140_SET_MODE_THREAD);
+#endif
 }
 
 void
@@ -1154,6 +1181,12 @@ QPDF::decryptStream(
     QPDFObjectHandle& stream_dict,
     std::vector<std::shared_ptr<Pipeline>>& heap)
 {
+#ifdef USE_CRYPTO_GNUTLS
+    unsigned oldmode = gnutls_fips140_mode_enabled();
+
+    gnutls_fips140_set_mode(GNUTLS_FIPS140_LAX, GNUTLS_FIPS140_SET_MODE_THREAD);
+#endif
+
     std::string type;
     if (stream_dict.getKey("/Type").isName()) {
         type = stream_dict.getKey("/Type").getName();
@@ -1262,6 +1295,10 @@ QPDF::decryptStream(
     }
     pipeline = new_pipeline.get();
     heap.push_back(new_pipeline);
+
+#ifdef USE_CRYPTO_GNUTLS
+    gnutls_fips140_set_mode(static_cast<gnutls_fips_mode_t>(oldmode), GNUTLS_FIPS140_SET_MODE_THREAD);
+#endif
 }
 
 void
diff -up qpdf-11.3.0/libqpdf/QPDFWriter.cc.relax qpdf-11.3.0/libqpdf/QPDFWriter.cc
--- qpdf-11.3.0/libqpdf/QPDFWriter.cc.relax	2023-02-25 22:24:41.000000000 +0100
+++ qpdf-11.3.0/libqpdf/QPDFWriter.cc	2023-03-02 08:56:48.559215611 +0100
@@ -26,6 +26,10 @@
 #include <stdexcept>
 #include <stdlib.h>
 
+#ifdef USE_CRYPTO_GNUTLS
+#include <gnutls/crypto.h>
+#endif
+
 QPDFWriter::ProgressReporter::~ProgressReporter()
 {
     // Must be explicit and not inline -- see QPDF_DLL_CLASS in
@@ -343,6 +347,13 @@ void
 QPDFWriter::setDeterministicID(bool val)
 {
     this->m->deterministic_id = val;
+
+#ifdef USE_CRYPTO_GNUTLS
+    if (val)
+	GNUTLS_FIPS140_SET_LAX_MODE();
+    else
+	GNUTLS_FIPS140_SET_STRICT_MODE();
+#endif
 }
 
 void
@@ -363,6 +374,13 @@ void
 QPDFWriter::setPreserveEncryption(bool val)
 {
     this->m->preserve_encryption = val;
+
+#ifdef USE_CRYPTO_GNUTLS
+    if (val)
+	GNUTLS_FIPS140_SET_STRICT_MODE();
+    else
+	GNUTLS_FIPS140_SET_LAX_MODE();
+#endif
 }
 
 void
@@ -2100,12 +2118,23 @@ QPDFWriter::generateID()
             }
         }
 
+#ifdef USE_CRYPTO_GNUTLS
+        unsigned oldmode = gnutls_fips140_mode_enabled();
+
+        gnutls_fips140_set_mode(GNUTLS_FIPS140_LAX, GNUTLS_FIPS140_SET_MODE_THREAD);
+#endif
+
         MD5 m;
         m.encodeString(seed.c_str());
         MD5::Digest digest;
         m.digest(digest);
         result =
             std::string(reinterpret_cast<char*>(digest), sizeof(MD5::Digest));
+
+#ifdef USE_CRYPTO_GNUTLS
+        gnutls_fips140_set_mode(static_cast<gnutls_fips_mode_t>(oldmode), GNUTLS_FIPS140_SET_MODE_THREAD);
+#endif
+
     }
 
     // If /ID already exists, follow the spec: use the original first