thunderbird/D161379.diff

diff -up comm/third_party/moz.build.D161379.diff comm/third_party/moz.build
--- comm/third_party/moz.build.D161379.diff	2022-10-14 21:45:15.000000000 +0200
+++ comm/third_party/moz.build	2022-11-10 11:49:44.194016978 +0100
@@ -11,9 +11,11 @@ if CONFIG["TB_LIBOTR_PREBUILT"]:
 
 if CONFIG["MZLA_LIBRNP"]:
     DIRS += [
-        "botan",
         "bzip2",
         "json-c",
         "rnp",
         "zlib",
     ]
+    if CONFIG["MZLA_LIBRNP_BACKEND"] == "botan":
+        DIRS += [ "botan" ]
+
diff -up comm/third_party/openpgp.configure.D161379.diff comm/third_party/openpgp.configure
--- comm/third_party/openpgp.configure.D161379.diff	2022-11-10 11:49:37.605024129 +0100
+++ comm/third_party/openpgp.configure	2022-11-10 11:49:44.194016978 +0100
@@ -199,16 +199,136 @@ with only_when(in_tree_librnp):
         set_config("MZLA_BZIP2_CFLAGS", bzip2_flags.cflags)
         set_config("MZLA_BZIP2_LIBS", bzip2_flags.ldflags)
 
-    # BOTAN --with-system-botan
-    system_lib_option(
-        "--with-system-botan",
-        help="Use system Botan for librnp (located with pkgconfig)",
-    )
-
-    botan_pkg = pkg_check_modules(
-        "MZLA_BOTAN", "botan-2 >= 2.8.0", when="--with-system-botan"
-    )
-    set_config("MZLA_SYSTEM_BOTAN", depends_if(botan_pkg)(lambda _: True))
+    # librnp crypto backend selection
+    option("--with-librnp-backend",
+           help="Build librnp with the selected backend: {botan, openssl}",
+           default="botan")
+
+    @depends("--with-librnp-backend")
+    def librnp_backend(backend):
+        allowed = ("botan", "openssl")
+        if backend[0] in allowed:
+            return backend[0]
+        else:
+            die(f"Unsupported librnp backend {backend[0]}.")
+
+    set_config("MZLA_LIBRNP_BACKEND", librnp_backend)
+
+    @depends(librnp_backend)
+    def rnp_botan(backend):
+        return backend == "botan"
+
+    @depends(librnp_backend)
+    def rnp_openssl(backend):
+        return backend == "openssl"
+
+    # Botan backend (--with-system-botan)
+    with only_when(rnp_botan):
+        system_lib_option(
+            "--with-system-botan",
+            help="Use system Botan for librnp (located with pkgconfig)",
+        )
+
+        botan_pkg = pkg_check_modules(
+            "MZLA_BOTAN", "botan-2 >= 2.8.0", when="--with-system-botan"
+        )
+        set_config("MZLA_SYSTEM_BOTAN", depends_if(botan_pkg)(lambda _: True))
+
+
+    # OpenSSL backend
+    with only_when(rnp_openssl):
+        option(
+            "--with-openssl",
+            nargs=1,
+            help="OpenSSL library prefix (when not found by pkgconfig)"
+        )
+        openssl_pkg = pkg_check_modules(
+            "MZLA_LIBRNP_OPENSSL",
+            "openssl > 1.1.1",
+            allow_missing=True,
+            config=False
+        )
+        @depends_if("--with-openssl", openssl_pkg)
+        @imports(_from="os.path", _import="isdir")
+        @imports(_from="os.path", _import="join")
+        def openssl_flags(openssl_prefix, openssl_pkg):
+            if openssl_prefix:
+                openssl_prefix = openssl_prefix[0]
+                include = join(openssl_prefix, "include")
+                lib = join(openssl_prefix, "lib")
+                if not isdir(lib):
+                    lib = join(openssl_prefix, "lib64")
+                if isdir(include) and isdir(lib):
+                    log.info(f"Using OpenSSL at {openssl_prefix}.")
+                    return namespace(
+                        cflags=(f"-I{include}",),
+                        ldflags=(f"-L{lib}", "-lssl", "-lcrypto"),
+                    )
+            if openssl_pkg:
+                return namespace(
+                    cflags=openssl_pkg.cflags,
+                    ldflags=openssl_pkg.libs,
+                )
+        set_config("MZLA_LIBRNP_OPENSSL_CFLAGS", openssl_flags.cflags)
+        set_config("MZLA_LIBRNP_OPENSSL_LIBS", openssl_flags.ldflags)
+
+
+        @depends(c_compiler, openssl_flags)
+        @imports(_from="textwrap", _import="dedent")
+        def openssl_version(compiler, openssl_flags):
+            log.info("Checking for OpenSSL >= 1.1.1")
+            if openssl_flags is None:
+                die("OpenSSL not found. Must be locatable with pkg-config or use --with-openssl.")
+
+            def ossl_hexver(hex_str):
+                # See opensshlv.h for description of OPENSSL_VERSION_NUMBER
+                MIN_OSSL_VER = 0x1010100f  # Version 1.1.1
+                ver_as_int = int(hex_str[:-1], 16)
+                ossl_major = (ver_as_int & 0xf0000000) >> 28
+                ossl_minor = (ver_as_int & 0x0ff00000) >> 20
+                ossl_fix = (ver_as_int & 0x000ff000) >> 12
+                ossl_patch = chr(96 + (ver_as_int & 0x00000ff0) >> 4)  # as a letter a-z
+                ver_as_str = f"{ossl_major}.{ossl_minor}.{ossl_fix}{ossl_patch}"
+                if ver_as_int < MIN_OSSL_VER:
+                    die(f"OpenSSL version {ver_as_str} is too old.")
+                return ver_as_str
+
+            check = dedent(
+                """\
+            #include <openssl/opensslv.h>
+            #ifdef OPENSSL_VERSION_STR
+            OPENSSL_VERSION_STR
+            #elif defined(OPENSSL_VERSION_NUMBER)
+            OPENSSL_VERSION_NUMBER
+            #else
+            #error Unable to determine OpenSSL version.
+            #endif
+                """
+            )
+            result = try_preprocess(
+                compiler.wrapper
+                + [compiler.compiler]
+                + compiler.flags
+                + list(openssl_flags.cflags),
+                "C",
+                check
+            )
+            if result:
+                openssl_ver = result.splitlines()[-1]
+                if openssl_ver.startswith("0x"):
+                    # OpenSSL 1.x.x - like 0x1010107fL
+                    openssl_ver = ossl_hexver(openssl_ver)
+                else:
+                    # OpenSSL 3.x.x - quoted version like "3.0.7"
+                    openssl_ver = openssl_ver.replace('"', "")
+                    major_version = openssl_ver.split(".")[0]
+                    if major_version != "3":
+                        die("Unrecognized OpenSSL version {openssl_version} found. Require >= 1.1.1 or 3.x.x")
+
+                log.info(f"Found OpenSSL {openssl_ver}.")
+                return openssl_ver
+
+        set_config("MZLA_LIBRNP_OPENSSL_VERSION", openssl_version)
 
     # Checks for building librnp itself
     # =================================
diff -up comm/third_party/rnp/moz.build.D161379.diff comm/third_party/rnp/moz.build
--- comm/third_party/rnp/moz.build.D161379.diff	2022-11-10 11:49:43.682017534 +0100
+++ comm/third_party/rnp/moz.build	2022-11-10 11:51:22.878909880 +0100
@@ -36,17 +36,53 @@ if CONFIG["CC_TYPE"] == "clang-cl":
         "/EHs",
     ]
 
+LOCAL_INCLUDES = [
+    "include",
+    "src",
+    "src/common",
+    "src/lib",
+]
+
+IQuote(
+    "{}/src/lib".format(OBJDIR),
+    "{}/src/lib".format(SRCDIR),
+)
+
+# Set up defines for src/lib/config.h
 rnp_defines = {
     "HAVE_BZLIB_H": True,
     "HAVE_ZLIB_H": True,
-    "CRYPTO_BACKEND_OPENSSL": True,
-    "ENABLE_AEAD": True,
-    "ENABLE_TWOFISH": True,
-    "ENABLE_BRAINPOOL": True,
     "ENABLE_IDEA": True,
     "PACKAGE_BUGREPORT": '"https://bugzilla.mozilla.org/enter_bug.cgi?product=Thunderbird"',
     "PACKAGE_STRING": '"rnp {}"'.format(CONFIG["MZLA_LIBRNP_FULL_VERSION"])
 }
+if CONFIG["MZLA_LIBRNP_BACKEND"] == "botan":
+    LOCAL_INCLUDES += ["!../botan/build/include"]
+    if CONFIG["MZLA_SYSTEM_BOTAN"]:
+        CXXFLAGS += CONFIG["MZLA_BOTAN_CFLAGS"]
+
+    rnp_defines.update({
+        "CRYPTO_BACKEND_BOTAN": True,
+        "ENABLE_AEAD": True,
+        "ENABLE_TWOFISH": True,
+        "ENABLE_BRAINPOOL": True,
+    })
+elif CONFIG["MZLA_LIBRNP_BACKEND"] == "openssl":
+    CXXFLAGS += CONFIG["MZLA_LIBRNP_OPENSSL_CFLAGS"]
+    OS_LIBS += CONFIG["MZLA_LIBRNP_OPENSSL_LIBS"]
+
+    rnp_defines.update({
+        "CRYPTO_BACKEND_OPENSSL": True,
+        # Not supported with RNP+OpenSSL https://github.com/rnpgp/rnp/issues/1642
+        "ENABLE_AEAD": False,
+        # Not supported by OpenSSL https://github.com/openssl/openssl/issues/2046
+        "ENABLE_TWOFISH": False,
+        # Supported, but not with RHEL's OpenSSL, disabled for now;
+        "ENABLE_BRAINPOOL": False,
+    })
+    if CONFIG["MZLA_LIBRNP_OPENSSL_VERSION"][0] == "3":
+        rnp_defines["CRYPTO_BACKEND_OPENSSL3"] = True
+
 GeneratedFile(
     "src/lib/config.h",
     script="/comm/python/rocbuild/process_cmake_define_files.py",
@@ -57,23 +93,6 @@ GeneratedFile(
     ],
 )
 
-LOCAL_INCLUDES = [
-    "include",
-    "src",
-    "src/common",
-    "src/lib",
-]
-
-IQuote(
-    "{}/src/lib".format(OBJDIR),
-    "{}/src/lib".format(SRCDIR),
-)
-
-if CONFIG["MZLA_SYSTEM_BOTAN"]:
-    CXXFLAGS += CONFIG["MZLA_BOTAN_CFLAGS"]
-else:
-    LOCAL_INCLUDES += ["!../botan/build/include"]
-
 if CONFIG["MOZ_SYSTEM_ZLIB"]:
     CXXFLAGS += CONFIG["MOZ_ZLIB_CFLAGS"]
 else:
@@ -109,29 +128,16 @@ SOURCES += [
     "src/common/time-utils.cpp",
     "src/lib/crypto.cpp",
     "src/lib/crypto/backend_version.cpp",
-    "src/lib/crypto/bn.cpp",
     "src/lib/crypto/cipher.cpp",
-    "src/lib/crypto/cipher_botan.cpp",
-    "src/lib/crypto/dsa.cpp",
-    "src/lib/crypto/ec.cpp",
     "src/lib/crypto/ec_curves.cpp",
-    "src/lib/crypto/ecdh.cpp",
     "src/lib/crypto/ecdh_utils.cpp",
-    "src/lib/crypto/ecdsa.cpp",
-    "src/lib/crypto/eddsa.cpp",
-    "src/lib/crypto/elgamal.cpp",
-    "src/lib/crypto/hash.cpp",
     "src/lib/crypto/hash_common.cpp",
     "src/lib/crypto/hash_sha1cd.cpp",
-    "src/lib/crypto/mem.cpp",
     "src/lib/crypto/mpi.cpp",
-    "src/lib/crypto/rng.cpp",
-    "src/lib/crypto/rsa.cpp",
     "src/lib/crypto/s2k.cpp",
     "src/lib/crypto/sha1cd/sha1.c",
     "src/lib/crypto/sha1cd/ubc_check.c",
     "src/lib/crypto/signatures.cpp",
-    "src/lib/crypto/symmetric.cpp",
     "src/lib/fingerprint.cpp",
     "src/lib/generate-key.cpp",
     "src/lib/json-utils.cpp",
@@ -159,4 +165,40 @@ SOURCES += [
     "src/librepgp/stream-write.cpp",
 ]
 
+if CONFIG["MZLA_LIBRNP_BACKEND"] == "botan":
+    SOURCES += [
+        "src/lib/crypto/bn.cpp",
+        "src/lib/crypto/cipher_botan.cpp",
+        "src/lib/crypto/dsa.cpp",
+        "src/lib/crypto/ec.cpp",
+        "src/lib/crypto/ecdh.cpp",
+        "src/lib/crypto/ecdsa.cpp",
+        "src/lib/crypto/eddsa.cpp",
+        "src/lib/crypto/elgamal.cpp",
+        "src/lib/crypto/hash.cpp",
+        "src/lib/crypto/mem.cpp",
+        "src/lib/crypto/rng.cpp",
+        "src/lib/crypto/rsa.cpp",
+        "src/lib/crypto/symmetric.cpp",
+    ]
+if CONFIG["MZLA_LIBRNP_BACKEND"] == "openssl":
+    SOURCES += [
+        "src/lib/crypto/bn_ossl.cpp",
+        "src/lib/crypto/cipher_ossl.cpp",
+        "src/lib/crypto/dl_ossl.cpp",
+        "src/lib/crypto/dsa_ossl.cpp",
+        "src/lib/crypto/ec_ossl.cpp",
+        "src/lib/crypto/ecdh_ossl.cpp",
+        "src/lib/crypto/ecdsa_ossl.cpp",
+        "src/lib/crypto/eddsa_ossl.cpp",
+        "src/lib/crypto/elgamal_ossl.cpp",
+        "src/lib/crypto/hash_crc24.cpp",
+        "src/lib/crypto/hash_ossl.cpp",
+        "src/lib/crypto/mem_ossl.cpp",
+        "src/lib/crypto/rng_ossl.cpp",
+        "src/lib/crypto/rsa_ossl.cpp",
+        "src/lib/crypto/s2k_ossl.cpp",
+        "src/lib/crypto/symmetric_ossl.cpp",
+    ]
+
 DIRS += ["src/rnp", "src/rnpkeys"]