From fffa3dc40d30155a17487ba3edacb320cb29b873 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matej=20Mu=C5=BEila?= <mmuzila@redhat.com>
Date: Wed, 4 Oct 2023 13:20:28 +0200
Subject: [PATCH] (CVE-2023-40745 CVE-2023-41175) raw2tiff: fix integer
 overflow and bypass of the check (fixes #592)

See merge request libtiff/libtiff!516

(cherry picked from commit 6ac40fefa1763ad08c6e9654d10910e462426f82)
---
 tools/raw2tiff.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/tools/raw2tiff.c b/tools/raw2tiff.c
index 0c6001f7..747ede7f 100644
--- a/tools/raw2tiff.c
+++ b/tools/raw2tiff.c
@@ -36,6 +36,7 @@
 #include <sys/types.h>
 #include <math.h>
 #include <ctype.h>
+#include <limits.h>
 
 #ifdef HAVE_UNISTD_H
 # include <unistd.h>
@@ -101,6 +102,7 @@ main(int argc, char* argv[])
 	int	fd;
 	char	*outfilename = NULL;
 	TIFF	*out;
+	uint32_t temp_limit_check = 0;     /* temp for integer overflow checking*/
 
 	uint32_t row, col, band;
 	int	c;
@@ -217,6 +219,33 @@ main(int argc, char* argv[])
 	if (guessSize(fd, dtype, hdr_size, nbands, swab, &width, &length) < 0)
 		return EXIT_FAILURE;
 
+    /* check for integer overflow in */
+    /* hdr_size + (*width) * (*length) * nbands * depth */
+
+    if ((width == 0) || (length == 0) ){
+        fprintf(stderr, "Too large nbands value specified.\n");
+        return (EXIT_FAILURE);
+    }
+    
+    temp_limit_check = nbands * depth;
+
+    if ( !temp_limit_check || length > ( UINT_MAX / temp_limit_check ) )  {
+        fprintf(stderr, "Too large length size specified.\n");
+        return (EXIT_FAILURE);
+    }
+    temp_limit_check = temp_limit_check * length;
+
+    if ( !temp_limit_check || width > ( UINT_MAX / temp_limit_check ) )  {
+        fprintf(stderr, "Too large width size specified.\n");
+        return (EXIT_FAILURE);
+    }
+    temp_limit_check = temp_limit_check * width;
+
+    if ( !temp_limit_check || hdr_size > ( UINT_MAX - temp_limit_check ) )  {
+        fprintf(stderr, "Too large header size specified.\n");
+        return (EXIT_FAILURE);
+    }
+
 	if (outfilename == NULL)
 		outfilename = argv[optind+1];
 	out = TIFFOpen(outfilename, "w");