Do strip and tile size calculations in unsigned arithmetic, and then
complain if the result overflows signed int32, because callers of these
functions expect signed results (tsize_t is signed).  CVE-2012-2088

NB: must be applied after libtiff-subsampling.patch to avoid fuzz issues.


diff -Naur tiff-3.9.4.orig/libtiff/tif_strip.c tiff-3.9.4/libtiff/tif_strip.c
--- tiff-3.9.4.orig/libtiff/tif_strip.c	2010-06-08 14:50:43.000000000 -0400
+++ tiff-3.9.4/libtiff/tif_strip.c	2012-06-27 12:37:55.054788399 -0400
@@ -107,6 +107,7 @@
 TIFFVStripSize(TIFF* tif, uint32 nrows)
 {
 	TIFFDirectory *td = &tif->tif_dir;
+	uint32 stripsize;
 
 	if (nrows == (uint32) -1)
 		nrows = td->td_imagelength;
@@ -122,7 +123,7 @@
 		 * YCbCr data for the extended image.
 		 */
 		uint16 ycbcrsubsampling[2];
-		tsize_t w, scanline, samplingarea;
+		uint32 w, scanline, samplingarea;
 
 		TIFFGetFieldDefaulted(tif, TIFFTAG_YCBCRSUBSAMPLING,
 				      ycbcrsubsampling + 0,
@@ -141,13 +142,27 @@
 		nrows = TIFFroundup(nrows, ycbcrsubsampling[1]);
 		/* NB: don't need TIFFhowmany here 'cuz everything is rounded */
 		scanline = multiply(tif, nrows, scanline, "TIFFVStripSize");
-		return ((tsize_t)
-		    summarize(tif, scanline,
-			      multiply(tif, 2, scanline / samplingarea,
-				       "TIFFVStripSize"), "TIFFVStripSize"));
+		/* a zero anywhere in here means overflow, must return zero */
+		if (scanline > 0) {
+			uint32 extra =
+			    multiply(tif, 2, scanline / samplingarea,
+				     "TIFFVStripSize");
+			if (extra > 0)
+				stripsize = summarize(tif, scanline, extra,
+						      "TIFFVStripSize");
+			else
+				stripsize = 0;
+		} else
+			stripsize = 0;
 	} else
-		return ((tsize_t) multiply(tif, nrows, TIFFScanlineSize(tif),
-					   "TIFFVStripSize"));
+		stripsize = multiply(tif, nrows, TIFFScanlineSize(tif),
+				     "TIFFVStripSize");
+	/* Because tsize_t is signed, we might have conversion overflow */
+	if (((tsize_t) stripsize) < 0) {
+		TIFFErrorExt(tif->tif_clientdata, tif->tif_name, "Integer overflow in %s", "TIFFVStripSize");
+		stripsize = 0;
+	}
+	return (tsize_t) stripsize;
 }
 
 
diff -Naur tiff-3.9.4.orig/libtiff/tif_tile.c tiff-3.9.4/libtiff/tif_tile.c
--- tiff-3.9.4.orig/libtiff/tif_tile.c	2010-06-08 14:50:43.000000000 -0400
+++ tiff-3.9.4/libtiff/tif_tile.c	2012-06-27 12:37:55.055788446 -0400
@@ -174,7 +174,7 @@
 TIFFTileRowSize(TIFF* tif)
 {
 	TIFFDirectory *td = &tif->tif_dir;
-	tsize_t rowsize;
+	uint32 rowsize;
 	
 	if (td->td_tilelength == 0 || td->td_tilewidth == 0)
 		return ((tsize_t) 0);
@@ -193,7 +193,7 @@
 TIFFVTileSize(TIFF* tif, uint32 nrows)
 {
 	TIFFDirectory *td = &tif->tif_dir;
-	tsize_t tilesize;
+	uint32 tilesize;
 
 	if (td->td_tilelength == 0 || td->td_tilewidth == 0 ||
 	    td->td_tiledepth == 0)
@@ -209,12 +209,12 @@
 		 * horizontal/vertical subsampling area include
 		 * YCbCr data for the extended image.
 		 */
-		tsize_t w =
+		uint32 w =
 		    TIFFroundup(td->td_tilewidth, td->td_ycbcrsubsampling[0]);
-		tsize_t rowsize =
+		uint32 rowsize =
 		    TIFFhowmany8(multiply(tif, w, td->td_bitspersample,
 					  "TIFFVTileSize"));
-		tsize_t samplingarea =
+		uint32 samplingarea =
 		    td->td_ycbcrsubsampling[0]*td->td_ycbcrsubsampling[1];
 		if (samplingarea == 0) {
 			TIFFErrorExt(tif->tif_clientdata, tif->tif_name, "Invalid YCbCr subsampling");
@@ -223,15 +223,27 @@
 		nrows = TIFFroundup(nrows, td->td_ycbcrsubsampling[1]);
 		/* NB: don't need TIFFhowmany here 'cuz everything is rounded */
 		tilesize = multiply(tif, nrows, rowsize, "TIFFVTileSize");
-		tilesize = summarize(tif, tilesize,
-				     multiply(tif, 2, tilesize / samplingarea,
-					      "TIFFVTileSize"),
+		/* a zero anywhere in here means overflow, must return zero */
+		if (tilesize > 0) {
+			uint32 extra =
+			    multiply(tif, 2, tilesize / samplingarea,
 				     "TIFFVTileSize");
+			if (extra > 0)
+				tilesize = summarize(tif, tilesize, extra,
+						     "TIFFVTileSize");
+			else
+				tilesize = 0;
+		}
 	} else
 		tilesize = multiply(tif, nrows, TIFFTileRowSize(tif),
 				    "TIFFVTileSize");
-	return ((tsize_t)
-	    multiply(tif, tilesize, td->td_tiledepth, "TIFFVTileSize"));
+	tilesize = multiply(tif, tilesize, td->td_tiledepth, "TIFFVTileSize");
+	/* Because tsize_t is signed, we might have conversion overflow */
+	if (((tsize_t) tilesize) < 0) {
+		TIFFErrorExt(tif->tif_clientdata, tif->tif_name, "Integer overflow in %s", "TIFFVTileSize");
+		tilesize = 0;
+	}
+	return (tsize_t) tilesize;
 }
 
 /*