rpms/hplip
7
0
Fork 0
Code Issues Pull Requests Releases Activity
7a1aca9995
hplip/hplip-hpcups-integer-overflow.patch

496 lines
17 KiB
Diff
Raw Blame History

commit cc245a1117ae478e916662a7d9bded65b55765b8
Author: Zdenek Dohnal <zdohnal@redhat.com>
Date: Mon May 25 15:27:09 2026 +0200
3.26.4
diff --git a/common/utils.h b/common/utils.h
index 97742bd93..579821078 100644
--- a/common/utils.h
+++ b/common/utils.h
@@ -4,6 +4,10 @@
#include <stdio.h>
#include <stdarg.h>
#include <syslog.h>
+#include <stdint.h>
+#include <limits.h>
+#include <stdbool.h>
+#include <stddef.h>
//#include "hpmud.h"
#define _STRINGIZE(x) #x
@@ -54,6 +58,52 @@ enum UTILS_PLUGIN_LIBRARY_TYPE
};
+/* Safe multiplication helpers - prevent integer overflow */
+
+/**
+ * safe_mul_size_t - Safely multiply two size_t values
+ * @a: First operand
+ * @b: Second operand
+ * @out: Output buffer for result
+ * Returns: true if multiplication succeeded, false if overflow detected
+ */
+static inline bool safe_mul_size_t(size_t a, size_t b, size_t *out)
+{
+ if (!out)
+ return false;
+ if (a == 0 || b == 0)
+ {
+ *out = 0;
+ return true;
+ }
+ if (a > ((size_t)-1) / b)
+ return false;
+ *out = a * b;
+ return true;
+}
+
+/**
+ * safe_mul_int_positive - Safely multiply two positive integers
+ * @a: First operand (must be >= 0)
+ * @b: Second operand (must be >= 0)
+ * @out: Output buffer for result
+ * Returns: true if multiplication succeeded, false if negative input or overflow detected
+ */
+static inline bool safe_mul_int_positive(int a, int b, int *out)
+{
+ if (!out || a < 0 || b < 0)
+ return false;
+ if (a == 0 || b == 0)
+ {
+ *out = 0;
+ return true;
+ }
+ if (a > INT_MAX / b)
+ return false;
+ *out = a * b;
+ return true;
+}
+
#ifdef __cplusplus
extern "C" {
#endif
diff --git a/prnt/hpcups/Hbpl1.cpp b/prnt/hpcups/Hbpl1.cpp
index 74a67f04a..d25b36142 100644
--- a/prnt/hpcups/Hbpl1.cpp
+++ b/prnt/hpcups/Hbpl1.cpp
@@ -130,8 +130,19 @@ DRIVER_ERROR Hbpl1::StartJob(SystemServices *pSystemServices, JobAttributes *pJA
m_PrintinGrayscale = m_JA.integer_values[3]; // cupsInterger3 value
m_pSystemServices = pSystemServices; //Reset and UEL not required
err = m_pHbpl1Wrapper->StartJob((void**)&m_pOutBuffer, &m_OutBuffSize);
- err = sendBuffer(static_cast<const BYTE *>(m_pOutBuffer), m_OutBuffSize);
- m_pHbpl1Wrapper->FreeBuffer(m_pOutBuffer, m_OutBuffSize);
+ if (err != NO_ERROR)
+ {
+ return err;
+ }
+ if (m_pOutBuffer != NULL && m_OutBuffSize > 0)
+ {
+ err = sendBuffer(static_cast<const BYTE *>(m_pOutBuffer), m_OutBuffSize);
+ m_pHbpl1Wrapper->FreeBuffer(m_pOutBuffer, m_OutBuffSize);
+ if (err != NO_ERROR)
+ {
+ return err;
+ }
+ }
if (m_PrintinGrayscale == ON){ //Grayscale = ON
m_ColorMode = COLORTYPE_BOTH;
@@ -156,8 +167,15 @@ DRIVER_ERROR Hbpl1::EndJob()
}
err = m_pHbpl1Wrapper->EndJob((void**)&m_pOutBuffer, &m_OutBuffSize);
- err = sendBuffer(static_cast<const BYTE *>(m_pOutBuffer), m_OutBuffSize);
- m_pHbpl1Wrapper->FreeBuffer(m_pOutBuffer, m_OutBuffSize);
+ if (err != NO_ERROR)
+ {
+ return err;
+ }
+ if (m_pOutBuffer != NULL && m_OutBuffSize > 0)
+ {
+ err = sendBuffer(static_cast<const BYTE *>(m_pOutBuffer), m_OutBuffSize);
+ m_pHbpl1Wrapper->FreeBuffer(m_pOutBuffer, m_OutBuffSize);
+ }
return err;
}
@@ -167,8 +185,15 @@ DRIVER_ERROR Hbpl1::StartPage (JobAttributes *pJA)
DRIVER_ERROR err = NO_ERROR;
err = m_pHbpl1Wrapper->StartPage((void**)&m_pOutBuffer, &m_OutBuffSize);
- err = sendBuffer(static_cast<const BYTE *>(m_pOutBuffer), m_OutBuffSize);
- m_pHbpl1Wrapper->FreeBuffer(m_pOutBuffer, m_OutBuffSize);
+ if (err != NO_ERROR)
+ {
+ return err;
+ }
+ if (m_pOutBuffer != NULL && m_OutBuffSize > 0)
+ {
+ err = sendBuffer(static_cast<const BYTE *>(m_pOutBuffer), m_OutBuffSize);
+ m_pHbpl1Wrapper->FreeBuffer(m_pOutBuffer, m_OutBuffSize);
+ }
return err;
}
@@ -188,29 +213,51 @@ And will reset scan lines counter to zero for next page(HPLIP-1041).
************************************************************************************/
DRIVER_ERROR Hbpl1::FormFeed ()
{
+ DRIVER_ERROR err = NO_ERROR;
if (0 != m_numScanLines && m_pbyStripData && 0 != m_nStripSize)
{
++m_nBandCount;
- m_pHbpl1Wrapper->Encapsulate(m_pbyStripData, m_nStripSize, m_nStripHeight, (void**)&m_pOutBuffer, &m_OutBuffSize);
- sendBuffer(m_pOutBuffer, m_OutBuffSize);
+ err = m_pHbpl1Wrapper->Encapsulate(m_pbyStripData, m_nStripSize, m_nStripHeight, (void**)&m_pOutBuffer, &m_OutBuffSize);
+ if (err != NO_ERROR)
+ return err;
+ if (m_pOutBuffer != NULL && m_OutBuffSize > 0)
+ {
+ err = sendBuffer(m_pOutBuffer, m_OutBuffSize);
+ if (err != NO_ERROR)
+ return err;
+ }
memset(m_pbyStripData,0xFF,m_nStripSize);
}
while(m_nBandCount < m_numStrips)
{
++m_nBandCount;
- m_pHbpl1Wrapper->Encapsulate(m_pbyStripData, m_nStripSize, m_nStripHeight, (void**)&m_pOutBuffer, &m_OutBuffSize);
- sendBuffer(m_pOutBuffer, m_OutBuffSize);
+ err = m_pHbpl1Wrapper->Encapsulate(m_pbyStripData, m_nStripSize, m_nStripHeight, (void**)&m_pOutBuffer, &m_OutBuffSize);
+ if (err != NO_ERROR)
+ return err;
+ if (m_pOutBuffer != NULL && m_OutBuffSize > 0)
+ {
+ err = sendBuffer(m_pOutBuffer, m_OutBuffSize);
+ if (err != NO_ERROR)
+ return err;
+ }
}
- m_pHbpl1Wrapper->EndPage((void**)&m_pOutBuffer, &m_OutBuffSize);
- sendBuffer(m_pOutBuffer, m_OutBuffSize);
+ err = m_pHbpl1Wrapper->EndPage((void**)&m_pOutBuffer, &m_OutBuffSize);
+ if (err != NO_ERROR)
+ return err;
+ if (m_pOutBuffer != NULL && m_OutBuffSize > 0)
+ {
+ err = sendBuffer(m_pOutBuffer, m_OutBuffSize);
+ if (err != NO_ERROR)
+ return err;
+ }
m_pHbpl1Wrapper->FreeBuffer(m_pOutBuffer,m_OutBuffSize);
m_nBandCount = 0;
m_numScanLines = 0;
- return NO_ERROR;
+ return err;
}
diff --git a/prnt/hpcups/Hbpl1_Wrapper.cpp b/prnt/hpcups/Hbpl1_Wrapper.cpp
index 996bfa9f2..9cbf69566 100644
--- a/prnt/hpcups/Hbpl1_Wrapper.cpp
+++ b/prnt/hpcups/Hbpl1_Wrapper.cpp
@@ -81,19 +81,31 @@ void Hbpl1Wrapper::FreeStripBuffer(void)
DRIVER_ERROR Hbpl1Wrapper::StartJob(void **pOutBuffer, int *pOutBufferSize)
{
- DRIVER_ERROR err = NO_ERROR;
-
- m_pPCLmGenerator->StartJob(pOutBuffer,pOutBufferSize,false);
- return err;
+ int ret = m_pPCLmGenerator->StartJob(pOutBuffer,pOutBufferSize,false);
+ if (ret != success)
+ {
+ if (pOutBuffer)
+ *pOutBuffer = NULL;
+ if (pOutBufferSize)
+ *pOutBufferSize = 0;
+ return SYSTEM_ERROR;
+ }
+ return NO_ERROR;
}
DRIVER_ERROR Hbpl1Wrapper::EndJob(void **pOutBuffer, int *pOutBufferSize)
{
- DRIVER_ERROR err = NO_ERROR;
-
- m_pPCLmGenerator->EndJob(pOutBuffer,pOutBufferSize);
- return err;
+ int ret = m_pPCLmGenerator->EndJob(pOutBuffer,pOutBufferSize);
+ if (ret != success)
+ {
+ if (pOutBuffer)
+ *pOutBuffer = NULL;
+ if (pOutBufferSize)
+ *pOutBufferSize = 0;
+ return SYSTEM_ERROR;
+ }
+ return NO_ERROR;
}
@@ -177,8 +189,15 @@ DRIVER_ERROR Hbpl1Wrapper::StartPage(void **pOutBuffer, int *pOutBufferSize)
PCLmPageContent.duplexDisposition = (duplexDispositionEnum)o_Hbpl1->m_JA.args_duplex_mode;
- m_pPCLmGenerator->StartPage(&PCLmSContent,true,pOutBuffer,pOutBufferSize);
-
+ int ret = m_pPCLmGenerator->StartPage(&PCLmSContent,true,pOutBuffer,pOutBufferSize);
+ if (ret != success)
+ {
+ if (pOutBuffer)
+ *pOutBuffer = NULL;
+ if (pOutBufferSize)
+ *pOutBufferSize = 0;
+ return SYSTEM_ERROR;
+ }
return err;
}
@@ -186,9 +205,16 @@ DRIVER_ERROR Hbpl1Wrapper::StartPage(void **pOutBuffer, int *pOutBufferSize)
DRIVER_ERROR Hbpl1Wrapper::EndPage(void **pOutBuffer, int *pOutBufferSize)
{
- DRIVER_ERROR err = NO_ERROR;
- m_pPCLmGenerator->EndPage(pOutBuffer, pOutBufferSize);
- return err;
+ int ret = m_pPCLmGenerator->EndPage(pOutBuffer, pOutBufferSize);
+ if (ret != success)
+ {
+ if (pOutBuffer)
+ *pOutBuffer = NULL;
+ if (pOutBufferSize)
+ *pOutBufferSize = 0;
+ return SYSTEM_ERROR;
+ }
+ return NO_ERROR;
}
@@ -199,9 +225,16 @@ DRIVER_ERROR Hbpl1Wrapper::FormFeed()
DRIVER_ERROR Hbpl1Wrapper::Encapsulate (void *pInBuffer, int inBufferSize, int numLines, void **pOutBuffer, int *pOutBufferSize)
{
- DRIVER_ERROR err = NO_ERROR;
- m_pPCLmGenerator->Encapsulate(pInBuffer, inBufferSize, numLines, pOutBuffer, pOutBufferSize);
- return err;
+ int ret = m_pPCLmGenerator->Encapsulate(pInBuffer, inBufferSize, numLines, pOutBuffer, pOutBufferSize);
+ if (ret != success)
+ {
+ if (pOutBuffer)
+ *pOutBuffer = NULL;
+ if (pOutBufferSize)
+ *pOutBufferSize = 0;
+ return SYSTEM_ERROR;
+ }
+ return NO_ERROR;
}
DRIVER_ERROR Hbpl1Wrapper::SkipLines (int iSkipLines)
diff --git a/prnt/hpcups/genPCLm.cpp b/prnt/hpcups/genPCLm.cpp
index bae3010ad..0e1650cf0 100644
--- a/prnt/hpcups/genPCLm.cpp
+++ b/prnt/hpcups/genPCLm.cpp
@@ -131,6 +131,7 @@
#include <fcntl.h>
#include <assert.h>
#include <math.h>
+#include <limits.h>
#include <zlib.h>
//#include <unistd.h>
@@ -1670,7 +1671,12 @@ int PCLmGenerator::StartPage(PCLmPageSetup *PCLmPageContent, void **pOutBuffer,
destColorSpace=PCLmPageContent->dstColorSpaceSpefication;
// Calculate how large the output buffer needs to be based upon the page specifications
- int tmp_outBuffSize=mediaWidthInPixels*currStripHeight*dstNumComponents;
+ int tmp_outBuffSize=0;
+ if(!safe_mul_int_positive(mediaWidthInPixels,currStripHeight,&tmp_outBuffSize) ||
+ !safe_mul_int_positive(tmp_outBuffSize,dstNumComponents,&tmp_outBuffSize))
+ {
+ return(errorOutAndCleanUp());
+ }
if(tmp_outBuffSize>currOutBuffSize)
{
@@ -1738,7 +1744,14 @@ int PCLmGenerator::StartPage(PCLmPageSetup *PCLmPageContent, void **pOutBuffer,
{
// We need to pad the scratchBuffer size to allow for compression expansion (RLE can create
// compressed segments that are slightly larger than the source.
- scratchBuffer=(ubyte*)malloc(currStripHeight*mediaWidthInPixels*srcNumComponents*2);
+ size_t scratchSize=0;
+ if(currStripHeight<=0 || mediaWidthInPixels<=0 || srcNumComponents<=0 ||
+ !safe_mul_size_t((size_t)currStripHeight, (size_t)mediaWidthInPixels, &scratchSize) ||
+ !safe_mul_size_t(scratchSize, (size_t)srcNumComponents, &scratchSize) ||
+ !safe_mul_size_t(scratchSize, 2u, &scratchSize))
+ return(errorOutAndCleanUp());
+
+ scratchBuffer=(ubyte*)malloc(scratchSize);
if(!scratchBuffer)
return(errorOutAndCleanUp());
/*if(DebugIt2)
@@ -1794,7 +1807,9 @@ int PCLmGenerator::SkipLines(int iSkipLines)
int PCLmGenerator::Encapsulate(void *pInBuffer, int inBufferSize, int thisHeight, void **pOutBuffer, int *iOutBufferSize)
{
int result=0, numCompBytes;
- int scanlineWidth=mediaWidthInPixels*srcNumComponents;
+ int scanlineWidth=0;
+ if(!safe_mul_int_positive(mediaWidthInPixels, srcNumComponents, &scanlineWidth))
+ return(errorOutAndCleanUp());
int compSize;
// int numLinesThisCall=inBufferSize/(currSourceWidth*srcNumComponents);
int numLinesThisCall=thisHeight;
@@ -1884,7 +1899,8 @@ int PCLmGenerator::Encapsulate(void *pInBuffer, int inBufferSize, int thisHeigh
{
colorConvertSource(sourceColorSpace, grayScale, (ubyte*)localInBuffer, currSourceWidth, numLinesThisCall);
// Adjust the scanline width accordingly
- scanlineWidth = mediaWidthInPixels * dstNumComponents;
+ if(!safe_mul_int_positive(mediaWidthInPixels, dstNumComponents, &scanlineWidth))
+ return(errorOutAndCleanUp());
}
if(leftMarginInPix)
@@ -1899,7 +1915,11 @@ int PCLmGenerator::Encapsulate(void *pInBuffer, int inBufferSize, int thisHeigh
}
#ifdef SUPPORT_WHITE_STRIPS
- bool whiteStrip=isWhiteStrip(pInBuffer, thisHeight*currSourceWidth*srcNumComponents);
+ int whiteStripLen=0;
+ if(!safe_mul_int_positive(thisHeight, currSourceWidth, &whiteStripLen) ||
+ !safe_mul_int_positive(whiteStripLen, srcNumComponents, &whiteStripLen))
+ return(errorOutAndCleanUp());
+ bool whiteStrip=isWhiteStrip(pInBuffer, whiteStripLen);
if(DebugIt2)
{
if(whiteStrip){
@@ -1918,9 +1938,14 @@ int PCLmGenerator::Encapsulate(void *pInBuffer, int inBufferSize, int thisHeigh
if(firstStrip && topMarginInPix)
{
ubyte whitePt=0xff;
+ size_t tmpStripSize=0;
+ if(!safe_mul_size_t((size_t)scanlineWidth, (size_t)topMarginInPix, &tmpStripSize))
+ return(errorOutAndCleanUp());
- ubyte *tmpStrip=(ubyte*)malloc(scanlineWidth*topMarginInPix);
- memset(tmpStrip,whitePt,scanlineWidth*topMarginInPix);
+ ubyte *tmpStrip=(ubyte*)malloc(tmpStripSize);
+ if(!tmpStrip)
+ return(errorOutAndCleanUp());
+ memset(tmpStrip,whitePt,tmpStripSize);
for(sint32 stripCntr=0; stripCntr<numFullInjectedStrips;stripCntr++)
@@ -1985,17 +2010,25 @@ int PCLmGenerator::Encapsulate(void *pInBuffer, int inBufferSize, int thisHeigh
}
else if(currCompressionDisposition==compressFlate)
{
- uint32 len=numLinesThisCall*scanlineWidth;
+ int sourceLen=0;
+ if(!safe_mul_int_positive(numLinesThisCall, scanlineWidth, &sourceLen))
+ return(errorOutAndCleanUp());
+ uint32 len=(uint32)sourceLen;
uLongf destSize=len;
if(firstStrip && topMarginInPix)
{
ubyte whitePt=0xff;
+ size_t tmpStripSize=0;
+ if(!safe_mul_size_t((size_t)scanlineWidth, (size_t)topMarginInPix, &tmpStripSize))
+ return(errorOutAndCleanUp());
// We need to inject a blank image-strip with a height==topMarginInPix
- ubyte *tmpStrip=(ubyte*)malloc(scanlineWidth*topMarginInPix);
+ ubyte *tmpStrip=(ubyte*)malloc(tmpStripSize);
+ if(!tmpStrip)
+ return(errorOutAndCleanUp());
uLongf tmpDestSize=destSize;
- memset(tmpStrip,whitePt,scanlineWidth*topMarginInPix);
+ memset(tmpStrip,whitePt,tmpStripSize);
for(sint32 stripCntr=0; stripCntr<numFullInjectedStrips;stripCntr++)
{
@@ -2013,12 +2046,12 @@ int PCLmGenerator::Encapsulate(void *pInBuffer, int inBufferSize, int thisHeigh
if(newStripPtr)
{
- result=compress((Bytef*)scratchBuffer,&destSize,(const Bytef*)newStripPtr,scanlineWidth*numLinesThisCall);
+ result=compress((Bytef*)scratchBuffer,&destSize,(const Bytef*)newStripPtr,(uLong)sourceLen);
if(DebugIt2)
writeOutputFile(destSize, scratchBuffer, m_pPCLmSSettings->user_name);
if(DebugIt2)
{
- dbglog("Allocated zlib dest buffer of size %d\n",numLinesThisCall*scanlineWidth);
+ dbglog("Allocated zlib dest buffer of size %d\n",sourceLen);
dbglog("zlib compression return result=%d, compSize=%d\n",result,(int)destSize);
}
free(newStripPtr);
@@ -2026,12 +2059,12 @@ int PCLmGenerator::Encapsulate(void *pInBuffer, int inBufferSize, int thisHeigh
}
else
{
- result=compress((Bytef*)scratchBuffer, &destSize, (const Bytef*)localInBuffer, scanlineWidth*numLinesThisCall);
+ result=compress((Bytef*)scratchBuffer, &destSize, (const Bytef*)localInBuffer, (uLong)sourceLen);
if(DebugIt2)
writeOutputFile(destSize, scratchBuffer, m_pPCLmSSettings->user_name);
if(DebugIt2)
{
- dbglog("Allocated zlib dest buffer of size %d\n",numLinesThisCall*scanlineWidth);
+ dbglog("Allocated zlib dest buffer of size %d\n",sourceLen);
dbglog("zlib compression return result=%d, compSize=%d\n",result,(int)destSize);
}
}
@@ -2040,14 +2073,23 @@ int PCLmGenerator::Encapsulate(void *pInBuffer, int inBufferSize, int thisHeigh
else if(currCompressionDisposition==compressRLE)
{
+ int sourceLen=0;
+ if(!safe_mul_int_positive(numLinesThisCall, scanlineWidth, &sourceLen))
+ return(errorOutAndCleanUp());
+
if(firstStrip && topMarginInPix)
{
ubyte whitePt=0xff;
+ size_t tmpStripSize=0;
+ if(!safe_mul_size_t((size_t)scanlineWidth, (size_t)topMarginInPix, &tmpStripSize))
+ return(errorOutAndCleanUp());
// We need to inject a blank image-strip with a height==topMarginInPix
- ubyte *tmpStrip=(ubyte*)malloc(scanlineWidth*topMarginInPix);
- memset(tmpStrip,whitePt,scanlineWidth*topMarginInPix);
+ ubyte *tmpStrip=(ubyte*)malloc(tmpStripSize);
+ if(!tmpStrip)
+ return(errorOutAndCleanUp());
+ memset(tmpStrip,whitePt,tmpStripSize);
for(sint32 stripCntr=0; stripCntr<numFullInjectedStrips;stripCntr++)
{
@@ -2067,16 +2109,16 @@ int PCLmGenerator::Encapsulate(void *pInBuffer, int inBufferSize, int thisHeigh
if(newStripPtr)
{
- compSize=HPRunLen_Encode((ubyte*)newStripPtr, scratchBuffer, scanlineWidth*numLinesThisCall);
+ compSize=HPRunLen_Encode((ubyte*)newStripPtr, scratchBuffer, sourceLen);
free(newStripPtr);
newStripPtr = NULL;
}
else
- compSize=HPRunLen_Encode((ubyte*)localInBuffer, scratchBuffer, scanlineWidth*numLinesThisCall);
+ compSize=HPRunLen_Encode((ubyte*)localInBuffer, scratchBuffer, sourceLen);
if(DebugIt2)
{
- dbglog("Allocated rle dest buffer of size %d\n",numLinesThisCall*scanlineWidth);
+ dbglog("Allocated rle dest buffer of size %d\n",sourceLen);
dbglog("rle compression return size=%d=%d\n",result,(int)compSize);
}
injectRLEStrip(scratchBuffer, compSize, mediaWidthInPixels, numLinesThisCall, destColorSpace, whiteStrip);
Reference in New Issue View Git Blame Copy Permalink