From 9207290587ea8c8703486d60877731228eed80ce Mon Sep 17 00:00:00 2001 From: Stephan Bergmann Date: Fri, 24 Aug 2018 10:27:01 +0200 Subject: [PATCH 4/5] iRelated rhbz#1618703: Properly handle failure en-/decoding PDF file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ...when e.g. FIPS mode makes the various calls to rtl_cipher_initARCFOUR fail. Reviewed-on: https://gerrit.libreoffice.org/59543 Tested-by: Jenkins Reviewed-by: Stephan Bergmann (cherry picked from commit 185a14525f114e58b48236284ed8e8644bc40e48) Reviewed-on: https://gerrit.libreoffice.org/59573 Reviewed-by: Caolán McNamara Tested-by: Caolán McNamara (cherry picked from commit 68ffc5c83ca73c58439b7c9935283541f007db44) Conflicts: filter/source/pdf/impdialog.cxx sdext/source/pdfimport/pdfparse/pdfentries.cxx vcl/source/gdi/pdfwriter_impl2.cxx Change-Id: Id1b2222249c151470e233ab814b21228f3a8b561 --- filter/source/pdf/impdialog.cxx | 7 +++ .../source/pdfimport/pdfparse/pdfentries.cxx | 42 +++++++++++----- vcl/source/gdi/pdfwriter_impl2.cxx | 48 +++++++++++-------- 3 files changed, 67 insertions(+), 30 deletions(-) diff --git a/filter/source/pdf/impdialog.cxx b/filter/source/pdf/impdialog.cxx index 82aaf012505a..736ac0c15f26 100644 --- a/filter/source/pdf/impdialog.cxx +++ b/filter/source/pdf/impdialog.cxx @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -1351,6 +1352,12 @@ IMPL_LINK_NOARG(ImpPDFTabSecurityPage, ClickmaPbSetPwdHdl, Button*, void) mbHaveOwnerPassword = !aOwnerPW.isEmpty(); mxPreparedPasswords = vcl::PDFWriter::InitEncryption( aOwnerPW, aUserPW, true ); + if (!mxPreparedPasswords.is()) { + OUString msg; + ErrorHandler::GetErrorString(ERRCODE_IO_NOTSUPPORTED, msg); //TOOD: handle failure + ScopedVclPtrInstance(this, msg, VclMessageType::Error)->Execute(); + return; + } if( mbHaveOwnerPassword ) { diff --git a/sdext/source/pdfimport/pdfparse/pdfentries.cxx b/sdext/source/pdfimport/pdfparse/pdfentries.cxx index 16563868f25c..074fb669c8da 100644 --- a/sdext/source/pdfimport/pdfparse/pdfentries.cxx +++ b/sdext/source/pdfimport/pdfparse/pdfentries.cxx @@ -1159,9 +1159,13 @@ static bool check_user_password( const OString& rPwd, PDFFileImplData* pData ) { // see PDF reference 1.4 Algorithm 3.4 // encrypt pad string - rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode, - aKey, nKeyLen, - nullptr, 0 ); + if (rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode, + aKey, nKeyLen, + nullptr, 0 ) + != rtl_Cipher_E_None) + { + return false; //TODO: differentiate "failed to decrypt" from "wrong password" + } rtl_cipher_encodeARCFOUR( pData->m_aCipher, nPadString, sizeof( nPadString ), nEncryptedEntry, sizeof( nEncryptedEntry ) ); bValid = (memcmp( nEncryptedEntry, pData->m_aUEntry, 32 ) == 0); @@ -1172,8 +1176,12 @@ static bool check_user_password( const OString& rPwd, PDFFileImplData* pData ) rtl_digest_updateMD5( pData->m_aDigest, nPadString, sizeof( nPadString ) ); rtl_digest_updateMD5( pData->m_aDigest, pData->m_aDocID.getStr(), pData->m_aDocID.getLength() ); rtl_digest_getMD5( pData->m_aDigest, nEncryptedEntry, sizeof(nEncryptedEntry) ); - rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode, - aKey, sizeof(aKey), nullptr, 0 ); + if (rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode, + aKey, sizeof(aKey), nullptr, 0 ) + != rtl_Cipher_E_None) + { + return false; //TODO: differentiate "failed to decrypt" from "wrong password" + } rtl_cipher_encodeARCFOUR( pData->m_aCipher, nEncryptedEntry, 16, nEncryptedEntry, 16 ); // encrypt in place @@ -1183,8 +1191,12 @@ static bool check_user_password( const OString& rPwd, PDFFileImplData* pData ) for( sal_uInt32 j = 0; j < sizeof(aTempKey); j++ ) aTempKey[j] = static_cast( aKey[j] ^ i ); - rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode, - aTempKey, sizeof(aTempKey), nullptr, 0 ); + if (rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode, + aTempKey, sizeof(aTempKey), nullptr, 0 ) + != rtl_Cipher_E_None) + { + return false; //TODO: differentiate "failed to decrypt" from "wrong password" + } rtl_cipher_encodeARCFOUR( pData->m_aCipher, nEncryptedEntry, 16, nEncryptedEntry, 16 ); // encrypt in place @@ -1230,8 +1242,12 @@ bool PDFFile::setupDecryptionData( const OString& rPwd ) const sal_uInt32 nKeyLen = password_to_key( rPwd, aKey, m_pData.get(), true ); if( m_pData->m_nStandardRevision == 2 ) { - rtl_cipher_initARCFOUR( m_pData->m_aCipher, rtl_Cipher_DirectionDecode, - aKey, nKeyLen, nullptr, 0 ); + if (rtl_cipher_initARCFOUR( m_pData->m_aCipher, rtl_Cipher_DirectionDecode, + aKey, nKeyLen, nullptr, 0 ) + != rtl_Cipher_E_None) + { + return false; //TODO: differentiate "failed to decrypt" from "wrong password" + } rtl_cipher_decodeARCFOUR( m_pData->m_aCipher, m_pData->m_aOEntry, 32, nPwd, 32 ); @@ -1244,8 +1260,12 @@ bool PDFFile::setupDecryptionData( const OString& rPwd ) const sal_uInt8 nTempKey[ENCRYPTION_KEY_LEN]; for( unsigned int j = 0; j < sizeof(nTempKey); j++ ) nTempKey[j] = sal_uInt8(aKey[j] ^ i); - rtl_cipher_initARCFOUR( m_pData->m_aCipher, rtl_Cipher_DirectionDecode, - nTempKey, nKeyLen, nullptr, 0 ); + if (rtl_cipher_initARCFOUR( m_pData->m_aCipher, rtl_Cipher_DirectionDecode, + nTempKey, nKeyLen, nullptr, 0 ) + != rtl_Cipher_E_None) + { + return false; //TODO: differentiate "failed to decrypt" from "wrong password" + } rtl_cipher_decodeARCFOUR( m_pData->m_aCipher, nPwd, 32, nPwd, 32 ); // decrypt inplace diff --git a/vcl/source/gdi/pdfwriter_impl2.cxx b/vcl/source/gdi/pdfwriter_impl2.cxx index e4f567d6bfd9..e85cf15e4395 100644 --- a/vcl/source/gdi/pdfwriter_impl2.cxx +++ b/vcl/source/gdi/pdfwriter_impl2.cxx @@ -1443,29 +1443,39 @@ bool PDFWriterImpl::computeODictionaryValue( const sal_uInt8* i_pPaddedOwnerPass //Step 4, the key is in nMD5Sum //step 5 already done, data is in i_pPaddedUserPassword //step 6 - rtl_cipher_initARCFOUR( aCipher, rtl_Cipher_DirectionEncode, - nMD5Sum, i_nKeyLength , nullptr, 0 ); - // encrypt the user password using the key set above - rtl_cipher_encodeARCFOUR( aCipher, i_pPaddedUserPassword, ENCRYPTED_PWD_SIZE, // the data to be encrypted - &io_rOValue[0], sal_Int32(io_rOValue.size()) ); //encrypted data - //Step 7, only if 128 bit - if( i_nKeyLength == SECUR_128BIT_KEY ) + if (rtl_cipher_initARCFOUR( aCipher, rtl_Cipher_DirectionEncode, + nMD5Sum, i_nKeyLength , nullptr, 0 ) + == rtl_Cipher_E_None) { - sal_uInt32 i, y; - sal_uInt8 nLocalKey[ SECUR_128BIT_KEY ]; // 16 = 128 bit key - - for( i = 1; i <= 19; i++ ) // do it 19 times, start with 1 + // encrypt the user password using the key set above + rtl_cipher_encodeARCFOUR( aCipher, i_pPaddedUserPassword, ENCRYPTED_PWD_SIZE, // the data to be encrypted + &io_rOValue[0], sal_Int32(io_rOValue.size()) ); //encrypted data + //Step 7, only if 128 bit + if( i_nKeyLength == SECUR_128BIT_KEY ) { - for( y = 0; y < sizeof( nLocalKey ); y++ ) - nLocalKey[y] = (sal_uInt8)( nMD5Sum[y] ^ i ); - - rtl_cipher_initARCFOUR( aCipher, rtl_Cipher_DirectionEncode, - nLocalKey, SECUR_128BIT_KEY, nullptr, 0 ); //destination data area, on init can be NULL - rtl_cipher_encodeARCFOUR( aCipher, &io_rOValue[0], sal_Int32(io_rOValue.size()), // the data to be encrypted - &io_rOValue[0], sal_Int32(io_rOValue.size()) ); // encrypted data, can be the same as the input, encrypt "in place" - //step 8, store in class data member + sal_uInt32 i, y; + sal_uInt8 nLocalKey[ SECUR_128BIT_KEY ]; // 16 = 128 bit key + + for( i = 1; i <= 19; i++ ) // do it 19 times, start with 1 + { + for( y = 0; y < sizeof( nLocalKey ); y++ ) + nLocalKey[y] = (sal_uInt8)( nMD5Sum[y] ^ i ); + + if (rtl_cipher_initARCFOUR( aCipher, rtl_Cipher_DirectionEncode, + nLocalKey, SECUR_128BIT_KEY, nullptr, 0 ) //destination data area, on init can be NULL + != rtl_Cipher_E_None) + { + bSuccess = false; + break; + } + rtl_cipher_encodeARCFOUR( aCipher, &io_rOValue[0], sal_Int32(io_rOValue.size()), // the data to be encrypted + &io_rOValue[0], sal_Int32(io_rOValue.size()) ); // encrypted data, can be the same as the input, encrypt "in place" + //step 8, store in class data member + } } } + else + bSuccess = false; } else bSuccess = false; -- 2.17.1