xref: /aoo41x/main/oox/source/core/binarycodec.cxx (revision ca5ec200)
1 /**************************************************************
2  *
3  * Licensed to the Apache Software Foundation (ASF) under one
4  * or more contributor license agreements.  See the NOTICE file
5  * distributed with this work for additional information
6  * regarding copyright ownership.  The ASF licenses this file
7  * to you under the Apache License, Version 2.0 (the
8  * "License"); you may not use this file except in compliance
9  * with the License.  You may obtain a copy of the License at
10  *
11  *   http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing,
14  * software distributed under the License is distributed on an
15  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16  * KIND, either express or implied.  See the License for the
17  * specific language governing permissions and limitations
18  * under the License.
19  *
20  *************************************************************/
21 
22 
23 
24 #include "oox/core/binarycodec.hxx"
25 
26 #include <algorithm>
27 #include <string.h>
28 #include "oox/helper/attributelist.hxx"
29 
30 #include <comphelper/sequenceashashmap.hxx>
31 #include <comphelper/docpasswordhelper.hxx>
32 
33 using namespace ::com::sun::star;
34 
35 namespace oox {
36 namespace core {
37 
38 // ============================================================================
39 
40 namespace {
41 
42 /** Rotates rnValue left by nBits bits. */
43 template< typename Type >
lclRotateLeft(Type & rnValue,size_t nBits)44 inline void lclRotateLeft( Type& rnValue, size_t nBits )
45 {
46     OSL_ENSURE( nBits < sizeof( Type ) * 8, "lclRotateLeft - rotation count overflow" );
47     rnValue = static_cast< Type >( (rnValue << nBits) | (rnValue >> (sizeof( Type ) * 8 - nBits)) );
48 }
49 
50 /** Rotates the lower nWidth bits of rnValue left by nBits bits. */
51 template< typename Type >
lclRotateLeft(Type & rnValue,size_t nBits,size_t nWidth)52 inline void lclRotateLeft( Type& rnValue, size_t nBits, size_t nWidth )
53 {
54     OSL_ENSURE( (nBits < nWidth) && (nWidth < sizeof( Type ) * 8), "lclRotateLeft - rotation count overflow" );
55     Type nMask = static_cast< Type >( (1UL << nWidth) - 1 );
56     rnValue = static_cast< Type >(
57         ((rnValue << nBits) | ((rnValue & nMask) >> (nWidth - nBits))) & nMask );
58 }
59 
lclGetLen(const sal_uInt8 * pnPassData,sal_Int32 nBufferSize)60 sal_Int32 lclGetLen( const sal_uInt8* pnPassData, sal_Int32 nBufferSize )
61 {
62     sal_Int32 nLen = 0;
63     while( (nLen < nBufferSize) && pnPassData[ nLen ] ) ++nLen;
64     return nLen;
65 }
66 
lclGetKey(const sal_uInt8 * pnPassData,sal_Int32 nBufferSize)67 sal_uInt16 lclGetKey( const sal_uInt8* pnPassData, sal_Int32 nBufferSize )
68 {
69     sal_Int32 nLen = lclGetLen( pnPassData, nBufferSize );
70     if( nLen <= 0 ) return 0;
71 
72     sal_uInt16 nKey = 0;
73     sal_uInt16 nKeyBase = 0x8000;
74     sal_uInt16 nKeyEnd = 0xFFFF;
75     const sal_uInt8* pnChar = pnPassData + nLen - 1;
76     for( sal_Int32 nIndex = 0; nIndex < nLen; ++nIndex, --pnChar )
77     {
78         sal_uInt8 cChar = *pnChar & 0x7F;
79         for( size_t nBit = 0; nBit < 8; ++nBit )
80         {
81             lclRotateLeft( nKeyBase, 1 );
82             if( nKeyBase & 1 ) nKeyBase ^= 0x1020;
83             if( cChar & 1 ) nKey ^= nKeyBase;
84             cChar >>= 1;
85             lclRotateLeft( nKeyEnd, 1 );
86             if( nKeyEnd & 1 ) nKeyEnd ^= 0x1020;
87         }
88     }
89     return nKey ^ nKeyEnd;
90 }
91 
lclGetHash(const sal_uInt8 * pnPassData,sal_Int32 nBufferSize)92 sal_uInt16 lclGetHash( const sal_uInt8* pnPassData, sal_Int32 nBufferSize )
93 {
94     sal_Int32 nLen = lclGetLen( pnPassData, nBufferSize );
95 
96     sal_uInt16 nHash = static_cast< sal_uInt16 >( nLen );
97     if( nLen > 0 )
98         nHash ^= 0xCE4B;
99 
100     const sal_uInt8* pnChar = pnPassData;
101     for( sal_Int32 nIndex = 0; nIndex < nLen; ++nIndex, ++pnChar )
102     {
103         sal_uInt16 cChar = *pnChar;
104         size_t nRot = static_cast< size_t >( (nIndex + 1) % 15 );
105         lclRotateLeft( cChar, nRot, 15 );
106         nHash ^= cChar;
107     }
108     return nHash;
109 }
110 
111 } // namespace
112 
113 // ============================================================================
114 
getPasswordHash(const AttributeList & rAttribs,sal_Int32 nElement)115 /*static*/ sal_uInt16 CodecHelper::getPasswordHash( const AttributeList& rAttribs, sal_Int32 nElement )
116 {
117     sal_Int32 nPasswordHash = rAttribs.getIntegerHex( nElement, 0 );
118     OSL_ENSURE( (0 <= nPasswordHash) && (nPasswordHash <= SAL_MAX_UINT16), "CodecHelper::getPasswordHash - invalid password hash" );
119     return static_cast< sal_uInt16 >( ((0 <= nPasswordHash) && (nPasswordHash <= SAL_MAX_UINT16)) ? nPasswordHash : 0 );
120 }
121 
122 // ============================================================================
123 
BinaryCodec_XOR(CodecType eCodecType)124 BinaryCodec_XOR::BinaryCodec_XOR( CodecType eCodecType ) :
125     meCodecType( eCodecType ),
126     mnOffset( 0 ),
127     mnBaseKey( 0 ),
128     mnHash( 0 )
129 {
130     (void)memset( mpnKey, 0, sizeof( mpnKey ) );
131 }
132 
~BinaryCodec_XOR()133 BinaryCodec_XOR::~BinaryCodec_XOR()
134 {
135     (void)memset( mpnKey, 0, sizeof( mpnKey ) );
136     mnBaseKey = mnHash = 0;
137 }
138 
initKey(const sal_uInt8 pnPassData[16])139 void BinaryCodec_XOR::initKey( const sal_uInt8 pnPassData[ 16 ] )
140 {
141     // calculate base key and hash from passed password
142     mnBaseKey = lclGetKey( pnPassData, 16 );
143     mnHash = lclGetHash( pnPassData, 16 );
144 
145      static const sal_uInt8 spnFillChars[] =
146     {
147         0xBB, 0xFF, 0xFF, 0xBA,
148         0xFF, 0xFF, 0xB9, 0x80,
149         0x00, 0xBE, 0x0F, 0x00,
150         0xBF, 0x0F, 0x00
151     };
152 
153     (void)memcpy( mpnKey, pnPassData, 16 );
154     sal_Int32 nIndex;
155     sal_Int32 nLen = lclGetLen( pnPassData, 16 );
156     const sal_uInt8* pnFillChar = spnFillChars;
157     for( nIndex = nLen; nIndex < static_cast< sal_Int32 >( sizeof( mpnKey ) ); ++nIndex, ++pnFillChar )
158         mpnKey[ nIndex ] = *pnFillChar;
159 
160     // rotation of key values is application dependent
161     size_t nRotateSize = 0;
162     switch( meCodecType )
163     {
164         case CODEC_WORD:    nRotateSize = 7;    break;
165         case CODEC_EXCEL:   nRotateSize = 2;    break;
166         // compiler will warn, if new codec type is introduced and not handled here
167     }
168 
169     // use little-endian base key to create key array
170     sal_uInt8 pnBaseKeyLE[ 2 ];
171     pnBaseKeyLE[ 0 ] = static_cast< sal_uInt8 >( mnBaseKey );
172     pnBaseKeyLE[ 1 ] = static_cast< sal_uInt8 >( mnBaseKey >> 8 );
173     sal_uInt8* pnKeyChar = mpnKey;
174     for( nIndex = 0; nIndex < static_cast< sal_Int32 >( sizeof( mpnKey ) ); ++nIndex, ++pnKeyChar )
175     {
176         *pnKeyChar ^= pnBaseKeyLE[ nIndex & 1 ];
177         lclRotateLeft( *pnKeyChar, nRotateSize );
178     }
179 }
180 
initCodec(const uno::Sequence<beans::NamedValue> & aData)181 bool BinaryCodec_XOR::initCodec( const uno::Sequence< beans::NamedValue >& aData )
182 {
183     bool bResult = sal_False;
184 
185     ::comphelper::SequenceAsHashMap aHashData( aData );
186     uno::Sequence< sal_Int8 > aKey = aHashData.getUnpackedValueOrDefault( ::rtl::OUString( RTL_CONSTASCII_USTRINGPARAM( "XOR95EncryptionKey" ) ), uno::Sequence< sal_Int8 >() );
187 
188     if ( aKey.getLength() == 16 )
189     {
190         (void)memcpy( mpnKey, aKey.getConstArray(), 16 );
191         bResult = sal_True;
192 
193         mnBaseKey = (sal_uInt16)aHashData.getUnpackedValueOrDefault( ::rtl::OUString( RTL_CONSTASCII_USTRINGPARAM( "XOR95BaseKey" ) ), (sal_Int16)0 );
194         mnHash = (sal_uInt16)aHashData.getUnpackedValueOrDefault( ::rtl::OUString( RTL_CONSTASCII_USTRINGPARAM( "XOR95PasswordHash" ) ), (sal_Int16)0 );
195     }
196     else
197         OSL_ENSURE( sal_False, "Unexpected key size!\n" );
198 
199     return bResult;
200 }
201 
getEncryptionData()202 uno::Sequence< beans::NamedValue > BinaryCodec_XOR::getEncryptionData()
203 {
204     ::comphelper::SequenceAsHashMap aHashData;
205     aHashData[ ::rtl::OUString( RTL_CONSTASCII_USTRINGPARAM( "XOR95EncryptionKey" ) ) ] <<= uno::Sequence<sal_Int8>( (sal_Int8*)mpnKey, 16 );
206     aHashData[ ::rtl::OUString( RTL_CONSTASCII_USTRINGPARAM( "XOR95BaseKey" ) ) ] <<= (sal_Int16)mnBaseKey;
207     aHashData[ ::rtl::OUString( RTL_CONSTASCII_USTRINGPARAM( "XOR95PasswordHash" ) ) ] <<= (sal_Int16)mnHash;
208 
209     return aHashData.getAsConstNamedValueList();
210 }
211 
verifyKey(sal_uInt16 nKey,sal_uInt16 nHash) const212 bool BinaryCodec_XOR::verifyKey( sal_uInt16 nKey, sal_uInt16 nHash ) const
213 {
214     return (nKey == mnBaseKey) && (nHash == mnHash);
215 }
216 
startBlock()217 void BinaryCodec_XOR::startBlock()
218 {
219     mnOffset = 0;
220 }
221 
decode(sal_uInt8 * pnDestData,const sal_uInt8 * pnSrcData,sal_Int32 nBytes)222 bool BinaryCodec_XOR::decode( sal_uInt8* pnDestData, const sal_uInt8* pnSrcData, sal_Int32 nBytes )
223 {
224     const sal_uInt8* pnCurrKey = mpnKey + mnOffset;
225     const sal_uInt8* pnKeyLast = mpnKey + 0x0F;
226 
227     // switch/case outside of the for loop (performance)
228     const sal_uInt8* pnSrcDataEnd = pnSrcData + nBytes;
229     switch( meCodecType )
230     {
231         case CODEC_WORD:
232         {
233             for( ; pnSrcData < pnSrcDataEnd; ++pnSrcData, ++pnDestData )
234             {
235                 sal_uInt8 nData = *pnSrcData ^ *pnCurrKey;
236                 if( (*pnSrcData != 0) && (nData != 0) )
237                     *pnDestData = nData;
238                 if( pnCurrKey < pnKeyLast ) ++pnCurrKey; else pnCurrKey = mpnKey;
239             }
240         }
241         break;
242         case CODEC_EXCEL:
243         {
244             for( ; pnSrcData < pnSrcDataEnd; ++pnSrcData, ++pnDestData )
245             {
246                 *pnDestData = *pnSrcData;
247                 lclRotateLeft( *pnDestData, 3 );
248                 *pnDestData ^= *pnCurrKey;
249                 if( pnCurrKey < pnKeyLast ) ++pnCurrKey; else pnCurrKey = mpnKey;
250             }
251         }
252         break;
253         // compiler will warn, if new codec type is introduced and not handled here
254     }
255 
256     // update offset and leave
257     return skip( nBytes );
258 }
259 
skip(sal_Int32 nBytes)260 bool BinaryCodec_XOR::skip( sal_Int32 nBytes )
261 {
262     mnOffset = static_cast< sal_Int32 >( (mnOffset + nBytes) & 0x0F );
263     return true;
264 }
265 
266 // ============================================================================
267 
BinaryCodec_RCF()268 BinaryCodec_RCF::BinaryCodec_RCF()
269 {
270     mhCipher = rtl_cipher_create( rtl_Cipher_AlgorithmARCFOUR, rtl_Cipher_ModeStream );
271     OSL_ENSURE( mhCipher != 0, "BinaryCodec_RCF::BinaryCodec_RCF - cannot create cipher" );
272 
273     mhDigest = rtl_digest_create( rtl_Digest_AlgorithmMD5 );
274     OSL_ENSURE( mhDigest != 0, "BinaryCodec_RCF::BinaryCodec_RCF - cannot create digest" );
275 
276     (void)memset( mpnDigestValue, 0, sizeof( mpnDigestValue ) );
277     (void)memset (mpnUnique, 0, sizeof(mpnUnique));
278 }
279 
~BinaryCodec_RCF()280 BinaryCodec_RCF::~BinaryCodec_RCF()
281 {
282     (void)memset( mpnDigestValue, 0, sizeof( mpnDigestValue ) );
283     (void)memset (mpnUnique, 0, sizeof(mpnUnique));
284     rtl_digest_destroy( mhDigest );
285     rtl_cipher_destroy( mhCipher );
286 }
287 
initCodec(const uno::Sequence<beans::NamedValue> & aData)288 bool BinaryCodec_RCF::initCodec( const uno::Sequence< beans::NamedValue >& aData )
289 {
290     bool bResult = sal_False;
291 
292     ::comphelper::SequenceAsHashMap aHashData( aData );
293     uno::Sequence< sal_Int8 > aKey = aHashData.getUnpackedValueOrDefault( ::rtl::OUString( RTL_CONSTASCII_USTRINGPARAM( "STD97EncryptionKey" ) ), uno::Sequence< sal_Int8 >() );
294 
295     if ( aKey.getLength() == RTL_DIGEST_LENGTH_MD5 )
296     {
297         (void)memcpy( mpnDigestValue, aKey.getConstArray(), RTL_DIGEST_LENGTH_MD5 );
298         uno::Sequence< sal_Int8 > aUniqueID = aHashData.getUnpackedValueOrDefault( ::rtl::OUString( RTL_CONSTASCII_USTRINGPARAM( "STD97UniqueID" ) ), uno::Sequence< sal_Int8 >() );
299         if ( aUniqueID.getLength() == 16 )
300         {
301             (void)memcpy( mpnUnique, aUniqueID.getConstArray(), 16 );
302             bResult = sal_False;
303         }
304         else
305             OSL_ENSURE( sal_False, "Unexpected document ID!\n" );
306     }
307     else
308         OSL_ENSURE( sal_False, "Unexpected key size!\n" );
309 
310     return bResult;
311 }
312 
getEncryptionData()313 uno::Sequence< beans::NamedValue > BinaryCodec_RCF::getEncryptionData()
314 {
315     ::comphelper::SequenceAsHashMap aHashData;
316     aHashData[ ::rtl::OUString( RTL_CONSTASCII_USTRINGPARAM( "STD97EncryptionKey" ) ) ] <<= uno::Sequence< sal_Int8 >( (sal_Int8*)mpnDigestValue, RTL_DIGEST_LENGTH_MD5 );
317     aHashData[ ::rtl::OUString( RTL_CONSTASCII_USTRINGPARAM( "STD97UniqueID" ) ) ] <<= uno::Sequence< sal_Int8 >( (sal_Int8*)mpnUnique, 16 );
318 
319     return aHashData.getAsConstNamedValueList();
320 }
321 
initKey(const sal_uInt16 pnPassData[16],const sal_uInt8 pnSalt[16])322 void BinaryCodec_RCF::initKey( const sal_uInt16 pnPassData[ 16 ], const sal_uInt8 pnSalt[ 16 ] )
323 {
324     uno::Sequence< sal_Int8 > aKey = ::comphelper::DocPasswordHelper::GenerateStd97Key( pnPassData, uno::Sequence< sal_Int8 >( (sal_Int8*)pnSalt, 16 ) );
325     // Fill raw digest of above updates into DigestValue.
326 
327     if ( aKey.getLength() == sizeof(mpnDigestValue) )
328         (void)memcpy ( mpnDigestValue, (const sal_uInt8*)aKey.getConstArray(), sizeof(mpnDigestValue) );
329     else
330         memset( mpnDigestValue, 0, sizeof(mpnDigestValue) );
331 
332     (void)memcpy( mpnUnique, pnSalt, 16 );
333 }
334 
verifyKey(const sal_uInt8 pnVerifier[16],const sal_uInt8 pnVerifierHash[16])335 bool BinaryCodec_RCF::verifyKey( const sal_uInt8 pnVerifier[ 16 ], const sal_uInt8 pnVerifierHash[ 16 ] )
336 {
337     if( !startBlock( 0 ) )
338         return false;
339 
340     sal_uInt8 pnDigest[ RTL_DIGEST_LENGTH_MD5 ];
341     sal_uInt8 pnBuffer[ 64 ];
342 
343     // decode salt data into buffer
344     rtl_cipher_decode( mhCipher, pnVerifier, 16, pnBuffer, sizeof( pnBuffer ) );
345 
346     pnBuffer[ 16 ] = 0x80;
347     (void)memset( pnBuffer + 17, 0, sizeof( pnBuffer ) - 17 );
348     pnBuffer[ 56 ] = 0x80;
349 
350     // fill raw digest of buffer into digest
351     rtl_digest_updateMD5( mhDigest, pnBuffer, sizeof( pnBuffer ) );
352     rtl_digest_rawMD5( mhDigest, pnDigest, sizeof( pnDigest ) );
353 
354     // decode original salt digest into buffer
355     rtl_cipher_decode( mhCipher, pnVerifierHash, 16, pnBuffer, sizeof( pnBuffer ) );
356 
357     // compare buffer with computed digest
358     bool bResult = memcmp( pnBuffer, pnDigest, sizeof( pnDigest ) ) == 0;
359 
360     // erase buffer and digest arrays and leave
361     (void)memset( pnBuffer, 0, sizeof( pnBuffer ) );
362     (void)memset( pnDigest, 0, sizeof( pnDigest ) );
363     return bResult;
364 }
365 
startBlock(sal_Int32 nCounter)366 bool BinaryCodec_RCF::startBlock( sal_Int32 nCounter )
367 {
368     // initialize key data array
369     sal_uInt8 pnKeyData[ 64 ];
370     (void)memset( pnKeyData, 0, sizeof( pnKeyData ) );
371 
372     // fill 40 bit of digest value into [0..4]
373     (void)memcpy( pnKeyData, mpnDigestValue, 5 );
374 
375     // fill little-endian counter into [5..8], static_cast masks out unneeded bits
376     pnKeyData[ 5 ] = static_cast< sal_uInt8 >( nCounter );
377     pnKeyData[ 6 ] = static_cast< sal_uInt8 >( nCounter >> 8 );
378     pnKeyData[ 7 ] = static_cast< sal_uInt8 >( nCounter >> 16 );
379     pnKeyData[ 8 ] = static_cast< sal_uInt8 >( nCounter >> 24 );
380 
381     pnKeyData[ 9 ] = 0x80;
382     pnKeyData[ 56 ] = 0x48;
383 
384     // fill raw digest of key data into key data
385     (void)rtl_digest_updateMD5( mhDigest, pnKeyData, sizeof( pnKeyData ) );
386     (void)rtl_digest_rawMD5( mhDigest, pnKeyData, RTL_DIGEST_LENGTH_MD5 );
387 
388     // initialize cipher with key data (for decoding)
389     rtlCipherError eResult =
390         rtl_cipher_init( mhCipher, rtl_Cipher_DirectionDecode, pnKeyData, RTL_DIGEST_LENGTH_MD5, 0, 0 );
391 
392     // rrase key data array and leave
393     (void)memset( pnKeyData, 0, sizeof( pnKeyData ) );
394     return eResult == rtl_Cipher_E_None;
395 }
396 
decode(sal_uInt8 * pnDestData,const sal_uInt8 * pnSrcData,sal_Int32 nBytes)397 bool BinaryCodec_RCF::decode( sal_uInt8* pnDestData, const sal_uInt8* pnSrcData, sal_Int32 nBytes )
398 {
399     rtlCipherError eResult = rtl_cipher_decode( mhCipher,
400         pnSrcData, static_cast< sal_Size >( nBytes ),
401         pnDestData, static_cast< sal_Size >( nBytes ) );
402     return eResult == rtl_Cipher_E_None;
403 }
404 
skip(sal_Int32 nBytes)405 bool BinaryCodec_RCF::skip( sal_Int32 nBytes )
406 {
407     // decode dummy data in memory to update internal state of RC4 cipher
408     sal_uInt8 pnDummy[ 1024 ];
409     sal_Int32 nBytesLeft = nBytes;
410     bool bResult = true;
411     while( bResult && (nBytesLeft > 0) )
412     {
413         sal_Int32 nBlockLen = ::std::min( nBytesLeft, static_cast< sal_Int32 >( sizeof( pnDummy ) ) );
414         bResult = decode( pnDummy, pnDummy, nBlockLen );
415         nBytesLeft -= nBlockLen;
416     }
417     return bResult;
418 }
419 
420 // ============================================================================
421 
422 } // namespace core
423 } // namespace oox
424