1 /**************************************************************
2 *
3 * Licensed to the Apache Software Foundation (ASF) under one
4 * or more contributor license agreements. See the NOTICE file
5 * distributed with this work for additional information
6 * regarding copyright ownership. The ASF licenses this file
7 * to you under the Apache License, Version 2.0 (the
8 * "License"); you may not use this file except in compliance
9 * with the License. You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing,
14 * software distributed under the License is distributed on an
15 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16 * KIND, either express or implied. See the License for the
17 * specific language governing permissions and limitations
18 * under the License.
19 *
20 *************************************************************/
21
22
23 #include <string.h>
24
25 #include <sal/types.h>
26
27 #include <rtl/textenc.h>
28 #include <rtl/tencinfo.h>
29
30
31 #include <com/sun/star/io/XInputStream.hpp>
32
33 using namespace rtl;
34 using namespace ::com::sun::star::uno;
35 using namespace ::com::sun::star::io;
36
37 #include "xml2utf.hxx"
38
39 namespace sax_expatwrap {
40
readAndConvert(Sequence<sal_Int8> & seq,sal_Int32 nMaxToRead)41 sal_Int32 XMLFile2UTFConverter::readAndConvert( Sequence<sal_Int8> &seq , sal_Int32 nMaxToRead )
42 {
43
44 Sequence<sal_Int8> seqIn;
45
46 if( ! m_in.is() ) {
47 throw NotConnectedException();
48 }
49 if( ! m_bStarted ) {
50 nMaxToRead = Max( 512 , nMaxToRead ); // it should be possible to find the encoding attribute
51 // within the first 512 bytes == 128 chars in UCS-4
52 }
53
54 sal_Int32 nRead;
55 Sequence< sal_Int8 > seqStart;
56 while( sal_True )
57 {
58 nRead = m_in->readSomeBytes( seq , nMaxToRead );
59
60 if( nRead + seqStart.getLength())
61 {
62 // if nRead is 0, the file is already eof.
63 if( ! m_bStarted && nRead )
64 {
65 // ensure that enough data is available to parse encoding
66 if( seqStart.getLength() )
67 {
68 // prefix with what we had so far.
69 sal_Int32 nLength = seq.getLength();
70 seq.realloc( seqStart.getLength() + nLength );
71
72 memmove (seq.getArray() + seqStart.getLength(),
73 seq.getConstArray(),
74 nLength);
75 memcpy (seq.getArray(),
76 seqStart.getConstArray(),
77 seqStart.getLength());
78 }
79
80 // autodetection with the first bytes
81 if( ! isEncodingRecognizable( seq ) )
82 {
83 // remember what we have so far.
84 seqStart = seq;
85
86 // read more !
87 continue;
88 }
89 if( scanForEncoding( seq ) || m_sEncoding.getLength() ) {
90 // initialize decoding
91 initializeDecoding();
92 }
93 nRead = seq.getLength();
94 seqStart = Sequence < sal_Int8 > ();
95 }
96
97 // do the encoding
98 if( m_pText2Unicode && m_pUnicode2Text &&
99 m_pText2Unicode->canContinue() && m_pUnicode2Text->canContinue() ) {
100
101 Sequence<sal_Unicode> seqUnicode = m_pText2Unicode->convert( seq );
102 seq = m_pUnicode2Text->convert( seqUnicode.getConstArray(), seqUnicode.getLength() );
103 }
104
105 if( ! m_bStarted )
106 {
107 // it must now be ensured, that no encoding attribute exist anymore
108 // ( otherwise the expat-Parser will crash )
109 // This must be done after decoding !
110 // ( e.g. Files decoded in ucs-4 cannot be read properly )
111 m_bStarted = sal_True;
112 removeEncoding( seq );
113 }
114 nRead = seq.getLength();
115 }
116
117 break;
118 }
119 return nRead;
120 }
121
122
~XMLFile2UTFConverter()123 XMLFile2UTFConverter::~XMLFile2UTFConverter()
124 {
125 if( m_pText2Unicode )
126 delete m_pText2Unicode;
127 if( m_pUnicode2Text )
128 delete m_pUnicode2Text;
129 }
130
131
removeEncoding(Sequence<sal_Int8> & seq)132 void XMLFile2UTFConverter::removeEncoding( Sequence<sal_Int8> &seq )
133 {
134 const sal_Int8 *pSource = seq.getArray();
135 if( ! strncmp( (const char * ) pSource , "<?xml" , 4) )
136 {
137
138 // scan for encoding
139 OString str( (sal_Char * ) pSource , seq.getLength() );
140
141 // cut sequence to first line break
142 // find first line break;
143 int nMax = str.indexOf( 10 );
144 if( nMax >= 0 )
145 {
146 str = str.copy( 0 , nMax );
147 }
148
149 int nFound = str.indexOf( " encoding" );
150 if( nFound >= 0 ) {
151 int nStop;
152 int nStart = str.indexOf( "\"" , nFound );
153 if( nStart < 0 || str.indexOf( "'" , nFound ) < nStart )
154 {
155 nStart = str.indexOf( "'" , nFound );
156 nStop = str.indexOf( "'" , nStart +1 );
157 }
158 else
159 {
160 nStop = str.indexOf( "\"" , nStart +1);
161 }
162
163 if( nStart >= 0 && nStop >= 0 && nStart+1 < nStop )
164 {
165 // remove encoding tag from file
166 memmove( &( seq.getArray()[nFound] ) ,
167 &( seq.getArray()[nStop+1]) ,
168 seq.getLength() - nStop -1);
169 seq.realloc( seq.getLength() - ( nStop+1 - nFound ) );
170 // str = String( (char * ) seq.getArray() , seq.getLen() );
171 }
172 }
173 }
174 }
175
176 // Checks, if enough data has been accumulated to recognize the encoding
isEncodingRecognizable(const Sequence<sal_Int8> & seq)177 sal_Bool XMLFile2UTFConverter::isEncodingRecognizable( const Sequence< sal_Int8 > &seq)
178 {
179 const sal_Int8 *pSource = seq.getConstArray();
180 sal_Bool bCheckIfFirstClosingBracketExsists = sal_False;
181
182 if( seq.getLength() < 8 ) {
183 // no recognition possible, when less than 8 bytes are available
184 return sal_False;
185 }
186
187 if( ! strncmp( (const char * ) pSource , "<?xml" , 4 ) ) {
188 // scan if the <?xml tag finishes within this buffer
189 bCheckIfFirstClosingBracketExsists = sal_True;
190 }
191 else if( ('<' == pSource[0] || '<' == pSource[2] ) &&
192 ( ('?' == pSource[4] || '?' == pSource[6] ) ) )
193 {
194 // check for utf-16
195 bCheckIfFirstClosingBracketExsists = sal_True;
196 }
197 else if( ( '<' == pSource[1] || '<' == pSource[3] ) &&
198 ( '?' == pSource[5] || '?' == pSource[7] ) )
199 {
200 // check for
201 bCheckIfFirstClosingBracketExsists = sal_True;
202 }
203
204 if( bCheckIfFirstClosingBracketExsists )
205 {
206 for( sal_Int32 i = 0; i < seq.getLength() ; i ++ )
207 {
208 // whole <?xml tag is valid
209 if( '>' == pSource[ i ] )
210 {
211 return sal_True;
212 }
213 }
214 return sal_False;
215 }
216
217 // No <? tag in front, no need for a bigger buffer
218 return sal_True;
219 }
220
scanForEncoding(Sequence<sal_Int8> & seq)221 sal_Bool XMLFile2UTFConverter::scanForEncoding( Sequence< sal_Int8 > &seq )
222 {
223 const sal_uInt8 *pSource = reinterpret_cast<const sal_uInt8*>( seq.getConstArray() );
224 sal_Bool bReturn = sal_True;
225
226 if( seq.getLength() < 4 ) {
227 // no recognition possible, when less than 4 bytes are available
228 return sal_False;
229 }
230
231 // first level : detect possible file formats
232 if( ! strncmp( (const char * ) pSource , "<?xml" , 4 ) ) {
233
234 // scan for encoding
235 OString str( (const sal_Char *) pSource , seq.getLength() );
236
237 // cut sequence to first line break
238 //find first line break;
239 int nMax = str.indexOf( 10 );
240 if( nMax >= 0 )
241 {
242 str = str.copy( 0 , nMax );
243 }
244
245 int nFound = str.indexOf( " encoding" );
246 if( nFound < str.getLength() ) {
247 int nStop;
248 int nStart = str.indexOf( "\"" , nFound );
249 if( nStart < 0 || str.indexOf( "'" , nFound ) < nStart )
250 {
251 nStart = str.indexOf( "'" , nFound );
252 nStop = str.indexOf( "'" , nStart +1 );
253 }
254 else
255 {
256 nStop = str.indexOf( "\"" , nStart +1);
257 }
258 if( nStart >= 0 && nStop >= 0 && nStart+1 < nStop )
259 {
260 // encoding found finally
261 m_sEncoding = str.copy( nStart+1 , nStop - nStart - 1 );
262 }
263 }
264 }
265 else if( 0xFE == pSource[0] &&
266 0xFF == pSource[1] ) {
267 // UTF-16 big endian
268 // conversion is done so that encoding information can be easily extracted
269 m_sEncoding = "utf-16";
270 }
271 else if( 0xFF == pSource[0] &&
272 0xFE == pSource[1] ) {
273 // UTF-16 little endian
274 // conversion is done so that encoding information can be easily extracted
275 m_sEncoding = "utf-16";
276 }
277 else if( 0x00 == pSource[0] && 0x3c == pSource[1] && 0x00 == pSource[2] && 0x3f == pSource[3] ) {
278 // UTF-16 big endian without byte order mark (this is (strictly speaking) an error.)
279 // The byte order mark is simply added
280
281 // simply add the byte order mark !
282 seq.realloc( seq.getLength() + 2 );
283 memmove( &( seq.getArray()[2] ) , seq.getArray() , seq.getLength() - 2 );
284 ((sal_uInt8*)seq.getArray())[0] = 0xFE;
285 ((sal_uInt8*)seq.getArray())[1] = 0xFF;
286
287 m_sEncoding = "utf-16";
288 }
289 else if( 0x3c == pSource[0] && 0x00 == pSource[1] && 0x3f == pSource[2] && 0x00 == pSource[3] ) {
290 // UTF-16 little endian without byte order mark (this is (strictly speaking) an error.)
291 // The byte order mark is simply added
292
293 seq.realloc( seq.getLength() + 2 );
294 memmove( &( seq.getArray()[2] ) , seq.getArray() , seq.getLength() - 2 );
295 ((sal_uInt8*)seq.getArray())[0] = 0xFF;
296 ((sal_uInt8*)seq.getArray())[1] = 0xFE;
297
298 m_sEncoding = "utf-16";
299 }
300 else if( 0xEF == pSource[0] &&
301 0xBB == pSource[1] &&
302 0xBF == pSource[2] )
303 {
304 // UTF-8 BOM (byte order mark); signifies utf-8, and not byte order
305 // The BOM is removed.
306 memmove( seq.getArray(), &( seq.getArray()[3] ), seq.getLength()-3 );
307 seq.realloc( seq.getLength() - 3 );
308 m_sEncoding = "utf-8";
309 }
310 else if( 0x00 == pSource[0] && 0x00 == pSource[1] && 0x00 == pSource[2] && 0x3c == pSource[3] ) {
311 // UCS-4 big endian
312 m_sEncoding = "ucs-4";
313 }
314 else if( 0x3c == pSource[0] && 0x00 == pSource[1] && 0x00 == pSource[2] && 0x00 == pSource[3] ) {
315 // UCS-4 little endian
316 m_sEncoding = "ucs-4";
317 }
318 else if( 0x4c == pSource[0] && 0x6f == pSource[1] &&
319 0xa7 == static_cast<unsigned char> (pSource[2]) &&
320 0x94 == static_cast<unsigned char> (pSource[3]) ) {
321 // EBCDIC
322 bReturn = sal_False; // must be extended
323 }
324 else {
325 // other
326 // UTF8 is directly recognized by the parser.
327 bReturn = sal_False;
328 }
329
330 return bReturn;
331 }
332
initializeDecoding()333 void XMLFile2UTFConverter::initializeDecoding()
334 {
335
336 if( m_sEncoding.getLength() )
337 {
338 rtl_TextEncoding encoding = rtl_getTextEncodingFromMimeCharset( m_sEncoding.getStr() );
339 if( encoding != RTL_TEXTENCODING_UTF8 )
340 {
341 m_pText2Unicode = new Text2UnicodeConverter( m_sEncoding );
342 m_pUnicode2Text = new Unicode2TextConverter( RTL_TEXTENCODING_UTF8 );
343 }
344 }
345 }
346
347
348 //----------------------------------------------
349 //
350 // Text2UnicodeConverter
351 //
352 //----------------------------------------------
Text2UnicodeConverter(const OString & sEncoding)353 Text2UnicodeConverter::Text2UnicodeConverter( const OString &sEncoding )
354 {
355 rtl_TextEncoding encoding = rtl_getTextEncodingFromMimeCharset( sEncoding.getStr() );
356 if( RTL_TEXTENCODING_DONTKNOW == encoding )
357 {
358 m_bCanContinue = sal_False;
359 m_bInitialized = sal_False;
360 }
361 else
362 {
363 init( encoding );
364 }
365 }
366
~Text2UnicodeConverter()367 Text2UnicodeConverter::~Text2UnicodeConverter()
368 {
369 if( m_bInitialized )
370 {
371 rtl_destroyTextToUnicodeContext( m_convText2Unicode , m_contextText2Unicode );
372 rtl_destroyUnicodeToTextConverter( m_convText2Unicode );
373 }
374 }
375
init(rtl_TextEncoding encoding)376 void Text2UnicodeConverter::init( rtl_TextEncoding encoding )
377 {
378 m_bCanContinue = sal_True;
379 m_bInitialized = sal_True;
380
381 m_convText2Unicode = rtl_createTextToUnicodeConverter(encoding);
382 m_contextText2Unicode = rtl_createTextToUnicodeContext( m_convText2Unicode );
383 m_rtlEncoding = encoding;
384 }
385
386
convert(const Sequence<sal_Int8> & seqText)387 Sequence<sal_Unicode> Text2UnicodeConverter::convert( const Sequence<sal_Int8> &seqText )
388 {
389 sal_uInt32 uiInfo;
390 sal_Size nSrcCvtBytes = 0;
391 sal_Size nTargetCount = 0;
392 sal_Size nSourceCount = 0;
393
394 // the whole source size
395 sal_Int32 nSourceSize = seqText.getLength() + m_seqSource.getLength();
396 Sequence<sal_Unicode> seqUnicode ( nSourceSize );
397
398 const sal_Int8 *pbSource = seqText.getConstArray();
399 sal_Int8 *pbTempMem = 0;
400
401 if( m_seqSource.getLength() ) {
402 // put old rest and new byte sequence into one array
403 pbTempMem = new sal_Int8[ nSourceSize ];
404 memcpy( pbTempMem , m_seqSource.getConstArray() , m_seqSource.getLength() );
405 memcpy( &(pbTempMem[ m_seqSource.getLength() ]) , seqText.getConstArray() , seqText.getLength() );
406 pbSource = pbTempMem;
407
408 // set to zero again
409 m_seqSource = Sequence< sal_Int8 >();
410 }
411
412 while( sal_True ) {
413
414 /* All invalid characters are transformed to the unicode undefined char */
415 nTargetCount += rtl_convertTextToUnicode(
416 m_convText2Unicode,
417 m_contextText2Unicode,
418 ( const sal_Char * ) &( pbSource[nSourceCount] ),
419 nSourceSize - nSourceCount ,
420 &( seqUnicode.getArray()[ nTargetCount ] ),
421 seqUnicode.getLength() - nTargetCount,
422 RTL_TEXTTOUNICODE_FLAGS_UNDEFINED_DEFAULT |
423 RTL_TEXTTOUNICODE_FLAGS_MBUNDEFINED_DEFAULT |
424 RTL_TEXTTOUNICODE_FLAGS_INVALID_DEFAULT,
425 &uiInfo,
426 &nSrcCvtBytes );
427 nSourceCount += nSrcCvtBytes;
428
429 if( uiInfo & RTL_TEXTTOUNICODE_INFO_DESTBUFFERTOSMALL ) {
430 // save necessary bytes for next conversion
431 seqUnicode.realloc( seqUnicode.getLength() * 2 );
432 continue;
433 }
434 break;
435 }
436 if( uiInfo & RTL_TEXTTOUNICODE_INFO_SRCBUFFERTOSMALL ) {
437 m_seqSource.realloc( nSourceSize - nSourceCount );
438 memcpy( m_seqSource.getArray() , &(pbSource[nSourceCount]) , nSourceSize-nSourceCount );
439 }
440
441
442 if( pbTempMem ) {
443 delete [] pbTempMem;
444 }
445
446 // set to correct unicode size
447 seqUnicode.realloc( nTargetCount );
448
449 return seqUnicode;
450 }
451
452
453
454 //----------------------------------------------
455 //
456 // Unicode2TextConverter
457 //
458 //----------------------------------------------
Unicode2TextConverter(rtl_TextEncoding encoding)459 Unicode2TextConverter::Unicode2TextConverter( rtl_TextEncoding encoding )
460 {
461 init( encoding );
462 }
463
464
~Unicode2TextConverter()465 Unicode2TextConverter::~Unicode2TextConverter()
466 {
467 if( m_bInitialized ) {
468 rtl_destroyUnicodeToTextContext( m_convUnicode2Text , m_contextUnicode2Text );
469 rtl_destroyUnicodeToTextConverter( m_convUnicode2Text );
470 }
471 }
472
473
convert(const sal_Unicode * puSource,sal_Int32 nSourceSize)474 Sequence<sal_Int8> Unicode2TextConverter::convert(const sal_Unicode *puSource , sal_Int32 nSourceSize)
475 {
476 sal_Unicode *puTempMem = 0;
477
478 if( m_seqSource.getLength() ) {
479 // For surrogates !
480 // put old rest and new byte sequence into one array
481 // In general when surrogates are used, they should be rarely
482 // cut off between two convert()-calls. So this code is used
483 // rarely and the extra copy is acceptable.
484 puTempMem = new sal_Unicode[ nSourceSize + m_seqSource.getLength()];
485 memcpy( puTempMem ,
486 m_seqSource.getConstArray() ,
487 m_seqSource.getLength() * sizeof( sal_Unicode ) );
488 memcpy(
489 &(puTempMem[ m_seqSource.getLength() ]) ,
490 puSource ,
491 nSourceSize*sizeof( sal_Unicode ) );
492 puSource = puTempMem;
493 nSourceSize += m_seqSource.getLength();
494
495 m_seqSource = Sequence< sal_Unicode > ();
496 }
497
498
499 sal_Size nTargetCount = 0;
500 sal_Size nSourceCount = 0;
501
502 sal_uInt32 uiInfo;
503 sal_Size nSrcCvtChars;
504
505 // take nSourceSize * 3 as preference
506 // this is an upper boundary for converting to utf8,
507 // which most often used as the target.
508 sal_Int32 nSeqSize = nSourceSize * 3;
509
510 Sequence<sal_Int8> seqText( nSeqSize );
511 sal_Char *pTarget = (sal_Char *) seqText.getArray();
512 while( sal_True ) {
513
514 nTargetCount += rtl_convertUnicodeToText(
515 m_convUnicode2Text,
516 m_contextUnicode2Text,
517 &( puSource[nSourceCount] ),
518 nSourceSize - nSourceCount ,
519 &( pTarget[nTargetCount] ),
520 nSeqSize - nTargetCount,
521 RTL_UNICODETOTEXT_FLAGS_UNDEFINED_DEFAULT |
522 RTL_UNICODETOTEXT_FLAGS_INVALID_DEFAULT ,
523 &uiInfo,
524 &nSrcCvtChars);
525 nSourceCount += nSrcCvtChars;
526
527 if( uiInfo & RTL_UNICODETOTEXT_INFO_DESTBUFFERTOSMALL ) {
528 nSeqSize = nSeqSize *2;
529 seqText.realloc( nSeqSize ); // double array size
530 pTarget = ( sal_Char * ) seqText.getArray();
531 continue;
532 }
533 break;
534 }
535
536 // for surrogates
537 if( uiInfo & RTL_UNICODETOTEXT_INFO_SRCBUFFERTOSMALL ) {
538 m_seqSource.realloc( nSourceSize - nSourceCount );
539 memcpy( m_seqSource.getArray() ,
540 &(puSource[nSourceCount]),
541 (nSourceSize - nSourceCount) * sizeof( sal_Unicode ) );
542 }
543
544 if( puTempMem ) {
545 delete [] puTempMem;
546 }
547
548 // reduce the size of the buffer (fast, no copy necessary)
549 seqText.realloc( nTargetCount );
550
551 return seqText;
552 }
553
init(rtl_TextEncoding encoding)554 void Unicode2TextConverter::init( rtl_TextEncoding encoding )
555 {
556 m_bCanContinue = sal_True;
557 m_bInitialized = sal_True;
558
559 m_convUnicode2Text = rtl_createUnicodeToTextConverter( encoding );
560 m_contextUnicode2Text = rtl_createUnicodeToTextContext( m_convUnicode2Text );
561 m_rtlEncoding = encoding;
562 };
563
564
565 }
566