1*b1cdbd2cSJim Jagielski /**************************************************************
2*b1cdbd2cSJim Jagielski *
3*b1cdbd2cSJim Jagielski * Licensed to the Apache Software Foundation (ASF) under one
4*b1cdbd2cSJim Jagielski * or more contributor license agreements. See the NOTICE file
5*b1cdbd2cSJim Jagielski * distributed with this work for additional information
6*b1cdbd2cSJim Jagielski * regarding copyright ownership. The ASF licenses this file
7*b1cdbd2cSJim Jagielski * to you under the Apache License, Version 2.0 (the
8*b1cdbd2cSJim Jagielski * "License"); you may not use this file except in compliance
9*b1cdbd2cSJim Jagielski * with the License. You may obtain a copy of the License at
10*b1cdbd2cSJim Jagielski *
11*b1cdbd2cSJim Jagielski * http://www.apache.org/licenses/LICENSE-2.0
12*b1cdbd2cSJim Jagielski *
13*b1cdbd2cSJim Jagielski * Unless required by applicable law or agreed to in writing,
14*b1cdbd2cSJim Jagielski * software distributed under the License is distributed on an
15*b1cdbd2cSJim Jagielski * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16*b1cdbd2cSJim Jagielski * KIND, either express or implied. See the License for the
17*b1cdbd2cSJim Jagielski * specific language governing permissions and limitations
18*b1cdbd2cSJim Jagielski * under the License.
19*b1cdbd2cSJim Jagielski *
20*b1cdbd2cSJim Jagielski *************************************************************/
21*b1cdbd2cSJim Jagielski
22*b1cdbd2cSJim Jagielski
23*b1cdbd2cSJim Jagielski #include <string.h>
24*b1cdbd2cSJim Jagielski
25*b1cdbd2cSJim Jagielski #include <sal/types.h>
26*b1cdbd2cSJim Jagielski
27*b1cdbd2cSJim Jagielski #include <rtl/textenc.h>
28*b1cdbd2cSJim Jagielski #include <rtl/tencinfo.h>
29*b1cdbd2cSJim Jagielski
30*b1cdbd2cSJim Jagielski
31*b1cdbd2cSJim Jagielski #include <com/sun/star/io/XInputStream.hpp>
32*b1cdbd2cSJim Jagielski
33*b1cdbd2cSJim Jagielski using namespace rtl;
34*b1cdbd2cSJim Jagielski using namespace ::com::sun::star::uno;
35*b1cdbd2cSJim Jagielski using namespace ::com::sun::star::io;
36*b1cdbd2cSJim Jagielski
37*b1cdbd2cSJim Jagielski #include "xml2utf.hxx"
38*b1cdbd2cSJim Jagielski
39*b1cdbd2cSJim Jagielski namespace sax_expatwrap {
40*b1cdbd2cSJim Jagielski
readAndConvert(Sequence<sal_Int8> & seq,sal_Int32 nMaxToRead)41*b1cdbd2cSJim Jagielski sal_Int32 XMLFile2UTFConverter::readAndConvert( Sequence<sal_Int8> &seq , sal_Int32 nMaxToRead )
42*b1cdbd2cSJim Jagielski throw ( IOException, NotConnectedException , BufferSizeExceededException , RuntimeException )
43*b1cdbd2cSJim Jagielski {
44*b1cdbd2cSJim Jagielski
45*b1cdbd2cSJim Jagielski Sequence<sal_Int8> seqIn;
46*b1cdbd2cSJim Jagielski
47*b1cdbd2cSJim Jagielski if( ! m_in.is() ) {
48*b1cdbd2cSJim Jagielski throw NotConnectedException();
49*b1cdbd2cSJim Jagielski }
50*b1cdbd2cSJim Jagielski if( ! m_bStarted ) {
51*b1cdbd2cSJim Jagielski nMaxToRead = Max( 512 , nMaxToRead ); // it should be possible to find the encoding attribute
52*b1cdbd2cSJim Jagielski // within the first 512 bytes == 128 chars in UCS-4
53*b1cdbd2cSJim Jagielski }
54*b1cdbd2cSJim Jagielski
55*b1cdbd2cSJim Jagielski sal_Int32 nRead;
56*b1cdbd2cSJim Jagielski Sequence< sal_Int8 > seqStart;
57*b1cdbd2cSJim Jagielski while( sal_True )
58*b1cdbd2cSJim Jagielski {
59*b1cdbd2cSJim Jagielski nRead = m_in->readSomeBytes( seq , nMaxToRead );
60*b1cdbd2cSJim Jagielski
61*b1cdbd2cSJim Jagielski if( nRead + seqStart.getLength())
62*b1cdbd2cSJim Jagielski {
63*b1cdbd2cSJim Jagielski // if nRead is 0, the file is already eof.
64*b1cdbd2cSJim Jagielski if( ! m_bStarted && nRead )
65*b1cdbd2cSJim Jagielski {
66*b1cdbd2cSJim Jagielski // ensure that enough data is available to parse encoding
67*b1cdbd2cSJim Jagielski if( seqStart.getLength() )
68*b1cdbd2cSJim Jagielski {
69*b1cdbd2cSJim Jagielski // prefix with what we had so far.
70*b1cdbd2cSJim Jagielski sal_Int32 nLength = seq.getLength();
71*b1cdbd2cSJim Jagielski seq.realloc( seqStart.getLength() + nLength );
72*b1cdbd2cSJim Jagielski
73*b1cdbd2cSJim Jagielski memmove (seq.getArray() + seqStart.getLength(),
74*b1cdbd2cSJim Jagielski seq.getConstArray(),
75*b1cdbd2cSJim Jagielski nLength);
76*b1cdbd2cSJim Jagielski memcpy (seq.getArray(),
77*b1cdbd2cSJim Jagielski seqStart.getConstArray(),
78*b1cdbd2cSJim Jagielski seqStart.getLength());
79*b1cdbd2cSJim Jagielski }
80*b1cdbd2cSJim Jagielski
81*b1cdbd2cSJim Jagielski // autodetection with the first bytes
82*b1cdbd2cSJim Jagielski if( ! isEncodingRecognizable( seq ) )
83*b1cdbd2cSJim Jagielski {
84*b1cdbd2cSJim Jagielski // remember what we have so far.
85*b1cdbd2cSJim Jagielski seqStart = seq;
86*b1cdbd2cSJim Jagielski
87*b1cdbd2cSJim Jagielski // read more !
88*b1cdbd2cSJim Jagielski continue;
89*b1cdbd2cSJim Jagielski }
90*b1cdbd2cSJim Jagielski if( scanForEncoding( seq ) || m_sEncoding.getLength() ) {
91*b1cdbd2cSJim Jagielski // initialize decoding
92*b1cdbd2cSJim Jagielski initializeDecoding();
93*b1cdbd2cSJim Jagielski }
94*b1cdbd2cSJim Jagielski nRead = seq.getLength();
95*b1cdbd2cSJim Jagielski seqStart = Sequence < sal_Int8 > ();
96*b1cdbd2cSJim Jagielski }
97*b1cdbd2cSJim Jagielski
98*b1cdbd2cSJim Jagielski // do the encoding
99*b1cdbd2cSJim Jagielski if( m_pText2Unicode && m_pUnicode2Text &&
100*b1cdbd2cSJim Jagielski m_pText2Unicode->canContinue() && m_pUnicode2Text->canContinue() ) {
101*b1cdbd2cSJim Jagielski
102*b1cdbd2cSJim Jagielski Sequence<sal_Unicode> seqUnicode = m_pText2Unicode->convert( seq );
103*b1cdbd2cSJim Jagielski seq = m_pUnicode2Text->convert( seqUnicode.getConstArray(), seqUnicode.getLength() );
104*b1cdbd2cSJim Jagielski }
105*b1cdbd2cSJim Jagielski
106*b1cdbd2cSJim Jagielski if( ! m_bStarted )
107*b1cdbd2cSJim Jagielski {
108*b1cdbd2cSJim Jagielski // it must now be ensured, that no encoding attribute exist anymore
109*b1cdbd2cSJim Jagielski // ( otherwise the expat-Parser will crash )
110*b1cdbd2cSJim Jagielski // This must be done after decoding !
111*b1cdbd2cSJim Jagielski // ( e.g. Files decoded in ucs-4 cannot be read properly )
112*b1cdbd2cSJim Jagielski m_bStarted = sal_True;
113*b1cdbd2cSJim Jagielski removeEncoding( seq );
114*b1cdbd2cSJim Jagielski }
115*b1cdbd2cSJim Jagielski nRead = seq.getLength();
116*b1cdbd2cSJim Jagielski }
117*b1cdbd2cSJim Jagielski
118*b1cdbd2cSJim Jagielski break;
119*b1cdbd2cSJim Jagielski }
120*b1cdbd2cSJim Jagielski return nRead;
121*b1cdbd2cSJim Jagielski }
122*b1cdbd2cSJim Jagielski
123*b1cdbd2cSJim Jagielski
~XMLFile2UTFConverter()124*b1cdbd2cSJim Jagielski XMLFile2UTFConverter::~XMLFile2UTFConverter()
125*b1cdbd2cSJim Jagielski {
126*b1cdbd2cSJim Jagielski if( m_pText2Unicode )
127*b1cdbd2cSJim Jagielski delete m_pText2Unicode;
128*b1cdbd2cSJim Jagielski if( m_pUnicode2Text )
129*b1cdbd2cSJim Jagielski delete m_pUnicode2Text;
130*b1cdbd2cSJim Jagielski }
131*b1cdbd2cSJim Jagielski
132*b1cdbd2cSJim Jagielski
removeEncoding(Sequence<sal_Int8> & seq)133*b1cdbd2cSJim Jagielski void XMLFile2UTFConverter::removeEncoding( Sequence<sal_Int8> &seq )
134*b1cdbd2cSJim Jagielski {
135*b1cdbd2cSJim Jagielski const sal_Int8 *pSource = seq.getArray();
136*b1cdbd2cSJim Jagielski if( ! strncmp( (const char * ) pSource , "<?xml" , 4) )
137*b1cdbd2cSJim Jagielski {
138*b1cdbd2cSJim Jagielski
139*b1cdbd2cSJim Jagielski // scan for encoding
140*b1cdbd2cSJim Jagielski OString str( (sal_Char * ) pSource , seq.getLength() );
141*b1cdbd2cSJim Jagielski
142*b1cdbd2cSJim Jagielski // cut sequence to first line break
143*b1cdbd2cSJim Jagielski // find first line break;
144*b1cdbd2cSJim Jagielski int nMax = str.indexOf( 10 );
145*b1cdbd2cSJim Jagielski if( nMax >= 0 )
146*b1cdbd2cSJim Jagielski {
147*b1cdbd2cSJim Jagielski str = str.copy( 0 , nMax );
148*b1cdbd2cSJim Jagielski }
149*b1cdbd2cSJim Jagielski
150*b1cdbd2cSJim Jagielski int nFound = str.indexOf( " encoding" );
151*b1cdbd2cSJim Jagielski if( nFound >= 0 ) {
152*b1cdbd2cSJim Jagielski int nStop;
153*b1cdbd2cSJim Jagielski int nStart = str.indexOf( "\"" , nFound );
154*b1cdbd2cSJim Jagielski if( nStart < 0 || str.indexOf( "'" , nFound ) < nStart )
155*b1cdbd2cSJim Jagielski {
156*b1cdbd2cSJim Jagielski nStart = str.indexOf( "'" , nFound );
157*b1cdbd2cSJim Jagielski nStop = str.indexOf( "'" , nStart +1 );
158*b1cdbd2cSJim Jagielski }
159*b1cdbd2cSJim Jagielski else
160*b1cdbd2cSJim Jagielski {
161*b1cdbd2cSJim Jagielski nStop = str.indexOf( "\"" , nStart +1);
162*b1cdbd2cSJim Jagielski }
163*b1cdbd2cSJim Jagielski
164*b1cdbd2cSJim Jagielski if( nStart >= 0 && nStop >= 0 && nStart+1 < nStop )
165*b1cdbd2cSJim Jagielski {
166*b1cdbd2cSJim Jagielski // remove encoding tag from file
167*b1cdbd2cSJim Jagielski memmove( &( seq.getArray()[nFound] ) ,
168*b1cdbd2cSJim Jagielski &( seq.getArray()[nStop+1]) ,
169*b1cdbd2cSJim Jagielski seq.getLength() - nStop -1);
170*b1cdbd2cSJim Jagielski seq.realloc( seq.getLength() - ( nStop+1 - nFound ) );
171*b1cdbd2cSJim Jagielski // str = String( (char * ) seq.getArray() , seq.getLen() );
172*b1cdbd2cSJim Jagielski }
173*b1cdbd2cSJim Jagielski }
174*b1cdbd2cSJim Jagielski }
175*b1cdbd2cSJim Jagielski }
176*b1cdbd2cSJim Jagielski
177*b1cdbd2cSJim Jagielski // Checks, if enough data has been accumulated to recognize the encoding
isEncodingRecognizable(const Sequence<sal_Int8> & seq)178*b1cdbd2cSJim Jagielski sal_Bool XMLFile2UTFConverter::isEncodingRecognizable( const Sequence< sal_Int8 > &seq)
179*b1cdbd2cSJim Jagielski {
180*b1cdbd2cSJim Jagielski const sal_Int8 *pSource = seq.getConstArray();
181*b1cdbd2cSJim Jagielski sal_Bool bCheckIfFirstClosingBracketExsists = sal_False;
182*b1cdbd2cSJim Jagielski
183*b1cdbd2cSJim Jagielski if( seq.getLength() < 8 ) {
184*b1cdbd2cSJim Jagielski // no recognition possible, when less than 8 bytes are available
185*b1cdbd2cSJim Jagielski return sal_False;
186*b1cdbd2cSJim Jagielski }
187*b1cdbd2cSJim Jagielski
188*b1cdbd2cSJim Jagielski if( ! strncmp( (const char * ) pSource , "<?xml" , 4 ) ) {
189*b1cdbd2cSJim Jagielski // scan if the <?xml tag finishes within this buffer
190*b1cdbd2cSJim Jagielski bCheckIfFirstClosingBracketExsists = sal_True;
191*b1cdbd2cSJim Jagielski }
192*b1cdbd2cSJim Jagielski else if( ('<' == pSource[0] || '<' == pSource[2] ) &&
193*b1cdbd2cSJim Jagielski ( ('?' == pSource[4] || '?' == pSource[6] ) ) )
194*b1cdbd2cSJim Jagielski {
195*b1cdbd2cSJim Jagielski // check for utf-16
196*b1cdbd2cSJim Jagielski bCheckIfFirstClosingBracketExsists = sal_True;
197*b1cdbd2cSJim Jagielski }
198*b1cdbd2cSJim Jagielski else if( ( '<' == pSource[1] || '<' == pSource[3] ) &&
199*b1cdbd2cSJim Jagielski ( '?' == pSource[5] || '?' == pSource[7] ) )
200*b1cdbd2cSJim Jagielski {
201*b1cdbd2cSJim Jagielski // check for
202*b1cdbd2cSJim Jagielski bCheckIfFirstClosingBracketExsists = sal_True;
203*b1cdbd2cSJim Jagielski }
204*b1cdbd2cSJim Jagielski
205*b1cdbd2cSJim Jagielski if( bCheckIfFirstClosingBracketExsists )
206*b1cdbd2cSJim Jagielski {
207*b1cdbd2cSJim Jagielski for( sal_Int32 i = 0; i < seq.getLength() ; i ++ )
208*b1cdbd2cSJim Jagielski {
209*b1cdbd2cSJim Jagielski // whole <?xml tag is valid
210*b1cdbd2cSJim Jagielski if( '>' == pSource[ i ] )
211*b1cdbd2cSJim Jagielski {
212*b1cdbd2cSJim Jagielski return sal_True;
213*b1cdbd2cSJim Jagielski }
214*b1cdbd2cSJim Jagielski }
215*b1cdbd2cSJim Jagielski return sal_False;
216*b1cdbd2cSJim Jagielski }
217*b1cdbd2cSJim Jagielski
218*b1cdbd2cSJim Jagielski // No <? tag in front, no need for a bigger buffer
219*b1cdbd2cSJim Jagielski return sal_True;
220*b1cdbd2cSJim Jagielski }
221*b1cdbd2cSJim Jagielski
scanForEncoding(Sequence<sal_Int8> & seq)222*b1cdbd2cSJim Jagielski sal_Bool XMLFile2UTFConverter::scanForEncoding( Sequence< sal_Int8 > &seq )
223*b1cdbd2cSJim Jagielski {
224*b1cdbd2cSJim Jagielski const sal_uInt8 *pSource = reinterpret_cast<const sal_uInt8*>( seq.getConstArray() );
225*b1cdbd2cSJim Jagielski sal_Bool bReturn = sal_True;
226*b1cdbd2cSJim Jagielski
227*b1cdbd2cSJim Jagielski if( seq.getLength() < 4 ) {
228*b1cdbd2cSJim Jagielski // no recognition possible, when less than 4 bytes are available
229*b1cdbd2cSJim Jagielski return sal_False;
230*b1cdbd2cSJim Jagielski }
231*b1cdbd2cSJim Jagielski
232*b1cdbd2cSJim Jagielski // first level : detect possible file formats
233*b1cdbd2cSJim Jagielski if( ! strncmp( (const char * ) pSource , "<?xml" , 4 ) ) {
234*b1cdbd2cSJim Jagielski
235*b1cdbd2cSJim Jagielski // scan for encoding
236*b1cdbd2cSJim Jagielski OString str( (const sal_Char *) pSource , seq.getLength() );
237*b1cdbd2cSJim Jagielski
238*b1cdbd2cSJim Jagielski // cut sequence to first line break
239*b1cdbd2cSJim Jagielski //find first line break;
240*b1cdbd2cSJim Jagielski int nMax = str.indexOf( 10 );
241*b1cdbd2cSJim Jagielski if( nMax >= 0 )
242*b1cdbd2cSJim Jagielski {
243*b1cdbd2cSJim Jagielski str = str.copy( 0 , nMax );
244*b1cdbd2cSJim Jagielski }
245*b1cdbd2cSJim Jagielski
246*b1cdbd2cSJim Jagielski int nFound = str.indexOf( " encoding" );
247*b1cdbd2cSJim Jagielski if( nFound < str.getLength() ) {
248*b1cdbd2cSJim Jagielski int nStop;
249*b1cdbd2cSJim Jagielski int nStart = str.indexOf( "\"" , nFound );
250*b1cdbd2cSJim Jagielski if( nStart < 0 || str.indexOf( "'" , nFound ) < nStart )
251*b1cdbd2cSJim Jagielski {
252*b1cdbd2cSJim Jagielski nStart = str.indexOf( "'" , nFound );
253*b1cdbd2cSJim Jagielski nStop = str.indexOf( "'" , nStart +1 );
254*b1cdbd2cSJim Jagielski }
255*b1cdbd2cSJim Jagielski else
256*b1cdbd2cSJim Jagielski {
257*b1cdbd2cSJim Jagielski nStop = str.indexOf( "\"" , nStart +1);
258*b1cdbd2cSJim Jagielski }
259*b1cdbd2cSJim Jagielski if( nStart >= 0 && nStop >= 0 && nStart+1 < nStop )
260*b1cdbd2cSJim Jagielski {
261*b1cdbd2cSJim Jagielski // encoding found finally
262*b1cdbd2cSJim Jagielski m_sEncoding = str.copy( nStart+1 , nStop - nStart - 1 );
263*b1cdbd2cSJim Jagielski }
264*b1cdbd2cSJim Jagielski }
265*b1cdbd2cSJim Jagielski }
266*b1cdbd2cSJim Jagielski else if( 0xFE == pSource[0] &&
267*b1cdbd2cSJim Jagielski 0xFF == pSource[1] ) {
268*b1cdbd2cSJim Jagielski // UTF-16 big endian
269*b1cdbd2cSJim Jagielski // conversion is done so that encoding information can be easily extracted
270*b1cdbd2cSJim Jagielski m_sEncoding = "utf-16";
271*b1cdbd2cSJim Jagielski }
272*b1cdbd2cSJim Jagielski else if( 0xFF == pSource[0] &&
273*b1cdbd2cSJim Jagielski 0xFE == pSource[1] ) {
274*b1cdbd2cSJim Jagielski // UTF-16 little endian
275*b1cdbd2cSJim Jagielski // conversion is done so that encoding information can be easily extracted
276*b1cdbd2cSJim Jagielski m_sEncoding = "utf-16";
277*b1cdbd2cSJim Jagielski }
278*b1cdbd2cSJim Jagielski else if( 0x00 == pSource[0] && 0x3c == pSource[1] && 0x00 == pSource[2] && 0x3f == pSource[3] ) {
279*b1cdbd2cSJim Jagielski // UTF-16 big endian without byte order mark (this is (strictly speaking) an error.)
280*b1cdbd2cSJim Jagielski // The byte order mark is simply added
281*b1cdbd2cSJim Jagielski
282*b1cdbd2cSJim Jagielski // simply add the byte order mark !
283*b1cdbd2cSJim Jagielski seq.realloc( seq.getLength() + 2 );
284*b1cdbd2cSJim Jagielski memmove( &( seq.getArray()[2] ) , seq.getArray() , seq.getLength() - 2 );
285*b1cdbd2cSJim Jagielski ((sal_uInt8*)seq.getArray())[0] = 0xFE;
286*b1cdbd2cSJim Jagielski ((sal_uInt8*)seq.getArray())[1] = 0xFF;
287*b1cdbd2cSJim Jagielski
288*b1cdbd2cSJim Jagielski m_sEncoding = "utf-16";
289*b1cdbd2cSJim Jagielski }
290*b1cdbd2cSJim Jagielski else if( 0x3c == pSource[0] && 0x00 == pSource[1] && 0x3f == pSource[2] && 0x00 == pSource[3] ) {
291*b1cdbd2cSJim Jagielski // UTF-16 little endian without byte order mark (this is (strictly speaking) an error.)
292*b1cdbd2cSJim Jagielski // The byte order mark is simply added
293*b1cdbd2cSJim Jagielski
294*b1cdbd2cSJim Jagielski seq.realloc( seq.getLength() + 2 );
295*b1cdbd2cSJim Jagielski memmove( &( seq.getArray()[2] ) , seq.getArray() , seq.getLength() - 2 );
296*b1cdbd2cSJim Jagielski ((sal_uInt8*)seq.getArray())[0] = 0xFF;
297*b1cdbd2cSJim Jagielski ((sal_uInt8*)seq.getArray())[1] = 0xFE;
298*b1cdbd2cSJim Jagielski
299*b1cdbd2cSJim Jagielski m_sEncoding = "utf-16";
300*b1cdbd2cSJim Jagielski }
301*b1cdbd2cSJim Jagielski else if( 0xEF == pSource[0] &&
302*b1cdbd2cSJim Jagielski 0xBB == pSource[1] &&
303*b1cdbd2cSJim Jagielski 0xBF == pSource[2] )
304*b1cdbd2cSJim Jagielski {
305*b1cdbd2cSJim Jagielski // UTF-8 BOM (byte order mark); signifies utf-8, and not byte order
306*b1cdbd2cSJim Jagielski // The BOM is removed.
307*b1cdbd2cSJim Jagielski memmove( seq.getArray(), &( seq.getArray()[3] ), seq.getLength()-3 );
308*b1cdbd2cSJim Jagielski seq.realloc( seq.getLength() - 3 );
309*b1cdbd2cSJim Jagielski m_sEncoding = "utf-8";
310*b1cdbd2cSJim Jagielski }
311*b1cdbd2cSJim Jagielski else if( 0x00 == pSource[0] && 0x00 == pSource[1] && 0x00 == pSource[2] && 0x3c == pSource[3] ) {
312*b1cdbd2cSJim Jagielski // UCS-4 big endian
313*b1cdbd2cSJim Jagielski m_sEncoding = "ucs-4";
314*b1cdbd2cSJim Jagielski }
315*b1cdbd2cSJim Jagielski else if( 0x3c == pSource[0] && 0x00 == pSource[1] && 0x00 == pSource[2] && 0x00 == pSource[3] ) {
316*b1cdbd2cSJim Jagielski // UCS-4 little endian
317*b1cdbd2cSJim Jagielski m_sEncoding = "ucs-4";
318*b1cdbd2cSJim Jagielski }
319*b1cdbd2cSJim Jagielski else if( 0x4c == pSource[0] && 0x6f == pSource[1] &&
320*b1cdbd2cSJim Jagielski 0xa7 == static_cast<unsigned char> (pSource[2]) &&
321*b1cdbd2cSJim Jagielski 0x94 == static_cast<unsigned char> (pSource[3]) ) {
322*b1cdbd2cSJim Jagielski // EBCDIC
323*b1cdbd2cSJim Jagielski bReturn = sal_False; // must be extended
324*b1cdbd2cSJim Jagielski }
325*b1cdbd2cSJim Jagielski else {
326*b1cdbd2cSJim Jagielski // other
327*b1cdbd2cSJim Jagielski // UTF8 is directly recognized by the parser.
328*b1cdbd2cSJim Jagielski bReturn = sal_False;
329*b1cdbd2cSJim Jagielski }
330*b1cdbd2cSJim Jagielski
331*b1cdbd2cSJim Jagielski return bReturn;
332*b1cdbd2cSJim Jagielski }
333*b1cdbd2cSJim Jagielski
initializeDecoding()334*b1cdbd2cSJim Jagielski void XMLFile2UTFConverter::initializeDecoding()
335*b1cdbd2cSJim Jagielski {
336*b1cdbd2cSJim Jagielski
337*b1cdbd2cSJim Jagielski if( m_sEncoding.getLength() )
338*b1cdbd2cSJim Jagielski {
339*b1cdbd2cSJim Jagielski rtl_TextEncoding encoding = rtl_getTextEncodingFromMimeCharset( m_sEncoding.getStr() );
340*b1cdbd2cSJim Jagielski if( encoding != RTL_TEXTENCODING_UTF8 )
341*b1cdbd2cSJim Jagielski {
342*b1cdbd2cSJim Jagielski m_pText2Unicode = new Text2UnicodeConverter( m_sEncoding );
343*b1cdbd2cSJim Jagielski m_pUnicode2Text = new Unicode2TextConverter( RTL_TEXTENCODING_UTF8 );
344*b1cdbd2cSJim Jagielski }
345*b1cdbd2cSJim Jagielski }
346*b1cdbd2cSJim Jagielski }
347*b1cdbd2cSJim Jagielski
348*b1cdbd2cSJim Jagielski
349*b1cdbd2cSJim Jagielski //----------------------------------------------
350*b1cdbd2cSJim Jagielski //
351*b1cdbd2cSJim Jagielski // Text2UnicodeConverter
352*b1cdbd2cSJim Jagielski //
353*b1cdbd2cSJim Jagielski //----------------------------------------------
Text2UnicodeConverter(const OString & sEncoding)354*b1cdbd2cSJim Jagielski Text2UnicodeConverter::Text2UnicodeConverter( const OString &sEncoding )
355*b1cdbd2cSJim Jagielski {
356*b1cdbd2cSJim Jagielski rtl_TextEncoding encoding = rtl_getTextEncodingFromMimeCharset( sEncoding.getStr() );
357*b1cdbd2cSJim Jagielski if( RTL_TEXTENCODING_DONTKNOW == encoding )
358*b1cdbd2cSJim Jagielski {
359*b1cdbd2cSJim Jagielski m_bCanContinue = sal_False;
360*b1cdbd2cSJim Jagielski m_bInitialized = sal_False;
361*b1cdbd2cSJim Jagielski }
362*b1cdbd2cSJim Jagielski else
363*b1cdbd2cSJim Jagielski {
364*b1cdbd2cSJim Jagielski init( encoding );
365*b1cdbd2cSJim Jagielski }
366*b1cdbd2cSJim Jagielski }
367*b1cdbd2cSJim Jagielski
~Text2UnicodeConverter()368*b1cdbd2cSJim Jagielski Text2UnicodeConverter::~Text2UnicodeConverter()
369*b1cdbd2cSJim Jagielski {
370*b1cdbd2cSJim Jagielski if( m_bInitialized )
371*b1cdbd2cSJim Jagielski {
372*b1cdbd2cSJim Jagielski rtl_destroyTextToUnicodeContext( m_convText2Unicode , m_contextText2Unicode );
373*b1cdbd2cSJim Jagielski rtl_destroyUnicodeToTextConverter( m_convText2Unicode );
374*b1cdbd2cSJim Jagielski }
375*b1cdbd2cSJim Jagielski }
376*b1cdbd2cSJim Jagielski
init(rtl_TextEncoding encoding)377*b1cdbd2cSJim Jagielski void Text2UnicodeConverter::init( rtl_TextEncoding encoding )
378*b1cdbd2cSJim Jagielski {
379*b1cdbd2cSJim Jagielski m_bCanContinue = sal_True;
380*b1cdbd2cSJim Jagielski m_bInitialized = sal_True;
381*b1cdbd2cSJim Jagielski
382*b1cdbd2cSJim Jagielski m_convText2Unicode = rtl_createTextToUnicodeConverter(encoding);
383*b1cdbd2cSJim Jagielski m_contextText2Unicode = rtl_createTextToUnicodeContext( m_convText2Unicode );
384*b1cdbd2cSJim Jagielski m_rtlEncoding = encoding;
385*b1cdbd2cSJim Jagielski }
386*b1cdbd2cSJim Jagielski
387*b1cdbd2cSJim Jagielski
convert(const Sequence<sal_Int8> & seqText)388*b1cdbd2cSJim Jagielski Sequence<sal_Unicode> Text2UnicodeConverter::convert( const Sequence<sal_Int8> &seqText )
389*b1cdbd2cSJim Jagielski {
390*b1cdbd2cSJim Jagielski sal_uInt32 uiInfo;
391*b1cdbd2cSJim Jagielski sal_Size nSrcCvtBytes = 0;
392*b1cdbd2cSJim Jagielski sal_Size nTargetCount = 0;
393*b1cdbd2cSJim Jagielski sal_Size nSourceCount = 0;
394*b1cdbd2cSJim Jagielski
395*b1cdbd2cSJim Jagielski // the whole source size
396*b1cdbd2cSJim Jagielski sal_Int32 nSourceSize = seqText.getLength() + m_seqSource.getLength();
397*b1cdbd2cSJim Jagielski Sequence<sal_Unicode> seqUnicode ( nSourceSize );
398*b1cdbd2cSJim Jagielski
399*b1cdbd2cSJim Jagielski const sal_Int8 *pbSource = seqText.getConstArray();
400*b1cdbd2cSJim Jagielski sal_Int8 *pbTempMem = 0;
401*b1cdbd2cSJim Jagielski
402*b1cdbd2cSJim Jagielski if( m_seqSource.getLength() ) {
403*b1cdbd2cSJim Jagielski // put old rest and new byte sequence into one array
404*b1cdbd2cSJim Jagielski pbTempMem = new sal_Int8[ nSourceSize ];
405*b1cdbd2cSJim Jagielski memcpy( pbTempMem , m_seqSource.getConstArray() , m_seqSource.getLength() );
406*b1cdbd2cSJim Jagielski memcpy( &(pbTempMem[ m_seqSource.getLength() ]) , seqText.getConstArray() , seqText.getLength() );
407*b1cdbd2cSJim Jagielski pbSource = pbTempMem;
408*b1cdbd2cSJim Jagielski
409*b1cdbd2cSJim Jagielski // set to zero again
410*b1cdbd2cSJim Jagielski m_seqSource = Sequence< sal_Int8 >();
411*b1cdbd2cSJim Jagielski }
412*b1cdbd2cSJim Jagielski
413*b1cdbd2cSJim Jagielski while( sal_True ) {
414*b1cdbd2cSJim Jagielski
415*b1cdbd2cSJim Jagielski /* All invalid characters are transformed to the unicode undefined char */
416*b1cdbd2cSJim Jagielski nTargetCount += rtl_convertTextToUnicode(
417*b1cdbd2cSJim Jagielski m_convText2Unicode,
418*b1cdbd2cSJim Jagielski m_contextText2Unicode,
419*b1cdbd2cSJim Jagielski ( const sal_Char * ) &( pbSource[nSourceCount] ),
420*b1cdbd2cSJim Jagielski nSourceSize - nSourceCount ,
421*b1cdbd2cSJim Jagielski &( seqUnicode.getArray()[ nTargetCount ] ),
422*b1cdbd2cSJim Jagielski seqUnicode.getLength() - nTargetCount,
423*b1cdbd2cSJim Jagielski RTL_TEXTTOUNICODE_FLAGS_UNDEFINED_DEFAULT |
424*b1cdbd2cSJim Jagielski RTL_TEXTTOUNICODE_FLAGS_MBUNDEFINED_DEFAULT |
425*b1cdbd2cSJim Jagielski RTL_TEXTTOUNICODE_FLAGS_INVALID_DEFAULT,
426*b1cdbd2cSJim Jagielski &uiInfo,
427*b1cdbd2cSJim Jagielski &nSrcCvtBytes );
428*b1cdbd2cSJim Jagielski nSourceCount += nSrcCvtBytes;
429*b1cdbd2cSJim Jagielski
430*b1cdbd2cSJim Jagielski if( uiInfo & RTL_TEXTTOUNICODE_INFO_DESTBUFFERTOSMALL ) {
431*b1cdbd2cSJim Jagielski // save necessary bytes for next conversion
432*b1cdbd2cSJim Jagielski seqUnicode.realloc( seqUnicode.getLength() * 2 );
433*b1cdbd2cSJim Jagielski continue;
434*b1cdbd2cSJim Jagielski }
435*b1cdbd2cSJim Jagielski break;
436*b1cdbd2cSJim Jagielski }
437*b1cdbd2cSJim Jagielski if( uiInfo & RTL_TEXTTOUNICODE_INFO_SRCBUFFERTOSMALL ) {
438*b1cdbd2cSJim Jagielski m_seqSource.realloc( nSourceSize - nSourceCount );
439*b1cdbd2cSJim Jagielski memcpy( m_seqSource.getArray() , &(pbSource[nSourceCount]) , nSourceSize-nSourceCount );
440*b1cdbd2cSJim Jagielski }
441*b1cdbd2cSJim Jagielski
442*b1cdbd2cSJim Jagielski
443*b1cdbd2cSJim Jagielski if( pbTempMem ) {
444*b1cdbd2cSJim Jagielski delete [] pbTempMem;
445*b1cdbd2cSJim Jagielski }
446*b1cdbd2cSJim Jagielski
447*b1cdbd2cSJim Jagielski // set to correct unicode size
448*b1cdbd2cSJim Jagielski seqUnicode.realloc( nTargetCount );
449*b1cdbd2cSJim Jagielski
450*b1cdbd2cSJim Jagielski return seqUnicode;
451*b1cdbd2cSJim Jagielski }
452*b1cdbd2cSJim Jagielski
453*b1cdbd2cSJim Jagielski
454*b1cdbd2cSJim Jagielski
455*b1cdbd2cSJim Jagielski //----------------------------------------------
456*b1cdbd2cSJim Jagielski //
457*b1cdbd2cSJim Jagielski // Unicode2TextConverter
458*b1cdbd2cSJim Jagielski //
459*b1cdbd2cSJim Jagielski //----------------------------------------------
Unicode2TextConverter(rtl_TextEncoding encoding)460*b1cdbd2cSJim Jagielski Unicode2TextConverter::Unicode2TextConverter( rtl_TextEncoding encoding )
461*b1cdbd2cSJim Jagielski {
462*b1cdbd2cSJim Jagielski init( encoding );
463*b1cdbd2cSJim Jagielski }
464*b1cdbd2cSJim Jagielski
465*b1cdbd2cSJim Jagielski
~Unicode2TextConverter()466*b1cdbd2cSJim Jagielski Unicode2TextConverter::~Unicode2TextConverter()
467*b1cdbd2cSJim Jagielski {
468*b1cdbd2cSJim Jagielski if( m_bInitialized ) {
469*b1cdbd2cSJim Jagielski rtl_destroyUnicodeToTextContext( m_convUnicode2Text , m_contextUnicode2Text );
470*b1cdbd2cSJim Jagielski rtl_destroyUnicodeToTextConverter( m_convUnicode2Text );
471*b1cdbd2cSJim Jagielski }
472*b1cdbd2cSJim Jagielski }
473*b1cdbd2cSJim Jagielski
474*b1cdbd2cSJim Jagielski
convert(const sal_Unicode * puSource,sal_Int32 nSourceSize)475*b1cdbd2cSJim Jagielski Sequence<sal_Int8> Unicode2TextConverter::convert(const sal_Unicode *puSource , sal_Int32 nSourceSize)
476*b1cdbd2cSJim Jagielski {
477*b1cdbd2cSJim Jagielski sal_Unicode *puTempMem = 0;
478*b1cdbd2cSJim Jagielski
479*b1cdbd2cSJim Jagielski if( m_seqSource.getLength() ) {
480*b1cdbd2cSJim Jagielski // For surrogates !
481*b1cdbd2cSJim Jagielski // put old rest and new byte sequence into one array
482*b1cdbd2cSJim Jagielski // In general when surrogates are used, they should be rarely
483*b1cdbd2cSJim Jagielski // cut off between two convert()-calls. So this code is used
484*b1cdbd2cSJim Jagielski // rarely and the extra copy is acceptable.
485*b1cdbd2cSJim Jagielski puTempMem = new sal_Unicode[ nSourceSize + m_seqSource.getLength()];
486*b1cdbd2cSJim Jagielski memcpy( puTempMem ,
487*b1cdbd2cSJim Jagielski m_seqSource.getConstArray() ,
488*b1cdbd2cSJim Jagielski m_seqSource.getLength() * sizeof( sal_Unicode ) );
489*b1cdbd2cSJim Jagielski memcpy(
490*b1cdbd2cSJim Jagielski &(puTempMem[ m_seqSource.getLength() ]) ,
491*b1cdbd2cSJim Jagielski puSource ,
492*b1cdbd2cSJim Jagielski nSourceSize*sizeof( sal_Unicode ) );
493*b1cdbd2cSJim Jagielski puSource = puTempMem;
494*b1cdbd2cSJim Jagielski nSourceSize += m_seqSource.getLength();
495*b1cdbd2cSJim Jagielski
496*b1cdbd2cSJim Jagielski m_seqSource = Sequence< sal_Unicode > ();
497*b1cdbd2cSJim Jagielski }
498*b1cdbd2cSJim Jagielski
499*b1cdbd2cSJim Jagielski
500*b1cdbd2cSJim Jagielski sal_Size nTargetCount = 0;
501*b1cdbd2cSJim Jagielski sal_Size nSourceCount = 0;
502*b1cdbd2cSJim Jagielski
503*b1cdbd2cSJim Jagielski sal_uInt32 uiInfo;
504*b1cdbd2cSJim Jagielski sal_Size nSrcCvtChars;
505*b1cdbd2cSJim Jagielski
506*b1cdbd2cSJim Jagielski // take nSourceSize * 3 as preference
507*b1cdbd2cSJim Jagielski // this is an upper boundary for converting to utf8,
508*b1cdbd2cSJim Jagielski // which most often used as the target.
509*b1cdbd2cSJim Jagielski sal_Int32 nSeqSize = nSourceSize * 3;
510*b1cdbd2cSJim Jagielski
511*b1cdbd2cSJim Jagielski Sequence<sal_Int8> seqText( nSeqSize );
512*b1cdbd2cSJim Jagielski sal_Char *pTarget = (sal_Char *) seqText.getArray();
513*b1cdbd2cSJim Jagielski while( sal_True ) {
514*b1cdbd2cSJim Jagielski
515*b1cdbd2cSJim Jagielski nTargetCount += rtl_convertUnicodeToText(
516*b1cdbd2cSJim Jagielski m_convUnicode2Text,
517*b1cdbd2cSJim Jagielski m_contextUnicode2Text,
518*b1cdbd2cSJim Jagielski &( puSource[nSourceCount] ),
519*b1cdbd2cSJim Jagielski nSourceSize - nSourceCount ,
520*b1cdbd2cSJim Jagielski &( pTarget[nTargetCount] ),
521*b1cdbd2cSJim Jagielski nSeqSize - nTargetCount,
522*b1cdbd2cSJim Jagielski RTL_UNICODETOTEXT_FLAGS_UNDEFINED_DEFAULT |
523*b1cdbd2cSJim Jagielski RTL_UNICODETOTEXT_FLAGS_INVALID_DEFAULT ,
524*b1cdbd2cSJim Jagielski &uiInfo,
525*b1cdbd2cSJim Jagielski &nSrcCvtChars);
526*b1cdbd2cSJim Jagielski nSourceCount += nSrcCvtChars;
527*b1cdbd2cSJim Jagielski
528*b1cdbd2cSJim Jagielski if( uiInfo & RTL_UNICODETOTEXT_INFO_DESTBUFFERTOSMALL ) {
529*b1cdbd2cSJim Jagielski nSeqSize = nSeqSize *2;
530*b1cdbd2cSJim Jagielski seqText.realloc( nSeqSize ); // double array size
531*b1cdbd2cSJim Jagielski pTarget = ( sal_Char * ) seqText.getArray();
532*b1cdbd2cSJim Jagielski continue;
533*b1cdbd2cSJim Jagielski }
534*b1cdbd2cSJim Jagielski break;
535*b1cdbd2cSJim Jagielski }
536*b1cdbd2cSJim Jagielski
537*b1cdbd2cSJim Jagielski // for surrogates
538*b1cdbd2cSJim Jagielski if( uiInfo & RTL_UNICODETOTEXT_INFO_SRCBUFFERTOSMALL ) {
539*b1cdbd2cSJim Jagielski m_seqSource.realloc( nSourceSize - nSourceCount );
540*b1cdbd2cSJim Jagielski memcpy( m_seqSource.getArray() ,
541*b1cdbd2cSJim Jagielski &(puSource[nSourceCount]),
542*b1cdbd2cSJim Jagielski (nSourceSize - nSourceCount) * sizeof( sal_Unicode ) );
543*b1cdbd2cSJim Jagielski }
544*b1cdbd2cSJim Jagielski
545*b1cdbd2cSJim Jagielski if( puTempMem ) {
546*b1cdbd2cSJim Jagielski delete [] puTempMem;
547*b1cdbd2cSJim Jagielski }
548*b1cdbd2cSJim Jagielski
549*b1cdbd2cSJim Jagielski // reduce the size of the buffer (fast, no copy necessary)
550*b1cdbd2cSJim Jagielski seqText.realloc( nTargetCount );
551*b1cdbd2cSJim Jagielski
552*b1cdbd2cSJim Jagielski return seqText;
553*b1cdbd2cSJim Jagielski }
554*b1cdbd2cSJim Jagielski
init(rtl_TextEncoding encoding)555*b1cdbd2cSJim Jagielski void Unicode2TextConverter::init( rtl_TextEncoding encoding )
556*b1cdbd2cSJim Jagielski {
557*b1cdbd2cSJim Jagielski m_bCanContinue = sal_True;
558*b1cdbd2cSJim Jagielski m_bInitialized = sal_True;
559*b1cdbd2cSJim Jagielski
560*b1cdbd2cSJim Jagielski m_convUnicode2Text = rtl_createUnicodeToTextConverter( encoding );
561*b1cdbd2cSJim Jagielski m_contextUnicode2Text = rtl_createUnicodeToTextContext( m_convUnicode2Text );
562*b1cdbd2cSJim Jagielski m_rtlEncoding = encoding;
563*b1cdbd2cSJim Jagielski };
564*b1cdbd2cSJim Jagielski
565*b1cdbd2cSJim Jagielski
566*b1cdbd2cSJim Jagielski }
567