xref: /aoo42x/main/l10ntools/inc/export.hxx (revision 983d4c8a) 
1 /**************************************************************
2  *
3  * Licensed to the Apache Software Foundation (ASF) under one
4  * or more contributor license agreements.  See the NOTICE file
5  * distributed with this work for additional information
6  * regarding copyright ownership.  The ASF licenses this file
7  * to you under the Apache License, Version 2.0 (the
8  * "License"); you may not use this file except in compliance
9  * with the License.  You may obtain a copy of the License at
10  *
11  *   http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing,
14  * software distributed under the License is distributed on an
15  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16  * KIND, either express or implied.  See the License for the
17  * specific language governing permissions and limitations
18  * under the License.
19  *
20  *************************************************************/
21 
22 
23 
24 #ifndef _EXPORT_HXX
25 #define _EXPORT_HXX
26 
27 #ifndef L10NTOOLS_DIRECTORY_HXX
28 #define L10NTOOLS_DIRECTORY_HXX
29 #include <l10ntools/directory.hxx>
30 #endif
31 
32 
33 // #define MERGE_SOURCE_LANGUAGES <- To merge en-US and de resource
34 
35 #include <tools/string.hxx>
36 #include <tools/list.hxx>
37 #include <tools/stream.hxx>
38 #include <tools/fsys.hxx>
39 #include <osl/file.hxx>
40 #include <osl/file.h>
41 
42 #include <hash_map> /* std::hashmap*/
43 #include <iterator> /* std::iterator*/
44 #include <set>      /* std::set*/
45 #include <vector>   /* std::vector*/
46 #include <queue>
47 #include <string>
48 
49 #include <unistd.h>
50 #ifdef WNT
51 #include <direct.h>
52 #endif
53 
54 #define NO_TRANSLATE_ISO		"x-no-translate"
55 
56 #define JAPANESE_ISO "ja"
57 
58 
59 struct eqstr{
60   sal_Bool operator()(const char* s1, const char* s2) const{
61     return strcmp(s1,s2)==0;
62   }
63 };
64 
65 struct equalByteString{
66         bool operator()( const ByteString& rKey1, const ByteString& rKey2 ) const {
67             return rKey1.CompareTo( rKey2 )==COMPARE_EQUAL;
68     }
69 };
70 struct lessByteString{
71         bool operator()( const ByteString& rKey1, const ByteString& rKey2 ) const {
72             return rKey1.CompareTo( rKey2 )==COMPARE_LESS;
73     }
74 };
75 
76 struct hashByteString{
77     size_t operator()( const ByteString& rName ) const{
78                 std::hash< const char* > myHash;
79                 return myHash( rName.GetBuffer() );
80     }
81 };
82 
83 class PFormEntrys;
84 class MergeData;
85 typedef std::set<ByteString , lessByteString > ByteStringSet;
86 
87 typedef std::hash_map<ByteString , ByteString , hashByteString,equalByteString>
88                                 ByteStringHashMap;
89 
90 typedef std::hash_map<ByteString , bool , hashByteString,equalByteString>
91                                 ByteStringBoolHashMap;
92 
93 typedef std::hash_map<ByteString , PFormEntrys* , hashByteString,equalByteString>
94                                 PFormEntrysHashMap;
95 
96 typedef std::hash_map<ByteString , MergeData* , hashByteString,equalByteString>
97                                 MergeDataHashMap;
98 
99 #define SOURCE_LANGUAGE ByteString("en-US")
100 #define LIST_REFID  "LIST_REFID"
101 
102 typedef ByteStringHashMap ExportListEntry;
103 
104 DECLARE_LIST( ExportListBase, ExportListEntry * )
105 
106 //
107 // class ExportList
108 //
109 
110 class ExportList : public ExportListBase
111 {
112 private:
113 	sal_uLong nSourceLanguageListEntryCount;
114 
115 public:
116 	ExportList() : ExportListBase() { nSourceLanguageListEntryCount = 0; }
117 	sal_uLong GetSourceLanguageListEntryCount() { return nSourceLanguageListEntryCount; }
118 	void NewSourceLanguageListEntry() { nSourceLanguageListEntryCount++; }
119 };
120 
121 #define REFID_NONE 0xFFFF
122 
123 //
124 // struct ResData
125 //
126 
127 /******************************************************************************
128 * Purpose: holds mandatory data to export a single res (used with ResStack)
129 ******************************************************************************/
130 
131 #define ID_LEVEL_NULL		0x0000
132 #define ID_LEVEL_AUTOID		0x0001
133 #define ID_LEVEL_TEXT		0x0002
134 #define ID_LEVEL_FIELDNAME	0x0003
135 #define ID_LEVEL_ACCESSPATH	0x0004
136 #define ID_LEVEL_IDENTIFIER 0x0005
137 #define ID_LEVEL_LISTINDEX	0x0006
138 
139 class ResData
140 {
141 public:
142 	~ResData();
143 	sal_Bool SetId( const ByteString &rId, sal_uInt16 nLevel );
144 
145     sal_uInt16 nWidth;
146 	sal_uInt16 nChildIndex;
147 	sal_uInt16 nIdLevel;
148 	sal_Bool bChild;
149 	sal_Bool bChildWithText;
150 
151 	sal_Bool bText;
152 	sal_Bool bHelpText;
153 	sal_Bool bQuickHelpText;
154 	sal_Bool bTitle;
155 	sal_Bool bList;
156 
157 	sal_Bool bRestMerged;
158 
159     ByteString sResTyp;
160 	ByteString sId;
161 	ByteString sGId;
162 	ByteString sHelpId;
163 	ByteString sFilename;
164 
165     ByteStringHashMap sText;
166     sal_uInt16 nTextRefId;
167 
168 	ByteStringHashMap sHelpText;
169     sal_uInt16 nHelpTextRefId;
170 
171 	ByteStringHashMap sQuickHelpText;
172     sal_uInt16 nQuickHelpTextRefId;
173 
174 	ByteStringHashMap sTitle;
175     sal_uInt16 nTitleRefId;
176 
177 	ByteString sTextTyp;
178 	ByteStringHashMap aFallbackData;
179 	ByteStringHashMap aMergedLanguages;
180 
181 	ExportList	*pStringList;
182 	ExportList	*pUIEntries;
183 	ExportList	*pItemList;
184     ExportList	*pFilterList;
185     ExportList  *pPairedList;
186 
187     ByteString sPForm;
188 
189 	void Dump();
190 	void addFallbackData( ByteString& sId , const ByteString& sText );
191 	bool getFallbackData( ByteString& sId , ByteString& sText);
192 
193 	void addMergedLanguage( ByteString& sLang );
194 	bool isMerged( ByteString& sLang );
195 	ResData( const ByteString &rPF, const ByteString &rGId )
196 			:
197             nWidth( 0 ),
198             nChildIndex( 0 ),
199             nIdLevel( ID_LEVEL_NULL ),
200             bChild( sal_False ),
201             bChildWithText( sal_False ),
202             bText( sal_False ),
203             bHelpText( sal_False ),
204             bQuickHelpText( sal_False ),
205             bTitle( sal_False ),
206             bList( sal_False ),
207             bRestMerged( sal_False ),
208             sGId( rGId ),
209             nTextRefId( REFID_NONE ),
210             nHelpTextRefId( REFID_NONE ),
211             nQuickHelpTextRefId( REFID_NONE ),
212             nTitleRefId( REFID_NONE ),
213             sTextTyp( "Text" ),
214             pStringList( NULL ),
215             pUIEntries( NULL ),
216             pItemList( NULL ),
217             pFilterList( NULL ),
218             pPairedList( NULL ),
219             sPForm( rPF )
220 	{
221 		sGId.EraseAllChars( '\r' );
222 		sPForm.EraseAllChars( '\r' );
223 	};
224 	ResData( const ByteString &rPF, const ByteString &rGId , const ByteString &rFilename )
225 			:
226             nChildIndex( 0 ),
227             nIdLevel( ID_LEVEL_NULL ),
228             bChild( sal_False ),
229             bChildWithText( sal_False ),
230             bText( sal_False ),
231             bHelpText( sal_False ),
232             bQuickHelpText( sal_False ),
233             bTitle( sal_False ),
234             bList( sal_False ),
235             bRestMerged( sal_False ),
236             sGId( rGId ),
237             sFilename( rFilename ),
238             nTextRefId( REFID_NONE ),
239             nHelpTextRefId( REFID_NONE ),
240             nQuickHelpTextRefId( REFID_NONE ),
241             nTitleRefId( REFID_NONE ),
242             sTextTyp( "Text" ),
243             pStringList( NULL ),
244             pUIEntries( NULL ),
245             pItemList( NULL ),
246             pFilterList( NULL ),
247             pPairedList( NULL ),
248             sPForm( rPF )
249 
250 	{
251 		sGId.EraseAllChars( '\r' );
252 		sPForm.EraseAllChars( '\r' );
253 	};
254 
255 
256 };
257 
258 
259 //
260 // class Export
261 //
262 
263 /******************************************************************************
264 * Purpose: syntax check and export of *.src, called from lexer
265 ******************************************************************************/
266 
267 #define LIST_NON 					0x0000
268 #define LIST_STRING 				0x0001
269 #define LIST_FILTER					0x0002
270 #define LIST_ITEM					0x0004
271 #define LIST_PAIRED                 0x0005
272 #define LIST_UIENTRIES				0x0008
273 #define STRING_TYP_TEXT				0x0010
274 #define	STRING_TYP_HELPTEXT			0x0020
275 #define STRING_TYP_QUICKHELPTEXT	0x0040
276 #define STRING_TYP_TITLE			0x0080
277 
278 #define MERGE_MODE_NORMAL			0x0000
279 #define MERGE_MODE_LIST				0x0001
280 
281 DECLARE_LIST( ResStack, ResData * )
282 // forwards
283 class WordTransformer;
284 class ParserQueue;
285 
286 class Export
287 {
288 private:
289 	WordTransformer	*pWordTransformer;
290 
291 	CharSet	aCharSet;					// used charset in src
292 
293 	SvFileStream aOutput;
294 
295 	ResStack aResStack;					// stack for parsing recursive
296 
297 	ByteString sActPForm;				// hold cur. system
298 
299 	sal_Bool bDefine;						// cur. res. in a define?
300 	sal_Bool bNextMustBeDefineEOL;			// define but no \ at lineend
301 	sal_uLong nLevel;						// res. recursiv? how deep?
302 	sal_uInt16 nList;						// cur. res. is String- or FilterList
303     ByteString nListLang;
304     sal_uLong nListIndex;
305 	sal_uLong nListLevel;
306     bool bSkipFile;
307 	ByteString sProject;
308 	ByteString sRoot;
309 	sal_Bool bEnableExport;
310 	sal_Bool bMergeMode;
311 	ByteString sMergeSrc;
312 	ByteString sLastListLine;
313 	sal_Bool bError;						// any errors while export?
314 	sal_Bool bReadOver;
315 	sal_Bool bDontWriteOutput;
316 	ByteString sLastTextTyp;
317     static bool isInitialized;
318 	ByteString sFilename;
319 
320 
321 public:
322 	ParserQueue* pParseQueue; // public ?
323     static ByteString sLanguages; // public ?
324     static ByteString sForcedLanguages; // public ?
325 
326 
327     static bool skipProject( ByteString sPrj ) ;
328 	static void InitLanguages( bool bMergeMode = false );
329     static void InitForcedLanguages( bool bMergeMode = false );
330     static std::vector<ByteString> GetLanguages();
331     static std::vector<ByteString> GetForcedLanguages();
332 
333     static void SetLanguages( std::vector<ByteString> val );
334     static void RemoveUTF8ByteOrderMarker( ByteString &rString );
335     static bool hasUTF8ByteOrderMarker( const ByteString &rString );
336     static void RemoveUTF8ByteOrderMarkerFromFile( const ByteString &rFilename );
337     static bool fileHasUTF8ByteOrderMarker( const ByteString &rString );
338 	static ByteString GetIsoLangByIndex( sal_uInt16 nIndex );
339 	static void QuotHTML( ByteString &rString );
340     static bool CopyFile( const ByteString& source , const ByteString& dest );
341 
342 	static void QuotHTMLXRM( ByteString &rString );
343     static void UnquotHTML( ByteString &rString );
344 
345     static const char* GetEnv( const char *pVar );
346 	static int getCurrentDirectory( rtl::OUString& base_fqurl , rtl::OUString& base );
347 
348     static bool isSourceLanguage( const ByteString &sLanguage );
349 	static bool isAllowed( const ByteString &sLanguage );
350 
351     static bool LanguageAllowed( const ByteString &nLanguage );
352     static void Languages( std::vector<ByteString>::const_iterator& begin , std::vector<ByteString>::const_iterator& end );
353 	static void getRandomName( const ByteString& sPrefix , ByteString& sRandStr , const ByteString& sPostfix  );
354     static void getRandomName( ByteString& sRandStr );
355     static void getCurrentDir( std::string& dir );
356 
357     static void replaceEncoding( ByteString& rString );
358 
359 	static ByteString GetFallbackLanguage( const ByteString nLanguage );
360 	static void FillInFallbacks( ResData *pResData );
361     static void FillInListFallbacks( ExportList *pList, const ByteString &nSource, const ByteString &nFallback );
362     static ByteString GetTimeStamp();
363 	static sal_Bool ConvertLineEnds( ByteString sSource, ByteString sDestination );
364 	static ByteString GetNativeFile( ByteString sSource );
365 	static DirEntry GetTempFile();
366 
367 	static void DumpExportList( ByteString& sListName , ExportList& aList );
368 	static ByteString DumpMap( ByteString& sMapName , ByteStringHashMap& aMap );
369 
370 private:
371     static std::vector<ByteString> aLanguages;
372     static std::vector<ByteString> aForcedLanguages;
373 
374 	sal_Bool ListExists( ResData *pResData, sal_uInt16 nLst );
375 
376 	sal_Bool WriteData( ResData *pResData, sal_Bool bCreateNew = sal_False );// called befor dest. cur ResData
377 	sal_Bool WriteExportList( ResData *pResData, ExportList *pExportList,
378 						const ByteString &rTyp, sal_Bool bCreateNew = sal_False );
379 
380 	ByteString MergePairedList( ByteString& sLine , ByteString& sText );
381 
382 	ByteString FullId();					// creates cur. GID
383 
384 	bool PairedListFallback( ByteString& sText , ResData& aResData );
385 
386 	ByteString GetPairedListID		( const ByteString& sText );
387     ByteString GetPairedListString	( const ByteString& sText );
388 	ByteString StripList	( const ByteString& sText );
389 
390 	void UnmergeUTF8( ByteString& sOrig );
391 	void InsertListEntry( const ByteString &rText, const ByteString &rLine );
392 	void CleanValue( ByteString &rValue );
393 	ByteString GetText( const ByteString &rSource, int nToken );
394 
395 	sal_Bool PrepareTextToMerge( ByteString &rText, sal_uInt16 nTyp,
396 		ByteString &nLangIndex, ResData *pResData );
397 
398 	void MergeRest( ResData *pResData, sal_uInt16 nMode = MERGE_MODE_NORMAL );
399 	void ConvertMergeContent( ByteString &rText );
400 
401   	void WriteToMerged( const ByteString &rText , bool bSDFContent );
402 	void SetChildWithText();
403 
404 	void CutComment( ByteString &rText );
405 
406 public:
407 	Export( const ByteString &rOutput, sal_Bool bWrite,
408 			const ByteString &rPrj, const ByteString &rPrjRoot , const ByteString& rFile );
409 	Export( const ByteString &rOutput, sal_Bool bWrite,
410 			const ByteString &rPrj, const ByteString &rPrjRoot,
411 			const ByteString &rMergeSource , const ByteString& rFile );
412 	~Export();
413 
414 	void Init();
415 	int Execute( int nToken, const char * pToken );	// called from lexer
416 	void SetError() { bError = sal_True; }
417 	sal_Bool GetError() { return bError; }
418 };
419 
420 
421 //
422 // class PFormEntrys
423 //
424 
425 /******************************************************************************
426 * Purpose: holds information of data to merge (one pform)
427 ******************************************************************************/
428 
429 class PFormEntrys : public ByteString
430 {
431 friend class MergeDataFile;
432 private:
433 	ByteString sHelpText; // empty string
434 	ByteStringHashMap sText;
435 	ByteStringBoolHashMap bTextFirst;
436 	ByteStringHashMap sQuickHelpText;
437 	ByteStringBoolHashMap bQuickHelpTextFirst;
438 	ByteStringHashMap sTitle;
439 	ByteStringBoolHashMap bTitleFirst;
440 
441 public:
442 	PFormEntrys( const ByteString &rPForm ) : ByteString( rPForm ) {};
443 	ByteString Dump();
444 	void InsertEntry(
445                     const ByteString &nId ,
446                     const ByteString &rText,
447 					const ByteString &rQuickHelpText,
448 					const ByteString &rTitle
449                     )
450 		{
451 			sText[ nId ] = rText;
452 			bTextFirst[ nId ] = true;
453 			sQuickHelpText[ nId ] = rQuickHelpText;
454 			bQuickHelpTextFirst[ nId ] = true;
455 			sTitle[ nId ] = rTitle;
456 			bTitleFirst[ nId ] = true;
457 		}
458      sal_Bool GetText( ByteString &rReturn, sal_uInt16 nTyp, const ByteString &nLangIndex, sal_Bool bDel = sal_False );
459      sal_Bool GetTransex3Text( ByteString &rReturn, sal_uInt16 nTyp, const ByteString &nLangIndex, sal_Bool bDel = sal_False );
460 
461 };
462 
463 //
464 // class MergeData
465 //
466 
467 /******************************************************************************
468 * Purpose: holds information of data to merge (one ressource)
469 ******************************************************************************/
470 
471 class MergeDataFile;
472 
473 class MergeData
474 {
475 friend class MergeDataFile;
476 private:
477 	ByteString sTyp;
478 	ByteString sGID;
479 	ByteString sLID;
480     ByteString sFilename;
481     PFormEntrysHashMap aMap;
482 public:
483 	MergeData( const ByteString &rTyp, const ByteString &rGID, const ByteString &rLID , const ByteString &rFilename )
484 			: sTyp( rTyp ), sGID( rGID ), sLID( rLID ) , sFilename( rFilename ) {};
485 	~MergeData();
486 	PFormEntrys* InsertEntry( const ByteString &rPForm );
487 	PFormEntrys* GetPFormEntrys( ResData *pResData );
488 
489     void Insert( const ByteString& rPFO , PFormEntrys* pfEntrys );
490     PFormEntrys* GetPFObject( const ByteString& rPFO );
491 
492 	ByteString Dump();
493 	sal_Bool operator==( ResData *pData );
494 };
495 
496 //
497 // class MergeDataFile
498 //
499 
500 /******************************************************************************
501 * Purpose: holds information of data to merge
502 ******************************************************************************/
503 
504 class MergeDataFile
505 {
506     private:
507         sal_Bool bErrorLog;
508         ByteString sErrorLog;
509         SvFileStream aErrLog;
510         MergeDataHashMap aMap;
511         std::set<ByteString> aLanguageSet;
512 
513         MergeData *GetMergeData( ResData *pResData , bool bCaseSensitve = false );
514         void InsertEntry( const ByteString &rTYP, const ByteString &rGID, const ByteString &rLID,
515             const ByteString &rPFO,
516             const ByteString &nLang, const ByteString &rTEXT,
517             const ByteString &rQHTEXT, const ByteString &rTITLE,
518             const ByteString &sFilename, bool bCaseSensitive
519             );
520         ByteString Dump();
521         void WriteError( const ByteString &rLine );
522 
523     public:
524         MergeDataFile( const ByteString &rFileName, const ByteString& rFile , sal_Bool bErrLog, CharSet aCharSet, bool bCaseSensitive = false );
525         ~MergeDataFile();
526 
527         std::vector<ByteString> GetLanguages();
528 
529         PFormEntrys *GetPFormEntrys( ResData *pResData );
530         PFormEntrys *GetPFormEntrysCaseSensitive( ResData *pResData );
531 
532         static ByteString CreateKey( const ByteString& rTYP , const ByteString& rGID , const ByteString& rLID , const ByteString& rFilename , bool bCaseSensitive = false );
533 };
534 
535 
536 class QueueEntry
537 {
538 public:
539     QueueEntry( int nTypVal , ByteString sLineVal ): nTyp( nTypVal ) , sLine( sLineVal ){};
540     int nTyp;
541     ByteString sLine;
542 };
543 
544 class ParserQueue
545 {
546 public:
547 
548     ParserQueue( Export& aExportObj );
549     ~ParserQueue();
550 
551     inline void Push( const QueueEntry& aEntry );
552     bool bCurrentIsM;  // public ?
553     bool bNextIsM;   // public ?
554     bool bLastWasM;   // public ?
555     bool bMflag;   // public ?
556 
557     void Close();
558 private:
559     // Future / Next
560     std::queue<QueueEntry>* aQueueNext;
561     // Current
562     std::queue<QueueEntry>* aQueueCur;
563     // Ref
564     std::queue<QueueEntry>* aQref;
565 
566     Export& aExport;
567     bool bStart;
568     bool bStartNext;
569 
570     inline void Pop( std::queue<QueueEntry>& aQueue );
571 
572 };
573 #endif
574 
575