xref: /aoo4110/main/l10ntools/inc/export.hxx (revision b1cdbd2c) 
1 /**************************************************************
2  *
3  * Licensed to the Apache Software Foundation (ASF) under one
4  * or more contributor license agreements.  See the NOTICE file
5  * distributed with this work for additional information
6  * regarding copyright ownership.  The ASF licenses this file
7  * to you under the Apache License, Version 2.0 (the
8  * "License"); you may not use this file except in compliance
9  * with the License.  You may obtain a copy of the License at
10  *
11  *   http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing,
14  * software distributed under the License is distributed on an
15  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16  * KIND, either express or implied.  See the License for the
17  * specific language governing permissions and limitations
18  * under the License.
19  *
20  *************************************************************/
21 
22 
23 
24 #ifndef _EXPORT_HXX
25 #define _EXPORT_HXX
26 
27 #include <l10ntools/directory.hxx>
28 
29 #include <tagtest.hxx>
30 
31 // #define MERGE_SOURCE_LANGUAGES <- To merge en-US and de resource
32 
33 #include <tools/string.hxx>
34 #include <tools/list.hxx>
35 #include <tools/stream.hxx>
36 #include <tools/fsys.hxx>
37 #include <osl/file.hxx>
38 #include <osl/file.h>
39 
40 #include <hash_map> /* std::hashmap*/
41 #include <iterator> /* std::iterator*/
42 #include <set>      /* std::set*/
43 #include <vector>   /* std::vector*/
44 #include <queue>
45 #include <string>
46 
47 #include <unistd.h>
48 #ifdef WNT
49 #include <direct.h>
50 #endif
51 
52 #define NO_TRANSLATE_ISO		"x-no-translate"
53 
54 #define JAPANESE_ISO "ja"
55 
56 class PFormEntrys;
57 class MergeData;
58 typedef std::set<ByteString , lessByteString > ByteStringSet;
59 
60 typedef std::hash_map<ByteString , ByteString , hashByteString,equalByteString>
61                                 ByteStringHashMap;
62 
63 typedef std::hash_map<ByteString , bool , hashByteString,equalByteString>
64                                 ByteStringBoolHashMap;
65 
66 typedef std::hash_map<ByteString , PFormEntrys* , hashByteString,equalByteString>
67                                 PFormEntrysHashMap;
68 
69 typedef std::hash_map<ByteString , MergeData* , hashByteString,equalByteString>
70                                 MergeDataHashMap;
71 
72 #define SOURCE_LANGUAGE ByteString("en-US")
73 #define LIST_REFID  "LIST_REFID"
74 
75 typedef ByteStringHashMap ExportListEntry;
76 
77 DECLARE_LIST( ExportListBase, ExportListEntry * )
78 
79 //
80 // class ExportList
81 //
82 
83 class ExportList : public ExportListBase
84 {
85 private:
86 	sal_uLong nSourceLanguageListEntryCount;
87 
88 public:
ExportList()89 	ExportList() : ExportListBase() { nSourceLanguageListEntryCount = 0; }
GetSourceLanguageListEntryCount()90 	sal_uLong GetSourceLanguageListEntryCount() { return nSourceLanguageListEntryCount; }
NewSourceLanguageListEntry()91 	void NewSourceLanguageListEntry() { nSourceLanguageListEntryCount++; }
92 };
93 
94 #define REFID_NONE 0xFFFF
95 
96 //
97 // struct ResData
98 //
99 
100 /******************************************************************************
101 * Purpose: holds mandatory data to export a single res (used with ResStack)
102 ******************************************************************************/
103 
104 #define ID_LEVEL_NULL		0x0000
105 #define ID_LEVEL_AUTOID		0x0001
106 #define ID_LEVEL_TEXT		0x0002
107 #define ID_LEVEL_FIELDNAME	0x0003
108 #define ID_LEVEL_ACCESSPATH	0x0004
109 #define ID_LEVEL_IDENTIFIER 0x0005
110 #define ID_LEVEL_LISTINDEX	0x0006
111 
112 class ResData
113 {
114 public:
115 	~ResData();
116 	sal_Bool SetId( const ByteString &rId, sal_uInt16 nLevel );
117 
118     sal_uInt16 nWidth;
119 	sal_uInt16 nChildIndex;
120 	sal_uInt16 nIdLevel;
121 	sal_Bool bChild;
122 	sal_Bool bChildWithText;
123 
124 	sal_Bool bText;
125 	sal_Bool bHelpText;
126 	sal_Bool bQuickHelpText;
127 	sal_Bool bTitle;
128 	sal_Bool bList;
129 
130 	sal_Bool bRestMerged;
131 
132     ByteString sResTyp;
133 	ByteString sId;
134 	ByteString sGId;
135 	ByteString sHelpId;
136 	ByteString sFilename;
137 
138     ByteStringHashMap sText;
139     sal_uInt16 nTextRefId;
140 
141 	ByteStringHashMap sHelpText;
142     sal_uInt16 nHelpTextRefId;
143 
144 	ByteStringHashMap sQuickHelpText;
145     sal_uInt16 nQuickHelpTextRefId;
146 
147 	ByteStringHashMap sTitle;
148     sal_uInt16 nTitleRefId;
149 
150 	ByteString sTextTyp;
151 	ByteStringHashMap aFallbackData;
152 	ByteStringHashMap aMergedLanguages;
153 
154 	ExportList	*pStringList;
155 	ExportList	*pUIEntries;
156 	ExportList	*pItemList;
157     ExportList	*pFilterList;
158     ExportList  *pPairedList;
159 
160     ByteString sPForm;
161 
162 	void Dump();
163 	void addFallbackData( ByteString& sId , const ByteString& sText );
164 	bool getFallbackData( ByteString& sId , ByteString& sText);
165 
166 	void addMergedLanguage( ByteString& sLang );
167 	bool isMerged( ByteString& sLang );
ResData(const ByteString & rPF,const ByteString & rGId)168 	ResData( const ByteString &rPF, const ByteString &rGId )
169 			:
170             nWidth( 0 ),
171             nChildIndex( 0 ),
172             nIdLevel( ID_LEVEL_NULL ),
173             bChild( sal_False ),
174             bChildWithText( sal_False ),
175             bText( sal_False ),
176             bHelpText( sal_False ),
177             bQuickHelpText( sal_False ),
178             bTitle( sal_False ),
179             bList( sal_False ),
180             bRestMerged( sal_False ),
181             sGId( rGId ),
182             nTextRefId( REFID_NONE ),
183             nHelpTextRefId( REFID_NONE ),
184             nQuickHelpTextRefId( REFID_NONE ),
185             nTitleRefId( REFID_NONE ),
186             sTextTyp( "Text" ),
187             pStringList( NULL ),
188             pUIEntries( NULL ),
189             pItemList( NULL ),
190             pFilterList( NULL ),
191             pPairedList( NULL ),
192             sPForm( rPF )
193 	{
194 		sGId.EraseAllChars( '\r' );
195 		sPForm.EraseAllChars( '\r' );
196 	};
ResData(const ByteString & rPF,const ByteString & rGId,const ByteString & rFilename)197 	ResData( const ByteString &rPF, const ByteString &rGId , const ByteString &rFilename )
198 			:
199             nChildIndex( 0 ),
200             nIdLevel( ID_LEVEL_NULL ),
201             bChild( sal_False ),
202             bChildWithText( sal_False ),
203             bText( sal_False ),
204             bHelpText( sal_False ),
205             bQuickHelpText( sal_False ),
206             bTitle( sal_False ),
207             bList( sal_False ),
208             bRestMerged( sal_False ),
209             sGId( rGId ),
210             sFilename( rFilename ),
211             nTextRefId( REFID_NONE ),
212             nHelpTextRefId( REFID_NONE ),
213             nQuickHelpTextRefId( REFID_NONE ),
214             nTitleRefId( REFID_NONE ),
215             sTextTyp( "Text" ),
216             pStringList( NULL ),
217             pUIEntries( NULL ),
218             pItemList( NULL ),
219             pFilterList( NULL ),
220             pPairedList( NULL ),
221             sPForm( rPF )
222 
223 	{
224 		sGId.EraseAllChars( '\r' );
225 		sPForm.EraseAllChars( '\r' );
226 	};
227 
228 
229 };
230 
231 
232 //
233 // class Export
234 //
235 
236 /******************************************************************************
237 * Purpose: syntax check and export of *.src, called from lexer
238 ******************************************************************************/
239 
240 #define LIST_NON 					0x0000
241 #define LIST_STRING 				0x0001
242 #define LIST_FILTER					0x0002
243 #define LIST_ITEM					0x0004
244 #define LIST_PAIRED                 0x0005
245 #define LIST_UIENTRIES				0x0008
246 #define STRING_TYP_TEXT				0x0010
247 #define	STRING_TYP_HELPTEXT			0x0020
248 #define STRING_TYP_QUICKHELPTEXT	0x0040
249 #define STRING_TYP_TITLE			0x0080
250 
251 #define MERGE_MODE_NORMAL			0x0000
252 #define MERGE_MODE_LIST				0x0001
253 
254 DECLARE_LIST( ResStack, ResData * )
255 // forwards
256 class WordTransformer;
257 class ParserQueue;
258 
259 class Export
260 {
261 private:
262 	WordTransformer	*pWordTransformer;
263 
264 	CharSet	aCharSet;					// used charset in src
265 
266 	SvFileStream aOutput;
267 
268 	ResStack aResStack;					// stack for parsing recursive
269 
270 	ByteString sActPForm;				// hold cur. system
271 
272 	sal_Bool bDefine;						// cur. res. in a define?
273 	sal_Bool bNextMustBeDefineEOL;			// define but no \ at lineend
274 	sal_uLong nLevel;						// res. recursiv? how deep?
275 	sal_uInt16 nList;						// cur. res. is String- or FilterList
276     ByteString nListLang;
277     sal_uLong nListIndex;
278 	sal_uLong nListLevel;
279     bool bSkipFile;
280 	ByteString sProject;
281 	ByteString sRoot;
282 	sal_Bool bEnableExport;
283 	sal_Bool bMergeMode;
284 	ByteString sMergeSrc;
285 	ByteString sLastListLine;
286 	sal_Bool bError;						// any errors while export?
287 	sal_Bool bReadOver;
288 	sal_Bool bDontWriteOutput;
289 	ByteString sLastTextTyp;
290     static bool isInitialized;
291 	ByteString sFilename;
292 
293 
294 public:
295 	ParserQueue* pParseQueue; // public ?
296     static ByteString sLanguages; // public ?
297     static ByteString sForcedLanguages; // public ?
298 
299 
300     static bool skipProject( ByteString sPrj ) ;
301 	static void InitLanguages( bool bMergeMode = false );
302     static void InitForcedLanguages( bool bMergeMode = false );
303     static std::vector<ByteString> GetLanguages();
304     static std::vector<ByteString> GetForcedLanguages();
305 
306     static void SetLanguages( std::vector<ByteString> val );
307     static void RemoveUTF8ByteOrderMarker( ByteString &rString );
308     static bool hasUTF8ByteOrderMarker( const ByteString &rString );
309     static void RemoveUTF8ByteOrderMarkerFromFile( const ByteString &rFilename );
310     static bool fileHasUTF8ByteOrderMarker( const ByteString &rString );
311 	static ByteString GetIsoLangByIndex( sal_uInt16 nIndex );
312 	static void QuotHTML( ByteString &rString );
313     static bool CopyFile( const ByteString& source , const ByteString& dest );
314 
315 	static void QuotHTMLXRM( ByteString &rString );
316     static void UnquotHTML( ByteString &rString );
317 
318     static const char* GetEnv( const char *pVar );
319 	static int getCurrentDirectory( rtl::OUString& base_fqurl , rtl::OUString& base );
320 
321     static bool isSourceLanguage( const ByteString &sLanguage );
322 	static bool isAllowed( const ByteString &sLanguage );
323 
324     static bool LanguageAllowed( const ByteString &nLanguage );
325     static void Languages( std::vector<ByteString>::const_iterator& begin , std::vector<ByteString>::const_iterator& end );
326 	static void getRandomName( const ByteString& sPrefix , ByteString& sRandStr , const ByteString& sPostfix  );
327     static void getRandomName( ByteString& sRandStr );
328     static void getCurrentDir( std::string& dir );
329 
330     static void replaceEncoding( ByteString& rString );
331 
332 	static ByteString GetFallbackLanguage( const ByteString nLanguage );
333 	static void FillInFallbacks( ResData *pResData );
334     static void FillInListFallbacks( ExportList *pList, const ByteString &nSource, const ByteString &nFallback );
335     static ByteString GetTimeStamp();
336 	static sal_Bool ConvertLineEnds( ByteString sSource, ByteString sDestination );
337 	static ByteString GetNativeFile( ByteString sSource );
338 	static DirEntry GetTempFile();
339 
340 	static void DumpExportList( ByteString& sListName , ExportList& aList );
341 	static ByteString DumpMap( ByteString& sMapName , ByteStringHashMap& aMap );
342 
343 private:
344     static std::vector<ByteString> aLanguages;
345     static std::vector<ByteString> aForcedLanguages;
346 
347 	sal_Bool ListExists( ResData *pResData, sal_uInt16 nLst );
348 
349 	sal_Bool WriteData( ResData *pResData, sal_Bool bCreateNew = sal_False );// called befor dest. cur ResData
350 	sal_Bool WriteExportList( ResData *pResData, ExportList *pExportList,
351 						const ByteString &rTyp, sal_Bool bCreateNew = sal_False );
352 
353 	ByteString MergePairedList( ByteString& sLine , ByteString& sText );
354 
355 	ByteString FullId();					// creates cur. GID
356 
357 	bool PairedListFallback( ByteString& sText , ResData& aResData );
358 
359 	ByteString GetPairedListID		( const ByteString& sText );
360     ByteString GetPairedListString	( const ByteString& sText );
361 	ByteString StripList	( const ByteString& sText );
362 
363 	void UnmergeUTF8( ByteString& sOrig );
364 	void InsertListEntry( const ByteString &rText, const ByteString &rLine );
365 	void CleanValue( ByteString &rValue );
366 	ByteString GetText( const ByteString &rSource, int nToken );
367 
368 	sal_Bool PrepareTextToMerge( ByteString &rText, sal_uInt16 nTyp,
369 		ByteString &nLangIndex, ResData *pResData );
370 
371 	void MergeRest( ResData *pResData, sal_uInt16 nMode = MERGE_MODE_NORMAL );
372 	void ConvertMergeContent( ByteString &rText );
373 
374   	void WriteToMerged( const ByteString &rText , bool bSDFContent );
375 	void SetChildWithText();
376 
377 	void CutComment( ByteString &rText );
378 
379 public:
380 	Export( const ByteString &rOutput, sal_Bool bWrite,
381 			const ByteString &rPrj, const ByteString &rPrjRoot , const ByteString& rFile );
382 	Export( const ByteString &rOutput, sal_Bool bWrite,
383 			const ByteString &rPrj, const ByteString &rPrjRoot,
384 			const ByteString &rMergeSource , const ByteString& rFile );
385 	~Export();
386 
387 	void Init();
388 	int Execute( int nToken, const char * pToken );	// called from lexer
SetError()389 	void SetError() { bError = sal_True; }
GetError()390 	sal_Bool GetError() { return bError; }
391 };
392 
393 
394 //
395 // class PFormEntrys
396 //
397 
398 /******************************************************************************
399 * Purpose: holds information of data to merge (one pform)
400 ******************************************************************************/
401 
402 class PFormEntrys : public ByteString
403 {
404 friend class MergeDataFile;
405 private:
406 	ByteString sHelpText; // empty string
407 	ByteStringHashMap sText;
408 	ByteStringBoolHashMap bTextFirst;
409 	ByteStringHashMap sQuickHelpText;
410 	ByteStringBoolHashMap bQuickHelpTextFirst;
411 	ByteStringHashMap sTitle;
412 	ByteStringBoolHashMap bTitleFirst;
413 
414 public:
PFormEntrys(const ByteString & rPForm)415 	PFormEntrys( const ByteString &rPForm ) : ByteString( rPForm ) {};
416 	ByteString Dump();
InsertEntry(const ByteString & nId,const ByteString & rText,const ByteString & rQuickHelpText,const ByteString & rTitle)417 	void InsertEntry(
418                     const ByteString &nId ,
419                     const ByteString &rText,
420 					const ByteString &rQuickHelpText,
421 					const ByteString &rTitle
422                     )
423 		{
424 			sText[ nId ] = rText;
425 			bTextFirst[ nId ] = true;
426 			sQuickHelpText[ nId ] = rQuickHelpText;
427 			bQuickHelpTextFirst[ nId ] = true;
428 			sTitle[ nId ] = rTitle;
429 			bTitleFirst[ nId ] = true;
430 		}
431      sal_Bool GetText( ByteString &rReturn, sal_uInt16 nTyp, const ByteString &nLangIndex, sal_Bool bDel = sal_False );
432      sal_Bool GetTransex3Text( ByteString &rReturn, sal_uInt16 nTyp, const ByteString &nLangIndex, sal_Bool bDel = sal_False );
433 
434 };
435 
436 //
437 // class MergeData
438 //
439 
440 /******************************************************************************
441 * Purpose: holds information of data to merge (one ressource)
442 ******************************************************************************/
443 
444 class MergeDataFile;
445 
446 class MergeData
447 {
448 friend class MergeDataFile;
449 private:
450 	ByteString sTyp;
451 	ByteString sGID;
452 	ByteString sLID;
453     ByteString sFilename;
454     PFormEntrysHashMap aMap;
455 public:
MergeData(const ByteString & rTyp,const ByteString & rGID,const ByteString & rLID,const ByteString & rFilename)456 	MergeData( const ByteString &rTyp, const ByteString &rGID, const ByteString &rLID , const ByteString &rFilename )
457 			: sTyp( rTyp ), sGID( rGID ), sLID( rLID ) , sFilename( rFilename ) {};
458 	~MergeData();
459 	PFormEntrys* InsertEntry( const ByteString &rPForm );
460 	PFormEntrys* GetPFormEntrys( ResData *pResData );
461 
462     void Insert( const ByteString& rPFO , PFormEntrys* pfEntrys );
463     PFormEntrys* GetPFObject( const ByteString& rPFO );
464 
465 	ByteString Dump();
466 	sal_Bool operator==( ResData *pData );
467 };
468 
469 //
470 // class MergeDataFile
471 //
472 
473 /******************************************************************************
474 * Purpose: holds information of data to merge
475 ******************************************************************************/
476 
477 class MergeDataFile
478 {
479     private:
480         sal_Bool bErrorLog;
481         ByteString sErrorLog;
482         SvFileStream aErrLog;
483         MergeDataHashMap aMap;
484         std::set<ByteString> aLanguageSet;
485 
486         MergeData *GetMergeData( ResData *pResData , bool bCaseSensitve = false );
487         void InsertEntry( const ByteString &rTYP, const ByteString &rGID, const ByteString &rLID,
488             const ByteString &rPFO,
489             const ByteString &nLang, const ByteString &rTEXT,
490             const ByteString &rQHTEXT, const ByteString &rTITLE,
491             const ByteString &sFilename, bool bCaseSensitive
492             );
493         ByteString Dump();
494         void WriteError( const ByteString &rLine );
495 
496     public:
497         MergeDataFile( const ByteString &rFileName, const ByteString& rFile , sal_Bool bErrLog, CharSet aCharSet, bool bCaseSensitive = false );
498         ~MergeDataFile();
499 
500         std::vector<ByteString> GetLanguages();
501 
502         PFormEntrys *GetPFormEntrys( ResData *pResData );
503         PFormEntrys *GetPFormEntrysCaseSensitive( ResData *pResData );
504 
505         static ByteString CreateKey( const ByteString& rTYP , const ByteString& rGID , const ByteString& rLID , const ByteString& rFilename , bool bCaseSensitive = false );
506 };
507 
508 
509 class QueueEntry
510 {
511 public:
QueueEntry(int nTypVal,ByteString sLineVal)512     QueueEntry( int nTypVal , ByteString sLineVal ): nTyp( nTypVal ) , sLine( sLineVal ){};
513     int nTyp;
514     ByteString sLine;
515 };
516 
517 class ParserQueue
518 {
519 public:
520 
521     ParserQueue( Export& aExportObj );
522     ~ParserQueue();
523 
524     inline void Push( const QueueEntry& aEntry );
525     bool bCurrentIsM;  // public ?
526     bool bNextIsM;   // public ?
527     bool bLastWasM;   // public ?
528     bool bMflag;   // public ?
529 
530     void Close();
531 private:
532     // Future / Next
533     std::queue<QueueEntry>* aQueueNext;
534     // Current
535     std::queue<QueueEntry>* aQueueCur;
536     // Ref
537     std::queue<QueueEntry>* aQref;
538 
539     Export& aExport;
540     bool bStart;
541     bool bStartNext;
542 
543     inline void Pop( std::queue<QueueEntry>& aQueue );
544 
545 };
546 #endif
547 
548