1:
2eval 'exec perl -wS $0 ${1+"$@"}'
3    if 0;
4
5
6#*************************************************************************
7#
8# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
9#
10# Copyright 2000, 2010 Oracle and/or its affiliates.
11#
12# OpenOffice.org - a multi-platform office productivity suite
13#
14# This file is part of OpenOffice.org.
15#
16# OpenOffice.org is free software: you can redistribute it and/or modify
17# it under the terms of the GNU Lesser General Public License version 3
18# only, as published by the Free Software Foundation.
19#
20# OpenOffice.org is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
23# GNU Lesser General Public License version 3 for more details
24# (a copy is included in the LICENSE file that accompanied this code).
25#
26# You should have received a copy of the GNU Lesser General Public License
27# version 3 along with OpenOffice.org.  If not, see
28# <http://www.openoffice.org/license.html>
29# for a copy of the LGPLv3 License.
30#
31#*************************************************************************
32
33use strict;
34use Getopt::Long;
35use IO::Handle;
36use File::Find;
37use File::Temp;
38use File::Copy;
39use File::Glob qw(:glob csh_glob);
40use Cwd;
41
42# ver 1.1
43#
44#### module lookup
45#use lib ("$ENV{SOLARENV}/bin/modules", "$ENV{COMMON_ENV_TOOLS}/modules");
46
47#### module lookup
48# OOo conform
49my @lib_dirs;
50BEGIN {
51    if ( !defined($ENV{SOLARENV}) ) {
52        die "No environment found (environment variable SOLARENV is undefined)";
53    }
54    push(@lib_dirs, "$ENV{SOLARENV}/bin/modules");
55    push(@lib_dirs, "$ENV{COMMON_ENV_TOOLS}/modules") if defined($ENV{COMMON_ENV_TOOLS});
56}
57use lib (@lib_dirs);
58
59#### globals ####
60my $sdffile = '';
61my $no_sort = '';
62my $outputfile = '';
63my $mode = '';
64my $bVerbose="0";
65my $srcpath = '';
66my $languages;
67#my %sl_modules;     # Contains all modules where en-US and de is source language
68my $use_default_date = '0';
69
70         #         (                           leftpart                                                     )            (           rightpart                    )
71         #            prj      file      dummy     type       gid       lid      helpid    pform     width      lang       text    helptext  qhelptext   title    timestamp
72my $sdf_regex  = "((([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*))\t([^\t]*)\t(([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t)([^\t]*))";
73my $file_types = "(src|hrc|xcs|xcu|lng|ulf|xrm|xhp|xcd|xgf|xxl|xrb)";
74# Always use this date to prevent cvs conflicts
75my $default_date = "2002-02-02 02:02:02";
76
77#### main ####
78parse_options();
79
80#%sl_modules = fetch_sourcelanguage_dirlist();
81
82
83if   ( $mode eq "merge"    )    {
84    merge_gsicheck();
85    splitfile( $sdffile );
86    unlink $sdffile;             # remove temp file!
87}
88elsif( $mode eq "extract"  )    {
89    collectfiles( $outputfile );
90}
91else                            {
92    usage();
93}
94
95exit(0);
96
97#########################################################
98sub splitfile{
99
100    my $lastFile        = '';
101    my $currentFile     = '';
102    my $cur_sdffile     = '';
103    my $last_sdffile    = '';
104    my $delim;
105    my $badDelim;
106    my $start           = 'TRUE';
107    my %index  = ();
108    my %block;
109
110    STDOUT->autoflush( 1 );
111
112    #print STDOUT "Open File $sdffile\n";
113    open MYFILE , "< $sdffile"
114    or die "Can't open '$sdffile'\n";
115
116    while( <MYFILE>){
117         if( /$sdf_regex/ ){
118            my $line           = defined $_ ? $_ : '';
119            my $prj            = defined $3 ? $3 : '';
120            my $file           = defined $4 ? $4 : '';
121            my $type           = defined $6 ? $6 : '';
122            my $gid            = defined $7 ? $7 : '';
123            my $lid            = defined $8 ? $8 : '';
124            my $lang           = defined $12 ? $12 : '';
125            my $plattform      = defined $10 ? $10 : '';
126            my $helpid         = defined $9 ? $9 : '';
127
128            next if( $prj eq "binfilter" );     # Don't merge strings into binfilter module
129	        chomp( $line );
130            $currentFile  = $srcpath . '\\' . $prj . '\\' . $file;
131            $currentFile  =~ s/\\/\//g;
132
133            $cur_sdffile = $currentFile;
134            #if( $cur_sdffile =~ /\.$file_types[\s]*$/ ){
135                $cur_sdffile =~ s/\/[^\/]*\.$file_types[\s]*$/\/localize.sdf/;
136            #}
137
138            # Set default date
139            if( $line =~ /(.*)\t[^\t\$]*$/ ){
140                $line = $1."\t".$default_date;
141            }
142
143            if( $start ){
144                $start='';
145                $lastFile = $currentFile; # ?
146                $last_sdffile = $cur_sdffile;
147            }
148
149            if( $lang eq "en-US" ){}
150            elsif( $cur_sdffile eq $last_sdffile )
151            {
152                $block{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } =  $line ;
153            }
154            else
155            {
156                writesdf( $lastFile , \%block );
157                $lastFile = $currentFile; #?
158                $last_sdffile = $cur_sdffile;
159                %block = ();
160                #if( ! $lang eq "en-US"  ) {
161                $block{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } =  $line ;
162                #}
163
164            }
165        } #else { print STDOUT "splitfile REGEX kaputt\n";}
166
167    }
168    writesdf( $lastFile , \%block );
169    %block = ();
170    close( MYFILE );
171
172}
173#########################################################
174
175sub writesdf{
176
177    my $lastFile        = shift;
178    my $blockhash_ref   = shift;
179    my $localizeFile    = $lastFile;
180    my %index=();
181
182    if( $localizeFile =~ /\.$file_types[\s]*$/ ){
183        $localizeFile =~ s/\/[^\/]*\.$file_types[\s]*$/\/localize.sdf/;
184        }else {
185            print STDERR "Strange filetype found '$localizeFile'\n";
186            return;
187        }
188        if( open DESTFILE , "< $localizeFile" ){
189
190        #or die "Can't open/create '\$localizeFile'";
191
192        #### Build hash
193        while(<DESTFILE>){
194         if( /$sdf_regex/ ){
195            my $line           = defined $_ ? $_ : '';
196            my $prj            = defined $3 ? $3 : '';
197            my $file           = defined $4 ? $4 : '';
198            my $type           = defined $6 ? $6 : '';
199            my $gid            = defined $7 ? $7 : '';
200            my $lid            = defined $8 ? $8 : '';
201            my $lang           = defined $12 ? $12 : '';
202            my $plattform      = defined $10 ? $10 : '';
203            my $helpid         = defined $9 ? $9 : '';
204
205            chomp( $line );
206            $index{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } =  $line ;
207
208         } #else { print STDOUT "writesdf REGEX kaputt $_\n";}
209
210        }
211        close( DESTFILE );
212    }
213    #### Copy new strings
214    my @mykeys = keys( %{ $blockhash_ref } );
215    my $isDirty = "FALSE";
216    foreach my $key( @mykeys ){
217        if( ! defined $index{ $key } ){
218            # Add new entry
219            $index{ $key }  = $blockhash_ref->{ $key} ;
220            $isDirty = "TRUE";
221        }elsif( $index{ $key } ne $blockhash_ref->{ $key } ){
222            # Overwrite old entry
223            $index{ $key } = $blockhash_ref->{ $key };
224            $isDirty = "TRUE";
225        }else {
226        }
227    }
228
229    #### Write file
230
231    if( !$bVerbose ){ print STDOUT "."; }
232    if( $isDirty eq "TRUE" ){
233        if( $bVerbose ){ print STDOUT "$localizeFile\n"; }
234        if( open DESTFILE , "+> $localizeFile" ){
235            print DESTFILE get_license_header();
236            @mykeys = sort keys( %index );
237            foreach my $key( @mykeys ){
238                print DESTFILE ( $index{ $key } , "\n" );
239            }
240            close DESTFILE;
241         }else {
242            print STDOUT "WARNING: File $localizeFile is not writable , try to merge ...\n";
243            my ( $TMPFILE , $tmpfile ) = File::Temp::tempfile();
244            if( open DESTFILE , "+> $tmpfile " ){
245                @mykeys = keys( %index );
246                foreach my $key( @mykeys ){
247                    print DESTFILE ( $index{ $key } , "\n" );
248                }
249                close DESTFILE;
250                if( move( $localizeFile , $localizeFile.".backup" ) ){
251                    if( copy( $tmpfile , $localizeFile ) ){
252                        unlink $localizeFile.".backup";
253                    } else { print STDERR "Can't open/create '$localizeFile', original file is renamed to  $localizeFile.backup\n"; }
254                } else { print STDERR "Can't open/create '$localizeFile'\n"; }
255            }else{
256                print STDERR "WARNING: Can't open/create '$localizeFile'\n";
257            }
258            unlink $tmpfile;
259        }
260    }
261#    if( $no_sort eq '' ){
262#        sort_outfile( $localizeFile );
263#    }
264}
265
266sub get_license_header{
267    return
268"#\n".
269"#    ####    ###     #   #   ###   #####    #####  ####   #####  #####  \n".
270"#    #   #  #   #    ##  #  #   #    #      #      #   #    #      #    \n".
271"#    #   #  #   #    # # #  #   #    #      ###    #   #    #      #    \n".
272"#    #   #  #   #    #  ##  #   #    #      #      #   #    #      #    \n".
273"#    ####    ###     #   #   ###     #      #####  ####   #####    #    \n".
274"#\n".
275"#    DO NOT EDIT! This file will be overwritten by localisation process\n".
276"#\n".
277"#*************************************************************************\n".
278"#\n".
279"# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.\n".
280"# \n".
281"# Copyright 2000, 2010 Oracle and/or its affiliates.\n".
282"#\n".
283"# OpenOffice.org - a multi-platform office productivity suite\n".
284"#\n".
285"# This file is part of OpenOffice.org.\n".
286"#\n".
287"# OpenOffice.org is free software: you can redistribute it and/or modify\n".
288"# it under the terms of the GNU Lesser General Public License version 3\n".
289"# only, as published by the Free Software Foundation.\n".
290"#\n".
291"# OpenOffice.org is distributed in the hope that it will be useful,\n".
292"# but WITHOUT ANY WARRANTY; without even the implied warranty of\n".
293"# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n".
294"# GNU Lesser General Public License version 3 for more details\n".
295"# (a copy is included in the LICENSE file that accompanied this code).\n".
296"#\n".
297"# You should have received a copy of the GNU Lesser General Public License\n".
298"# version 3 along with OpenOffice.org.  If not, see\n".
299"# <http://www.openoffice.org/license.html>\n".
300"# for a copy of the LGPLv3 License.\n".
301"#\n".
302"#*************************************************************************\n";
303}
304######## Check input sdf file and use only the correct part
305sub merge_gsicheck{
306    my $command = '';
307    my ( $TMPHANDLE , $tmpfile ) = File::Temp::tempfile();
308    my ( $TMPHANDLE2 , $tmpfile2 ) = File::Temp::tempfile();
309    close ( $TMPHANDLE );
310    close ( $TMPHANDLE2 );
311
312    unlink $tmpfile2;
313    my $output2 = `cat $sdffile | sort > $tmpfile2`;
314    my $rc2 = $? << 8;
315    if( $rc2 ne  0 ){
316        printf("ERROR: Failed -> cat $sdffile | sort > $tmpfile2\n$output2\n");
317        exit( -1 );
318    }
319
320#    if( $ENV{WRAPCMD} ){
321#        $command = "$ENV{WRAPCMD} gsicheck";
322#    }else{
323#        $command = "gsicheck";
324#    }
325#    my $errfile = $tmpfile.".err";
326#    $command .= " -k -c -wcf $tmpfile -wef $errfile -l \"\" $tmpfile2";
327#    my $output = `$command`;
328#    my $rc = $? << 8;
329#    if ( $output ne "" ){
330#        print STDOUT "### gsicheck ###\n";
331#        print STDOUT "### The file $errfile have been written containing the errors in your sdf file. Those lines will not be merged: ###\n\n";
332#        print STDOUT "$output\n";
333#        print STDOUT "################\n";
334#
335#    }else{
336#        # Remove the 0 Byte file
337#        unlink $errfile;
338#    }
339    $sdffile = $tmpfile2;
340#    unlink $tmpfile2;
341}
342#########################################################
343sub collectfiles{
344    print STDOUT "### Localize\n";
345    my @sdfparticles;
346    my $localizehash_ref;
347    my ( $bAll , $bUseLocalize, $langhash_ref , $bHasSourceLanguage , $bFakeEnglish ) = parseLanguages();
348
349    # Enable autoflush on STDOUT
350    # $| = 1;
351    STDOUT->autoflush( 1 );
352
353    ### Search sdf particles
354    print STDOUT "### Searching sdf particles\n";
355    my $working_path = getcwd();
356    chdir $srcpath;
357    find sub {
358        my $file = $File::Find::name;
359        if( -f && $file =~ /.*localize.sdf$/ ) {
360            push   @sdfparticles , $file;
361            if( $bVerbose eq "1" ) { print STDOUT "$file\n"; }
362            else { print ".";  }
363
364         }
365    } , getcwd() ;#"."; #$srcpath;
366    chdir $working_path;
367
368    my $nFound  = $#sdfparticles +1;
369    print "\n    $nFound files found !\n";
370
371    my ( $LOCALIZEPARTICLE , $localizeSDF ) = File::Temp::tempfile();
372    close( $LOCALIZEPARTICLE );
373
374    my ( $ALLPARTICLES_MERGED , $particleSDF_merged )     = File::Temp::tempfile();
375    close( $ALLPARTICLES_MERGED );
376    my ( $LOCALIZE_LOG , $my_localize_log ) = File::Temp::tempfile();
377    close( $LOCALIZE_LOG );
378
379    ## Get the localize de,en-US extract
380    if( $bAll || $bUseLocalize ){
381        print "### Fetching source language strings\n";
382        my $command = "";
383        my $args    = "";
384
385        if( $ENV{WRAPCMD} ){
386            $command = "$ENV{WRAPCMD} localize_sl";
387        }else{
388            $command = "localize_sl";
389        }
390
391        # -e
392        # if ( -x $command ){
393        if( $command ){
394            if( !$bVerbose  ){ $args .= " -QQ -skip_links "; }
395            $args .= " -e -f $localizeSDF -l ";
396            my $bFlag="";
397            if( $bAll ) {$args .= " en-US";}
398            else{
399              my @list;
400              foreach my $isokey ( keys( %{ $langhash_ref } ) ){
401                push @list , $isokey;
402                if( $langhash_ref->{ $isokey } ne "" ){
403                    push @list , $langhash_ref->{ $isokey };
404                }
405              }
406              remove_duplicates( \@list );
407              foreach my $isokey ( @list ){
408                switch :{
409                    #( $isokey=~ /^de$/i  )
410                    #    && do{
411                    #            if( $bFlag eq "TRUE" ){ $args .= ",de"; }
412                    #            else  {
413                    #                $args .=  "de";  $bFlag = "TRUE";
414                    #             }
415                    #          };
416                        ( $isokey=~ /^en-US$/i  )
417                        && do{
418                                if( $bFlag eq "TRUE" ){ $args .= ",en-US"; }
419                                else {
420                                    $args .= "en-US";  $bFlag = "TRUE";
421                                 }
422                              };
423
424                    } #switch
425                } #foreach
426              } # if
427        } # if
428        if ( $bVerbose ) { print STDOUT $command.$args."\n"; }
429
430        my $rc = system( $command.$args );
431
432        #my $output = `$command.$args`;
433        #my $rc = $? << 8;
434
435        if( $rc < 0 ){    print STDERR "ERROR: localize rc = $rc\n"; exit( -1 ); }
436        ( $localizehash_ref )  = read_file( $localizeSDF , $langhash_ref );
437
438    }
439    ## Get sdf particles
440   open ALLPARTICLES_MERGED , "+>> $particleSDF_merged"
441    or die "Can't open $particleSDF_merged";
442
443    ## Fill fackback hash
444    my( $fallbackhashhash_ref ) = fetch_fallback( \@sdfparticles , $localizeSDF ,  $langhash_ref );
445#    my( $fallbackhashhash_ref ) = fetch_fallback( \@sdfparticles , $localizeSDF , $langhash_ref );
446    my %block;
447    my $cur_fallback;
448    if( !$bAll) {
449        foreach my $cur_lang ( keys( %{ $langhash_ref } ) ){
450            #print STDOUT "DBG: G1 cur_lang=$cur_lang\n";
451            $cur_fallback = $langhash_ref->{ $cur_lang };
452            if( $cur_fallback ne "" ){
453                # Insert fallback strings
454                #print STDOUT "DBG: Renaming $cur_fallback to $cur_lang in fallbackhash\n";
455                rename_language(  $fallbackhashhash_ref ,  $cur_fallback , $cur_lang );
456            }
457            foreach my $currentfile ( @sdfparticles ){
458                if ( open MYFILE , "< $currentfile" ) {
459                    while(<MYFILE>){
460                        if( /$sdf_regex/ ){
461                            my $line           = defined $_ ? $_ : '';
462                            my $prj            = defined $3 ? $3 : '';
463                            my $file           = defined $4 ? $4 : '';
464                            my $type           = defined $6 ? $6 : '';
465                            my $gid            = defined $7 ? $7 : '';
466                            my $lid            = defined $8 ? $8 : '';
467                            my $lang           = defined $12 ? $12 : '';
468                            my $plattform      = defined $10 ? $10 : '';
469                            my $helpid         = defined $9 ? $9 : '';
470
471                            chomp( $line );
472
473                            if ( $lang eq $cur_lang ){
474                                # Overwrite fallback strings with collected strings
475                                #if( ( !has_two_sourcelanguages( $cur_lang) && $cur_lang eq "de" ) || $cur_lang ne "en-US" ){
476                                     $fallbackhashhash_ref->{ $cur_lang }{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } =  $line ;
477                                     #}
478
479                            }
480                        }
481                    }
482                }else { print STDERR "WARNING: Can't open file $currentfile"; }
483            }
484
485            foreach my $line ( keys( %{$fallbackhashhash_ref->{ $cur_lang } } )) {
486                if( #$cur_lang ne "de" &&
487                    $cur_lang ne "en-US" ){
488                    print ALLPARTICLES_MERGED ( $fallbackhashhash_ref->{ $cur_lang }{ $line }, "\n" );
489                }
490             }
491        }
492    } else {
493        foreach my $currentfile ( @sdfparticles ){
494            if ( open MYFILE , "< $currentfile" ) {
495                while( <MYFILE> ){
496                    print ALLPARTICLES_MERGED ( $_, "\n" );  # recheck de / en-US !
497                }
498            }
499            else { print STDERR "WARNING: Can't open file $currentfile"; }
500        }
501    }
502    close ALLPARTICLES_MERGED;
503
504
505    # Hash of array
506    my %output;
507    my @order;
508
509    ## Join both
510    if( $outputfile ){
511        if( open DESTFILE , "+> $outputfile" ){
512            if( !open LOCALIZEPARTICLE ,  "< $localizeSDF" ) { print STDERR "ERROR: Can't open file $localizeSDF\n"; }
513            if( !open ALLPARTICLES_MERGED , "< $particleSDF_merged" ) { print STDERR "ERROR: Can't open file $particleSDF_merged\n"; }
514
515            # Insert localize
516            my $extract_date="";
517            while ( <LOCALIZEPARTICLE> ){
518                if( /$sdf_regex/ ){
519                    my $leftpart       = defined $2 ? $2 : '';
520                    my $lang           = defined $12 ? $12 : '';
521                    my $rightpart      = defined $13 ? $13 : '';
522                    my $timestamp      = defined $18 ? $18 : '';
523
524                    my $prj            = defined $3 ? $3 : '';
525                    my $file           = defined $4 ? $4 : '';
526                    my $type           = defined $6 ? $6 : '';
527                    my $gid            = defined $7 ? $7 : '';
528                    my $lid            = defined $8 ? $8 : '';
529                    #my $lang           = defined $12 ? $12 : '';
530                    my $plattform      = defined $10 ? $10 : '';
531                    my $helpid         = defined $9 ? $9 : '';
532
533
534                    if( $use_default_date )
535                    {
536                        $extract_date = "$default_date\n" ;
537                    }
538                    elsif( $extract_date eq "" ) {
539                        $extract_date = $timestamp ;
540                        $extract_date =~ tr/\r\n//d;
541                        $extract_date .= "\n";
542                    }
543
544                    if( $bAll ){ print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date ; }
545                    else {
546                        foreach my $sLang ( keys( %{ $langhash_ref } ) ){
547                            if( $sLang=~ /all/i )                       {
548                                push @{ $output{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } } ,  $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
549                                #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date;
550                            }
551                            #if( $sLang eq "de" && $lang eq "de" )       {
552                            #    push @{ $output{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } } ,  $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
553                                #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date;
554                                #}
555                            if( $sLang eq "en-US" && $lang eq "en-US" ) {
556                                push @order , $prj.$gid.$lid.$file.$type.$plattform.$helpid;
557                                if( !$bFakeEnglish ){ push @{ $output{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } } ,  $leftpart."\t".$lang."\t".$rightpart.$extract_date ; }
558                                #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date;
559                            }
560
561                        }
562                    }
563                }
564            }
565            # Insert particles
566            while ( <ALLPARTICLES_MERGED> ){
567                if( /$sdf_regex/ ){
568                    my $leftpart       = defined $2 ? $2 : '';
569                    my $prj            = defined $3 ? $3 : '';
570                    my $lang           = defined $12 ? $12 : '';
571                    my $rightpart      = defined $13 ? $13 : '';
572                    my $timestamp      = defined $18 ? $18 : '';
573
574                    #my $prj            = defined $3 ? $3 : '';
575                    my $file           = defined $4 ? $4 : '';
576                    my $type           = defined $6 ? $6 : '';
577                    my $gid            = defined $7 ? $7 : '';
578                    my $lid            = defined $8 ? $8 : '';
579                    #my $lang           = defined $12 ? $12 : '';
580                    my $plattform      = defined $10 ? $10 : '';
581                    my $helpid         = defined $9 ? $9 : '';
582
583
584                    if( $use_default_date )
585                    {
586                        $extract_date = "$default_date\n" ;
587                    }
588                    elsif( $extract_date eq "" )
589                    {
590                        $extract_date = $timestamp;
591                    }
592
593                    if( ! ( $prj =~ /binfilter/i ) ) {
594                        push @{ $output{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } } , $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
595                        #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
596                    }
597                 }
598            }
599
600            # Write!
601            foreach my $curkey ( @order ){
602                foreach my $curlist ( $output{ $curkey } ){
603                    foreach my $line ( @{$curlist} ){
604                        print DESTFILE $line;
605                    }
606                }
607            }
608
609        }else { print STDERR "Can't open $outputfile";}
610    }
611    close DESTFILE;
612    close LOCALIZEPARTICLE;
613    close ALLPARTICLES_MERGED;
614
615    #print STDOUT "DBG: \$localizeSDF $localizeSDF \$particleSDF_merged $particleSDF_merged\n";
616    unlink $localizeSDF , $particleSDF_merged ,  $my_localize_log;
617
618    #sort_outfile( $outputfile );
619    #remove_obsolete( $outputfile ) , if $bHasSourceLanguage ne "";
620    }
621
622#########################################################
623sub remove_obsolete{
624    my $outfile = shift;
625    my @lines;
626    my $enusleftpart;
627    my @good_lines;
628
629    print STDOUT "### Removing obsolete strings\n";
630
631    # Kick out all strings without en-US reference
632    if ( open ( SORTEDFILE , "< $outfile" ) ){
633        while( <SORTEDFILE> ){
634            if( /$sdf_regex/ ){
635                my $line           = defined $_ ? $_ : '';
636                my $language       = defined $12 ? $12 : '';
637                my $prj            = defined $3 ? $3 : '';
638                my $file           = defined $4 ? $4 : '';
639                my $type           = defined $6 ? $6 : '';
640                my $gid            = defined $7 ? $7 : '';
641                my $lid            = defined $8 ? $8 : '';
642                my $plattform      = defined $10 ? $10 : '';
643                my $helpid         = defined $9 ? $9 : '';
644
645                my $leftpart = $prj.$gid.$lid.$file.$type.$plattform.$helpid;
646
647                if( $language eq "en-US" ){                 # source string found, 1. entry
648                    $enusleftpart = $leftpart;
649                    push @good_lines , $line;
650                }else{
651                    if( !defined $enusleftpart or !defined $leftpart ){
652                        print STDERR "BADLINE: $line\n";
653                        print STDERR "\$enusleftpart = $enusleftpart\n";
654                        print STDERR "\$leftpart = $leftpart\n";
655                    }
656                    if( $enusleftpart eq $leftpart ){   # matching language
657                        push @good_lines , $line;
658                    }
659                    #else{
660                    #    print STDERR "OUT:  \$enusleftpart=$enusleftpart \$leftpart=$leftpart \$line=$line\n";
661                    #}
662                }
663            }
664        }
665        close SORTEDFILE;
666    } else { print STDERR "ERROR: Can't open file $outfile\n";}
667
668    # Write file
669    if ( open ( SORTEDFILE , "> $outfile" ) ){
670        foreach my $newline ( @good_lines ) {
671            print SORTEDFILE $newline;
672        }
673        close SORTEDFILE;
674    } else { print STDERR "ERROR: Can't open file $outfile\n";}
675
676}
677#########################################################
678sub sort_outfile{
679        my $outfile = shift;
680        print STDOUT "### Sorting ... $outfile ...";
681        my @lines;
682        my @sorted_lines;
683
684
685        #if ( open ( SORTEDFILE , "< $outputfile" ) ){
686        if ( open ( SORTEDFILE , "< $outfile" ) ){
687            my $line;
688            while ( <SORTEDFILE> ){
689                $line = $_;
690                if( $line =~ /^[^\#]/ ){
691                    push @lines , $line;
692                }
693            }
694            close SORTEDFILE;
695            @sorted_lines = sort {
696                my $xa_lang          = "";
697                my $xa_left_part    = "";
698                my $xa_right_part    = "";
699                my $xa_timestamp     = "";
700                my $xb_lang          = "";
701                my $xb_left_part    = "";
702                my $xb_right_part    = "";
703                my $xb_timestamp     = "";
704                my $xa               = "";
705                my $xb               = "";
706                my @alist;
707                my @blist;
708
709                if( $a=~ /$sdf_regex/ ){
710                    $xa_left_part       = defined $2 ? $2 : '';
711                    $xa_lang           = defined $12 ? $12 : '';
712                    $xa_right_part     = defined $13 ? $13 : '';
713                    $xa_left_part = remove_last_column( $xa_left_part );
714
715                }
716                if( $b=~ /$sdf_regex/ ){
717                    $xb_left_part       = defined $2 ? $2 : '';
718                    $xb_lang           = defined $12 ? $12 : '';
719                    $xb_right_part     = defined $13 ? $13 : '';
720                    $xb_left_part = remove_last_column( $xb_left_part );
721
722
723                }
724                if( (  $xa_left_part cmp $xb_left_part ) == 0 ){         # Left part equal
725                     if( ( $xa_lang cmp $xb_lang ) == 0 ){               # Lang equal
726                         return ( $xa_right_part cmp $xb_right_part );   # Right part compare
727                    }
728                    elsif( $xa_lang eq "en-US" ) { return -1; }        # en-US wins
729                    elsif( $xb_lang eq "en-US" ) { return 1;  }        # en-US wins
730                    else { return $xa_lang cmp $xb_lang; }             # lang compare
731                }
732                else {
733                    return $xa_left_part cmp $xb_left_part;        # Left part compare
734                }
735            } @lines;
736
737            if ( open ( SORTEDFILE , "> $outfile" ) ){
738                print SORTEDFILE get_license_header();
739                foreach my $newline ( @sorted_lines ) {
740                    print SORTEDFILE $newline;
741                    #print STDOUT $newline;
742                }
743            }
744            close SORTEDFILE;
745        } else { print STDERR "WARNING: Can't open file $outfile\n";}
746	print "done\n";
747
748}
749#########################################################
750sub remove_last_column{
751    my $string                  = shift;
752    my @alist = split ( "\t" , $string );
753    pop @alist;
754    return join( "\t" , @alist );
755}
756
757#########################################################
758sub rename_language{
759    my $fallbackhashhash_ref    = shift;
760    my $cur_fallback            = shift;
761    my $cur_lang                = shift;
762    my $line;
763
764    foreach my $key( keys ( %{ $fallbackhashhash_ref->{ $cur_fallback } } ) ){
765        $line = $fallbackhashhash_ref->{ $cur_fallback }{ $key };
766        if( $line =~ /$sdf_regex/ ){
767            my $leftpart       = defined $2 ? $2 : '';
768            my $lang           = defined $12 ? $12 : '';
769            my $rightpart      = defined $13 ? $13 : '';
770
771            $fallbackhashhash_ref->{ $cur_lang }{ $key } = $leftpart."\t".$cur_lang."\t".$rightpart;
772        }
773    }
774}
775
776############################################################
777sub remove_duplicates{
778    my $list_ref    = shift;
779    my %tmphash;
780    foreach my $key ( @{ $list_ref } ){ $tmphash{ $key } = '' ; }
781    @{$list_ref} = keys( %tmphash );
782}
783
784##############################################################
785sub fetch_fallback{
786    my $sdfparticleslist_ref   = shift;
787    my $localizeSDF            = shift;
788    my $langhash_ref           = shift;
789    my %fallbackhashhash;
790    my $cur_lang;
791    my @langlist;
792
793    foreach my $key ( keys ( %{ $langhash_ref } ) ){
794        $cur_lang = $langhash_ref->{ $key };
795        if ( $cur_lang ne "" ) {
796            push @langlist , $cur_lang;
797        }
798    }
799    remove_duplicates( \@langlist );
800    foreach  $cur_lang ( @langlist ){
801        if( $cur_lang eq "en-US" ){
802            read_fallbacks_from_source( $localizeSDF , $cur_lang , \%fallbackhashhash );
803        }
804    }
805
806    # remove de / en-US
807    my @tmplist;
808    foreach $cur_lang( @langlist ){
809        if(  $cur_lang ne "en-US" ){
810           push @tmplist , $cur_lang;
811
812        }
813    }
814    @langlist = @tmplist;
815    if ( $#langlist +1 ){
816        read_fallbacks_from_particles( $sdfparticleslist_ref , \@langlist , \%fallbackhashhash );
817
818    }
819    return (\%fallbackhashhash);
820}
821
822#########################################################
823sub write_file{
824
825    my $localizeFile = shift;
826    my $index_ref    = shift;
827
828    if( open DESTFILE , "+> $localizeFile" ){
829        foreach my $key( %{ $index_ref } ){
830            print DESTFILE ($index_ref->{ $key }, "\n" );
831        }
832        close DESTFILE;
833    }else {
834      print STDERR "Can't open/create '$localizeFile'";
835    }
836}
837
838#########################################################
839sub read_file{
840
841    my $sdffile         = shift;
842    my $langhash_ref    = shift;
843    my %block           = ();
844
845    open MYFILE , "< $sdffile"
846        or die "Can't open '$sdffile'\n";
847        while( <MYFILE>){
848          if( /$sdf_regex/ ){
849            my $line           = defined $_ ? $_ : '';
850            my $prj            = defined $3 ? $3 : '';
851            my $file           = defined $4 ? $4 : '';
852            my $type           = defined $6 ? $6 : '';
853            my $gid            = defined $7 ? $7 : '';
854            my $lid            = defined $8 ? $8 : '';
855            my $plattform      = defined $10 ? $10 : '';
856            my $lang           = defined $12 ? $12 : '';
857            my $helpid         = defined $9 ? $9 : '';
858
859            foreach my $isolang ( keys ( %{ $langhash_ref } ) ){
860                if( $isolang=~ /$lang/i || $isolang=~ /all/i ) { $block{$prj.$gid.$lid.$file.$type.$plattform.$helpid } =  $line ; }
861            }
862        }
863    }
864    return (\%block);
865}
866
867#########################################################
868sub read_fallbacks_from_particles{
869
870    my $sdfparticleslist_ref    = shift;
871    my $isolanglist_ref         = shift;
872    my $fallbackhashhash_ref    = shift;
873    my $block_ref;
874    foreach my $currentfile ( @{ $sdfparticleslist_ref } ){
875        if ( open MYFILE , "< $currentfile" ) {
876            while(<MYFILE>){
877                if( /$sdf_regex/ ){
878                    my $line           = defined $_ ? $_ : '';
879                    my $prj            = defined $3 ? $3 : '';
880                    my $file           = defined $4 ? $4 : '';
881                    my $type           = defined $6 ? $6 : '';
882                    my $gid            = defined $7 ? $7 : '';
883                    my $lid            = defined $8 ? $8 : '';
884                    my $lang           = defined $12 ? $12 : '';
885                    my $plattform      = defined $10 ? $10 : '';
886                    my $helpid         = defined $9 ? $9 : '';
887
888                    chomp( $line );
889
890                    foreach my $isolang ( @{$isolanglist_ref}  ){
891                        if( $isolang=~ /$lang/i ) {
892                            $fallbackhashhash_ref->{ $isolang }{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } =  $line ;
893                        }
894                    }
895                }
896            }
897       }else { print STDERR "WARNING: Can't open file $currentfile"; }
898    }
899}
900
901#########################################################
902sub read_fallbacks_from_source{
903
904    my $sdffile                 = shift;
905    my $isolang                 = shift;
906    my $fallbackhashhash_ref    = shift;
907    my $block_ref;
908    # read fallback for single file
909    open MYFILE , "< $sdffile"
910        or die "Can't open '$sdffile'\n";
911
912    while( <MYFILE>){
913          if( /$sdf_regex/ ){
914            my $line           = defined $_ ? $_ : '';
915            my $prj            = defined $3 ? $3 : '';
916            my $file           = defined $4 ? $4 : '';
917            my $type           = defined $6 ? $6 : '';
918            my $gid            = defined $7 ? $7 : '';
919            my $lid            = defined $8 ? $8 : '';
920            my $helpid         = defined $9 ? $9 : '';
921            my $lang           = defined $12 ? $12 : '';
922            my $plattform      = defined $10 ? $10 : '';
923
924            chomp( $line );
925            if( $isolang=~ /$lang/i ) { $fallbackhashhash_ref->{ $isolang }{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } =  $line ;
926            }
927        }
928    }
929}
930
931#########################################################
932sub parseLanguages{
933
934    my $bAll;
935    my $bUseLocalize;
936    my $bHasSourceLanguage="";
937    my $bFakeEnglish="";
938    my %langhash;
939    my $iso="";
940    my $fallback="";
941
942    #### -l all
943    if(   $languages=~ /all/ ){
944        $bAll = "TRUE";
945        $bHasSourceLanguage = "TRUE";
946    }
947    ### -l fr=de,de
948    elsif( $languages=~ /.*,.*/ ){
949        my @tmpstr =  split "," , $languages;
950        for my $lang ( @tmpstr ){
951            if( $lang=~ /([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)(=([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*))?/ ){
952                $iso        = $1;
953                $fallback   = $4;
954
955                if( ( $iso && $iso=~ /(en-US)/i )  || ( $fallback && $fallback=~ /(en-US)/i ) ) {
956                    $bUseLocalize = "TRUE";
957                }
958                if( ( $iso && $iso=~ /(en-US)/i ) ) {
959                    $bHasSourceLanguage = "TRUE";
960                }
961             if( $fallback ) { $langhash{ $iso } = $fallback;   }
962             else            { $langhash{ $iso } = "";          }
963            }
964        }
965    }
966    ### -l de
967    else{
968        if( $languages=~ /([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)(=([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*))?/ ){
969            $iso        = $1;
970            $fallback   = $4;
971
972            if( ( $iso && $iso=~ /(en-US)/i )  || ( $fallback && $fallback=~ /(en-US)/i ) ) {
973                $bUseLocalize = "TRUE";
974
975            }
976            if( ( $iso && $iso=~ /(en-US)/i )  ) {
977                $bHasSourceLanguage = "TRUE";
978            }
979
980             if( $fallback ) { $langhash{ $iso } = $fallback;   }
981             else            { $langhash{ $iso } = "";          }
982        }
983    }
984    # HACK en-US always needed!
985    if( !$bHasSourceLanguage ){
986        #$bHasSourceLanguage = "TRUE";
987        $bUseLocalize = "TRUE";
988        $bFakeEnglish = "TRUE";
989        $langhash{ "en-US" } = "";
990    }
991    return ( $bAll ,  $bUseLocalize , \%langhash , $bHasSourceLanguage, $bFakeEnglish);
992}
993
994#########################################################
995sub parse_options{
996
997    my $help;
998    my $merge;
999    my $extract;
1000    my $success = GetOptions('f=s' => \$sdffile , 'l=s' => \$languages , 's=s' => \$srcpath ,  'h' => \$help , 'v' => \$bVerbose ,
1001                             'm' => \$merge , 'e' => \$extract , 'x' => \$no_sort , 'd' => \$use_default_date );
1002    $outputfile = $sdffile;
1003
1004    #print STDOUT "DBG: lang = $languages\n";
1005    if( !$srcpath ){
1006        #$srcpath = "$ENV{SRC_ROOT}";
1007        if( !$srcpath ){
1008	        print STDERR "No path to the source root found!\n\n";
1009	        usage();
1010            exit(1);
1011        }
1012    }
1013    if( $help || !$success || $#ARGV > 1 || ( !$sdffile ) ){
1014        usage();
1015        exit(1);
1016    }
1017    if( $merge && $sdffile && ! ( -r $sdffile)){
1018        print STDERR "Can't open file '$sdffile'\n";
1019        exit(1);
1020    }
1021    if( !( $languages=~ /[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*(=[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)?(,[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*(=[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)?)*/ ) ){
1022        print STDERR "Please check the -l iso code\n";
1023        exit(1);
1024    }
1025    if( ( !$merge && !$extract ) || ( $merge && $extract ) ){ usage();exit( -1 );}
1026    if( $extract ){ $mode = "extract"; }
1027    else          { $mode = "merge";   }
1028}
1029
1030#########################################################
1031sub usage{
1032
1033    print STDERR "Usage: localize.pl\n";
1034    print STDERR "Split or collect SDF files\n";
1035    print STDERR "           merge: -m -f <sdffile>    -l l1[=f1][,l2[=f2]][...] [ -s <sourceroot> ]\n";
1036    print STDERR "         extract: -e -f <outputfile> -l <lang> [ -s <sourceroot> ] [-d]\n";
1037    print STDERR "Options:\n";
1038    print STDERR "    -h              help\n";
1039    print STDERR "    -m              Merge mode\n";
1040    print STDERR "    -e              Extract mode\n";
1041    print STDERR "    -f <sdffile>    To split a big SDF file into particles\n";
1042    print STDERR "       <outputfile> To collect and join all particles to one big file\n";
1043    print STDERR "    -s <sourceroot> Path to the modules, if no \$SRC_ROOT is set\n";
1044    print STDERR "    -l ( all | <isocode> | <isocode>=fallback ) comma seperated languages\n";
1045    print STDERR "    -d              Use default date in extracted sdf file\n";
1046    print STDERR "    -v              Verbose\n";
1047    print STDERR "\nExample:\n";
1048    print STDERR "\nlocalize -e -l en-US,pt-BR=en-US -f my.sdf\n( Extract en-US and pt-BR with en-US fallback )\n";
1049    print STDERR "\nlocalize -m -l cs -f my.sdf\n( Merge cs translation into the sourcecode ) \n";
1050}
1051
1052#            my $line           = defined $_ ? $_ : '';
1053#            my $leftpart       = defined $2 ? $2 : '';
1054#            my $prj            = defined $3 ? $3 : '';
1055#            my $file           = defined $4 ? $4 : '';
1056#            my $dummy          = defined $5 ? $5 : '';
1057#            my $type           = defined $6 ? $6 : '';
1058#            my $gid            = defined $7 ? $7 : '';
1059#            my $lid            = defined $8 ? $8 : '';
1060#            my $helpid         = defined $9 ? $9 : '';
1061#            my $plattform      = defined $10 ? $10 : '';
1062#            my $width          = defined $11 ? $11 : '';
1063#            my $lang           = defined $12 ? $12 : '';
1064#            my $rightpart      = defined $13 ? $13 : '';
1065#            my $text           = defined $14 ? $14 : '';
1066#            my $helptext       = defined $15 ? $15 : '';
1067#            my $quickhelptext  = defined $16 ? $16 : '';
1068#            my $title          = defined $17 ? $17 : '';
1069#            my $timestamp      = defined $18 ? $18 : '';
1070
1071