1:
2eval 'exec perl -wS $0 ${1+"$@"}'
3    if 0;
4
5
6#**************************************************************
7#
8#  Licensed to the Apache Software Foundation (ASF) under one
9#  or more contributor license agreements.  See the NOTICE file
10#  distributed with this work for additional information
11#  regarding copyright ownership.  The ASF licenses this file
12#  to you under the Apache License, Version 2.0 (the
13#  "License"); you may not use this file except in compliance
14#  with the License.  You may obtain a copy of the License at
15#
16#    http://www.apache.org/licenses/LICENSE-2.0
17#
18#  Unless required by applicable law or agreed to in writing,
19#  software distributed under the License is distributed on an
20#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
21#  KIND, either express or implied.  See the License for the
22#  specific language governing permissions and limitations
23#  under the License.
24#
25#**************************************************************
26
27
28
29use strict;
30use Getopt::Long;
31use IO::Handle;
32use File::Find;
33use File::Temp;
34use File::Copy;
35use File::Glob qw(:glob csh_glob);
36use Cwd;
37
38# ver 1.1
39#
40#### module lookup
41#use lib ("$ENV{SOLARENV}/bin/modules", "$ENV{COMMON_ENV_TOOLS}/modules");
42
43#### module lookup
44# OOo conform
45my @lib_dirs;
46BEGIN {
47    if ( !defined($ENV{SOLARENV}) ) {
48        die "No environment found (environment variable SOLARENV is undefined)";
49    }
50    push(@lib_dirs, "$ENV{SOLARENV}/bin/modules");
51    push(@lib_dirs, "$ENV{COMMON_ENV_TOOLS}/modules") if defined($ENV{COMMON_ENV_TOOLS});
52}
53use lib (@lib_dirs);
54
55#### globals ####
56my $sdffile = '';
57my $no_sort = '';
58my $outputfile = '';
59my $mode = '';
60my $bVerbose="0";
61my $srcpath = '';
62my $languages;
63#my %sl_modules;     # Contains all modules where en-US and de is source language
64my $use_default_date = '0';
65
66         #         (                           leftpart                                                     )            (           rightpart                    )
67         #            prj      file      dummy     type       gid       lid      helpid    pform     width      lang       text    helptext  qhelptext   title    timestamp
68my $sdf_regex  = "((([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*))\t([^\t]*)\t(([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t)([^\t]*))";
69my $file_types = "(src|hrc|xcs|xcu|lng|ulf|xrm|xhp|xcd|xgf|xxl|xrb)";
70# Always use this date to prevent cvs conflicts
71my $default_date = "2002-02-02 02:02:02";
72
73#### main ####
74parse_options();
75
76#%sl_modules = fetch_sourcelanguage_dirlist();
77
78
79if   ( $mode eq "merge"    )    {
80    merge_gsicheck();
81    splitfile( $sdffile );
82    unlink $sdffile;             # remove temp file!
83}
84elsif( $mode eq "extract"  )    {
85    collectfiles( $outputfile );
86}
87else                            {
88    usage();
89}
90
91exit(0);
92
93#########################################################
94sub splitfile{
95
96    my $lastFile        = '';
97    my $currentFile     = '';
98    my $cur_sdffile     = '';
99    my $last_sdffile    = '';
100    my $delim;
101    my $badDelim;
102    my $start           = 'TRUE';
103    my %index  = ();
104    my %block;
105
106    STDOUT->autoflush( 1 );
107
108    #print STDOUT "Open File $sdffile\n";
109    open MYFILE , "< $sdffile"
110    or die "Can't open '$sdffile'\n";
111
112    while( <MYFILE>){
113         if( /$sdf_regex/ ){
114            my $line           = defined $_ ? $_ : '';
115            my $prj            = defined $3 ? $3 : '';
116            my $file           = defined $4 ? $4 : '';
117            my $type           = defined $6 ? $6 : '';
118            my $gid            = defined $7 ? $7 : '';
119            my $lid            = defined $8 ? $8 : '';
120            my $lang           = defined $12 ? $12 : '';
121            my $platform       = defined $10 ? $10 : '';
122            my $helpid         = defined $9 ? $9 : '';
123
124            next if( $prj eq "binfilter" );     # Don't merge strings into binfilter module
125	        chomp( $line );
126            $currentFile  = $srcpath . '\\' . $prj . '\\' . $file;
127            $currentFile  =~ s/\\/\//g;
128
129            $cur_sdffile = $currentFile;
130            #if( $cur_sdffile =~ /\.$file_types[\s]*$/ ){
131                $cur_sdffile =~ s/\/[^\/]*\.$file_types[\s]*$/\/localize.sdf/;
132            #}
133
134            # Set default date
135            if( $line =~ /(.*)\t[^\t\$]*$/ ){
136                $line = $1."\t".$default_date;
137            }
138
139            if( $start ){
140                $start='';
141                $lastFile = $currentFile; # ?
142                $last_sdffile = $cur_sdffile;
143            }
144
145            if( $lang eq "en-US" ){}
146            elsif( $cur_sdffile eq $last_sdffile )
147            {
148                $block{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$platform\t$lang" } =  $line ;
149            }
150            else
151            {
152                writesdf( $lastFile , \%block );
153                $lastFile = $currentFile; #?
154                $last_sdffile = $cur_sdffile;
155                %block = ();
156                #if( ! $lang eq "en-US"  ) {
157                $block{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$platform\t$lang" } =  $line ;
158                #}
159
160            }
161        } #else { print STDOUT "splitfile REGEX kaputt\n";}
162
163    }
164    writesdf( $lastFile , \%block );
165    %block = ();
166    close( MYFILE );
167
168}
169#########################################################
170
171sub writesdf{
172
173    my $lastFile        = shift;
174    my $blockhash_ref   = shift;
175    my $localizeFile    = $lastFile;
176    my %index=();
177
178    if( $localizeFile =~ /\.$file_types[\s]*$/ ){
179        $localizeFile =~ s/\/[^\/]*\.$file_types[\s]*$/\/localize.sdf/;
180        }else {
181            print STDERR "Strange filetype found '$localizeFile'\n";
182            return;
183        }
184        if( open DESTFILE , "< $localizeFile" ){
185
186        #or die "Can't open/create '\$localizeFile'";
187
188        #### Build hash
189        while(<DESTFILE>){
190         if( /$sdf_regex/ ){
191            my $line           = defined $_ ? $_ : '';
192            my $prj            = defined $3 ? $3 : '';
193            my $file           = defined $4 ? $4 : '';
194            my $type           = defined $6 ? $6 : '';
195            my $gid            = defined $7 ? $7 : '';
196            my $lid            = defined $8 ? $8 : '';
197            my $lang           = defined $12 ? $12 : '';
198            my $platform       = defined $10 ? $10 : '';
199            my $helpid         = defined $9 ? $9 : '';
200
201            chomp( $line );
202            $index{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$platform\t$lang" } =  $line ;
203
204         } #else { print STDOUT "writesdf REGEX kaputt $_\n";}
205
206        }
207        close( DESTFILE );
208    }
209    #### Copy new strings
210    my @mykeys = keys( %{ $blockhash_ref } );
211    my $isDirty = "FALSE";
212    foreach my $key( @mykeys ){
213        if( ! defined $index{ $key } ){
214            # Add new entry
215            $index{ $key }  = $blockhash_ref->{ $key} ;
216            $isDirty = "TRUE";
217        }elsif( $index{ $key } ne $blockhash_ref->{ $key } ){
218            # Overwrite old entry
219            $index{ $key } = $blockhash_ref->{ $key };
220            $isDirty = "TRUE";
221        }else {
222        }
223    }
224
225    #### Write file
226
227    if( !$bVerbose ){ print STDOUT "."; }
228    if( $isDirty eq "TRUE" ){
229        if( $bVerbose ){ print STDOUT "$localizeFile\n"; }
230        if( open DESTFILE , "+> $localizeFile" ){
231            print DESTFILE get_license_header();
232            @mykeys = sort keys( %index );
233            foreach my $key( @mykeys ){
234                print DESTFILE ( $index{ $key } , "\n" );
235            }
236            close DESTFILE;
237         }else {
238            print STDOUT "WARNING: File $localizeFile is not writable , try to merge ...\n";
239            my ( $TMPFILE , $tmpfile ) = File::Temp::tempfile();
240            if( open DESTFILE , "+> $tmpfile " ){
241                @mykeys = keys( %index );
242                foreach my $key( @mykeys ){
243                    print DESTFILE ( $index{ $key } , "\n" );
244                }
245                close DESTFILE;
246                if( move( $localizeFile , $localizeFile.".backup" ) ){
247                    if( copy( $tmpfile , $localizeFile ) ){
248                        unlink $localizeFile.".backup";
249                    } else { print STDERR "Can't open/create '$localizeFile', original file is renamed to  $localizeFile.backup\n"; }
250                } else { print STDERR "Can't open/create '$localizeFile'\n"; }
251            }else{
252                print STDERR "WARNING: Can't open/create '$localizeFile'\n";
253            }
254            unlink $tmpfile;
255        }
256    }
257#    if( $no_sort eq '' ){
258#        sort_outfile( $localizeFile );
259#    }
260}
261
262sub get_license_header{
263    return
264"#\n".
265"#    ####    ###     #   #   ###   #####    #####  ####   #####  #####  \n".
266"#    #   #  #   #    ##  #  #   #    #      #      #   #    #      #    \n".
267"#    #   #  #   #    # # #  #   #    #      ###    #   #    #      #    \n".
268"#    #   #  #   #    #  ##  #   #    #      #      #   #    #      #    \n".
269"#    ####    ###     #   #   ###     #      #####  ####   #####    #    \n".
270"#\n".
271"#    DO NOT EDIT! This file will be overwritten by localization process\n".
272"#\n".
273"#**************************************************************\n".
274"#  \n".
275"#  Licensed to the Apache Software Foundation (ASF) under one\n".
276"#  or more contributor license agreements.  See the NOTICE file\n".
277"#  distributed with this work for additional information\n".
278"#  regarding copyright ownership.  The ASF licenses this file\n".
279"#  to you under the Apache License, Version 2.0 (the\n".
280"#  "License"); you may not use this file except in compliance\n".
281"#  with the License.  You may obtain a copy of the License at\n".
282"#  \n".
283"#    http://www.apache.org/licenses/LICENSE-2.0\n".
284"#  \n".
285"#  Unless required by applicable law or agreed to in writing,\n".
286"#  software distributed under the License is distributed on an\n".
287"#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n".
288"#  KIND, either express or implied.  See the License for the\n".
289"#  specific language governing permissions and limitations\n".
290"#  under the License.\n".
291"#  \n".
292"#**************************************************************\n";
293}
294######## Check input sdf file and use only the correct part
295sub merge_gsicheck{
296    my $command = '';
297    my ( $TMPHANDLE , $tmpfile ) = File::Temp::tempfile();
298    my ( $TMPHANDLE2 , $tmpfile2 ) = File::Temp::tempfile();
299    close ( $TMPHANDLE );
300    close ( $TMPHANDLE2 );
301
302    unlink $tmpfile2;
303    my $output2 = `cat $sdffile | sort > $tmpfile2`;
304    my $rc2 = $? << 8;
305    if( $rc2 ne  0 ){
306        printf("ERROR: Failed -> cat $sdffile | sort > $tmpfile2\n$output2\n");
307        exit( -1 );
308    }
309
310#    if( $ENV{WRAPCMD} ){
311#        $command = "$ENV{WRAPCMD} gsicheck";
312#    }else{
313#        $command = "gsicheck";
314#    }
315#    my $errfile = $tmpfile.".err";
316#    $command .= " -k -c -wcf $tmpfile -wef $errfile -l \"\" $tmpfile2";
317#    my $output = `$command`;
318#    my $rc = $? << 8;
319#    if ( $output ne "" ){
320#        print STDOUT "### gsicheck ###\n";
321#        print STDOUT "### The file $errfile have been written containing the errors in your sdf file. Those lines will not be merged: ###\n\n";
322#        print STDOUT "$output\n";
323#        print STDOUT "################\n";
324#
325#    }else{
326#        # Remove the 0 Byte file
327#        unlink $errfile;
328#    }
329    $sdffile = $tmpfile2;
330#    unlink $tmpfile2;
331}
332#########################################################
333sub collectfiles{
334    print STDOUT "### Localize\n";
335    my @sdfparticles;
336    my $localizehash_ref;
337    my ( $bAll , $bUseLocalize, $langhash_ref , $bHasSourceLanguage , $bFakeEnglish ) = parseLanguages();
338
339    # Enable autoflush on STDOUT
340    # $| = 1;
341    STDOUT->autoflush( 1 );
342
343    ### Search sdf particles
344    print STDOUT "### Searching sdf particles\n";
345    my $working_path = getcwd();
346    chdir $srcpath;
347    find sub {
348        my $file = $File::Find::name;
349        if( -f && $file =~ /.*localize.sdf$/ ) {
350            push   @sdfparticles , $file;
351            if( $bVerbose eq "1" ) { print STDOUT "$file\n"; }
352            else { print ".";  }
353
354         }
355    } , getcwd() ;#"."; #$srcpath;
356    chdir $working_path;
357
358    my $nFound  = $#sdfparticles +1;
359    print "\n    $nFound files found !\n";
360
361    my ( $LOCALIZEPARTICLE , $localizeSDF ) = File::Temp::tempfile();
362    close( $LOCALIZEPARTICLE );
363
364    my ( $ALLPARTICLES_MERGED , $particleSDF_merged )     = File::Temp::tempfile();
365    close( $ALLPARTICLES_MERGED );
366    my ( $LOCALIZE_LOG , $my_localize_log ) = File::Temp::tempfile();
367    close( $LOCALIZE_LOG );
368
369    ## Get the localize de,en-US extract
370    if( $bAll || $bUseLocalize ){
371        print "### Fetching source language strings\n";
372        my $command = "";
373        my $args    = "";
374
375        if( $ENV{WRAPCMD} ){
376            $command = "$ENV{WRAPCMD} localize_sl";
377        }else{
378            $command = "localize_sl";
379        }
380
381        # -e
382        # if ( -x $command ){
383        if( $command ){
384            if( !$bVerbose  ){ $args .= " -QQ -skip_links "; }
385            $args .= " -e -f $localizeSDF -l ";
386            my $bFlag="";
387            if( $bAll ) {$args .= " en-US";}
388            else{
389              my @list;
390              foreach my $isokey ( keys( %{ $langhash_ref } ) ){
391                push @list , $isokey;
392                if( $langhash_ref->{ $isokey } ne "" ){
393                    push @list , $langhash_ref->{ $isokey };
394                }
395              }
396              remove_duplicates( \@list );
397              foreach my $isokey ( @list ){
398                switch :{
399                    #( $isokey=~ /^de$/i  )
400                    #    && do{
401                    #            if( $bFlag eq "TRUE" ){ $args .= ",de"; }
402                    #            else  {
403                    #                $args .=  "de";  $bFlag = "TRUE";
404                    #             }
405                    #          };
406                        ( $isokey=~ /^en-US$/i  )
407                        && do{
408                                if( $bFlag eq "TRUE" ){ $args .= ",en-US"; }
409                                else {
410                                    $args .= "en-US";  $bFlag = "TRUE";
411                                 }
412                              };
413
414                    } #switch
415                } #foreach
416              } # if
417        } # if
418        if ( $bVerbose ) { print STDOUT $command.$args."\n"; }
419
420        my $rc = system( $command.$args );
421
422        #my $output = `$command.$args`;
423        #my $rc = $? << 8;
424
425        if( $rc < 0 ){    print STDERR "ERROR: localize rc = $rc\n"; exit( -1 ); }
426        ( $localizehash_ref )  = read_file( $localizeSDF , $langhash_ref );
427
428    }
429    ## Get sdf particles
430   open ALLPARTICLES_MERGED , "+>> $particleSDF_merged"
431    or die "Can't open $particleSDF_merged";
432
433    ## Fill fackback hash
434    my( $fallbackhashhash_ref ) = fetch_fallback( \@sdfparticles , $localizeSDF ,  $langhash_ref );
435#    my( $fallbackhashhash_ref ) = fetch_fallback( \@sdfparticles , $localizeSDF , $langhash_ref );
436    my %block;
437    my $cur_fallback;
438    if( !$bAll) {
439        foreach my $cur_lang ( keys( %{ $langhash_ref } ) ){
440            #print STDOUT "DBG: G1 cur_lang=$cur_lang\n";
441            $cur_fallback = $langhash_ref->{ $cur_lang };
442            if( $cur_fallback ne "" ){
443                # Insert fallback strings
444                #print STDOUT "DBG: Renaming $cur_fallback to $cur_lang in fallbackhash\n";
445                rename_language(  $fallbackhashhash_ref ,  $cur_fallback , $cur_lang );
446            }
447            foreach my $currentfile ( @sdfparticles ){
448                if ( open MYFILE , "< $currentfile" ) {
449                    while(<MYFILE>){
450                        if( /$sdf_regex/ ){
451                            my $line           = defined $_ ? $_ : '';
452                            my $prj            = defined $3 ? $3 : '';
453                            my $file           = defined $4 ? $4 : '';
454                            my $type           = defined $6 ? $6 : '';
455                            my $gid            = defined $7 ? $7 : '';
456                            my $lid            = defined $8 ? $8 : '';
457                            my $lang           = defined $12 ? $12 : '';
458                            my $platform       = defined $10 ? $10 : '';
459                            my $helpid         = defined $9 ? $9 : '';
460
461                            chomp( $line );
462
463                            if ( $lang eq $cur_lang ){
464                                # Overwrite fallback strings with collected strings
465                                #if( ( !has_two_sourcelanguages( $cur_lang) && $cur_lang eq "de" ) || $cur_lang ne "en-US" ){
466                                     $fallbackhashhash_ref->{ $cur_lang }{ $prj.$gid.$lid.$file.$type.$platform.$helpid } =  $line ;
467                                     #}
468
469                            }
470                        }
471                    }
472                }else { print STDERR "WARNING: Can't open file $currentfile"; }
473            }
474
475            foreach my $line ( keys( %{$fallbackhashhash_ref->{ $cur_lang } } )) {
476                if( #$cur_lang ne "de" &&
477                    $cur_lang ne "en-US" ){
478                    print ALLPARTICLES_MERGED ( $fallbackhashhash_ref->{ $cur_lang }{ $line }, "\n" );
479                }
480             }
481        }
482    } else {
483        foreach my $currentfile ( @sdfparticles ){
484            if ( open MYFILE , "< $currentfile" ) {
485                while( <MYFILE> ){
486                    print ALLPARTICLES_MERGED ( $_, "\n" );  # recheck de / en-US !
487                }
488            }
489            else { print STDERR "WARNING: Can't open file $currentfile"; }
490        }
491    }
492    close ALLPARTICLES_MERGED;
493
494
495    # Hash of array
496    my %output;
497    my @order;
498
499    ## Join both
500    if( $outputfile ){
501        if( open DESTFILE , "+> $outputfile" ){
502            if( !open LOCALIZEPARTICLE ,  "< $localizeSDF" ) { print STDERR "ERROR: Can't open file $localizeSDF\n"; }
503            if( !open ALLPARTICLES_MERGED , "< $particleSDF_merged" ) { print STDERR "ERROR: Can't open file $particleSDF_merged\n"; }
504
505            # Insert localize
506            my $extract_date="";
507            while ( <LOCALIZEPARTICLE> ){
508                if( /$sdf_regex/ ){
509                    my $leftpart       = defined $2 ? $2 : '';
510                    my $lang           = defined $12 ? $12 : '';
511                    my $rightpart      = defined $13 ? $13 : '';
512                    my $timestamp      = defined $18 ? $18 : '';
513
514                    my $prj            = defined $3 ? $3 : '';
515                    my $file           = defined $4 ? $4 : '';
516                    my $type           = defined $6 ? $6 : '';
517                    my $gid            = defined $7 ? $7 : '';
518                    my $lid            = defined $8 ? $8 : '';
519                    #my $lang           = defined $12 ? $12 : '';
520                    my $platform       = defined $10 ? $10 : '';
521                    my $helpid         = defined $9 ? $9 : '';
522
523
524                    if( $use_default_date )
525                    {
526                        $extract_date = "$default_date\n" ;
527                    }
528                    elsif( $extract_date eq "" ) {
529                        $extract_date = $timestamp ;
530                        $extract_date =~ tr/\r\n//d;
531                        $extract_date .= "\n";
532                    }
533
534                    if( $bAll ){ print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date ; }
535                    else {
536                        foreach my $sLang ( keys( %{ $langhash_ref } ) ){
537                            if( $sLang=~ /all/i )                       {
538                                push @{ $output{ $prj.$gid.$lid.$file.$type.$platform.$helpid } } ,  $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
539                                #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date;
540                            }
541                            #if( $sLang eq "de" && $lang eq "de" )       {
542                            #    push @{ $output{ $prj.$gid.$lid.$file.$type.$platform.$helpid } } ,  $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
543                                #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date;
544                                #}
545                            if( $sLang eq "en-US" && $lang eq "en-US" ) {
546                                push @order , $prj.$gid.$lid.$file.$type.$platform.$helpid;
547                                if( !$bFakeEnglish ){ push @{ $output{ $prj.$gid.$lid.$file.$type.$platform.$helpid } } ,  $leftpart."\t".$lang."\t".$rightpart.$extract_date ; }
548                                #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date;
549                            }
550
551                        }
552                    }
553                }
554            }
555            # Insert particles
556            while ( <ALLPARTICLES_MERGED> ){
557                if( /$sdf_regex/ ){
558                    my $leftpart       = defined $2 ? $2 : '';
559                    my $prj            = defined $3 ? $3 : '';
560                    my $lang           = defined $12 ? $12 : '';
561                    my $rightpart      = defined $13 ? $13 : '';
562                    my $timestamp      = defined $18 ? $18 : '';
563
564                    #my $prj            = defined $3 ? $3 : '';
565                    my $file           = defined $4 ? $4 : '';
566                    my $type           = defined $6 ? $6 : '';
567                    my $gid            = defined $7 ? $7 : '';
568                    my $lid            = defined $8 ? $8 : '';
569                    #my $lang           = defined $12 ? $12 : '';
570                    my $platform       = defined $10 ? $10 : '';
571                    my $helpid         = defined $9 ? $9 : '';
572
573
574                    if( $use_default_date )
575                    {
576                        $extract_date = "$default_date\n" ;
577                    }
578                    elsif( $extract_date eq "" )
579                    {
580                        $extract_date = $timestamp;
581                    }
582
583                    if( ! ( $prj =~ /binfilter/i ) ) {
584                        push @{ $output{ $prj.$gid.$lid.$file.$type.$platform.$helpid } } , $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
585                        #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
586                    }
587                 }
588            }
589
590            # Write!
591            foreach my $curkey ( @order ){
592                foreach my $curlist ( $output{ $curkey } ){
593                    foreach my $line ( @{$curlist} ){
594                        print DESTFILE $line;
595                    }
596                }
597            }
598
599        }else { print STDERR "Can't open $outputfile";}
600    }
601    close DESTFILE;
602    close LOCALIZEPARTICLE;
603    close ALLPARTICLES_MERGED;
604
605    #print STDOUT "DBG: \$localizeSDF $localizeSDF \$particleSDF_merged $particleSDF_merged\n";
606    unlink $localizeSDF , $particleSDF_merged ,  $my_localize_log;
607
608    #sort_outfile( $outputfile );
609    #remove_obsolete( $outputfile ) , if $bHasSourceLanguage ne "";
610    }
611
612#########################################################
613sub remove_obsolete{
614    my $outfile = shift;
615    my @lines;
616    my $enusleftpart;
617    my @good_lines;
618
619    print STDOUT "### Removing obsolete strings\n";
620
621    # Kick out all strings without en-US reference
622    if ( open ( SORTEDFILE , "< $outfile" ) ){
623        while( <SORTEDFILE> ){
624            if( /$sdf_regex/ ){
625                my $line           = defined $_ ? $_ : '';
626                my $language       = defined $12 ? $12 : '';
627                my $prj            = defined $3 ? $3 : '';
628                my $file           = defined $4 ? $4 : '';
629                my $type           = defined $6 ? $6 : '';
630                my $gid            = defined $7 ? $7 : '';
631                my $lid            = defined $8 ? $8 : '';
632                my $platform       = defined $10 ? $10 : '';
633                my $helpid         = defined $9 ? $9 : '';
634
635                my $leftpart = $prj.$gid.$lid.$file.$type.$platform.$helpid;
636
637                if( $language eq "en-US" ){                 # source string found, 1. entry
638                    $enusleftpart = $leftpart;
639                    push @good_lines , $line;
640                }else{
641                    if( !defined $enusleftpart or !defined $leftpart ){
642                        print STDERR "BADLINE: $line\n";
643                        print STDERR "\$enusleftpart = $enusleftpart\n";
644                        print STDERR "\$leftpart = $leftpart\n";
645                    }
646                    if( $enusleftpart eq $leftpart ){   # matching language
647                        push @good_lines , $line;
648                    }
649                    #else{
650                    #    print STDERR "OUT:  \$enusleftpart=$enusleftpart \$leftpart=$leftpart \$line=$line\n";
651                    #}
652                }
653            }
654        }
655        close SORTEDFILE;
656    } else { print STDERR "ERROR: Can't open file $outfile\n";}
657
658    # Write file
659    if ( open ( SORTEDFILE , "> $outfile" ) ){
660        foreach my $newline ( @good_lines ) {
661            print SORTEDFILE $newline;
662        }
663        close SORTEDFILE;
664    } else { print STDERR "ERROR: Can't open file $outfile\n";}
665
666}
667#########################################################
668sub sort_outfile{
669        my $outfile = shift;
670        print STDOUT "### Sorting ... $outfile ...";
671        my @lines;
672        my @sorted_lines;
673
674
675        #if ( open ( SORTEDFILE , "< $outputfile" ) ){
676        if ( open ( SORTEDFILE , "< $outfile" ) ){
677            my $line;
678            while ( <SORTEDFILE> ){
679                $line = $_;
680                if( $line =~ /^[^\#]/ ){
681                    push @lines , $line;
682                }
683            }
684            close SORTEDFILE;
685            @sorted_lines = sort {
686                my $xa_lang          = "";
687                my $xa_left_part    = "";
688                my $xa_right_part    = "";
689                my $xa_timestamp     = "";
690                my $xb_lang          = "";
691                my $xb_left_part    = "";
692                my $xb_right_part    = "";
693                my $xb_timestamp     = "";
694                my $xa               = "";
695                my $xb               = "";
696                my @alist;
697                my @blist;
698
699                if( $a=~ /$sdf_regex/ ){
700                    $xa_left_part       = defined $2 ? $2 : '';
701                    $xa_lang           = defined $12 ? $12 : '';
702                    $xa_right_part     = defined $13 ? $13 : '';
703                    $xa_left_part = remove_last_column( $xa_left_part );
704
705                }
706                if( $b=~ /$sdf_regex/ ){
707                    $xb_left_part       = defined $2 ? $2 : '';
708                    $xb_lang           = defined $12 ? $12 : '';
709                    $xb_right_part     = defined $13 ? $13 : '';
710                    $xb_left_part = remove_last_column( $xb_left_part );
711
712
713                }
714                if( (  $xa_left_part cmp $xb_left_part ) == 0 ){         # Left part equal
715                     if( ( $xa_lang cmp $xb_lang ) == 0 ){               # Lang equal
716                         return ( $xa_right_part cmp $xb_right_part );   # Right part compare
717                    }
718                    elsif( $xa_lang eq "en-US" ) { return -1; }        # en-US wins
719                    elsif( $xb_lang eq "en-US" ) { return 1;  }        # en-US wins
720                    else { return $xa_lang cmp $xb_lang; }             # lang compare
721                }
722                else {
723                    return $xa_left_part cmp $xb_left_part;        # Left part compare
724                }
725            } @lines;
726
727            if ( open ( SORTEDFILE , "> $outfile" ) ){
728                print SORTEDFILE get_license_header();
729                foreach my $newline ( @sorted_lines ) {
730                    print SORTEDFILE $newline;
731                    #print STDOUT $newline;
732                }
733            }
734            close SORTEDFILE;
735        } else { print STDERR "WARNING: Can't open file $outfile\n";}
736	print "done\n";
737
738}
739#########################################################
740sub remove_last_column{
741    my $string                  = shift;
742    my @alist = split ( "\t" , $string );
743    pop @alist;
744    return join( "\t" , @alist );
745}
746
747#########################################################
748sub rename_language{
749    my $fallbackhashhash_ref    = shift;
750    my $cur_fallback            = shift;
751    my $cur_lang                = shift;
752    my $line;
753
754    foreach my $key( keys ( %{ $fallbackhashhash_ref->{ $cur_fallback } } ) ){
755        $line = $fallbackhashhash_ref->{ $cur_fallback }{ $key };
756        if( $line =~ /$sdf_regex/ ){
757            my $leftpart       = defined $2 ? $2 : '';
758            my $lang           = defined $12 ? $12 : '';
759            my $rightpart      = defined $13 ? $13 : '';
760
761            $fallbackhashhash_ref->{ $cur_lang }{ $key } = $leftpart."\t".$cur_lang."\t".$rightpart;
762        }
763    }
764}
765
766############################################################
767sub remove_duplicates{
768    my $list_ref    = shift;
769    my %tmphash;
770    foreach my $key ( @{ $list_ref } ){ $tmphash{ $key } = '' ; }
771    @{$list_ref} = keys( %tmphash );
772}
773
774##############################################################
775sub fetch_fallback{
776    my $sdfparticleslist_ref   = shift;
777    my $localizeSDF            = shift;
778    my $langhash_ref           = shift;
779    my %fallbackhashhash;
780    my $cur_lang;
781    my @langlist;
782
783    foreach my $key ( keys ( %{ $langhash_ref } ) ){
784        $cur_lang = $langhash_ref->{ $key };
785        if ( $cur_lang ne "" ) {
786            push @langlist , $cur_lang;
787        }
788    }
789    remove_duplicates( \@langlist );
790    foreach  $cur_lang ( @langlist ){
791        if( $cur_lang eq "en-US" ){
792            read_fallbacks_from_source( $localizeSDF , $cur_lang , \%fallbackhashhash );
793        }
794    }
795
796    # remove de / en-US
797    my @tmplist;
798    foreach $cur_lang( @langlist ){
799        if(  $cur_lang ne "en-US" ){
800           push @tmplist , $cur_lang;
801
802        }
803    }
804    @langlist = @tmplist;
805    if ( $#langlist +1 ){
806        read_fallbacks_from_particles( $sdfparticleslist_ref , \@langlist , \%fallbackhashhash );
807
808    }
809    return (\%fallbackhashhash);
810}
811
812#########################################################
813sub write_file{
814
815    my $localizeFile = shift;
816    my $index_ref    = shift;
817
818    if( open DESTFILE , "+> $localizeFile" ){
819        foreach my $key( %{ $index_ref } ){
820            print DESTFILE ($index_ref->{ $key }, "\n" );
821        }
822        close DESTFILE;
823    }else {
824      print STDERR "Can't open/create '$localizeFile'";
825    }
826}
827
828#########################################################
829sub read_file{
830
831    my $sdffile         = shift;
832    my $langhash_ref    = shift;
833    my %block           = ();
834
835    open MYFILE , "< $sdffile"
836        or die "Can't open '$sdffile'\n";
837        while( <MYFILE>){
838          if( /$sdf_regex/ ){
839            my $line           = defined $_ ? $_ : '';
840            my $prj            = defined $3 ? $3 : '';
841            my $file           = defined $4 ? $4 : '';
842            my $type           = defined $6 ? $6 : '';
843            my $gid            = defined $7 ? $7 : '';
844            my $lid            = defined $8 ? $8 : '';
845            my $platform       = defined $10 ? $10 : '';
846            my $lang           = defined $12 ? $12 : '';
847            my $helpid         = defined $9 ? $9 : '';
848
849            foreach my $isolang ( keys ( %{ $langhash_ref } ) ){
850                if( $isolang=~ /$lang/i || $isolang=~ /all/i ) { $block{$prj.$gid.$lid.$file.$type.$platform.$helpid } =  $line ; }
851            }
852        }
853    }
854    return (\%block);
855}
856
857#########################################################
858sub read_fallbacks_from_particles{
859
860    my $sdfparticleslist_ref    = shift;
861    my $isolanglist_ref         = shift;
862    my $fallbackhashhash_ref    = shift;
863    my $block_ref;
864    foreach my $currentfile ( @{ $sdfparticleslist_ref } ){
865        if ( open MYFILE , "< $currentfile" ) {
866            while(<MYFILE>){
867                if( /$sdf_regex/ ){
868                    my $line           = defined $_ ? $_ : '';
869                    my $prj            = defined $3 ? $3 : '';
870                    my $file           = defined $4 ? $4 : '';
871                    my $type           = defined $6 ? $6 : '';
872                    my $gid            = defined $7 ? $7 : '';
873                    my $lid            = defined $8 ? $8 : '';
874                    my $lang           = defined $12 ? $12 : '';
875                    my $platform       = defined $10 ? $10 : '';
876                    my $helpid         = defined $9 ? $9 : '';
877
878                    chomp( $line );
879
880                    foreach my $isolang ( @{$isolanglist_ref}  ){
881                        if( $isolang=~ /$lang/i ) {
882                            $fallbackhashhash_ref->{ $isolang }{ $prj.$gid.$lid.$file.$type.$platform.$helpid } =  $line ;
883                        }
884                    }
885                }
886            }
887       }else { print STDERR "WARNING: Can't open file $currentfile"; }
888    }
889}
890
891#########################################################
892sub read_fallbacks_from_source{
893
894    my $sdffile                 = shift;
895    my $isolang                 = shift;
896    my $fallbackhashhash_ref    = shift;
897    my $block_ref;
898    # read fallback for single file
899    open MYFILE , "< $sdffile"
900        or die "Can't open '$sdffile'\n";
901
902    while( <MYFILE>){
903          if( /$sdf_regex/ ){
904            my $line           = defined $_ ? $_ : '';
905            my $prj            = defined $3 ? $3 : '';
906            my $file           = defined $4 ? $4 : '';
907            my $type           = defined $6 ? $6 : '';
908            my $gid            = defined $7 ? $7 : '';
909            my $lid            = defined $8 ? $8 : '';
910            my $helpid         = defined $9 ? $9 : '';
911            my $lang           = defined $12 ? $12 : '';
912            my $platform       = defined $10 ? $10 : '';
913
914            chomp( $line );
915            if( $isolang=~ /$lang/i ) { $fallbackhashhash_ref->{ $isolang }{ $prj.$gid.$lid.$file.$type.$platform.$helpid } =  $line ;
916            }
917        }
918    }
919}
920
921#########################################################
922sub parseLanguages{
923
924    my $bAll;
925    my $bUseLocalize;
926    my $bHasSourceLanguage="";
927    my $bFakeEnglish="";
928    my %langhash;
929    my $iso="";
930    my $fallback="";
931
932    #### -l all
933    if(   $languages=~ /all/ ){
934        $bAll = "TRUE";
935        $bHasSourceLanguage = "TRUE";
936    }
937    ### -l fr=de,de
938    elsif( $languages=~ /.*,.*/ ){
939        my @tmpstr =  split "," , $languages;
940        for my $lang ( @tmpstr ){
941            if( $lang=~ /([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)(=([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*))?/ ){
942                $iso        = $1;
943                $fallback   = $4;
944
945                if( ( $iso && $iso=~ /(en-US)/i )  || ( $fallback && $fallback=~ /(en-US)/i ) ) {
946                    $bUseLocalize = "TRUE";
947                }
948                if( ( $iso && $iso=~ /(en-US)/i ) ) {
949                    $bHasSourceLanguage = "TRUE";
950                }
951             if( $fallback ) { $langhash{ $iso } = $fallback;   }
952             else            { $langhash{ $iso } = "";          }
953            }
954        }
955    }
956    ### -l de
957    else{
958        if( $languages=~ /([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)(=([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*))?/ ){
959            $iso        = $1;
960            $fallback   = $4;
961
962            if( ( $iso && $iso=~ /(en-US)/i )  || ( $fallback && $fallback=~ /(en-US)/i ) ) {
963                $bUseLocalize = "TRUE";
964
965            }
966            if( ( $iso && $iso=~ /(en-US)/i )  ) {
967                $bHasSourceLanguage = "TRUE";
968            }
969
970             if( $fallback ) { $langhash{ $iso } = $fallback;   }
971             else            { $langhash{ $iso } = "";          }
972        }
973    }
974    # HACK en-US always needed!
975    if( !$bHasSourceLanguage ){
976        #$bHasSourceLanguage = "TRUE";
977        $bUseLocalize = "TRUE";
978        $bFakeEnglish = "TRUE";
979        $langhash{ "en-US" } = "";
980    }
981    return ( $bAll ,  $bUseLocalize , \%langhash , $bHasSourceLanguage, $bFakeEnglish);
982}
983
984#########################################################
985sub parse_options{
986
987    my $help;
988    my $merge;
989    my $extract;
990    my $success = GetOptions('f=s' => \$sdffile , 'l=s' => \$languages , 's=s' => \$srcpath ,  'h' => \$help , 'v' => \$bVerbose ,
991                             'm' => \$merge , 'e' => \$extract , 'x' => \$no_sort , 'd' => \$use_default_date );
992    $outputfile = $sdffile;
993
994    #print STDOUT "DBG: lang = $languages\n";
995    if( !$srcpath ){
996        #$srcpath = "$ENV{SRC_ROOT}";
997        if( !$srcpath ){
998	        print STDERR "No path to the source root found!\n\n";
999	        usage();
1000            exit(1);
1001        }
1002    }
1003    if( $help || !$success || $#ARGV > 1 || ( !$sdffile ) ){
1004        usage();
1005        exit(1);
1006    }
1007    if( $merge && $sdffile && ! ( -r $sdffile)){
1008        print STDERR "Can't open file '$sdffile'\n";
1009        exit(1);
1010    }
1011    if( !( $languages=~ /[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*(=[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)?(,[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*(=[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)?)*/ ) ){
1012        print STDERR "Please check the -l iso code\n";
1013        exit(1);
1014    }
1015    if( ( !$merge && !$extract ) || ( $merge && $extract ) ){ usage();exit( -1 );}
1016    if( $extract ){ $mode = "extract"; }
1017    else          { $mode = "merge";   }
1018}
1019
1020#########################################################
1021sub usage{
1022
1023    print STDERR "Usage: localize.pl\n";
1024    print STDERR "Split or collect SDF files\n";
1025    print STDERR "           merge: -m -f <sdffile>    -l l1[=f1][,l2[=f2]][...] [ -s <sourceroot> ]\n";
1026    print STDERR "         extract: -e -f <outputfile> -l <lang> [ -s <sourceroot> ] [-d]\n";
1027    print STDERR "Options:\n";
1028    print STDERR "    -h              help\n";
1029    print STDERR "    -m              Merge mode\n";
1030    print STDERR "    -e              Extract mode\n";
1031    print STDERR "    -f <sdffile>    To split a big SDF file into particles\n";
1032    print STDERR "       <outputfile> To collect and join all particles to one big file\n";
1033    print STDERR "    -s <sourceroot> Path to the modules, if no \$SRC_ROOT is set\n";
1034    print STDERR "    -l ( all | <isocode> | <isocode>=fallback ) comma separated languages\n";
1035    print STDERR "    -d              Use default date in extracted sdf file\n";
1036    print STDERR "    -v              Verbose\n";
1037    print STDERR "\nExample:\n";
1038    print STDERR "\nlocalize -e -l en-US,pt-BR=en-US -f my.sdf\n( Extract en-US and pt-BR with en-US fallback )\n";
1039    print STDERR "\nlocalize -m -l cs -f my.sdf\n( Merge cs translation into the sourcecode ) \n";
1040}
1041
1042#            my $line           = defined $_ ? $_ : '';
1043#            my $leftpart       = defined $2 ? $2 : '';
1044#            my $prj            = defined $3 ? $3 : '';
1045#            my $file           = defined $4 ? $4 : '';
1046#            my $dummy          = defined $5 ? $5 : '';
1047#            my $type           = defined $6 ? $6 : '';
1048#            my $gid            = defined $7 ? $7 : '';
1049#            my $lid            = defined $8 ? $8 : '';
1050#            my $helpid         = defined $9 ? $9 : '';
1051#            my $platform       = defined $10 ? $10 : '';
1052#            my $width          = defined $11 ? $11 : '';
1053#            my $lang           = defined $12 ? $12 : '';
1054#            my $rightpart      = defined $13 ? $13 : '';
1055#            my $text           = defined $14 ? $14 : '';
1056#            my $helptext       = defined $15 ? $15 : '';
1057#            my $quickhelptext  = defined $16 ? $16 : '';
1058#            my $title          = defined $17 ? $17 : '';
1059#            my $timestamp      = defined $18 ? $18 : '';
1060
1061