1:
2eval 'exec perl -wS $0 ${1+"$@"}'
3    if 0;
4
5
6#**************************************************************
7#
8#  Licensed to the Apache Software Foundation (ASF) under one
9#  or more contributor license agreements.  See the NOTICE file
10#  distributed with this work for additional information
11#  regarding copyright ownership.  The ASF licenses this file
12#  to you under the Apache License, Version 2.0 (the
13#  "License"); you may not use this file except in compliance
14#  with the License.  You may obtain a copy of the License at
15#
16#    http://www.apache.org/licenses/LICENSE-2.0
17#
18#  Unless required by applicable law or agreed to in writing,
19#  software distributed under the License is distributed on an
20#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
21#  KIND, either express or implied.  See the License for the
22#  specific language governing permissions and limitations
23#  under the License.
24#
25#**************************************************************
26
27
28
29use strict;
30use Getopt::Long;
31use IO::Handle;
32use File::Find;
33use File::Temp;
34use File::Copy;
35use File::Glob qw(:glob csh_glob);
36use Cwd;
37
38# ver 1.1
39#
40#### module lookup
41#use lib ("$ENV{SOLARENV}/bin/modules", "$ENV{COMMON_ENV_TOOLS}/modules");
42
43#### module lookup
44# OOo conform
45my @lib_dirs;
46BEGIN {
47    if ( !defined($ENV{SOLARENV}) ) {
48        die "No environment found (environment variable SOLARENV is undefined)";
49    }
50    push(@lib_dirs, "$ENV{SOLARENV}/bin/modules");
51    push(@lib_dirs, "$ENV{COMMON_ENV_TOOLS}/modules") if defined($ENV{COMMON_ENV_TOOLS});
52}
53use lib (@lib_dirs);
54
55#### globals ####
56my $sdffile = '';
57my $no_sort = '';
58my $outputfile = '';
59my $mode = '';
60my $bVerbose="0";
61my $srcpath = '';
62my $languages;
63#my %sl_modules;     # Contains all modules where en-US and de is source language
64my $use_default_date = '0';
65
66         #         (                           leftpart                                                     )            (           rightpart                    )
67         #            prj      file      dummy     type       gid       lid      helpid    pform     width      lang       text    helptext  qhelptext   title    timestamp
68my $sdf_regex  = "((([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*))\t([^\t]*)\t(([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t)([^\t]*))";
69my $file_types = "(src|hrc|xcs|xcu|lng|ulf|xrm|xhp|xcd|xgf|xxl|xrb)";
70# Always use this date to prevent cvs conflicts
71my $default_date = "2002-02-02 02:02:02";
72
73#### main ####
74parse_options();
75
76#%sl_modules = fetch_sourcelanguage_dirlist();
77
78
79if   ( $mode eq "merge"    )    {
80    merge_gsicheck();
81    splitfile( $sdffile );
82    unlink $sdffile;             # remove temp file!
83}
84elsif( $mode eq "extract"  )    {
85    collectfiles( $outputfile );
86}
87else                            {
88    usage();
89}
90
91exit(0);
92
93#########################################################
94sub splitfile{
95
96    my $lastFile        = '';
97    my $currentFile     = '';
98    my $cur_sdffile     = '';
99    my $last_sdffile    = '';
100    my $delim;
101    my $badDelim;
102    my $start           = 'TRUE';
103    my %index  = ();
104    my %block;
105
106    STDOUT->autoflush( 1 );
107
108    #print STDOUT "Open File $sdffile\n";
109    open MYFILE , "< $sdffile"
110    or die "Can't open '$sdffile'\n";
111
112    while( <MYFILE>){
113         if( /$sdf_regex/ ){
114            my $line           = defined $_ ? $_ : '';
115            my $prj            = defined $3 ? $3 : '';
116            my $file           = defined $4 ? $4 : '';
117            my $type           = defined $6 ? $6 : '';
118            my $gid            = defined $7 ? $7 : '';
119            my $lid            = defined $8 ? $8 : '';
120            my $lang           = defined $12 ? $12 : '';
121            my $plattform      = defined $10 ? $10 : '';
122            my $helpid         = defined $9 ? $9 : '';
123
124            next if( $prj eq "binfilter" );     # Don't merge strings into binfilter module
125	        chomp( $line );
126            $currentFile  = $srcpath . '\\' . $prj . '\\' . $file;
127            $currentFile  =~ s/\\/\//g;
128
129            $cur_sdffile = $currentFile;
130            #if( $cur_sdffile =~ /\.$file_types[\s]*$/ ){
131                $cur_sdffile =~ s/\/[^\/]*\.$file_types[\s]*$/\/localize.sdf/;
132            #}
133
134            # Set default date
135            if( $line =~ /(.*)\t[^\t\$]*$/ ){
136                $line = $1."\t".$default_date;
137            }
138
139            if( $start ){
140                $start='';
141                $lastFile = $currentFile; # ?
142                $last_sdffile = $cur_sdffile;
143            }
144
145            if( $lang eq "en-US" ){}
146            elsif( $cur_sdffile eq $last_sdffile )
147            {
148                $block{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } =  $line ;
149            }
150            else
151            {
152                writesdf( $lastFile , \%block );
153                $lastFile = $currentFile; #?
154                $last_sdffile = $cur_sdffile;
155                %block = ();
156                #if( ! $lang eq "en-US"  ) {
157                $block{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } =  $line ;
158                #}
159
160            }
161        } #else { print STDOUT "splitfile REGEX kaputt\n";}
162
163    }
164    writesdf( $lastFile , \%block );
165    %block = ();
166    close( MYFILE );
167
168}
169#########################################################
170
171sub writesdf{
172
173    my $lastFile        = shift;
174    my $blockhash_ref   = shift;
175    my $localizeFile    = $lastFile;
176    my %index=();
177
178    if( $localizeFile =~ /\.$file_types[\s]*$/ ){
179        $localizeFile =~ s/\/[^\/]*\.$file_types[\s]*$/\/localize.sdf/;
180        }else {
181            print STDERR "Strange filetype found '$localizeFile'\n";
182            return;
183        }
184        if( open DESTFILE , "< $localizeFile" ){
185
186        #or die "Can't open/create '\$localizeFile'";
187
188        #### Build hash
189        while(<DESTFILE>){
190         if( /$sdf_regex/ ){
191            my $line           = defined $_ ? $_ : '';
192            my $prj            = defined $3 ? $3 : '';
193            my $file           = defined $4 ? $4 : '';
194            my $type           = defined $6 ? $6 : '';
195            my $gid            = defined $7 ? $7 : '';
196            my $lid            = defined $8 ? $8 : '';
197            my $lang           = defined $12 ? $12 : '';
198            my $plattform      = defined $10 ? $10 : '';
199            my $helpid         = defined $9 ? $9 : '';
200
201            chomp( $line );
202            $index{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } =  $line ;
203
204         } #else { print STDOUT "writesdf REGEX kaputt $_\n";}
205
206        }
207        close( DESTFILE );
208    }
209    #### Copy new strings
210    my @mykeys = keys( %{ $blockhash_ref } );
211    my $isDirty = "FALSE";
212    foreach my $key( @mykeys ){
213        if( ! defined $index{ $key } ){
214            # Add new entry
215            $index{ $key }  = $blockhash_ref->{ $key} ;
216            $isDirty = "TRUE";
217        }elsif( $index{ $key } ne $blockhash_ref->{ $key } ){
218            # Overwrite old entry
219            $index{ $key } = $blockhash_ref->{ $key };
220            $isDirty = "TRUE";
221        }else {
222        }
223    }
224
225    #### Write file
226
227    if( !$bVerbose ){ print STDOUT "."; }
228    if( $isDirty eq "TRUE" ){
229        if( $bVerbose ){ print STDOUT "$localizeFile\n"; }
230        if( open DESTFILE , "+> $localizeFile" ){
231            print DESTFILE get_license_header();
232            @mykeys = sort keys( %index );
233            foreach my $key( @mykeys ){
234                print DESTFILE ( $index{ $key } , "\n" );
235            }
236            close DESTFILE;
237         }else {
238            print STDOUT "WARNING: File $localizeFile is not writable , try to merge ...\n";
239            my ( $TMPFILE , $tmpfile ) = File::Temp::tempfile();
240            if( open DESTFILE , "+> $tmpfile " ){
241                @mykeys = keys( %index );
242                foreach my $key( @mykeys ){
243                    print DESTFILE ( $index{ $key } , "\n" );
244                }
245                close DESTFILE;
246                if( move( $localizeFile , $localizeFile.".backup" ) ){
247                    if( copy( $tmpfile , $localizeFile ) ){
248                        unlink $localizeFile.".backup";
249                    } else { print STDERR "Can't open/create '$localizeFile', original file is renamed to  $localizeFile.backup\n"; }
250                } else { print STDERR "Can't open/create '$localizeFile'\n"; }
251            }else{
252                print STDERR "WARNING: Can't open/create '$localizeFile'\n";
253            }
254            unlink $tmpfile;
255        }
256    }
257#    if( $no_sort eq '' ){
258#        sort_outfile( $localizeFile );
259#    }
260}
261
262sub get_license_header{
263    return
264"#\n".
265"#    ####    ###     #   #   ###   #####    #####  ####   #####  #####  \n".
266"#    #   #  #   #    ##  #  #   #    #      #      #   #    #      #    \n".
267"#    #   #  #   #    # # #  #   #    #      ###    #   #    #      #    \n".
268"#    #   #  #   #    #  ##  #   #    #      #      #   #    #      #    \n".
269"#    ####    ###     #   #   ###     #      #####  ####   #####    #    \n".
270"#\n".
271"#    DO NOT EDIT! This file will be overwritten by localisation process\n".
272"#\n".
273"#*************************************************************************\n".
274"#\n".
275"# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.\n".
276"# \n".
277"# Copyright 2000, 2010 Oracle and/or its affiliates.\n".
278"#\n".
279"# OpenOffice.org - a multi-platform office productivity suite\n".
280"#\n".
281"# This file is part of OpenOffice.org.\n".
282"#\n".
283"# OpenOffice.org is free software: you can redistribute it and/or modify\n".
284"# it under the terms of the GNU Lesser General Public License version 3\n".
285"# only, as published by the Free Software Foundation.\n".
286"#\n".
287"# OpenOffice.org is distributed in the hope that it will be useful,\n".
288"# but WITHOUT ANY WARRANTY; without even the implied warranty of\n".
289"# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n".
290"# GNU Lesser General Public License version 3 for more details\n".
291"# (a copy is included in the LICENSE file that accompanied this code).\n".
292"#\n".
293"# You should have received a copy of the GNU Lesser General Public License\n".
294"# version 3 along with OpenOffice.org.  If not, see\n".
295"# <http://www.openoffice.org/license.html>\n".
296"# for a copy of the LGPLv3 License.\n".
297"#\n".
298"#*************************************************************************\n";
299}
300######## Check input sdf file and use only the correct part
301sub merge_gsicheck{
302    my $command = '';
303    my ( $TMPHANDLE , $tmpfile ) = File::Temp::tempfile();
304    my ( $TMPHANDLE2 , $tmpfile2 ) = File::Temp::tempfile();
305    close ( $TMPHANDLE );
306    close ( $TMPHANDLE2 );
307
308    unlink $tmpfile2;
309    my $output2 = `cat $sdffile | sort > $tmpfile2`;
310    my $rc2 = $? << 8;
311    if( $rc2 ne  0 ){
312        printf("ERROR: Failed -> cat $sdffile | sort > $tmpfile2\n$output2\n");
313        exit( -1 );
314    }
315
316#    if( $ENV{WRAPCMD} ){
317#        $command = "$ENV{WRAPCMD} gsicheck";
318#    }else{
319#        $command = "gsicheck";
320#    }
321#    my $errfile = $tmpfile.".err";
322#    $command .= " -k -c -wcf $tmpfile -wef $errfile -l \"\" $tmpfile2";
323#    my $output = `$command`;
324#    my $rc = $? << 8;
325#    if ( $output ne "" ){
326#        print STDOUT "### gsicheck ###\n";
327#        print STDOUT "### The file $errfile have been written containing the errors in your sdf file. Those lines will not be merged: ###\n\n";
328#        print STDOUT "$output\n";
329#        print STDOUT "################\n";
330#
331#    }else{
332#        # Remove the 0 Byte file
333#        unlink $errfile;
334#    }
335    $sdffile = $tmpfile2;
336#    unlink $tmpfile2;
337}
338#########################################################
339sub collectfiles{
340    print STDOUT "### Localize\n";
341    my @sdfparticles;
342    my $localizehash_ref;
343    my ( $bAll , $bUseLocalize, $langhash_ref , $bHasSourceLanguage , $bFakeEnglish ) = parseLanguages();
344
345    # Enable autoflush on STDOUT
346    # $| = 1;
347    STDOUT->autoflush( 1 );
348
349    ### Search sdf particles
350    print STDOUT "### Searching sdf particles\n";
351    my $working_path = getcwd();
352    chdir $srcpath;
353    find sub {
354        my $file = $File::Find::name;
355        if( -f && $file =~ /.*localize.sdf$/ ) {
356            push   @sdfparticles , $file;
357            if( $bVerbose eq "1" ) { print STDOUT "$file\n"; }
358            else { print ".";  }
359
360         }
361    } , getcwd() ;#"."; #$srcpath;
362    chdir $working_path;
363
364    my $nFound  = $#sdfparticles +1;
365    print "\n    $nFound files found !\n";
366
367    my ( $LOCALIZEPARTICLE , $localizeSDF ) = File::Temp::tempfile();
368    close( $LOCALIZEPARTICLE );
369
370    my ( $ALLPARTICLES_MERGED , $particleSDF_merged )     = File::Temp::tempfile();
371    close( $ALLPARTICLES_MERGED );
372    my ( $LOCALIZE_LOG , $my_localize_log ) = File::Temp::tempfile();
373    close( $LOCALIZE_LOG );
374
375    ## Get the localize de,en-US extract
376    if( $bAll || $bUseLocalize ){
377        print "### Fetching source language strings\n";
378        my $command = "";
379        my $args    = "";
380
381        if( $ENV{WRAPCMD} ){
382            $command = "$ENV{WRAPCMD} localize_sl";
383        }else{
384            $command = "localize_sl";
385        }
386
387        # -e
388        # if ( -x $command ){
389        if( $command ){
390            if( !$bVerbose  ){ $args .= " -QQ -skip_links "; }
391            $args .= " -e -f $localizeSDF -l ";
392            my $bFlag="";
393            if( $bAll ) {$args .= " en-US";}
394            else{
395              my @list;
396              foreach my $isokey ( keys( %{ $langhash_ref } ) ){
397                push @list , $isokey;
398                if( $langhash_ref->{ $isokey } ne "" ){
399                    push @list , $langhash_ref->{ $isokey };
400                }
401              }
402              remove_duplicates( \@list );
403              foreach my $isokey ( @list ){
404                switch :{
405                    #( $isokey=~ /^de$/i  )
406                    #    && do{
407                    #            if( $bFlag eq "TRUE" ){ $args .= ",de"; }
408                    #            else  {
409                    #                $args .=  "de";  $bFlag = "TRUE";
410                    #             }
411                    #          };
412                        ( $isokey=~ /^en-US$/i  )
413                        && do{
414                                if( $bFlag eq "TRUE" ){ $args .= ",en-US"; }
415                                else {
416                                    $args .= "en-US";  $bFlag = "TRUE";
417                                 }
418                              };
419
420                    } #switch
421                } #foreach
422              } # if
423        } # if
424        if ( $bVerbose ) { print STDOUT $command.$args."\n"; }
425
426        my $rc = system( $command.$args );
427
428        #my $output = `$command.$args`;
429        #my $rc = $? << 8;
430
431        if( $rc < 0 ){    print STDERR "ERROR: localize rc = $rc\n"; exit( -1 ); }
432        ( $localizehash_ref )  = read_file( $localizeSDF , $langhash_ref );
433
434    }
435    ## Get sdf particles
436   open ALLPARTICLES_MERGED , "+>> $particleSDF_merged"
437    or die "Can't open $particleSDF_merged";
438
439    ## Fill fackback hash
440    my( $fallbackhashhash_ref ) = fetch_fallback( \@sdfparticles , $localizeSDF ,  $langhash_ref );
441#    my( $fallbackhashhash_ref ) = fetch_fallback( \@sdfparticles , $localizeSDF , $langhash_ref );
442    my %block;
443    my $cur_fallback;
444    if( !$bAll) {
445        foreach my $cur_lang ( keys( %{ $langhash_ref } ) ){
446            #print STDOUT "DBG: G1 cur_lang=$cur_lang\n";
447            $cur_fallback = $langhash_ref->{ $cur_lang };
448            if( $cur_fallback ne "" ){
449                # Insert fallback strings
450                #print STDOUT "DBG: Renaming $cur_fallback to $cur_lang in fallbackhash\n";
451                rename_language(  $fallbackhashhash_ref ,  $cur_fallback , $cur_lang );
452            }
453            foreach my $currentfile ( @sdfparticles ){
454                if ( open MYFILE , "< $currentfile" ) {
455                    while(<MYFILE>){
456                        if( /$sdf_regex/ ){
457                            my $line           = defined $_ ? $_ : '';
458                            my $prj            = defined $3 ? $3 : '';
459                            my $file           = defined $4 ? $4 : '';
460                            my $type           = defined $6 ? $6 : '';
461                            my $gid            = defined $7 ? $7 : '';
462                            my $lid            = defined $8 ? $8 : '';
463                            my $lang           = defined $12 ? $12 : '';
464                            my $plattform      = defined $10 ? $10 : '';
465                            my $helpid         = defined $9 ? $9 : '';
466
467                            chomp( $line );
468
469                            if ( $lang eq $cur_lang ){
470                                # Overwrite fallback strings with collected strings
471                                #if( ( !has_two_sourcelanguages( $cur_lang) && $cur_lang eq "de" ) || $cur_lang ne "en-US" ){
472                                     $fallbackhashhash_ref->{ $cur_lang }{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } =  $line ;
473                                     #}
474
475                            }
476                        }
477                    }
478                }else { print STDERR "WARNING: Can't open file $currentfile"; }
479            }
480
481            foreach my $line ( keys( %{$fallbackhashhash_ref->{ $cur_lang } } )) {
482                if( #$cur_lang ne "de" &&
483                    $cur_lang ne "en-US" ){
484                    print ALLPARTICLES_MERGED ( $fallbackhashhash_ref->{ $cur_lang }{ $line }, "\n" );
485                }
486             }
487        }
488    } else {
489        foreach my $currentfile ( @sdfparticles ){
490            if ( open MYFILE , "< $currentfile" ) {
491                while( <MYFILE> ){
492                    print ALLPARTICLES_MERGED ( $_, "\n" );  # recheck de / en-US !
493                }
494            }
495            else { print STDERR "WARNING: Can't open file $currentfile"; }
496        }
497    }
498    close ALLPARTICLES_MERGED;
499
500
501    # Hash of array
502    my %output;
503    my @order;
504
505    ## Join both
506    if( $outputfile ){
507        if( open DESTFILE , "+> $outputfile" ){
508            if( !open LOCALIZEPARTICLE ,  "< $localizeSDF" ) { print STDERR "ERROR: Can't open file $localizeSDF\n"; }
509            if( !open ALLPARTICLES_MERGED , "< $particleSDF_merged" ) { print STDERR "ERROR: Can't open file $particleSDF_merged\n"; }
510
511            # Insert localize
512            my $extract_date="";
513            while ( <LOCALIZEPARTICLE> ){
514                if( /$sdf_regex/ ){
515                    my $leftpart       = defined $2 ? $2 : '';
516                    my $lang           = defined $12 ? $12 : '';
517                    my $rightpart      = defined $13 ? $13 : '';
518                    my $timestamp      = defined $18 ? $18 : '';
519
520                    my $prj            = defined $3 ? $3 : '';
521                    my $file           = defined $4 ? $4 : '';
522                    my $type           = defined $6 ? $6 : '';
523                    my $gid            = defined $7 ? $7 : '';
524                    my $lid            = defined $8 ? $8 : '';
525                    #my $lang           = defined $12 ? $12 : '';
526                    my $plattform      = defined $10 ? $10 : '';
527                    my $helpid         = defined $9 ? $9 : '';
528
529
530                    if( $use_default_date )
531                    {
532                        $extract_date = "$default_date\n" ;
533                    }
534                    elsif( $extract_date eq "" ) {
535                        $extract_date = $timestamp ;
536                        $extract_date =~ tr/\r\n//d;
537                        $extract_date .= "\n";
538                    }
539
540                    if( $bAll ){ print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date ; }
541                    else {
542                        foreach my $sLang ( keys( %{ $langhash_ref } ) ){
543                            if( $sLang=~ /all/i )                       {
544                                push @{ $output{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } } ,  $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
545                                #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date;
546                            }
547                            #if( $sLang eq "de" && $lang eq "de" )       {
548                            #    push @{ $output{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } } ,  $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
549                                #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date;
550                                #}
551                            if( $sLang eq "en-US" && $lang eq "en-US" ) {
552                                push @order , $prj.$gid.$lid.$file.$type.$plattform.$helpid;
553                                if( !$bFakeEnglish ){ push @{ $output{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } } ,  $leftpart."\t".$lang."\t".$rightpart.$extract_date ; }
554                                #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date;
555                            }
556
557                        }
558                    }
559                }
560            }
561            # Insert particles
562            while ( <ALLPARTICLES_MERGED> ){
563                if( /$sdf_regex/ ){
564                    my $leftpart       = defined $2 ? $2 : '';
565                    my $prj            = defined $3 ? $3 : '';
566                    my $lang           = defined $12 ? $12 : '';
567                    my $rightpart      = defined $13 ? $13 : '';
568                    my $timestamp      = defined $18 ? $18 : '';
569
570                    #my $prj            = defined $3 ? $3 : '';
571                    my $file           = defined $4 ? $4 : '';
572                    my $type           = defined $6 ? $6 : '';
573                    my $gid            = defined $7 ? $7 : '';
574                    my $lid            = defined $8 ? $8 : '';
575                    #my $lang           = defined $12 ? $12 : '';
576                    my $plattform      = defined $10 ? $10 : '';
577                    my $helpid         = defined $9 ? $9 : '';
578
579
580                    if( $use_default_date )
581                    {
582                        $extract_date = "$default_date\n" ;
583                    }
584                    elsif( $extract_date eq "" )
585                    {
586                        $extract_date = $timestamp;
587                    }
588
589                    if( ! ( $prj =~ /binfilter/i ) ) {
590                        push @{ $output{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } } , $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
591                        #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
592                    }
593                 }
594            }
595
596            # Write!
597            foreach my $curkey ( @order ){
598                foreach my $curlist ( $output{ $curkey } ){
599                    foreach my $line ( @{$curlist} ){
600                        print DESTFILE $line;
601                    }
602                }
603            }
604
605        }else { print STDERR "Can't open $outputfile";}
606    }
607    close DESTFILE;
608    close LOCALIZEPARTICLE;
609    close ALLPARTICLES_MERGED;
610
611    #print STDOUT "DBG: \$localizeSDF $localizeSDF \$particleSDF_merged $particleSDF_merged\n";
612    unlink $localizeSDF , $particleSDF_merged ,  $my_localize_log;
613
614    #sort_outfile( $outputfile );
615    #remove_obsolete( $outputfile ) , if $bHasSourceLanguage ne "";
616    }
617
618#########################################################
619sub remove_obsolete{
620    my $outfile = shift;
621    my @lines;
622    my $enusleftpart;
623    my @good_lines;
624
625    print STDOUT "### Removing obsolete strings\n";
626
627    # Kick out all strings without en-US reference
628    if ( open ( SORTEDFILE , "< $outfile" ) ){
629        while( <SORTEDFILE> ){
630            if( /$sdf_regex/ ){
631                my $line           = defined $_ ? $_ : '';
632                my $language       = defined $12 ? $12 : '';
633                my $prj            = defined $3 ? $3 : '';
634                my $file           = defined $4 ? $4 : '';
635                my $type           = defined $6 ? $6 : '';
636                my $gid            = defined $7 ? $7 : '';
637                my $lid            = defined $8 ? $8 : '';
638                my $plattform      = defined $10 ? $10 : '';
639                my $helpid         = defined $9 ? $9 : '';
640
641                my $leftpart = $prj.$gid.$lid.$file.$type.$plattform.$helpid;
642
643                if( $language eq "en-US" ){                 # source string found, 1. entry
644                    $enusleftpart = $leftpart;
645                    push @good_lines , $line;
646                }else{
647                    if( !defined $enusleftpart or !defined $leftpart ){
648                        print STDERR "BADLINE: $line\n";
649                        print STDERR "\$enusleftpart = $enusleftpart\n";
650                        print STDERR "\$leftpart = $leftpart\n";
651                    }
652                    if( $enusleftpart eq $leftpart ){   # matching language
653                        push @good_lines , $line;
654                    }
655                    #else{
656                    #    print STDERR "OUT:  \$enusleftpart=$enusleftpart \$leftpart=$leftpart \$line=$line\n";
657                    #}
658                }
659            }
660        }
661        close SORTEDFILE;
662    } else { print STDERR "ERROR: Can't open file $outfile\n";}
663
664    # Write file
665    if ( open ( SORTEDFILE , "> $outfile" ) ){
666        foreach my $newline ( @good_lines ) {
667            print SORTEDFILE $newline;
668        }
669        close SORTEDFILE;
670    } else { print STDERR "ERROR: Can't open file $outfile\n";}
671
672}
673#########################################################
674sub sort_outfile{
675        my $outfile = shift;
676        print STDOUT "### Sorting ... $outfile ...";
677        my @lines;
678        my @sorted_lines;
679
680
681        #if ( open ( SORTEDFILE , "< $outputfile" ) ){
682        if ( open ( SORTEDFILE , "< $outfile" ) ){
683            my $line;
684            while ( <SORTEDFILE> ){
685                $line = $_;
686                if( $line =~ /^[^\#]/ ){
687                    push @lines , $line;
688                }
689            }
690            close SORTEDFILE;
691            @sorted_lines = sort {
692                my $xa_lang          = "";
693                my $xa_left_part    = "";
694                my $xa_right_part    = "";
695                my $xa_timestamp     = "";
696                my $xb_lang          = "";
697                my $xb_left_part    = "";
698                my $xb_right_part    = "";
699                my $xb_timestamp     = "";
700                my $xa               = "";
701                my $xb               = "";
702                my @alist;
703                my @blist;
704
705                if( $a=~ /$sdf_regex/ ){
706                    $xa_left_part       = defined $2 ? $2 : '';
707                    $xa_lang           = defined $12 ? $12 : '';
708                    $xa_right_part     = defined $13 ? $13 : '';
709                    $xa_left_part = remove_last_column( $xa_left_part );
710
711                }
712                if( $b=~ /$sdf_regex/ ){
713                    $xb_left_part       = defined $2 ? $2 : '';
714                    $xb_lang           = defined $12 ? $12 : '';
715                    $xb_right_part     = defined $13 ? $13 : '';
716                    $xb_left_part = remove_last_column( $xb_left_part );
717
718
719                }
720                if( (  $xa_left_part cmp $xb_left_part ) == 0 ){         # Left part equal
721                     if( ( $xa_lang cmp $xb_lang ) == 0 ){               # Lang equal
722                         return ( $xa_right_part cmp $xb_right_part );   # Right part compare
723                    }
724                    elsif( $xa_lang eq "en-US" ) { return -1; }        # en-US wins
725                    elsif( $xb_lang eq "en-US" ) { return 1;  }        # en-US wins
726                    else { return $xa_lang cmp $xb_lang; }             # lang compare
727                }
728                else {
729                    return $xa_left_part cmp $xb_left_part;        # Left part compare
730                }
731            } @lines;
732
733            if ( open ( SORTEDFILE , "> $outfile" ) ){
734                print SORTEDFILE get_license_header();
735                foreach my $newline ( @sorted_lines ) {
736                    print SORTEDFILE $newline;
737                    #print STDOUT $newline;
738                }
739            }
740            close SORTEDFILE;
741        } else { print STDERR "WARNING: Can't open file $outfile\n";}
742	print "done\n";
743
744}
745#########################################################
746sub remove_last_column{
747    my $string                  = shift;
748    my @alist = split ( "\t" , $string );
749    pop @alist;
750    return join( "\t" , @alist );
751}
752
753#########################################################
754sub rename_language{
755    my $fallbackhashhash_ref    = shift;
756    my $cur_fallback            = shift;
757    my $cur_lang                = shift;
758    my $line;
759
760    foreach my $key( keys ( %{ $fallbackhashhash_ref->{ $cur_fallback } } ) ){
761        $line = $fallbackhashhash_ref->{ $cur_fallback }{ $key };
762        if( $line =~ /$sdf_regex/ ){
763            my $leftpart       = defined $2 ? $2 : '';
764            my $lang           = defined $12 ? $12 : '';
765            my $rightpart      = defined $13 ? $13 : '';
766
767            $fallbackhashhash_ref->{ $cur_lang }{ $key } = $leftpart."\t".$cur_lang."\t".$rightpart;
768        }
769    }
770}
771
772############################################################
773sub remove_duplicates{
774    my $list_ref    = shift;
775    my %tmphash;
776    foreach my $key ( @{ $list_ref } ){ $tmphash{ $key } = '' ; }
777    @{$list_ref} = keys( %tmphash );
778}
779
780##############################################################
781sub fetch_fallback{
782    my $sdfparticleslist_ref   = shift;
783    my $localizeSDF            = shift;
784    my $langhash_ref           = shift;
785    my %fallbackhashhash;
786    my $cur_lang;
787    my @langlist;
788
789    foreach my $key ( keys ( %{ $langhash_ref } ) ){
790        $cur_lang = $langhash_ref->{ $key };
791        if ( $cur_lang ne "" ) {
792            push @langlist , $cur_lang;
793        }
794    }
795    remove_duplicates( \@langlist );
796    foreach  $cur_lang ( @langlist ){
797        if( $cur_lang eq "en-US" ){
798            read_fallbacks_from_source( $localizeSDF , $cur_lang , \%fallbackhashhash );
799        }
800    }
801
802    # remove de / en-US
803    my @tmplist;
804    foreach $cur_lang( @langlist ){
805        if(  $cur_lang ne "en-US" ){
806           push @tmplist , $cur_lang;
807
808        }
809    }
810    @langlist = @tmplist;
811    if ( $#langlist +1 ){
812        read_fallbacks_from_particles( $sdfparticleslist_ref , \@langlist , \%fallbackhashhash );
813
814    }
815    return (\%fallbackhashhash);
816}
817
818#########################################################
819sub write_file{
820
821    my $localizeFile = shift;
822    my $index_ref    = shift;
823
824    if( open DESTFILE , "+> $localizeFile" ){
825        foreach my $key( %{ $index_ref } ){
826            print DESTFILE ($index_ref->{ $key }, "\n" );
827        }
828        close DESTFILE;
829    }else {
830      print STDERR "Can't open/create '$localizeFile'";
831    }
832}
833
834#########################################################
835sub read_file{
836
837    my $sdffile         = shift;
838    my $langhash_ref    = shift;
839    my %block           = ();
840
841    open MYFILE , "< $sdffile"
842        or die "Can't open '$sdffile'\n";
843        while( <MYFILE>){
844          if( /$sdf_regex/ ){
845            my $line           = defined $_ ? $_ : '';
846            my $prj            = defined $3 ? $3 : '';
847            my $file           = defined $4 ? $4 : '';
848            my $type           = defined $6 ? $6 : '';
849            my $gid            = defined $7 ? $7 : '';
850            my $lid            = defined $8 ? $8 : '';
851            my $plattform      = defined $10 ? $10 : '';
852            my $lang           = defined $12 ? $12 : '';
853            my $helpid         = defined $9 ? $9 : '';
854
855            foreach my $isolang ( keys ( %{ $langhash_ref } ) ){
856                if( $isolang=~ /$lang/i || $isolang=~ /all/i ) { $block{$prj.$gid.$lid.$file.$type.$plattform.$helpid } =  $line ; }
857            }
858        }
859    }
860    return (\%block);
861}
862
863#########################################################
864sub read_fallbacks_from_particles{
865
866    my $sdfparticleslist_ref    = shift;
867    my $isolanglist_ref         = shift;
868    my $fallbackhashhash_ref    = shift;
869    my $block_ref;
870    foreach my $currentfile ( @{ $sdfparticleslist_ref } ){
871        if ( open MYFILE , "< $currentfile" ) {
872            while(<MYFILE>){
873                if( /$sdf_regex/ ){
874                    my $line           = defined $_ ? $_ : '';
875                    my $prj            = defined $3 ? $3 : '';
876                    my $file           = defined $4 ? $4 : '';
877                    my $type           = defined $6 ? $6 : '';
878                    my $gid            = defined $7 ? $7 : '';
879                    my $lid            = defined $8 ? $8 : '';
880                    my $lang           = defined $12 ? $12 : '';
881                    my $plattform      = defined $10 ? $10 : '';
882                    my $helpid         = defined $9 ? $9 : '';
883
884                    chomp( $line );
885
886                    foreach my $isolang ( @{$isolanglist_ref}  ){
887                        if( $isolang=~ /$lang/i ) {
888                            $fallbackhashhash_ref->{ $isolang }{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } =  $line ;
889                        }
890                    }
891                }
892            }
893       }else { print STDERR "WARNING: Can't open file $currentfile"; }
894    }
895}
896
897#########################################################
898sub read_fallbacks_from_source{
899
900    my $sdffile                 = shift;
901    my $isolang                 = shift;
902    my $fallbackhashhash_ref    = shift;
903    my $block_ref;
904    # read fallback for single file
905    open MYFILE , "< $sdffile"
906        or die "Can't open '$sdffile'\n";
907
908    while( <MYFILE>){
909          if( /$sdf_regex/ ){
910            my $line           = defined $_ ? $_ : '';
911            my $prj            = defined $3 ? $3 : '';
912            my $file           = defined $4 ? $4 : '';
913            my $type           = defined $6 ? $6 : '';
914            my $gid            = defined $7 ? $7 : '';
915            my $lid            = defined $8 ? $8 : '';
916            my $helpid         = defined $9 ? $9 : '';
917            my $lang           = defined $12 ? $12 : '';
918            my $plattform      = defined $10 ? $10 : '';
919
920            chomp( $line );
921            if( $isolang=~ /$lang/i ) { $fallbackhashhash_ref->{ $isolang }{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } =  $line ;
922            }
923        }
924    }
925}
926
927#########################################################
928sub parseLanguages{
929
930    my $bAll;
931    my $bUseLocalize;
932    my $bHasSourceLanguage="";
933    my $bFakeEnglish="";
934    my %langhash;
935    my $iso="";
936    my $fallback="";
937
938    #### -l all
939    if(   $languages=~ /all/ ){
940        $bAll = "TRUE";
941        $bHasSourceLanguage = "TRUE";
942    }
943    ### -l fr=de,de
944    elsif( $languages=~ /.*,.*/ ){
945        my @tmpstr =  split "," , $languages;
946        for my $lang ( @tmpstr ){
947            if( $lang=~ /([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)(=([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*))?/ ){
948                $iso        = $1;
949                $fallback   = $4;
950
951                if( ( $iso && $iso=~ /(en-US)/i )  || ( $fallback && $fallback=~ /(en-US)/i ) ) {
952                    $bUseLocalize = "TRUE";
953                }
954                if( ( $iso && $iso=~ /(en-US)/i ) ) {
955                    $bHasSourceLanguage = "TRUE";
956                }
957             if( $fallback ) { $langhash{ $iso } = $fallback;   }
958             else            { $langhash{ $iso } = "";          }
959            }
960        }
961    }
962    ### -l de
963    else{
964        if( $languages=~ /([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)(=([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*))?/ ){
965            $iso        = $1;
966            $fallback   = $4;
967
968            if( ( $iso && $iso=~ /(en-US)/i )  || ( $fallback && $fallback=~ /(en-US)/i ) ) {
969                $bUseLocalize = "TRUE";
970
971            }
972            if( ( $iso && $iso=~ /(en-US)/i )  ) {
973                $bHasSourceLanguage = "TRUE";
974            }
975
976             if( $fallback ) { $langhash{ $iso } = $fallback;   }
977             else            { $langhash{ $iso } = "";          }
978        }
979    }
980    # HACK en-US always needed!
981    if( !$bHasSourceLanguage ){
982        #$bHasSourceLanguage = "TRUE";
983        $bUseLocalize = "TRUE";
984        $bFakeEnglish = "TRUE";
985        $langhash{ "en-US" } = "";
986    }
987    return ( $bAll ,  $bUseLocalize , \%langhash , $bHasSourceLanguage, $bFakeEnglish);
988}
989
990#########################################################
991sub parse_options{
992
993    my $help;
994    my $merge;
995    my $extract;
996    my $success = GetOptions('f=s' => \$sdffile , 'l=s' => \$languages , 's=s' => \$srcpath ,  'h' => \$help , 'v' => \$bVerbose ,
997                             'm' => \$merge , 'e' => \$extract , 'x' => \$no_sort , 'd' => \$use_default_date );
998    $outputfile = $sdffile;
999
1000    #print STDOUT "DBG: lang = $languages\n";
1001    if( !$srcpath ){
1002        #$srcpath = "$ENV{SRC_ROOT}";
1003        if( !$srcpath ){
1004	        print STDERR "No path to the source root found!\n\n";
1005	        usage();
1006            exit(1);
1007        }
1008    }
1009    if( $help || !$success || $#ARGV > 1 || ( !$sdffile ) ){
1010        usage();
1011        exit(1);
1012    }
1013    if( $merge && $sdffile && ! ( -r $sdffile)){
1014        print STDERR "Can't open file '$sdffile'\n";
1015        exit(1);
1016    }
1017    if( !( $languages=~ /[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*(=[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)?(,[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*(=[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)?)*/ ) ){
1018        print STDERR "Please check the -l iso code\n";
1019        exit(1);
1020    }
1021    if( ( !$merge && !$extract ) || ( $merge && $extract ) ){ usage();exit( -1 );}
1022    if( $extract ){ $mode = "extract"; }
1023    else          { $mode = "merge";   }
1024}
1025
1026#########################################################
1027sub usage{
1028
1029    print STDERR "Usage: localize.pl\n";
1030    print STDERR "Split or collect SDF files\n";
1031    print STDERR "           merge: -m -f <sdffile>    -l l1[=f1][,l2[=f2]][...] [ -s <sourceroot> ]\n";
1032    print STDERR "         extract: -e -f <outputfile> -l <lang> [ -s <sourceroot> ] [-d]\n";
1033    print STDERR "Options:\n";
1034    print STDERR "    -h              help\n";
1035    print STDERR "    -m              Merge mode\n";
1036    print STDERR "    -e              Extract mode\n";
1037    print STDERR "    -f <sdffile>    To split a big SDF file into particles\n";
1038    print STDERR "       <outputfile> To collect and join all particles to one big file\n";
1039    print STDERR "    -s <sourceroot> Path to the modules, if no \$SRC_ROOT is set\n";
1040    print STDERR "    -l ( all | <isocode> | <isocode>=fallback ) comma seperated languages\n";
1041    print STDERR "    -d              Use default date in extracted sdf file\n";
1042    print STDERR "    -v              Verbose\n";
1043    print STDERR "\nExample:\n";
1044    print STDERR "\nlocalize -e -l en-US,pt-BR=en-US -f my.sdf\n( Extract en-US and pt-BR with en-US fallback )\n";
1045    print STDERR "\nlocalize -m -l cs -f my.sdf\n( Merge cs translation into the sourcecode ) \n";
1046}
1047
1048#            my $line           = defined $_ ? $_ : '';
1049#            my $leftpart       = defined $2 ? $2 : '';
1050#            my $prj            = defined $3 ? $3 : '';
1051#            my $file           = defined $4 ? $4 : '';
1052#            my $dummy          = defined $5 ? $5 : '';
1053#            my $type           = defined $6 ? $6 : '';
1054#            my $gid            = defined $7 ? $7 : '';
1055#            my $lid            = defined $8 ? $8 : '';
1056#            my $helpid         = defined $9 ? $9 : '';
1057#            my $plattform      = defined $10 ? $10 : '';
1058#            my $width          = defined $11 ? $11 : '';
1059#            my $lang           = defined $12 ? $12 : '';
1060#            my $rightpart      = defined $13 ? $13 : '';
1061#            my $text           = defined $14 ? $14 : '';
1062#            my $helptext       = defined $15 ? $15 : '';
1063#            my $quickhelptext  = defined $16 ? $16 : '';
1064#            my $title          = defined $17 ? $17 : '';
1065#            my $timestamp      = defined $18 ? $18 : '';
1066
1067