xref: /aoo41x/main/l10ntools/scripts/localize.pl (revision 8e9e5c11)
1:
2eval 'exec perl -wS $0 ${1+"$@"}'
3    if 0;
4
5
6#**************************************************************
7#
8#  Licensed to the Apache Software Foundation (ASF) under one
9#  or more contributor license agreements.  See the NOTICE file
10#  distributed with this work for additional information
11#  regarding copyright ownership.  The ASF licenses this file
12#  to you under the Apache License, Version 2.0 (the
13#  "License"); you may not use this file except in compliance
14#  with the License.  You may obtain a copy of the License at
15#
16#    http://www.apache.org/licenses/LICENSE-2.0
17#
18#  Unless required by applicable law or agreed to in writing,
19#  software distributed under the License is distributed on an
20#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
21#  KIND, either express or implied.  See the License for the
22#  specific language governing permissions and limitations
23#  under the License.
24#
25#**************************************************************
26
27
28
29use strict;
30use Getopt::Long;
31use IO::Handle;
32use File::Find;
33use File::Temp;
34use File::Path;
35use File::Copy;
36use File::Glob qw(:glob csh_glob);
37use Cwd;
38
39my $CVS_BINARY = "/usr/bin/cvs";
40# ver 1.1
41#
42#### module lookup
43#use lib ("$ENV{SOLARENV}/bin/modules", "$ENV{COMMON_ENV_TOOLS}/modules");
44
45#### module lookup
46# OOo conform
47my @lib_dirs;
48BEGIN {
49    if ( !defined($ENV{SOLARENV}) ) {
50        die "No environment found (environment variable SOLARENV is undefined)";
51    }
52    push(@lib_dirs, "$ENV{SOLARENV}/bin/modules");
53    push(@lib_dirs, "$ENV{COMMON_ENV_TOOLS}/modules") if defined($ENV{COMMON_ENV_TOOLS});
54}
55use lib (@lib_dirs);
56
57#### globals ####
58my $sdffile                 = '';
59my $no_sort                 = '';
60my $create_dirs             = '';
61my $multi_localize_files    = '';
62my $module_to_merge         = '';
63my $sort_sdf_before         = '';
64my $outputfile              = '';
65my $no_gsicheck             = '';
66my $mode                    = '';
67my $bVerbose                = "0";
68my $srcpath                 = '';
69my $languages;
70#my %sl_modules;     # Contains all modules where en-US and de is source language
71my $use_default_date = '0';
72my $force_ooo_module = '0';
73my %is_ooo_module;
74my %is_so_module;
75
76         #         (                           leftpart                                                     )            (           rightpart                    )
77         #            prj      file      dummy     type       gid       lid      helpid    pform     width      lang       text    helptext  qhelptext   title    timestamp
78my $sdf_regex  = "((([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*))\t([^\t]*)\t(([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t)([^\t]*))";
79my $file_types = "(src|hrc|xcs|xcu|lng|ulf|xrm|xhp|xcd|xgf|xxl|xrb)";
80# Always use this date to prevent cvs conflicts
81my $default_date = "2002-02-02 02:02:02";
82my @sdfparticles;
83
84#### main ####
85parse_options();
86
87my $binpath = '';
88if( defined $ENV{UPDMINOREXT} )
89{
90    $binpath = $ENV{SOLARVER}."/".$ENV{INPATH}."/bin".$ENV{UPDMINOREXT}."/" ;
91}
92else
93{
94    $binpath = $ENV{SOLARVER}."/".$ENV{INPATH}."/bin/" ;
95}
96
97#%sl_modules = fetch_sourcelanguage_dirlist();
98
99
100if   ( $mode eq "merge"    )    {
101    if ( ! $no_gsicheck ){
102        merge_gsicheck();
103    }
104    splitfile( $sdffile );
105    if ( ! $no_gsicheck ){
106        unlink $sdffile;             # remove temp file!
107    }
108}
109elsif( $mode eq "extract"  )    {
110    collectfiles( $outputfile );
111}
112else                            {
113    usage();
114}
115
116exit(0);
117
118#########################################################
119sub splitfile{
120
121    my $lastFile        = '';
122    my $currentFile     = '';
123    my $cur_sdffile     = '';
124    my $last_sdffile    = '';
125    my $delim;
126    my $badDelim;
127    my $start           = 'TRUE';
128    my %index  = ();
129    my %block;
130
131    STDOUT->autoflush( 1 );
132
133    #print STDOUT "Open File $sdffile\n";
134    open MYFILE , "< $sdffile"
135    or die "Can't open '$sdffile'\n";
136
137#    my %lang_hash;
138    my %string_hash_ooo;
139    my %string_hash_so;
140    my %so_modules;
141    $so_modules{ "extras_full" } = "TRUE";
142
143    while( <MYFILE>){
144         if( /$sdf_regex/ ){
145            my $line           = defined $_ ? $_ : '';
146            my $prj            = defined $3 ? $3 : '';
147            my $file           = defined $4 ? $4 : '';
148            my $type           = defined $6 ? $6 : '';
149            my $gid            = defined $7 ? $7 : '';
150            my $lid            = defined $8 ? $8 : '';
151            my $lang           = defined $12 ? $12 : '';
152            my $plattform      = defined $10 ? $10 : '';
153            my $helpid         = defined $9 ? $9 : '';
154            next if( $prj eq "binfilter" );     # Don't merge strings into binfilter module
155	        chomp( $line );
156
157            if( $force_ooo_module )
158            {
159                $string_hash_ooo { $lang }{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } = $line;
160            }
161            else
162            {
163                $string_hash_so{ $lang }{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } = $line;
164            }
165        }
166    }
167    close( MYFILE );
168
169    if( !defined $ENV{SOURCE_ROOT_DIR} ){
170        print "Error, no SOURCE_ROOT_DIR in env found.\n";
171        exit( -1 );
172    }
173    my $src_root = $ENV{SOURCE_ROOT_DIR};
174    my $ooo_src_root = $src_root."/l10n";
175    my $so_l10n_path  = $src_root."/sun/l10n_so/source";
176    my $ooo_l10n_path = $ooo_src_root."/l10n/source";
177
178    #print "$so_l10n_path\n";
179    #print "$ooo_l10n_path\n";
180
181    if( $force_ooo_module )
182    {
183        write_sdf( \%string_hash_ooo , $ooo_l10n_path );
184    }
185    else
186    {
187        write_sdf( \%string_hash_so , $so_l10n_path );
188    }
189}
190
191sub write_sdf
192{
193    my $string_hash         = shift;
194    my $l10n_file           = shift;
195
196    foreach my $lang( keys( %{ $string_hash } ) )
197    {
198        my @sdf_file;
199        next , if( $lang eq "en-US" );
200
201        mkdir $l10n_file."/$lang";
202        # mkdir!!!!
203        my $current_l10n_file = $l10n_file."/$lang/localize.sdf";
204        print "Writing '$current_l10n_file'\n";
205        if( open DESTFILE , "< $current_l10n_file" ){
206
207            while(<DESTFILE>){
208                if( /$sdf_regex/ ){
209                    my $line           = defined $_ ? $_ : '';
210                    my $prj            = defined $3 ? $3 : '';
211                    my $file           = defined $4 ? $4 : '';
212                    my $type           = defined $6 ? $6 : '';
213                    my $gid            = defined $7 ? $7 : '';
214                    my $lid            = defined $8 ? $8 : '';
215                    my $lang           = defined $12 ? $12 : '';
216                    my $plattform      = defined $10 ? $10 : '';
217                    my $helpid         = defined $9 ? $9 : '';
218
219                    chomp( $line );
220                    if ( defined $string_hash->{ $lang }{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } )
221                    {
222                        # Changed String!
223                        push @sdf_file , $string_hash->{ $lang }{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } ;
224                        $string_hash->{ $lang }{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } = undef;
225                    }
226                    else
227                    {
228                        # No new string
229                        push @sdf_file , $line;
230                    }
231                }
232            }
233        }
234        close( DESTFILE );
235        #Now just append the enw strings
236        #FIXME!!! Implement insertion in the correct order
237        foreach my $key ( keys ( %{ $string_hash->{ $lang } } ) )
238        {
239            push @sdf_file , $string_hash->{ $lang }{ $key } , if ( defined $string_hash->{ $lang }{ $key } );
240            #print "WARNING: Not defined = ".$string_hash->{ $lang }{ $key }."\n", if( ! defined  $string_hash->{ $lang }{ $key } );
241        }
242
243        # Write the new file
244        my ( $TMPFILE , $tmpfile ) = File::Temp::tempfile();
245        if( open DESTFILE , "+> $tmpfile " ){
246            print DESTFILE get_license_header();
247            foreach my $string( @sdf_file ){
248                print DESTFILE "$string\n";
249            }
250            close ( DESTFILE );
251            if( move( $current_l10n_file , $current_l10n_file.".backup" ) ){
252                if( copy( $tmpfile , $current_l10n_file ) ){
253                    unlink $l10n_file.".backup";
254                 } else { print STDERR "Can't open/create '$l10n_file', original file is renamed to $l10n_file.backup\n"; }
255            } else { print STDERR "Can't open/create '$l10n_file'\n"; }
256         }else{
257            print STDERR "WARNING: Can't open/create '$l10n_file'\n";
258         }
259         unlink $tmpfile;
260     }
261}
262
263#########################################################
264
265sub get_license_header{
266    return
267"#\n".
268"#    ####    ###     #   #   ###   #####    #####  ####   #####  #####  \n".
269"#    #   #  #   #    ##  #  #   #    #      #      #   #    #      #    \n".
270"#    #   #  #   #    # # #  #   #    #      ###    #   #    #      #    \n".
271"#    #   #  #   #    #  ##  #   #    #      #      #   #    #      #    \n".
272"#    ####    ###     #   #   ###     #      #####  ####   #####    #    \n".
273"#\n".
274"#    DO NOT EDIT! This file will be overwritten by localisation process\n".
275"#\n".
276"#**************************************************************\n".
277"#  \n".
278"#  Licensed to the Apache Software Foundation (ASF) under one\n".
279"#  or more contributor license agreements.  See the NOTICE file\n".
280"#  distributed with this work for additional information\n".
281"#  regarding copyright ownership.  The ASF licenses this file\n".
282"#  to you under the Apache License, Version 2.0 (the\n".
283"#  "License"); you may not use this file except in compliance\n".
284"#  with the License.  You may obtain a copy of the License at\n".
285"#  \n".
286"#    http://www.apache.org/licenses/LICENSE-2.0\n".
287"#  \n".
288"#  Unless required by applicable law or agreed to in writing,\n".
289"#  software distributed under the License is distributed on an\n".
290"#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n".
291"#  KIND, either express or implied.  See the License for the\n".
292"#  specific language governing permissions and limitations\n".
293"#  under the License.\n".
294"#  \n".
295"#**************************************************************\n";
296}
297######## Check input sdf file and use only the correct part
298sub merge_gsicheck{
299    my $command = '';
300    my ( $TMPHANDLE , $tmpfile ) = File::Temp::tempfile();
301    close ( $TMPHANDLE );
302
303    $command = "$ENV{WRAPCMD} " if( $ENV{WRAPCMD} );
304    $command .= "$ENV{SOLARVER}/$ENV{INPATH}/bin/gsicheck";
305
306    my $errfile = $sdffile.".err";
307    $command .= " -k -c -wcf $tmpfile -wef $errfile -l \"\" $sdffile";
308    #my $rc = system( $command );
309    my $output = `$command`;
310    my $rc = $? << 8;
311    if ( $output ne "" ){
312        print STDOUT "### gsicheck ###\n";
313        print STDOUT "### The file $errfile have been written containing the errors in your sdf file. Those lines will not be merged: ###\n\n";
314        print STDOUT "$output\n";
315        print STDOUT "################\n";
316
317    }else{
318        # Remove the 0 Byte file
319        unlink $errfile;
320    }
321    $sdffile = $tmpfile;
322}
323#########################################################
324# find search function
325sub wanted
326{
327    my $file = $File::Find::name;
328    if( -f $file && $file =~ /.*localize.sdf$/ && !( $file =~ /.*\.svn.*/ ) ) {
329        push   @sdfparticles , $file;
330        if( $bVerbose eq "1" ) { print STDOUT "$file\n"; }
331        else { print ".";  }
332    }
333}
334
335sub add_paths
336{
337    my $langhash_ref            = shift;
338    my $root_dir = $ENV{ SRC_ROOT };
339    my $ooo_l10n_dir = "$root_dir/l10n/source";
340    my $so_l10n_dir  = "$root_dir/l10n_so/source";
341
342    if( -e $ooo_l10n_dir )
343    {
344        foreach my $lang ( keys( %{ $langhash_ref } ) )
345        {
346            my $loc_file = "$ooo_l10n_dir/$lang/localize.sdf";
347            if( -e $loc_file )
348            {
349                push @sdfparticles , "$ooo_l10n_dir/$lang/localize.sdf";
350            }
351            else { print "WARNING: $loc_file not found ....\n"; }
352        }
353    }
354    else { die "ERROR: Can not find directory $ooo_l10n_dir!!!" }
355    if( -e $so_l10n_dir )
356    {
357        foreach my $lang ( keys( %{ $langhash_ref } ) )
358        {
359            my $loc_file = "$so_l10n_dir/$lang/localize.sdf";
360            if( -e $loc_file )
361            {
362                push @sdfparticles , "$ooo_l10n_dir/$lang/localize.sdf";
363            }
364            else { #print "WARNING: $loc_file not found ....\n";
365            }
366        }
367
368    }
369}
370sub collectfiles{
371    print STDOUT "### Localize\n";
372    my $localizehash_ref;
373    my ( $bAll , $bUseLocalize, $langhash_ref , $bHasSourceLanguage , $bFakeEnglish ) = parseLanguages();
374
375    # Enable autoflush on STDOUT
376    # $| = 1;
377    STDOUT->autoflush( 1 );
378
379    my $working_path = getcwd();
380    chdir $ENV{SOURCE_ROOT_DIR}, if defined $ENV{SOURCE_ROOT_DIR};
381    add_paths( $langhash_ref );
382
383    my ( $LOCALIZEPARTICLE , $localizeSDF ) = File::Temp::tempfile();
384    close( $LOCALIZEPARTICLE );
385
386    my ( $ALLPARTICLES_MERGED , $particleSDF_merged )     = File::Temp::tempfile();
387    close( $ALLPARTICLES_MERGED );
388    my ( $LOCALIZE_LOG , $my_localize_log ) = File::Temp::tempfile();
389    close( $LOCALIZE_LOG );
390
391    ## Get the localize en-US extract
392    if( $bAll || $bUseLocalize ){
393        print "### Fetching source language strings\n";
394        my $command = "";
395        my $args    = "";
396
397        if( $ENV{WRAPCMD} ){
398            $command = $ENV{WRAPCMD}.$binpath."localize_sl";
399        }else{
400            $command = $binpath."localize_sl";
401        }
402        print $command;
403        # -e
404        # if ( -x $command ){
405        if( $command ){
406            if( !$bVerbose  ){ $args .= " "; }
407            $args .= " -e -f $localizeSDF -l ";
408            my $bFlag="";
409            if( $bAll ) {$args .= " en-US";}
410            else{
411              my @list;
412              foreach my $isokey ( keys( %{ $langhash_ref } ) ){
413                push @list , $isokey;
414                if( $langhash_ref->{ $isokey } ne "" ){
415                    push @list , $langhash_ref->{ $isokey };
416                }
417              }
418              remove_duplicates( \@list );
419              foreach my $isokey ( @list ){
420                switch :{
421                       ( $isokey=~ /^en-US$/i  )
422                        && do{
423                                if( $bFlag eq "TRUE" ){ $args .= ",en-US"; }
424                                else {
425                                    $args .= "en-US";  $bFlag = "TRUE";
426                                 }
427                              };
428
429                    } #switch
430                } #foreach
431              } # if
432        } # if
433        if ( $bVerbose ) { print STDOUT $command.$args."\n"; }
434
435        my $rc = system( $command.$args );
436
437        if( $rc < 0 ){    print STDERR "ERROR: localize rc = $rc\n"; exit( -1 ); }
438        ( $localizehash_ref )  = read_file( $localizeSDF , $langhash_ref );
439
440    }
441    ## Get sdf particles
442#*****************
443    open ALLPARTICLES_MERGED , "+>> $particleSDF_merged"
444    or die "Can't open $particleSDF_merged";
445
446    ## Fill fackback hash
447    my( $fallbackhashhash_ref ) = fetch_fallback( \@sdfparticles , $localizeSDF ,  $langhash_ref );
448    my %block;
449    my $cur_fallback;
450    if( !$bAll) {
451        foreach my $cur_lang ( keys( %{ $langhash_ref } ) ){
452            #print STDOUT "DBG: G1 cur_lang=$cur_lang\n";
453            $cur_fallback = $langhash_ref->{ $cur_lang };
454            if( $cur_fallback ne "" ){
455                # Insert fallback strings
456                #print STDOUT "DBG: Renaming $cur_fallback to $cur_lang in fallbackhash\n";
457                rename_language(  $fallbackhashhash_ref ,  $cur_fallback , $cur_lang );
458            }
459            foreach my $currentfile ( @sdfparticles ){
460                if ( open MYFILE , "< $currentfile" ) {
461                    while(<MYFILE>){
462                        if( /$sdf_regex/ ){
463                            my $line           = defined $_ ? $_ : '';
464                            my $prj            = defined $3 ? $3 : '';
465                            my $file           = defined $4 ? $4 : '';
466                            my $type           = defined $6 ? $6 : '';
467                            my $gid            = defined $7 ? $7 : '';
468                            my $lid            = defined $8 ? $8 : '';
469                            my $lang           = defined $12 ? $12 : '';
470                            my $plattform      = defined $10 ? $10 : '';
471                            my $helpid         = defined $9 ? $9 : '';
472
473                            chomp( $line );
474
475                            if ( $lang eq $cur_lang ){
476                                # Overwrite fallback strings with collected strings
477                                #if( ( !has_two_sourcelanguages( $cur_lang) && $cur_lang eq "de" ) || $cur_lang ne "en-US" ){
478                                     $fallbackhashhash_ref->{ $cur_lang }{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } =  $line ;
479                                     #}
480
481                            }
482                        }
483                    }
484                }else { print STDERR "WARNING: Can't open file $currentfile"; }
485            }
486
487            foreach my $line ( keys( %{$fallbackhashhash_ref->{ $cur_lang } } )) {
488                if( #$cur_lang ne "de" &&
489                    $cur_lang ne "en-US" ){
490                    print ALLPARTICLES_MERGED ( $fallbackhashhash_ref->{ $cur_lang }{ $line }, "\n" );
491                }
492             }
493        }
494    } else {
495        foreach my $currentfile ( @sdfparticles ){
496            if ( open MYFILE , "< $currentfile" ) {
497                while( <MYFILE> ){
498                    print ALLPARTICLES_MERGED ( $_, "\n" );  # recheck de / en-US !
499                }
500            }
501            else { print STDERR "WARNING: Can't open file $currentfile"; }
502        }
503    }
504    close ALLPARTICLES_MERGED;
505
506    # Hash of array
507    my %output;
508    my @order;
509
510    ## Join both
511    if( $outputfile ){
512        if( open DESTFILE , "+> $outputfile" ){
513            if( !open LOCALIZEPARTICLE ,  "< $localizeSDF" ) { print STDERR "ERROR: Can't open file $localizeSDF\n"; }
514            if( !open ALLPARTICLES_MERGED , "< $particleSDF_merged" ) { print STDERR "ERROR: Can't open file $particleSDF_merged\n"; }
515
516            # Insert localize
517            my $extract_date="";
518            while ( <LOCALIZEPARTICLE> ){
519                if( /$sdf_regex/ ){
520                    my $leftpart       = defined $2 ? $2 : '';
521                    my $lang           = defined $12 ? $12 : '';
522                    my $rightpart      = defined $13 ? $13 : '';
523                    my $timestamp      = defined $18 ? $18 : '';
524
525                    my $prj            = defined $3 ? $3 : '';
526                    my $file           = defined $4 ? $4 : '';
527                    my $type           = defined $6 ? $6 : '';
528                    my $gid            = defined $7 ? $7 : '';
529                    my $lid            = defined $8 ? $8 : '';
530                    #my $lang           = defined $12 ? $12 : '';
531                    my $plattform      = defined $10 ? $10 : '';
532                    my $helpid         = defined $9 ? $9 : '';
533
534
535                    if( $use_default_date )
536                    {
537                        $extract_date = "$default_date\n" ;
538                    }
539                    elsif( $extract_date eq "" ) {
540                        $extract_date = $timestamp ;
541                        $extract_date =~ tr/\r\n//d;
542                        $extract_date .= "\n";
543                    }
544
545                    if( $bAll ){ print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date ; }
546                    else {
547                        foreach my $sLang ( keys( %{ $langhash_ref } ) ){
548                            if( $sLang=~ /all/i )                       {
549                                push @{ $output{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } } ,  $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
550                                #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date;
551                            }
552                            #if( $sLang eq "de" && $lang eq "de" )       {
553                            #    push @{ $output{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } } ,  $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
554                                #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date;
555                                #}
556                            if( $sLang eq "en-US" && $lang eq "en-US" ) {
557                                push @order , $prj.$gid.$lid.$file.$type.$plattform.$helpid;
558                                if( !$bFakeEnglish ){ push @{ $output{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } } ,  $leftpart."\t".$lang."\t".$rightpart.$extract_date ; }
559                                #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date;
560                            }
561
562                        }
563                    }
564                }
565            }
566            # Insert particles
567            while ( <ALLPARTICLES_MERGED> ){
568                if( /$sdf_regex/ ){
569                    my $leftpart       = defined $2 ? $2 : '';
570                    my $prj            = defined $3 ? $3 : '';
571                    my $lang           = defined $12 ? $12 : '';
572                    my $rightpart      = defined $13 ? $13 : '';
573                    my $timestamp      = defined $18 ? $18 : '';
574
575                    #my $prj            = defined $3 ? $3 : '';
576                    my $file           = defined $4 ? $4 : '';
577                    my $type           = defined $6 ? $6 : '';
578                    my $gid            = defined $7 ? $7 : '';
579                    my $lid            = defined $8 ? $8 : '';
580                    #my $lang           = defined $12 ? $12 : '';
581                    my $plattform      = defined $10 ? $10 : '';
582                    my $helpid         = defined $9 ? $9 : '';
583
584
585                    if( $use_default_date )
586                    {
587                        $extract_date = "$default_date\n" ;
588                    }
589                    elsif( $extract_date eq "" )
590                    {
591                        $extract_date = $timestamp;
592                    }
593
594                    if( ! ( $prj =~ /binfilter/i ) ) {
595                        push @{ $output{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } } , $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
596                        #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
597                    }
598                 }
599            }
600
601            # Write!
602            foreach my $curkey ( @order ){
603                foreach my $curlist ( $output{ $curkey } ){
604                    foreach my $line ( @{$curlist} ){
605                        print DESTFILE $line;
606                    }
607                }
608            }
609
610        }else { print STDERR "Can't open $outputfile";}
611    }
612    close DESTFILE;
613    close LOCALIZEPARTICLE;
614    close ALLPARTICLES_MERGED;
615    chdir $working_path;
616
617    #print STDOUT "DBG: \$localizeSDF $localizeSDF \$particleSDF_merged $particleSDF_merged\n";
618    unlink $localizeSDF , $particleSDF_merged ,  $my_localize_log;
619
620    #sort_outfile( $outputfile );
621    #remove_obsolete( $outputfile ) , if $bHasSourceLanguage ne "";
622    }
623
624#########################################################
625sub remove_obsolete{
626    my $outfile = shift;
627    my @lines;
628    my $enusleftpart;
629    my @good_lines;
630
631    print STDOUT "### Removing obsolete strings\n";
632
633    # Kick out all strings without en-US reference
634    if ( open ( SORTEDFILE , "< $outfile" ) ){
635        while( <SORTEDFILE> ){
636            if( /$sdf_regex/ ){
637                my $line           = defined $_ ? $_ : '';
638                my $language       = defined $12 ? $12 : '';
639                my $prj            = defined $3 ? $3 : '';
640                my $file           = defined $4 ? $4 : '';
641                my $type           = defined $6 ? $6 : '';
642                my $gid            = defined $7 ? $7 : '';
643                my $lid            = defined $8 ? $8 : '';
644                my $plattform      = defined $10 ? $10 : '';
645                my $helpid         = defined $9 ? $9 : '';
646
647                my $leftpart = $prj.$gid.$lid.$file.$type.$plattform.$helpid;
648
649                if( $language eq "en-US" ){                 # source string found, 1. entry
650                    $enusleftpart = $leftpart;
651                    push @good_lines , $line;
652                }else{
653                    if( !defined $enusleftpart or !defined $leftpart ){
654                        print STDERR "BADLINE: $line\n";
655                        print STDERR "\$enusleftpart = $enusleftpart\n";
656                        print STDERR "\$leftpart = $leftpart\n";
657                    }
658                    if( $enusleftpart eq $leftpart ){   # matching language
659                        push @good_lines , $line;
660                    }
661                    #else{
662                    #    print STDERR "OUT:  \$enusleftpart=$enusleftpart \$leftpart=$leftpart \$line=$line\n";
663                    #}
664                }
665            }
666        }
667        close SORTEDFILE;
668    } else { print STDERR "ERROR: Can't open file $outfile\n";}
669
670    # Write file
671    if ( open ( SORTEDFILE , "> $outfile" ) ){
672        foreach my $newline ( @good_lines ) {
673            print SORTEDFILE $newline;
674        }
675        close SORTEDFILE;
676    } else { print STDERR "ERROR: Can't open file $outfile\n";}
677
678}
679#########################################################
680sub sort_outfile{
681        my $outfile = shift;
682        print STDOUT "### Sorting ... $outfile ...";
683        my @lines;
684        my @sorted_lines;
685
686
687        #if ( open ( SORTEDFILE , "< $outputfile" ) ){
688        if ( open ( SORTEDFILE , "< $outfile" ) ){
689            my $line;
690            while ( <SORTEDFILE> ){
691                $line = $_;
692                if( $line =~ /^[^\#]/ ){
693                    push @lines , $line;
694                }
695            }
696            close SORTEDFILE;
697            @sorted_lines = sort {
698                my $xa_lang          = "";
699                my $xa_left_part    = "";
700                my $xa_right_part    = "";
701                my $xa_timestamp     = "";
702                my $xb_lang          = "";
703                my $xb_left_part    = "";
704                my $xb_right_part    = "";
705                my $xb_timestamp     = "";
706                my $xa               = "";
707                my $xb               = "";
708                my @alist;
709                my @blist;
710
711                if( $a=~ /$sdf_regex/ ){
712                    $xa_left_part       = defined $2 ? $2 : '';
713                    $xa_lang           = defined $12 ? $12 : '';
714                    $xa_right_part     = defined $13 ? $13 : '';
715                    $xa_left_part = remove_last_column( $xa_left_part );
716
717                }
718                if( $b=~ /$sdf_regex/ ){
719                    $xb_left_part       = defined $2 ? $2 : '';
720                    $xb_lang           = defined $12 ? $12 : '';
721                    $xb_right_part     = defined $13 ? $13 : '';
722                    $xb_left_part = remove_last_column( $xb_left_part );
723
724
725                }
726                if( (  $xa_left_part cmp $xb_left_part ) == 0 ){         # Left part equal
727                     if( ( $xa_lang cmp $xb_lang ) == 0 ){               # Lang equal
728                         return ( $xa_right_part cmp $xb_right_part );   # Right part compare
729                    }
730                    elsif( $xa_lang eq "en-US" ) { return -1; }        # en-US wins
731                    elsif( $xb_lang eq "en-US" ) { return 1;  }        # en-US wins
732                    else { return $xa_lang cmp $xb_lang; }             # lang compare
733                }
734                else {
735                    return $xa_left_part cmp $xb_left_part;        # Left part compare
736                }
737            } @lines;
738
739            if ( open ( SORTEDFILE , "> $outfile" ) ){
740                print SORTEDFILE get_license_header();
741                foreach my $newline ( @sorted_lines ) {
742                    print SORTEDFILE $newline;
743                    #print STDOUT $newline;
744                }
745            }
746            close SORTEDFILE;
747        } else { print STDERR "WARNING: Can't open file $outfile\n";}
748	print "done\n";
749
750}
751#########################################################
752sub remove_last_column{
753    my $string                  = shift;
754    my @alist = split ( "\t" , $string );
755    pop @alist;
756    return join( "\t" , @alist );
757}
758
759#########################################################
760sub rename_language{
761    my $fallbackhashhash_ref    = shift;
762    my $cur_fallback            = shift;
763    my $cur_lang                = shift;
764    my $line;
765
766    foreach my $key( keys ( %{ $fallbackhashhash_ref->{ $cur_fallback } } ) ){
767        $line = $fallbackhashhash_ref->{ $cur_fallback }{ $key };
768        if( $line =~ /$sdf_regex/ ){
769            my $leftpart       = defined $2 ? $2 : '';
770            my $lang           = defined $12 ? $12 : '';
771            my $rightpart      = defined $13 ? $13 : '';
772
773            $fallbackhashhash_ref->{ $cur_lang }{ $key } = $leftpart."\t".$cur_lang."\t".$rightpart;
774        }
775    }
776}
777
778############################################################
779sub remove_duplicates{
780    my $list_ref    = shift;
781    my %tmphash;
782    foreach my $key ( @{ $list_ref } ){ $tmphash{ $key } = '' ; }
783    @{$list_ref} = keys( %tmphash );
784}
785
786##############################################################
787sub fetch_fallback{
788    my $sdfparticleslist_ref   = shift;
789    my $localizeSDF            = shift;
790    my $langhash_ref           = shift;
791    my %fallbackhashhash;
792    my $cur_lang;
793    my @langlist;
794
795    foreach my $key ( keys ( %{ $langhash_ref } ) ){
796        $cur_lang = $langhash_ref->{ $key };
797        if ( $cur_lang ne "" ) {
798            push @langlist , $cur_lang;
799        }
800    }
801    remove_duplicates( \@langlist );
802    foreach  $cur_lang ( @langlist ){
803        if( $cur_lang eq "en-US" ){
804            read_fallbacks_from_source( $localizeSDF , $cur_lang , \%fallbackhashhash );
805        }
806    }
807
808    # remove de / en-US
809    my @tmplist;
810    foreach $cur_lang( @langlist ){
811        if(  $cur_lang ne "en-US" ){
812           push @tmplist , $cur_lang;
813
814        }
815    }
816    @langlist = @tmplist;
817    if ( $#langlist +1 ){
818        read_fallbacks_from_particles( $sdfparticleslist_ref , \@langlist , \%fallbackhashhash );
819
820    }
821    return (\%fallbackhashhash);
822}
823
824#########################################################
825sub write_file{
826
827    my $localizeFile = shift;
828    my $index_ref    = shift;
829
830    if( open DESTFILE , "+> $localizeFile" ){
831        foreach my $key( %{ $index_ref } ){
832            print DESTFILE ($index_ref->{ $key }, "\n" );
833        }
834        close DESTFILE;
835    }else {
836      print STDERR "Can't open/create '$localizeFile'";
837    }
838}
839
840#########################################################
841sub read_file{
842
843    my $sdffile         = shift;
844    my $langhash_ref    = shift;
845    my %block           = ();
846
847    open MYFILE , "< $sdffile"
848        or die "Can't open '$sdffile'\n";
849        while( <MYFILE>){
850          if( /$sdf_regex/ ){
851            my $line           = defined $_ ? $_ : '';
852            my $prj            = defined $3 ? $3 : '';
853            my $file           = defined $4 ? $4 : '';
854            my $type           = defined $6 ? $6 : '';
855            my $gid            = defined $7 ? $7 : '';
856            my $lid            = defined $8 ? $8 : '';
857            my $plattform      = defined $10 ? $10 : '';
858            my $lang           = defined $12 ? $12 : '';
859            my $helpid         = defined $9 ? $9 : '';
860
861            foreach my $isolang ( keys ( %{ $langhash_ref } ) ){
862                if( $isolang=~ /$lang/i || $isolang=~ /all/i ) { $block{$prj.$gid.$lid.$file.$type.$plattform.$helpid } =  $line ; }
863            }
864        }
865    }
866    return (\%block);
867}
868
869#########################################################
870sub read_fallbacks_from_particles{
871
872    my $sdfparticleslist_ref    = shift;
873    my $isolanglist_ref         = shift;
874    my $fallbackhashhash_ref    = shift;
875    my $block_ref;
876    foreach my $currentfile ( @{ $sdfparticleslist_ref } ){
877        if ( open MYFILE , "< $currentfile" ) {
878            while(<MYFILE>){
879                if( /$sdf_regex/ ){
880                    my $line           = defined $_ ? $_ : '';
881                    my $prj            = defined $3 ? $3 : '';
882                    my $file           = defined $4 ? $4 : '';
883                    my $type           = defined $6 ? $6 : '';
884                    my $gid            = defined $7 ? $7 : '';
885                    my $lid            = defined $8 ? $8 : '';
886                    my $lang           = defined $12 ? $12 : '';
887                    my $plattform      = defined $10 ? $10 : '';
888                    my $helpid         = defined $9 ? $9 : '';
889
890                    chomp( $line );
891
892                    foreach my $isolang ( @{$isolanglist_ref}  ){
893                        if( $isolang=~ /$lang/i ) {
894                            $fallbackhashhash_ref->{ $isolang }{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } =  $line ;
895                        }
896                    }
897                }
898            }
899       }else { print STDERR "WARNING: Can't open file $currentfile"; }
900    }
901}
902
903#########################################################
904sub read_fallbacks_from_source{
905
906    my $sdffile                 = shift;
907    my $isolang                 = shift;
908    my $fallbackhashhash_ref    = shift;
909    my $block_ref;
910    # read fallback for single file
911    open MYFILE , "< $sdffile"
912        or die "Can't open '$sdffile'\n";
913
914    while( <MYFILE>){
915          if( /$sdf_regex/ ){
916            my $line           = defined $_ ? $_ : '';
917            my $prj            = defined $3 ? $3 : '';
918            my $file           = defined $4 ? $4 : '';
919            my $type           = defined $6 ? $6 : '';
920            my $gid            = defined $7 ? $7 : '';
921            my $lid            = defined $8 ? $8 : '';
922            my $helpid         = defined $9 ? $9 : '';
923            my $lang           = defined $12 ? $12 : '';
924            my $plattform      = defined $10 ? $10 : '';
925
926            chomp( $line );
927            if( $isolang=~ /$lang/i ) { $fallbackhashhash_ref->{ $isolang }{ $prj.$gid.$lid.$file.$type.$plattform.$helpid } =  $line ;
928            }
929        }
930    }
931}
932
933#########################################################
934sub parseLanguages{
935
936    my $bAll;
937    my $bUseLocalize;
938    my $bHasSourceLanguage="";
939    my $bFakeEnglish="";
940    my %langhash;
941    my $iso="";
942    my $fallback="";
943
944    #### -l all
945    if(   $languages=~ /all/ ){
946        $bAll = "TRUE";
947        $bHasSourceLanguage = "TRUE";
948    }
949    ### -l fr=de,de
950    elsif( $languages=~ /.*,.*/ ){
951        my @tmpstr =  split "," , $languages;
952        for my $lang ( @tmpstr ){
953            if( $lang=~ /([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)(=([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*))?/ ){
954                $iso        = $1;
955                $fallback   = $4;
956
957                if( ( $iso && $iso=~ /(en-US)/i )  || ( $fallback && $fallback=~ /(en-US)/i ) ) {
958                    $bUseLocalize = "TRUE";
959                }
960                if( ( $iso && $iso=~ /(en-US)/i ) ) {
961                    $bHasSourceLanguage = "TRUE";
962                }
963             if( $fallback ) { $langhash{ $iso } = $fallback;   }
964             else            { $langhash{ $iso } = "";          }
965            }
966        }
967    }
968    ### -l de
969    else{
970        if( $languages=~ /([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)(=([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*))?/ ){
971            $iso        = $1;
972            $fallback   = $4;
973
974            if( ( $iso && $iso=~ /(en-US)/i )  || ( $fallback && $fallback=~ /(en-US)/i ) ) {
975                $bUseLocalize = "TRUE";
976
977            }
978            if( ( $iso && $iso=~ /(en-US)/i )  ) {
979                $bHasSourceLanguage = "TRUE";
980            }
981
982             if( $fallback ) { $langhash{ $iso } = $fallback;   }
983             else            { $langhash{ $iso } = "";          }
984        }
985    }
986    # HACK en-US always needed!
987    if( !$bHasSourceLanguage ){
988        #$bHasSourceLanguage = "TRUE";
989        $bUseLocalize = "TRUE";
990        $bFakeEnglish = "TRUE";
991        $langhash{ "en-US" } = "";
992    }
993    return ( $bAll ,  $bUseLocalize , \%langhash , $bHasSourceLanguage, $bFakeEnglish);
994}
995
996#########################################################
997sub parse_options{
998
999    my $help;
1000    my $merge;
1001    my $extract;
1002    my $success = GetOptions('f=s' => \$sdffile , 'l=s' => \$languages , 's=s' => \$srcpath ,  'h' => \$help , 'v' => \$bVerbose ,
1003                             'm' => \$merge , 'e' => \$extract , 'x' => \$no_sort , 'd' => \$use_default_date , 'c' => \$create_dirs ,
1004                             'n' => \$no_gsicheck , 'o' => \$force_ooo_module );
1005    $outputfile = $sdffile;
1006
1007    #print STDOUT "DBG: lang = $languages\n";
1008    if( !$srcpath ){
1009        $srcpath = "$ENV{SRC_ROOT}";
1010        if( !$srcpath ){
1011	        print STDERR "No path to the source root found!\n\n";
1012	        usage();
1013            exit(1);
1014        }
1015    }
1016    if( $help ){
1017        usage();
1018        exit(0);
1019    }
1020    if( !$success || $#ARGV > 1 || ( !$sdffile ) ){
1021        usage();
1022        exit(1);
1023    }
1024    if( $merge && $sdffile && ! ( -r $sdffile)){
1025        print STDERR "Can't open file '$sdffile'\n";
1026        exit(1);
1027    }
1028    if( !( $languages=~ /[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*(=[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)?(,[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*(=[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)?)*/ ) ){
1029        print STDERR "Please check the -l iso code\n";
1030        exit(1);
1031    }
1032    if( ( !$merge && !$extract ) || ( $merge && $extract ) ){ usage();exit( -1 );}
1033    if( $extract ){ $mode = "extract"; }
1034    else          { $mode = "merge";   }
1035}
1036#my $multi_localize_files    = ''; h
1037#my $module_to_merge         = ''; i
1038#my $sort_sdf_before         = ''; g
1039
1040#########################################################
1041sub usage{
1042
1043    print STDERR "Usage: localize.pl\n";
1044    print STDERR "Split or collect SDF files\n";
1045    print STDERR "           merge: -m -f <sdffile>    -l l1[=f1][,l2[=f2]][...] [ -s <sourceroot> ] [ -c ]\n";
1046    print STDERR "         extract: -e -f <outputfile> -l <lang> [ -s <sourceroot> ] [-d]\n";
1047    print STDERR "Options:\n";
1048    print STDERR "    -h              help\n";
1049    print STDERR "    -m              Merge mode\n";
1050    print STDERR "    -e              Extract mode\n";
1051    print STDERR "    -f <sdffile>    To split a big SDF file into particles\n";
1052    print STDERR "       <outputfile> To collect and join all particles to one big file\n";
1053    print STDERR "    -s <sourceroot> Path to the modules, if no \$SRC_ROOT is set\n";
1054    print STDERR "    -l ( all | <isocode> | <isocode>=fallback ) comma seperated languages\n";
1055    print STDERR "    -d              Use default date in extracted sdf file\n";
1056    print STDERR "    -c              Create needed directories\n";
1057    print STDERR "    -g              Sort sdf file before mergeing\n";
1058    print STDERR "    -h              File with localize.sdf's\n!";
1059    print STDERR "    -n              No gsicheck\n";
1060    print STDERR "    -i              Module to merge\n";
1061    print STDERR "    -o              force using ooo localization from the l10n module instead of l10n_so; \n";
1062    print STDERR "                    useful if the type can't be detected by the .svn tags; \n";
1063    print STDERR "    -v              Verbose\n";
1064    print STDERR "\nExample:\n";
1065    print STDERR "\nlocalize -e -l en-US,pt-BR=en-US -f my.sdf\n( Extract en-US and pt-BR with en-US fallback )\n";
1066    print STDERR "\nlocalize -m -l cs -f my.sdf\n( Merge cs translation into the sourcecode ) \n";
1067}
1068
1069