1cdf0e10cSrcweir:
2cdf0e10cSrcweireval 'exec perl -wS $0 ${1+"$@"}'
3cdf0e10cSrcweir    if 0;
4cdf0e10cSrcweir
5cdf0e10cSrcweir
67e90fac2SAndrew Rist#**************************************************************
77e90fac2SAndrew Rist#
87e90fac2SAndrew Rist#  Licensed to the Apache Software Foundation (ASF) under one
97e90fac2SAndrew Rist#  or more contributor license agreements.  See the NOTICE file
107e90fac2SAndrew Rist#  distributed with this work for additional information
117e90fac2SAndrew Rist#  regarding copyright ownership.  The ASF licenses this file
127e90fac2SAndrew Rist#  to you under the Apache License, Version 2.0 (the
137e90fac2SAndrew Rist#  "License"); you may not use this file except in compliance
147e90fac2SAndrew Rist#  with the License.  You may obtain a copy of the License at
157e90fac2SAndrew Rist#
167e90fac2SAndrew Rist#    http://www.apache.org/licenses/LICENSE-2.0
177e90fac2SAndrew Rist#
187e90fac2SAndrew Rist#  Unless required by applicable law or agreed to in writing,
197e90fac2SAndrew Rist#  software distributed under the License is distributed on an
207e90fac2SAndrew Rist#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
217e90fac2SAndrew Rist#  KIND, either express or implied.  See the License for the
227e90fac2SAndrew Rist#  specific language governing permissions and limitations
237e90fac2SAndrew Rist#  under the License.
247e90fac2SAndrew Rist#
257e90fac2SAndrew Rist#**************************************************************
267e90fac2SAndrew Rist
277e90fac2SAndrew Rist
28cdf0e10cSrcweir
29cdf0e10cSrcweiruse strict;
30cdf0e10cSrcweiruse Getopt::Long;
31cdf0e10cSrcweiruse IO::Handle;
32cdf0e10cSrcweiruse File::Find;
33cdf0e10cSrcweiruse File::Temp;
34cdf0e10cSrcweiruse File::Copy;
35cdf0e10cSrcweiruse File::Glob qw(:glob csh_glob);
36cdf0e10cSrcweiruse Cwd;
37cdf0e10cSrcweir
38cdf0e10cSrcweir# ver 1.1
39cdf0e10cSrcweir#
40cdf0e10cSrcweir#### module lookup
41cdf0e10cSrcweir#use lib ("$ENV{SOLARENV}/bin/modules", "$ENV{COMMON_ENV_TOOLS}/modules");
42cdf0e10cSrcweir
43cdf0e10cSrcweir#### module lookup
44cdf0e10cSrcweir# OOo conform
45cdf0e10cSrcweirmy @lib_dirs;
46cdf0e10cSrcweirBEGIN {
47cdf0e10cSrcweir    if ( !defined($ENV{SOLARENV}) ) {
48cdf0e10cSrcweir        die "No environment found (environment variable SOLARENV is undefined)";
49cdf0e10cSrcweir    }
50cdf0e10cSrcweir    push(@lib_dirs, "$ENV{SOLARENV}/bin/modules");
51cdf0e10cSrcweir    push(@lib_dirs, "$ENV{COMMON_ENV_TOOLS}/modules") if defined($ENV{COMMON_ENV_TOOLS});
52cdf0e10cSrcweir}
53cdf0e10cSrcweiruse lib (@lib_dirs);
54cdf0e10cSrcweir
55cdf0e10cSrcweir#### globals ####
56cdf0e10cSrcweirmy $sdffile = '';
57cdf0e10cSrcweirmy $no_sort = '';
58cdf0e10cSrcweirmy $outputfile = '';
59cdf0e10cSrcweirmy $mode = '';
60cdf0e10cSrcweirmy $bVerbose="0";
61cdf0e10cSrcweirmy $srcpath = '';
62cdf0e10cSrcweirmy $languages;
63cdf0e10cSrcweir#my %sl_modules;     # Contains all modules where en-US and de is source language
64cdf0e10cSrcweirmy $use_default_date = '0';
65cdf0e10cSrcweir
66cdf0e10cSrcweir         #         (                           leftpart                                                     )            (           rightpart                    )
67cdf0e10cSrcweir         #            prj      file      dummy     type       gid       lid      helpid    pform     width      lang       text    helptext  qhelptext   title    timestamp
68cdf0e10cSrcweirmy $sdf_regex  = "((([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*))\t([^\t]*)\t(([^\t]*)\t([^\t]*)\t([^\t]*)\t([^\t]*)\t)([^\t]*))";
69cdf0e10cSrcweirmy $file_types = "(src|hrc|xcs|xcu|lng|ulf|xrm|xhp|xcd|xgf|xxl|xrb)";
70cdf0e10cSrcweir# Always use this date to prevent cvs conflicts
71cdf0e10cSrcweirmy $default_date = "2002-02-02 02:02:02";
72cdf0e10cSrcweir
73cdf0e10cSrcweir#### main ####
74cdf0e10cSrcweirparse_options();
75cdf0e10cSrcweir
76cdf0e10cSrcweir#%sl_modules = fetch_sourcelanguage_dirlist();
77cdf0e10cSrcweir
78cdf0e10cSrcweir
79cdf0e10cSrcweirif   ( $mode eq "merge"    )    {
80cdf0e10cSrcweir    merge_gsicheck();
81cdf0e10cSrcweir    splitfile( $sdffile );
82cdf0e10cSrcweir    unlink $sdffile;             # remove temp file!
83cdf0e10cSrcweir}
84cdf0e10cSrcweirelsif( $mode eq "extract"  )    {
85cdf0e10cSrcweir    collectfiles( $outputfile );
86cdf0e10cSrcweir}
87cdf0e10cSrcweirelse                            {
88cdf0e10cSrcweir    usage();
89cdf0e10cSrcweir}
90cdf0e10cSrcweir
91cdf0e10cSrcweirexit(0);
92cdf0e10cSrcweir
93cdf0e10cSrcweir#########################################################
94cdf0e10cSrcweirsub splitfile{
95cdf0e10cSrcweir
96cdf0e10cSrcweir    my $lastFile        = '';
97cdf0e10cSrcweir    my $currentFile     = '';
98cdf0e10cSrcweir    my $cur_sdffile     = '';
99cdf0e10cSrcweir    my $last_sdffile    = '';
100cdf0e10cSrcweir    my $delim;
101cdf0e10cSrcweir    my $badDelim;
102cdf0e10cSrcweir    my $start           = 'TRUE';
103cdf0e10cSrcweir    my %index  = ();
104cdf0e10cSrcweir    my %block;
105cdf0e10cSrcweir
106cdf0e10cSrcweir    STDOUT->autoflush( 1 );
107cdf0e10cSrcweir
108cdf0e10cSrcweir    #print STDOUT "Open File $sdffile\n";
109cdf0e10cSrcweir    open MYFILE , "< $sdffile"
110cdf0e10cSrcweir    or die "Can't open '$sdffile'\n";
111cdf0e10cSrcweir
112cdf0e10cSrcweir    while( <MYFILE>){
113cdf0e10cSrcweir         if( /$sdf_regex/ ){
114cdf0e10cSrcweir            my $line           = defined $_ ? $_ : '';
115cdf0e10cSrcweir            my $prj            = defined $3 ? $3 : '';
116cdf0e10cSrcweir            my $file           = defined $4 ? $4 : '';
117cdf0e10cSrcweir            my $type           = defined $6 ? $6 : '';
118cdf0e10cSrcweir            my $gid            = defined $7 ? $7 : '';
119cdf0e10cSrcweir            my $lid            = defined $8 ? $8 : '';
120cdf0e10cSrcweir            my $lang           = defined $12 ? $12 : '';
1216d678023Smseidel            my $platform       = defined $10 ? $10 : '';
122cdf0e10cSrcweir            my $helpid         = defined $9 ? $9 : '';
123cdf0e10cSrcweir
124cdf0e10cSrcweir            next if( $prj eq "binfilter" );     # Don't merge strings into binfilter module
125cdf0e10cSrcweir	        chomp( $line );
126cdf0e10cSrcweir            $currentFile  = $srcpath . '\\' . $prj . '\\' . $file;
127cdf0e10cSrcweir            $currentFile  =~ s/\\/\//g;
128cdf0e10cSrcweir
129cdf0e10cSrcweir            $cur_sdffile = $currentFile;
130cdf0e10cSrcweir            #if( $cur_sdffile =~ /\.$file_types[\s]*$/ ){
131cdf0e10cSrcweir                $cur_sdffile =~ s/\/[^\/]*\.$file_types[\s]*$/\/localize.sdf/;
132cdf0e10cSrcweir            #}
133cdf0e10cSrcweir
134cdf0e10cSrcweir            # Set default date
135cdf0e10cSrcweir            if( $line =~ /(.*)\t[^\t\$]*$/ ){
136cdf0e10cSrcweir                $line = $1."\t".$default_date;
137cdf0e10cSrcweir            }
138cdf0e10cSrcweir
139cdf0e10cSrcweir            if( $start ){
140cdf0e10cSrcweir                $start='';
141cdf0e10cSrcweir                $lastFile = $currentFile; # ?
142cdf0e10cSrcweir                $last_sdffile = $cur_sdffile;
143cdf0e10cSrcweir            }
144cdf0e10cSrcweir
145cdf0e10cSrcweir            if( $lang eq "en-US" ){}
146cdf0e10cSrcweir            elsif( $cur_sdffile eq $last_sdffile )
147cdf0e10cSrcweir            {
1486d678023Smseidel                $block{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$platform\t$lang" } =  $line ;
149cdf0e10cSrcweir            }
150cdf0e10cSrcweir            else
151cdf0e10cSrcweir            {
152cdf0e10cSrcweir                writesdf( $lastFile , \%block );
153cdf0e10cSrcweir                $lastFile = $currentFile; #?
154cdf0e10cSrcweir                $last_sdffile = $cur_sdffile;
155cdf0e10cSrcweir                %block = ();
156cdf0e10cSrcweir                #if( ! $lang eq "en-US"  ) {
1576d678023Smseidel                $block{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$platform\t$lang" } =  $line ;
158cdf0e10cSrcweir                #}
159cdf0e10cSrcweir
160cdf0e10cSrcweir            }
161cdf0e10cSrcweir        } #else { print STDOUT "splitfile REGEX kaputt\n";}
162cdf0e10cSrcweir
163cdf0e10cSrcweir    }
164cdf0e10cSrcweir    writesdf( $lastFile , \%block );
165cdf0e10cSrcweir    %block = ();
166cdf0e10cSrcweir    close( MYFILE );
167cdf0e10cSrcweir
168cdf0e10cSrcweir}
169cdf0e10cSrcweir#########################################################
170cdf0e10cSrcweir
171cdf0e10cSrcweirsub writesdf{
172cdf0e10cSrcweir
173cdf0e10cSrcweir    my $lastFile        = shift;
174cdf0e10cSrcweir    my $blockhash_ref   = shift;
175cdf0e10cSrcweir    my $localizeFile    = $lastFile;
176cdf0e10cSrcweir    my %index=();
177cdf0e10cSrcweir
178cdf0e10cSrcweir    if( $localizeFile =~ /\.$file_types[\s]*$/ ){
179cdf0e10cSrcweir        $localizeFile =~ s/\/[^\/]*\.$file_types[\s]*$/\/localize.sdf/;
180cdf0e10cSrcweir        }else {
181cdf0e10cSrcweir            print STDERR "Strange filetype found '$localizeFile'\n";
182cdf0e10cSrcweir            return;
183cdf0e10cSrcweir        }
184cdf0e10cSrcweir        if( open DESTFILE , "< $localizeFile" ){
185cdf0e10cSrcweir
186cdf0e10cSrcweir        #or die "Can't open/create '\$localizeFile'";
187cdf0e10cSrcweir
188cdf0e10cSrcweir        #### Build hash
189cdf0e10cSrcweir        while(<DESTFILE>){
190cdf0e10cSrcweir         if( /$sdf_regex/ ){
191cdf0e10cSrcweir            my $line           = defined $_ ? $_ : '';
192cdf0e10cSrcweir            my $prj            = defined $3 ? $3 : '';
193cdf0e10cSrcweir            my $file           = defined $4 ? $4 : '';
194cdf0e10cSrcweir            my $type           = defined $6 ? $6 : '';
195cdf0e10cSrcweir            my $gid            = defined $7 ? $7 : '';
196cdf0e10cSrcweir            my $lid            = defined $8 ? $8 : '';
197cdf0e10cSrcweir            my $lang           = defined $12 ? $12 : '';
1986d678023Smseidel            my $platform       = defined $10 ? $10 : '';
199cdf0e10cSrcweir            my $helpid         = defined $9 ? $9 : '';
200cdf0e10cSrcweir
201cdf0e10cSrcweir            chomp( $line );
2026d678023Smseidel            $index{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$platform\t$lang" } =  $line ;
203cdf0e10cSrcweir
204cdf0e10cSrcweir         } #else { print STDOUT "writesdf REGEX kaputt $_\n";}
205cdf0e10cSrcweir
206cdf0e10cSrcweir        }
207cdf0e10cSrcweir        close( DESTFILE );
208cdf0e10cSrcweir    }
209cdf0e10cSrcweir    #### Copy new strings
210cdf0e10cSrcweir    my @mykeys = keys( %{ $blockhash_ref } );
211cdf0e10cSrcweir    my $isDirty = "FALSE";
212cdf0e10cSrcweir    foreach my $key( @mykeys ){
213cdf0e10cSrcweir        if( ! defined $index{ $key } ){
214cdf0e10cSrcweir            # Add new entry
215cdf0e10cSrcweir            $index{ $key }  = $blockhash_ref->{ $key} ;
216cdf0e10cSrcweir            $isDirty = "TRUE";
217cdf0e10cSrcweir        }elsif( $index{ $key } ne $blockhash_ref->{ $key } ){
218cdf0e10cSrcweir            # Overwrite old entry
219cdf0e10cSrcweir            $index{ $key } = $blockhash_ref->{ $key };
220cdf0e10cSrcweir            $isDirty = "TRUE";
221cdf0e10cSrcweir        }else {
222cdf0e10cSrcweir        }
223cdf0e10cSrcweir    }
224cdf0e10cSrcweir
225cdf0e10cSrcweir    #### Write file
226cdf0e10cSrcweir
227cdf0e10cSrcweir    if( !$bVerbose ){ print STDOUT "."; }
228cdf0e10cSrcweir    if( $isDirty eq "TRUE" ){
229cdf0e10cSrcweir        if( $bVerbose ){ print STDOUT "$localizeFile\n"; }
230cdf0e10cSrcweir        if( open DESTFILE , "+> $localizeFile" ){
231cdf0e10cSrcweir            print DESTFILE get_license_header();
232cdf0e10cSrcweir            @mykeys = sort keys( %index );
233cdf0e10cSrcweir            foreach my $key( @mykeys ){
234cdf0e10cSrcweir                print DESTFILE ( $index{ $key } , "\n" );
235cdf0e10cSrcweir            }
236cdf0e10cSrcweir            close DESTFILE;
237cdf0e10cSrcweir         }else {
238cdf0e10cSrcweir            print STDOUT "WARNING: File $localizeFile is not writable , try to merge ...\n";
239cdf0e10cSrcweir            my ( $TMPFILE , $tmpfile ) = File::Temp::tempfile();
240cdf0e10cSrcweir            if( open DESTFILE , "+> $tmpfile " ){
241cdf0e10cSrcweir                @mykeys = keys( %index );
242cdf0e10cSrcweir                foreach my $key( @mykeys ){
243cdf0e10cSrcweir                    print DESTFILE ( $index{ $key } , "\n" );
244cdf0e10cSrcweir                }
245cdf0e10cSrcweir                close DESTFILE;
246cdf0e10cSrcweir                if( move( $localizeFile , $localizeFile.".backup" ) ){
247cdf0e10cSrcweir                    if( copy( $tmpfile , $localizeFile ) ){
248cdf0e10cSrcweir                        unlink $localizeFile.".backup";
249cdf0e10cSrcweir                    } else { print STDERR "Can't open/create '$localizeFile', original file is renamed to  $localizeFile.backup\n"; }
250cdf0e10cSrcweir                } else { print STDERR "Can't open/create '$localizeFile'\n"; }
251cdf0e10cSrcweir            }else{
252cdf0e10cSrcweir                print STDERR "WARNING: Can't open/create '$localizeFile'\n";
253cdf0e10cSrcweir            }
254cdf0e10cSrcweir            unlink $tmpfile;
255cdf0e10cSrcweir        }
256cdf0e10cSrcweir    }
257cdf0e10cSrcweir#    if( $no_sort eq '' ){
258cdf0e10cSrcweir#        sort_outfile( $localizeFile );
259cdf0e10cSrcweir#    }
260cdf0e10cSrcweir}
261cdf0e10cSrcweir
262cdf0e10cSrcweirsub get_license_header{
263cdf0e10cSrcweir    return
264cdf0e10cSrcweir"#\n".
265cdf0e10cSrcweir"#    ####    ###     #   #   ###   #####    #####  ####   #####  #####  \n".
266cdf0e10cSrcweir"#    #   #  #   #    ##  #  #   #    #      #      #   #    #      #    \n".
267cdf0e10cSrcweir"#    #   #  #   #    # # #  #   #    #      ###    #   #    #      #    \n".
268cdf0e10cSrcweir"#    #   #  #   #    #  ##  #   #    #      #      #   #    #      #    \n".
269cdf0e10cSrcweir"#    ####    ###     #   #   ###     #      #####  ####   #####    #    \n".
270cdf0e10cSrcweir"#\n".
271*0c423017Smseidel"#    DO NOT EDIT! This file will be overwritten by localization process\n".
272cdf0e10cSrcweir"#\n".
2738e9e5c11SAndrew Rist"#**************************************************************\n".
2748e9e5c11SAndrew Rist"#  \n".
2758e9e5c11SAndrew Rist"#  Licensed to the Apache Software Foundation (ASF) under one\n".
2768e9e5c11SAndrew Rist"#  or more contributor license agreements.  See the NOTICE file\n".
2778e9e5c11SAndrew Rist"#  distributed with this work for additional information\n".
2788e9e5c11SAndrew Rist"#  regarding copyright ownership.  The ASF licenses this file\n".
2798e9e5c11SAndrew Rist"#  to you under the Apache License, Version 2.0 (the\n".
2808e9e5c11SAndrew Rist"#  "License"); you may not use this file except in compliance\n".
2818e9e5c11SAndrew Rist"#  with the License.  You may obtain a copy of the License at\n".
2828e9e5c11SAndrew Rist"#  \n".
2838e9e5c11SAndrew Rist"#    http://www.apache.org/licenses/LICENSE-2.0\n".
2848e9e5c11SAndrew Rist"#  \n".
2858e9e5c11SAndrew Rist"#  Unless required by applicable law or agreed to in writing,\n".
2868e9e5c11SAndrew Rist"#  software distributed under the License is distributed on an\n".
2878e9e5c11SAndrew Rist"#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n".
2888e9e5c11SAndrew Rist"#  KIND, either express or implied.  See the License for the\n".
2898e9e5c11SAndrew Rist"#  specific language governing permissions and limitations\n".
2908e9e5c11SAndrew Rist"#  under the License.\n".
2918e9e5c11SAndrew Rist"#  \n".
2928e9e5c11SAndrew Rist"#**************************************************************\n";
293cdf0e10cSrcweir}
294cdf0e10cSrcweir######## Check input sdf file and use only the correct part
295cdf0e10cSrcweirsub merge_gsicheck{
296cdf0e10cSrcweir    my $command = '';
297cdf0e10cSrcweir    my ( $TMPHANDLE , $tmpfile ) = File::Temp::tempfile();
298cdf0e10cSrcweir    my ( $TMPHANDLE2 , $tmpfile2 ) = File::Temp::tempfile();
299cdf0e10cSrcweir    close ( $TMPHANDLE );
300cdf0e10cSrcweir    close ( $TMPHANDLE2 );
301cdf0e10cSrcweir
302cdf0e10cSrcweir    unlink $tmpfile2;
303cdf0e10cSrcweir    my $output2 = `cat $sdffile | sort > $tmpfile2`;
304cdf0e10cSrcweir    my $rc2 = $? << 8;
305cdf0e10cSrcweir    if( $rc2 ne  0 ){
306cdf0e10cSrcweir        printf("ERROR: Failed -> cat $sdffile | sort > $tmpfile2\n$output2\n");
307cdf0e10cSrcweir        exit( -1 );
308cdf0e10cSrcweir    }
309cdf0e10cSrcweir
310cdf0e10cSrcweir#    if( $ENV{WRAPCMD} ){
311cdf0e10cSrcweir#        $command = "$ENV{WRAPCMD} gsicheck";
312cdf0e10cSrcweir#    }else{
313cdf0e10cSrcweir#        $command = "gsicheck";
314cdf0e10cSrcweir#    }
315cdf0e10cSrcweir#    my $errfile = $tmpfile.".err";
316cdf0e10cSrcweir#    $command .= " -k -c -wcf $tmpfile -wef $errfile -l \"\" $tmpfile2";
317cdf0e10cSrcweir#    my $output = `$command`;
318cdf0e10cSrcweir#    my $rc = $? << 8;
319cdf0e10cSrcweir#    if ( $output ne "" ){
320cdf0e10cSrcweir#        print STDOUT "### gsicheck ###\n";
321cdf0e10cSrcweir#        print STDOUT "### The file $errfile have been written containing the errors in your sdf file. Those lines will not be merged: ###\n\n";
322cdf0e10cSrcweir#        print STDOUT "$output\n";
323cdf0e10cSrcweir#        print STDOUT "################\n";
324cdf0e10cSrcweir#
325cdf0e10cSrcweir#    }else{
326cdf0e10cSrcweir#        # Remove the 0 Byte file
327cdf0e10cSrcweir#        unlink $errfile;
328cdf0e10cSrcweir#    }
329cdf0e10cSrcweir    $sdffile = $tmpfile2;
330cdf0e10cSrcweir#    unlink $tmpfile2;
331cdf0e10cSrcweir}
332cdf0e10cSrcweir#########################################################
333cdf0e10cSrcweirsub collectfiles{
334cdf0e10cSrcweir    print STDOUT "### Localize\n";
335cdf0e10cSrcweir    my @sdfparticles;
336cdf0e10cSrcweir    my $localizehash_ref;
337cdf0e10cSrcweir    my ( $bAll , $bUseLocalize, $langhash_ref , $bHasSourceLanguage , $bFakeEnglish ) = parseLanguages();
338cdf0e10cSrcweir
339cdf0e10cSrcweir    # Enable autoflush on STDOUT
340cdf0e10cSrcweir    # $| = 1;
341cdf0e10cSrcweir    STDOUT->autoflush( 1 );
342cdf0e10cSrcweir
343cdf0e10cSrcweir    ### Search sdf particles
344cdf0e10cSrcweir    print STDOUT "### Searching sdf particles\n";
345cdf0e10cSrcweir    my $working_path = getcwd();
346cdf0e10cSrcweir    chdir $srcpath;
347cdf0e10cSrcweir    find sub {
348cdf0e10cSrcweir        my $file = $File::Find::name;
349cdf0e10cSrcweir        if( -f && $file =~ /.*localize.sdf$/ ) {
350cdf0e10cSrcweir            push   @sdfparticles , $file;
351cdf0e10cSrcweir            if( $bVerbose eq "1" ) { print STDOUT "$file\n"; }
352cdf0e10cSrcweir            else { print ".";  }
353cdf0e10cSrcweir
354cdf0e10cSrcweir         }
355cdf0e10cSrcweir    } , getcwd() ;#"."; #$srcpath;
356cdf0e10cSrcweir    chdir $working_path;
357cdf0e10cSrcweir
358cdf0e10cSrcweir    my $nFound  = $#sdfparticles +1;
359cdf0e10cSrcweir    print "\n    $nFound files found !\n";
360cdf0e10cSrcweir
361cdf0e10cSrcweir    my ( $LOCALIZEPARTICLE , $localizeSDF ) = File::Temp::tempfile();
362cdf0e10cSrcweir    close( $LOCALIZEPARTICLE );
363cdf0e10cSrcweir
364cdf0e10cSrcweir    my ( $ALLPARTICLES_MERGED , $particleSDF_merged )     = File::Temp::tempfile();
365cdf0e10cSrcweir    close( $ALLPARTICLES_MERGED );
366cdf0e10cSrcweir    my ( $LOCALIZE_LOG , $my_localize_log ) = File::Temp::tempfile();
367cdf0e10cSrcweir    close( $LOCALIZE_LOG );
368cdf0e10cSrcweir
369cdf0e10cSrcweir    ## Get the localize de,en-US extract
370cdf0e10cSrcweir    if( $bAll || $bUseLocalize ){
371cdf0e10cSrcweir        print "### Fetching source language strings\n";
372cdf0e10cSrcweir        my $command = "";
373cdf0e10cSrcweir        my $args    = "";
374cdf0e10cSrcweir
375cdf0e10cSrcweir        if( $ENV{WRAPCMD} ){
376cdf0e10cSrcweir            $command = "$ENV{WRAPCMD} localize_sl";
377cdf0e10cSrcweir        }else{
378cdf0e10cSrcweir            $command = "localize_sl";
379cdf0e10cSrcweir        }
380cdf0e10cSrcweir
381cdf0e10cSrcweir        # -e
382cdf0e10cSrcweir        # if ( -x $command ){
383cdf0e10cSrcweir        if( $command ){
384cdf0e10cSrcweir            if( !$bVerbose  ){ $args .= " -QQ -skip_links "; }
385cdf0e10cSrcweir            $args .= " -e -f $localizeSDF -l ";
386cdf0e10cSrcweir            my $bFlag="";
387cdf0e10cSrcweir            if( $bAll ) {$args .= " en-US";}
388cdf0e10cSrcweir            else{
389cdf0e10cSrcweir              my @list;
390cdf0e10cSrcweir              foreach my $isokey ( keys( %{ $langhash_ref } ) ){
391cdf0e10cSrcweir                push @list , $isokey;
392cdf0e10cSrcweir                if( $langhash_ref->{ $isokey } ne "" ){
393cdf0e10cSrcweir                    push @list , $langhash_ref->{ $isokey };
394cdf0e10cSrcweir                }
395cdf0e10cSrcweir              }
396cdf0e10cSrcweir              remove_duplicates( \@list );
397cdf0e10cSrcweir              foreach my $isokey ( @list ){
398cdf0e10cSrcweir                switch :{
399cdf0e10cSrcweir                    #( $isokey=~ /^de$/i  )
400cdf0e10cSrcweir                    #    && do{
401cdf0e10cSrcweir                    #            if( $bFlag eq "TRUE" ){ $args .= ",de"; }
402cdf0e10cSrcweir                    #            else  {
403cdf0e10cSrcweir                    #                $args .=  "de";  $bFlag = "TRUE";
404cdf0e10cSrcweir                    #             }
405cdf0e10cSrcweir                    #          };
406cdf0e10cSrcweir                        ( $isokey=~ /^en-US$/i  )
407cdf0e10cSrcweir                        && do{
408cdf0e10cSrcweir                                if( $bFlag eq "TRUE" ){ $args .= ",en-US"; }
409cdf0e10cSrcweir                                else {
410cdf0e10cSrcweir                                    $args .= "en-US";  $bFlag = "TRUE";
411cdf0e10cSrcweir                                 }
412cdf0e10cSrcweir                              };
413cdf0e10cSrcweir
414cdf0e10cSrcweir                    } #switch
415cdf0e10cSrcweir                } #foreach
416cdf0e10cSrcweir              } # if
417cdf0e10cSrcweir        } # if
418cdf0e10cSrcweir        if ( $bVerbose ) { print STDOUT $command.$args."\n"; }
419cdf0e10cSrcweir
420cdf0e10cSrcweir        my $rc = system( $command.$args );
421cdf0e10cSrcweir
422cdf0e10cSrcweir        #my $output = `$command.$args`;
423cdf0e10cSrcweir        #my $rc = $? << 8;
424cdf0e10cSrcweir
425cdf0e10cSrcweir        if( $rc < 0 ){    print STDERR "ERROR: localize rc = $rc\n"; exit( -1 ); }
426cdf0e10cSrcweir        ( $localizehash_ref )  = read_file( $localizeSDF , $langhash_ref );
427cdf0e10cSrcweir
428cdf0e10cSrcweir    }
429cdf0e10cSrcweir    ## Get sdf particles
430cdf0e10cSrcweir   open ALLPARTICLES_MERGED , "+>> $particleSDF_merged"
431cdf0e10cSrcweir    or die "Can't open $particleSDF_merged";
432cdf0e10cSrcweir
433cdf0e10cSrcweir    ## Fill fackback hash
434cdf0e10cSrcweir    my( $fallbackhashhash_ref ) = fetch_fallback( \@sdfparticles , $localizeSDF ,  $langhash_ref );
435cdf0e10cSrcweir#    my( $fallbackhashhash_ref ) = fetch_fallback( \@sdfparticles , $localizeSDF , $langhash_ref );
436cdf0e10cSrcweir    my %block;
437cdf0e10cSrcweir    my $cur_fallback;
438cdf0e10cSrcweir    if( !$bAll) {
439cdf0e10cSrcweir        foreach my $cur_lang ( keys( %{ $langhash_ref } ) ){
440cdf0e10cSrcweir            #print STDOUT "DBG: G1 cur_lang=$cur_lang\n";
441cdf0e10cSrcweir            $cur_fallback = $langhash_ref->{ $cur_lang };
442cdf0e10cSrcweir            if( $cur_fallback ne "" ){
443cdf0e10cSrcweir                # Insert fallback strings
444cdf0e10cSrcweir                #print STDOUT "DBG: Renaming $cur_fallback to $cur_lang in fallbackhash\n";
445cdf0e10cSrcweir                rename_language(  $fallbackhashhash_ref ,  $cur_fallback , $cur_lang );
446cdf0e10cSrcweir            }
447cdf0e10cSrcweir            foreach my $currentfile ( @sdfparticles ){
448cdf0e10cSrcweir                if ( open MYFILE , "< $currentfile" ) {
449cdf0e10cSrcweir                    while(<MYFILE>){
450cdf0e10cSrcweir                        if( /$sdf_regex/ ){
451cdf0e10cSrcweir                            my $line           = defined $_ ? $_ : '';
452cdf0e10cSrcweir                            my $prj            = defined $3 ? $3 : '';
453cdf0e10cSrcweir                            my $file           = defined $4 ? $4 : '';
454cdf0e10cSrcweir                            my $type           = defined $6 ? $6 : '';
455cdf0e10cSrcweir                            my $gid            = defined $7 ? $7 : '';
456cdf0e10cSrcweir                            my $lid            = defined $8 ? $8 : '';
457cdf0e10cSrcweir                            my $lang           = defined $12 ? $12 : '';
4586d678023Smseidel                            my $platform       = defined $10 ? $10 : '';
459cdf0e10cSrcweir                            my $helpid         = defined $9 ? $9 : '';
460cdf0e10cSrcweir
461cdf0e10cSrcweir                            chomp( $line );
462cdf0e10cSrcweir
463cdf0e10cSrcweir                            if ( $lang eq $cur_lang ){
464cdf0e10cSrcweir                                # Overwrite fallback strings with collected strings
465cdf0e10cSrcweir                                #if( ( !has_two_sourcelanguages( $cur_lang) && $cur_lang eq "de" ) || $cur_lang ne "en-US" ){
4666d678023Smseidel                                     $fallbackhashhash_ref->{ $cur_lang }{ $prj.$gid.$lid.$file.$type.$platform.$helpid } =  $line ;
467cdf0e10cSrcweir                                     #}
468cdf0e10cSrcweir
469cdf0e10cSrcweir                            }
470cdf0e10cSrcweir                        }
471cdf0e10cSrcweir                    }
472cdf0e10cSrcweir                }else { print STDERR "WARNING: Can't open file $currentfile"; }
473cdf0e10cSrcweir            }
474cdf0e10cSrcweir
475cdf0e10cSrcweir            foreach my $line ( keys( %{$fallbackhashhash_ref->{ $cur_lang } } )) {
476cdf0e10cSrcweir                if( #$cur_lang ne "de" &&
477cdf0e10cSrcweir                    $cur_lang ne "en-US" ){
478cdf0e10cSrcweir                    print ALLPARTICLES_MERGED ( $fallbackhashhash_ref->{ $cur_lang }{ $line }, "\n" );
479cdf0e10cSrcweir                }
480cdf0e10cSrcweir             }
481cdf0e10cSrcweir        }
482cdf0e10cSrcweir    } else {
483cdf0e10cSrcweir        foreach my $currentfile ( @sdfparticles ){
484cdf0e10cSrcweir            if ( open MYFILE , "< $currentfile" ) {
485cdf0e10cSrcweir                while( <MYFILE> ){
486cdf0e10cSrcweir                    print ALLPARTICLES_MERGED ( $_, "\n" );  # recheck de / en-US !
487cdf0e10cSrcweir                }
488cdf0e10cSrcweir            }
489cdf0e10cSrcweir            else { print STDERR "WARNING: Can't open file $currentfile"; }
490cdf0e10cSrcweir        }
491cdf0e10cSrcweir    }
492cdf0e10cSrcweir    close ALLPARTICLES_MERGED;
493cdf0e10cSrcweir
494cdf0e10cSrcweir
495cdf0e10cSrcweir    # Hash of array
496cdf0e10cSrcweir    my %output;
497cdf0e10cSrcweir    my @order;
498cdf0e10cSrcweir
499cdf0e10cSrcweir    ## Join both
500cdf0e10cSrcweir    if( $outputfile ){
501cdf0e10cSrcweir        if( open DESTFILE , "+> $outputfile" ){
502cdf0e10cSrcweir            if( !open LOCALIZEPARTICLE ,  "< $localizeSDF" ) { print STDERR "ERROR: Can't open file $localizeSDF\n"; }
503cdf0e10cSrcweir            if( !open ALLPARTICLES_MERGED , "< $particleSDF_merged" ) { print STDERR "ERROR: Can't open file $particleSDF_merged\n"; }
504cdf0e10cSrcweir
505cdf0e10cSrcweir            # Insert localize
506cdf0e10cSrcweir            my $extract_date="";
507cdf0e10cSrcweir            while ( <LOCALIZEPARTICLE> ){
508cdf0e10cSrcweir                if( /$sdf_regex/ ){
509cdf0e10cSrcweir                    my $leftpart       = defined $2 ? $2 : '';
510cdf0e10cSrcweir                    my $lang           = defined $12 ? $12 : '';
511cdf0e10cSrcweir                    my $rightpart      = defined $13 ? $13 : '';
512cdf0e10cSrcweir                    my $timestamp      = defined $18 ? $18 : '';
513cdf0e10cSrcweir
514cdf0e10cSrcweir                    my $prj            = defined $3 ? $3 : '';
515cdf0e10cSrcweir                    my $file           = defined $4 ? $4 : '';
516cdf0e10cSrcweir                    my $type           = defined $6 ? $6 : '';
517cdf0e10cSrcweir                    my $gid            = defined $7 ? $7 : '';
518cdf0e10cSrcweir                    my $lid            = defined $8 ? $8 : '';
519cdf0e10cSrcweir                    #my $lang           = defined $12 ? $12 : '';
5206d678023Smseidel                    my $platform       = defined $10 ? $10 : '';
521cdf0e10cSrcweir                    my $helpid         = defined $9 ? $9 : '';
522cdf0e10cSrcweir
523cdf0e10cSrcweir
524cdf0e10cSrcweir                    if( $use_default_date )
525cdf0e10cSrcweir                    {
526cdf0e10cSrcweir                        $extract_date = "$default_date\n" ;
527cdf0e10cSrcweir                    }
528cdf0e10cSrcweir                    elsif( $extract_date eq "" ) {
529cdf0e10cSrcweir                        $extract_date = $timestamp ;
530cdf0e10cSrcweir                        $extract_date =~ tr/\r\n//d;
531cdf0e10cSrcweir                        $extract_date .= "\n";
532cdf0e10cSrcweir                    }
533cdf0e10cSrcweir
534cdf0e10cSrcweir                    if( $bAll ){ print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date ; }
535cdf0e10cSrcweir                    else {
536cdf0e10cSrcweir                        foreach my $sLang ( keys( %{ $langhash_ref } ) ){
537cdf0e10cSrcweir                            if( $sLang=~ /all/i )                       {
5386d678023Smseidel                                push @{ $output{ $prj.$gid.$lid.$file.$type.$platform.$helpid } } ,  $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
539cdf0e10cSrcweir                                #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date;
540cdf0e10cSrcweir                            }
541cdf0e10cSrcweir                            #if( $sLang eq "de" && $lang eq "de" )       {
5426d678023Smseidel                            #    push @{ $output{ $prj.$gid.$lid.$file.$type.$platform.$helpid } } ,  $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
543cdf0e10cSrcweir                                #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date;
544cdf0e10cSrcweir                                #}
545cdf0e10cSrcweir                            if( $sLang eq "en-US" && $lang eq "en-US" ) {
5466d678023Smseidel                                push @order , $prj.$gid.$lid.$file.$type.$platform.$helpid;
5476d678023Smseidel                                if( !$bFakeEnglish ){ push @{ $output{ $prj.$gid.$lid.$file.$type.$platform.$helpid } } ,  $leftpart."\t".$lang."\t".$rightpart.$extract_date ; }
548cdf0e10cSrcweir                                #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date;
549cdf0e10cSrcweir                            }
550cdf0e10cSrcweir
551cdf0e10cSrcweir                        }
552cdf0e10cSrcweir                    }
553cdf0e10cSrcweir                }
554cdf0e10cSrcweir            }
555cdf0e10cSrcweir            # Insert particles
556cdf0e10cSrcweir            while ( <ALLPARTICLES_MERGED> ){
557cdf0e10cSrcweir                if( /$sdf_regex/ ){
558cdf0e10cSrcweir                    my $leftpart       = defined $2 ? $2 : '';
559cdf0e10cSrcweir                    my $prj            = defined $3 ? $3 : '';
560cdf0e10cSrcweir                    my $lang           = defined $12 ? $12 : '';
561cdf0e10cSrcweir                    my $rightpart      = defined $13 ? $13 : '';
562cdf0e10cSrcweir                    my $timestamp      = defined $18 ? $18 : '';
563cdf0e10cSrcweir
564cdf0e10cSrcweir                    #my $prj            = defined $3 ? $3 : '';
565cdf0e10cSrcweir                    my $file           = defined $4 ? $4 : '';
566cdf0e10cSrcweir                    my $type           = defined $6 ? $6 : '';
567cdf0e10cSrcweir                    my $gid            = defined $7 ? $7 : '';
568cdf0e10cSrcweir                    my $lid            = defined $8 ? $8 : '';
569cdf0e10cSrcweir                    #my $lang           = defined $12 ? $12 : '';
5706d678023Smseidel                    my $platform       = defined $10 ? $10 : '';
571cdf0e10cSrcweir                    my $helpid         = defined $9 ? $9 : '';
572cdf0e10cSrcweir
573cdf0e10cSrcweir
574cdf0e10cSrcweir                    if( $use_default_date )
575cdf0e10cSrcweir                    {
576cdf0e10cSrcweir                        $extract_date = "$default_date\n" ;
577cdf0e10cSrcweir                    }
578cdf0e10cSrcweir                    elsif( $extract_date eq "" )
579cdf0e10cSrcweir                    {
580cdf0e10cSrcweir                        $extract_date = $timestamp;
581cdf0e10cSrcweir                    }
582cdf0e10cSrcweir
583cdf0e10cSrcweir                    if( ! ( $prj =~ /binfilter/i ) ) {
5846d678023Smseidel                        push @{ $output{ $prj.$gid.$lid.$file.$type.$platform.$helpid } } , $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
585cdf0e10cSrcweir                        #print DESTFILE $leftpart."\t".$lang."\t".$rightpart.$extract_date ;
586cdf0e10cSrcweir                    }
587cdf0e10cSrcweir                 }
588cdf0e10cSrcweir            }
589cdf0e10cSrcweir
590cdf0e10cSrcweir            # Write!
591cdf0e10cSrcweir            foreach my $curkey ( @order ){
592cdf0e10cSrcweir                foreach my $curlist ( $output{ $curkey } ){
593cdf0e10cSrcweir                    foreach my $line ( @{$curlist} ){
594cdf0e10cSrcweir                        print DESTFILE $line;
595cdf0e10cSrcweir                    }
596cdf0e10cSrcweir                }
597cdf0e10cSrcweir            }
598cdf0e10cSrcweir
599cdf0e10cSrcweir        }else { print STDERR "Can't open $outputfile";}
600cdf0e10cSrcweir    }
601cdf0e10cSrcweir    close DESTFILE;
602cdf0e10cSrcweir    close LOCALIZEPARTICLE;
603cdf0e10cSrcweir    close ALLPARTICLES_MERGED;
604cdf0e10cSrcweir
605cdf0e10cSrcweir    #print STDOUT "DBG: \$localizeSDF $localizeSDF \$particleSDF_merged $particleSDF_merged\n";
606cdf0e10cSrcweir    unlink $localizeSDF , $particleSDF_merged ,  $my_localize_log;
607cdf0e10cSrcweir
608cdf0e10cSrcweir    #sort_outfile( $outputfile );
609cdf0e10cSrcweir    #remove_obsolete( $outputfile ) , if $bHasSourceLanguage ne "";
610cdf0e10cSrcweir    }
611cdf0e10cSrcweir
612cdf0e10cSrcweir#########################################################
613cdf0e10cSrcweirsub remove_obsolete{
614cdf0e10cSrcweir    my $outfile = shift;
615cdf0e10cSrcweir    my @lines;
616cdf0e10cSrcweir    my $enusleftpart;
617cdf0e10cSrcweir    my @good_lines;
618cdf0e10cSrcweir
619cdf0e10cSrcweir    print STDOUT "### Removing obsolete strings\n";
620cdf0e10cSrcweir
621cdf0e10cSrcweir    # Kick out all strings without en-US reference
622cdf0e10cSrcweir    if ( open ( SORTEDFILE , "< $outfile" ) ){
623cdf0e10cSrcweir        while( <SORTEDFILE> ){
624cdf0e10cSrcweir            if( /$sdf_regex/ ){
625cdf0e10cSrcweir                my $line           = defined $_ ? $_ : '';
626cdf0e10cSrcweir                my $language       = defined $12 ? $12 : '';
627cdf0e10cSrcweir                my $prj            = defined $3 ? $3 : '';
628cdf0e10cSrcweir                my $file           = defined $4 ? $4 : '';
629cdf0e10cSrcweir                my $type           = defined $6 ? $6 : '';
630cdf0e10cSrcweir                my $gid            = defined $7 ? $7 : '';
631cdf0e10cSrcweir                my $lid            = defined $8 ? $8 : '';
6326d678023Smseidel                my $platform       = defined $10 ? $10 : '';
633cdf0e10cSrcweir                my $helpid         = defined $9 ? $9 : '';
634cdf0e10cSrcweir
6356d678023Smseidel                my $leftpart = $prj.$gid.$lid.$file.$type.$platform.$helpid;
636cdf0e10cSrcweir
637cdf0e10cSrcweir                if( $language eq "en-US" ){                 # source string found, 1. entry
638cdf0e10cSrcweir                    $enusleftpart = $leftpart;
639cdf0e10cSrcweir                    push @good_lines , $line;
640cdf0e10cSrcweir                }else{
641cdf0e10cSrcweir                    if( !defined $enusleftpart or !defined $leftpart ){
642cdf0e10cSrcweir                        print STDERR "BADLINE: $line\n";
643cdf0e10cSrcweir                        print STDERR "\$enusleftpart = $enusleftpart\n";
644cdf0e10cSrcweir                        print STDERR "\$leftpart = $leftpart\n";
645cdf0e10cSrcweir                    }
646cdf0e10cSrcweir                    if( $enusleftpart eq $leftpart ){   # matching language
647cdf0e10cSrcweir                        push @good_lines , $line;
648cdf0e10cSrcweir                    }
649cdf0e10cSrcweir                    #else{
650cdf0e10cSrcweir                    #    print STDERR "OUT:  \$enusleftpart=$enusleftpart \$leftpart=$leftpart \$line=$line\n";
651cdf0e10cSrcweir                    #}
652cdf0e10cSrcweir                }
653cdf0e10cSrcweir            }
654cdf0e10cSrcweir        }
655cdf0e10cSrcweir        close SORTEDFILE;
656cdf0e10cSrcweir    } else { print STDERR "ERROR: Can't open file $outfile\n";}
657cdf0e10cSrcweir
658cdf0e10cSrcweir    # Write file
659cdf0e10cSrcweir    if ( open ( SORTEDFILE , "> $outfile" ) ){
660cdf0e10cSrcweir        foreach my $newline ( @good_lines ) {
661cdf0e10cSrcweir            print SORTEDFILE $newline;
662cdf0e10cSrcweir        }
663cdf0e10cSrcweir        close SORTEDFILE;
664cdf0e10cSrcweir    } else { print STDERR "ERROR: Can't open file $outfile\n";}
665cdf0e10cSrcweir
666cdf0e10cSrcweir}
667cdf0e10cSrcweir#########################################################
668cdf0e10cSrcweirsub sort_outfile{
669cdf0e10cSrcweir        my $outfile = shift;
670cdf0e10cSrcweir        print STDOUT "### Sorting ... $outfile ...";
671cdf0e10cSrcweir        my @lines;
672cdf0e10cSrcweir        my @sorted_lines;
673cdf0e10cSrcweir
674cdf0e10cSrcweir
675cdf0e10cSrcweir        #if ( open ( SORTEDFILE , "< $outputfile" ) ){
676cdf0e10cSrcweir        if ( open ( SORTEDFILE , "< $outfile" ) ){
677cdf0e10cSrcweir            my $line;
678cdf0e10cSrcweir            while ( <SORTEDFILE> ){
679cdf0e10cSrcweir                $line = $_;
680cdf0e10cSrcweir                if( $line =~ /^[^\#]/ ){
681cdf0e10cSrcweir                    push @lines , $line;
682cdf0e10cSrcweir                }
683cdf0e10cSrcweir            }
684cdf0e10cSrcweir            close SORTEDFILE;
685cdf0e10cSrcweir            @sorted_lines = sort {
686cdf0e10cSrcweir                my $xa_lang          = "";
687cdf0e10cSrcweir                my $xa_left_part    = "";
688cdf0e10cSrcweir                my $xa_right_part    = "";
689cdf0e10cSrcweir                my $xa_timestamp     = "";
690cdf0e10cSrcweir                my $xb_lang          = "";
691cdf0e10cSrcweir                my $xb_left_part    = "";
692cdf0e10cSrcweir                my $xb_right_part    = "";
693cdf0e10cSrcweir                my $xb_timestamp     = "";
694cdf0e10cSrcweir                my $xa               = "";
695cdf0e10cSrcweir                my $xb               = "";
696cdf0e10cSrcweir                my @alist;
697cdf0e10cSrcweir                my @blist;
698cdf0e10cSrcweir
699cdf0e10cSrcweir                if( $a=~ /$sdf_regex/ ){
700cdf0e10cSrcweir                    $xa_left_part       = defined $2 ? $2 : '';
701cdf0e10cSrcweir                    $xa_lang           = defined $12 ? $12 : '';
702cdf0e10cSrcweir                    $xa_right_part     = defined $13 ? $13 : '';
703cdf0e10cSrcweir                    $xa_left_part = remove_last_column( $xa_left_part );
704cdf0e10cSrcweir
705cdf0e10cSrcweir                }
706cdf0e10cSrcweir                if( $b=~ /$sdf_regex/ ){
707cdf0e10cSrcweir                    $xb_left_part       = defined $2 ? $2 : '';
708cdf0e10cSrcweir                    $xb_lang           = defined $12 ? $12 : '';
709cdf0e10cSrcweir                    $xb_right_part     = defined $13 ? $13 : '';
710cdf0e10cSrcweir                    $xb_left_part = remove_last_column( $xb_left_part );
711cdf0e10cSrcweir
712cdf0e10cSrcweir
713cdf0e10cSrcweir                }
714cdf0e10cSrcweir                if( (  $xa_left_part cmp $xb_left_part ) == 0 ){         # Left part equal
715cdf0e10cSrcweir                     if( ( $xa_lang cmp $xb_lang ) == 0 ){               # Lang equal
716cdf0e10cSrcweir                         return ( $xa_right_part cmp $xb_right_part );   # Right part compare
717cdf0e10cSrcweir                    }
718cdf0e10cSrcweir                    elsif( $xa_lang eq "en-US" ) { return -1; }        # en-US wins
719cdf0e10cSrcweir                    elsif( $xb_lang eq "en-US" ) { return 1;  }        # en-US wins
720cdf0e10cSrcweir                    else { return $xa_lang cmp $xb_lang; }             # lang compare
721cdf0e10cSrcweir                }
722cdf0e10cSrcweir                else {
723cdf0e10cSrcweir                    return $xa_left_part cmp $xb_left_part;        # Left part compare
724cdf0e10cSrcweir                }
725cdf0e10cSrcweir            } @lines;
726cdf0e10cSrcweir
727cdf0e10cSrcweir            if ( open ( SORTEDFILE , "> $outfile" ) ){
728cdf0e10cSrcweir                print SORTEDFILE get_license_header();
729cdf0e10cSrcweir                foreach my $newline ( @sorted_lines ) {
730cdf0e10cSrcweir                    print SORTEDFILE $newline;
731cdf0e10cSrcweir                    #print STDOUT $newline;
732cdf0e10cSrcweir                }
733cdf0e10cSrcweir            }
734cdf0e10cSrcweir            close SORTEDFILE;
735cdf0e10cSrcweir        } else { print STDERR "WARNING: Can't open file $outfile\n";}
736cdf0e10cSrcweir	print "done\n";
737cdf0e10cSrcweir
738cdf0e10cSrcweir}
739cdf0e10cSrcweir#########################################################
740cdf0e10cSrcweirsub remove_last_column{
741cdf0e10cSrcweir    my $string                  = shift;
742cdf0e10cSrcweir    my @alist = split ( "\t" , $string );
743cdf0e10cSrcweir    pop @alist;
744cdf0e10cSrcweir    return join( "\t" , @alist );
745cdf0e10cSrcweir}
746cdf0e10cSrcweir
747cdf0e10cSrcweir#########################################################
748cdf0e10cSrcweirsub rename_language{
749cdf0e10cSrcweir    my $fallbackhashhash_ref    = shift;
750cdf0e10cSrcweir    my $cur_fallback            = shift;
751cdf0e10cSrcweir    my $cur_lang                = shift;
752cdf0e10cSrcweir    my $line;
753cdf0e10cSrcweir
754cdf0e10cSrcweir    foreach my $key( keys ( %{ $fallbackhashhash_ref->{ $cur_fallback } } ) ){
755cdf0e10cSrcweir        $line = $fallbackhashhash_ref->{ $cur_fallback }{ $key };
756cdf0e10cSrcweir        if( $line =~ /$sdf_regex/ ){
757cdf0e10cSrcweir            my $leftpart       = defined $2 ? $2 : '';
758cdf0e10cSrcweir            my $lang           = defined $12 ? $12 : '';
759cdf0e10cSrcweir            my $rightpart      = defined $13 ? $13 : '';
760cdf0e10cSrcweir
761cdf0e10cSrcweir            $fallbackhashhash_ref->{ $cur_lang }{ $key } = $leftpart."\t".$cur_lang."\t".$rightpart;
762cdf0e10cSrcweir        }
763cdf0e10cSrcweir    }
764cdf0e10cSrcweir}
765cdf0e10cSrcweir
766cdf0e10cSrcweir############################################################
767cdf0e10cSrcweirsub remove_duplicates{
768cdf0e10cSrcweir    my $list_ref    = shift;
769cdf0e10cSrcweir    my %tmphash;
770cdf0e10cSrcweir    foreach my $key ( @{ $list_ref } ){ $tmphash{ $key } = '' ; }
771cdf0e10cSrcweir    @{$list_ref} = keys( %tmphash );
772cdf0e10cSrcweir}
773cdf0e10cSrcweir
774cdf0e10cSrcweir##############################################################
775cdf0e10cSrcweirsub fetch_fallback{
776cdf0e10cSrcweir    my $sdfparticleslist_ref   = shift;
777cdf0e10cSrcweir    my $localizeSDF            = shift;
778cdf0e10cSrcweir    my $langhash_ref           = shift;
779cdf0e10cSrcweir    my %fallbackhashhash;
780cdf0e10cSrcweir    my $cur_lang;
781cdf0e10cSrcweir    my @langlist;
782cdf0e10cSrcweir
783cdf0e10cSrcweir    foreach my $key ( keys ( %{ $langhash_ref } ) ){
784cdf0e10cSrcweir        $cur_lang = $langhash_ref->{ $key };
785cdf0e10cSrcweir        if ( $cur_lang ne "" ) {
786cdf0e10cSrcweir            push @langlist , $cur_lang;
787cdf0e10cSrcweir        }
788cdf0e10cSrcweir    }
789cdf0e10cSrcweir    remove_duplicates( \@langlist );
790cdf0e10cSrcweir    foreach  $cur_lang ( @langlist ){
791cdf0e10cSrcweir        if( $cur_lang eq "en-US" ){
792cdf0e10cSrcweir            read_fallbacks_from_source( $localizeSDF , $cur_lang , \%fallbackhashhash );
793cdf0e10cSrcweir        }
794cdf0e10cSrcweir    }
795cdf0e10cSrcweir
796cdf0e10cSrcweir    # remove de / en-US
797cdf0e10cSrcweir    my @tmplist;
798cdf0e10cSrcweir    foreach $cur_lang( @langlist ){
799cdf0e10cSrcweir        if(  $cur_lang ne "en-US" ){
800cdf0e10cSrcweir           push @tmplist , $cur_lang;
801cdf0e10cSrcweir
802cdf0e10cSrcweir        }
803cdf0e10cSrcweir    }
804cdf0e10cSrcweir    @langlist = @tmplist;
805cdf0e10cSrcweir    if ( $#langlist +1 ){
806cdf0e10cSrcweir        read_fallbacks_from_particles( $sdfparticleslist_ref , \@langlist , \%fallbackhashhash );
807cdf0e10cSrcweir
808cdf0e10cSrcweir    }
809cdf0e10cSrcweir    return (\%fallbackhashhash);
810cdf0e10cSrcweir}
811cdf0e10cSrcweir
812cdf0e10cSrcweir#########################################################
813cdf0e10cSrcweirsub write_file{
814cdf0e10cSrcweir
815cdf0e10cSrcweir    my $localizeFile = shift;
816cdf0e10cSrcweir    my $index_ref    = shift;
817cdf0e10cSrcweir
818cdf0e10cSrcweir    if( open DESTFILE , "+> $localizeFile" ){
819cdf0e10cSrcweir        foreach my $key( %{ $index_ref } ){
820cdf0e10cSrcweir            print DESTFILE ($index_ref->{ $key }, "\n" );
821cdf0e10cSrcweir        }
822cdf0e10cSrcweir        close DESTFILE;
823cdf0e10cSrcweir    }else {
824cdf0e10cSrcweir      print STDERR "Can't open/create '$localizeFile'";
825cdf0e10cSrcweir    }
826cdf0e10cSrcweir}
827cdf0e10cSrcweir
828cdf0e10cSrcweir#########################################################
829cdf0e10cSrcweirsub read_file{
830cdf0e10cSrcweir
831cdf0e10cSrcweir    my $sdffile         = shift;
832cdf0e10cSrcweir    my $langhash_ref    = shift;
833cdf0e10cSrcweir    my %block           = ();
834cdf0e10cSrcweir
835cdf0e10cSrcweir    open MYFILE , "< $sdffile"
836cdf0e10cSrcweir        or die "Can't open '$sdffile'\n";
837cdf0e10cSrcweir        while( <MYFILE>){
838cdf0e10cSrcweir          if( /$sdf_regex/ ){
839cdf0e10cSrcweir            my $line           = defined $_ ? $_ : '';
840cdf0e10cSrcweir            my $prj            = defined $3 ? $3 : '';
841cdf0e10cSrcweir            my $file           = defined $4 ? $4 : '';
842cdf0e10cSrcweir            my $type           = defined $6 ? $6 : '';
843cdf0e10cSrcweir            my $gid            = defined $7 ? $7 : '';
844cdf0e10cSrcweir            my $lid            = defined $8 ? $8 : '';
8456d678023Smseidel            my $platform       = defined $10 ? $10 : '';
846cdf0e10cSrcweir            my $lang           = defined $12 ? $12 : '';
847cdf0e10cSrcweir            my $helpid         = defined $9 ? $9 : '';
848cdf0e10cSrcweir
849cdf0e10cSrcweir            foreach my $isolang ( keys ( %{ $langhash_ref } ) ){
8506d678023Smseidel                if( $isolang=~ /$lang/i || $isolang=~ /all/i ) { $block{$prj.$gid.$lid.$file.$type.$platform.$helpid } =  $line ; }
851cdf0e10cSrcweir            }
852cdf0e10cSrcweir        }
853cdf0e10cSrcweir    }
854cdf0e10cSrcweir    return (\%block);
855cdf0e10cSrcweir}
856cdf0e10cSrcweir
857cdf0e10cSrcweir#########################################################
858cdf0e10cSrcweirsub read_fallbacks_from_particles{
859cdf0e10cSrcweir
860cdf0e10cSrcweir    my $sdfparticleslist_ref    = shift;
861cdf0e10cSrcweir    my $isolanglist_ref         = shift;
862cdf0e10cSrcweir    my $fallbackhashhash_ref    = shift;
863cdf0e10cSrcweir    my $block_ref;
864cdf0e10cSrcweir    foreach my $currentfile ( @{ $sdfparticleslist_ref } ){
865cdf0e10cSrcweir        if ( open MYFILE , "< $currentfile" ) {
866cdf0e10cSrcweir            while(<MYFILE>){
867cdf0e10cSrcweir                if( /$sdf_regex/ ){
868cdf0e10cSrcweir                    my $line           = defined $_ ? $_ : '';
869cdf0e10cSrcweir                    my $prj            = defined $3 ? $3 : '';
870cdf0e10cSrcweir                    my $file           = defined $4 ? $4 : '';
871cdf0e10cSrcweir                    my $type           = defined $6 ? $6 : '';
872cdf0e10cSrcweir                    my $gid            = defined $7 ? $7 : '';
873cdf0e10cSrcweir                    my $lid            = defined $8 ? $8 : '';
874cdf0e10cSrcweir                    my $lang           = defined $12 ? $12 : '';
8756d678023Smseidel                    my $platform       = defined $10 ? $10 : '';
876cdf0e10cSrcweir                    my $helpid         = defined $9 ? $9 : '';
877cdf0e10cSrcweir
878cdf0e10cSrcweir                    chomp( $line );
879cdf0e10cSrcweir
880cdf0e10cSrcweir                    foreach my $isolang ( @{$isolanglist_ref}  ){
881cdf0e10cSrcweir                        if( $isolang=~ /$lang/i ) {
8826d678023Smseidel                            $fallbackhashhash_ref->{ $isolang }{ $prj.$gid.$lid.$file.$type.$platform.$helpid } =  $line ;
883cdf0e10cSrcweir                        }
884cdf0e10cSrcweir                    }
885cdf0e10cSrcweir                }
886cdf0e10cSrcweir            }
887cdf0e10cSrcweir       }else { print STDERR "WARNING: Can't open file $currentfile"; }
888cdf0e10cSrcweir    }
889cdf0e10cSrcweir}
890cdf0e10cSrcweir
891cdf0e10cSrcweir#########################################################
892cdf0e10cSrcweirsub read_fallbacks_from_source{
893cdf0e10cSrcweir
894cdf0e10cSrcweir    my $sdffile                 = shift;
895cdf0e10cSrcweir    my $isolang                 = shift;
896cdf0e10cSrcweir    my $fallbackhashhash_ref    = shift;
897cdf0e10cSrcweir    my $block_ref;
898cdf0e10cSrcweir    # read fallback for single file
899cdf0e10cSrcweir    open MYFILE , "< $sdffile"
900cdf0e10cSrcweir        or die "Can't open '$sdffile'\n";
901cdf0e10cSrcweir
902cdf0e10cSrcweir    while( <MYFILE>){
903cdf0e10cSrcweir          if( /$sdf_regex/ ){
904cdf0e10cSrcweir            my $line           = defined $_ ? $_ : '';
905cdf0e10cSrcweir            my $prj            = defined $3 ? $3 : '';
906cdf0e10cSrcweir            my $file           = defined $4 ? $4 : '';
907cdf0e10cSrcweir            my $type           = defined $6 ? $6 : '';
908cdf0e10cSrcweir            my $gid            = defined $7 ? $7 : '';
909cdf0e10cSrcweir            my $lid            = defined $8 ? $8 : '';
910cdf0e10cSrcweir            my $helpid         = defined $9 ? $9 : '';
911cdf0e10cSrcweir            my $lang           = defined $12 ? $12 : '';
9126d678023Smseidel            my $platform       = defined $10 ? $10 : '';
913cdf0e10cSrcweir
914cdf0e10cSrcweir            chomp( $line );
9156d678023Smseidel            if( $isolang=~ /$lang/i ) { $fallbackhashhash_ref->{ $isolang }{ $prj.$gid.$lid.$file.$type.$platform.$helpid } =  $line ;
916cdf0e10cSrcweir            }
917cdf0e10cSrcweir        }
918cdf0e10cSrcweir    }
919cdf0e10cSrcweir}
920cdf0e10cSrcweir
921cdf0e10cSrcweir#########################################################
922cdf0e10cSrcweirsub parseLanguages{
923cdf0e10cSrcweir
924cdf0e10cSrcweir    my $bAll;
925cdf0e10cSrcweir    my $bUseLocalize;
926cdf0e10cSrcweir    my $bHasSourceLanguage="";
927cdf0e10cSrcweir    my $bFakeEnglish="";
928cdf0e10cSrcweir    my %langhash;
929cdf0e10cSrcweir    my $iso="";
930cdf0e10cSrcweir    my $fallback="";
931cdf0e10cSrcweir
932cdf0e10cSrcweir    #### -l all
933cdf0e10cSrcweir    if(   $languages=~ /all/ ){
934cdf0e10cSrcweir        $bAll = "TRUE";
935cdf0e10cSrcweir        $bHasSourceLanguage = "TRUE";
936cdf0e10cSrcweir    }
937cdf0e10cSrcweir    ### -l fr=de,de
938cdf0e10cSrcweir    elsif( $languages=~ /.*,.*/ ){
939cdf0e10cSrcweir        my @tmpstr =  split "," , $languages;
940cdf0e10cSrcweir        for my $lang ( @tmpstr ){
941cdf0e10cSrcweir            if( $lang=~ /([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)(=([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*))?/ ){
942cdf0e10cSrcweir                $iso        = $1;
943cdf0e10cSrcweir                $fallback   = $4;
944cdf0e10cSrcweir
945cdf0e10cSrcweir                if( ( $iso && $iso=~ /(en-US)/i )  || ( $fallback && $fallback=~ /(en-US)/i ) ) {
946cdf0e10cSrcweir                    $bUseLocalize = "TRUE";
947cdf0e10cSrcweir                }
948cdf0e10cSrcweir                if( ( $iso && $iso=~ /(en-US)/i ) ) {
949cdf0e10cSrcweir                    $bHasSourceLanguage = "TRUE";
950cdf0e10cSrcweir                }
951cdf0e10cSrcweir             if( $fallback ) { $langhash{ $iso } = $fallback;   }
952cdf0e10cSrcweir             else            { $langhash{ $iso } = "";          }
953cdf0e10cSrcweir            }
954cdf0e10cSrcweir        }
955cdf0e10cSrcweir    }
956cdf0e10cSrcweir    ### -l de
957cdf0e10cSrcweir    else{
958cdf0e10cSrcweir        if( $languages=~ /([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)(=([a-zA-Z]{2,3}(-[a-zA-Z\-]*)*))?/ ){
959cdf0e10cSrcweir            $iso        = $1;
960cdf0e10cSrcweir            $fallback   = $4;
961cdf0e10cSrcweir
962cdf0e10cSrcweir            if( ( $iso && $iso=~ /(en-US)/i )  || ( $fallback && $fallback=~ /(en-US)/i ) ) {
963cdf0e10cSrcweir                $bUseLocalize = "TRUE";
964cdf0e10cSrcweir
965cdf0e10cSrcweir            }
966cdf0e10cSrcweir            if( ( $iso && $iso=~ /(en-US)/i )  ) {
967cdf0e10cSrcweir                $bHasSourceLanguage = "TRUE";
968cdf0e10cSrcweir            }
969cdf0e10cSrcweir
970cdf0e10cSrcweir             if( $fallback ) { $langhash{ $iso } = $fallback;   }
971cdf0e10cSrcweir             else            { $langhash{ $iso } = "";          }
972cdf0e10cSrcweir        }
973cdf0e10cSrcweir    }
974cdf0e10cSrcweir    # HACK en-US always needed!
975cdf0e10cSrcweir    if( !$bHasSourceLanguage ){
976cdf0e10cSrcweir        #$bHasSourceLanguage = "TRUE";
977cdf0e10cSrcweir        $bUseLocalize = "TRUE";
978cdf0e10cSrcweir        $bFakeEnglish = "TRUE";
979cdf0e10cSrcweir        $langhash{ "en-US" } = "";
980cdf0e10cSrcweir    }
981cdf0e10cSrcweir    return ( $bAll ,  $bUseLocalize , \%langhash , $bHasSourceLanguage, $bFakeEnglish);
982cdf0e10cSrcweir}
983cdf0e10cSrcweir
984cdf0e10cSrcweir#########################################################
985cdf0e10cSrcweirsub parse_options{
986cdf0e10cSrcweir
987cdf0e10cSrcweir    my $help;
988cdf0e10cSrcweir    my $merge;
989cdf0e10cSrcweir    my $extract;
990cdf0e10cSrcweir    my $success = GetOptions('f=s' => \$sdffile , 'l=s' => \$languages , 's=s' => \$srcpath ,  'h' => \$help , 'v' => \$bVerbose ,
991cdf0e10cSrcweir                             'm' => \$merge , 'e' => \$extract , 'x' => \$no_sort , 'd' => \$use_default_date );
992cdf0e10cSrcweir    $outputfile = $sdffile;
993cdf0e10cSrcweir
994cdf0e10cSrcweir    #print STDOUT "DBG: lang = $languages\n";
995cdf0e10cSrcweir    if( !$srcpath ){
996cdf0e10cSrcweir        #$srcpath = "$ENV{SRC_ROOT}";
997cdf0e10cSrcweir        if( !$srcpath ){
998cdf0e10cSrcweir	        print STDERR "No path to the source root found!\n\n";
999cdf0e10cSrcweir	        usage();
1000cdf0e10cSrcweir            exit(1);
1001cdf0e10cSrcweir        }
1002cdf0e10cSrcweir    }
1003cdf0e10cSrcweir    if( $help || !$success || $#ARGV > 1 || ( !$sdffile ) ){
1004cdf0e10cSrcweir        usage();
1005cdf0e10cSrcweir        exit(1);
1006cdf0e10cSrcweir    }
1007cdf0e10cSrcweir    if( $merge && $sdffile && ! ( -r $sdffile)){
1008cdf0e10cSrcweir        print STDERR "Can't open file '$sdffile'\n";
1009cdf0e10cSrcweir        exit(1);
1010cdf0e10cSrcweir    }
1011cdf0e10cSrcweir    if( !( $languages=~ /[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*(=[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)?(,[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*(=[a-zA-Z]{2,3}(-[a-zA-Z\-]*)*)?)*/ ) ){
1012cdf0e10cSrcweir        print STDERR "Please check the -l iso code\n";
1013cdf0e10cSrcweir        exit(1);
1014cdf0e10cSrcweir    }
1015cdf0e10cSrcweir    if( ( !$merge && !$extract ) || ( $merge && $extract ) ){ usage();exit( -1 );}
1016cdf0e10cSrcweir    if( $extract ){ $mode = "extract"; }
1017cdf0e10cSrcweir    else          { $mode = "merge";   }
1018cdf0e10cSrcweir}
1019cdf0e10cSrcweir
1020cdf0e10cSrcweir#########################################################
1021cdf0e10cSrcweirsub usage{
1022cdf0e10cSrcweir
1023cdf0e10cSrcweir    print STDERR "Usage: localize.pl\n";
1024cdf0e10cSrcweir    print STDERR "Split or collect SDF files\n";
1025cdf0e10cSrcweir    print STDERR "           merge: -m -f <sdffile>    -l l1[=f1][,l2[=f2]][...] [ -s <sourceroot> ]\n";
1026cdf0e10cSrcweir    print STDERR "         extract: -e -f <outputfile> -l <lang> [ -s <sourceroot> ] [-d]\n";
1027cdf0e10cSrcweir    print STDERR "Options:\n";
1028cdf0e10cSrcweir    print STDERR "    -h              help\n";
1029cdf0e10cSrcweir    print STDERR "    -m              Merge mode\n";
1030cdf0e10cSrcweir    print STDERR "    -e              Extract mode\n";
1031cdf0e10cSrcweir    print STDERR "    -f <sdffile>    To split a big SDF file into particles\n";
1032cdf0e10cSrcweir    print STDERR "       <outputfile> To collect and join all particles to one big file\n";
1033cdf0e10cSrcweir    print STDERR "    -s <sourceroot> Path to the modules, if no \$SRC_ROOT is set\n";
1034a893be29SPedro Giffuni    print STDERR "    -l ( all | <isocode> | <isocode>=fallback ) comma separated languages\n";
1035cdf0e10cSrcweir    print STDERR "    -d              Use default date in extracted sdf file\n";
1036cdf0e10cSrcweir    print STDERR "    -v              Verbose\n";
1037cdf0e10cSrcweir    print STDERR "\nExample:\n";
1038cdf0e10cSrcweir    print STDERR "\nlocalize -e -l en-US,pt-BR=en-US -f my.sdf\n( Extract en-US and pt-BR with en-US fallback )\n";
1039cdf0e10cSrcweir    print STDERR "\nlocalize -m -l cs -f my.sdf\n( Merge cs translation into the sourcecode ) \n";
1040cdf0e10cSrcweir}
1041cdf0e10cSrcweir
1042cdf0e10cSrcweir#            my $line           = defined $_ ? $_ : '';
1043cdf0e10cSrcweir#            my $leftpart       = defined $2 ? $2 : '';
1044cdf0e10cSrcweir#            my $prj            = defined $3 ? $3 : '';
1045cdf0e10cSrcweir#            my $file           = defined $4 ? $4 : '';
1046cdf0e10cSrcweir#            my $dummy          = defined $5 ? $5 : '';
1047cdf0e10cSrcweir#            my $type           = defined $6 ? $6 : '';
1048cdf0e10cSrcweir#            my $gid            = defined $7 ? $7 : '';
1049cdf0e10cSrcweir#            my $lid            = defined $8 ? $8 : '';
1050cdf0e10cSrcweir#            my $helpid         = defined $9 ? $9 : '';
10516d678023Smseidel#            my $platform       = defined $10 ? $10 : '';
1052cdf0e10cSrcweir#            my $width          = defined $11 ? $11 : '';
1053cdf0e10cSrcweir#            my $lang           = defined $12 ? $12 : '';
1054cdf0e10cSrcweir#            my $rightpart      = defined $13 ? $13 : '';
1055cdf0e10cSrcweir#            my $text           = defined $14 ? $14 : '';
1056cdf0e10cSrcweir#            my $helptext       = defined $15 ? $15 : '';
1057cdf0e10cSrcweir#            my $quickhelptext  = defined $16 ? $16 : '';
1058cdf0e10cSrcweir#            my $title          = defined $17 ? $17 : '';
1059cdf0e10cSrcweir#            my $timestamp      = defined $18 ? $18 : '';
1060cdf0e10cSrcweir
1061