X-Git-Url: https://git.openstreetmap.org/rails.git/blobdiff_plain/1facda11d6a1dbcbc1290eb447fcb0df7c24d6d5..b479b0c2b6b6d82942c0efbd4b0633e859a1ed41:/script/locale/merge-from-translatewiki diff --git a/script/locale/merge-from-translatewiki b/script/locale/merge-from-translatewiki index 83a66b246..afb3fca92 100644 --- a/script/locale/merge-from-translatewiki +++ b/script/locale/merge-from-translatewiki @@ -23,6 +23,7 @@ merge-from-translatewiki - Get new translations from L =head1 SYNOPSIS + # Run this normally, hopefully... merge-from-translatewiki --locales-dir=config/locales # Diff the existing files: @@ -52,6 +53,10 @@ Print this help message. The locales dir we'll merge stuff into. E.g. C. +=item --only-new + +Only import translations that don't exists for us yet. + =back =head1 AUTHOR @@ -66,6 +71,7 @@ Getopt::Long::Parser->new( )->getoptions( 'h|help' => \my $help, 'locales-dir=s' => \my $locales_dir, + 'only-new' => \my $only_new, ) or help(); # On --help @@ -104,7 +110,7 @@ for my $my_yaml_file (@my_yaml_files) { say "loaded my translations"; ## Write out merged data -for my $translatewiki_lang (@translatewiki_languages_codes) { +for my $translatewiki_lang (sort @translatewiki_languages_codes) { my $rails_lang = $translatewiki_lang; $rails_lang =~ s/(?<=-)(\w+)/\U$1\E/; my $out_file = catfile($locales_dir, $rails_lang . '.yml'); @@ -114,19 +120,32 @@ for my $translatewiki_lang (@translatewiki_languages_codes) { my $expanded = expand_hash($translatewiki_translations{$translatewiki_lang}); my $out = +{ $rails_lang => $expanded }; spit_out($out_file, $out); - } elsif (ref $my_translations{$translatewiki_lang} eq 'HASH') { - say STDERR "$rails_lang has existing translations. Merging"; + } elsif (ref $my_translations{$translatewiki_lang} eq 'HASH' and not $only_new) { + say STDERR "$rails_lang has existing translations. Merging the old translation with the new Translatewiki one"; # Get the data my %tw = %{ $translatewiki_translations{$translatewiki_lang} }; my %me = %{ $my_translations{$translatewiki_lang} }; + my %en = %{ $my_translations{en} }; # Use %tw to start with my %new = %tw; ### Merge stuff - # These keys shouldn't be removed + ## These keys shouldn't be removed but are due to + ## Translatewiki fail (they were missing in the original + ## import) my @url_keys = qw( + browse.relation_member.entry + changeset.changeset.id + geocoder.search_osm_namefinder.suffix_suburb + html.dir + layouts.intro_3_bytemark + layouts.intro_3_ucl + layouts.project_name.h1 + layouts.project_name.title + printable_name.with_version + site.edit.anon_edits layouts.help_wiki_url layouts.shop_url notifier.gpx_notification.failure.import_failures_url @@ -139,14 +158,45 @@ for my $translatewiki_lang (@translatewiki_languages_codes) { for my $key (@url_keys) { if ( exists $me{$key} and not exists $new{$key} ) { - $new{$key} = $me{$key}; + $new{$key} = $me{$key} if $me{$key} ne $en{$key}; + } + } + + ## When foo exists in this file but only foo.one, foo,other + ## etc in English or the original file we don't want to throw away what we have + my @plural_keys = qw( zero one many few other two ); + + while (my ($me_k, $me_v) = each %me) { + if (not exists $tw{ $me_k } and + not exists $en{ $me_k } and + ( + exists $en{ $me_k . '.zero' } or + exists $en{ $me_k . '.one' } or + exists $en{ $me_k . '.many' } or + exists $en{ $me_k . '.few' } or + exists $en{ $me_k . '.other' } or + exists $en{ $me_k . '.two' })) { + #say STDERR "Bringing back nuked plural form '$me_k' Setting it to '$me{ $me_k }'"; + $new{ $me_k } = $me{ $me_k }; + } + } + + # Both arrays and strings are supported in the site key. Avoid removing e.g.: + # -site.key.table.entry.school: 學校；大學 + # Just because en.yml has site.key.table.entry.school.0 and site.key.table.entry.school.1 + while (my ($me_k, $me_v) = each %me) { + next unless $me_k =~ /^site\.key\.table\.entry/; + next if $me_k =~ /\.\d+$/; + + if (ref $en{ $me_k } eq 'ARRAY' and not ref $me{ $me_k }) { + $new{ $me_k } = $me{ $me_k }; } } my $expanded = expand_hash( \%new ); my $out = +{ $rails_lang => $expanded }; spit_out($out_file, $out); - } else { + } elsif (not $only_new) { die "Internal error on $translatewiki_lang"; } } @@ -157,7 +207,6 @@ sub spit_out my $yaml_out = Dump $data; open my $fh, ">", $file; - say $fh "# Imported at " . (scalar localtime) . " from Translatewiki.net"; print $fh $yaml_out; close $fh; } @@ -199,16 +248,46 @@ sub expand_hash my ($flat_hash) = @_; my %new_hash; while (my ($k, $v) = each %$flat_hash) { + #say "Inserting $k=$v"; insert_string_deep(\%new_hash, $k, $v); } \%new_hash; } -sub insert_string_deep { - my ($h, $ks, $v) = @_; - my $p = \$h; $p = \$$p->{$_} for split /\./, $ks; - $$p = $v; +# Fails under strict in certain cases: +## Inserting browse.start_rjs.object_list.history.type.way=Vía [[id]] +## Inserting activerecord.models.relation_tag=Etiqueta de la relación +## Inserting browse.changeset_details.has_nodes.one=Tiene el siguiente {{count}} nodo: +## Can't use string ("Tiene {{count}} nodos:") as a HASH ref while "strict refs" in use at script/locale/merge-from-translatewiki line 234. +# Line 234 = my $p = \$h; $p = \$$p->{$_} for split /\./, $ks; + +# sub insert_string_deep_X { +# my ($h, $ks, $v) = @_; +# my $p = \$h; $p = \$$p->{$_} for split /\./, $ks; +# $$p = $v; +# } + +sub insert_string_deep +{ + my ($hash, $key, $value) = @_; + + my @key = split /\./, $key; + my $h = $hash; + + my $i = 0; + for my $k (@key) { + $i ++; + if ($i == @key) { + $h->{$k} = $value; + } else { + if (ref $h->{$k}) { + $h = $h->{$k}; + } else { + $h = $h->{$k} = {}; + } + } + } } # @@ -226,9 +305,12 @@ sub get_translatewiki_translations } my %translatewiki_languages; - say "All languages are: @languages"; + my $all_count = scalar @languages; + say "Translatewiki has $all_count languages I'm about to get"; + my $count = 0; for my $lang (@languages) { - say STDERR "Getting language $lang from Translatewiki"; + $count ++; + say STDERR "Getting language $count/$all_count ($lang) from Translatewiki"; my $yaml = get_language_from_translatewiki($lang); my $flat_data = load_and_flatten_yaml($yaml);