Add support for compressed request bodies

[rails.git] / script / misc / update-wiki-pages

diff --git a/script/misc/update-wiki-pages b/script/misc/update-wiki-pages
old mode 100644 (file)
new mode 100755 (executable)
index 15dd1b6..0bef30d
--- a/script/misc/update-wiki-pages
+++ b/script/misc/update-wiki-pages
@@ -1,4 +1,5 @@
 #!/usr/bin/env perl
 #!/usr/bin/env perl

+use 5.010;

 use strict;
 use warnings;
 
 use strict;
 use warnings;
 


@@ -19,19 +20,15 @@ use Test::More 'no_plan';
 
 =head1 NAME
 
 
 =head1 NAME
 

-update-wiki-pages - Screen-scrape the wiki for key/value wiki description pages
+update-wiki-pages - Scrape the wiki for key/value wiki description pages

 
 =head1 SYNOPSIS
 
     perl script/misc/update-wiki-pages config/wiki_pages.yml 
 
 
 =head1 SYNOPSIS
 
     perl script/misc/update-wiki-pages config/wiki_pages.yml 
 

-=head1 BUGS
+Or with prove(1):

 
 

-This will break if there are more than 500 key or value pages. Paging
-needs to be implemenented.
-
-That or using a proper API or something (if it's there) or making a
-direct query to the wiki database.
+    prove -e 'perl script/misc/update-wiki-pages' config/wiki_pages.yml

 
 =cut
 
 
 =cut
 


@@ -45,7 +42,10 @@ Getopt::Long::Parser->new(
 # On --help
 help() if $help;
 
 # On --help
 help() if $help;
 

-help() unless $ARGV[0];
+my $out_file = $ARGV[0];
+$out_file //= 'config/wiki_pages.yml';
+
+help() unless -f $out_file;

 
 # Get a API interface
 my $mw = MediaWiki::API->new();
 
 # Get a API interface
 my $mw = MediaWiki::API->new();


@@ -63,12 +63,13 @@ for my $lang ('', map { "${_}:" } qw[ Pt Fi De It HU Cz Fr RU Pl ]) {
     # Key pages
     ok(1, "    Getting key pages");
     my $cnt = stick_content_in_hash("key", "Template:${lang}KeyDescription", \%feature);
     # Key pages
     ok(1, "    Getting key pages");
     my $cnt = stick_content_in_hash("key", "Template:${lang}KeyDescription", \%feature);

+    $cnt += stick_content_in_hash("key", "Template:${lang}Feature", \%feature);

     ok(1, "    Got $cnt key pages");
     $count{key} += $cnt;
 
     # Value pages
     ok(1, "    Getting value pages");
     ok(1, "    Got $cnt key pages");
     $count{key} += $cnt;
 
     # Value pages
     ok(1, "    Getting value pages");

-    my $cnt = stick_content_in_hash("tag", "Template:${lang}ValueDescription", \%feature);
+    $cnt = stick_content_in_hash("tag", "Template:${lang}ValueDescription", \%feature);

     ok(1, "    Got $cnt value pages");
     $count{value} += $cnt;
 }
     ok(1, "    Got $cnt value pages");
     $count{value} += $cnt;
 }


@@ -76,7 +77,7 @@ for my $lang ('', map { "${_}:" } qw[ Pt Fi De It HU Cz Fr RU Pl ]) {
 ok(1, "Got a total of $count{$_} ${_}s") for qw[ key value ];
 
 # Dump to .yml file
 ok(1, "Got a total of $count{$_} ${_}s") for qw[ key value ];
 
 # Dump to .yml file

-open my $out, ">", $ARGV[0] or die "Can't open file '$ARGV[0]' supplied on the command line";
+open my $out, ">", $out_file or die "Can't open file '$out_file' supplied on the command line";

 say $out "# THIS FILE IS AUTOGENERATED WITH THE script/misc/update-wiki-pages";
 say $out "# PROGRAM DO NOT MANUALLY EDIT IT";
 say $out "";
 say $out "# THIS FILE IS AUTOGENERATED WITH THE script/misc/update-wiki-pages";
 say $out "# PROGRAM DO NOT MANUALLY EDIT IT";
 say $out "";


@@ -97,29 +98,54 @@ sub stick_content_in_hash
     };
 
     my $count = 0;
     };
 
     my $count = 0;

+
+    my $process_link = sub {
+        my $link = shift;
+        $count++;
+        ok(1, "    ... got $count links") if $count % 200 == 0;
+        my $title = $link->{title};
+        my $lang;
+        my $key_name;
+        if ($title =~ /^$ukey:(?<key_name>.*?)$/) {
+            # English by default
+            $lang = "en";
+            $key_name = $space_to_underscore->($+{key_name});
+        } elsif ($title =~ /^(?<lang>[^:]+):$ukey:(?<key_name>.*?)$/) {
+            $lang = lc $+{lang};
+            $key_name = $space_to_underscore->($+{key_name});
+        }
+        if ($lang && !exists($hash->{$lang}->{$key}->{$key_name})) {
+            $hash->{$lang}->{$key}->{$key_name} = $title;
+        }
+    };
+

     get_embeddedin(
         $title,
         sub {
     get_embeddedin(
         $title,
         sub {

-            my ($links) = @_;
-            my (@links) = @$links;
-            ok(1, "    ... got " . scalar(@links) . " more links");
-            for my $link (@links) {
-                $count++;
-                my $title = $link->{title};
-
-                if ($title =~ /^$ukey:(?<key_name>.*?)$/) {
-                    # English by default
-                    $hash->{en}->{$key}->{ $space_to_underscore->($+{key_name}) } = $title;
-                } elsif ($title =~ /^(?<lang>[^:]+):$ukey:(?<key_name>.*?)$/) {
-                    $hash->{lc $+{lang}}->{$key}->{ $space_to_underscore->($+{key_name}) } = $title;
+            my $link = shift;
+            $process_link->($link);
+            get_redirects(
+                $link->{title},
+                sub {
+                    my $link = shift;
+                    $process_link->($link) if exists($link->{redirect});

                 }
                 }

-            }
+            );

         }
     );
 
     return $count;
 }
 
         }
     );
 
     return $count;
 }
 

+sub process_list
+{
+    my $callback = shift;
+    my $links = shift;
+    for my $link (@$links) {
+        $callback->($link);
+    }
+}
+

 sub get_embeddedin
 {
     my ($title, $callback) = @_;
 sub get_embeddedin
 {
     my ($title, $callback) = @_;


@@ -135,7 +161,28 @@ sub get_embeddedin
         },
         {
             max => '0',
         },
         {
             max => '0',

-            hook => $callback,
+            hook => sub { process_list($callback, @_) },
+            skip_encoding => 1,
+        }
+    ) || die $mw->{error}->{code} . ': ' . $mw->{error}->{details};
+}
+
+sub get_redirects
+{
+    my ($title, $callback) = @_;
+    my $articles = $mw->list(
+        {
+            action => 'query',
+            list => 'backlinks',
+            bltitle => $title,
+            blfilterredir => 'redirects',
+            # Doesn't work for De:* and anything non-en. Odd.
+            # einamespace => '0|8',
+            bllimit => '200',
+        },
+        {
+            max => '0',
+            hook => sub { process_list($callback, @_) },

             skip_encoding => 1,
         }
     ) || die $mw->{error}->{code} . ': ' . $mw->{error}->{details};
             skip_encoding => 1,
         }
     ) || die $mw->{error}->{code} . ': ' . $mw->{error}->{details};