#!/usr/bin/env perl use strict; use warnings; use Pod::Usage (); use Getopt::Long (); BEGIN { eval "require MediaWiki::API; require YAML::XS;" or do { print "You have to install some modules via CPAN to run this:\n"; print " sudo cpanp MediaWiki::API YAML::XS\n"; exit 1; }; } use MediaWiki::API; use YAML::XS qw(Dump); use Test::More 'no_plan'; =head1 NAME update-key-and-tag-description-pages-from-wiki - Screen-scrape the wiki for key/value wiki description pages =head1 SYNOPSIS perl script/misc/update-key-and-tag-description-pages-from-wiki config/wiki-tag-and-key-description.yml =head1 BUGS This will break if there are more than 500 key or value pages. Paging needs to be implemenented. That or using a proper API or something (if it's there) or making a direct query to the wiki database. =cut # Get the command-line options Getopt::Long::Parser->new( config => [ qw< bundling no_ignore_case no_require_order pass_through > ], )->getoptions( 'h|help' => \my $help, ) or help(); # On --help help() if $help; help() unless $ARGV[0]; # Get a API interface my $mw = MediaWiki::API->new(); ok($mw, "Got a MediaWiki API"); $mw->{config}->{api_url} = 'http://wiki.openstreetmap.org/w/api.php'; # All our goodies my (%feature, %count); # This is what you get on: ## http://wiki.openstreetmap.org/w/index.php?search=Template:KeyDescription&fulltext=Search&fulltext=Search for my $lang ('', map { "${_}:" } qw[ Pt Fi De It HU Cz Fr RU Pl ]) { ok(1, " Templates for language '$lang'"); # Key pages ok(1, " Getting key pages"); my $cnt = stick_content_in_hash("key", "Template:${lang}KeyDescription", \%feature); ok(1, " Got $cnt key pages"); $count{key} += $cnt; # Value pages ok(1, " Getting value pages"); my $cnt = stick_content_in_hash("tag", "Template:${lang}ValueDescription", \%feature); ok(1, " Got $cnt value pages"); $count{value} += $cnt; } ok(1, "Got a total of $count{$_} ${_}s") for qw[ key value ]; # Dump to .yml file open my $out, ">", $ARGV[0] or die "Can't open file '$ARGV[0]' supplied on the command line"; say $out "# THIS FILE IS AUTOGENERATED WITH THE script/misc/update-key-and-tag-description-pages-from-wiki"; say $out "# PROGRAM DO NOT MANUALLY EDIT IT"; say $out ""; say $out Dump(\%feature); close $out; exit 0; sub stick_content_in_hash { my ($key, $title, $hash) = @_; my $ukey = ucfirst $key; my $space_to_underscore = sub { my $txt = shift; $txt =~ s/ /_/g; $txt; }; my $count = 0; get_embeddedin( $title, sub { my ($links) = @_; my (@links) = @$links; ok(1, " ... got " . scalar(@links) . " more links"); for my $link (@links) { $count++; my $title = $link->{title}; if ($title =~ /^$ukey:(?.*?)$/) { # English by default $hash->{en}->{$key}->{ $space_to_underscore->($+{key_name}) } = $title; } elsif ($title =~ /^(?[^:]+):$ukey:(?.*?)$/) { $hash->{lc $+{lang}}->{$key}->{ $space_to_underscore->($+{key_name}) } = $title; } } } ); return $count; } sub get_embeddedin { my ($title, $callback) = @_; my $articles = $mw->list( { action => 'query', list => 'embeddedin', eititle => $title, eifilterredir => 'nonredirects', # Doesn't work for De:* and anything non-en. Odd. # einamespace => '0|8', eilimit => '200', }, { max => '0', hook => $callback, skip_encoding => 1, } ) || die $mw->{error}->{code} . ': ' . $mw->{error}->{details}; } sub help { my %arg = @_; Pod::Usage::pod2usage( -verbose => $arg{ verbose }, -exitval => $arg{ exitval } || 0, ); }