update documentation for new wikipedia data

diff --git a/docs/admin/Import-and-Update.md b/docs/admin/Import-and-Update.md
index 757dab695d46ccf74963ea29b8ac938a0ab1b8df..a38d8eea22cbf2dce1fa653cda6890a5e6e07b56 100644 (file)
--- a/docs/admin/Import-and-Update.md
+++ b/docs/admin/Import-and-Update.md
@@ -33,7 +33,7 @@ the directory exists. There should be at least 40GB of free space.
 
 ## Downloading additional data
 
 
 ## Downloading additional data
 

-### Wikipedia rankings
+### Wikipedia/Wikidata rankings

 
 Wikipedia can be used as an optional auxiliary data source to help indicate
 the importance of OSM features. Nominatim will work without this information
 
 Wikipedia can be used as an optional auxiliary data source to help indicate
 the importance of OSM features. Nominatim will work without this information


@@ -41,15 +41,14 @@ but it will improve the quality of the results if this is installed.
 This data is available as a binary download:
 
     cd $NOMINATIM_SOURCE_DIR/data
 This data is available as a binary download:
 
     cd $NOMINATIM_SOURCE_DIR/data

-    wget https://www.nominatim.org/data/wikipedia_article.sql.bin
-    wget https://www.nominatim.org/data/wikipedia_redirect.sql.bin
+    wget https://www.nominatim.org/data/wikimedia_importance.sql.gz

 
 

-Combined the 2 files are around 1.5GB and add around 30GB to the install
+The file is about 1GB and it adds around 10GB to the install

 size of Nominatim. They also increase the install time by an hour or so.
 
 size of Nominatim. They also increase the install time by an hour or so.
 

-*NOTE:* you'll need to download the Wikipedia rankings before performing
-the initial import of the data if you want the rankings applied to the
-loaded data.
+*NOTE:* if you forgot to download the wikipedia rankings, you can also add
+them after the import by running `./utils/setup.php --import-wikipedia-articles`
+and then `./utils/update.php --recompute-importance`.

 
 ### Great Britain, USA postcodes
 
 
 ### Great Britain, USA postcodes
 

diff --git a/docs/admin/Migration.md b/docs/admin/Migration.md
index f3668357fd1ffa36b5f595710f4e53d781bb1526..e6b6d1026e215414538d8d7d64d9ec3e90c63c06 100644 (file)
--- a/docs/admin/Migration.md
+++ b/docs/admin/Migration.md
@@ -6,6 +6,21 @@ to newer versions of Nominatim.
 SQL statements should be executed from the PostgreSQL commandline. Execute
 `psql nominatim` to enter command line mode.
 
 SQL statements should be executed from the PostgreSQL commandline. Execute
 `psql nominatim` to enter command line mode.
 

+## 3.4.0 -> master
+
+### New Wikipedia/Wikidata importance tables
+
+The `wikipedia_*` tables have a new format that also includes references to
+Wikidata. You need to update the computation functions and the tables as
+follows:
+
+  * download the new Wikipedia tables as described in the import section
+  * reimport the tables: `./utils/setup.php --import-wikipedia-articles`
+  * update the functions: `./utils/setup.php --create-functions --enable-diff-updates`
+  * compute importance: `./utils/update.php --recompute-importance`
+
+The last step takes about 10 hours on the full planet.
+

 ## 3.3.0 -> 3.4.0
 
 ### Reorganisation of location_area_country table
 ## 3.3.0 -> 3.4.0
 
 ### Reorganisation of location_area_country table

diff --git a/lib/setup/SetupClass.php b/lib/setup/SetupClass.php
index 818aeeb7d1b5cdb083d025cd9c3790b77e4c292d..c1c15d9a5f846384db100800b75ccb6fadd937fd 100755 (executable)
--- a/lib/setup/SetupClass.php
+++ b/lib/setup/SetupClass.php
@@ -323,19 +323,12 @@ class SetupFunctions
 
     public function importWikipediaArticles()
     {
 
     public function importWikipediaArticles()
     {

-        $sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikipedia_article.sql.bin';
-        $sWikiRedirectsFile = CONST_Wikipedia_Data_Path.'/wikipedia_redirect.sql.bin';
+        $sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikimedia_importance.sql.gz';

         if (file_exists($sWikiArticlesFile)) {
         if (file_exists($sWikiArticlesFile)) {

-            info('Importing wikipedia articles');
+            info('Importing wikipedia articles and redirects');

             $this->pgsqlRunDropAndRestore($sWikiArticlesFile);
         } else {
             $this->pgsqlRunDropAndRestore($sWikiArticlesFile);
         } else {

-            warn('wikipedia article dump file not found - places will have default importance');
-        }
-        if (file_exists($sWikiRedirectsFile)) {
-            info('Importing wikipedia redirects');
-            $this->pgsqlRunDropAndRestore($sWikiRedirectsFile);
-        } else {
-            warn('wikipedia redirect dump file not found - some place importance values may be missing');
+            warn('wikipedia importance dump file not found - places will have default importance');

         }
     }
 
         }
     }
 

diff --git a/utils/update.php b/utils/update.php
index 31c911ccccb85b737d26f88a07e5c9d9f1f23bef..735f4638a7a8a59f758f3db9783a0c647f8fead0 100644 (file)
--- a/utils/update.php
+++ b/utils/update.php
@@ -322,7 +322,7 @@ if ($aResult['update-address-levels']) {
 }
 
 if ($aResult['recompute-importance']) {
 }
 
 if ($aResult['recompute-importance']) {

-    echo 'Updating importance values for database.\n';
+    echo "Updating importance values for database.\n";

     $oDB = new Nominatim\DB();
     $oDB->connect();
 
     $oDB = new Nominatim\DB();
     $oDB->connect();