## Downloading additional data
-### Wikipedia rankings
+### Wikipedia/Wikidata rankings
Wikipedia can be used as an optional auxiliary data source to help indicate
the importance of OSM features. Nominatim will work without this information
This data is available as a binary download:
cd $NOMINATIM_SOURCE_DIR/data
- wget https://www.nominatim.org/data/wikipedia_article.sql.bin
- wget https://www.nominatim.org/data/wikipedia_redirect.sql.bin
+ wget https://www.nominatim.org/data/wikimedia-importance.sql.gz
-Combined the 2 files are around 1.5GB and add around 30GB to the install
-size of Nominatim. They also increase the install time by an hour or so.
+The file is about 400MB and adds around 4GB to Nominatim database.
-*NOTE:* you'll need to download the Wikipedia rankings before performing
-the initial import of the data if you want the rankings applied to the
-loaded data.
+*NOTE:* if you forgot to download the wikipedia rankings, you can also add
+them after the import by running `./utils/setup.php --import-wikipedia-articles`
+and then `./utils/update.php --recompute-importance`.
### Great Britain, USA postcodes
SQL statements should be executed from the PostgreSQL commandline. Execute
`psql nominatim` to enter command line mode.
+## 3.4.0 -> master
+
+### New Wikipedia/Wikidata importance tables
+
+The `wikipedia_*` tables have a new format that also includes references to
+Wikidata. You need to update the computation functions and the tables as
+follows:
+
+ * download the new Wikipedia tables as described in the import section
+ * reimport the tables: `./utils/setup.php --import-wikipedia-articles`
+ * update the functions: `./utils/setup.php --create-functions --enable-diff-updates`
+ * compute importance: `./utils/update.php --recompute-importance`
+
+The last step takes about 10 hours on the full planet.
+
## 3.3.0 -> 3.4.0
### Reorganisation of location_area_country table
if ($this->bNoPartitions) {
$this->pgsqlRunScript('update country_name set partition = 0');
}
-
- // the following will be needed by createFunctions later but
- // is only defined in the subsequently called createTables
- // Create dummies here that will be overwritten by the proper
- // versions in create-tables.
- $this->pgsqlRunScript('CREATE TABLE IF NOT EXISTS place_boundingbox ()');
- $this->pgsqlRunScript('CREATE TYPE wikipedia_article_match AS ()', false);
}
public function importData($sOSMFile)
public function importWikipediaArticles()
{
- $sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikipedia_article.sql.bin';
- $sWikiRedirectsFile = CONST_Wikipedia_Data_Path.'/wikipedia_redirect.sql.bin';
+ $sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikimedia-importance.sql.gz';
if (file_exists($sWikiArticlesFile)) {
- info('Importing wikipedia articles');
- $this->pgsqlRunDropAndRestore($sWikiArticlesFile);
- } else {
- warn('wikipedia article dump file not found - places will have default importance');
- }
- if (file_exists($sWikiRedirectsFile)) {
- info('Importing wikipedia redirects');
- $this->pgsqlRunDropAndRestore($sWikiRedirectsFile);
+ info('Importing wikipedia articles and redirects');
+ $this->pgExec('DROP TABLE IF EXISTS wikipedia_article');
+ $this->pgExec('DROP TABLE IF EXISTS wikipedia_redirect');
+ $this->pgsqlRunScriptFile($sWikiArticlesFile);
} else {
- warn('wikipedia redirect dump file not found - some place importance values may be missing');
+ warn('wikipedia importance dump file not found - places will have default importance');
}
}
echo '.';
$this->pgExec('TRUNCATE place_addressline');
echo '.';
- $this->pgExec('TRUNCATE place_boundingbox');
- echo '.';
$this->pgExec('TRUNCATE location_area');
echo '.';
if (!$this->dbReverseOnly()) {
}
}
- private function pgsqlRunDropAndRestore($sDumpFile)
- {
- $sCMD = 'pg_restore'
- .' -p '.escapeshellarg($this->aDSNInfo['port'])
- .' -d '.escapeshellarg($this->aDSNInfo['database'])
- .' --no-owner -Fc --clean '.escapeshellarg($sDumpFile);
- if ($this->oDB->getPostgresVersion() >= 9.04) {
- $sCMD .= ' --if-exists';
- }
- if (isset($this->aDSNInfo['hostspec'])) {
- $sCMD .= ' -h '.escapeshellarg($this->aDSNInfo['hostspec']);
- }
- if (isset($this->aDSNInfo['username'])) {
- $sCMD .= ' -U '.escapeshellarg($this->aDSNInfo['username']);
- }
-
- $this->runWithPgEnv($sCMD);
- }
-
private function pgsqlRunScript($sScript, $bfatal = true)
{
runSQLScript(
END LOOP;
NEW.importance := null;
- select language||':'||title,importance from get_wikipedia_match(NEW.extratags, NEW.country_code) INTO NEW.wikipedia,NEW.importance;
- IF NEW.importance IS NULL THEN
- select language||':'||title,importance from wikipedia_article where osm_type = NEW.osm_type and osm_id = NEW.osm_id order by importance desc limit 1 INTO NEW.wikipedia,NEW.importance;
- END IF;
+ SELECT wikipedia, importance
+ FROM compute_importance(NEW.extratags, NEW.country_code, NEW.osm_type, NEW.osm_id)
+ INTO NEW.wikipedia,NEW.importance;
--DEBUG: RAISE WARNING 'Importance computed from wikipedia: %', NEW.importance;
-- mark the linked place (excludes from search results)
UPDATE placex set linked_place_id = NEW.place_id where place_id = linkedPlacex.place_id;
- -- keep a note of the node id in case we need it for wikipedia in a bit
- linked_node_id := linkedPlacex.osm_id;
- select language||':'||title,importance from get_wikipedia_match(linkedPlacex.extratags, NEW.country_code) INTO linked_wikipedia,linked_importance;
+ select wikipedia, importance
+ FROM compute_importance(linkedPlacex.extratags, NEW.country_code,
+ 'N', linkedPlacex.osm_id)
+ INTO linked_wikipedia,linked_importance;
--DEBUG: RAISE WARNING 'Linked label member';
END LOOP;
-- mark the linked place (excludes from search results)
UPDATE placex set linked_place_id = NEW.place_id where place_id = linkedPlacex.place_id;
- -- keep a note of the node id in case we need it for wikipedia in a bit
- linked_node_id := linkedPlacex.osm_id;
- select language||':'||title,importance from get_wikipedia_match(linkedPlacex.extratags, NEW.country_code) INTO linked_wikipedia,linked_importance;
+ select wikipedia, importance
+ FROM compute_importance(linkedPlacex.extratags, NEW.country_code,
+ 'N', linkedPlacex.osm_id)
+ INTO linked_wikipedia,linked_importance;
--DEBUG: RAISE WARNING 'Linked admin_center';
END IF;
-- mark the linked place (excludes from search results)
UPDATE placex set linked_place_id = NEW.place_id where place_id = linkedPlacex.place_id;
- -- keep a note of the node id in case we need it for wikipedia in a bit
- linked_node_id := linkedPlacex.osm_id;
- select language||':'||title,importance from get_wikipedia_match(linkedPlacex.extratags, NEW.country_code) INTO linked_wikipedia,linked_importance;
+ select wikipedia, importance
+ FROM compute_importance(linkedPlacex.extratags, NEW.country_code,
+ 'N', linkedPlacex.osm_id)
+ INTO linked_wikipedia,linked_importance;
--DEBUG: RAISE WARNING 'Linked named place';
END LOOP;
END IF;
(NEW.importance is null or NEW.importance < linked_importance) THEN
NEW.importance = linked_importance;
END IF;
-
- -- Still null? how about looking it up by the node id
- IF NEW.importance IS NULL THEN
- --DEBUG: RAISE WARNING 'Looking up importance by linked node id';
- select language||':'||title,importance from wikipedia_article where osm_type = 'N'::char(1) and osm_id = linked_node_id order by importance desc limit 1 INTO NEW.wikipedia,NEW.importance;
- END IF;
-
END IF;
-- make sure all names are in the word table
$$
LANGUAGE plpgsql IMMUTABLE;
-DROP TYPE wikipedia_article_match CASCADE;
+DROP TYPE IF EXISTS wikipedia_article_match CASCADE;
create type wikipedia_article_match as (
language TEXT,
title TEXT,
$$
LANGUAGE plpgsql;
+DROP TYPE IF EXISTS place_importance CASCADE;
+create type place_importance as (
+ importance FLOAT,
+ wikipedia TEXT
+);
+
+CREATE OR REPLACE FUNCTION compute_importance(extratags HSTORE, country_code varchar(2), osm_type varchar(1), osm_id BIGINT)
+ RETURNS place_importance
+ AS $$
+DECLARE
+ match RECORD;
+ result place_importance;
+BEGIN
+ FOR match IN SELECT * FROM get_wikipedia_match(extratags, country_code)
+ WHERE language is not NULL
+ LOOP
+ result.importance := match.importance;
+ result.wikipedia := match.language || ':' || match.title;
+ RETURN result;
+ END LOOP;
+
+ IF extratags ? 'wikidata' THEN
+ FOR match IN SELECT * FROM wikipedia_article
+ WHERE wd_page_title = extratags->'wikidata'
+ ORDER BY language = 'en' DESC, langcount DESC LIMIT 1 LOOP
+ result.importance := match.importance;
+ result.wikipedia := match.language || ':' || match.title;
+ RETURN result;
+ END LOOP;
+ END IF;
+
+ RETURN null;
+END;
+$$
+LANGUAGE plpgsql;
+
CREATE OR REPLACE FUNCTION quad_split_geometry(geometry GEOMETRY, maxarea FLOAT, maxdepth INTEGER)
RETURNS SETOF GEOMETRY
AS $$
lon double precision,
importance double precision,
osm_type character(1),
- osm_id bigint
+ osm_id bigint,
+ wd_page_title text,
+ instance_of text
);
ALTER TABLE ONLY wikipedia_article ADD CONSTRAINT wikipedia_article_pkey PRIMARY KEY (language, title);
CREATE INDEX idx_wikipedia_article_osm_id ON wikipedia_article USING btree (osm_type, osm_id);
| en |
Then result addresses contain
| ID | state |
- | 0 | Salto |
+ | 0 | Florida |
When sending json search query "25 de Mayo" with address
| accept-language | viewbox |
- | en | -56.35879,-34.18330,-56.31618,-34.20815 |
+ | en | -57.95468,-31.39261,-57.94741,-31.39490 |
Then result addresses contain
| ID | state |
- | 0 | Florida |
+ | 0 | Salto |
Scenario: viewboxes cannot be points
When sending json search query "foo"
array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'),
array('update-address-levels', '', 0, 1, 0, 0, 'bool', 'Reimport address level configuration (EXPERT)'),
+ array('recompute-importance', '', 0, 1, 0, 0, 'bool', 'Recompute place importances'),
array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'),
);
$oAlParser->createTable($oDB, 'address_levels');
}
+if ($aResult['recompute-importance']) {
+ echo "Updating importance values for database.\n";
+ $oDB = new Nominatim\DB();
+ $oDB->connect();
+
+ $sSQL = 'ALTER TABLE placex DISABLE TRIGGER ALL;';
+ $sSQL .= 'UPDATE placex SET (wikipedia, importance) =';
+ $sSQL .= ' (SELECT wikipedia, importance';
+ $sSQL .= ' FROM compute_importance(extratags, country_code, osm_type, osm_id));';
+ $sSQL .= 'UPDATE placex s SET wikipedia = d.wikipedia, importance = d.importance';
+ $sSQL .= ' FROM placex d';
+ $sSQL .= ' WHERE s.place_id = d.linked_place_id and d.wikipedia is not null';
+ $sSQL .= ' and (s.wikipedia is null or s.importance < d.importance);';
+ $sSQL .= 'ALTER TABLE placex ENABLE TRIGGER ALL;';
+ $oDB->exec($sSQL);
+}
+
if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
//
if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) {
projects / nominatim.git / commitdiff