download() {
echo "Downloading $1"
- wget --quiet --no-clobber --tries 3
+ wget --quiet --no-clobber --tries 3 "$1"
}
# languages to process (refer to List of Wikipedias here: https://en.wikipedia.org/wiki/List_of_Wikipedias)
#!/bin/bash
psqlcmd() {
- psql --quiet wikiprocessingdb
+ psql --quiet wikiprocessingdb |& \
+ grep -v 'does not exist, skipping' |& \
+ grep -v 'violates check constraint' |& \
+ grep -vi 'Failing row contains'
}
mysql2pgsqlcmd() {
- ./mysql2pgsql.perl /dev/stdin /dev/stdout
+ ./mysql2pgsql.perl --nodrop /dev/stdin /dev/stdout
}
download() {
echo "Downloading $1"
- wget --quiet --no-clobber --tries 3
+ wget --quiet --no-clobber --tries=3 "$1"
}
do
echo "Language: $i"
+ # We pre-create the table schema. This allows us to
+ # 1. Skip index creation. Most queries we do are full table scans
+ # 2. Add constrain to only import namespace=0 (wikipedia articles)
+ # Both cuts down data size considerably (50%+)
+
echo "Importing ${i}wiki-latest-pagelinks"
- gzip -dc ${i}wiki-latest-pagelinks.sql.gz | sed "s/\`pagelinks\`/\`${i}pagelinks\`/g" | mysql2pgsqlcmd | psqlcmd
+
+ echo "DROP TABLE IF EXISTS ${i}pagelinks;" | psqlcmd
+ echo "CREATE TABLE ${i}pagelinks (
+ pl_from int NOT NULL DEFAULT '0',
+ pl_namespace int NOT NULL DEFAULT '0',
+ pl_title text NOT NULL DEFAULT '',
+ pl_from_namespace int NOT NULL DEFAULT '0'
+ );" | psqlcmd
+
+ time \
+ gzip -dc ${i}wiki-latest-pagelinks.sql.gz | \
+ sed "s/\`pagelinks\`/\`${i}pagelinks\`/g" | \
+ mysql2pgsqlcmd | \
+ grep -v '^CREATE INDEX ' | \
+ psqlcmd
+
+
+
echo "Importing ${i}wiki-latest-page"
- gzip -dc ${i}wiki-latest-page.sql.gz | sed "s/\`page\`/\`${i}page\`/g" | mysql2pgsqlcmd | psqlcmd
+
+ # autoincrement serial8 4byte
+ echo "DROP TABLE IF EXISTS ${i}page;" | psqlcmd
+ echo "CREATE TABLE ${i}page (
+ page_id int NOT NULL,
+ page_namespace int NOT NULL DEFAULT '0',
+ page_title text NOT NULL DEFAULT '',
+ page_restrictions text NOT NULL,
+ page_is_redirect smallint NOT NULL DEFAULT '0',
+ page_is_new smallint NOT NULL DEFAULT '0',
+ page_random double precision NOT NULL DEFAULT '0',
+ page_touched text NOT NULL DEFAULT '',
+ page_links_updated text DEFAULT NULL,
+ page_latest int NOT NULL DEFAULT '0',
+ page_len int NOT NULL DEFAULT '0',
+ page_content_model text DEFAULT NULL,
+ page_lang text DEFAULT NULL
+ );" | psqlcmd
+
+ time \
+ gzip -dc ${i}wiki-latest-page.sql.gz | \
+ sed "s/\`page\`/\`${i}page\`/g" | \
+ mysql2pgsqlcmd | \
+ grep -v '^CREATE INDEX ' | \
+ psqlcmd
+
+
+
echo "Importing ${i}wiki-latest-langlinks"
- gzip -dc ${i}wiki-latest-langlinks.sql.gz | sed "s/\`langlinks\`/\`${i}langlinks\`/g" | mysql2pgsqlcmd | psqlcmd
+
+ echo "DROP TABLE IF EXISTS ${i}langlinks;" | psqlcmd
+ echo "CREATE TABLE ${i}langlinks (
+ ll_from int NOT NULL DEFAULT '0',
+ ll_lang text NOT NULL DEFAULT '',
+ ll_title text NOT NULL DEFAULT ''
+ );" | psqlcmd
+
+ time \
+ gzip -dc ${i}wiki-latest-langlinks.sql.gz | \
+ sed "s/\`langlinks\`/\`${i}langlinks\`/g" | \
+ mysql2pgsqlcmd | \
+ grep -v '^CREATE INDEX ' | \
+ psqlcmd
+
+
+
+
echo "Importing ${i}wiki-latest-redirect"
- gzip -dc ${i}wiki-latest-redirect.sql.gz | sed "s/\`redirect\`/\`${i}redirect\`/g" | mysql2pgsqlcmd | psqlcmd
+
+ echo "DROP TABLE IF EXISTS ${i}redirect;" | psqlcmd
+ echo "CREATE TABLE ${i}redirect (
+ rd_from int NOT NULL DEFAULT '0',
+ rd_namespace int NOT NULL DEFAULT '0',
+ rd_title text NOT NULL DEFAULT '',
+ rd_interwiki text DEFAULT NULL,
+ rd_fragment text DEFAULT NULL
+ );" | psqlcmd
+
+ time \
+ gzip -dc ${i}wiki-latest-redirect.sql.gz | \
+ sed "s/\`redirect\`/\`${i}redirect\`/g" | \
+ mysql2pgsqlcmd | \
+ grep -v '^CREATE INDEX ' | \
+ psqlcmd
done
echo "CREATE TABLE ${i}pagelinkcount
AS
SELECT pl_title AS title,
- COUNT(*) AS count
+ COUNT(*) AS count,
+ 0::bigint as othercount
FROM ${i}pagelinks
WHERE pl_namespace = 0
GROUP BY pl_title
AND rd_namespace = 0
;" | psqlcmd
- echo "ALTER TABLE ${i}pagelinkcount
- ADD COLUMN othercount integer
- ;" | psqlcmd
+done
- echo "UPDATE ${i}pagelinkcount
- SET othercount = 0
- ;" | psqlcmd
+for i in "${LANGUAGES[@]}"
+do
for j in "${LANGUAGES[@]}"
do
echo "UPDATE ${i}pagelinkcount
echo "DROP TABLE ${i}redirect;" | psqlcmd
echo "DROP TABLE ${i}pagelinkcount;" | psqlcmd
done
+
+echo "all done."
if ($sType == 'T') return 'way';
if ($sType == 'I') return 'way';
+ // not handled: P, L
+
return '';
}
return '';
}
-function detailsLink($aFeature, $sTitle = false)
+function detailsLink($aFeature, $sTitle = false, $sExtraProperties = false)
{
if (!$aFeature['place_id']) return '';
- return '<a href="details.php?place_id='.$aFeature['place_id'].'">'.($sTitle?$sTitle:$aFeature['place_id']).'</a>';
+ $sHtml = '<a ';
+ if ($sExtraProperties) {
+ $sHtml .= $sExtraProperties.' ';
+ }
+
+ $sHtml .= 'href="details.php?place_id='.$aFeature['place_id'].'">'.($sTitle?$sTitle:$aFeature['place_id']).'</a>';
+
+ return $sHtml;
}
-function detailsPermaLink($aFeature, $sRefText = false)
+function detailsPermaLink($aFeature, $sRefText = false, $sExtraProperties = false)
{
$sOSMType = formatOSMType($aFeature['osm_type'], false);
if ($sOSMType) {
- $sLabel = $sRefText ? $sRefText : $sOSMType.' '.$aFeature['osm_id'];
- return '<a href="details.php?osmtype='.$aFeature['osm_type'].'&osmid='.$aFeature['osm_id'].'&class='.$aFeature['class'].'">'.$sLabel.'</a>';
+ $sHtml = '<a ';
+ if ($sExtraProperties) {
+ $sHtml .= $sExtraProperties.' ';
+ }
+ $sHtml .= 'href="details.php?osmtype='.$aFeature['osm_type']
+ .'&osmid='.$aFeature['osm_id'].'&class='.$aFeature['class'].'">';
+
+ if ($sRefText) {
+ $sHtml .= $sRefText.'</a>';
+ } else {
+ $sHtml .= $sOSMType.' '.$aFeature['osm_id'].'</a>';
+ }
+
+ return $sHtml;
}
- return '';
+ return detailsLink($aFeature, $sRefText, $sExtraProperties);
}
else
echo ' <span class="type">('.ucwords(str_replace('_',' ',$aResult['type'])).')</span>';
echo '<p>'.$aResult['lat'].','.$aResult['lon'].'</p>';
- echo ' <a class="btn btn-default btn-xs details" href="details.php?osmtype='.$aResult['osm_type'].'&osmid='.$aResult['osm_id'].'&class='.$aResult['class'].'">details</a>';
+ echo detailsPermaLink($aResult, 'details', 'class="btn btn-default btn-xs details"');
echo '</div>';
?>
</div>
echo ' <span class="type">('.ucwords(str_replace('_',' ',$aResult['class'])).')</span>';
else
echo ' <span class="type">('.ucwords(str_replace('_',' ',$aResult['type'])).')</span>';
- echo ' <a class="btn btn-default btn-xs details" href="details.php?osmtype='.$aResult['osm_type'].'&osmid='.$aResult['osm_id'].'&class='.$aResult['class'].'">details</a>';
+ echo detailsPermaLink($aResult, 'details', 'class="btn btn-default btn-xs details"');
echo '</div>';
$i = $i+1;
}
planet extract. A precompiled PBF with the necessary data can be downloaded from
https://www.nominatim.org/data/test/nominatim-api-testdata.pbf
+You need at least 2GB RAM and 10GB discspace.
+
The polygons defining the extract can be found in the test/testdb
directory. There is also a reduced set of wikipedia data for this extract,
which you need to import as well. For Tiger tests the data of South Dakota
is required. Get the Tiger files `46*`.
-The official test dataset is derived from the 160725 planet. Newer
+ cd Nominatim/data
+ wget https://nominatim.org/data/tiger2018-nominatim-preprocessed.tar.gz
+ tar xvf tiger2018-nominatim-preprocessed.tar.gz --wildcards --no-anchored '46*'
+ rm tiger2018-nominatim-preprocessed.tar.gz
+
+The official test dataset is derived from the 180924 planet. Newer
planets are likely to work as well but you may see isolated test
failures where the data has changed. To recreate the input data
for the test database run:
--- /dev/null
+<?php
+
+namespace Nominatim;
+
+require_once(CONST_BasePath.'/lib/output.php');
+
+class OutputTest extends \PHPUnit\Framework\TestCase
+{
+ public function testDetailsPermaLinkNode()
+ {
+ $aFeature = array('osm_type' => 'N', 'osm_id'=> 38274, 'class' => 'place');
+ $this->assertSame(
+ detailsPermaLink($aFeature),
+ '<a href="details.php?osmtype=N&osmid=38274&class=place">node 38274</a>'
+ );
+ }
+
+ public function testDetailsPermaLinkWay()
+ {
+ $aFeature = array('osm_type' => 'W', 'osm_id'=> 65, 'class' => 'highway');
+ $this->assertSame(
+ detailsPermaLink($aFeature),
+ '<a href="details.php?osmtype=W&osmid=65&class=highway">way 65</a>'
+ );
+ }
+
+ public function testDetailsPermaLinkRelation()
+ {
+ $aFeature = array('osm_type' => 'R', 'osm_id'=> 9908, 'class' => 'waterway');
+ $this->assertSame(
+ detailsPermaLink($aFeature),
+ '<a href="details.php?osmtype=R&osmid=9908&class=waterway">relation 9908</a>'
+ );
+ }
+
+ public function testDetailsPermaLinkTiger()
+ {
+ $aFeature = array('osm_type' => 'T', 'osm_id'=> 2, 'place_id' => 334);
+ $this->assertSame(
+ detailsPermaLink($aFeature, 'foo'),
+ '<a href="details.php?place_id=334">foo</a>'
+ );
+ }
+
+ public function testDetailsPermaLinkInterpolation()
+ {
+ $aFeature = array('osm_type' => 'I', 'osm_id'=> 400, 'place_id' => 3);
+ $this->assertSame(
+ detailsPermaLink($aFeature, 'foo'),
+ '<a href="details.php?place_id=3">foo</a>'
+ );
+ }
+
+ public function testDetailsPermaLinkWithExtraPropertiesNode()
+ {
+ $aFeature = array('osm_type' => 'N', 'osm_id'=> 2, 'class' => 'amenity');
+ $this->assertSame(
+ detailsPermaLink($aFeature, 'something', 'class="xtype"'),
+ '<a class="xtype" href="details.php?osmtype=N&osmid=2&class=amenity">something</a>'
+ );
+ }
+
+ public function testDetailsPermaLinkWithExtraPropertiesTiger()
+ {
+ $aFeature = array('osm_type' => 'T', 'osm_id'=> 5, 'place_id' => 46);
+ $this->assertSame(
+ detailsPermaLink($aFeature, 'something', 'class="xtype"'),
+ '<a class="xtype" href="details.php?place_id=46">something</a>'
+ );
+ }
+}
$sSQL .= ' ORDER BY class ASC';
$sPlaceId = $oDB->getOne($sSQL, array(':type' => $sOsmType, ':id' => $iOsmId));
+
+ // Nothing? Maybe it's an interpolation.
+ // XXX Simply returns the first parent street it finds. It should
+ // get a house number and get the right interpolation.
+ if (!$sPlaceId && $sOsmType == 'W' && (!$sClass || $sClass == 'place')) {
+ $sSQL = 'SELECT place_id FROM location_property_osmline'
+ .' WHERE osm_id = :id LIMIT 1';
+ $sPlaceId = $oDB->getOne($sSQL, array(':id' => $iOsmId));
+ }
+
// Be nice about our error messages for broken geometry
if (!$sPlaceId) {
projects / nominatim.git / commitdiff