project(nominatim)
set(NOMINATIM_VERSION_MAJOR 3)
-set(NOMINATIM_VERSION_MINOR 3)
+set(NOMINATIM_VERSION_MINOR 4)
set(NOMINATIM_VERSION_PATCH 0)
set(NOMINATIM_VERSION "${NOMINATIM_VERSION_MAJOR}.${NOMINATIM_VERSION_MINOR}.${NOMINATIM_VERSION_PATCH}")
+3.4.0
+
+ * increase required version for PostgreSQL(9.3), PostGIS(2.2) and PHP(7.0)
+ * better error reporting for out-of-memory errors
+ * exclude postcode ranges separated by colon from centre point calculation
+ * update osm2pgsql, better handling of imports without flatnode file
+ * switch to more efficient algorithm for word set computation
+ * use only boundries for country and state parts of addresses
+ * improve updates of addresses with housenumbers and interpolations
+ * remove country from place_addressline table and use country_code instead
+ * optimise indexes on search_name partition tables
+ * improve searching of attached streets for large objects like airports
+ * drop support for python 2
+ * new scripts for importing Wikidata for importance
+ * create and drop indexes concurrently to not clash with auto vacuum
+ * various documentation improvements
+
+
3.3.0
* zoom 17 in reverse now zooms in on minor streets
Wikidata
---
-This script downloads and processes Wikidata to enrich the previously created Wekipedia tables for use in Nominatim.
+This script downloads and processes Wikidata to enrich the previously created Wikipedia tables for use in Nominatim.
#### Import & Process Wikidata
COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_CURRENT_SOURCE_DIR}/api ${CMAKE_CURRENT_BINARY_DIR}/api
COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_CURRENT_SOURCE_DIR}/index.md ${CMAKE_CURRENT_BINARY_DIR}/index.md
COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_CURRENT_SOURCE_DIR}/extra.css ${CMAKE_CURRENT_BINARY_DIR}/extra.css
+ COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_CURRENT_SOURCE_DIR}/styles.css ${CMAKE_CURRENT_BINARY_DIR}/styles.css
COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_CURRENT_SOURCE_DIR}/data-sources/overview.md ${CMAKE_CURRENT_BINARY_DIR}/data-sources/overview.md
COMMAND ${CMAKE_COMMAND} -E create_symlink ${PROJECT_SOURCE_DIR}/data-sources/us-tiger/README.md ${CMAKE_CURRENT_BINARY_DIR}/data-sources/US-Tiger.md
COMMAND ${CMAKE_COMMAND} -E create_symlink ${PROJECT_SOURCE_DIR}/data-sources/gb-postcodes/README.md ${CMAKE_CURRENT_BINARY_DIR}/data-sources/GB-Postcodes.md
SQL statements should be executed from the PostgreSQL commandline. Execute
`psql nominatim` to enter command line mode.
-## 3.3.0 -> master
+## 3.3.0 -> 3.4.0
### Reorganisation of location_area_country table
- 'Installation on Ubuntu 16' : 'appendix/Install-on-Ubuntu-16.md'
- 'Installation on Ubuntu 18' : 'appendix/Install-on-Ubuntu-18.md'
markdown_extensions:
- - codehilite:
- use_pygments: False
+ - codehilite
- toc:
permalink:
-extra_css: [extra.css]
+extra_css: [extra.css, styles.css]
--- /dev/null
+.codehilite .hll { background-color: #ffffcc }
+.codehilite { background: #f0f0f0; }
+.codehilite .c { color: #60a0b0; font-style: italic } /* Comment */
+.codehilite .err { /* border: 1px solid #FF0000 */ } /* Error */
+.codehilite .k { color: #007020; font-weight: bold } /* Keyword */
+.codehilite .o { color: #666666 } /* Operator */
+.codehilite .ch { color: #60a0b0; font-style: italic } /* Comment.Hashbang */
+.codehilite .cm { color: #60a0b0; font-style: italic } /* Comment.Multiline */
+.codehilite .cp { color: #007020 } /* Comment.Preproc */
+.codehilite .cpf { color: #60a0b0; font-style: italic } /* Comment.PreprocFile */
+.codehilite .c1 { color: #60a0b0; font-style: italic } /* Comment.Single */
+.codehilite .cs { color: #60a0b0; background-color: #fff0f0 } /* Comment.Special */
+.codehilite .gd { color: #A00000 } /* Generic.Deleted */
+.codehilite .ge { font-style: italic } /* Generic.Emph */
+.codehilite .gr { color: #FF0000 } /* Generic.Error */
+.codehilite .gh { color: #000080; font-weight: bold } /* Generic.Heading */
+.codehilite .gi { color: #00A000 } /* Generic.Inserted */
+.codehilite .go { color: #888888 } /* Generic.Output */
+.codehilite .gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */
+.codehilite .gs { font-weight: bold } /* Generic.Strong */
+.codehilite .gu { color: #800080; font-weight: bold } /* Generic.Subheading */
+.codehilite .gt { color: #0044DD } /* Generic.Traceback */
+.codehilite .kc { color: #007020; font-weight: bold } /* Keyword.Constant */
+.codehilite .kd { color: #007020; font-weight: bold } /* Keyword.Declaration */
+.codehilite .kn { color: #007020; font-weight: bold } /* Keyword.Namespace */
+.codehilite .kp { color: #007020 } /* Keyword.Pseudo */
+.codehilite .kr { color: #007020; font-weight: bold } /* Keyword.Reserved */
+.codehilite .kt { color: #902000 } /* Keyword.Type */
+.codehilite .m { color: #40a070 } /* Literal.Number */
+.codehilite .s { color: #4070a0 } /* Literal.String */
+.codehilite .na { color: #4070a0 } /* Name.Attribute */
+.codehilite .nb { color: #007020 } /* Name.Builtin */
+.codehilite .nc { color: #0e84b5; font-weight: bold } /* Name.Class */
+.codehilite .no { color: #60add5 } /* Name.Constant */
+.codehilite .nd { color: #555555; font-weight: bold } /* Name.Decorator */
+.codehilite .ni { color: #d55537; font-weight: bold } /* Name.Entity */
+.codehilite .ne { color: #007020 } /* Name.Exception */
+.codehilite .nf { color: #06287e } /* Name.Function */
+.codehilite .nl { color: #002070; font-weight: bold } /* Name.Label */
+.codehilite .nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */
+.codehilite .nt { color: #062873; font-weight: bold } /* Name.Tag */
+.codehilite .nv { color: #bb60d5 } /* Name.Variable */
+.codehilite .ow { color: #007020; font-weight: bold } /* Operator.Word */
+.codehilite .w { color: #bbbbbb } /* Text.Whitespace */
+.codehilite .mb { color: #40a070 } /* Literal.Number.Bin */
+.codehilite .mf { color: #40a070 } /* Literal.Number.Float */
+.codehilite .mh { color: #40a070 } /* Literal.Number.Hex */
+.codehilite .mi { color: #40a070 } /* Literal.Number.Integer */
+.codehilite .mo { color: #40a070 } /* Literal.Number.Oct */
+.codehilite .sa { color: #4070a0 } /* Literal.String.Affix */
+.codehilite .sb { color: #4070a0 } /* Literal.String.Backtick */
+.codehilite .sc { color: #4070a0 } /* Literal.String.Char */
+.codehilite .dl { color: #4070a0 } /* Literal.String.Delimiter */
+.codehilite .sd { color: #4070a0; font-style: italic } /* Literal.String.Doc */
+.codehilite .s2 { color: #4070a0 } /* Literal.String.Double */
+.codehilite .se { color: #4070a0; font-weight: bold } /* Literal.String.Escape */
+.codehilite .sh { color: #4070a0 } /* Literal.String.Heredoc */
+.codehilite .si { color: #70a0d0; font-style: italic } /* Literal.String.Interpol */
+.codehilite .sx { color: #c65d09 } /* Literal.String.Other */
+.codehilite .sr { color: #235388 } /* Literal.String.Regex */
+.codehilite .s1 { color: #4070a0 } /* Literal.String.Single */
+.codehilite .ss { color: #517918 } /* Literal.String.Symbol */
+.codehilite .bp { color: #007020 } /* Name.Builtin.Pseudo */
+.codehilite .fm { color: #06287e } /* Name.Function.Magic */
+.codehilite .vc { color: #bb60d5 } /* Name.Variable.Class */
+.codehilite .vg { color: #bb60d5 } /* Name.Variable.Global */
+.codehilite .vi { color: #bb60d5 } /* Name.Variable.Instance */
+.codehilite .vm { color: #bb60d5 } /* Name.Variable.Magic */
+.codehilite .il { color: #40a070 } /* Literal.Number.Integer.Long */
}
foreach ($aLanguages as $sLanguage => $fLanguagePref) {
- $aLangPrefOrder['short_name:'.$sLanguage] = 'short_name:'.$sLanguage;
$aLangPrefOrder['name:'.$sLanguage] = 'name:'.$sLanguage;
}
- $aLangPrefOrder['short_name'] = 'short_name';
$aLangPrefOrder['name'] = 'name';
$aLangPrefOrder['brand'] = 'brand';
foreach ($aLanguages as $sLanguage => $fLanguagePref) {
$aLangPrefOrder['official_name:'.$sLanguage] = 'official_name:'.$sLanguage;
+ $aLangPrefOrder['short_name:'.$sLanguage] = 'short_name:'.$sLanguage;
}
$aLangPrefOrder['official_name'] = 'official_name';
+ $aLangPrefOrder['short_name'] = 'short_name';
$aLangPrefOrder['ref'] = 'ref';
$aLangPrefOrder['type'] = 'type';
return $aLangPrefOrder;
-Subproject commit 90e17f0e8c793487ca39cbf95501cd9c5daa33e2
+Subproject commit 8d9087f1111f4a062158e8e6b10bfbceed90899b
-- Indices used only during search and update.
-- These indices are created only after the indexing process is done.
-CREATE INDEX idx_word_word_id on word USING BTREE (word_id) {ts:search-index};
+CREATE INDEX CONCURRENTLY idx_word_word_id on word USING BTREE (word_id) {ts:search-index};
-CREATE INDEX idx_place_addressline_address_place_id on place_addressline USING BTREE (address_place_id) {ts:search-index};
+CREATE INDEX CONCURRENTLY idx_place_addressline_address_place_id on place_addressline USING BTREE (address_place_id) {ts:search-index};
-DROP INDEX IF EXISTS idx_placex_rank_search;
-CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search) {ts:search-index};
-CREATE INDEX idx_placex_rank_address ON placex USING BTREE (rank_address) {ts:search-index};
-CREATE INDEX idx_placex_pendingsector ON placex USING BTREE (rank_search,geometry_sector) {ts:address-index} where indexed_status > 0;
-CREATE INDEX idx_placex_parent_place_id ON placex USING BTREE (parent_place_id) {ts:search-index} where parent_place_id IS NOT NULL;
+DROP INDEX CONCURRENTLY IF EXISTS idx_placex_rank_search;
+CREATE INDEX CONCURRENTLY idx_placex_rank_search ON placex USING BTREE (rank_search) {ts:search-index};
+CREATE INDEX CONCURRENTLY idx_placex_rank_address ON placex USING BTREE (rank_address) {ts:search-index};
+CREATE INDEX CONCURRENTLY idx_placex_pendingsector ON placex USING BTREE (rank_search,geometry_sector) {ts:address-index} where indexed_status > 0;
+CREATE INDEX CONCURRENTLY idx_placex_parent_place_id ON placex USING BTREE (parent_place_id) {ts:search-index} where parent_place_id IS NOT NULL;
-CREATE INDEX idx_placex_geometry_reverse_lookupPoint
+CREATE INDEX CONCURRENTLY idx_placex_geometry_reverse_lookupPoint
ON placex USING gist (geometry) {ts:search-index}
WHERE (name is not null or housenumber is not null or rank_address between 26 and 27)
AND class not in ('railway','tunnel','bridge','man_made')
AND rank_address >= 26 AND indexed_status = 0 AND linked_place_id is null;
-CREATE INDEX idx_placex_geometry_reverse_lookupPolygon
+CREATE INDEX CONCURRENTLY idx_placex_geometry_reverse_lookupPolygon
ON placex USING gist (geometry) {ts:search-index}
WHERE St_GeometryType(geometry) in ('ST_Polygon', 'ST_MultiPolygon')
AND rank_address between 4 and 25 AND type != 'postcode'
AND name is not null AND indexed_status = 0 AND linked_place_id is null;
-CREATE INDEX idx_placex_geometry_reverse_placeNode
+CREATE INDEX CONCURRENTLY idx_placex_geometry_reverse_placeNode
ON placex USING gist (geometry) {ts:search-index}
WHERE osm_type = 'N' AND rank_search between 5 and 25
AND class = 'place' AND type != 'postcode'
GRANT SELECT ON table country_osm_grid to "{www-user}";
-CREATE INDEX idx_location_area_country_place_id ON location_area_country USING BTREE (place_id) {ts:address-index};
+CREATE INDEX CONCURRENTLY idx_location_area_country_place_id ON location_area_country USING BTREE (place_id) {ts:address-index};
-CREATE INDEX idx_osmline_parent_place_id ON location_property_osmline USING BTREE (parent_place_id) {ts:search-index};
-CREATE INDEX idx_osmline_parent_osm_id ON location_property_osmline USING BTREE (osm_id) {ts:search-index};
+CREATE INDEX CONCURRENTLY idx_osmline_parent_place_id ON location_property_osmline USING BTREE (parent_place_id) {ts:search-index};
+CREATE INDEX CONCURRENTLY idx_osmline_parent_osm_id ON location_property_osmline USING BTREE (osm_id) {ts:search-index};
-DROP INDEX IF EXISTS place_id_idx;
-CREATE UNIQUE INDEX idx_place_osm_unique on place using btree(osm_id,osm_type,class,type) {ts:address-index};
+DROP INDEX CONCURRENTLY IF EXISTS place_id_idx;
+CREATE UNIQUE INDEX CONCURRENTLY idx_place_osm_unique on place using btree(osm_id,osm_type,class,type) {ts:address-index};
-CREATE UNIQUE INDEX idx_postcode_id ON location_postcode USING BTREE (place_id) {ts:search-index};
-CREATE INDEX idx_postcode_postcode ON location_postcode USING BTREE (postcode) {ts:search-index};
+CREATE UNIQUE INDEX CONCURRENTLY idx_postcode_id ON location_postcode USING BTREE (place_id) {ts:search-index};
+CREATE INDEX CONCURRENTLY idx_postcode_postcode ON location_postcode USING BTREE (postcode) {ts:search-index};
-- Indices used for /search API.
-- These indices are created only after the indexing process is done.
-CREATE INDEX idx_search_name_nameaddress_vector ON search_name USING GIN (nameaddress_vector) WITH (fastupdate = off) {ts:search-index};
-CREATE INDEX idx_search_name_name_vector ON search_name USING GIN (name_vector) WITH (fastupdate = off) {ts:search-index};
-CREATE INDEX idx_search_name_centroid ON search_name USING GIST (centroid) {ts:search-index};
+CREATE INDEX CONCURRENTLY idx_search_name_nameaddress_vector ON search_name USING GIN (nameaddress_vector) WITH (fastupdate = off) {ts:search-index};
+CREATE INDEX CONCURRENTLY idx_search_name_name_vector ON search_name USING GIN (name_vector) WITH (fastupdate = off) {ts:search-index};
+CREATE INDEX CONCURRENTLY idx_search_name_centroid ON search_name USING GIST (centroid) {ts:search-index};
foreach (explode(',', $sLanguageIn) as $sLanguage) {
$sURL = 'https://wiki.openstreetmap.org/wiki/Special:Export/Nominatim/Special_Phrases/'.strtoupper($sLanguage);
$sWikiPageXML = file_get_contents($sURL);
- if (preg_match_all('#\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([\\-YN])#', $sWikiPageXML, $aMatches, PREG_SET_ORDER)) {
- foreach ($aMatches as $aMatch) {
- $sLabel = trim($aMatch[1]);
- if ($oNormalizer !== null) {
- $sTrans = pg_escape_string($oNormalizer->transliterate($sLabel));
- } else {
- $sTrans = null;
- }
- $sClass = trim($aMatch[2]);
- $sType = trim($aMatch[3]);
- // hack around a bug where building=yes was imported with
- // quotes into the wiki
- $sType = preg_replace('/"/', '', $sType);
- // sanity check, in case somebody added garbage in the wiki
- if (preg_match('/^\\w+$/', $sClass) < 1
- || preg_match('/^\\w+$/', $sType) < 1
- ) {
- trigger_error("Bad class/type for language $sLanguage: $sClass=$sType");
- exit;
- }
- // blacklisting: disallow certain class/type combinations
- if (isset($aTagsBlacklist[$sClass]) && in_array($sType, $aTagsBlacklist[$sClass])) {
- // fwrite(STDERR, "Blacklisted: ".$sClass."/".$sType."\n");
- continue;
- }
- // whitelisting: if class is in whitelist, allow only tags in the list
- if (isset($aTagsWhitelist[$sClass]) && !in_array($sType, $aTagsWhitelist[$sClass])) {
- // fwrite(STDERR, "Non-Whitelisted: ".$sClass."/".$sType."\n");
- continue;
- }
- $aPairs[$sClass.'|'.$sType] = array($sClass, $sType);
-
- switch (trim($aMatch[4])) {
- case 'near':
- echo "select getorcreate_amenityoperator(make_standard_name('".pg_escape_string($sLabel)."'), '$sTrans', '$sClass', '$sType', 'near');\n";
- break;
- case 'in':
- echo "select getorcreate_amenityoperator(make_standard_name('".pg_escape_string($sLabel)."'), '$sTrans', '$sClass', '$sType', 'in');\n";
- break;
- default:
- echo "select getorcreate_amenity(make_standard_name('".pg_escape_string($sLabel)."'), '$sTrans', '$sClass', '$sType');\n";
- break;
- }
+
+ if (!preg_match_all(
+ '#\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([\\-YN])#',
+ $sWikiPageXML,
+ $aMatches,
+ PREG_SET_ORDER
+ )) {
+ continue;
+ }
+
+ foreach ($aMatches as $aMatch) {
+ $sLabel = trim($aMatch[1]);
+ if ($oNormalizer !== null) {
+ $sTrans = pg_escape_string($oNormalizer->transliterate($sLabel));
+ } else {
+ $sTrans = null;
+ }
+ $sClass = trim($aMatch[2]);
+ $sType = trim($aMatch[3]);
+ // hack around a bug where building=yes was imported with
+ // quotes into the wiki
+ $sType = preg_replace('/("|")/', '', $sType);
+ // sanity check, in case somebody added garbage in the wiki
+ if (preg_match('/^\\w+$/', $sClass) < 1
+ || preg_match('/^\\w+$/', $sType) < 1
+ ) {
+ trigger_error("Bad class/type for language $sLanguage: $sClass=$sType");
+ exit;
+ }
+ // blacklisting: disallow certain class/type combinations
+ if (isset($aTagsBlacklist[$sClass]) && in_array($sType, $aTagsBlacklist[$sClass])) {
+ // fwrite(STDERR, "Blacklisted: ".$sClass."/".$sType."\n");
+ continue;
+ }
+ // whitelisting: if class is in whitelist, allow only tags in the list
+ if (isset($aTagsWhitelist[$sClass]) && !in_array($sType, $aTagsWhitelist[$sClass])) {
+ // fwrite(STDERR, "Non-Whitelisted: ".$sClass."/".$sType."\n");
+ continue;
+ }
+ $aPairs[$sClass.'|'.$sType] = array($sClass, $sType);
+
+ switch (trim($aMatch[4])) {
+ case 'near':
+ printf(
+ "SELECT getorcreate_amenityoperator(make_standard_name('%s'), '%s', '%s', '%s', 'near');\n",
+ pg_escape_string($sLabel),
+ $sTrans,
+ $sClass,
+ $sType
+ );
+ break;
+ case 'in':
+ printf(
+ "SELECT getorcreate_amenityoperator(make_standard_name('%s'), '%s', '%s', '%s', 'in');\n",
+ pg_escape_string($sLabel),
+ $sTrans,
+ $sClass,
+ $sType
+ );
+ break;
+ default:
+ printf(
+ "SELECT getorcreate_amenity(make_standard_name('%s'), '%s', '%s', '%s');\n",
+ pg_escape_string($sLabel),
+ $sTrans,
+ $sClass,
+ $sType
+ );
+ break;
}
}
}
- echo 'create index idx_placex_classtype on placex (class, type);';
+ echo 'CREATE INDEX idx_placex_classtype ON placex (class, type);';
foreach ($aPairs as $aPair) {
- echo 'create table place_classtype_'.pg_escape_string($aPair[0]).'_'.pg_escape_string($aPair[1]);
- if (CONST_Tablespace_Aux_Data)
- echo ' tablespace '.CONST_Tablespace_Aux_Data;
- echo ' as select place_id as place_id,st_centroid(geometry) as centroid from placex where ';
- echo "class = '".pg_escape_string($aPair[0])."' and type = '".pg_escape_string($aPair[1])."'";
- echo ";\n";
-
- echo 'CREATE INDEX idx_place_classtype_'.pg_escape_string($aPair[0]).'_'.pg_escape_string($aPair[1]).'_centroid ';
- echo 'ON place_classtype_'.pg_escape_string($aPair[0]).'_'.pg_escape_string($aPair[1]).' USING GIST (centroid)';
- if (CONST_Tablespace_Aux_Index)
- echo ' tablespace '.CONST_Tablespace_Aux_Index;
- echo ";\n";
-
- echo 'CREATE INDEX idx_place_classtype_'.pg_escape_string($aPair[0]).'_'.pg_escape_string($aPair[1]).'_place_id ';
- echo 'ON place_classtype_'.pg_escape_string($aPair[0]).'_'.pg_escape_string($aPair[1]).' USING btree(place_id)';
- if (CONST_Tablespace_Aux_Index)
- echo ' tablespace '.CONST_Tablespace_Aux_Index;
- echo ";\n";
-
- echo 'GRANT SELECT ON place_classtype_'.pg_escape_string($aPair[0]).'_'.pg_escape_string($aPair[1]).' TO "'.CONST_Database_Web_User."\";\n";
+ $sql_tablespace = CONST_Tablespace_Aux_Data ? ' TABLESPACE '.CONST_Tablespace_Aux_Data : '';
+
+ printf(
+ 'CREATE TABLE place_classtype_%s_%s'
+ . $sql_tablespace
+ . ' AS'
+ . ' SELECT place_id AS place_id,st_centroid(geometry) AS centroid FROM placex'
+ . " WHERE class = '%s' AND type = '%s'"
+ . ";\n",
+ pg_escape_string($aPair[0]),
+ pg_escape_string($aPair[1]),
+ pg_escape_string($aPair[0]),
+ pg_escape_string($aPair[1])
+ );
+
+ printf(
+ 'CREATE INDEX idx_place_classtype_%s_%s_centroid'
+ . ' ON place_classtype_%s_%s USING GIST (centroid)'
+ . $sql_tablespace
+ . ";\n",
+ pg_escape_string($aPair[0]),
+ pg_escape_string($aPair[1]),
+ pg_escape_string($aPair[0]),
+ pg_escape_string($aPair[1])
+ );
+
+ printf(
+ 'CREATE INDEX idx_place_classtype_%s_%s_place_id'
+ . ' ON place_classtype_%s_%s USING btree(place_id)'
+ . $sql_tablespace
+ . ";\n",
+ pg_escape_string($aPair[0]),
+ pg_escape_string($aPair[1]),
+ pg_escape_string($aPair[0]),
+ pg_escape_string($aPair[1])
+ );
+
+ printf(
+ 'GRANT SELECT ON place_classtype_%s_%s TO "%s"'
+ . ";\n",
+ pg_escape_string($aPair[0]),
+ pg_escape_string($aPair[1]),
+ CONST_Database_Web_User
+ );
}
- echo 'drop index idx_placex_classtype;';
+ echo 'DROP INDEX idx_placex_classtype;';
}
projects / nominatim.git / commitdiff