steps:
- name: Install prerequisits
run: |
- sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev python3-psycopg2 python3-pyosmium php-symfony-dotenv
+ sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev python3-psycopg2 python3-pyosmium python3-dotenv
shell: bash
- name: Configure
- name: Build
run: |
make -j2 all
- ./utils/setup.php --setup-website
+ ./nominatim refresh --website
shell: bash
working-directory: build
- name: Install test prerequsites
run: |
- sudo apt-get install -y -qq php-codesniffer python3-tidylib
+ sudo apt-get install -y -qq php-codesniffer
sudo pip3 install behave
- name: PHP linting
shell: bash
- name: Import
- run: php ./utils/setup.php --osm-file ../monaco-latest.osm.pbf --osm2pgsql-cache 500 --all
+ run: ./nominatim import --osm-file ../monaco-latest.osm.pbf
working-directory: build
- name: Import special phrases
- run: php ./utils/specialphrases.php --wiki-import | psql -d nominatim
+ run: ./nominatim special-phrases --from-wiki | psql -d nominatim
working-directory: build
- name: Check import
- run: php ./utils/check_import_finished.php
+ run: ./nominatim check-database
working-directory: build
- name: Run update
run: |
- php ./utils/update.php --init-updates
- php ./utils/update.php --import-osmosis
+ ./nominatim replication --init
+ ./nominatim replication --once
working-directory: build
- name: Run reverse-only import
run : |
dropdb nominatim
- php ./utils/setup.php --osm-file ../monaco-latest.osm.pbf --reverse-only --all
+ php ./nominatim import --osm-file ../monaco-latest.osm.pbf --reverse-only
working-directory: build
set(CUSTOMSCRIPTS
check_import_finished.php
country_languages.php
- importWikipedia.php
export.php
query.php
setup.php
configure_file(${PROJECT_SOURCE_DIR}/cmake/script.tmpl
${PROJECT_BINARY_DIR}/utils/${script_source})
endforeach()
+
+ configure_file(${PROJECT_SOURCE_DIR}/cmake/tool.tmpl
+ ${PROJECT_BINARY_DIR}/nominatim)
endif()
#-----------------------------------------------------------------------------
#!@PHP_BIN@ -Cq
<?php
+require('@CMAKE_SOURCE_DIR@/lib/dotenv_loader.php');
+
@define('CONST_Default_ModulePath', '@CMAKE_BINARY_DIR@/module');
@define('CONST_Default_Osm2pgsql', '@CMAKE_BINARY_DIR@/osm2pgsql/osm2pgsql');
@define('CONST_BinDir', '@CMAKE_SOURCE_DIR@/utils');
-@define('CONST_LibDir', '@CMAKE_SOURCE_DIR@/lib');
@define('CONST_DataDir', '@CMAKE_SOURCE_DIR@');
-require_once(CONST_BinDir.'/@script_source@');
+loadDotEnv();
+
+require_once('@CMAKE_SOURCE_DIR@/lib/admin/@script_source@');
--- /dev/null
+#!/usr/bin/env python3
+import sys
+
+sys.path.insert(1, '@CMAKE_SOURCE_DIR@')
+
+from nominatim import cli
+
+exit(cli.nominatim(module_dir='@CMAKE_BINARY_DIR@/module',
+ osm2pgsql_path='@CMAKE_BINARY_DIR@/osm2pgsql/osm2pgsql',
+ phplib_dir='@CMAKE_SOURCE_DIR@/lib',
+ data_dir='@CMAKE_SOURCE_DIR@'))
BASEURL="https://download.geofabrik.de"
DOWNCOUNTRYPOSTFIX="-latest.osm.pbf"
-
+
+### Setting up multiple regions
+
!!! tip
- If your database already exists and you want to add more countries, replace the setting up part
+ If your database already exists and you want to add more countries,
+ replace the setting up part
`${SETUPFILE} --osm-file ${UPDATEDIR}/tmp/combined.osm.pbf --all 2>&1`
with `${UPDATEFILE} --import-file ${UPDATEDIR}/tmp/combined.osm.pbf --index --index-instances N 2>&1`
where N is the numbers of CPUs in your system.
-### Setting up multiple regions
-
Run the following command from your Nominatim directory after configuring the file.
bash ./utils/import_multiple_regions.sh
then you can resume with the following command:
```sh
-./utils/setup.php --index --create-search-indices --create-country-names
+nominatim import --continue indexing
```
If the reported rank is 26 or higher, you can also safely add `--index-noanalyse`.
If it happened during index creation you can try rerunning the step with
```sh
-./utils/setup.php --create-search-indices --ignore-errors
+nominatim import --continue indexing
```
Otherwise it's best to start the full setup from the beginning.
### I see the error: "function transliteration(text) does not exist"
-Reinstall the nominatim functions with `setup.php --create--functions`
+Reinstall the nominatim functions with `nominatim refresh --functions`
and check for any errors, e.g. a missing `nominatim.so` file.
### I see the error: "ERROR: mmap (remap) failed"
### nominatim UPDATE failed: ERROR: buffer 179261 is not owned by resource owner Portal
-Several users [reported this](https://github.com/openstreetmap/Nominatim/issues/1168) during the initial import of the database. It's
+Several users [reported this](https://github.com/openstreetmap/Nominatim/issues/1168)
+during the initial import of the database. It's
something PostgreSQL internal Nominatim doesn't control. And PostgreSQL forums
suggest it's threading related but definitely some kind of crash of a process.
Users reported either rebooting the server, different hardware or just trying
### I forgot to delete the flatnodes file before starting an import.
That's fine. For each import the flatnodes file get overwritten.
-See [https://help.openstreetmap.org/questions/52419/nominatim-flatnode-storage]()
+See [https://help.openstreetmap.org/questions/52419/nominatim-flatnode-storage](https://help.openstreetmap.org/questions/52419/nominatim-flatnode-storage)
for more information.
## Configuration setup in `.env`
The Nominatim server can be customized via a `.env` in the build directory.
-This is a file in [dotenv](https://symfony.com/doc/4.3/components/dotenv.html) format
+This is a file in [dotenv](https://github.com/theskumar/python-dotenv) format
which looks the same as variable settings in a standard shell environment.
You can also set the same configuration via environment variables. All
settings have a `NOMINATIM_` prefix to avoid conflicts with other environment
!!! tip
If you forgot to download the wikipedia rankings, you can also add
importances after the import. Download the files, then run
- `./utils/setup.php --import-wikipedia-articles`
- and `./utils/update.php --recompute-importance`.
+ `./nominatim refresh --wiki-data --importance`.
### Great Britain, USA postcodes
About half of the data in Nominatim's database is not really used for serving
the API. It is only there to allow the data to be updated from the latest
changes from OSM. For many uses these dynamic updates are not really required.
-If you don't plan to apply updates, the dynamic part of the database can be
-safely dropped using the following command:
+If you don't plan to apply updates, you can run the import with the
+`--no-updates` parameter. This will drop the dynamic part of the database as
+soon as it is not required anymore.
+
+You can also drop the dynamic part later using the following command:
```
-./utils/setup.php --drop
+./nominatim freeze
```
Note that you still need to provide for sufficient disk space for the initial
from the **build directory** to start the import:
```sh
-./utils/setup.php --osm-file <data file> --all 2>&1 | tee setup.log
+./nominatim import --osm-file <data file> 2>&1 | tee setup.log
```
### Notes on full planet imports
Run this script to verify all required tables and indices got created successfully.
```sh
-./utils/check_import_finished.php
+./nominatim check-database
```
-### Setting up the website
-
-Run the following command to set up the configuration file for the API frontend
-`settings/settings-frontend.php`. These settings are used in website/*.php files.
-
-```sh
-./utils/setup.php --setup-website
-```
-!!! Note
- This step is not necessary if you use `--all` option while setting up the DB.
+### Testing the installation
Now you can try out your installation by running:
of forward geocoding in particular under high load. To recompute word counts run:
```sh
-./utils/update.php --recompute-word-counts
+./nominatim refresh --word-counts
```
This will take a couple of hours for a full planet installation. You can
[special key phrases](https://wiki.openstreetmap.org/wiki/Nominatim/Special_Phrases)
you also need to enable these key phrases like this:
- ./utils/specialphrases.php --wiki-import > specialphrases.sql
+ ./nominatim special-phrases --from-wiki > specialphrases.sql
psql -d nominatim -f specialphrases.sql
Note that this command downloads the phrases from the wiki link above. You
1. Get preprocessed TIGER 2019 data and unpack it into the
data directory in your Nominatim sources:
- cd Nominatim/data
wget https://nominatim.org/data/tiger2019-nominatim-preprocessed.tar.gz
tar xf tiger2019-nominatim-preprocessed.tar.gz
2. Import the data into your Nominatim database:
- ./utils/setup.php --import-tiger-data
+ ./nominatim add-data --tiger-data tiger
3. Enable use of the Tiger data in your `.env` by adding:
4. Apply the new settings:
```sh
- ./utils/setup.php --create-functions --enable-diff-updates --create-partition-functions
+ ./nominatim refresh --functions
```
* [PostgreSQL](https://www.postgresql.org) (9.3+)
* [PostGIS](https://postgis.net) (2.2+)
- * [Python 3](https://www.python.org/)
+ * [Python 3](https://www.python.org/) (3.4+)
* [Psycopg2](https://www.psycopg.org)
* [PHP](https://php.net) (7.0 or later)
* PHP-pgsql
* PHP-intl (bundled with PHP)
- * [PHP Symphony Dotenv](https://symfony.com/doc/4.3/components/dotenv.html)
+ * [Python Dotenv](https://github.com/theskumar/python-dotenv)
For running continuous updates:
SQL statements should be executed from the PostgreSQL commandline. Execute
`psql nominatim` to enter command line mode.
+## 3.6.0 -> master
+
+### Introducing `nominatim` command line tool
+
+The various php utilities have been replaced with a single `nominatim`
+command line tool. Make sure to adapt any scripts. There is no direct 1:1
+matching between the old utilities and the commands of nominatim CLI. The
+following list gives you a list of nominatim sub-commands that contain
+functionality of each script:
+
+* ./utils/setup.php: `import`, `freeze`, `refresh`
+* ./utils/update.php: `replication`, `add-data`, `index`, `refresh`
+* ./utils/specialphrases.php: `special-phrases`
+* ./utils/check_import_finished.php: `check-database`
+* ./utils/warm.php: `warm`
+* ./utils/export.php: `export`
+
+Try `nominatim <command> --help` for more information about each subcommand.
+
+`./utils/query.php` no longer exists in its old form. `nominatim search`
+provides a replacement.
+
## 3.5.0 -> 3.6.0
### Change of layout of search_name_* tables
# Updating the Database
There are many different ways to update your Nominatim database.
-The following section describes how to keep it up-to-date with Pyosmium.
-For a list of other methods see the output of `./utils/update.php --help`.
+The following section describes how to keep it up-to-date using
+an [online replication service for OpenStreetMap data](https://wiki.openstreetmap.org/wiki/Planet.osm/diffs)
+For a list of other methods to add or update data see the output of
+`nominatim add-data --help`.
!!! important
If you have configured a flatnode file for the import, then you
To set up the update process now run the following command:
- ./utils/update.php --init-updates
+ ./nominatim replication --init
It outputs the date where updates will start. Recheck that this date is
what you expect.
-The `--init-updates` command needs to be rerun whenever the replication service
-is changed.
+The `replication --init` command needs to be rerun whenever the replication
+service is changed.
#### Updating Nominatim
The following command will keep your database constantly up to date:
- ./utils/update.php --import-osmosis-all
-
-(Note that even though the old name "import-osmosis-all" has been kept for
-compatibility reasons, Osmosis is not required to run this - it uses pyosmium
-behind the scenes.)
+ ./nominatim replication
If you have imported multiple country extracts and want to keep them
up-to-date, [Advanced installations section](Advanced-Installations.md) contains instructions
suited for these kinds of queries.
That said if you installed your own Nominatim instance you can use the
-`/utils/export.php` PHP script as basis to return such lists.
+`nominatim export` PHP script as basis to return such lists.
## Regular updating calculated postcodes
The script to rerun the calculation is
-`build/utils/update.php --calculate-postcodes`
+`nominatim refresh --postcodes`
and runs once per night on nominatim.openstreetmap.org.
<?php
+@define('CONST_LibDir', dirname(dirname(__FILE__)));
require_once(CONST_LibDir.'/init-cmd.php');
echo $term_colors['red'].$message.$term_colors['normal']."\n";
};
-
$oDB = new Nominatim\DB;
<?php
+@define('CONST_LibDir', dirname(dirname(__FILE__)));
require_once(CONST_LibDir.'/init-cmd.php');
<?php
+ @define('CONST_LibDir', dirname(dirname(__FILE__)));
// Script to extract structured city and street data
// from a running nominatim instance as CSV data
<?php
+@define('CONST_LibDir', dirname(dirname(__FILE__)));
require_once(CONST_LibDir.'/init-cmd.php');
require_once(CONST_LibDir.'/Geocode.php');
<?php
+@define('CONST_LibDir', dirname(dirname(__FILE__)));
require_once(CONST_LibDir.'/init-cmd.php');
require_once(CONST_LibDir.'/setup/SetupClass.php');
<?php
+@define('CONST_LibDir', dirname(dirname(__FILE__)));
require_once(CONST_LibDir.'/init-cmd.php');
ini_set('memory_limit', '800M');
<?php
+@define('CONST_LibDir', dirname(dirname(__FILE__)));
require_once(CONST_LibDir.'/init-cmd.php');
require_once(CONST_LibDir.'/setup_functions.php');
<?php
+@define('CONST_LibDir', dirname(dirname(__FILE__)));
require_once(CONST_LibDir.'/init-cmd.php');
require_once(CONST_LibDir.'/log.php');
loadSettings($aCMDResult['project-dir'] ?? getcwd());
+@define('CONST_Database_DSN', getSetting('DATABASE_DSN'));
+@define('CONST_Default_Language', getSetting('DEFAULT_LANGUAGE', false));
+@define('CONST_Log_DB', getSettingBool('LOG_DB'));
+@define('CONST_Log_File', getSetting('LOG_FILE', false));
+@define('CONST_Max_Word_Frequency', getSetting('MAX_WORD_FREQUENCY'));
+@define('CONST_NoAccessControl', getSettingBool('CORS_NOACCESSCONTROL'));
+@define('CONST_Places_Max_ID_count', getSetting('LOOKUP_MAX_COUNT'));
+@define('CONST_PolygonOutput_MaximumTypes', getSetting('POLYGON_OUTPUT_MAX_TYPES'));
+@define('CONST_Search_BatchMode', getSettingBool('SEARCH_BATCH_MODE'));
+@define('CONST_Search_NameOnlySearchFrequencyThreshold', getSetting('SEARCH_NAME_ONLY_THRESHOLD'));
+@define('CONST_Term_Normalization_Rules', getSetting('TERM_NORMALIZATION'));
+@define('CONST_Use_Aux_Location_data', getSettingBool('USE_AUX_LOCATION_DATA'));
+@define('CONST_Use_US_Tiger_Data', getSettingBool('USE_US_TIGER_DATA'));
+@define('CONST_MapIcon_URL', getSetting('MAPICON_URL', false));
+
+
$oDB = new Nominatim\DB();
$oDB->connect();
--- /dev/null
+<?php
+
+require('Symfony/Component/Dotenv/autoload.php');
+
+function loadDotEnv()
+{
+ $dotenv = new \Symfony\Component\Dotenv\Dotenv();
+ $dotenv->load(CONST_DataDir.'/settings/env.defaults');
+
+ if (file_exists('.env')) {
+ $dotenv->load('.env');
+ }
+}
<?php
-require('Symfony/Component/Dotenv/autoload.php');
-
function loadSettings($sProjectDir)
{
@define('CONST_InstallDir', $sProjectDir);
-
- $dotenv = new \Symfony\Component\Dotenv\Dotenv();
- $dotenv->load(CONST_DataDir.'/settings/env.defaults');
-
- if (file_exists($sProjectDir.'/.env')) {
- $dotenv->load($sProjectDir.'/.env');
- }
+ // Temporary hack to set the direcory via environment instead of
+ // the installed scripts. Neither setting is part of the official
+ // set of settings.
+ defined('CONST_DataDir') or define('CONST_DataDir', $_SERVER['NOMINATIM_DATADIR']);
+ defined('CONST_BinDir') or define('CONST_BinDir', $_SERVER['NOMINATIM_BINDIR']);
}
function getSetting($sConfName, $sDefault = null)
function getSettingConfig($sConfName, $sSystemConfig)
{
- $sValue = $_ENV['NOMINATIM_'.$sConfName];
+ $sValue = $_SERVER['NOMINATIM_'.$sConfName];
if (!$sValue) {
return CONST_DataDir.'/settings/'.$sSystemConfig;
$this->iCacheMemory = getCacheMemoryMB();
}
- $this->sModulePath = getSetting('DATABASE_MODULE_PATH', CONST_Default_ModulePath);
+ $this->sModulePath = getSetting('DATABASE_MODULE_PATH');
+ if (!$this->sModulePath) {
+ $this->sModulePath = CONST_Default_ModulePath;
+ }
info('module path: ' . $this->sModulePath);
// parse database string
function getOsm2pgsqlBinary()
{
- return getSetting('OSM2PGSQL_BINARY', CONST_Default_Osm2pgsql);
+ $sBinary = getSetting('OSM2PGSQL_BINARY');
+
+ return $sBinary ? $sBinary : CONST_Default_Osm2pgsql;
}
function getImportStyle()
--- /dev/null
+"""
+Module with functions for importing, updating Nominatim databases
+as well as general maintenance helpers.
+"""
--- /dev/null
+"""
+Helper functions for executing external programs.
+"""
+import subprocess
+
+def run_legacy_script(script, *args, nominatim_env=None, throw_on_fail=False):
+ """ Run a Nominatim PHP script with the given arguments.
+
+ Returns the exit code of the script. If `throw_on_fail` is True
+ then throw a `CalledProcessError` on a non-zero exit.
+ """
+ cmd = ['/usr/bin/env', 'php', '-Cq',
+ nominatim_env.phplib_dir / 'admin' / script]
+ cmd.extend([str(a) for a in args])
+
+ env = nominatim_env.config.get_os_env()
+ env['NOMINATIM_DATADIR'] = str(nominatim_env.data_dir)
+ env['NOMINATIM_BINDIR'] = str(nominatim_env.data_dir / 'utils')
+ if not env['NOMINATIM_DATABASE_MODULE_PATH']:
+ env['NOMINATIM_DATABASE_MODULE_PATH'] = nominatim_env.module_dir
+ if not env['NOMINATIM_OSM2PGSQL_BINARY']:
+ env['NOMINATIM_OSM2PGSQL_BINARY'] = nominatim_env.osm2pgsql_path
+
+ proc = subprocess.run(cmd, cwd=str(nominatim_env.project_dir), env=env)
+
+ if throw_on_fail:
+ proc.check_returncode()
+
+ return proc.returncode
--- /dev/null
+"""
+Command-line interface to the Nominatim functions for import, update,
+database administration and querying.
+"""
+import sys
+import os
+import argparse
+import logging
+from pathlib import Path
+
+from .config import Configuration
+from .admin.exec_utils import run_legacy_script
+
+class CommandlineParser:
+ """ Wraps some of the common functions for parsing the command line
+ and setting up subcommands.
+ """
+ def __init__(self, prog, description):
+ self.parser = argparse.ArgumentParser(
+ prog=prog,
+ description=description,
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+
+ self.subs = self.parser.add_subparsers(title='available commands',
+ dest='subcommand')
+
+ # Arguments added to every sub-command
+ self.default_args = argparse.ArgumentParser(add_help=False)
+ group = self.default_args.add_argument_group('Default arguments')
+ group.add_argument('-h', '--help', action='help',
+ help='Show this help message and exit')
+ group.add_argument('-q', '--quiet', action='store_const', const=0,
+ dest='verbose', default=1,
+ help='Print only error messages')
+ group.add_argument('-v', '--verbose', action='count', default=1,
+ help='Increase verboseness of output')
+ group.add_argument('--project-dir', metavar='DIR', default='.',
+ help='Base directory of the Nominatim installation (default:.)')
+ group.add_argument('-j', '--threads', metavar='NUM', type=int,
+ help='Number of parallel threads to use')
+
+
+ def add_subcommand(self, name, cmd):
+ """ Add a subcommand to the parser. The subcommand must be a class
+ with a function add_args() that adds the parameters for the
+ subcommand and a run() function that executes the command.
+ """
+ parser = self.subs.add_parser(name, parents=[self.default_args],
+ help=cmd.__doc__.split('\n', 1)[0],
+ description=cmd.__doc__,
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ add_help=False)
+ parser.set_defaults(command=cmd)
+ cmd.add_args(parser)
+
+ def run(self, **kwargs):
+ """ Parse the command line arguments of the program and execute the
+ appropriate subcommand.
+ """
+ args = self.parser.parse_args()
+
+ if args.subcommand is None:
+ return self.parser.print_help()
+
+ for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir'):
+ setattr(args, arg, Path(kwargs[arg]))
+ args.project_dir = Path(args.project_dir)
+
+ logging.basicConfig(stream=sys.stderr,
+ format='%(asctime)s %(levelname)s: %(message)s',
+ datefmt='%Y-%m-%d %H:%M:%S',
+ level=max(4 - args.verbose, 1) * 10)
+
+ args.config = Configuration(args.project_dir, args.data_dir / 'settings')
+
+ return args.command.run(args)
+
+##### Subcommand classes
+#
+# Each class needs to implement two functions: add_args() adds the CLI parameters
+# for the subfunction, run() executes the subcommand.
+#
+# The class documentation doubles as the help text for the command. The
+# first line is also used in the summary when calling the program without
+# a subcommand.
+#
+# No need to document the functions each time.
+# pylint: disable=C0111
+
+
+class SetupAll:
+ """\
+ Create a new Nominatim database from an OSM file.
+ """
+
+ @staticmethod
+ def add_args(parser):
+ group_name = parser.add_argument_group('Required arguments')
+ group = group_name.add_mutually_exclusive_group(required=True)
+ group.add_argument('--osm-file',
+ help='OSM file to be imported.')
+ group.add_argument('--continue', dest='continue_at',
+ choices=['load-data', 'indexing', 'db-postprocess'],
+ help='Continue an import that was interrupted')
+ group = parser.add_argument_group('Optional arguments')
+ group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
+ help='Size of cache to be used by osm2pgsql (in MB)')
+ group.add_argument('--reverse-only', action='store_true',
+ help='Do not create tables and indexes for searching')
+ group.add_argument('--enable-debug-statements', action='store_true',
+ help='Include debug warning statements in SQL code')
+ group.add_argument('--no-partitions', action='store_true',
+ help="""Do not partition search indices
+ (speeds up import of single country extracts)""")
+ group.add_argument('--no-updates', action='store_true',
+ help="""Do not keep tables that are only needed for
+ updating the database later""")
+ group = parser.add_argument_group('Expert options')
+ group.add_argument('--ignore-errors', action='store_true',
+ help='Continue import even when errors in SQL are present')
+ group.add_argument('--index-noanalyse', action='store_true',
+ help='Do not perform analyse operations during index')
+
+
+ @staticmethod
+ def run(args):
+ params = ['setup.php']
+ if args.osm_file:
+ params.extend(('--all', '--osm-file', args.osm_file))
+ else:
+ if args.continue_at == 'load-data':
+ params.append('--load-data')
+ if args.continue_at in ('load-data', 'indexing'):
+ params.append('--index')
+ params.extend(('--create-search-indices', '--create-country-names',
+ '--setup-website'))
+ if args.osm2pgsql_cache:
+ params.extend(('--osm2pgsql-cache', args.osm2pgsql_cache))
+ if args.reverse_only:
+ params.append('--reverse-only')
+ if args.enable_debug_statements:
+ params.append('--enable-debug-statements')
+ if args.no_partitions:
+ params.append('--no-partitions')
+ if args.no_updates:
+ params.append('--drop')
+ if args.ignore_errors:
+ params.append('--ignore-errors')
+ if args.index_noanalyse:
+ params.append('--index-noanalyse')
+
+ return run_legacy_script(*params, nominatim_env=args)
+
+
+class SetupFreeze:
+ """\
+ Make database read-only.
+
+ About half of data in the Nominatim database is kept only to be able to
+ keep the data up-to-date with new changes made in OpenStreetMap. This
+ command drops all this data and only keeps the part needed for geocoding
+ itself.
+
+ This command has the same effect as the `--no-updates` option for imports.
+ """
+
+ @staticmethod
+ def add_args(parser):
+ pass # No options
+
+ @staticmethod
+ def run(args):
+ return run_legacy_script('setup.php', '--drop', nominatim_env=args)
+
+
+class SetupSpecialPhrases:
+ """\
+ Maintain special phrases.
+ """
+
+ @staticmethod
+ def add_args(parser):
+ group = parser.add_argument_group('Input arguments')
+ group.add_argument('--from-wiki', action='store_true',
+ help='Pull special phrases from the OSM wiki.')
+ group = parser.add_argument_group('Output arguments')
+ group.add_argument('-o', '--output', default='-',
+ type=argparse.FileType('w', encoding='UTF-8'),
+ help="""File to write the preprocessed phrases to.
+ If omitted, it will be written to stdout.""")
+
+ @staticmethod
+ def run(args):
+ if args.output.name != '<stdout>':
+ raise NotImplementedError('Only output to stdout is currently implemented.')
+ return run_legacy_script('specialphrases.php', '--wiki-import', nominatim_env=args)
+
+
+class UpdateReplication:
+ """\
+ Update the database using an online replication service.
+ """
+
+ @staticmethod
+ def add_args(parser):
+ group = parser.add_argument_group('Arguments for initialisation')
+ group.add_argument('--init', action='store_true',
+ help='Initialise the update process')
+ group.add_argument('--no-update-functions', dest='update_functions',
+ action='store_false',
+ help="""Do not update the trigger function to
+ support differential updates.""")
+ group = parser.add_argument_group('Arguments for updates')
+ group.add_argument('--check-for-updates', action='store_true',
+ help='Check if new updates are available and exit')
+ group.add_argument('--once', action='store_true',
+ help="""Download and apply updates only once. When
+ not set, updates are continuously applied""")
+ group.add_argument('--no-index', action='store_false', dest='do_index',
+ help="""Do not index the new data. Only applicable
+ together with --once""")
+
+ @staticmethod
+ def run(args):
+ params = ['update.php']
+ if args.init:
+ params.append('--init-updates')
+ if not args.update_functions:
+ params.append('--no-update-functions')
+ elif args.check_for_updates:
+ params.append('--check-for-updates')
+ else:
+ if args.once:
+ params.append('--import-osmosis')
+ else:
+ params.append('--import-osmosis-all')
+ if not args.do_index:
+ params.append('--no-index')
+
+ return run_legacy_script(*params, nominatim_env=args)
+
+
+class UpdateAddData:
+ """\
+ Add additional data from a file or an online source.
+
+ Data is only imported, not indexed. You need to call `nominatim-update index`
+ to complete the process.
+ """
+
+ @staticmethod
+ def add_args(parser):
+ group_name = parser.add_argument_group('Source')
+ group = group_name.add_mutually_exclusive_group(required=True)
+ group.add_argument('--file', metavar='FILE',
+ help='Import data from an OSM file')
+ group.add_argument('--diff', metavar='FILE',
+ help='Import data from an OSM diff file')
+ group.add_argument('--node', metavar='ID', type=int,
+ help='Import a single node from the API')
+ group.add_argument('--way', metavar='ID', type=int,
+ help='Import a single way from the API')
+ group.add_argument('--relation', metavar='ID', type=int,
+ help='Import a single relation from the API')
+ group.add_argument('--tiger-data', metavar='DIR',
+ help='Add housenumbers from the US TIGER census database.')
+ group = parser.add_argument_group('Extra arguments')
+ group.add_argument('--use-main-api', action='store_true',
+ help='Use OSM API instead of Overpass to download objects')
+
+ @staticmethod
+ def run(args):
+ if args.tiger_data:
+ os.environ['NOMINATIM_TIGER_DATA_PATH'] = args.tiger_data
+ return run_legacy_script('setup.php', '--import-tiger-data', nominatim_env=args)
+
+ params = ['update.php']
+ if args.file:
+ params.extend(('--import-file', args.file))
+ elif args.diff:
+ params.extend(('--import-diff', args.diff))
+ elif args.node:
+ params.extend(('--import-node', args.node))
+ elif args.way:
+ params.extend(('--import-way', args.way))
+ elif args.relation:
+ params.extend(('--import-relation', args.relation))
+ if args.use_main_api:
+ params.append('--use-main-api')
+ return run_legacy_script(*params, nominatim_env=args)
+
+
+class UpdateIndex:
+ """\
+ Reindex all new and modified data.
+ """
+
+ @staticmethod
+ def add_args(parser):
+ pass
+
+ @staticmethod
+ def run(args):
+ return run_legacy_script('update.php', '--index', nominatim_env=args)
+
+
+class UpdateRefresh:
+ """\
+ Recompute auxiliary data used by the indexing process.
+
+ These functions must not be run in parallel with other update commands.
+ """
+
+ @staticmethod
+ def add_args(parser):
+ group = parser.add_argument_group('Data arguments')
+ group.add_argument('--postcodes', action='store_true',
+ help='Update postcode centroid table')
+ group.add_argument('--word-counts', action='store_true',
+ help='Compute frequency of full-word search terms')
+ group.add_argument('--address-levels', action='store_true',
+ help='Reimport address level configuration')
+ group.add_argument('--functions', action='store_true',
+ help='Update the PL/pgSQL functions in the database')
+ group.add_argument('--wiki-data', action='store_true',
+ help='Update Wikipedia/data importance numbers.')
+ group.add_argument('--importance', action='store_true',
+ help='Recompute place importances (expensive!)')
+ group.add_argument('--website', action='store_true',
+ help='Refresh the directory that serves the scripts for the web API')
+ group = parser.add_argument_group('Arguments for function refresh')
+ group.add_argument('--no-diff-updates', action='store_false', dest='diffs',
+ help='Do not enable code for propagating updates')
+ group.add_argument('--enable-debug-statements', action='store_true',
+ help='Enable debug warning statements in functions')
+
+ @staticmethod
+ def run(args):
+ if args.postcodes:
+ run_legacy_script('update.php', '--calculate-postcodes',
+ nominatim_env=args, throw_on_fail=True)
+ if args.word_counts:
+ run_legacy_script('update.php', '--recompute-word-counts',
+ nominatim_env=args, throw_on_fail=True)
+ if args.address_levels:
+ run_legacy_script('update.php', '--update-address-levels',
+ nominatim_env=args, throw_on_fail=True)
+ if args.functions:
+ params = ['setup.php', '--create-functions', '--create-partition-functions']
+ if args.diffs:
+ params.append('--enable-diff-updates')
+ if args.enable_debug_statements:
+ params.append('--enable-debug-statements')
+ run_legacy_script(*params, nominatim_env=args, throw_on_fail=True)
+ if args.wiki_data:
+ run_legacy_script('setup.php', '--import-wikipedia-articles',
+ nominatim_env=args, throw_on_fail=True)
+ # Attention: importance MUST come after wiki data import.
+ if args.importance:
+ run_legacy_script('update.php', '--recompute-importance',
+ nominatim_env=args, throw_on_fail=True)
+ if args.website:
+ run_legacy_script('setup.php', '--setup-website',
+ nominatim_env=args, throw_on_fail=True)
+
+
+class AdminCheckDatabase:
+ """\
+ Check that the database is complete and operational.
+ """
+
+ @staticmethod
+ def add_args(parser):
+ pass # No options
+
+ @staticmethod
+ def run(args):
+ return run_legacy_script('check_import_finished.php', nominatim_env=args)
+
+
+class AdminWarm:
+ """\
+ Warm database caches for search and reverse queries.
+ """
+
+ @staticmethod
+ def add_args(parser):
+ group = parser.add_argument_group('Target arguments')
+ group.add_argument('--search-only', action='store_const', dest='target',
+ const='search',
+ help="Only pre-warm tables for search queries")
+ group.add_argument('--reverse-only', action='store_const', dest='target',
+ const='reverse',
+ help="Only pre-warm tables for reverse queries")
+
+ @staticmethod
+ def run(args):
+ params = ['warm.php']
+ if args.target == 'reverse':
+ params.append('--reverse-only')
+ if args.target == 'search':
+ params.append('--search-only')
+ return run_legacy_script(*params, nominatim_env=args)
+
+
+class QueryExport:
+ """\
+ Export addresses as CSV file from a Nominatim database.
+ """
+
+ @staticmethod
+ def add_args(parser):
+ group = parser.add_argument_group('Output arguments')
+ group.add_argument('--output-type', default='street',
+ choices=('continent', 'country', 'state', 'county',
+ 'city', 'suburb', 'street', 'path'),
+ help='Type of places to output (default: street)')
+ group.add_argument('--output-format',
+ default='street;suburb;city;county;state;country',
+ help="""Semicolon-separated list of address types
+ (see --output-type). Multiple ranks can be
+ merged into one column by simply using a
+ comma-separated list.""")
+ group.add_argument('--output-all-postcodes', action='store_true',
+ help="""List all postcodes for address instead of
+ just the most likely one""")
+ group.add_argument('--language',
+ help="""Preferred language for output
+ (use local name, if omitted)""")
+ group = parser.add_argument_group('Filter arguments')
+ group.add_argument('--restrict-to-country', metavar='COUNTRY_CODE',
+ help='Export only objects within country')
+ group.add_argument('--restrict-to-osm-node', metavar='ID', type=int,
+ help='Export only children of this OSM node')
+ group.add_argument('--restrict-to-osm-way', metavar='ID', type=int,
+ help='Export only children of this OSM way')
+ group.add_argument('--restrict-to-osm-relation', metavar='ID', type=int,
+ help='Export only children of this OSM relation')
+
+
+ @staticmethod
+ def run(args):
+ params = ['export.php',
+ '--output-type', args.output_type,
+ '--output-format', args.output_format]
+ if args.output_all_postcodes:
+ params.append('--output-all-postcodes')
+ if args.language:
+ params.extend(('--language', args.language))
+ if args.restrict_to_country:
+ params.extend(('--restrict-to-country', args.restrict_to_country))
+ if args.restrict_to_osm_node:
+ params.extend(('--restrict-to-osm-node', args.restrict_to_osm_node))
+ if args.restrict_to_osm_way:
+ params.extend(('--restrict-to-osm-way', args.restrict_to_osm_way))
+ if args.restrict_to_osm_relation:
+ params.extend(('--restrict-to-osm-relation', args.restrict_to_osm_relation))
+
+ return run_legacy_script(*params, nominatim_env=args)
+
+class QueryTodo:
+ """\
+ Todo
+ """
+ @staticmethod
+ def add_args(parser):
+ pass
+
+ @staticmethod
+ def run(args): # pylint: disable=W0613
+ print("TODO: searching")
+
+
+def nominatim(**kwargs):
+ """\
+ Command-line tools for importing, updating, administrating and
+ querying the Nominatim database.
+ """
+ parser = CommandlineParser('nominatim', nominatim.__doc__)
+
+ parser.add_subcommand('import', SetupAll)
+ parser.add_subcommand('freeze', SetupFreeze)
+ parser.add_subcommand('replication', UpdateReplication)
+
+ parser.add_subcommand('check-database', AdminCheckDatabase)
+ parser.add_subcommand('warm', AdminWarm)
+
+ parser.add_subcommand('special-phrases', SetupSpecialPhrases)
+
+ parser.add_subcommand('add-data', UpdateAddData)
+ parser.add_subcommand('index', UpdateIndex)
+ parser.add_subcommand('refresh', UpdateRefresh)
+
+ parser.add_subcommand('export', QueryExport)
+ parser.add_subcommand('search', QueryTodo)
+ parser.add_subcommand('reverse', QueryTodo)
+ parser.add_subcommand('lookup', QueryTodo)
+ parser.add_subcommand('details', QueryTodo)
+ parser.add_subcommand('status', QueryTodo)
+
+ return parser.run(**kwargs)
--- /dev/null
+"""
+Nominatim configuration accessor.
+"""
+import os
+
+from dotenv import dotenv_values
+
+class Configuration:
+ """ Load and manage the project configuration.
+
+ Nominatim uses dotenv to configure the software. Configuration options
+ are resolved in the following order:
+
+ * from the OS environment
+ * from the .env file in the project directory of the installation
+ * from the default installation in the configuration directory
+
+ All Nominatim configuration options are prefixed with 'NOMINATIM_' to
+ avoid conflicts with other environment variables.
+ """
+
+ def __init__(self, project_dir, config_dir):
+ self._config = dotenv_values(str((config_dir / 'env.defaults').resolve()))
+ if project_dir is not None:
+ self._config.update(dotenv_values(str((project_dir / '.env').resolve())))
+
+ def __getattr__(self, name):
+ name = 'NOMINATIM_' + name
+
+ return os.environ.get(name) or self._config[name]
+
+ def get_os_env(self):
+ """ Return a copy of the OS environment with the Nominatim configuration
+ merged in.
+ """
+ env = dict(self._config)
+ env.update(os.environ)
+
+ return env
-from behave import *
from pathlib import Path
+from behave import *
+
from steps.geometry_factory import GeometryFactory
from steps.nominatim_environment import NominatimEnvironment
-import os
from pathlib import Path
+import sys
import tempfile
import psycopg2
import psycopg2.extras
+sys.path.insert(1, str((Path(__file__) / '..' / '..' / '..' / '..').resolve()))
+
+from nominatim.config import Configuration
from steps.utils import run_script
class NominatimEnvironment:
self.code_coverage_path = config['PHPCOV']
self.code_coverage_id = 1
+ self.default_config = Configuration(None, self.src_dir / 'settings').get_os_env()
self.test_env = None
self.template_db_done = False
self.api_db_done = False
and dsn == self.test_env['NOMINATIM_DATABASE_DSN']:
return # environment already set uo
- self.test_env = os.environ
+ self.test_env = dict(self.default_config)
self.test_env['NOMINATIM_DATABASE_DSN'] = dsn
self.test_env['NOMINATIM_FLATNODE_FILE'] = ''
self.test_env['NOMINATIM_IMPORT_STYLE'] = 'full'
self.test_env['NOMINATIM_USE_US_TIGER_DATA'] = 'yes'
+ self.test_env['NOMINATIM_DATADIR'] = self.src_dir
+ self.test_env['NOMINATIM_BINDIR'] = self.src_dir / 'utils'
+ self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.build_dir / 'module'
+ self.test_env['NOMINATIM_OSM2PGSQL_BINARY'] = self.build_dir / 'osm2pgsql' / 'osm2pgsql'
if self.server_module_path:
self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path
""" Run one of the Nominatim utility scripts with the given arguments.
"""
cmd = ['/usr/bin/env', 'php', '-Cq']
- cmd.append((Path(self.build_dir) / 'utils' / '{}.php'.format(script)).resolve())
+ cmd.append((Path(self.src_dir) / 'lib' / 'admin' / '{}.php'.format(script)).resolve())
cmd.extend(['--' + x for x in args])
for k, v in kwargs.items():
cmd.extend(('--' + k.replace('_', '-'), str(v)))
if self.website_dir is not None:
cwd = self.website_dir.name
else:
- cwd = self.build_dir
+ cwd = None
run_script(cmd, cwd=cwd, env=self.test_env)
""" Query directly via PHP script.
"""
cmd = ['/usr/bin/env', 'php']
- cmd.append(os.path.join(context.nominatim.build_dir, 'utils', 'query.php'))
+ cmd.append(context.nominatim.src_dir / 'lib' / 'admin' / 'query.php')
if query:
cmd.extend(['--search', query])
# add more parameters in table form
if dups:
cmd.extend(('--dedupe', '0'))
- outp, err = run_script(cmd, cwd=context.nominatim.build_dir)
+ outp, err = run_script(cmd, cwd=context.nominatim.website_dir.name,
+ env=context.nominatim.test_env)
context.response = SearchResponse(outp, 'json')
# Installing the Required Software
# ================================
#
-# !!! caution
-# These instructions are currently broken because they do not
-# include installation of the required PHP library symfony-dotenv.
-#
# These instructions expect that you have a freshly installed CentOS version 7.
# Make sure all packages are up-to-date by running:
#
python3-pip python3-setuptools python3-devel \
expat-devel zlib-devel
- pip3 install --user psycopg2
+ pip3 install --user psycopg2 python-dotenv
#
# Installing the Required Software
# ================================
#
-# !!! caution
-# These instructions are currently broken because they do not
-# include installation of the required PHP library symfony-dotenv.
-#
# These instructions expect that you have a freshly installed CentOS version 8.
# Make sure all packages are up-to-date by running:
#
python3-pip python3-setuptools python3-devel \
expat-devel zlib-devel
- pip3 install --user psycopg2
+ pip3 install --user psycopg2 python-dotenv
#
libbz2-dev libpq-dev libproj-dev \
postgresql-server-dev-10 postgresql-10-postgis-2.4 \
postgresql-contrib-10 postgresql-10-postgis-scripts \
- php php-pgsql php-intl php-symfony-dotenv \
+ php php-pgsql php-intl python3-dotenv \
python3-psycopg2 git
libbz2-dev libpq-dev libproj-dev \
postgresql-server-dev-12 postgresql-12-postgis-3 \
postgresql-contrib-12 postgresql-12-postgis-3-scripts \
- php php-pgsql php-intl php-symfony-dotenv \
+ php php-pgsql php-intl python3-dotenv \
python3-psycopg2 git
#
projects / nominatim.git / commitdiff