From: Sarah Hoffmann Date: Thu, 2 Aug 2018 21:16:07 +0000 (+0200) Subject: Merge pull request #1095 from estadtherr/remote_postgres_pr X-Git-Tag: v3.2.0~36 X-Git-Url: https://git.openstreetmap.org/nominatim.git/commitdiff_plain/7f10264fb6d4cc89ca818b822e8db10aeefd969e?hp=7e0fdf5928a0c726b22ad969267a3b9e0402c823 Merge pull request #1095 from estadtherr/remote_postgres_pr Enable Postgres to run on a different host than the web server --- diff --git a/data/words.sql b/data/words.sql index 85578f20..ac250739 100644 --- a/data/words.sql +++ b/data/words.sql @@ -18,12 +18,13 @@ SET default_with_oids = false; -- Name: word_frequencies; Type: TABLE; Schema: public; Owner: -; Tablespace: -- +DROP TABLE IF EXISTS word_frequencies; + CREATE TABLE word_frequencies ( word_token text, count bigint ); - -- -- Data for Name: word_frequencies; Type: TABLE DATA; Schema: public; Owner: - -- diff --git a/lib/cmd.php b/lib/cmd.php index 28d56f2e..9ec290d1 100644 --- a/lib/cmd.php +++ b/lib/cmd.php @@ -158,6 +158,16 @@ function runSQLScript($sScript, $bfatal = true, $bVerbose = false, $bIgnoreError $aDSNInfo = DB::parseDSN(CONST_Database_DSN); if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database']; + if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { + $sCMD .= ' -h ' . $aDSNInfo['hostspec']; + } + if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { + $sCMD .= ' -U ' . $aDSNInfo['username']; + } + $aProcEnv = null; + if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); + } if (!$bVerbose) { $sCMD .= ' -q'; } @@ -170,15 +180,15 @@ function runSQLScript($sScript, $bfatal = true, $bVerbose = false, $bIgnoreError 2 => STDERR ); $ahPipes = null; - $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes); + $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv); if (!is_resource($hProcess)) { fail('unable to start pgsql'); } while (strlen($sScript)) { - $written = fwrite($ahPipes[0], $sScript); - if ($written <= 0) break; - $sScript = substr($sScript, $written); + $iWritten = fwrite($ahPipes[0], $sScript); + if ($iWritten <= 0) break; + $sScript = substr($sScript, $iWritten); } fclose($ahPipes[0]); $iReturn = proc_close($hProcess); @@ -186,3 +196,22 @@ function runSQLScript($sScript, $bfatal = true, $bVerbose = false, $bIgnoreError fail("pgsql returned with error code ($iReturn)"); } } + + +function runWithEnv($sCmd, $aEnv) +{ + $aFDs = array( + 0 => array('pipe', 'r'), + 1 => STDOUT, + 2 => STDERR); + $aPipes = null; + $hProc = @proc_open($sCmd, $aFDs, $aPipes, null, $aEnv); + if (!is_resource($hProc)) { + fail('unable to run command:' . $sCmd); + } + + fclose($aPipes[0]); // no stdin + + $iStat = proc_close($hProc); + return $iStat; +} diff --git a/nominatim/index.c b/nominatim/index.c index c16aba9e..bb553f7e 100644 --- a/nominatim/index.c +++ b/nominatim/index.c @@ -262,13 +262,16 @@ struct index_thread_data * thread_data, const char *structuredoutputfile) void nominatim_index(int rank_min, int rank_max, int num_threads, const char *conninfo, const char *structuredoutputfile) { - struct index_thread_data * thread_data; + struct index_thread_data *thread_data; PGconn *conn; - PGresult * res; + PGresult *res; + int num_rows = 0, status_code = 0; + int db_has_locale = 0; + char *result_string = NULL; int rank; - + int i; xmlTextWriterPtr writer; @@ -283,6 +286,23 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co exit(EXIT_FAILURE); } + res = PQexec(conn, "SHOW lc_messages"); + status_code = PQresultStatus(res); + if (status_code != PGRES_TUPLES_OK && status_code != PGRES_SINGLE_TUPLE) { + fprintf(stderr, "Failed determining database locale: %s\n", PQerrorMessage(conn)); + exit(EXIT_FAILURE); + } + num_rows = PQntuples(res); + if (num_rows > 0) + { + result_string = PQgetvalue(res, 0, 0); + if (result_string && (strlen(result_string) > 0) && (strcasecmp(result_string, "C") != 0)) + { + // non-default locale if the result exists, is non-empty, and is not "C" + db_has_locale = 1; + } + } + pg_prepare_params[0] = PG_OID_INT4; res = PQprepare(conn, "index_sectors", "select geometry_sector,count(*) from placex where rank_search = $1 and indexed_status > 0 group by geometry_sector order by geometry_sector", @@ -392,19 +412,20 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co } PQclear(res); - // Make sure the error message is not localized as we parse it later. - res = PQexec(thread_data[i].conn, "SET lc_messages TO 'C'"); - if (PQresultStatus(res) != PGRES_COMMAND_OK) + if (db_has_locale) { - fprintf(stderr, "Failed to set langauge: %s\n", PQerrorMessage(thread_data[i].conn)); - exit(EXIT_FAILURE); + // Make sure the error message is not localized as we parse it later. + res = PQexec(thread_data[i].conn, "SET lc_messages TO 'C'"); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + fprintf(stderr, "Failed to set langauge: %s\n", PQerrorMessage(thread_data[i].conn)); + exit(EXIT_FAILURE); + } + PQclear(res); } - PQclear(res); - nominatim_exportCreatePreparedQueries(thread_data[i].conn); } - fprintf(stderr, "Starting indexing rank (%i to %i) using %i threads\n", rank_min, rank_max, num_threads); for (rank = rank_min; rank <= rank_max; rank++) diff --git a/settings/defaults.php b/settings/defaults.php index 2d8f47d0..8cdbcb5a 100644 --- a/settings/defaults.php +++ b/settings/defaults.php @@ -9,6 +9,7 @@ if (isset($_GET['debug']) && $_GET['debug']) @define('CONST_Debug', true); @define('CONST_Debug', false); @define('CONST_Database_DSN', 'pgsql://@/nominatim'); // ://:@:/ @define('CONST_Database_Web_User', 'www-data'); +@define('CONST_Database_Module_Path', CONST_InstallPath.'/module'); @define('CONST_Max_Word_Frequency', '50000'); @define('CONST_Limit_Reindexing', true); // Restrict search languages. diff --git a/sql/partition-tables.src.sql b/sql/partition-tables.src.sql index 61ed5281..20dafcd7 100644 --- a/sql/partition-tables.src.sql +++ b/sql/partition-tables.src.sql @@ -48,6 +48,7 @@ CREATE INDEX idx_search_name_-partition-_place_id ON search_name_-partition- USI CREATE INDEX idx_search_name_-partition-_centroid ON search_name_-partition- USING GIST (centroid) {ts:address-index}; CREATE INDEX idx_search_name_-partition-_name_vector ON search_name_-partition- USING GIN (name_vector) WITH (fastupdate = off) {ts:address-index}; +DROP TABLE IF EXISTS location_road_-partition-; CREATE TABLE location_road_-partition- ( place_id BIGINT, partition SMALLINT, diff --git a/test/README.md b/test/README.md index 2a357e40..0487fd40 100644 --- a/test/README.md +++ b/test/README.md @@ -73,6 +73,11 @@ The tests can be configured with a set of environment variables: the test databases (db tests) * `TEST_DB` - name of test database (db tests) * `ABI_TEST_DB` - name of the database containing the API test data (api tests) + * `DB_HOST` - (optional) hostname of database host + * `DB_USER` - (optional) username of database login + * `DB_PASS` - (optional) password for database login + * `SERVER_MODULE_PATH` - (optional) path on the Postgres server to Nominatim + * module shared library file * `TEST_SETTINGS_TEMPLATE` - file to write temporary Nominatim settings to * `REMOVE_TEMPLATE` - if true, the template database will not be reused during the next run. Reusing the base templates speeds up tests diff --git a/test/bdd/environment.py b/test/bdd/environment.py index 162346de..fdc65a5e 100644 --- a/test/bdd/environment.py +++ b/test/bdd/environment.py @@ -14,10 +14,14 @@ userconfig = { 'BUILDDIR' : os.path.join(os.path.split(__file__)[0], "../../build"), 'REMOVE_TEMPLATE' : False, 'KEEP_TEST_DB' : False, + 'DB_HOST' : None, + 'DB_USER' : None, + 'DB_PASS' : None, 'TEMPLATE_DB' : 'test_template_nominatim', 'TEST_DB' : 'test_nominatim', 'API_TEST_DB' : 'test_api_nominatim', 'TEST_SETTINGS_FILE' : '/tmp/nominatim_settings.php', + 'SERVER_MODULE_PATH' : None, 'PHPCOV' : False, # set to output directory to enable code coverage } @@ -30,9 +34,13 @@ class NominatimEnvironment(object): def __init__(self, config): self.build_dir = os.path.abspath(config['BUILDDIR']) self.src_dir = os.path.abspath(os.path.join(os.path.split(__file__)[0], "../..")) + self.db_host = config['DB_HOST'] + self.db_user = config['DB_USER'] + self.db_pass = config['DB_PASS'] self.template_db = config['TEMPLATE_DB'] self.test_db = config['TEST_DB'] self.api_test_db = config['API_TEST_DB'] + self.server_module_path = config['SERVER_MODULE_PATH'] self.local_settings_file = config['TEST_SETTINGS_FILE'] self.reuse_template = not config['REMOVE_TEMPLATE'] self.keep_scenario_db = config['KEEP_TEST_DB'] @@ -42,6 +50,17 @@ class NominatimEnvironment(object): self.template_db_done = False + def connect_database(self, dbname): + dbargs = {'database': dbname} + if self.db_host: + dbargs['host'] = self.db_host + if self.db_user: + dbargs['user'] = self.db_user + if self.db_pass: + dbargs['password'] = self.db_pass + conn = psycopg2.connect(**dbargs) + return conn + def next_code_coverage_file(self): fn = os.path.join(self.code_coverage_path, "%06d.cov" % self.code_coverage_id) self.code_coverage_id += 1 @@ -50,7 +69,11 @@ class NominatimEnvironment(object): def write_nominatim_config(self, dbname): f = open(self.local_settings_file, 'w') - f.write(" getProcessorCount()) { - $iInstances = getProcessorCount(); - warn("resetting threads to $iInstances"); -} // Assume we can steal all the cache memory in the box (unless told otherwise) if (isset($aCMDResult['osm2pgsql-cache'])) { @@ -82,6 +79,9 @@ if (isset($aCMDResult['osm2pgsql-cache'])) { $iCacheMemory = getCacheMemoryMB(); } +$sModulePath = CONST_Database_Module_Path; +info('module path: ' . $sModulePath); + $aDSNInfo = DB::parseDSN(CONST_Database_DSN); if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; @@ -92,7 +92,22 @@ if ($aCMDResult['create-db'] || $aCMDResult['all']) { if (!PEAR::isError($oDB)) { fail('database already exists ('.CONST_Database_DSN.')'); } - passthruCheckReturn('createdb -E UTF-8 -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']); + + $sCreateDBCmd = 'createdb -E UTF-8 -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']; + if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { + $sCreateDBCmd .= ' -U ' . $aDSNInfo['username']; + } + if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { + $sCreateDBCmd .= ' -h ' . $aDSNInfo['hostspec']; + } + + $aProcEnv = null; + if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); + } + + $result = runWithEnv($sCreateDBCmd, $aProcEnv); + if ($result != 0) fail('Error executing external command: '.$sCreateDBCmd); } if ($aCMDResult['setup-db'] || $aCMDResult['all']) { @@ -140,17 +155,8 @@ if ($aCMDResult['setup-db'] || $aCMDResult['all']) { exit(1); } - // Try accessing the C module, so we know early if something is wrong - // and can simply error out. - $sSQL = "CREATE FUNCTION nominatim_test_import_func(text) RETURNS text AS '"; - $sSQL .= CONST_InstallPath."/module/nominatim.so', 'transliteration' LANGUAGE c IMMUTABLE STRICT"; - $sSQL .= ';DROP FUNCTION nominatim_test_import_func(text);'; - $oResult = $oDB->query($sSQL); - - if (PEAR::isError($oResult)) { - echo "\nERROR: Failed to load nominatim module. Reason:\n"; - echo $oResult->userinfo."\n\n"; - exit(1); + if (!checkModulePresence()) { + fail('error loading nominatim.so module'); } if (!file_exists(CONST_ExtraDataPath.'/country_osm_grid.sql.gz')) { @@ -180,8 +186,8 @@ if ($aCMDResult['setup-db'] || $aCMDResult['all']) { // is only defined in the subsequently called create_tables. // Create dummies here that will be overwritten by the proper // versions in create-tables. - pgsqlRunScript('CREATE TABLE place_boundingbox ()'); - pgsqlRunScript('create type wikipedia_article_match as ()'); + pgsqlRunScript('CREATE TABLE IF NOT EXISTS place_boundingbox ()'); + pgsqlRunScript('CREATE TYPE wikipedia_article_match AS ()', false); } if ($aCMDResult['import-data'] || $aCMDResult['all']) { @@ -209,8 +215,20 @@ if ($aCMDResult['import-data'] || $aCMDResult['all']) { $osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1'; $osm2pgsql .= ' -C '.$iCacheMemory; $osm2pgsql .= ' -P '.$aDSNInfo['port']; + if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { + $osm2pgsql .= ' -U ' . $aDSNInfo['username']; + } + if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { + $osm2pgsql .= ' -H ' . $aDSNInfo['hostspec']; + } + + $aProcEnv = null; + if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); + } + $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file']; - passthruCheckReturn($osm2pgsql); + runWithEnv($osm2pgsql, $aProcEnv); $oDB =& getDB(); if (!$aCMDResult['ignore-errors'] && !chksql($oDB->getRow('select * from place limit 1'))) { @@ -221,9 +239,11 @@ if ($aCMDResult['import-data'] || $aCMDResult['all']) { if ($aCMDResult['create-functions'] || $aCMDResult['all']) { info('Create Functions'); $bDidSomething = true; - if (!file_exists(CONST_InstallPath.'/module/nominatim.so')) { - fail('nominatim module not built'); + + if (!checkModulePresence()) { + fail('error loading nominatim.so module'); } + create_sql_functions($aCMDResult); } @@ -413,14 +433,23 @@ if ($aCMDResult['load-data'] || $aCMDResult['all']) { fail(pg_last_error($aDBInstances[$iLoadThreads]->connection)); } - $bAnyBusy = true; - while ($bAnyBusy) { - $bAnyBusy = false; - for ($i = 0; $i <= $iLoadThreads; $i++) { - if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true; + $bFailed = false; + for ($i = 0; $i <= $iLoadThreads; $i++) { + while (($hPGresult = pg_get_result($aDBInstances[$i]->connection)) !== false) { + $resultStatus = pg_result_status($hPGresult); + // PGSQL_EMPTY_QUERY, PGSQL_COMMAND_OK, PGSQL_TUPLES_OK, + // PGSQL_COPY_OUT, PGSQL_COPY_IN, PGSQL_BAD_RESPONSE, + // PGSQL_NONFATAL_ERROR and PGSQL_FATAL_ERROR + echo 'Query result ' . $i . ' is: ' . $resultStatus . "\n"; + if ($resultStatus != PGSQL_COMMAND_OK && $resultStatus != PGSQL_TUPLES_OK) { + $resultError = pg_result_error($hPGresult); + echo '-- error text ' . $i . ': ' . $resultError . "\n"; + $bFailed = true; + } } - sleep(1); - echo '.'; + } + if ($bFailed) { + fail('SQL errors loading placex and/or location_property_osmline tables'); } echo "\n"; info('Reanalysing database'); @@ -579,14 +608,34 @@ if ($aCMDResult['index'] || $aCMDResult['all']) { $bDidSomething = true; $sOutputFile = ''; $sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$iInstances.$sOutputFile; + if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { + $sBaseCmd .= ' -H ' . $aDSNInfo['hostspec']; + } + if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { + $sBaseCmd .= ' -U ' . $aDSNInfo['username']; + } + $aProcEnv = null; + if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); + } + info('Index ranks 0 - 4'); - passthruCheckReturn($sBaseCmd.' -R 4'); + $iStatus = runWithEnv($sBaseCmd.' -R 4', $aProcEnv); + if ($iStatus != 0) { + fail('error status ' . $iStatus . ' running nominatim!'); + } if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE'); info('Index ranks 5 - 25'); - passthruCheckReturn($sBaseCmd.' -r 5 -R 25'); + $iStatus = runWithEnv($sBaseCmd.' -r 5 -R 25', $aProcEnv); + if ($iStatus != 0) { + fail('error status ' . $iStatus . ' running nominatim!'); + } if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE'); info('Index ranks 26 - 30'); - passthruCheckReturn($sBaseCmd.' -r 26'); + $iStatus = runWithEnv($sBaseCmd.' -r 26', $aProcEnv); + if ($iStatus != 0) { + fail('error status ' . $iStatus . ' running nominatim!'); + } info('Index postcodes'); $oDB =& getDB(); @@ -722,6 +771,16 @@ function pgsqlRunScriptFile($sFilename) if (!$aCMDResult['verbose']) { $sCMD .= ' -q'; } + if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { + $sCMD .= ' -h ' . $aDSNInfo['hostspec']; + } + if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { + $sCMD .= ' -U ' . $aDSNInfo['username']; + } + $aProcEnv = null; + if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); + } $ahGzipPipes = null; if (preg_match('/\\.gz$/', $sFilename)) { @@ -745,10 +804,9 @@ function pgsqlRunScriptFile($sFilename) 2 => array('file', '/dev/null', 'a') ); $ahPipes = null; - $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes); + $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv); if (!is_resource($hProcess)) fail('unable to start pgsql'); - // TODO: error checking while (!feof($ahPipes[1])) { echo fread($ahPipes[1], 4096); @@ -830,32 +888,24 @@ function pgsqlRunDropAndRestore($sDumpFile) $aDSNInfo = DB::parseDSN(CONST_Database_DSN); if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile; - - $aDescriptors = array( - 0 => array('pipe', 'r'), - 1 => array('pipe', 'w'), - 2 => array('file', '/dev/null', 'a') - ); - $ahPipes = null; - $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes); - if (!is_resource($hProcess)) fail('unable to start pg_restore'); - - fclose($ahPipes[0]); - - // TODO: error checking - while (!feof($ahPipes[1])) { - echo fread($ahPipes[1], 4096); + if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { + $sCMD .= ' -h ' . $aDSNInfo['hostspec']; + } + if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { + $sCMD .= ' -U ' . $aDSNInfo['username']; + } + $aProcEnv = null; + if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); } - fclose($ahPipes[1]); - $iReturn = proc_close($hProcess); + $iReturn = runWithEnv($sCMD, $aProcEnv); } -function passthruCheckReturn($cmd) +function passthruCheckReturn($sCmd) { - $result = -1; - passthru($cmd, $result); - if ($result != 0) fail('Error executing external command: '.$cmd); + $iResult = -1; + passthru($sCmd, $iResult); } function replace_tablespace($sTemplate, $sTablespace, $sSql) @@ -871,8 +921,9 @@ function replace_tablespace($sTemplate, $sTablespace, $sSql) function create_sql_functions($aCMDResult) { + global $sModulePath; $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql'); - $sTemplate = str_replace('{modulepath}', CONST_InstallPath.'/module', $sTemplate); + $sTemplate = str_replace('{modulepath}', $sModulePath, $sTemplate); if ($aCMDResult['enable-diff-updates']) { $sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate); } @@ -890,3 +941,26 @@ function create_sql_functions($aCMDResult) } pgsqlRunScript($sTemplate); } + +function checkModulePresence() +{ + // Try accessing the C module, so we know early if something is wrong + // and can simply error out. + global $sModulePath; + $sSQL = "CREATE FUNCTION nominatim_test_import_func(text) RETURNS text AS '"; + $sSQL .= $sModulePath."/nominatim.so', 'transliteration' LANGUAGE c IMMUTABLE STRICT"; + $sSQL .= ';DROP FUNCTION nominatim_test_import_func(text);'; + + $oDB =& getDB(); + $oResult = $oDB->query($sSQL); + + $bResult = true; + + if (PEAR::isError($oResult)) { + echo "\nERROR: Failed to load nominatim module. Reason:\n"; + echo $oResult->userinfo."\n\n"; + $bResult = false; + } + + return $bResult; +} diff --git a/utils/update.php b/utils/update.php index 6caa7e4b..c70bdf54 100755 --- a/utils/update.php +++ b/utils/update.php @@ -5,6 +5,8 @@ require_once(dirname(dirname(__FILE__)).'/settings/settings.php'); require_once(CONST_BasePath.'/lib/init-cmd.php'); ini_set('memory_limit', '800M'); +# (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help) + $aCMDOptions = array( 'Import / update / index osm data', @@ -14,6 +16,7 @@ $aCMDOptions array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'), array('check-for-updates', '', 0, 1, 0, 0, 'bool', 'Check if new updates are available'), + array('no-update-functions', '', 0, 1, 0, 0, 'bool', 'Do not update trigger functions to support differential updates (assuming the diff update logic is already present)'), array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'), array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'), array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'), @@ -56,6 +59,17 @@ if ($iCacheMemory + 500 > getTotalMemoryMB()) { echo "WARNING: resetting cache memory to $iCacheMemory\n"; } $sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas --number-processes 1 -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port']; +if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { + $sOsm2pgsqlCmd .= ' -U ' . $aDSNInfo['username']; +} +if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { + $sOsm2pgsqlCmd .= ' -H ' . $aDSNInfo['hostspec']; +} +$aProcEnv = null; +if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); +} + if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { $sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File; } @@ -84,11 +98,13 @@ if ($aResult['init-updates']) { echo "and have set up CONST_Pyosmium_Binary to point to pyosmium-get-changes.\n"; fail('pyosmium-get-changes not found or not usable'); } - $sSetup = CONST_InstallPath.'/utils/setup.php'; - $iRet = -1; - passthru($sSetup.' --create-functions --enable-diff-updates', $iRet); - if ($iRet != 0) { - fail('Error running setup script'); + if (!$aResult['no-update-functions']) { + $sSetup = CONST_InstallPath.'/utils/setup.php'; + $iRet = -1; + passthru($sSetup.' --create-functions --enable-diff-updates', $iRet); + if ($iRet != 0) { + fail('Error running setup script'); + } } $sDatabaseDate = getDatabaseDate($oDB); @@ -137,7 +153,7 @@ if (isset($aResult['import-diff']) || isset($aResult['import-file'])) { // Import the file $sCMD = $sOsm2pgsqlCmd.' '.$sNextFile; echo $sCMD."\n"; - exec($sCMD, $sJunk, $iErrorLevel); + $iErrorLevel = runWithEnv($sCMD, $aProcEnv); if ($iErrorLevel) { fail("Error from osm2pgsql, $iErrorLevel\n"); @@ -189,7 +205,7 @@ if ($bHaveDiff) { // import generated change file $sCMD = $sOsm2pgsqlCmd.' '.$sTemporaryFile; echo $sCMD."\n"; - exec($sCMD, $sJunk, $iErrorLevel); + $iErrorLevel = runWithEnv($sCMD, $aProcEnv); if ($iErrorLevel) { fail("osm2pgsql exited with error level $iErrorLevel\n"); } @@ -273,7 +289,15 @@ if ($aResult['recompute-word-counts']) { } if ($aResult['index']) { - passthru(CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']); + $sCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']; + if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { + $sCmd .= ' -H ' . $aDSNInfo['hostspec']; + } + if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { + $sCmd .= ' -U ' . $aDSNInfo['username']; + } + + runWithEnv($sCmd, $aProcEnv); } if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { @@ -287,6 +311,12 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { $sCMDDownload = CONST_Pyosmium_Binary.' --server '.CONST_Replication_Url.' -o '.$sImportFile.' -s '.CONST_Replication_Max_Diff_size; $sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile; $sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances']; + if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { + $sCMDIndex .= ' -H ' . $aDSNInfo['hostspec']; + } + if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { + $sCMDIndex .= ' -U ' . $aDSNInfo['username']; + } while (true) { $fStartTime = time(); @@ -354,7 +384,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { $fCMDStartTime = time(); echo $sCMDImport."\n"; unset($sJunk); - exec($sCMDImport, $sJunk, $iErrorLevel); + $iErrorLevel = runWithEnv($sCMDImport, $aProcEnv); if ($iErrorLevel) { echo "Error executing osm2pgsql: $iErrorLevel\n"; exit($iErrorLevel); @@ -383,7 +413,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { $fCMDStartTime = time(); echo "$sThisIndexCmd\n"; - exec($sThisIndexCmd, $sJunk, $iErrorLevel); + $iErrorLevel = runWithEnv($sThisIndexCmd, $aProcEnv); if ($iErrorLevel) { echo "Error: $iErrorLevel\n"; exit($iErrorLevel);