From 3afd12f977e8793f0f63a76e133d5e7ca11b0a3d Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 2 Oct 2018 22:59:10 +0200 Subject: [PATCH] simplify constructor of SetupFunctions Also cleans up spacing. --- lib/setup/SetupClass.php | 21 ++++-------- utils/setup.php | 70 ++++++++++++++++++++-------------------- utils/update.php | 67 ++++++++++++++++++++------------------ 3 files changed, 77 insertions(+), 81 deletions(-) diff --git a/lib/setup/SetupClass.php b/lib/setup/SetupClass.php index fbcf9b5c..bd53260e 100755 --- a/lib/setup/SetupClass.php +++ b/lib/setup/SetupClass.php @@ -16,14 +16,13 @@ class SetupFunctions protected $oDB = null; // set in setupDB (earliest) or later in loadData, importData, drop, createSqlFunctions, importTigerData // pgsqlRunPartitionScript, calculatePostcodes, ..if no already set - public function __construct($callingFunction, array $aCMDResult = array()) + public function __construct(array $aCMDResult) { // by default, use all but one processor, but never more than 15. $this->iInstances = isset($aCMDResult['threads']) ? $aCMDResult['threads'] : (min(16, getProcessorCount()) - 1); - if ($this->iInstances < 1) { $this->iInstances = 1; warn('resetting threads to '.$this->iInstances); @@ -38,11 +37,11 @@ class SetupFunctions $this->sModulePath = CONST_Database_Module_Path; info('module path: ' . $this->sModulePath); - + // prepares DB for import or update, sets the Data Source Name $this->aDSNInfo = \DB::parseDSN(CONST_Database_DSN); if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432; - + // setting member variables based on command line options stored in $aCMDResult $this->sVerbose = $aCMDResult['verbose']; @@ -62,13 +61,10 @@ class SetupFunctions } else { $this->bNoPartitions = false; } - - // if class is instantiated by update.php, we have to set EnableDiffUpdates to true - // otherwise set to value provided by setup.php's command line arg array - if ($callingFunction == 'update') { - $this->bEnableDiffUpdates = true; - } elseif ($callingFunction == 'setup') { + if (isset($aCMDResult['enable-diff-updates'])) { $this->bEnableDiffUpdates = $aCMDResult['enable-diff-updates']; + } else { + $this->bEnableDiffUpdates = false; } } @@ -158,7 +154,6 @@ class SetupFunctions $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql.gz'); $this->pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_table.sql'); - if (file_exists(CONST_BasePath.'/data/gb_postcode_data.sql.gz')) { $this->pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_data.sql.gz'); } else { @@ -192,8 +187,6 @@ class SetupFunctions fail("osm2pgsql not found in '$osm2pgsql'"); } - - if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File; } @@ -749,7 +742,7 @@ class SetupFunctions } $iReturn = runWithEnv($sCMD, $aProcEnv); // /lib/cmd.php "function runWithEnv($sCmd, $aEnv)" } - + private function pgsqlRunScript($sScript, $bfatal = true) { runSQLScript( diff --git a/utils/setup.php b/utils/setup.php index bdd3f03b..ea3e14a3 100755 --- a/utils/setup.php +++ b/utils/setup.php @@ -15,40 +15,40 @@ use Nominatim\Setup\SetupFunctions as SetupFunctions; // (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help) $aCMDOptions = array( - 'Create and setup nominatim search system', - array('help', 'h', 0, 1, 0, 0, false, 'Show Help'), - array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), - array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), - - array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'), - array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'), - - array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'), - - array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'), - array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'), - array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'), - array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'), - array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'), - array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'), - array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'), - array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'), - array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'), - array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'), - array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'), - array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'), - array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'), - array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'), - array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'), - array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'), - array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'), - array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'), - array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'), - array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'), - array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'), - array('create-country-names', '', 0, 1, 0, 0, 'bool', 'Create default list of searchable country names'), - array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'), -); + 'Create and setup nominatim search system', + array('help', 'h', 0, 1, 0, 0, false, 'Show Help'), + array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), + array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), + + array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'), + array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'), + + array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'), + + array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'), + array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'), + array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'), + array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'), + array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'), + array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'), + array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'), + array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'), + array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'), + array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'), + array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'), + array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'), + array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'), + array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'), + array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'), + array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'), + array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'), + array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'), + array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'), + array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'), + array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'), + array('create-country-names', '', 0, 1, 0, 0, 'bool', 'Create default list of searchable country names'), + array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'), + ); // $aCMDOptions passed to getCmdOpt by reference getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true); @@ -71,7 +71,7 @@ if ($aCMDResult['osmosis-init']) { // ****************************************************** // instantiate Setup class -$cSetup = new SetupFunctions('setup', $aCMDResult); +$cSetup = new SetupFunctions($aCMDResult); // ******************************************************* // go through complete process if 'all' is selected or start selected functions diff --git a/utils/update.php b/utils/update.php index dc8389ab..31f32f60 100755 --- a/utils/update.php +++ b/utils/update.php @@ -13,37 +13,37 @@ use Nominatim\Setup\SetupFunctions as SetupFunctions; // (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help) $aCMDOptions = array( - 'Import / update / index osm data', - array('help', 'h', 0, 1, 0, 0, false, 'Show Help'), - array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), - array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), - - array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'), - array('check-for-updates', '', 0, 1, 0, 0, 'bool', 'Check if new updates are available'), - array('no-update-functions', '', 0, 1, 0, 0, 'bool', 'Do not update trigger functions to support differential updates (assuming the diff update logic is already present)'), - array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'), - array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'), - array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'), - - array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Update postcode centroid table'), - - array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'), - array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'), - array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'), - - array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'), - array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'), - array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'), - array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'), - - array('index', '', 0, 1, 0, 0, 'bool', 'Index'), - array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'), - array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'), - - array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'), - array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'), - array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'), -); + 'Import / update / index osm data', + array('help', 'h', 0, 1, 0, 0, false, 'Show Help'), + array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), + array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), + + array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'), + array('check-for-updates', '', 0, 1, 0, 0, 'bool', 'Check if new updates are available'), + array('no-update-functions', '', 0, 1, 0, 0, 'bool', 'Do not update trigger functions to support differential updates (assuming the diff update logic is already present)'), + array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'), + array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'), + array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'), + + array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Update postcode centroid table'), + + array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'), + array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'), + array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'), + + array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'), + array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'), + array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'), + array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'), + + array('index', '', 0, 1, 0, 0, 'bool', 'Index'), + array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'), + array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'), + + array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'), + array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'), + array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'), + ); getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true); @@ -106,7 +106,10 @@ if ($aResult['init-updates']) { if (!$aResult['no-update-functions']) { // instantiate setupClass to use the function therein - $cSetup = new SetupFunctions('update'); + $cSetup = new SetupFunctions(array( + 'enable-diff-updates' => true, + 'verbose' => $aResult['verbose'] + )); $cSetup->createFunctions(); } -- 2.43.2