From: Sarah Hoffmann Date: Fri, 22 Oct 2021 12:41:14 +0000 (+0200) Subject: replace NOMINATIM_PHRASE_CONFIG with command line option X-Git-Tag: v4.0.0~13^2~3 X-Git-Url: https://git.openstreetmap.org/nominatim.git/commitdiff_plain/c77df2d1ebb40bbdf28aa337490e7af2861b334d replace NOMINATIM_PHRASE_CONFIG with command line option --- diff --git a/docs/admin/Migration.md b/docs/admin/Migration.md index a8c1375d..8458e3d9 100644 --- a/docs/admin/Migration.md +++ b/docs/admin/Migration.md @@ -15,6 +15,20 @@ breaking changes. **Please read them before running the migration.** If you are migrating from a version <3.6, then you still have to follow the manual migration steps up to 3.6. +## 3.7.0 -> master + +### NOMINATIM_PHRASE_CONFIG removed + +Custom blacklist configurations for special phrases now need to be handed +with the `--config` parameter to `nominatim special-phrases`. Alternatively +you can put your custom configuration in the project directory in a file +named `phrase-settings.json`. + +Version 3.8 also removes the automatic converter for the php format of +the configuration in older versions. If you are updating from Nominatim < 3.7 +and still work with a custom `phrase-settings.php`, you need to manually +convert it into a json format. + ## 3.6.0 -> 3.7.0 ### New format and name of configuration file diff --git a/docs/customize/Settings.md b/docs/customize/Settings.md index 5796ed50..f34f85b1 100644 --- a/docs/customize/Settings.md +++ b/docs/customize/Settings.md @@ -303,19 +303,6 @@ Set a custom location for the [wikipedia ranking file](../admin/Import.md#wikipediawikidata-rankings). When unset, Nominatim expects the data to be saved in the project directory. -#### NOMINATIM_PHRASE_CONFIG - -| Summary | | -| -------------- | --------------------------------------------------- | -| **Description:** | Configuration file for special phrase imports | -| **Format:** | path | -| **Default:** | _empty_ (use default settings) | - -The _phrase_config_ file configures black and white lists of tag types, -so that some of them can be ignored, when loading special phrases from -the OSM wiki. The default settings can be found in the configuration -directory as `phrase-settings.json`. - #### NOMINATIM_ADDRESS_LEVEL_CONFIG | Summary | | diff --git a/lib-php/migration/PhraseSettingsToJson.php b/lib-php/migration/PhraseSettingsToJson.php deleted file mode 100644 index ac6e6213..00000000 --- a/lib-php/migration/PhraseSettingsToJson.php +++ /dev/null @@ -1,21 +0,0 @@ - None: + def __init__(self, config, db_connection, sp_loader) -> None: self.config = config - self.phplib_dir = phplib_dir self.db_connection = db_connection self.sp_loader = sp_loader self.statistics_handler = SpecialPhrasesImporterStatistics() @@ -101,13 +94,8 @@ class SPImporter(): """ Load white and black lists from phrases-settings.json. """ - settings_path = (self.config.config_dir / 'phrase-settings.json').resolve() + settings = self.config.load_sub_configuration('phrase-settings.json') - if self.config.PHRASE_CONFIG: - settings_path = self._convert_php_settings_if_needed(self.config.PHRASE_CONFIG) - - with settings_path.open("r") as json_settings: - settings = json.load(json_settings) return settings['blackList'], settings['whiteList'] def _check_sanity(self, phrase): @@ -255,29 +243,3 @@ class SPImporter(): for table in self.table_phrases_to_delete: self.statistics_handler.notify_one_table_deleted() db_cursor.drop_table(table) - - - def _convert_php_settings_if_needed(self, file_path): - """ - Convert php settings file of special phrases to json file if it is still in php format. - """ - if not isfile(file_path): - raise UsageError(str(file_path) + ' is not a valid file.') - - file, extension = os.path.splitext(file_path) - json_file_path = Path(file + '.json').resolve() - - if extension not in ('.php', '.json'): - raise UsageError('The custom NOMINATIM_PHRASE_CONFIG file has not a valid extension.') - - if extension == '.php' and not isfile(json_file_path): - try: - subprocess.run(['/usr/bin/env', 'php', '-Cq', - (self.phplib_dir / 'migration/PhraseSettingsToJson.php').resolve(), - file_path], check=True) - LOG.warning('special_phrase configuration file has been converted to json.') - except subprocess.CalledProcessError: - LOG.error('Error while converting %s to json.', file_path) - raise - - return json_file_path diff --git a/settings/env.defaults b/settings/env.defaults index 3fb128dc..2ece74dc 100644 --- a/settings/env.defaults +++ b/settings/env.defaults @@ -89,8 +89,8 @@ NOMINATIM_TIGER_DATA_PATH= NOMINATIM_WIKIPEDIA_DATA_PATH= # Configuration file for special phrase import. -# When unset, the internal default settings from 'settings/phrase-settings.json' -# are used. +# OBSOLETE: use `nominatim special-phrases --config ` or simply put +# a custom phrase-settings.json into your project directory. NOMINATIM_PHRASE_CONFIG= # Configuration file for rank assignments. diff --git a/settings/phrase-settings.json b/settings/phrase-settings.json index a097dca4..5d3ef6eb 100644 --- a/settings/phrase-settings.json +++ b/settings/phrase-settings.json @@ -6,7 +6,7 @@ "Also use this list to exclude an entire class from special phrases." ], "blackList": { - "bounday": [ + "boundary": [ "administrative" ], "place": [ diff --git a/test/python/test_tools_import_special_phrases.py b/test/python/test_tools_import_special_phrases.py index f0a34b08..7c3d0646 100644 --- a/test/python/test_tools_import_special_phrases.py +++ b/test/python/test_tools_import_special_phrases.py @@ -17,30 +17,12 @@ def testfile_dir(src_dir): @pytest.fixture -def sp_importer(temp_db_conn, def_config, temp_phplib_dir_with_migration): +def sp_importer(temp_db_conn, def_config): """ Return an instance of SPImporter. """ loader = SPWikiLoader(def_config, ['en']) - return SPImporter(def_config, temp_phplib_dir_with_migration, temp_db_conn, loader) - - -@pytest.fixture -def temp_phplib_dir_with_migration(src_dir, tmp_path): - """ - Return temporary phpdir with migration subdirectory and - PhraseSettingsToJson.php script inside. - """ - migration_file = (src_dir / 'lib-php' / 'migration' / 'PhraseSettingsToJson.php').resolve() - - phpdir = tmp_path / 'tempphp' - phpdir.mkdir() - - (phpdir / 'migration').mkdir() - migration_dest_path = (phpdir / 'migration' / 'PhraseSettingsToJson.php').resolve() - copyfile(str(migration_file), str(migration_dest_path)) - - return phpdir + return SPImporter(def_config, temp_db_conn, loader) @pytest.fixture @@ -90,49 +72,6 @@ def test_load_white_and_black_lists(sp_importer): assert isinstance(black_list, dict) and isinstance(white_list, dict) -def test_convert_php_settings(sp_importer, testfile_dir, tmp_path): - """ - Test that _convert_php_settings_if_needed() convert the given - php file to a json file. - """ - php_file = (testfile_dir / 'phrase_settings.php').resolve() - - temp_settings = (tmp_path / 'phrase_settings.php').resolve() - copyfile(php_file, temp_settings) - sp_importer._convert_php_settings_if_needed(temp_settings) - - assert (tmp_path / 'phrase_settings.json').is_file() - -def test_convert_settings_wrong_file(sp_importer): - """ - Test that _convert_php_settings_if_needed() raise an exception - if the given file is not a valid file. - """ - with pytest.raises(UsageError, match='random_file is not a valid file.'): - sp_importer._convert_php_settings_if_needed('random_file') - -def test_convert_settings_json_already_exist(sp_importer, testfile_dir): - """ - Test that if we give to '_convert_php_settings_if_needed' a php file path - and that a the corresponding json file already exists, it is returned. - """ - php_file = (testfile_dir / 'phrase_settings.php').resolve() - json_file = (testfile_dir / 'phrase_settings.json').resolve() - - returned = sp_importer._convert_php_settings_if_needed(php_file) - - assert returned == json_file - -def test_convert_settings_giving_json(sp_importer, testfile_dir): - """ - Test that if we give to '_convert_php_settings_if_needed' a json file path - the same path is directly returned - """ - json_file = (testfile_dir / 'phrase_settings.json').resolve() - - returned = sp_importer._convert_php_settings_if_needed(json_file) - - assert returned == json_file def test_create_place_classtype_indexes(temp_db_with_extensions, temp_db_conn, table_factory, sp_importer):