from nominatim.db import utils as db_utils
from nominatim.db.connection import connect
+from nominatim.errors import UsageError
+
+def _flatten_name_list(names):
+ if names is None:
+ return {}
+
+ if not isinstance(names, dict):
+ raise UsageError("Expected key-value list for names in country_settings.py")
+
+ flat = {}
+ for prefix, remain in names.items():
+ if isinstance(remain, str):
+ flat[prefix] = remain
+ elif not isinstance(remain, dict):
+ raise UsageError("Entries in names must be key-value lists.")
+ else:
+ for suffix, name in remain.items():
+ if suffix == 'default':
+ flat[prefix] = name
+ else:
+ flat[f'{prefix}:{suffix}'] = name
+
+ return flat
+
class _CountryInfo:
def __init__(self):
self._info = {}
- self._key_prefix = 'name'
+
def load(self, config):
""" Load the country properties from the configuration files,
"""
if not self._info:
self._info = config.load_sub_configuration('country_settings.yaml')
- # Convert languages into a list for simpler handling.
for prop in self._info.values():
+ # Convert languages into a list for simpler handling.
if 'languages' not in prop:
prop['languages'] = []
elif not isinstance(prop['languages'], list):
prop['languages'] = [x.strip()
for x in prop['languages'].split(',')]
- if 'names' not in prop or prop['names'] is None:
- prop['names'] = {self._key_prefix: {}}
+ prop['names'] = _flatten_name_list(prop.get('names'))
+
def items(self):
""" Return tuples of (country_code, property dict) as iterable.
"""
return self._info.items()
- def key_prefix(self):
- """ Return the prefix that will be attached to the keys of the country
- names values when storing them in the database
+ def get(self, country_code):
+ """ Get country information for the country with the given country code.
"""
- return self._key_prefix
+ return self._info.get(country_code, {})
+
_COUNTRY_INFO = _CountryInfo()
"""
db_utils.execute_file(dsn, sql_dir / 'country_osm_grid.sql.gz')
- def add_prefix_to_keys(names, prefix):
- return {prefix+':'+k: v for k, v in names.items()}
-
params = []
for ccode, props in _COUNTRY_INFO.items():
if ccode is not None and props is not None:
partition = props.get('partition')
lang = props['languages'][0] if len(
props['languages']) == 1 else None
- name = add_prefix_to_keys(props.get('names').get(
- _COUNTRY_INFO.key_prefix()), _COUNTRY_INFO.key_prefix())
- params.append((ccode, name, lang, partition))
+
+ params.append((ccode, props['names'], lang, partition))
with connect(dsn) as conn:
with conn.cursor() as cur:
psycopg2.extras.register_hstore(cur)
languages = languages.split(',')
def _include_key(key):
- return key == _COUNTRY_INFO.key_prefix() or \
- (key.startswith(_COUNTRY_INFO.key_prefix()+':') and
- (not languages or key[len(_COUNTRY_INFO.key_prefix())+1:] in languages))
+ return ':' not in key or not languages or \
+ key[key.index(':') + 1:] in languages
with conn.cursor() as cur:
psycopg2.extras.register_hstore(cur)
with tokenizer.name_analyzer() as analyzer:
for code, name in cur:
names = {'countrycode': code}
- if code == 'gb':
- names['short_name'] = 'UK'
- if code == 'us':
- names['short_name'] = 'United States'
# country names (only in languages as provided)
if name:
- names.update(((k, v)
- for k, v in name.items() if _include_key(k)))
+ names.update({k : v for k, v in name.items() if _include_key(k)})
analyzer.add_country_names(code, names)
"""
Tests for function that handle country properties.
"""
-
+from textwrap import dedent
import pytest
from nominatim.tools import country_info
-
-def read_config(def_config):
+@pytest.fixture
+def loaded_country(def_config):
country_info.setup_country_config(def_config)
+@pytest.fixture
+def env_with_country_config(project_env):
+
+ def _mk_config(cfg):
+ (project_env.project_dir / 'country_settings.yaml').write_text(dedent(cfg))
+
+ return project_env
+
+ return _mk_config
+
+
@pytest.mark.parametrize("no_partitions", (True, False))
def test_setup_country_tables(src_dir, temp_db_with_extensions, dsn, temp_db_cursor,
- def_config, no_partitions):
- read_config(def_config)
+ loaded_country, no_partitions):
country_info.setup_country_tables(dsn, src_dir / 'data', no_partitions)
assert temp_db_cursor.table_exists('country_name')
@pytest.mark.parametrize("languages", (None, ' fr,en'))
def test_create_country_names(temp_db_with_extensions, temp_db_conn, temp_db_cursor,
- table_factory, tokenizer_mock, languages, def_config):
- read_config(def_config)
+ table_factory, tokenizer_mock, languages, loaded_country):
table_factory('country_name', 'country_code varchar(2), name hstore',
content=(('us', '"name"=>"us1","name:af"=>"us2"'),
for k, v in tokenizer.analyser_cache['countries']}
if languages:
- assert result_set == {'us': set(('us', 'us1', 'United States')),
+ assert result_set == {'us': set(('us', 'us1')),
'fr': set(('fr', 'Fra', 'Fren'))}
else:
- assert result_set == {'us': set(('us', 'us1', 'us2', 'United States')),
+ assert result_set == {'us': set(('us', 'us1', 'us2')),
'fr': set(('fr', 'Fra', 'Fren'))}
-def test_setup_country_config_languages_not_loaded(project_env):
- (project_env.project_dir / 'country_settings.yaml').write_text("""
-de:
- partition: 3
- names:
- name:
- default: Deutschland
-""")
- country_info._COUNTRY_INFO._info = None
- country_info.setup_country_config(project_env)
- assert country_info._COUNTRY_INFO._info == {'de': {'partition': 3,
- 'languages': [], 'names': {'name': {'default': 'Deutschland'}}}}
-
-
-def test_setup_country_config_name_not_loaded(project_env):
- (project_env.project_dir / 'country_settings.yaml').write_text("""
-de:
- partition: 3
- languages: de
- names:
-""")
- country_info._COUNTRY_INFO._info = None
- country_info.setup_country_config(project_env)
- assert country_info._COUNTRY_INFO._info == {'de': {'partition': 3,
- 'languages': ['de'], 'names': {'name': {}}}}
-
-
-def test_setup_country_config_names_not_loaded(project_env):
- (project_env.project_dir / 'country_settings.yaml').write_text("""
-de:
- partition: 3
- languages: de
-""")
- country_info._COUNTRY_INFO._info = None
- country_info.setup_country_config(project_env)
- assert country_info._COUNTRY_INFO._info == {'de': {'partition': 3,
- 'languages': ['de'], 'names': {'name': {}}}}
-
-
-def test_setup_country_config_special_character(project_env):
- (project_env.project_dir / 'country_settings.yaml').write_text("""
-bq:
- partition: 250
- languages: nl
- names:
- name:
- default: "\\N"
-""")
- country_info._COUNTRY_INFO._info = None
- country_info.setup_country_config(project_env)
- assert country_info._COUNTRY_INFO._info == {'bq': {'partition': 250,
- 'languages': ['nl'], 'names': {'name': {'default': '\x85'}}}}
+def test_setup_country_names_prefixes(env_with_country_config):
+ config = env_with_country_config("""\
+ es:
+ names:
+ name:
+ en: Spain
+ de: Spanien
+ default: Espagñe
+ us:
+ names:
+ short_name:
+ default: USA
+ name:
+ default: United States
+ en: United States
+ """)
+ info = country_info._CountryInfo()
+ info.load(config)
+
+ assert info.get('es')['names'] == {"name": "Espagñe",
+ "name:en": "Spain",
+ "name:de": "Spanien"}
+ assert info.get('us')['names'] == {"name": "United States",
+ "name:en": "United States",
+ "short_name": "USA"}
+ assert 'names' not in info.get('xx')
+
+
+def test_setup_country_config_languages_not_loaded(env_with_country_config):
+ config = env_with_country_config("""\
+ de:
+ partition: 3
+ names:
+ name:
+ default: Deutschland
+ """)
+ info = country_info._CountryInfo()
+ info.load(config)
+ assert dict(info.items()) == {'de': {'partition': 3,
+ 'languages': [],
+ 'names': {'name': 'Deutschland'}}}
+
+
+def test_setup_country_config_name_not_loaded(env_with_country_config):
+ config = env_with_country_config("""\
+ de:
+ partition: 3
+ languages: de
+ names:
+ """)
+
+ info = country_info._CountryInfo()
+ info.load(config)
+
+ assert dict(info.items()) == {'de': {'partition': 3,
+ 'languages': ['de'],
+ 'names': {}
+ }}
+
+
+def test_setup_country_config_names_not_loaded(env_with_country_config):
+ config = env_with_country_config("""
+ de:
+ partition: 3
+ languages: de
+ """)
+
+ info = country_info._CountryInfo()
+ info.load(config)
+
+ assert dict(info.items()) == {'de': {'partition': 3,
+ 'languages': ['de'],
+ 'names': {}
+ }}
+
+
+def test_setup_country_config_special_character(env_with_country_config):
+ config = env_with_country_config("""
+ bq:
+ partition: 250
+ languages: nl
+ names:
+ name:
+ default: "\\N"
+ """)
+
+ info = country_info._CountryInfo()
+ info.load(config)
+
+ assert dict(info.items()) == {'bq': {'partition': 250,
+ 'languages': ['nl'],
+ 'names': {'name': '\x85'}
+ }}
projects / nominatim.git / commitdiff