From 8f63f9516b276af8391d71e285db44a7b7214803 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 19 Apr 2021 09:23:37 +0200 Subject: [PATCH] simplify interface for adding tiger data Also simplifies tests using existing fixtures. --- nominatim/cli.py | 7 ++--- nominatim/tools/tiger_data.py | 10 +++---- test/python/conftest.py | 7 ++++- test/python/test_tools_tiger_data.py | 41 ++++++++++++---------------- 4 files changed, 31 insertions(+), 34 deletions(-) diff --git a/nominatim/cli.py b/nominatim/cli.py index 47b25c17..55f51aac 100644 --- a/nominatim/cli.py +++ b/nominatim/cli.py @@ -148,11 +148,8 @@ class UpdateAddData: @staticmethod def run(args): if args.tiger_data: - return tiger_data.add_tiger_data(args.config.get_libpq_dsn(), - args.tiger_data, - args.threads or 1, - args.config, - args.sqllib_dir) + return tiger_data.add_tiger_data(args.tiger_data, + args.config, args.threads or 1) params = ['update.php'] if args.file: diff --git a/nominatim/tools/tiger_data.py b/nominatim/tools/tiger_data.py index c1de3615..90789e79 100644 --- a/nominatim/tools/tiger_data.py +++ b/nominatim/tools/tiger_data.py @@ -76,17 +76,17 @@ def handle_unregister_connection_pool(sel, place_threads): conn.close() place_threads -= 1 -def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir): - """ Import tiger data from directory or tar file +def add_tiger_data(data_dir, config, threads): + """ Import tiger data from directory or tar file `data dir`. """ - + dsn = config.get_libpq_dsn() sql_files, tar = handle_tarfile_or_directory(data_dir) if not sql_files: return with connect(dsn) as conn: - sql = SQLPreprocessor(conn, config, sqllib_dir) + sql = SQLPreprocessor(conn, config, config.lib_dir.sql) sql.run_sql_file(conn, 'tiger_import_start.sql') # Reading sql_files and then for each file line handling @@ -116,5 +116,5 @@ def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir): print('\n') LOG.warning("Creating indexes on Tiger data") with connect(dsn) as conn: - sql = SQLPreprocessor(conn, config, sqllib_dir) + sql = SQLPreprocessor(conn, config, config.lib_dir.sql) sql.run_sql_file(conn, 'tiger_import_finish.sql') diff --git a/test/python/conftest.py b/test/python/conftest.py index 0d1cd2f3..2a2d2129 100644 --- a/test/python/conftest.py +++ b/test/python/conftest.py @@ -126,7 +126,12 @@ def table_factory(temp_db_cursor): @pytest.fixture def def_config(): - return Configuration(None, SRC_DIR.resolve() / 'settings') + cfg = Configuration(None, SRC_DIR.resolve() / 'settings') + cfg.set_libdirs(module='.', osm2pgsql='.', + php=SRC_DIR / 'lib-php', + sql=SRC_DIR / 'lib-sql', + data=SRC_DIR / 'data') + return cfg @pytest.fixture def src_dir(): diff --git a/test/python/test_tools_tiger_data.py b/test/python/test_tools_tiger_data.py index 1366fe3e..f97a5713 100644 --- a/test/python/test_tools_tiger_data.py +++ b/test/python/test_tools_tiger_data.py @@ -10,35 +10,31 @@ from nominatim.tools import tiger_data, database_import @pytest.mark.parametrize("threads", (1, 5)) -def test_add_tiger_data(dsn, src_dir, def_config, tmp_path, sql_preprocessor, - temp_db_cursor, threads, temp_db): - temp_db_cursor.execute('CREATE EXTENSION hstore') - temp_db_cursor.execute('CREATE EXTENSION postgis') +def test_add_tiger_data(def_config, tmp_path, sql_preprocessor, + temp_db_cursor, threads, temp_db_with_extensions): temp_db_cursor.execute('CREATE TABLE place (id INT)') sqlfile = tmp_path / '1010.sql' sqlfile.write_text("""INSERT INTO place values (1); INSERT INTO non_existant_table values (1);""") - tiger_data.add_tiger_data(dsn, str(tmp_path), threads, def_config, src_dir / 'lib-sql') + tiger_data.add_tiger_data(str(tmp_path), def_config, threads) assert temp_db_cursor.table_rows('place') == 1 + @pytest.mark.parametrize("threads", (1, 5)) -def test_add_tiger_data_bad_file(dsn, src_dir, def_config, tmp_path, sql_preprocessor, - temp_db_cursor, threads, temp_db): - temp_db_cursor.execute('CREATE EXTENSION hstore') - temp_db_cursor.execute('CREATE EXTENSION postgis') +def test_add_tiger_data_bad_file(def_config, tmp_path, sql_preprocessor, + temp_db_cursor, threads, temp_db_with_extensions): temp_db_cursor.execute('CREATE TABLE place (id INT)') sqlfile = tmp_path / '1010.txt' sqlfile.write_text("""Random text""") - tiger_data.add_tiger_data(dsn, str(tmp_path), threads, def_config, src_dir / 'lib-sql') + tiger_data.add_tiger_data(str(tmp_path), def_config, threads) assert temp_db_cursor.table_rows('place') == 0 + @pytest.mark.parametrize("threads", (1, 5)) -def test_add_tiger_data_tarfile(dsn, src_dir, def_config, tmp_path, - temp_db_cursor, threads, temp_db, sql_preprocessor): - temp_db_cursor.execute('CREATE EXTENSION hstore') - temp_db_cursor.execute('CREATE EXTENSION postgis') +def test_add_tiger_data_tarfile(def_config, tmp_path, temp_db_cursor, + threads, temp_db_with_extensions, sql_preprocessor): temp_db_cursor.execute('CREATE TABLE place (id INT)') sqlfile = tmp_path / '1010.sql' sqlfile.write_text("""INSERT INTO place values (1); @@ -46,21 +42,20 @@ def test_add_tiger_data_tarfile(dsn, src_dir, def_config, tmp_path, tar = tarfile.open("sample.tar.gz", "w:gz") tar.add(sqlfile) tar.close() - tiger_data.add_tiger_data(dsn, str(src_dir / 'sample.tar.gz'), threads, def_config, src_dir / 'lib-sql') - + tiger_data.add_tiger_data(str(tmp_path), def_config, threads) + assert temp_db_cursor.table_rows('place') == 1 + @pytest.mark.parametrize("threads", (1, 5)) -def test_add_tiger_data_bad_tarfile(dsn, src_dir, def_config, tmp_path, - temp_db_cursor, threads, temp_db, sql_preprocessor): - temp_db_cursor.execute('CREATE EXTENSION hstore') - temp_db_cursor.execute('CREATE EXTENSION postgis') +def test_add_tiger_data_bad_tarfile(def_config, tmp_path, temp_db_cursor, threads, + temp_db_with_extensions, sql_preprocessor): temp_db_cursor.execute('CREATE TABLE place (id INT)') sqlfile = tmp_path / '1010.txt' sqlfile.write_text("""Random text""") tar = tarfile.open("sample.tar.gz", "w:gz") tar.add(sqlfile) tar.close() - tiger_data.add_tiger_data(dsn, str(src_dir / 'sample.tar.gz'), threads, def_config, src_dir / 'lib-sql') - - assert temp_db_cursor.table_rows('place') == 0 \ No newline at end of file + tiger_data.add_tiger_data(str(tmp_path), def_config, threads) + + assert temp_db_cursor.table_rows('place') == 0 -- 2.45.1