X-Git-Url: https://git.openstreetmap.org/nominatim.git/blobdiff_plain/2af82975cd968ec09683ae5b16a9aa157a7f2176..122c4618b9f091f52d5064b16780ab581f2ceff5:/nominatim/tools/tiger_data.py diff --git a/nominatim/tools/tiger_data.py b/nominatim/tools/tiger_data.py index 521d11c4..26f6beb2 100644 --- a/nominatim/tools/tiger_data.py +++ b/nominatim/tools/tiger_data.py @@ -3,7 +3,6 @@ Functions for setting up and importing a new Nominatim database. """ import logging import os -import time import tarfile import selectors @@ -11,6 +10,9 @@ from ..db.connection import connect from ..db.async_connection import DBConnection from ..db.sql_preprocessor import SQLPreprocessor +# pylint: disable=R0912 +# pylint: disable=R0914,R0915,W0702 + LOG = logging.getLogger() @@ -19,22 +21,22 @@ def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir): """ # Handling directory or tarball file. is_tarfile = False - if(data_dir.endswith('.tar.gz')): + if data_dir.endswith('.tar.gz'): is_tarfile = True tar = tarfile.open(data_dir) sql_files = [i for i in tar.getmembers() if i.name.endswith('.sql')] - LOG.warning(f'Found {len(sql_files)} SQL files in tarfile with path {data_dir}') - if(not len(sql_files)): - LOG.warning(f'Tiger data import selected but no files found in tarfile with path {data_dir}') + LOG.warning("Found %d SQL files in tarfile with path %s", len(sql_files), data_dir) + if not sql_files: + LOG.warning("Tiger data import selected but no files in tarfile's path %s", data_dir) return else: files = os.listdir(data_dir) sql_files = [i for i in files if i.endswith('.sql')] - LOG.warning(f'Found {len(sql_files)} SQL files in path {data_dir}') - if(not len(sql_files)): - LOG.warning(f'Tiger data import selected but no files found in path {data_dir}') + LOG.warning("Found %d SQL files in path %s", len(sql_files), data_dir) + if not sql_files: + LOG.warning("Tiger data import selected but no files found in path %s", data_dir) return - + with connect(dsn) as conn: sql = SQLPreprocessor(conn, config, sqllib_dir) sql.run_sql_file(conn, 'tiger_import_start.sql') @@ -44,7 +46,7 @@ def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir): sel = selectors.DefaultSelector() place_threads = max(1, threads - 1) for sql_file in sql_files: - if(not is_tarfile): + if not is_tarfile: file_path = os.path.join(data_dir, sql_file) file = open(file_path) else: @@ -52,17 +54,17 @@ def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir): lines = 0 end_of_file = False total_used_threads = place_threads - while(True): - if(end_of_file): + while True : + if end_of_file: break for imod in range(place_threads): conn = DBConnection(dsn) conn.connect() sql_query = file.readline() - lines+=1 + lines += 1 - if(not sql_query): + if not sql_query: end_of_file = True total_used_threads = imod break @@ -70,11 +72,11 @@ def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir): conn.perform(sql_query) sel.register(conn, selectors.EVENT_READ, conn) - if(lines==1000): + if lines == 1000: print('. ', end='', flush=True) - lines=0 + lines = 0 - todo = min(place_threads,total_used_threads) + todo = min(place_threads, total_used_threads) while todo > 0: for key, _ in sel.select(1): try: @@ -84,9 +86,9 @@ def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir): conn.close() todo -= 1 except: - todo -=1 + todo -= 1 - if(is_tarfile): + if is_tarfile: tar.close() print('\n') LOG.warning("Creating indexes on Tiger data")