From 8f0885f6cb24f545a2f5021d53d8aec64a72bf9b Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Thu, 28 Jan 2021 14:34:17 +0100 Subject: [PATCH] port check-for-update function to python --- lib/admin/update.php | 13 +--------- nominatim/cli.py | 16 ++++++------ nominatim/db/status.py | 13 ++++++++++ nominatim/tools/replication.py | 25 +++++++++++++++++++ test/python/test_cli.py | 5 ++-- test/python/test_db_status.py | 12 +++++++++ test/python/test_tools_replication.py | 36 +++++++++++++++++++++++++++ utils/check_server_for_updates.py | 24 ------------------ 8 files changed, 98 insertions(+), 46 deletions(-) delete mode 100755 utils/check_server_for_updates.py diff --git a/lib/admin/update.php b/lib/admin/update.php index 04eb7019..a2ff6158 100644 --- a/lib/admin/update.php +++ b/lib/admin/update.php @@ -126,18 +126,7 @@ if ($aResult['init-updates']) { } if ($aResult['check-for-updates']) { - $aLastState = $oDB->getRow('SELECT sequence_id FROM import_status'); - - if (!$aLastState['sequence_id']) { - fail('Updates not set up. Please run ./utils/update.php --init-updates.'); - } - - $oCmd = (new \Nominatim\Shell(CONST_BinDir.'/check_server_for_updates.py')) - ->addParams($sBaseURL) - ->addParams($aLastState['sequence_id']); - $iRet = $oCmd->run(); - - exit($iRet); + exit((clone($oNominatimCmd))->addParams('replication', '--check-for-updates')->run()); } if (isset($aResult['import-diff']) || isset($aResult['import-file'])) { diff --git a/nominatim/cli.py b/nominatim/cli.py index a9ffef33..edfddcbc 100644 --- a/nominatim/cli.py +++ b/nominatim/cli.py @@ -257,14 +257,16 @@ class UpdateReplication: return 0 if args.check_for_updates: - params.append('--check-for-updates') + ret = replication.check_for_updates(conn, args.config.REPLICATION_URL) + conn.close() + return ret + + if args.once: + params.append('--import-osmosis') else: - if args.once: - params.append('--import-osmosis') - else: - params.append('--import-osmosis-all') - if not args.do_index: - params.append('--no-index') + params.append('--import-osmosis-all') + if not args.do_index: + params.append('--no-index') return run_legacy_script(*params, nominatim_env=args) diff --git a/nominatim/db/status.py b/nominatim/db/status.py index af4b85c3..a0454771 100644 --- a/nominatim/db/status.py +++ b/nominatim/db/status.py @@ -48,3 +48,16 @@ def set_status(conn, date, seq=None, indexed=True): VALUES (%s, %s, %s)""", (date, seq, indexed)) conn.commit() + + +def get_status(conn): + """ Return the current status as a triple of (date, sequence, indexed). + If status has not been set up yet, a triple of None is returned. + """ + with conn.cursor() as cur: + cur.execute("SELECT * FROM import_status LIMIT 1") + if cur.rowcount < 1: + return None, None, None + + row = cur.fetchone() + return row['lastimportdate'], row['sequence_id'], row['indexed'] diff --git a/nominatim/tools/replication.py b/nominatim/tools/replication.py index 86405168..f278556a 100644 --- a/nominatim/tools/replication.py +++ b/nominatim/tools/replication.py @@ -32,3 +32,28 @@ def init_replication(conn, base_url): status.set_status(conn, date=date, seq=seq) LOG.warning("Updates intialised at sequence %s (%s)", seq, date) + + +def check_for_updates(conn, base_url): + """ Check if new data is available from the replication service at the + given base URL. + """ + _, seq, _ = status.get_status(conn) + + if seq is None: + LOG.error("Replication not set up. " + "Please run 'nominatim replication --init' first.") + return 254 + + state = ReplicationServer(base_url).get_state_info() + + if state is None: + LOG.error("Cannot get state for URL %s.", base_url) + return 253 + + if state.sequence <= seq: + LOG.warning("Database is up to date.") + return 1 + + LOG.warning("New data available (%i => %i).", seq, state.sequence) + return 0 diff --git a/test/python/test_cli.py b/test/python/test_cli.py index ed46eba3..71e3ff65 100644 --- a/test/python/test_cli.py +++ b/test/python/test_cli.py @@ -120,7 +120,6 @@ def test_refresh_command(monkeypatch, temp_db, command, func): monkeypatch.setattr(nominatim.tools.refresh, func, func_mock) assert 0 == call_nominatim('refresh', '--' + command) - assert func_mock.called == 1 @@ -132,14 +131,14 @@ def test_refresh_importance_computed_after_wiki_import(mock_run_legacy, temp_db) @pytest.mark.parametrize("params,func", [ - (('--init', '--no-update-functions'), 'init_replication') + (('--init', '--no-update-functions'), 'init_replication'), + (('--check-for-updates',), 'check_for_updates') ]) def test_replication_command(monkeypatch, temp_db, params, func): func_mock = MockParamCapture() monkeypatch.setattr(nominatim.tools.replication, func, func_mock) assert 0 == call_nominatim('replication', *params) - assert func_mock.called == 1 diff --git a/test/python/test_db_status.py b/test/python/test_db_status.py index d73b099e..9631170a 100644 --- a/test/python/test_db_status.py +++ b/test/python/test_db_status.py @@ -72,3 +72,15 @@ def test_set_status_filled_table(status_table, temp_db_conn, temp_db_cursor): assert temp_db_cursor.rowcount == 1 assert temp_db_cursor.fetchone() == [date, 456, False] + + +def test_get_status_empty_table(status_table, temp_db_conn): + assert nominatim.db.status.get_status(temp_db_conn) == (None, None, None) + + +def test_get_status_success(status_table, temp_db_conn): + date = dt.datetime.fromordinal(1000000).replace(tzinfo=dt.timezone.utc) + nominatim.db.status.set_status(temp_db_conn, date=date, seq=667, indexed=False) + + assert nominatim.db.status.get_status(temp_db_conn) == \ + (date, 667, False) diff --git a/test/python/test_tools_replication.py b/test/python/test_tools_replication.py index f8a2e92b..e06eda59 100644 --- a/test/python/test_tools_replication.py +++ b/test/python/test_tools_replication.py @@ -4,8 +4,10 @@ Tests for replication functionality. import datetime as dt import pytest +from osmium.replication.server import OsmosisState import nominatim.tools.replication +import nominatim.db.status as status OSM_NODE_DATA = """\ @@ -39,3 +41,37 @@ def test_init_replication_success(monkeypatch, status_table, place_row, temp_db_ expected_date = dt.datetime.fromisoformat('2006-01-27T19:09:10').replace(tzinfo=dt.timezone.utc) assert temp_db_cursor.rowcount == 1 assert temp_db_cursor.fetchone() == [expected_date, 234, True] + + +def test_check_for_updates_empty_status_table(status_table, temp_db_conn): + assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == 254 + + +def test_check_for_updates_seq_not_set(status_table, temp_db_conn): + status.set_status(temp_db_conn, dt.datetime.now().replace(tzinfo=dt.timezone.utc)) + + assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == 254 + + +def test_check_for_updates_no_state(monkeypatch, status_table, temp_db_conn): + status.set_status(temp_db_conn, + dt.datetime.now().replace(tzinfo=dt.timezone.utc), + seq=345) + + monkeypatch.setattr(nominatim.tools.replication.ReplicationServer, + "get_state_info", lambda self: None) + + assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == 253 + + +@pytest.mark.parametrize("server_sequence,result", [(344, 1), (345, 1), (346, 0)]) +def test_check_for_updates_no_new_data(monkeypatch, status_table, temp_db_conn, + server_sequence, result): + date = dt.datetime.now().replace(tzinfo=dt.timezone.utc) + status.set_status(temp_db_conn, date, seq=345) + + monkeypatch.setattr(nominatim.tools.replication.ReplicationServer, + "get_state_info", + lambda self: OsmosisState(server_sequence, date)) + + assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == result diff --git a/utils/check_server_for_updates.py b/utils/check_server_for_updates.py deleted file mode 100755 index bcc9d0ba..00000000 --- a/utils/check_server_for_updates.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python3 - -import sys -from osmium.replication import server - -if __name__ == '__main__': - if len(sys.argv) != 3: - print("Usage: python check_server_for_updates.py ") - sys.exit(254) - - seqid = int(sys.argv[2]) - - state = server.ReplicationServer(sys.argv[1]).get_state_info() - - if state is None: - print("ERROR: Cannot get state from URL %s." % (sys.argv[1], )) - sys.exit(253) - - if state.sequence <= seqid: - print("Database up to date.") - sys.exit(1) - - print("New data available (%i => %i)." % (seqid, state.sequence)) - sys.exit(0) -- 2.45.1