]> git.openstreetmap.org Git - nominatim.git/blob - test/python/test_tools_replication.py
Merge pull request #2475 from lonvia/catchup-mode
[nominatim.git] / test / python / test_tools_replication.py
1 """
2 Tests for replication functionality.
3 """
4 import datetime as dt
5 import time
6
7 import pytest
8 from osmium.replication.server import OsmosisState
9
10 import nominatim.tools.replication
11 import nominatim.db.status as status
12 from nominatim.errors import UsageError
13
14 OSM_NODE_DATA = """\
15 <osm version="0.6" generator="OpenStreetMap server" copyright="OpenStreetMap and contributors" attribution="http://www.openstreetmap.org/copyright" license="http://opendatacommons.org/licenses/odbl/1-0/">
16 <node id="100" visible="true" version="1" changeset="2047" timestamp="2006-01-27T22:09:10Z" user="Foo" uid="111" lat="48.7586670" lon="8.1343060">
17 </node>
18 </osm>
19 """
20
21 @pytest.fixture(autouse=True)
22 def setup_status_table(status_table):
23     pass
24
25 ### init replication
26
27 def test_init_replication_bad_base_url(monkeypatch, place_row, temp_db_conn):
28     place_row(osm_type='N', osm_id=100)
29
30     monkeypatch.setattr(nominatim.db.status, "get_url", lambda u: OSM_NODE_DATA)
31
32     with pytest.raises(UsageError, match="Failed to reach replication service"):
33         nominatim.tools.replication.init_replication(temp_db_conn, 'https://test.io')
34
35
36 def test_init_replication_success(monkeypatch, place_row, temp_db_conn, temp_db_cursor):
37     place_row(osm_type='N', osm_id=100)
38
39     monkeypatch.setattr(nominatim.db.status, "get_url", lambda u: OSM_NODE_DATA)
40     monkeypatch.setattr(nominatim.tools.replication.ReplicationServer,
41                         "timestamp_to_sequence",
42                         lambda self, date: 234)
43
44     nominatim.tools.replication.init_replication(temp_db_conn, 'https://test.io')
45
46     expected_date = dt.datetime.strptime('2006-01-27T19:09:10', status.ISODATE_FORMAT)\
47                         .replace(tzinfo=dt.timezone.utc)
48
49     assert temp_db_cursor.row_set("SELECT * FROM import_status") \
50              == {(expected_date, 234, True)}
51
52
53 ### checking for updates
54
55 def test_check_for_updates_empty_status_table(temp_db_conn):
56     assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == 254
57
58
59 def test_check_for_updates_seq_not_set(temp_db_conn):
60     status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc))
61
62     assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == 254
63
64
65 def test_check_for_updates_no_state(monkeypatch, temp_db_conn):
66     status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc), seq=345)
67
68     monkeypatch.setattr(nominatim.tools.replication.ReplicationServer,
69                         "get_state_info", lambda self: None)
70
71     assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == 253
72
73
74 @pytest.mark.parametrize("server_sequence,result", [(344, 2), (345, 2), (346, 0)])
75 def test_check_for_updates_no_new_data(monkeypatch, temp_db_conn,
76                                        server_sequence, result):
77     date = dt.datetime.now(dt.timezone.utc)
78     status.set_status(temp_db_conn, date, seq=345)
79
80     monkeypatch.setattr(nominatim.tools.replication.ReplicationServer,
81                         "get_state_info",
82                         lambda self: OsmosisState(server_sequence, date))
83
84     assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == result
85
86
87 ### updating
88
89 @pytest.fixture
90 def update_options(tmpdir):
91     return dict(base_url='https://test.io',
92                 indexed_only=False,
93                 update_interval=3600,
94                 import_file=tmpdir / 'foo.osm',
95                 max_diff_size=1)
96
97 def test_update_empty_status_table(temp_db_conn):
98     with pytest.raises(UsageError):
99         nominatim.tools.replication.update(temp_db_conn, {})
100
101
102 def test_update_already_indexed(temp_db_conn):
103     status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc), seq=34, indexed=False)
104
105     assert nominatim.tools.replication.update(temp_db_conn, dict(indexed_only=True)) \
106              == nominatim.tools.replication.UpdateState.MORE_PENDING
107
108
109 def test_update_no_data_no_sleep(monkeypatch, temp_db_conn, update_options):
110     date = dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=1)
111     status.set_status(temp_db_conn, date, seq=34)
112
113     monkeypatch.setattr(nominatim.tools.replication.ReplicationServer,
114                         "apply_diffs",
115                         lambda *args, **kwargs: None)
116
117     sleeptime = []
118     monkeypatch.setattr(time, 'sleep', sleeptime.append)
119
120     assert nominatim.tools.replication.update(temp_db_conn, update_options) \
121              == nominatim.tools.replication.UpdateState.NO_CHANGES
122
123     assert not sleeptime
124
125
126 def test_update_no_data_sleep(monkeypatch, temp_db_conn, update_options):
127     date = dt.datetime.now(dt.timezone.utc) - dt.timedelta(minutes=30)
128     status.set_status(temp_db_conn, date, seq=34)
129
130     monkeypatch.setattr(nominatim.tools.replication.ReplicationServer,
131                         "apply_diffs",
132                         lambda *args, **kwargs: None)
133
134     sleeptime = []
135     monkeypatch.setattr(time, 'sleep', sleeptime.append)
136
137     assert nominatim.tools.replication.update(temp_db_conn, update_options) \
138              == nominatim.tools.replication.UpdateState.NO_CHANGES
139
140     assert len(sleeptime) == 1
141     assert sleeptime[0] < 3600
142     assert sleeptime[0] > 0