]> git.openstreetmap.org Git - nominatim.git/blob - test/python/tools/test_migration.py
Merge pull request #3262 from lonvia/fix-category-search
[nominatim.git] / test / python / tools / test_migration.py
1 # SPDX-License-Identifier: GPL-2.0-only
2 #
3 # This file is part of Nominatim. (https://nominatim.org)
4 #
5 # Copyright (C) 2022 by the Nominatim developer community.
6 # For a full list of authors see the git log.
7 """
8 Tests for migration functions
9 """
10 import pytest
11 import psycopg2.extras
12
13 from nominatim.tools import migration
14 from nominatim.errors import UsageError
15 import nominatim.version
16
17 from mock_legacy_word_table import MockLegacyWordTable
18
19 class DummyTokenizer:
20
21     def update_sql_functions(self, config):
22         pass
23
24
25 @pytest.fixture
26 def postprocess_mock(monkeypatch):
27     monkeypatch.setattr(migration.refresh, 'create_functions', lambda *args: args)
28     monkeypatch.setattr(migration.tokenizer_factory, 'get_tokenizer_for_db',
29                         lambda *args: DummyTokenizer())
30
31 @pytest.fixture
32 def legacy_word_table(temp_db_conn):
33     return MockLegacyWordTable(temp_db_conn)
34
35
36 def test_no_migration_old_versions(temp_db_with_extensions, table_factory, def_config):
37     table_factory('country_name', 'name HSTORE, country_code TEXT')
38
39     with pytest.raises(UsageError, match='Migration not possible'):
40         migration.migrate(def_config, {})
41
42
43 def test_set_up_migration_for_36(temp_db_with_extensions, temp_db_cursor,
44                                  table_factory, def_config, monkeypatch,
45                                  postprocess_mock):
46     psycopg2.extras.register_hstore(temp_db_cursor)
47     # don't actually run any migration, except the property table creation
48     monkeypatch.setattr(migration, '_MIGRATION_FUNCTIONS',
49                         [((3, 5, 0, 99), migration.add_nominatim_property_table)])
50     # Use a r/o user name that always exists
51     monkeypatch.setenv('NOMINATIM_DATABASE_WEBUSER', 'postgres')
52
53     table_factory('country_name', 'name HSTORE, country_code TEXT',
54                   (({str(x): 'a' for x in range(200)}, 'gb'),))
55
56     assert not temp_db_cursor.table_exists('nominatim_properties')
57
58     assert migration.migrate(def_config, {}) == 0
59
60     assert temp_db_cursor.table_exists('nominatim_properties')
61
62     assert 1 == temp_db_cursor.scalar(""" SELECT count(*) FROM nominatim_properties
63                                           WHERE property = 'database_version'""")
64
65
66 def test_already_at_version(def_config, property_table):
67
68     property_table.set('database_version',
69                        '{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(nominatim.version.NOMINATIM_VERSION))
70
71     assert migration.migrate(def_config, {}) == 0
72
73
74 def test_run_single_migration(def_config, temp_db_cursor, property_table,
75                               monkeypatch, postprocess_mock):
76     oldversion = [x for x in nominatim.version.NOMINATIM_VERSION]
77     oldversion[0] -= 1
78     property_table.set('database_version',
79                        '{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(oldversion))
80
81     done = {'old': False, 'new': False}
82     def _migration(**_):
83         """ Dummy migration"""
84         done['new'] = True
85
86     def _old_migration(**_):
87         """ Dummy migration"""
88         done['old'] = True
89
90     oldversion[0] = 0
91     monkeypatch.setattr(migration, '_MIGRATION_FUNCTIONS',
92                         [(tuple(oldversion), _old_migration),
93                          (nominatim.version.NOMINATIM_VERSION, _migration)])
94
95     assert migration.migrate(def_config, {}) == 0
96
97     assert done['new']
98     assert not done['old']
99     assert property_table.get('database_version') == \
100            '{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(nominatim.version.NOMINATIM_VERSION)
101
102
103 ###### Tests for specific migrations
104 #
105 # Each migration should come with two tests:
106 #  1. Test that migration from old to new state works as expected.
107 #  2. Test that the migration can be rerun on the new state without side effects.
108
109
110 @pytest.mark.parametrize('in_attr', ('', 'with time zone'))
111 def test_import_status_timestamp_change(temp_db_conn, temp_db_cursor,
112                                         table_factory, in_attr):
113     table_factory('import_status',
114                   f"""lastimportdate timestamp {in_attr},
115                      sequence_id integer,
116                      indexed boolean""")
117
118     migration.import_status_timestamp_change(temp_db_conn)
119     temp_db_conn.commit()
120
121     assert temp_db_cursor.scalar("""SELECT data_type FROM information_schema.columns
122                                     WHERE table_name = 'import_status'
123                                       and column_name = 'lastimportdate'""")\
124             == 'timestamp with time zone'
125
126
127 def test_add_nominatim_property_table(temp_db_conn, temp_db_cursor,
128                                       def_config, monkeypatch):
129     # Use a r/o user name that always exists
130     monkeypatch.setenv('NOMINATIM_DATABASE_WEBUSER', 'postgres')
131
132     assert not temp_db_cursor.table_exists('nominatim_properties')
133
134     migration.add_nominatim_property_table(temp_db_conn, def_config)
135     temp_db_conn.commit()
136
137     assert temp_db_cursor.table_exists('nominatim_properties')
138
139
140 def test_add_nominatim_property_table_repeat(temp_db_conn, temp_db_cursor,
141                                              def_config, property_table):
142     assert temp_db_cursor.table_exists('nominatim_properties')
143
144     migration.add_nominatim_property_table(temp_db_conn, def_config)
145     temp_db_conn.commit()
146
147     assert temp_db_cursor.table_exists('nominatim_properties')
148
149
150 def test_change_housenumber_transliteration(temp_db_conn, temp_db_cursor,
151                                             legacy_word_table, placex_table):
152     placex_table.add(housenumber='3A')
153
154     temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION make_standard_name(name TEXT)
155                               RETURNS TEXT AS $$ SELECT lower(name) $$ LANGUAGE SQL """)
156     temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION getorcreate_housenumber_id(lookup_word TEXT)
157                               RETURNS INTEGER AS $$ SELECT 4325 $$ LANGUAGE SQL """)
158
159     migration.change_housenumber_transliteration(temp_db_conn)
160     temp_db_conn.commit()
161
162     assert temp_db_cursor.scalar('SELECT housenumber from placex') == '3a'
163
164     migration.change_housenumber_transliteration(temp_db_conn)
165     temp_db_conn.commit()
166
167     assert temp_db_cursor.scalar('SELECT housenumber from placex') == '3a'
168
169
170 def test_switch_placenode_geometry_index(temp_db_conn, temp_db_cursor, placex_table):
171     temp_db_cursor.execute("""CREATE INDEX idx_placex_adminname
172                               ON placex (place_id)""")
173
174     migration.switch_placenode_geometry_index(temp_db_conn)
175     temp_db_conn.commit()
176
177     assert temp_db_cursor.index_exists('placex', 'idx_placex_geometry_placenode')
178     assert not temp_db_cursor.index_exists('placex', 'idx_placex_adminname')
179
180
181 def test_switch_placenode_geometry_index_repeat(temp_db_conn, temp_db_cursor, placex_table):
182     temp_db_cursor.execute("""CREATE INDEX idx_placex_geometry_placenode
183                               ON placex (place_id)""")
184
185     migration.switch_placenode_geometry_index(temp_db_conn)
186     temp_db_conn.commit()
187
188     assert temp_db_cursor.index_exists('placex', 'idx_placex_geometry_placenode')
189     assert not temp_db_cursor.index_exists('placex', 'idx_placex_adminname')
190     assert temp_db_cursor.scalar("""SELECT indexdef from pg_indexes
191                                     WHERE tablename = 'placex'
192                                       and indexname = 'idx_placex_geometry_placenode'
193                                  """).endswith('(place_id)')
194
195
196 def test_install_legacy_tokenizer(temp_db_conn, temp_db_cursor, project_env,
197                                   property_table, table_factory, monkeypatch,
198                                   tmp_path):
199     table_factory('placex', 'place_id BIGINT')
200     table_factory('location_property_osmline', 'place_id BIGINT')
201
202     # Setting up the tokenizer is problematic
203     class MiniTokenizer:
204         def migrate_database(self, config):
205             pass
206
207     monkeypatch.setattr(migration.tokenizer_factory, 'create_tokenizer',
208                         lambda cfg, **kwargs: MiniTokenizer())
209
210     migration.install_legacy_tokenizer(temp_db_conn, project_env)
211     temp_db_conn.commit()
212
213
214
215 def test_install_legacy_tokenizer_repeat(temp_db_conn, temp_db_cursor,
216                                          def_config, property_table):
217
218     property_table.set('tokenizer', 'dummy')
219     migration.install_legacy_tokenizer(temp_db_conn, def_config)
220     temp_db_conn.commit()
221
222
223 def test_create_tiger_housenumber_index(temp_db_conn, temp_db_cursor, table_factory):
224     table_factory('location_property_tiger',
225                   'parent_place_id BIGINT, startnumber INT, endnumber INT')
226
227     migration.create_tiger_housenumber_index(temp_db_conn)
228     temp_db_conn.commit()
229
230     if temp_db_conn.server_version_tuple() >= (11, 0, 0):
231         assert temp_db_cursor.index_exists('location_property_tiger',
232                                            'idx_location_property_tiger_housenumber_migrated')
233
234     migration.create_tiger_housenumber_index(temp_db_conn)
235     temp_db_conn.commit()