]> git.openstreetmap.org Git - nominatim.git/blob - nominatim/tools/check_database.py
Merge pull request #2270 from lonvia/simplify-place-boundary-merge
[nominatim.git] / nominatim / tools / check_database.py
1 """
2 Collection of functions that check if the database is complete and functional.
3 """
4 from enum import Enum
5 from textwrap import dedent
6
7 import psycopg2
8
9 from ..db.connection import connect
10 from ..errors import UsageError
11
12 CHECKLIST = []
13
14 class CheckState(Enum):
15     """ Possible states of a check. FATAL stops check execution entirely.
16     """
17     OK = 0
18     FAIL = 1
19     FATAL = 2
20     NOT_APPLICABLE = 3
21
22 def _check(hint=None):
23     """ Decorator for checks. It adds the function to the list of
24         checks to execute and adds the code for printing progress messages.
25     """
26     def decorator(func):
27         title = func.__doc__.split('\n', 1)[0].strip()
28         def run_check(conn, config):
29             print(title, end=' ... ')
30             ret = func(conn, config)
31             if isinstance(ret, tuple):
32                 ret, params = ret
33             else:
34                 params = {}
35             if ret == CheckState.OK:
36                 print('\033[92mOK\033[0m')
37             elif ret == CheckState.NOT_APPLICABLE:
38                 print('not applicable')
39             else:
40                 print('\x1B[31mFailed\033[0m')
41                 if hint:
42                     print(dedent(hint.format(**params)))
43             return ret
44
45         CHECKLIST.append(run_check)
46         return run_check
47
48     return decorator
49
50 class _BadConnection: # pylint: disable=R0903
51
52     def __init__(self, msg):
53         self.msg = msg
54
55     def close(self):
56         """ Dummy function to provide the implementation.
57         """
58
59 def check_database(config):
60     """ Run a number of checks on the database and return the status.
61     """
62     try:
63         conn = connect(config.get_libpq_dsn()).connection
64     except UsageError as err:
65         conn = _BadConnection(str(err))
66
67     overall_result = 0
68     for check in CHECKLIST:
69         ret = check(conn, config)
70         if ret == CheckState.FATAL:
71             conn.close()
72             return 1
73         if ret in (CheckState.FATAL, CheckState.FAIL):
74             overall_result = 1
75
76     conn.close()
77     return overall_result
78
79
80 def _get_indexes(conn):
81     indexes = ['idx_word_word_id',
82                'idx_place_addressline_address_place_id',
83                'idx_placex_rank_search',
84                'idx_placex_rank_address',
85                'idx_placex_parent_place_id',
86                'idx_placex_geometry_reverse_lookuppolygon',
87                'idx_placex_geometry_placenode',
88                'idx_placex_housenumber',
89                'idx_osmline_parent_place_id',
90                'idx_osmline_parent_osm_id',
91                'idx_postcode_id',
92                'idx_postcode_postcode'
93               ]
94     if conn.table_exists('search_name'):
95         indexes.extend(('idx_search_name_nameaddress_vector',
96                         'idx_search_name_name_vector',
97                         'idx_search_name_centroid'))
98     if conn.table_exists('place'):
99         indexes.extend(('idx_placex_pendingsector',
100                         'idx_location_area_country_place_id',
101                         'idx_place_osm_unique'
102                        ))
103
104     return indexes
105
106
107 ### CHECK FUNCTIONS
108 #
109 # Functions are exectured in the order they appear here.
110
111 @_check(hint="""\
112              {error}
113
114              Hints:
115              * Is the database server started?
116              * Check the NOMINATIM_DATABASE_DSN variable in your local .env
117              * Try connecting to the database with the same settings
118
119              Project directory: {config.project_dir}
120              Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}
121              """)
122 def check_connection(conn, config):
123     """ Checking database connection
124     """
125     if isinstance(conn, _BadConnection):
126         return CheckState.FATAL, dict(error=conn.msg, config=config)
127
128     return CheckState.OK
129
130 @_check(hint="""\
131              placex table not found
132
133              Hints:
134              * Are you connecting to the right database?
135              * Did the import process finish without errors?
136
137              Project directory: {config.project_dir}
138              Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}
139              """)
140 def check_placex_table(conn, config):
141     """ Checking for placex table
142     """
143     if conn.table_exists('placex'):
144         return CheckState.OK
145
146     return CheckState.FATAL, dict(config=config)
147
148
149 @_check(hint="""placex table has no data. Did the import finish sucessfully?""")
150 def check_placex_size(conn, config): # pylint: disable=W0613
151     """ Checking for placex content
152     """
153     with conn.cursor() as cur:
154         cnt = cur.scalar('SELECT count(*) FROM (SELECT * FROM placex LIMIT 100) x')
155
156     return CheckState.OK if cnt > 0 else CheckState.FATAL
157
158
159 @_check(hint="""\
160              The Postgresql extension nominatim.so was not correctly loaded.
161
162              Error: {error}
163
164              Hints:
165              * Check the output of the CMmake/make installation step
166              * Does nominatim.so exist?
167              * Does nominatim.so exist on the database server?
168              * Can nominatim.so be accessed by the database user?
169              """)
170 def check_module(conn, config): # pylint: disable=W0613
171     """ Checking that nominatim.so module is installed
172     """
173     with conn.cursor() as cur:
174         try:
175             out = cur.scalar("SELECT make_standard_name('a')")
176         except psycopg2.ProgrammingError as err:
177             return CheckState.FAIL, dict(error=str(err))
178
179         if out != 'a':
180             return CheckState.FAIL, dict(error='Unexpected result for make_standard_name()')
181
182         return CheckState.OK
183
184
185 @_check(hint="""\
186              The indexing didn't finish. {count} entries are not yet indexed.
187
188              To index the remaining entries, run:   {index_cmd}
189              """)
190 def check_indexing(conn, config): # pylint: disable=W0613
191     """ Checking indexing status
192     """
193     with conn.cursor() as cur:
194         cnt = cur.scalar('SELECT count(*) FROM placex WHERE indexed_status > 0')
195
196     if cnt == 0:
197         return CheckState.OK
198
199     if conn.index_exists('idx_word_word_id'):
200         # Likely just an interrupted update.
201         index_cmd = 'nominatim index'
202     else:
203         # Looks like the import process got interrupted.
204         index_cmd = 'nominatim import --continue indexing'
205
206     return CheckState.FAIL, dict(count=cnt, index_cmd=index_cmd)
207
208
209 @_check(hint="""\
210              The following indexes are missing:
211                {indexes}
212
213              Rerun the index creation with:   nominatim import --continue db-postprocess
214              """)
215 def check_database_indexes(conn, config): # pylint: disable=W0613
216     """ Checking that database indexes are complete
217     """
218     missing = []
219     for index in _get_indexes(conn):
220         if not conn.index_exists(index):
221             missing.append(index)
222
223     if missing:
224         return CheckState.FAIL, dict(indexes='\n  '.join(missing))
225
226     return CheckState.OK
227
228
229 @_check(hint="""\
230              At least one index is invalid. That can happen, e.g. when index creation was
231              disrupted and later restarted. You should delete the affected indices
232              and recreate them.
233
234              Invalid indexes:
235                {indexes}
236              """)
237 def check_database_index_valid(conn, config): # pylint: disable=W0613
238     """ Checking that all database indexes are valid
239     """
240     with conn.cursor() as cur:
241         cur.execute(""" SELECT relname FROM pg_class, pg_index
242                         WHERE pg_index.indisvalid = false
243                         AND pg_index.indexrelid = pg_class.oid""")
244
245         broken = list(cur)
246
247     if broken:
248         return CheckState.FAIL, dict(indexes='\n  '.join(broken))
249
250     return CheckState.OK
251
252
253 @_check(hint="""\
254              {error}
255              Run TIGER import again:   nominatim add-data --tiger-data <DIR>
256              """)
257 def check_tiger_table(conn, config):
258     """ Checking TIGER external data table.
259     """
260     if not config.get_bool('USE_US_TIGER_DATA'):
261         return CheckState.NOT_APPLICABLE
262
263     if not conn.table_exists('location_property_tiger'):
264         return CheckState.FAIL, dict(error='TIGER data table not found.')
265
266     with conn.cursor() as cur:
267         if cur.scalar('SELECT count(*) FROM location_property_tiger') == 0:
268             return CheckState.FAIL, dict(error='TIGER data table is empty.')
269
270     return CheckState.OK