]> git.openstreetmap.org Git - nominatim.git/blob - nominatim/tools/check_database.py
Merge remote-tracking branch 'upstream/master'
[nominatim.git] / nominatim / tools / check_database.py
1 """
2 Collection of functions that check if the database is complete and functional.
3 """
4 from enum import Enum
5 from textwrap import dedent
6
7 import psycopg2
8
9 from ..db.connection import connect
10 from ..errors import UsageError
11
12 CHECKLIST = []
13
14 class CheckState(Enum):
15     """ Possible states of a check. FATAL stops check execution entirely.
16     """
17     OK = 0
18     FAIL = 1
19     FATAL = 2
20     NOT_APPLICABLE = 3
21
22 def _check(hint=None):
23     """ Decorator for checks. It adds the function to the list of
24         checks to execute and adds the code for printing progress messages.
25     """
26     def decorator(func):
27         title = func.__doc__.split('\n', 1)[0].strip()
28         def run_check(conn, config):
29             print(title, end=' ... ')
30             ret = func(conn, config)
31             if isinstance(ret, tuple):
32                 ret, params = ret
33             else:
34                 params = {}
35             if ret == CheckState.OK:
36                 print('\033[92mOK\033[0m')
37             elif ret == CheckState.NOT_APPLICABLE:
38                 print('not applicable')
39             else:
40                 print('\x1B[31mFailed\033[0m')
41                 if hint:
42                     print(dedent(hint.format(**params)))
43             return ret
44
45         CHECKLIST.append(run_check)
46         return run_check
47
48     return decorator
49
50 class _BadConnection: # pylint: disable=R0903
51
52     def __init__(self, msg):
53         self.msg = msg
54
55     def close(self):
56         """ Dummy function to provide the implementation.
57         """
58
59 def check_database(config):
60     """ Run a number of checks on the database and return the status.
61     """
62     try:
63         conn = connect(config.get_libpq_dsn()).connection
64     except UsageError as err:
65         conn = _BadConnection(str(err))
66
67     overall_result = 0
68     for check in CHECKLIST:
69         ret = check(conn, config)
70         if ret == CheckState.FATAL:
71             conn.close()
72             return 1
73         if ret in (CheckState.FATAL, CheckState.FAIL):
74             overall_result = 1
75
76     conn.close()
77     return overall_result
78
79
80 def _get_indexes(conn):
81     indexes = ['idx_word_word_id',
82                'idx_place_addressline_address_place_id',
83                'idx_placex_rank_search',
84                'idx_placex_rank_address',
85                'idx_placex_parent_place_id',
86                'idx_placex_geometry_reverse_lookuppolygon',
87                'idx_placex_geometry_reverse_placenode',
88                'idx_osmline_parent_place_id',
89                'idx_osmline_parent_osm_id',
90                'idx_postcode_id',
91                'idx_postcode_postcode'
92               ]
93     if conn.table_exists('search_name'):
94         indexes.extend(('idx_search_name_nameaddress_vector',
95                         'idx_search_name_name_vector',
96                         'idx_search_name_centroid'))
97     if conn.table_exists('place'):
98         indexes.extend(('idx_placex_pendingsector',
99                         'idx_location_area_country_place_id',
100                         'idx_place_osm_unique'
101                        ))
102
103     return indexes
104
105
106 ### CHECK FUNCTIONS
107 #
108 # Functions are exectured in the order they appear here.
109
110 @_check(hint="""\
111              {error}
112
113              Hints:
114              * Is the database server started?
115              * Check the NOMINATIM_DATABASE_DSN variable in your local .env
116              * Try connecting to the database with the same settings
117
118              Project directory: {config.project_dir}
119              Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}
120              """)
121 def check_connection(conn, config):
122     """ Checking database connection
123     """
124     if isinstance(conn, _BadConnection):
125         return CheckState.FATAL, dict(error=conn.msg, config=config)
126
127     return CheckState.OK
128
129 @_check(hint="""\
130              placex table not found
131
132              Hints:
133              * Are you connecting to the right database?
134              * Did the import process finish without errors?
135
136              Project directory: {config.project_dir}
137              Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}
138              """)
139 def check_placex_table(conn, config):
140     """ Checking for placex table
141     """
142     if conn.table_exists('placex'):
143         return CheckState.OK
144
145     return CheckState.FATAL, dict(config=config)
146
147
148 @_check(hint="""placex table has no data. Did the import finish sucessfully?""")
149 def check_placex_size(conn, config): # pylint: disable=W0613
150     """ Checking for placex content
151     """
152     with conn.cursor() as cur:
153         cnt = cur.scalar('SELECT count(*) FROM (SELECT * FROM placex LIMIT 100) x')
154
155     return CheckState.OK if cnt > 0 else CheckState.FATAL
156
157
158 @_check(hint="""\
159              The Postgresql extension nominatim.so was not correctly loaded.
160
161              Error: {error}
162
163              Hints:
164              * Check the output of the CMmake/make installation step
165              * Does nominatim.so exist?
166              * Does nominatim.so exist on the database server?
167              * Can nominatim.so be accessed by the database user?
168              """)
169 def check_module(conn, config): # pylint: disable=W0613
170     """ Checking that nominatim.so module is installed
171     """
172     with conn.cursor() as cur:
173         try:
174             out = cur.scalar("SELECT make_standard_name('a')")
175         except psycopg2.ProgrammingError as err:
176             return CheckState.FAIL, dict(error=str(err))
177
178         if out != 'a':
179             return CheckState.FAIL, dict(error='Unexpected result for make_standard_name()')
180
181         return CheckState.OK
182
183
184 @_check(hint="""\
185              The indexing didn't finish. {count} entries are not yet indexed.
186
187              To index the remaining entries, run:   {index_cmd}
188              """)
189 def check_indexing(conn, config): # pylint: disable=W0613
190     """ Checking indexing status
191     """
192     with conn.cursor() as cur:
193         cnt = cur.scalar('SELECT count(*) FROM placex WHERE indexed_status > 0')
194
195     if cnt == 0:
196         return CheckState.OK
197
198     if conn.index_exists('idx_word_word_id'):
199         # Likely just an interrupted update.
200         index_cmd = 'nominatim index'
201     else:
202         # Looks like the import process got interrupted.
203         index_cmd = 'nominatim import --continue indexing'
204
205     return CheckState.FAIL, dict(count=cnt, index_cmd=index_cmd)
206
207
208 @_check(hint="""\
209              The following indexes are missing:
210                {indexes}
211
212              Rerun the index creation with:   nominatim import --continue db-postprocess
213              """)
214 def check_database_indexes(conn, config): # pylint: disable=W0613
215     """ Checking that database indexes are complete
216     """
217     missing = []
218     for index in _get_indexes(conn):
219         if not conn.index_exists(index):
220             missing.append(index)
221
222     if missing:
223         return CheckState.FAIL, dict(indexes='\n  '.join(missing))
224
225     return CheckState.OK
226
227
228 @_check(hint="""\
229              At least one index is invalid. That can happen, e.g. when index creation was
230              disrupted and later restarted. You should delete the affected indices
231              and recreate them.
232
233              Invalid indexes:
234                {indexes}
235              """)
236 def check_database_index_valid(conn, config): # pylint: disable=W0613
237     """ Checking that all database indexes are valid
238     """
239     with conn.cursor() as cur:
240         cur.execute(""" SELECT relname FROM pg_class, pg_index
241                         WHERE pg_index.indisvalid = false
242                         AND pg_index.indexrelid = pg_class.oid""")
243
244         broken = list(cur)
245
246     if broken:
247         return CheckState.FAIL, dict(indexes='\n  '.join(broken))
248
249     return CheckState.OK
250
251
252 @_check(hint="""\
253              {error}
254              Run TIGER import again:   nominatim add-data --tiger-data <DIR>
255              """)
256 def check_tiger_table(conn, config):
257     """ Checking TIGER external data table.
258     """
259     if not config.get_bool('USE_US_TIGER_DATA'):
260         return CheckState.NOT_APPLICABLE
261
262     if not conn.table_exists('location_property_tiger'):
263         return CheckState.FAIL, dict(error='TIGER data table not found.')
264
265     with conn.cursor() as cur:
266         if cur.scalar('SELECT count(*) FROM location_property_tiger') == 0:
267             return CheckState.FAIL, dict(error='TIGER data table is empty.')
268
269     return CheckState.OK