]> git.openstreetmap.org Git - nominatim.git/blob - nominatim/tools/check_database.py
do not check for extra housenumber index for reverse-only
[nominatim.git] / nominatim / tools / check_database.py
1 """
2 Collection of functions that check if the database is complete and functional.
3 """
4 from enum import Enum
5 from textwrap import dedent
6
7 import psycopg2
8
9 from nominatim.db.connection import connect
10 from nominatim.errors import UsageError
11
12 CHECKLIST = []
13
14 class CheckState(Enum):
15     """ Possible states of a check. FATAL stops check execution entirely.
16     """
17     OK = 0
18     FAIL = 1
19     FATAL = 2
20     NOT_APPLICABLE = 3
21
22 def _check(hint=None):
23     """ Decorator for checks. It adds the function to the list of
24         checks to execute and adds the code for printing progress messages.
25     """
26     def decorator(func):
27         title = func.__doc__.split('\n', 1)[0].strip()
28         def run_check(conn, config):
29             print(title, end=' ... ')
30             ret = func(conn, config)
31             if isinstance(ret, tuple):
32                 ret, params = ret
33             else:
34                 params = {}
35             if ret == CheckState.OK:
36                 print('\033[92mOK\033[0m')
37             elif ret == CheckState.NOT_APPLICABLE:
38                 print('not applicable')
39             else:
40                 print('\x1B[31mFailed\033[0m')
41                 if hint:
42                     print(dedent(hint.format(**params)))
43             return ret
44
45         CHECKLIST.append(run_check)
46         return run_check
47
48     return decorator
49
50 class _BadConnection:
51
52     def __init__(self, msg):
53         self.msg = msg
54
55     def close(self):
56         """ Dummy function to provide the implementation.
57         """
58
59 def check_database(config):
60     """ Run a number of checks on the database and return the status.
61     """
62     try:
63         conn = connect(config.get_libpq_dsn()).connection
64     except UsageError as err:
65         conn = _BadConnection(str(err))
66
67     overall_result = 0
68     for check in CHECKLIST:
69         ret = check(conn, config)
70         if ret == CheckState.FATAL:
71             conn.close()
72             return 1
73         if ret in (CheckState.FATAL, CheckState.FAIL):
74             overall_result = 1
75
76     conn.close()
77     return overall_result
78
79
80 def _get_indexes(conn):
81     indexes = ['idx_word_word_id',
82                'idx_place_addressline_address_place_id',
83                'idx_placex_rank_search',
84                'idx_placex_rank_address',
85                'idx_placex_parent_place_id',
86                'idx_placex_geometry_reverse_lookuppolygon',
87                'idx_placex_geometry_placenode',
88                'idx_osmline_parent_place_id',
89                'idx_osmline_parent_osm_id',
90                'idx_postcode_id',
91                'idx_postcode_postcode'
92               ]
93     if conn.table_exists('search_name'):
94         indexes.extend(('idx_search_name_nameaddress_vector',
95                         'idx_search_name_name_vector',
96                         'idx_search_name_centroid'))
97         if conn.server_version_tuple() >= (11, 0, 0):
98             indexes.extend(('idx_placex_housenumber',
99                             'idx_osmline_parent_osm_id_with_hnr'))
100     if conn.table_exists('place'):
101         indexes.extend(('idx_placex_pendingsector',
102                         'idx_location_area_country_place_id',
103                         'idx_place_osm_unique'
104                        ))
105
106     return indexes
107
108
109 ### CHECK FUNCTIONS
110 #
111 # Functions are exectured in the order they appear here.
112
113 @_check(hint="""\
114              {error}
115
116              Hints:
117              * Is the database server started?
118              * Check the NOMINATIM_DATABASE_DSN variable in your local .env
119              * Try connecting to the database with the same settings
120
121              Project directory: {config.project_dir}
122              Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}
123              """)
124 def check_connection(conn, config):
125     """ Checking database connection
126     """
127     if isinstance(conn, _BadConnection):
128         return CheckState.FATAL, dict(error=conn.msg, config=config)
129
130     return CheckState.OK
131
132 @_check(hint="""\
133              placex table not found
134
135              Hints:
136              * Are you connecting to the right database?
137              * Did the import process finish without errors?
138
139              Project directory: {config.project_dir}
140              Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}
141              """)
142 def check_placex_table(conn, config):
143     """ Checking for placex table
144     """
145     if conn.table_exists('placex'):
146         return CheckState.OK
147
148     return CheckState.FATAL, dict(config=config)
149
150
151 @_check(hint="""placex table has no data. Did the import finish sucessfully?""")
152 def check_placex_size(conn, config): # pylint: disable=W0613
153     """ Checking for placex content
154     """
155     with conn.cursor() as cur:
156         cnt = cur.scalar('SELECT count(*) FROM (SELECT * FROM placex LIMIT 100) x')
157
158     return CheckState.OK if cnt > 0 else CheckState.FATAL
159
160
161 @_check(hint="""\
162              The Postgresql extension nominatim.so was not correctly loaded.
163
164              Error: {error}
165
166              Hints:
167              * Check the output of the CMmake/make installation step
168              * Does nominatim.so exist?
169              * Does nominatim.so exist on the database server?
170              * Can nominatim.so be accessed by the database user?
171              """)
172 def check_module(conn, config): # pylint: disable=W0613
173     """ Checking that nominatim.so module is installed
174     """
175     with conn.cursor() as cur:
176         try:
177             out = cur.scalar("SELECT make_standard_name('a')")
178         except psycopg2.ProgrammingError as err:
179             return CheckState.FAIL, dict(error=str(err))
180
181         if out != 'a':
182             return CheckState.FAIL, dict(error='Unexpected result for make_standard_name()')
183
184         return CheckState.OK
185
186
187 @_check(hint="""\
188              The indexing didn't finish. {count} entries are not yet indexed.
189
190              To index the remaining entries, run:   {index_cmd}
191              """)
192 def check_indexing(conn, config): # pylint: disable=W0613
193     """ Checking indexing status
194     """
195     with conn.cursor() as cur:
196         cnt = cur.scalar('SELECT count(*) FROM placex WHERE indexed_status > 0')
197
198     if cnt == 0:
199         return CheckState.OK
200
201     if conn.index_exists('idx_word_word_id'):
202         # Likely just an interrupted update.
203         index_cmd = 'nominatim index'
204     else:
205         # Looks like the import process got interrupted.
206         index_cmd = 'nominatim import --continue indexing'
207
208     return CheckState.FAIL, dict(count=cnt, index_cmd=index_cmd)
209
210
211 @_check(hint="""\
212              The following indexes are missing:
213                {indexes}
214
215              Rerun the index creation with:   nominatim import --continue db-postprocess
216              """)
217 def check_database_indexes(conn, config): # pylint: disable=W0613
218     """ Checking that database indexes are complete
219     """
220     missing = []
221     for index in _get_indexes(conn):
222         if not conn.index_exists(index):
223             missing.append(index)
224
225     if missing:
226         return CheckState.FAIL, dict(indexes='\n  '.join(missing))
227
228     return CheckState.OK
229
230
231 @_check(hint="""\
232              At least one index is invalid. That can happen, e.g. when index creation was
233              disrupted and later restarted. You should delete the affected indices
234              and recreate them.
235
236              Invalid indexes:
237                {indexes}
238              """)
239 def check_database_index_valid(conn, config): # pylint: disable=W0613
240     """ Checking that all database indexes are valid
241     """
242     with conn.cursor() as cur:
243         cur.execute(""" SELECT relname FROM pg_class, pg_index
244                         WHERE pg_index.indisvalid = false
245                         AND pg_index.indexrelid = pg_class.oid""")
246
247         broken = list(cur)
248
249     if broken:
250         return CheckState.FAIL, dict(indexes='\n  '.join(broken))
251
252     return CheckState.OK
253
254
255 @_check(hint="""\
256              {error}
257              Run TIGER import again:   nominatim add-data --tiger-data <DIR>
258              """)
259 def check_tiger_table(conn, config):
260     """ Checking TIGER external data table.
261     """
262     if not config.get_bool('USE_US_TIGER_DATA'):
263         return CheckState.NOT_APPLICABLE
264
265     if not conn.table_exists('location_property_tiger'):
266         return CheckState.FAIL, dict(error='TIGER data table not found.')
267
268     with conn.cursor() as cur:
269         if cur.scalar('SELECT count(*) FROM location_property_tiger') == 0:
270             return CheckState.FAIL, dict(error='TIGER data table is empty.')
271
272     return CheckState.OK