]> git.openstreetmap.org Git - nominatim.git/blob - nominatim/tools/check_database.py
Refactoring loading of external special phrases and importation process by introducin...
[nominatim.git] / nominatim / tools / check_database.py
1 """
2 Collection of functions that check if the database is complete and functional.
3 """
4 from enum import Enum
5 from textwrap import dedent
6
7 from nominatim.db.connection import connect
8 from nominatim.errors import UsageError
9 from nominatim.tokenizer import factory as tokenizer_factory
10
11 CHECKLIST = []
12
13 class CheckState(Enum):
14     """ Possible states of a check. FATAL stops check execution entirely.
15     """
16     OK = 0
17     FAIL = 1
18     FATAL = 2
19     NOT_APPLICABLE = 3
20
21 def _check(hint=None):
22     """ Decorator for checks. It adds the function to the list of
23         checks to execute and adds the code for printing progress messages.
24     """
25     def decorator(func):
26         title = func.__doc__.split('\n', 1)[0].strip()
27         def run_check(conn, config):
28             print(title, end=' ... ')
29             ret = func(conn, config)
30             if isinstance(ret, tuple):
31                 ret, params = ret
32             else:
33                 params = {}
34             if ret == CheckState.OK:
35                 print('\033[92mOK\033[0m')
36             elif ret == CheckState.NOT_APPLICABLE:
37                 print('not applicable')
38             else:
39                 print('\x1B[31mFailed\033[0m')
40                 if hint:
41                     print(dedent(hint.format(**params)))
42             return ret
43
44         CHECKLIST.append(run_check)
45         return run_check
46
47     return decorator
48
49 class _BadConnection:
50
51     def __init__(self, msg):
52         self.msg = msg
53
54     def close(self):
55         """ Dummy function to provide the implementation.
56         """
57
58 def check_database(config):
59     """ Run a number of checks on the database and return the status.
60     """
61     try:
62         conn = connect(config.get_libpq_dsn()).connection
63     except UsageError as err:
64         conn = _BadConnection(str(err))
65
66     overall_result = 0
67     for check in CHECKLIST:
68         ret = check(conn, config)
69         if ret == CheckState.FATAL:
70             conn.close()
71             return 1
72         if ret in (CheckState.FATAL, CheckState.FAIL):
73             overall_result = 1
74
75     conn.close()
76     return overall_result
77
78
79 def _get_indexes(conn):
80     indexes = ['idx_place_addressline_address_place_id',
81                'idx_placex_rank_search',
82                'idx_placex_rank_address',
83                'idx_placex_parent_place_id',
84                'idx_placex_geometry_reverse_lookuppolygon',
85                'idx_placex_geometry_placenode',
86                'idx_osmline_parent_place_id',
87                'idx_osmline_parent_osm_id',
88                'idx_postcode_id',
89                'idx_postcode_postcode'
90               ]
91     if conn.table_exists('search_name'):
92         indexes.extend(('idx_search_name_nameaddress_vector',
93                         'idx_search_name_name_vector',
94                         'idx_search_name_centroid'))
95         if conn.server_version_tuple() >= (11, 0, 0):
96             indexes.extend(('idx_placex_housenumber',
97                             'idx_osmline_parent_osm_id_with_hnr'))
98     if conn.table_exists('place'):
99         indexes.extend(('idx_placex_pendingsector',
100                         'idx_location_area_country_place_id',
101                         'idx_place_osm_unique'
102                        ))
103
104     return indexes
105
106
107 ### CHECK FUNCTIONS
108 #
109 # Functions are exectured in the order they appear here.
110
111 @_check(hint="""\
112              {error}
113
114              Hints:
115              * Is the database server started?
116              * Check the NOMINATIM_DATABASE_DSN variable in your local .env
117              * Try connecting to the database with the same settings
118
119              Project directory: {config.project_dir}
120              Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}
121              """)
122 def check_connection(conn, config):
123     """ Checking database connection
124     """
125     if isinstance(conn, _BadConnection):
126         return CheckState.FATAL, dict(error=conn.msg, config=config)
127
128     return CheckState.OK
129
130 @_check(hint="""\
131              placex table not found
132
133              Hints:
134              * Are you connecting to the right database?
135              * Did the import process finish without errors?
136
137              Project directory: {config.project_dir}
138              Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}
139              """)
140 def check_placex_table(conn, config):
141     """ Checking for placex table
142     """
143     if conn.table_exists('placex'):
144         return CheckState.OK
145
146     return CheckState.FATAL, dict(config=config)
147
148
149 @_check(hint="""placex table has no data. Did the import finish sucessfully?""")
150 def check_placex_size(conn, _):
151     """ Checking for placex content
152     """
153     with conn.cursor() as cur:
154         cnt = cur.scalar('SELECT count(*) FROM (SELECT * FROM placex LIMIT 100) x')
155
156     return CheckState.OK if cnt > 0 else CheckState.FATAL
157
158
159 @_check(hint="""{msg}""")
160 def check_tokenizer(_, config):
161     """ Checking that tokenizer works
162     """
163     try:
164         tokenizer = tokenizer_factory.get_tokenizer_for_db(config)
165     except UsageError:
166         return CheckState.FAIL, dict(msg="""\
167             Cannot load tokenizer. Did the import finish sucessfully?""")
168
169     result = tokenizer.check_database()
170
171     if result is None:
172         return CheckState.OK
173
174     return CheckState.FAIL, dict(msg=result)
175
176
177 @_check(hint="""\
178              The indexing didn't finish. {count} entries are not yet indexed.
179
180              To index the remaining entries, run:   {index_cmd}
181              """)
182 def check_indexing(conn, _):
183     """ Checking indexing status
184     """
185     with conn.cursor() as cur:
186         cnt = cur.scalar('SELECT count(*) FROM placex WHERE indexed_status > 0')
187
188     if cnt == 0:
189         return CheckState.OK
190
191     if conn.index_exists('idx_placex_rank_search'):
192         # Likely just an interrupted update.
193         index_cmd = 'nominatim index'
194     else:
195         # Looks like the import process got interrupted.
196         index_cmd = 'nominatim import --continue indexing'
197
198     return CheckState.FAIL, dict(count=cnt, index_cmd=index_cmd)
199
200
201 @_check(hint="""\
202              The following indexes are missing:
203                {indexes}
204
205              Rerun the index creation with:   nominatim import --continue db-postprocess
206              """)
207 def check_database_indexes(conn, _):
208     """ Checking that database indexes are complete
209     """
210     missing = []
211     for index in _get_indexes(conn):
212         if not conn.index_exists(index):
213             missing.append(index)
214
215     if missing:
216         return CheckState.FAIL, dict(indexes='\n  '.join(missing))
217
218     return CheckState.OK
219
220
221 @_check(hint="""\
222              At least one index is invalid. That can happen, e.g. when index creation was
223              disrupted and later restarted. You should delete the affected indices
224              and recreate them.
225
226              Invalid indexes:
227                {indexes}
228              """)
229 def check_database_index_valid(conn, _):
230     """ Checking that all database indexes are valid
231     """
232     with conn.cursor() as cur:
233         cur.execute(""" SELECT relname FROM pg_class, pg_index
234                         WHERE pg_index.indisvalid = false
235                         AND pg_index.indexrelid = pg_class.oid""")
236
237         broken = list(cur)
238
239     if broken:
240         return CheckState.FAIL, dict(indexes='\n  '.join(broken))
241
242     return CheckState.OK
243
244
245 @_check(hint="""\
246              {error}
247              Run TIGER import again:   nominatim add-data --tiger-data <DIR>
248              """)
249 def check_tiger_table(conn, config):
250     """ Checking TIGER external data table.
251     """
252     if not config.get_bool('USE_US_TIGER_DATA'):
253         return CheckState.NOT_APPLICABLE
254
255     if not conn.table_exists('location_property_tiger'):
256         return CheckState.FAIL, dict(error='TIGER data table not found.')
257
258     with conn.cursor() as cur:
259         if cur.scalar('SELECT count(*) FROM location_property_tiger') == 0:
260             return CheckState.FAIL, dict(error='TIGER data table is empty.')
261
262     return CheckState.OK