nominatim/tools/check_database.py

   1 """
   2 Collection of functions that check if the database is complete and functional.
   3 """
   4 from enum import Enum
   5 from textwrap import dedent
   6
   7 import psycopg2
   8
   9 from nominatim.db.connection import connect
  10 from nominatim.errors import UsageError
  11
  12 CHECKLIST = []
  13
  14 class CheckState(Enum):
  15     """ Possible states of a check. FATAL stops check execution entirely.
  16     """
  17     OK = 0
  18     FAIL = 1
  19     FATAL = 2
  20     NOT_APPLICABLE = 3
  21
  22 def _check(hint=None):
  23     """ Decorator for checks. It adds the function to the list of
  24         checks to execute and adds the code for printing progress messages.
  25     """
  26     def decorator(func):
  27         title = func.__doc__.split('\n', 1)[0].strip()
  28         def run_check(conn, config):
  29             print(title, end=' ... ')
  30             ret = func(conn, config)
  31             if isinstance(ret, tuple):
  32                 ret, params = ret
  33             else:
  34                 params = {}
  35             if ret == CheckState.OK:
  36                 print('\033[92mOK\033[0m')
  37             elif ret == CheckState.NOT_APPLICABLE:
  38                 print('not applicable')
  39             else:
  40                 print('\x1B[31mFailed\033[0m')
  41                 if hint:
  42                     print(dedent(hint.format(**params)))
  43             return ret
  44
  45         CHECKLIST.append(run_check)
  46         return run_check
  47
  48     return decorator
  49
  50 class _BadConnection: # pylint: disable=R0903
  51
  52     def __init__(self, msg):
  53         self.msg = msg
  54
  55     def close(self):
  56         """ Dummy function to provide the implementation.
  57         """
  58
  59 def check_database(config):
  60     """ Run a number of checks on the database and return the status.
  61     """
  62     try:
  63         conn = connect(config.get_libpq_dsn()).connection
  64     except UsageError as err:
  65         conn = _BadConnection(str(err))
  66
  67     overall_result = 0
  68     for check in CHECKLIST:
  69         ret = check(conn, config)
  70         if ret == CheckState.FATAL:
  71             conn.close()
  72             return 1
  73         if ret in (CheckState.FATAL, CheckState.FAIL):
  74             overall_result = 1
  75
  76     conn.close()
  77     return overall_result
  78
  79
  80 def _get_indexes(conn):
  81     indexes = ['idx_word_word_id',
  82                'idx_place_addressline_address_place_id',
  83                'idx_placex_rank_search',
  84                'idx_placex_rank_address',
  85                'idx_placex_parent_place_id',
  86                'idx_placex_geometry_reverse_lookuppolygon',
  87                'idx_placex_geometry_placenode',
  88                'idx_osmline_parent_place_id',
  89                'idx_osmline_parent_osm_id',
  90                'idx_postcode_id',
  91                'idx_postcode_postcode'
  92               ]
  93     if conn.table_exists('search_name'):
  94         indexes.extend(('idx_search_name_nameaddress_vector',
  95                         'idx_search_name_name_vector',
  96                         'idx_search_name_centroid'))
  97     if conn.table_exists('place'):
  98         indexes.extend(('idx_placex_pendingsector',
  99                         'idx_location_area_country_place_id',
 100                         'idx_place_osm_unique'
 101                        ))
 102     if conn.server_version_tuple() >= (11, 0, 0):
 103         indexes.extend(('idx_placex_housenumber',
 104                         'idx_osmline_parent_osm_id_with_hnr'))
 105
 106     return indexes
 107
 108
 109 ### CHECK FUNCTIONS
 110 #
 111 # Functions are exectured in the order they appear here.
 112
 113 @_check(hint="""\
 114              {error}
 115
 116              Hints:
 117              * Is the database server started?
 118              * Check the NOMINATIM_DATABASE_DSN variable in your local .env
 119              * Try connecting to the database with the same settings
 120
 121              Project directory: {config.project_dir}
 122              Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}
 123              """)
 124 def check_connection(conn, config):
 125     """ Checking database connection
 126     """
 127     if isinstance(conn, _BadConnection):
 128         return CheckState.FATAL, dict(error=conn.msg, config=config)
 129
 130     return CheckState.OK
 131
 132 @_check(hint="""\
 133              placex table not found
 134
 135              Hints:
 136              * Are you connecting to the right database?
 137              * Did the import process finish without errors?
 138
 139              Project directory: {config.project_dir}
 140              Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}
 141              """)
 142 def check_placex_table(conn, config):
 143     """ Checking for placex table
 144     """
 145     if conn.table_exists('placex'):
 146         return CheckState.OK
 147
 148     return CheckState.FATAL, dict(config=config)
 149
 150
 151 @_check(hint="""placex table has no data. Did the import finish sucessfully?""")
 152 def check_placex_size(conn, config): # pylint: disable=W0613
 153     """ Checking for placex content
 154     """
 155     with conn.cursor() as cur:
 156         cnt = cur.scalar('SELECT count(*) FROM (SELECT * FROM placex LIMIT 100) x')
 157
 158     return CheckState.OK if cnt > 0 else CheckState.FATAL
 159
 160
 161 @_check(hint="""\
 162              The Postgresql extension nominatim.so was not correctly loaded.
 163
 164              Error: {error}
 165
 166              Hints:
 167              * Check the output of the CMmake/make installation step
 168              * Does nominatim.so exist?
 169              * Does nominatim.so exist on the database server?
 170              * Can nominatim.so be accessed by the database user?
 171              """)
 172 def check_module(conn, config): # pylint: disable=W0613
 173     """ Checking that nominatim.so module is installed
 174     """
 175     with conn.cursor() as cur:
 176         try:
 177             out = cur.scalar("SELECT make_standard_name('a')")
 178         except psycopg2.ProgrammingError as err:
 179             return CheckState.FAIL, dict(error=str(err))
 180
 181         if out != 'a':
 182             return CheckState.FAIL, dict(error='Unexpected result for make_standard_name()')
 183
 184         return CheckState.OK
 185
 186
 187 @_check(hint="""\
 188              The indexing didn't finish. {count} entries are not yet indexed.
 189
 190              To index the remaining entries, run:   {index_cmd}
 191              """)
 192 def check_indexing(conn, config): # pylint: disable=W0613
 193     """ Checking indexing status
 194     """
 195     with conn.cursor() as cur:
 196         cnt = cur.scalar('SELECT count(*) FROM placex WHERE indexed_status > 0')
 197
 198     if cnt == 0:
 199         return CheckState.OK
 200
 201     if conn.index_exists('idx_word_word_id'):
 202         # Likely just an interrupted update.
 203         index_cmd = 'nominatim index'
 204     else:
 205         # Looks like the import process got interrupted.
 206         index_cmd = 'nominatim import --continue indexing'
 207
 208     return CheckState.FAIL, dict(count=cnt, index_cmd=index_cmd)
 209
 210
 211 @_check(hint="""\
 212              The following indexes are missing:
 213                {indexes}
 214
 215              Rerun the index creation with:   nominatim import --continue db-postprocess
 216              """)
 217 def check_database_indexes(conn, config): # pylint: disable=W0613
 218     """ Checking that database indexes are complete
 219     """
 220     missing = []
 221     for index in _get_indexes(conn):
 222         if not conn.index_exists(index):
 223             missing.append(index)
 224
 225     if missing:
 226         return CheckState.FAIL, dict(indexes='\n  '.join(missing))
 227
 228     return CheckState.OK
 229
 230
 231 @_check(hint="""\
 232              At least one index is invalid. That can happen, e.g. when index creation was
 233              disrupted and later restarted. You should delete the affected indices
 234              and recreate them.
 235
 236              Invalid indexes:
 237                {indexes}
 238              """)
 239 def check_database_index_valid(conn, config): # pylint: disable=W0613
 240     """ Checking that all database indexes are valid
 241     """
 242     with conn.cursor() as cur:
 243         cur.execute(""" SELECT relname FROM pg_class, pg_index
 244                         WHERE pg_index.indisvalid = false
 245                         AND pg_index.indexrelid = pg_class.oid""")
 246
 247         broken = list(cur)
 248
 249     if broken:
 250         return CheckState.FAIL, dict(indexes='\n  '.join(broken))
 251
 252     return CheckState.OK
 253
 254
 255 @_check(hint="""\
 256              {error}
 257              Run TIGER import again:   nominatim add-data --tiger-data <DIR>
 258              """)
 259 def check_tiger_table(conn, config):
 260     """ Checking TIGER external data table.
 261     """
 262     if not config.get_bool('USE_US_TIGER_DATA'):
 263         return CheckState.NOT_APPLICABLE
 264
 265     if not conn.table_exists('location_property_tiger'):
 266         return CheckState.FAIL, dict(error='TIGER data table not found.')
 267
 268     with conn.cursor() as cur:
 269         if cur.scalar('SELECT count(*) FROM location_property_tiger') == 0:
 270             return CheckState.FAIL, dict(error='TIGER data table is empty.')
 271
 272     return CheckState.OK