]> git.openstreetmap.org Git - nominatim.git/blob - nominatim/tools/refresh.py
Merge pull request #2323 from darkshredder/disable-search-reverse-only
[nominatim.git] / nominatim / tools / refresh.py
1 """
2 Functions for bringing auxiliary data in the database up-to-date.
3 """
4 import json
5 import logging
6 from textwrap import dedent
7
8 from psycopg2.extras import execute_values
9
10 from nominatim.db.utils import execute_file
11 from nominatim.db.sql_preprocessor import SQLPreprocessor
12 from nominatim.version import NOMINATIM_VERSION
13
14 LOG = logging.getLogger()
15
16
17 def recompute_word_counts(dsn, sql_dir):
18     """ Compute the frequency of full-word search terms.
19     """
20     execute_file(dsn, sql_dir / 'words_from_search_name.sql')
21
22
23 def _add_address_level_rows_from_entry(rows, entry):
24     """ Converts a single entry from the JSON format for address rank
25         descriptions into a flat format suitable for inserting into a
26         PostgreSQL table and adds these lines to `rows`.
27     """
28     countries = entry.get('countries') or (None, )
29     for key, values in entry['tags'].items():
30         for value, ranks in values.items():
31             if isinstance(ranks, list):
32                 rank_search, rank_address = ranks
33             else:
34                 rank_search = rank_address = ranks
35             if not value:
36                 value = None
37             for country in countries:
38                 rows.append((country, key, value, rank_search, rank_address))
39
40 def load_address_levels(conn, table, levels):
41     """ Replace the `address_levels` table with the contents of `levels'.
42
43         A new table is created any previously existing table is dropped.
44         The table has the following columns:
45             country, class, type, rank_search, rank_address
46     """
47     rows = []
48     for entry in levels:
49         _add_address_level_rows_from_entry(rows, entry)
50
51     with conn.cursor() as cur:
52         cur.execute('DROP TABLE IF EXISTS {}'.format(table))
53
54         cur.execute("""CREATE TABLE {} (country_code varchar(2),
55                                         class TEXT,
56                                         type TEXT,
57                                         rank_search SMALLINT,
58                                         rank_address SMALLINT)""".format(table))
59
60         execute_values(cur, "INSERT INTO {} VALUES %s".format(table), rows)
61
62         cur.execute('CREATE UNIQUE INDEX ON {} (country_code, class, type)'.format(table))
63
64     conn.commit()
65
66 def load_address_levels_from_file(conn, config_file):
67     """ Replace the `address_levels` table with the contents of the config
68         file.
69     """
70     with config_file.open('r') as fdesc:
71         load_address_levels(conn, 'address_levels', json.load(fdesc))
72
73
74 def create_functions(conn, config, enable_diff_updates=True, enable_debug=False):
75     """ (Re)create the PL/pgSQL functions.
76     """
77     sql = SQLPreprocessor(conn, config)
78
79     sql.run_sql_file(conn, 'functions.sql',
80                      disable_diff_updates=not enable_diff_updates,
81                      debug=enable_debug)
82
83
84
85 WEBSITE_SCRIPTS = (
86     'deletable.php',
87     'details.php',
88     'lookup.php',
89     'polygons.php',
90     'reverse.php',
91     'search.php',
92     'status.php'
93 )
94
95 # constants needed by PHP scripts: PHP name, config name, type
96 PHP_CONST_DEFS = (
97     ('Database_DSN', 'DATABASE_DSN', str),
98     ('Default_Language', 'DEFAULT_LANGUAGE', str),
99     ('Log_DB', 'LOG_DB', bool),
100     ('Log_File', 'LOG_FILE', str),
101     ('NoAccessControl', 'CORS_NOACCESSCONTROL', bool),
102     ('Places_Max_ID_count', 'LOOKUP_MAX_COUNT', int),
103     ('PolygonOutput_MaximumTypes', 'POLYGON_OUTPUT_MAX_TYPES', int),
104     ('Search_BatchMode', 'SEARCH_BATCH_MODE', bool),
105     ('Search_NameOnlySearchFrequencyThreshold', 'SEARCH_NAME_ONLY_THRESHOLD', str),
106     ('Use_US_Tiger_Data', 'USE_US_TIGER_DATA', bool),
107     ('MapIcon_URL', 'MAPICON_URL', str),
108 )
109
110
111 def import_wikipedia_articles(dsn, data_path, ignore_errors=False):
112     """ Replaces the wikipedia importance tables with new data.
113         The import is run in a single transaction so that the new data
114         is replace seemlessly.
115
116         Returns 0 if all was well and 1 if the importance file could not
117         be found. Throws an exception if there was an error reading the file.
118     """
119     datafile = data_path / 'wikimedia-importance.sql.gz'
120
121     if not datafile.exists():
122         return 1
123
124     pre_code = """BEGIN;
125                   DROP TABLE IF EXISTS "wikipedia_article";
126                   DROP TABLE IF EXISTS "wikipedia_redirect"
127                """
128     post_code = "COMMIT"
129     execute_file(dsn, datafile, ignore_errors=ignore_errors,
130                  pre_code=pre_code, post_code=post_code)
131
132     return 0
133
134
135 def recompute_importance(conn):
136     """ Recompute wikipedia links and importance for all entries in placex.
137         This is a long-running operations that must not be executed in
138         parallel with updates.
139     """
140     with conn.cursor() as cur:
141         cur.execute('ALTER TABLE placex DISABLE TRIGGER ALL')
142         cur.execute("""
143             UPDATE placex SET (wikipedia, importance) =
144                (SELECT wikipedia, importance
145                 FROM compute_importance(extratags, country_code, osm_type, osm_id))
146             """)
147         cur.execute("""
148             UPDATE placex s SET wikipedia = d.wikipedia, importance = d.importance
149              FROM placex d
150              WHERE s.place_id = d.linked_place_id and d.wikipedia is not null
151                    and (s.wikipedia is null or s.importance < d.importance);
152             """)
153
154         cur.execute('ALTER TABLE placex ENABLE TRIGGER ALL')
155     conn.commit()
156
157
158 def setup_website(basedir, config, conn):
159     """ Create the website script stubs.
160     """
161     if not basedir.exists():
162         LOG.info('Creating website directory.')
163         basedir.mkdir()
164
165     template = dedent("""\
166                       <?php
167
168                       @define('CONST_Debug', $_GET['debug'] ?? false);
169                       @define('CONST_LibDir', '{0}');
170                       @define('CONST_TokenizerDir', '{2}');
171                       @define('CONST_NominatimVersion', '{1[0]}.{1[1]}.{1[2]}-{1[3]}');
172
173                       """.format(config.lib_dir.php, NOMINATIM_VERSION,
174                                  config.project_dir / 'tokenizer'))
175
176     for php_name, conf_name, var_type in PHP_CONST_DEFS:
177         if var_type == bool:
178             varout = 'true' if config.get_bool(conf_name) else 'false'
179         elif var_type == int:
180             varout = getattr(config, conf_name)
181         elif not getattr(config, conf_name):
182             varout = 'false'
183         else:
184             varout = "'{}'".format(getattr(config, conf_name).replace("'", "\\'"))
185
186         template += "@define('CONST_{}', {});\n".format(php_name, varout)
187
188     template += "\nrequire_once('{}/website/{{}}');\n".format(config.lib_dir.php)
189
190     search_name_table_exists = bool(conn and conn.table_exists('search_name'))
191
192     for script in WEBSITE_SCRIPTS:
193         if not search_name_table_exists and script == 'search.php':
194             (basedir / script).write_text(template.format('reverse-only-search.php'), 'utf-8')
195         else:
196             (basedir / script).write_text(template.format(script), 'utf-8')