indexer: get rid of special handling of few places

[nominatim.git] / nominatim / nominatim.py

diff --git a/nominatim/nominatim.py b/nominatim/nominatim.py
index 0db0777dc17d20b87413a2bf55d08432dc55de24..f87203af36dfee91a818e4b8723d41436c11ccb1 100755 (executable)
--- a/nominatim/nominatim.py
+++ b/nominatim/nominatim.py
@@ -124,6 +124,15 @@ class DBConnection(object):
         self.wait()
 
         self.cursor = self.conn.cursor()
+        # Disable JIT and parallel workers as they are known to cause problems.
+        # Update pg_settings instead of using SET because it does not yield
+        # errors on older versions of Postgres where the settings are not
+        # implemented.
+        self.perform(
+            """ UPDATE pg_settings SET setting = -1 WHERE name = 'jit_above_cost';
+                UPDATE pg_settings SET setting = 0 
+                   WHERE name = 'max_parallel_workers_per_gather';""")
+        self.wait()
 
     def wait(self):
         """ Block until any pending operation is done.
@@ -225,31 +234,22 @@ class Indexer(object):
         done_tuples = 0
         rank_start_time = datetime.now()
 
-        sector_sql = obj.sql_sector_places()
-        index_sql = obj.sql_index_place()
         min_grouped_tuples = total_tuples - len(self.threads) * 1000
 
         next_info = 100 if log.isEnabledFor(logging.INFO) else total_tuples + 1
 
+        pcur = self.conn.cursor()
+
         for r in cur:
             sector = r[0]
-
-            # Should we do the remaining ones together?
-            do_all = done_tuples > min_grouped_tuples
-
-            pcur = self.conn.cursor(name='places')
-
-            if do_all:
-                pcur.execute(obj.sql_nosector_places())
-            else:
-                pcur.execute(sector_sql, (sector, ))
+            pcur.execute(obj.sql_sector_places(), (sector, ))
 
             for place in pcur:
                 place_id = place[0]
                 log.debug("Processing place {}".format(place_id))
                 thread = next(next_thread)
 
-                thread.perform(index_sql, (place_id,))
+                thread.perform(obj.sql_index_place(), (place_id,))
                 done_tuples += 1
 
                 if done_tuples >= next_info:
@@ -262,11 +262,7 @@ class Indexer(object):
                                    (total_tuples - done_tuples)/tuples_per_sec))
                     next_info += int(tuples_per_sec)
 
-            pcur.close()
-
-            if do_all:
-                break
-
+        pcur.close()
         cur.close()
 
         for t in self.threads:
@@ -297,7 +293,7 @@ class Indexer(object):
             if command_stat > 100000:
                 for t in self.threads:
                     while not t.is_done():
-                        wait_select(t.conn)
+                        t.wait()
                     t.connect()
                 command_stat = 0
                 ready = self.threads