]> git.openstreetmap.org Git - nominatim.git/commitdiff
add function for parallel execution of SQL scripts
authorSarah Hoffmann <lonvia@denofr.de>
Mon, 19 Sep 2022 09:52:17 +0000 (11:52 +0200)
committerSarah Hoffmann <lonvia@denofr.de>
Mon, 19 Sep 2022 09:52:17 +0000 (11:52 +0200)
nominatim/db/sql_preprocessor.py

index b450422da09396cb687cfe50750490311f82a31d..31b4a8c0f9a0042877cb652f6578fe2e896a899a 100644 (file)
@@ -11,6 +11,7 @@ from typing import Set, Dict, Any
 import jinja2
 
 from nominatim.db.connection import Connection
 import jinja2
 
 from nominatim.db.connection import Connection
+from nominatim.db.async_connection import WorkerPool
 from nominatim.config import Configuration
 
 def _get_partitions(conn: Connection) -> Set[int]:
 from nominatim.config import Configuration
 
 def _get_partitions(conn: Connection) -> Set[int]:
@@ -96,3 +97,21 @@ class SQLPreprocessor:
         with conn.cursor() as cur:
             cur.execute(sql)
         conn.commit()
         with conn.cursor() as cur:
             cur.execute(sql)
         conn.commit()
+
+
+    def run_parallel_sql_file(self, dsn: str, name: str, num_threads: int = 1,
+                              **kwargs: Any) -> None:
+        """ Execure the given SQL files using parallel asynchronous connections.
+            The keyword arguments may supply additional parameters for
+            preprocessing.
+
+            After preprocessing the SQL code is cut at lines containing only
+            '---'. Each chunk is sent to one of the `num_threads` workers.
+        """
+        sql = self.env.get_template(name).render(**kwargs)
+
+        parts = sql.split('\n---\n')
+
+        with WorkerPool(dsn, num_threads) as pool:
+            for part in parts:
+                pool.next_free_worker().perform(part)