factor out async connection handling into separate class

[nominatim.git] / test / python / test_indexing.py

diff --git a/test/python/test_indexing.py b/test/python/test_indexing.py
index 6b52a65ea6171d318e2d6d6d81a9a1ac51cbe2d2..6692eba61d349b8908415d793078fba17b0db3af 100644 (file)
--- a/test/python/test_indexing.py
+++ b/test/python/test_indexing.py
@@ -5,13 +5,14 @@ import itertools
 import psycopg2
 import pytest
 
-from nominatim.indexer.indexer import Indexer
+from nominatim.indexer import indexer
 
 class IndexerTestDB:
 
     def __init__(self, conn):
         self.placex_id = itertools.count(100000)
         self.osmline_id = itertools.count(500000)
+        self.postcode_id = itertools.count(700000)
 
         self.conn = conn
         self.conn.set_isolation_level(0)
@@ -31,6 +32,12 @@ class IndexerTestDB:
                                indexed_status SMALLINT,
                                indexed_date TIMESTAMP,
                                geometry_sector INTEGER)""")
+            cur.execute("""CREATE TABLE location_postcode (
+                               place_id BIGINT,
+                               indexed_status SMALLINT,
+                               indexed_date TIMESTAMP,
+                               country_code varchar(2),
+                               postcode TEXT)""")
             cur.execute("""CREATE OR REPLACE FUNCTION date_update() RETURNS TRIGGER
                            AS $$
                            BEGIN
@@ -39,10 +46,10 @@ class IndexerTestDB:
                              END IF;
                              RETURN NEW;
                            END; $$ LANGUAGE plpgsql;""")
-            cur.execute("""CREATE TRIGGER placex_update BEFORE UPDATE ON placex
-                           FOR EACH ROW EXECUTE PROCEDURE date_update()""")
-            cur.execute("""CREATE TRIGGER osmline_update BEFORE UPDATE ON location_property_osmline
-                           FOR EACH ROW EXECUTE PROCEDURE date_update()""")
+            for table in ('placex', 'location_property_osmline', 'location_postcode'):
+                cur.execute("""CREATE TRIGGER {0}_update BEFORE UPDATE ON {0}
+                               FOR EACH ROW EXECUTE PROCEDURE date_update()
+                            """.format(table))
 
     def scalar(self, query):
         with self.conn.cursor() as cur:
@@ -74,6 +81,15 @@ class IndexerTestDB:
                         (next_id, sector))
         return next_id
 
+    def add_postcode(self, country, postcode):
+        next_id = next(self.postcode_id)
+        with self.conn.cursor() as cur:
+            cur.execute("""INSERT INTO location_postcode
+                            (place_id, indexed_status, country_code, postcode)
+                            VALUES (%s, 1, %s, %s)""",
+                        (next_id, country, postcode))
+        return next_id
+
     def placex_unindexed(self):
         return self.scalar('SELECT count(*) from placex where indexed_status > 0')
 
@@ -87,7 +103,7 @@ def test_db(temp_db_conn):
 
 
 @pytest.mark.parametrize("threads", [1, 15])
-def test_index_full(test_db, threads):
+def test_index_all_by_rank(test_db, threads):
     for rank in range(31):
         test_db.add_place(rank_address=rank, rank_search=rank)
     test_db.add_osmline()
@@ -95,7 +111,7 @@ def test_index_full(test_db, threads):
     assert 31 == test_db.placex_unindexed()
     assert 1 == test_db.osmline_unindexed()
 
-    idx = Indexer('dbname=test_nominatim_python_unittest', threads)
+    idx = indexer.Indexer('dbname=test_nominatim_python_unittest', threads)
     idx.index_by_rank(0, 30)
 
     assert 0 == test_db.placex_unindexed()
@@ -134,7 +150,7 @@ def test_index_partial_without_30(test_db, threads):
     assert 31 == test_db.placex_unindexed()
     assert 1 == test_db.osmline_unindexed()
 
-    idx = Indexer('dbname=test_nominatim_python_unittest', threads)
+    idx = indexer.Indexer('dbname=test_nominatim_python_unittest', threads)
     idx.index_by_rank(4, 15)
 
     assert 19 == test_db.placex_unindexed()
@@ -154,7 +170,7 @@ def test_index_partial_with_30(test_db, threads):
     assert 31 == test_db.placex_unindexed()
     assert 1 == test_db.osmline_unindexed()
 
-    idx = Indexer('dbname=test_nominatim_python_unittest', threads)
+    idx = indexer.Indexer('dbname=test_nominatim_python_unittest', threads)
     idx.index_by_rank(28, 30)
 
     assert 27 == test_db.placex_unindexed()
@@ -175,7 +191,7 @@ def test_index_boundaries(test_db, threads):
     assert 37 == test_db.placex_unindexed()
     assert 1 == test_db.osmline_unindexed()
 
-    idx = Indexer('dbname=test_nominatim_python_unittest', threads)
+    idx = indexer.Indexer('dbname=test_nominatim_python_unittest', threads)
     idx.index_boundaries(0, 30)
 
     assert 31 == test_db.placex_unindexed()
@@ -184,3 +200,49 @@ def test_index_boundaries(test_db, threads):
     assert 0 == test_db.scalar("""
                     SELECT count(*) FROM placex
                       WHERE indexed_status = 0 AND class != 'boundary'""")
+
+
+@pytest.mark.parametrize("threads", [1, 15])
+def test_index_postcodes(test_db, threads):
+    for postcode in range(1000):
+        test_db.add_postcode('de', postcode)
+    for postcode in range(32000, 33000):
+        test_db.add_postcode('us', postcode)
+
+    idx = indexer.Indexer('dbname=test_nominatim_python_unittest', threads)
+    idx.index_postcodes()
+
+    assert 0 == test_db.scalar("""SELECT count(*) FROM location_postcode
+                                  WHERE indexed_status != 0""")
+
+
+@pytest.mark.parametrize("analyse", [True, False])
+def test_index_full(test_db, analyse):
+    for rank in range(4, 10):
+        test_db.add_admin(rank_address=rank, rank_search=rank)
+    for rank in range(31):
+        test_db.add_place(rank_address=rank, rank_search=rank)
+    test_db.add_osmline()
+    for postcode in range(1000):
+        test_db.add_postcode('de', postcode)
+
+    idx = indexer.Indexer('dbname=test_nominatim_python_unittest', 4)
+    idx.index_full(analyse=analyse)
+
+    assert 0 == test_db.placex_unindexed()
+    assert 0 == test_db.osmline_unindexed()
+    assert 0 == test_db.scalar("""SELECT count(*) FROM location_postcode
+                                  WHERE indexed_status != 0""")
+
+
+@pytest.mark.parametrize("threads", [1, 15])
+def test_index_reopen_connection(test_db, threads, monkeypatch):
+    monkeypatch.setattr(indexer.WorkerPool, "REOPEN_CONNECTIONS_AFTER", 15)
+
+    for _ in range(1000):
+        test_db.add_place(rank_address=30, rank_search=30)
+
+    idx = indexer.Indexer('dbname=test_nominatim_python_unittest', threads)
+    idx.index_by_rank(28, 30)
+
+    assert 0 == test_db.placex_unindexed()