Merge pull request #3373 from lonvia/restrict-man-made

[nominatim.git] / test / python / cli / test_cli.py

diff --git a/test/python/cli/test_cli.py b/test/python/cli/test_cli.py
index f4878b6186feaab16237a5169a528da074f5fb10..998bccb434c418eeaa076995f851a0cbaeab8ba9 100644 (file)
--- a/test/python/cli/test_cli.py
+++ b/test/python/cli/test_cli.py
@@ -1,257 +1,112 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.

 """
 Tests for command line interface wrapper.
 
 These tests just check that the various command line parameters route to the
 """
 Tests for command line interface wrapper.
 
 These tests just check that the various command line parameters route to the

-correct functionionality. They use a lot of monkeypatching to avoid executing
+correct functionality. They use a lot of monkeypatching to avoid executing

 the actual functions.
 """
 the actual functions.
 """

+import importlib

 import pytest
 
 import pytest
 

-import nominatim.db.properties
-import nominatim.cli
-import nominatim.clicmd.api
-import nominatim.clicmd.refresh
-import nominatim.clicmd.admin
-import nominatim.clicmd.setup

 import nominatim.indexer.indexer
 import nominatim.indexer.indexer

-import nominatim.tools.admin

 import nominatim.tools.add_osm_data
 import nominatim.tools.add_osm_data

-import nominatim.tools.check_database
-import nominatim.tools.database_import
-import nominatim.tools.country_info

 import nominatim.tools.freeze
 import nominatim.tools.freeze

-import nominatim.tools.refresh
-import nominatim.tools.postcodes
-import nominatim.tokenizer.factory

 
 
 
 

-class TestCli:
+def test_cli_help(cli_call, capsys):
+    """ Running nominatim tool without arguments prints help.
+    """
+    assert cli_call() == 1

 
 

-    @pytest.fixture(autouse=True)
-    def setup_cli_call(self, cli_call):
-        self.call_nominatim = cli_call
-
-
-    def test_cli_help(self, capsys):
-        """ Running nominatim tool without arguments prints help.
-        """
-        assert self.call_nominatim() == 1
+    captured = capsys.readouterr()
+    assert captured.out.startswith('usage:')

 
 

-        captured = capsys.readouterr()
-        assert captured.out.startswith('usage:')
+def test_cli_version(cli_call, capsys):
+    """ Running nominatim tool --version prints a version string.
+    """
+    assert cli_call('--version') == 0

 
 

+    captured = capsys.readouterr()
+    assert captured.out.startswith('Nominatim version')

 
 

-    @pytest.mark.parametrize("command,script", [
-                             (('export',), 'export')
-                             ])
-    def test_legacy_commands_simple(self, mock_run_legacy, command, script):
-        assert self.call_nominatim(*command) == 0
+@pytest.mark.parametrize("name,oid", [('file', 'foo.osm'), ('diff', 'foo.osc')])
+def test_cli_add_data_file_command(cli_call, mock_func_factory, name, oid):
+    mock_run_legacy = mock_func_factory(nominatim.tools.add_osm_data, 'add_data_from_file')
+    assert cli_call('add-data', '--' + name, str(oid)) == 0

 
 

-        assert mock_run_legacy.called == 1
-        assert mock_run_legacy.last_args[0] == script + '.php'
+    assert mock_run_legacy.called == 1

 
 
 
 

-    @pytest.mark.parametrize("params", [('--warm', ),
-                                        ('--warm', '--reverse-only'),
-                                        ('--warm', '--search-only')])
-    def test_admin_command_legacy(self, mock_func_factory, params):
-        mock_run_legacy = mock_func_factory(nominatim.clicmd.admin, 'run_legacy_script')
+@pytest.mark.parametrize("name,oid", [('node', 12), ('way', 8), ('relation', 32)])
+def test_cli_add_data_object_command(cli_call, mock_func_factory, name, oid):
+    mock_run_legacy = mock_func_factory(nominatim.tools.add_osm_data, 'add_osm_object')
+    assert cli_call('add-data', '--' + name, str(oid)) == 0

 
 

-        assert self.call_nominatim('admin', *params) == 0
+    assert mock_run_legacy.called == 1

 
 

-        assert mock_run_legacy.called == 1

 
 
 
 

-    def test_admin_command_check_database(self, mock_func_factory):
-        mock = mock_func_factory(nominatim.tools.check_database, 'check_database')
+def test_cli_add_data_tiger_data(cli_call, cli_tokenizer_mock, mock_func_factory):
+    mock = mock_func_factory(nominatim.tools.tiger_data, 'add_tiger_data')

 
 

-        assert self.call_nominatim('admin', '--check-database') == 0
-        assert mock.called == 1
+    assert cli_call('add-data', '--tiger-data', 'somewhere') == 0

 
 

+    assert mock.called == 1

 
 

-    @pytest.mark.parametrize("name,oid", [('file', 'foo.osm'), ('diff', 'foo.osc')])
-    def test_add_data_file_command(self, mock_func_factory, name, oid):
-        mock_run_legacy = mock_func_factory(nominatim.tools.add_osm_data, 'add_data_from_file')
-        assert self.call_nominatim('add-data', '--' + name, str(oid)) == 0

 
 

-        assert mock_run_legacy.called == 1
+def test_cli_serve_php(cli_call, mock_func_factory):
+    func = mock_func_factory(nominatim.cli, 'run_php_server')

 
 

+    cli_call('serve', '--engine', 'php') == 0

 
 

-    @pytest.mark.parametrize("name,oid", [('node', 12), ('way', 8), ('relation', 32)])
-    def test_add_data_object_command(self, mock_func_factory, name, oid):
-        mock_run_legacy = mock_func_factory(nominatim.tools.add_osm_data, 'add_osm_object')
-        assert self.call_nominatim('add-data', '--' + name, str(oid)) == 0
+    assert func.called == 1

 
 

-        assert mock_run_legacy.called == 1

 
 

+def test_cli_serve_starlette_custom_server(cli_call, mock_func_factory):
+    pytest.importorskip("starlette")
+    mod = pytest.importorskip("uvicorn")
+    func = mock_func_factory(mod, "run")

 
 

-    def test_serve_command(self, mock_func_factory):
-        func = mock_func_factory(nominatim.cli, 'run_php_server')
+    cli_call('serve', '--engine', 'starlette', '--server', 'foobar:4545') == 0

 
 

-        self.call_nominatim('serve')
-
-        assert func.called == 1
+    assert func.called == 1
+    assert func.last_kwargs['host'] == 'foobar'
+    assert func.last_kwargs['port'] == 4545

 
 
 
 

-@pytest.mark.parametrize("params", [('search', '--query', 'new'),
-                                    ('reverse', '--lat', '0', '--lon', '0'),
-                                    ('lookup', '--id', 'N1'),
-                                    ('details', '--node', '1'),
-                                    ('details', '--way', '1'),
-                                    ('details', '--relation', '1'),
-                                    ('details', '--place_id', '10001'),
-                                    ('status',)])
-class TestCliApiCall:
+def test_cli_serve_starlette_custom_server_bad_port(cli_call, mock_func_factory):
+    pytest.importorskip("starlette")
+    mod = pytest.importorskip("uvicorn")
+    func = mock_func_factory(mod, "run")

 
 

-    @pytest.fixture(autouse=True)
-    def setup_cli_call(self, cli_call):
-        self.call_nominatim = cli_call
+    cli_call('serve', '--engine', 'starlette', '--server', 'foobar:45:45') == 1

 
 

-    def test_api_commands_simple(self, mock_func_factory, params, tmp_path):
-        (tmp_path / 'website').mkdir()
-        (tmp_path / 'website' / (params[0] + '.php')).write_text('')
-        mock_run_api = mock_func_factory(nominatim.clicmd.api, 'run_api_script')

 
 

-        assert self.call_nominatim(*params, '--project-dir', str(tmp_path)) == 0
+@pytest.mark.parametrize("engine", ['falcon', 'starlette'])
+def test_cli_serve_uvicorn_based(cli_call, engine, mock_func_factory):
+    pytest.importorskip(engine)
+    mod = pytest.importorskip("uvicorn")
+    func = mock_func_factory(mod, "run")

 
 

-        assert mock_run_api.called == 1
-        assert mock_run_api.last_args[0] == params[0]
+    cli_call('serve', '--engine', engine) == 0

 
 

-
-    def test_bad_project_idr(self, mock_func_factory, params):
-        mock_run_api = mock_func_factory(nominatim.clicmd.api, 'run_api_script')
-
-        assert self.call_nominatim(*params) == 1
+    assert func.called == 1
+    assert func.last_kwargs['host'] == '127.0.0.1'
+    assert func.last_kwargs['port'] == 8088

 
 
 class TestCliWithDb:
 
     @pytest.fixture(autouse=True)
 
 
 class TestCliWithDb:
 
     @pytest.fixture(autouse=True)

-    def setup_cli_call(self, cli_call, temp_db):
+    def setup_cli_call(self, cli_call, temp_db, cli_tokenizer_mock):

         self.call_nominatim = cli_call
         self.call_nominatim = cli_call

-
-
-    @pytest.fixture(autouse=True)
-    def setup_tokenizer_mock(self, monkeypatch):
-        class DummyTokenizer:
-            def __init__(self, *args, **kwargs):
-                self.update_sql_functions_called = False
-                self.finalize_import_called = False
-                self.update_statistics_called = False
-
-            def update_sql_functions(self, *args):
-                self.update_sql_functions_called = True
-
-            def finalize_import(self, *args):
-                self.finalize_import_called = True
-
-            def update_statistics(self):
-                self.update_statistics_called = True
-
-
-        tok = DummyTokenizer()
-        monkeypatch.setattr(nominatim.tokenizer.factory, 'get_tokenizer_for_db',
-                            lambda *args: tok)
-        monkeypatch.setattr(nominatim.tokenizer.factory, 'create_tokenizer',
-                            lambda *args: tok)
-
-        self.tokenizer_mock = tok
-
-
-    def test_import_missing_file(self):
-        assert self.call_nominatim('import', '--osm-file', 'sfsafegwedgw.reh.erh') == 1
-
-
-    def test_import_bad_file(self):
-        assert self.call_nominatim('import', '--osm-file', '.') == 1
-
-
-    def test_import_full(self, mock_func_factory):
-        mocks = [
-            mock_func_factory(nominatim.tools.database_import, 'setup_database_skeleton'),
-            mock_func_factory(nominatim.tools.country_info, 'setup_country_tables'),
-            mock_func_factory(nominatim.tools.database_import, 'import_osm_data'),
-            mock_func_factory(nominatim.tools.refresh, 'import_wikipedia_articles'),
-            mock_func_factory(nominatim.tools.database_import, 'truncate_data_tables'),
-            mock_func_factory(nominatim.tools.database_import, 'load_data'),
-            mock_func_factory(nominatim.tools.database_import, 'create_tables'),
-            mock_func_factory(nominatim.tools.database_import, 'create_table_triggers'),
-            mock_func_factory(nominatim.tools.database_import, 'create_partition_tables'),
-            mock_func_factory(nominatim.tools.database_import, 'create_search_indices'),
-            mock_func_factory(nominatim.tools.country_info, 'create_country_names'),
-            mock_func_factory(nominatim.tools.refresh, 'load_address_levels_from_config'),
-            mock_func_factory(nominatim.tools.postcodes, 'update_postcodes'),
-            mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_full'),
-            mock_func_factory(nominatim.tools.refresh, 'setup_website'),
-            mock_func_factory(nominatim.db.properties, 'set_property')
-        ]
-
-        cf_mock = mock_func_factory(nominatim.tools.refresh, 'create_functions')
-
-        assert self.call_nominatim('import', '--osm-file', __file__) == 0
-        assert self.tokenizer_mock.finalize_import_called
-
-        assert cf_mock.called > 1
-
-        for mock in mocks:
-            assert mock.called == 1, "Mock '{}' not called".format(mock.func_name)
-
-
-    def test_import_continue_load_data(self, mock_func_factory):
-        mocks = [
-            mock_func_factory(nominatim.tools.database_import, 'truncate_data_tables'),
-            mock_func_factory(nominatim.tools.database_import, 'load_data'),
-            mock_func_factory(nominatim.tools.database_import, 'create_search_indices'),
-            mock_func_factory(nominatim.tools.country_info, 'create_country_names'),
-            mock_func_factory(nominatim.tools.postcodes, 'update_postcodes'),
-            mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_full'),
-            mock_func_factory(nominatim.tools.refresh, 'setup_website'),
-            mock_func_factory(nominatim.db.properties, 'set_property')
-        ]
-
-        assert self.call_nominatim('import', '--continue', 'load-data') == 0
-        assert self.tokenizer_mock.finalize_import_called
-
-        for mock in mocks:
-            assert mock.called == 1, "Mock '{}' not called".format(mock.func_name)
-
-
-    def test_import_continue_indexing(self, mock_func_factory, placex_table,
-                                      temp_db_conn):
-        mocks = [
-            mock_func_factory(nominatim.tools.database_import, 'create_search_indices'),
-            mock_func_factory(nominatim.tools.country_info, 'create_country_names'),
-            mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_full'),
-            mock_func_factory(nominatim.tools.refresh, 'setup_website'),
-            mock_func_factory(nominatim.db.properties, 'set_property')
-        ]
-
-        assert self.call_nominatim('import', '--continue', 'indexing') == 0
-
-        for mock in mocks:
-            assert mock.called == 1, "Mock '{}' not called".format(mock.func_name)
-
-        assert temp_db_conn.index_exists('idx_placex_pendingsector')
-
-        # Calling it again still works for the index
-        assert self.call_nominatim('import', '--continue', 'indexing') == 0
-        assert temp_db_conn.index_exists('idx_placex_pendingsector')
-
-
-    def test_import_continue_postprocess(self, mock_func_factory):
-        mocks = [
-            mock_func_factory(nominatim.tools.database_import, 'create_search_indices'),
-            mock_func_factory(nominatim.tools.country_info, 'create_country_names'),
-            mock_func_factory(nominatim.tools.refresh, 'setup_website'),
-            mock_func_factory(nominatim.db.properties, 'set_property')
-        ]
-
-        assert self.call_nominatim('import', '--continue', 'db-postprocess') == 0
-
-        assert self.tokenizer_mock.finalize_import_called
-
-        for mock in mocks:
-            assert mock.called == 1, "Mock '{}' not called".format(mock.func_name)
+        self.tokenizer_mock = cli_tokenizer_mock

 
 
     def test_freeze_command(self, mock_func_factory):
 
 
     def test_freeze_command(self, mock_func_factory):


@@ -264,15 +119,6 @@ class TestCliWithDb:
         assert mock_flatnode.called == 1
 
 
         assert mock_flatnode.called == 1
 
 

-
-    @pytest.mark.parametrize("func, params", [('analyse_indexing', ('--analyse-indexing', ))])
-    def test_admin_command_tool(self, mock_func_factory, func, params):
-        mock = mock_func_factory(nominatim.tools.admin, func)
-
-        assert self.call_nominatim('admin', *params) == 0
-        assert mock.called == 1
-
-

     @pytest.mark.parametrize("params,do_bnds,do_ranks", [
                               ([], 1, 1),
                               (['--boundaries-only'], 1, 0),
     @pytest.mark.parametrize("params,do_bnds,do_ranks", [
                               ([], 1, 1),
                               (['--boundaries-only'], 1, 0),


@@ -289,71 +135,27 @@ class TestCliWithDb:
         assert bnd_mock.called == do_bnds
         assert rank_mock.called == do_ranks
 
         assert bnd_mock.called == do_bnds
         assert rank_mock.called == do_ranks
 

-    @pytest.mark.parametrize("no_replace", [(True), (False)])
-    def test_special_phrases_wiki_command(self, mock_func_factory, no_replace):
+
+    def test_special_phrases_wiki_command(self, mock_func_factory):

         func = mock_func_factory(nominatim.clicmd.special_phrases.SPImporter, 'import_phrases')
 
         func = mock_func_factory(nominatim.clicmd.special_phrases.SPImporter, 'import_phrases')
 

-        if no_replace:
-            self.call_nominatim('special-phrases', '--import-from-wiki', '--no-replace')
-        else:
-            self.call_nominatim('special-phrases', '--import-from-wiki')
+        self.call_nominatim('special-phrases', '--import-from-wiki', '--no-replace')

 
         assert func.called == 1
 
 
         assert func.called == 1
 

-    @pytest.mark.parametrize("no_replace", [(True), (False)])
-    def test_special_phrases_csv_command(self, src_dir, mock_func_factory, no_replace):
+
+    def test_special_phrases_csv_command(self, src_dir, mock_func_factory):

         func = mock_func_factory(nominatim.clicmd.special_phrases.SPImporter, 'import_phrases')
         testdata = src_dir / 'test' / 'testdb'
         csv_path = str((testdata / 'full_en_phrases_test.csv').resolve())
 
         func = mock_func_factory(nominatim.clicmd.special_phrases.SPImporter, 'import_phrases')
         testdata = src_dir / 'test' / 'testdb'
         csv_path = str((testdata / 'full_en_phrases_test.csv').resolve())
 

-        if no_replace:
-            self.call_nominatim('special-phrases', '--import-from-csv', csv_path, '--no-replace')
-        else:
-            self.call_nominatim('special-phrases', '--import-from-csv', csv_path)
+        self.call_nominatim('special-phrases', '--import-from-csv', csv_path)

 
         assert func.called == 1
 
 
         assert func.called == 1
 

-    @pytest.mark.parametrize("command,func", [
-                             ('address-levels', 'load_address_levels_from_config'),
-                             ('wiki-data', 'import_wikipedia_articles'),
-                             ('importance', 'recompute_importance'),
-                             ('website', 'setup_website'),
-                             ])
-    def test_refresh_command(self, mock_func_factory, command, func):
-        func_mock = mock_func_factory(nominatim.tools.refresh, func)
-
-        assert self.call_nominatim('refresh', '--' + command) == 0
-        assert func_mock.called == 1
-
-
-    def test_refresh_word_count(self):
-        assert self.call_nominatim('refresh', '--word-count') == 0
-        assert self.tokenizer_mock.update_statistics_called
-
-
-    def test_refresh_postcodes(self, mock_func_factory, place_table):
-        func_mock = mock_func_factory(nominatim.tools.postcodes, 'update_postcodes')
-        idx_mock = mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_postcodes')
-
-        assert self.call_nominatim('refresh', '--postcodes') == 0
-        assert func_mock.called == 1
-        assert idx_mock.called == 1
-
-    def test_refresh_create_functions(self, mock_func_factory):
-        func_mock = mock_func_factory(nominatim.tools.refresh, 'create_functions')
-
-        assert self.call_nominatim('refresh', '--functions') == 0
-        assert func_mock.called == 1
-        assert self.tokenizer_mock.update_sql_functions_called
-
-
-    def test_refresh_importance_computed_after_wiki_import(self, monkeypatch):
-        calls = []
-        monkeypatch.setattr(nominatim.tools.refresh, 'import_wikipedia_articles',
-                            lambda *args, **kwargs: calls.append('import') or 0)
-        monkeypatch.setattr(nominatim.tools.refresh, 'recompute_importance',
-                            lambda *args, **kwargs: calls.append('update'))

 
 

-        assert self.call_nominatim('refresh', '--importance', '--wiki-data') == 0
+    def test_special_phrases_csv_bad_file(self, src_dir):
+        testdata = src_dir / 'something349053905.csv'

 
 

-        assert calls == ['import', 'update']
+        self.call_nominatim('special-phrases', '--import-from-csv',
+                            str(testdata.resolve())) == 1