From: Sarah Hoffmann <lonvia@denofr.de>
Date: Mon, 18 Jul 2022 07:38:08 +0000 (+0200)
Subject: Merge pull request #2761 from lonvia/repair-index-analysis
X-Git-Tag: v4.1.0~11
X-Git-Url: https://git.openstreetmap.org/nominatim.git/commitdiff_plain/b1903f0fbf1e41c594be9b20f00b8a621dfe7552?hp=4b12d52ef553a0b4990e8ea190037ec007cc231b

Merge pull request #2761 from lonvia/repair-index-analysis

Repair `admin --analyse-indexing`
---

diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml
index 4ce14f93..53b76a03 100644
--- a/.github/workflows/ci-tests.yml
+++ b/.github/workflows/ci-tests.yml
@@ -37,7 +37,7 @@ jobs:
         needs: create-archive
         strategy:
             matrix:
-                ubuntu: [18, 20]
+                ubuntu: [18, 20, 22]
                 include:
                     - ubuntu: 18
                       postgresql: 9.6
@@ -49,6 +49,11 @@ jobs:
                       postgis: 3
                       pytest: py.test-3
                       php: 7.4
+                    - ubuntu: 22
+                      postgresql: 14
+                      postgis: 3
+                      pytest: py.test-3
+                      php: 8.1
 
         runs-on: ubuntu-${{ matrix.ubuntu }}.04
 
@@ -65,6 +70,7 @@ jobs:
               with:
                   php-version: ${{ matrix.php }}
                   tools: phpunit, phpcs, composer
+                  ini-values: opcache.jit=disable
 
             - uses: actions/setup-python@v2
               with:
@@ -85,8 +91,12 @@ jobs:
               if: matrix.ubuntu == 20
 
             - name: Install test prerequsites
-              run: pip3 install pytest behave==1.2.6
-              if: matrix.ubuntu == 18
+              run: pip3 install pylint pytest behave==1.2.6
+              if: ${{ (matrix.ubuntu == 18) || (matrix.ubuntu == 22) }}
+
+            - name: Install test prerequsites
+              run: sudo apt-get install -y -qq python3-pytest
+              if: matrix.ubuntu == 22
 
             - name: Install latest pylint
               run: pip3 install pylint
@@ -102,7 +112,7 @@ jobs:
             - name: PHP unit tests
               run: phpunit ./
               working-directory: Nominatim/test/php
-              if: matrix.ubuntu == 20
+              if: ${{ (matrix.ubuntu == 20) || (matrix.ubuntu == 22) }}
 
             - name: Python unit tests
               run: $PYTEST test/python
@@ -161,7 +171,7 @@ jobs:
 
         strategy:
             matrix:
-                name: [Ubuntu-18, Ubuntu-20]
+                name: [Ubuntu-18, Ubuntu-20, Ubuntu-22]
                 include:
                     - name: Ubuntu-18
                       flavour: ubuntu
@@ -173,6 +183,11 @@ jobs:
                       image: "ubuntu:20.04"
                       ubuntu: 20
                       install_mode: install-apache
+                    - name: Ubuntu-22
+                      flavour: ubuntu
+                      image: "ubuntu:22.04"
+                      ubuntu: 22
+                      install_mode: install-apache
 
         container:
             image: ${{ matrix.image }}
diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt
index 3b9a316b..0ccc5974 100644
--- a/docs/CMakeLists.txt
+++ b/docs/CMakeLists.txt
@@ -26,6 +26,7 @@ ADD_CUSTOM_TARGET(doc
    COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Centos-8.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Centos-8.md
    COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-18.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-18.md
    COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-20.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-20.md
+   COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-22.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-22.md
    COMMAND PYTHONPATH=${PROJECT_SOURCE_DIR} mkdocs build -d ${CMAKE_CURRENT_BINARY_DIR}/../site-html -f ${CMAKE_CURRENT_BINARY_DIR}/../mkdocs.yml
 )
 
diff --git a/docs/admin/Installation.md b/docs/admin/Installation.md
index f5411604..96546cf3 100644
--- a/docs/admin/Installation.md
+++ b/docs/admin/Installation.md
@@ -89,8 +89,7 @@ your `postgresql.conf` file.
     work_mem = (50MB)
     effective_cache_size = (24GB)
     synchronous_commit = off
-    checkpoint_segments = 100 # only for postgresql <= 9.4
-    max_wal_size = 1GB # postgresql > 9.4
+    max_wal_size = 1GB
     checkpoint_timeout = 10min
     checkpoint_completion_target = 0.9
 
diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml
index a156fb71..48fe1d0d 100644
--- a/docs/mkdocs.yml
+++ b/docs/mkdocs.yml
@@ -46,6 +46,7 @@ nav:
         - 'Installation on CentOS 8' : 'appendix/Install-on-Centos-8.md'
         - 'Installation on Ubuntu 18' : 'appendix/Install-on-Ubuntu-18.md'
         - 'Installation on Ubuntu 20' : 'appendix/Install-on-Ubuntu-20.md'
+        - 'Installation on Ubuntu 22' : 'appendix/Install-on-Ubuntu-22.md'
 markdown_extensions:
     - codehilite
     - admonition
diff --git a/lib-php/PlaceLookup.php b/lib-php/PlaceLookup.php
index 715f1ced..33156537 100644
--- a/lib-php/PlaceLookup.php
+++ b/lib-php/PlaceLookup.php
@@ -445,7 +445,7 @@ class PlaceLookup
 
             if ($this->bExtraTags) {
                 if ($aPlace['extra']) {
-                    $aPlace['sExtraTags'] = json_decode($aPlace['extra']);
+                    $aPlace['sExtraTags'] = json_decode($aPlace['extra'], true);
                 } else {
                     $aPlace['sExtraTags'] = (object) array();
                 }
@@ -482,7 +482,7 @@ class PlaceLookup
             return (object) array();
         }
 
-        $aFullNames = json_decode($sNames);
+        $aFullNames = json_decode($sNames, true);
         $aNames = array();
 
         foreach ($aFullNames as $sKey => $sValue) {
diff --git a/lib-php/template/address-geocodejson.php b/lib-php/template/address-geocodejson.php
index 8a0a6289..d54aef40 100644
--- a/lib-php/template/address-geocodejson.php
+++ b/lib-php/template/address-geocodejson.php
@@ -56,7 +56,7 @@ if (empty($aPlace)) {
     }
 
     if (isset($aPlace['asgeojson'])) {
-        $aFilteredPlaces['geometry'] = json_decode($aPlace['asgeojson']);
+        $aFilteredPlaces['geometry'] = json_decode($aPlace['asgeojson'], true);
     } else {
         $aFilteredPlaces['geometry'] = array(
                                         'type' => 'Point',
diff --git a/lib-php/template/address-geojson.php b/lib-php/template/address-geojson.php
index 206b959f..dc3c3832 100644
--- a/lib-php/template/address-geojson.php
+++ b/lib-php/template/address-geojson.php
@@ -65,7 +65,7 @@ if (empty($aPlace)) {
     }
 
     if (isset($aPlace['asgeojson'])) {
-        $aFilteredPlaces['geometry'] = json_decode($aPlace['asgeojson']);
+        $aFilteredPlaces['geometry'] = json_decode($aPlace['asgeojson'], true);
     } else {
         $aFilteredPlaces['geometry'] = array(
                                         'type' => 'Point',
diff --git a/lib-php/template/address-json.php b/lib-php/template/address-json.php
index 1a429acb..0766eaf4 100644
--- a/lib-php/template/address-json.php
+++ b/lib-php/template/address-json.php
@@ -63,7 +63,7 @@ if (empty($aPlace)) {
     }
 
     if (isset($aPlace['asgeojson'])) {
-        $aFilteredPlaces['geojson'] = json_decode($aPlace['asgeojson']);
+        $aFilteredPlaces['geojson'] = json_decode($aPlace['asgeojson'], true);
     }
 
     if (isset($aPlace['assvg'])) {
diff --git a/lib-php/template/details-json.php b/lib-php/template/details-json.php
index 68a0ce25..ae80a85b 100644
--- a/lib-php/template/details-json.php
+++ b/lib-php/template/details-json.php
@@ -48,7 +48,7 @@ $aPlaceDetails['centroid'] = array(
                               'coordinates' => array( (float) $aPointDetails['lon'], (float) $aPointDetails['lat'] )
                              );
 
-$aPlaceDetails['geometry'] = json_decode($aPointDetails['asgeojson']);
+$aPlaceDetails['geometry'] = json_decode($aPointDetails['asgeojson'], true);
 
 $funcMapAddressLine = function ($aFull) {
     return array(
diff --git a/lib-php/template/search-batch-json.php b/lib-php/template/search-batch-json.php
index cbf25c87..430237a2 100644
--- a/lib-php/template/search-batch-json.php
+++ b/lib-php/template/search-batch-json.php
@@ -60,7 +60,7 @@ foreach ($aBatchResults as $aSearchResults) {
         }
 
         if (isset($aPointDetails['asgeojson'])) {
-            $aPlace['geojson'] = json_decode($aPointDetails['asgeojson']);
+            $aPlace['geojson'] = json_decode($aPointDetails['asgeojson'], true);
         }
 
         if (isset($aPointDetails['assvg'])) {
diff --git a/lib-php/template/search-geocodejson.php b/lib-php/template/search-geocodejson.php
index 5439e3cf..bba41a0d 100644
--- a/lib-php/template/search-geocodejson.php
+++ b/lib-php/template/search-geocodejson.php
@@ -46,7 +46,7 @@ foreach ($aSearchResults as $iResNum => $aPointDetails) {
     }
 
     if (isset($aPointDetails['asgeojson'])) {
-        $aPlace['geometry'] = json_decode($aPointDetails['asgeojson']);
+        $aPlace['geometry'] = json_decode($aPointDetails['asgeojson'], true);
     } else {
         $aPlace['geometry'] = array(
                                'type' => 'Point',
diff --git a/lib-php/template/search-geojson.php b/lib-php/template/search-geojson.php
index f517142f..7665700d 100644
--- a/lib-php/template/search-geojson.php
+++ b/lib-php/template/search-geojson.php
@@ -54,7 +54,7 @@ foreach ($aSearchResults as $iResNum => $aPointDetails) {
     }
 
     if (isset($aPointDetails['asgeojson'])) {
-        $aPlace['geometry'] = json_decode($aPointDetails['asgeojson']);
+        $aPlace['geometry'] = json_decode($aPointDetails['asgeojson'], true);
     } else {
         $aPlace['geometry'] = array(
                                'type' => 'Point',
diff --git a/lib-php/template/search-json.php b/lib-php/template/search-json.php
index 3f9e0a57..5fb13020 100644
--- a/lib-php/template/search-json.php
+++ b/lib-php/template/search-json.php
@@ -53,7 +53,7 @@ foreach ($aSearchResults as $iResNum => $aPointDetails) {
     }
 
     if (isset($aPointDetails['asgeojson'])) {
-        $aPlace['geojson'] = json_decode($aPointDetails['asgeojson']);
+        $aPlace['geojson'] = json_decode($aPointDetails['asgeojson'], true);
     }
 
     if (isset($aPointDetails['assvg'])) {
diff --git a/test/python/tools/test_exec_utils.py b/test/python/tools/test_exec_utils.py
index 78650180..26ea92b2 100644
--- a/test/python/tools/test_exec_utils.py
+++ b/test/python/tools/test_exec_utils.py
@@ -122,7 +122,13 @@ class TestRunApiScript:
 
     @staticmethod
     def test_fail_on_error_output(tmp_path):
-        (tmp_path / 'website' / 'bad.php').write_text("<?php\nfwrite(STDERR, 'WARNING'.PHP_EOL);")
+        # Starting PHP 8 the PHP CLI no longer has STDERR defined as constant
+        php = """
+        <?php
+        if(!defined('STDERR')) define('STDERR', fopen('php://stderr', 'wb'));
+        fwrite(STDERR, 'WARNING'.PHP_EOL);
+        """
+        (tmp_path / 'website' / 'bad.php').write_text(php)
 
         assert exec_utils.run_api_script('bad', tmp_path) == 1
 
diff --git a/vagrant/Install-on-Ubuntu-22.sh b/vagrant/Install-on-Ubuntu-22.sh
new file mode 100755
index 00000000..dbb70ffe
--- /dev/null
+++ b/vagrant/Install-on-Ubuntu-22.sh
@@ -0,0 +1,271 @@
+#!/bin/bash -e
+#
+# hacks for broken vagrant box      #DOCS:
+sudo rm -f /var/lib/dpkg/lock       #DOCS:
+export APT_LISTCHANGES_FRONTEND=none #DOCS:
+export DEBIAN_FRONTEND=noninteractive #DOCS:
+
+# *Note:* these installation instructions are also available in executable
+#         form for use with vagrant under vagrant/Install-on-Ubuntu-22.sh.
+#
+# Installing the Required Software
+# ================================
+#
+# These instructions expect that you have a freshly installed Ubuntu 22.04.
+#
+# Make sure all packages are up-to-date by running:
+#
+
+    sudo apt update -qq
+
+# Now you can install all packages needed for Nominatim:
+
+    sudo apt install -y php-cgi
+    sudo apt install -y build-essential cmake g++ libboost-dev libboost-system-dev \
+                        libboost-filesystem-dev libexpat1-dev zlib1g-dev \
+                        libbz2-dev libpq-dev libproj-dev \
+                        postgresql-server-dev-14 postgresql-14-postgis-3 \
+                        postgresql-contrib-14 postgresql-14-postgis-3-scripts \
+                        php php-pgsql php-intl libicu-dev python3-dotenv \
+                        python3-psycopg2 python3-psutil python3-jinja2 \
+                        python3-icu python3-datrie git
+
+#
+# System Configuration
+# ====================
+#
+# The following steps are meant to configure a fresh Ubuntu installation
+# for use with Nominatim. You may skip some of the steps if you have your
+# OS already configured.
+#
+# Creating Dedicated User Accounts
+# --------------------------------
+#
+# Nominatim will run as a global service on your machine. It is therefore
+# best to install it under its own separate user account. In the following
+# we assume this user is called nominatim and the installation will be in
+# /srv/nominatim. To create the user and directory run:
+#
+#     sudo useradd -d /srv/nominatim -s /bin/bash -m nominatim
+#
+# You may find a more suitable location if you wish.
+#
+# To be able to copy and paste instructions from this manual, export
+# user name and home directory now like this:
+#
+if [ "x$USERNAME" == "x" ]; then #DOCS:
+    export USERNAME=vagrant        #DOCS:    export USERNAME=nominatim
+    export USERHOME=/home/vagrant  #DOCS:    export USERHOME=/srv/nominatim
+fi                                 #DOCS:
+#
+# **Never, ever run the installation as a root user.** You have been warned.
+#
+# Make sure that system servers can read from the home directory:
+
+    chmod a+x $USERHOME
+
+# Setting up PostgreSQL
+# ---------------------
+#
+# Tune the postgresql configuration, which is located in 
+# `/etc/postgresql/14/main/postgresql.conf`. See section *Postgres Tuning* in
+# [the installation page](../admin/Installation.md#postgresql-tuning)
+# for the parameters to change.
+#
+# Restart the postgresql service after updating this config file.
+
+if [ "x$NOSYSTEMD" == "xyes" ]; then  #DOCS:
+    sudo pg_ctlcluster 14 main start  #DOCS:
+else                                  #DOCS:
+    sudo systemctl restart postgresql
+fi                                    #DOCS:
+#
+# Finally, we need to add two postgres users: one for the user that does
+# the import and another for the webserver which should access the database
+# for reading only:
+#
+
+    sudo -u postgres createuser -s $USERNAME
+    sudo -u postgres createuser www-data
+
+#
+# Installing Nominatim
+# ====================
+#
+# Building and Configuration
+# --------------------------
+#
+# Get the source code from Github and change into the source directory
+#
+if [ "x$1" == "xyes" ]; then  #DOCS:    :::sh
+    cd $USERHOME
+    git clone --recursive https://github.com/openstreetmap/Nominatim.git
+    cd Nominatim
+else                               #DOCS:
+    cd $USERHOME/Nominatim         #DOCS:
+fi                                 #DOCS:
+
+# When installing the latest source from github, you also need to
+# download the country grid:
+
+if [ ! -f data/country_osm_grid.sql.gz ]; then       #DOCS:    :::sh
+    wget -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
+fi                                 #DOCS:
+
+# The code must be built in a separate directory. Create this directory,
+# then configure and build Nominatim in there:
+
+    mkdir $USERHOME/build
+    cd $USERHOME/build
+    cmake $USERHOME/Nominatim
+    make
+    sudo make install
+
+# Nominatim is now ready to use. You can continue with
+# [importing a database from OSM data](../admin/Import.md). If you want to set up
+# a webserver first, continue reading.
+#
+# Setting up a webserver
+# ======================
+#
+# The webserver should serve the php scripts from the website directory of your
+# [project directory](../admin/Import.md#creating-the-project-directory).
+# This directory needs to exist when being configured.
+# Therefore set up a project directory and create a website directory:
+
+    mkdir $USERHOME/nominatim-project
+    mkdir $USERHOME/nominatim-project/website
+
+# The import process will populate the directory later.
+
+#
+# Option 1: Using Apache
+# ----------------------
+#
+if [ "x$2" == "xinstall-apache" ]; then #DOCS:
+#
+# Apache has a PHP module that can be used to serve Nominatim. To install them
+# run:
+
+    sudo apt install -y apache2 libapache2-mod-php
+
+# You need to create an alias to the website directory in your apache
+# configuration. Add a separate nominatim configuration to your webserver:
+
+#DOCS:```sh
+sudo tee /etc/apache2/conf-available/nominatim.conf << EOFAPACHECONF
+<Directory "$USERHOME/nominatim-project/website">
+  Options FollowSymLinks MultiViews
+  AddType text/html   .php
+  DirectoryIndex search.php
+  Require all granted
+</Directory>
+
+Alias /nominatim $USERHOME/nominatim-project/website
+EOFAPACHECONF
+#DOCS:```
+
+#
+# Then enable the configuration and restart apache
+#
+
+    sudo a2enconf nominatim
+if [ "x$NOSYSTEMD" == "xyes" ]; then  #DOCS:
+    sudo apache2ctl start             #DOCS:
+else                                  #DOCS:
+    sudo systemctl restart apache2
+fi                                    #DOCS:
+
+# The Nominatim API is now available at `http://localhost/nominatim/`.
+
+fi   #DOCS:
+
+#
+# Option 2: Using nginx
+# ---------------------
+#
+if [ "x$2" == "xinstall-nginx" ]; then #DOCS:
+
+# Nginx has no native support for php scripts. You need to set up php-fpm for
+# this purpose. First install nginx and php-fpm:
+
+    sudo apt install -y nginx php-fpm
+
+# You need to configure php-fpm to listen on a Unix socket.
+
+#DOCS:```sh
+sudo tee /etc/php/8.1/fpm/pool.d/www.conf << EOF_PHP_FPM_CONF
+[www]
+; Replace the tcp listener and add the unix socket
+listen = /var/run/php8.1-fpm.sock
+
+; Ensure that the daemon runs as the correct user
+listen.owner = www-data
+listen.group = www-data
+listen.mode = 0666
+
+; Unix user of FPM processes
+user = www-data
+group = www-data
+
+; Choose process manager type (static, dynamic, ondemand)
+pm = ondemand
+pm.max_children = 5
+EOF_PHP_FPM_CONF
+#DOCS:```
+
+# Then create a Nginx configuration to forward http requests to that socket.
+
+#DOCS:```sh
+sudo tee /etc/nginx/sites-available/default << EOF_NGINX_CONF
+server {
+    listen 80 default_server;
+    listen [::]:80 default_server;
+
+    root $USERHOME/nominatim-project/website;
+    index search.php index.html;
+    location / {
+        try_files \$uri \$uri/ @php;
+    }
+
+    location @php {
+        fastcgi_param SCRIPT_FILENAME "\$document_root\$uri.php";
+        fastcgi_param PATH_TRANSLATED "\$document_root\$uri.php";
+        fastcgi_param QUERY_STRING    \$args;
+        fastcgi_pass unix:/var/run/php8.1-fpm.sock;
+        fastcgi_index index.php;
+        include fastcgi_params;
+    }
+
+    location ~ [^/]\.php(/|$) {
+        fastcgi_split_path_info ^(.+?\.php)(/.*)$;
+        if (!-f \$document_root\$fastcgi_script_name) {
+            return 404;
+        }
+        fastcgi_pass unix:/var/run/php7.4-fpm.sock;
+        fastcgi_index search.php;
+        include fastcgi.conf;
+    }
+}
+EOF_NGINX_CONF
+#DOCS:```
+
+# If you have some errors, make sure that php8.1-fpm.sock is well under
+# /var/run/ and not under /var/run/php. Otherwise change the Nginx configuration
+# to /var/run/php/php8.1-fpm.sock.
+#
+# Enable the configuration and restart Nginx
+#
+
+if [ "x$NOSYSTEMD" == "xyes" ]; then  #DOCS:
+    sudo /usr/sbin/php-fpm8.1 --nodaemonize --fpm-config /etc/php/8.1/fpm/php-fpm.conf & #DOCS:
+    sudo /usr/sbin/nginx &            #DOCS:
+else                                  #DOCS:
+    sudo systemctl restart php8.1-fpm nginx
+fi                                    #DOCS:
+
+# The Nominatim API is now available at `http://localhost/`.
+
+
+
+fi   #DOCS: