From: Sarah Hoffmann <lonvia@denofr.de>
Date: Sat, 21 Aug 2021 08:45:22 +0000 (+0200)
Subject: CI: use vagrant scripts for import tests
X-Git-Tag: v4.0.0~35^2~3
X-Git-Url: https://git.openstreetmap.org/nominatim.git/commitdiff_plain/6ebbbfee61eed2ba3ef337833ad60c328fadd0f8
CI: use vagrant scripts for import tests
Use vanilla docker images of Ubuntu and leave the setup
to the vagrant scripts. Then do the usual import tests.
Also fixes a couple of issues found with the scripts
---
diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml
index 98ff4208..990f7ad6 100644
--- a/.github/workflows/ci-tests.yml
+++ b/.github/workflows/ci-tests.yml
@@ -4,6 +4,7 @@ on: [ push, pull_request ]
jobs:
tests:
+ if: False
strategy:
matrix:
ubuntu: [18, 20]
@@ -120,6 +121,7 @@ jobs:
if: matrix.ubuntu == 20
import:
+ if: False
strategy:
matrix:
ubuntu: [18, 20]
diff --git a/.github/workflows/vagrant-scripts.yml b/.github/workflows/vagrant-scripts.yml
new file mode 100644
index 00000000..5b2df2f4
--- /dev/null
+++ b/.github/workflows/vagrant-scripts.yml
@@ -0,0 +1,152 @@
+name: Install scripts
+
+on: [ push, pull_request ]
+
+jobs:
+ create-archive:
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v2
+ with:
+ submodules: true
+
+ - name: Get Date
+ id: get-date
+ run: |
+ echo "::set-output name=date::$(/bin/date -u "+%Y%W")"
+ shell: bash
+
+ - uses: actions/cache@v2
+ with:
+ path: |
+ data/country_grid.sql.gz
+ key: nominatim-country-data-${{ steps.get-date.outputs.date }}
+
+ - name: Package tarball
+ run: |
+ if [ ! -f data/country_grid.sql.gz ]; then
+ wget --no-verbose -O data/country_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
+ fi
+ cd ..
+ tar cjf nominatim-src.tar.bz2 Nominatim
+ mv nominatim-src.tar.bz2 Nominatim
+
+ - name: 'Upload Artifact'
+ uses: actions/upload-artifact@v2
+ with:
+ name: full-source
+ path: nominatim-src.tar.bz2
+ retention-days: 1
+
+
+ ubuntu:
+ runs-on: ubuntu-latest
+ needs: create-archive
+
+ strategy:
+ matrix:
+ ubuntu: [18, 20]
+ include:
+ - ubuntu: 18
+ install_mode:
+ - ubuntu: 20
+ install_mode: install-apache
+
+ container:
+ image: ubuntu:${{ matrix.ubuntu }}.04
+
+ defaults:
+ run:
+ shell: sudo -Hu nominatim bash --noprofile --norc -eo pipefail {0}
+
+ steps:
+ - name: Prepare container
+ run: |
+ export APT_LISTCHANGES_FRONTEND=none
+ export DEBIAN_FRONTEND=noninteractive
+ apt-get update -qq
+ apt-get install -y git sudo wget
+ ln -snf /usr/share/zoneinfo/$CONTAINER_TIMEZONE /etc/localtime && echo $CONTAINER_TIMEZONE > /etc/timezone
+ shell: bash
+
+ - name: Setup import user
+ run: |
+ useradd -m nominatim -G sudo
+ echo '%sudo ALL=(ALL:ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd
+ echo "/home/nominatim/Nominatim/vagrant/Install-on-Ubuntu-$UBUNTU.sh no $INSTALL_MODE" > /home/nominatim/vagrant.sh
+ shell: bash
+ env:
+ UBUNTU: ${{ matrix.ubuntu }}
+ INSTALL_MODE: ${{ matrix.install_mode }}
+
+ - uses: actions/download-artifact@v2
+ with:
+ name: full-source
+ path: /home/nominatim
+
+ - name: Get Date
+ id: get-date
+ run: |
+ echo "::set-output name=date::$(/bin/date -u "+%Y%W")"
+ shell: bash
+
+ - name: Install Nominatim
+ run: |
+ export USERNAME=nominatim
+ export USERHOME=/home/nominatim
+ export NOSYSTEMD=yes
+ tar xf nominatim-src.tar.bz2
+ . vagrant.sh
+ working-directory: /home/nominatim
+
+ - uses: actions/cache@v2
+ with:
+ path: |
+ monaco-latest.osm.pbf
+ key: nominatim-test-data-${{ steps.get-date.outputs.date }}
+
+ - name: Prepare import environment
+ run: |
+ if [ ! -f monaco-latest.osm.pbf ]; then
+ wget --no-verbose https://download.geofabrik.de/europe/monaco-latest.osm.pbf
+ fi
+ mkdir data-env data-env-reverse
+ chown nominatim. data-env data-env-reverse
+ shell: bash
+
+ - name: Import
+ run: nominatim import --osm-file ../monaco-latest.osm.pbf
+ working-directory: data-env
+
+ - name: Import special phrases
+ run: nominatim special-phrases --import-from-wiki
+ working-directory: data-env
+
+ - name: Check full import
+ run: nominatim admin --check-database
+ working-directory: data-env
+
+ - name: Warm up database
+ run: nominatim admin --warm
+ working-directory: data-env
+
+ - name: Run update
+ run: |
+ sudo apt-get install -y python3-pip
+ pip3 install --user osmium
+ nominatim replication --init
+ NOMINATIM_REPLICATION_MAX_DIFF=1 nominatim replication --once
+ working-directory: data-env
+
+ - name: Run reverse-only import
+ run : |
+ echo 'NOMINATIM_DATABASE_DSN="pgsql:dbname=reverse"' > .env
+ nominatim import --osm-file ../monaco-latest.osm.pbf --reverse-only --no-updates
+ working-directory: data-env-reverse
+ env:
+ NOMINATIM_DATABASE_DSN: pgsql:dbname=reverse
+
+ - name: Check reverse import
+ run: nominatim admin --check-database
+ working-directory: data-env-reverse
diff --git a/vagrant/Install-on-Ubuntu-18.sh b/vagrant/Install-on-Ubuntu-18.sh
index dadce086..9818c2a0 100755
--- a/vagrant/Install-on-Ubuntu-18.sh
+++ b/vagrant/Install-on-Ubuntu-18.sh
@@ -1,8 +1,7 @@
-#!/bin/bash
+#!/bin/bash -e
#
# hacks for broken vagrant box #DOCS:
sudo rm -f /var/lib/dpkg/lock #DOCS:
-sudo update-locale LANG=en_US.UTF-8 #DOCS:
export APT_LISTCHANGES_FRONTEND=none #DOCS:
export DEBIAN_FRONTEND=noninteractive #DOCS:
@@ -15,10 +14,9 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
#
# These instructions expect that you have a freshly installed Ubuntu 18.04.
#
-# Make sure all packages are are up-to-date by running:
+# Make sure all packages are up-to-date by running:
#
- sudo apt-get -o DPkg::options::="--force-confdef" -o DPkg::options::="--force-confold" --force-yes -fuy install grub-pc #DOCS:
sudo apt update -qq
# Now you can install all packages needed for Nominatim:
@@ -30,12 +28,12 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
postgresql-server-dev-10 postgresql-10-postgis-2.4 \
postgresql-contrib-10 postgresql-10-postgis-scripts \
php php-pgsql php-intl libicu-dev python3-pip \
- python3-psycopg2 python3-psutil python3-jinja2 python3-icu git
+ python3-psutil python3-jinja2 python3-icu git
# Some of the Python packages that come with Ubuntu 18.04 are too old, so
# install the latest version from pip:
- pip3 install python-dotenv datrie pyyaml
+ pip3 install --user python-dotenv datrie pyyaml psycopg2
#
# System Configuration
@@ -60,8 +58,10 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
# To be able to copy and paste instructions from this manual, export
# user name and home directory now like this:
#
+if [ "x$USERNAME" == "x" ]; then #DOCS:
export USERNAME=vagrant #DOCS: export USERNAME=nominatim
export USERHOME=/home/vagrant #DOCS: export USERHOME=/srv/nominatim
+fi #DOCS:
#
# **Never, ever run the installation as a root user.** You have been warned.
#
@@ -79,7 +79,11 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
#
# Restart the postgresql service after updating this config file.
- sudo systemctl restart postgresql
+if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
+ sudo pg_ctlcluster 10 main start
+else #DOCS:
+ sudo systemctl restart postgresql #DOCS:
+fi #DOCS:
#
# Finally, we need to add two postgres users: one for the user that does
@@ -133,11 +137,13 @@ fi #DOCS:
#
# The webserver should serve the php scripts from the website directory of your
# [project directory](../admin/Import.md#creating-the-project-directory).
-# Therefore set up a project directory and populate the website directory:
+# This directory needs to exist when being configured.
+# Therefore set up a project directory and create the website directory:
mkdir $USERHOME/nominatim-project
- cd $USERHOME/nominatim-project
- nominatim refresh --website
+ mkdir $USERHOME/nominatim-project/website
+
+# The import process will populate the directory later.
#
# Option 1: Using Apache
# ----------------------
@@ -170,7 +176,11 @@ EOFAPACHECONF
#
sudo a2enconf nominatim
+if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
+ sudo apache2ctl start #DOCS:
+else #DOCS:
sudo systemctl restart apache2
+fi #DOCS:
# The Nominatim API is now available at `http://localhost/nominatim/`.
@@ -250,7 +260,12 @@ EOF_NGINX_CONF
# Enable the configuration and restart Nginx
#
+if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
+ sudo /usr/sbin/php-fpm7.2 --nodaemonize --fpm-config /etc/php/7.2/fpm/php-fpm.conf & #DOCS:
+ sudo /usr/sbin/nginx & #DOCS:
+else #DOCS:
sudo systemctl restart php7.2-fpm nginx
+fi #DOCS:
# The Nominatim API is now available at `http://localhost/`.
diff --git a/vagrant/Install-on-Ubuntu-20.sh b/vagrant/Install-on-Ubuntu-20.sh
index bbad7a57..0850644d 100755
--- a/vagrant/Install-on-Ubuntu-20.sh
+++ b/vagrant/Install-on-Ubuntu-20.sh
@@ -1,8 +1,7 @@
-#!/bin/bash
+#!/bin/bash -e
#
# hacks for broken vagrant box #DOCS:
sudo rm -f /var/lib/dpkg/lock #DOCS:
-sudo update-locale LANG=en_US.UTF-8 #DOCS:
export APT_LISTCHANGES_FRONTEND=none #DOCS:
export DEBIAN_FRONTEND=noninteractive #DOCS:
@@ -14,14 +13,9 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
#
# These instructions expect that you have a freshly installed Ubuntu 20.04.
#
-# Make sure all packages are are up-to-date by running:
+# Make sure all packages are up-to-date by running:
#
-#DOCS: :::sh
- sudo apt-get \ #DOCS:
- -o DPkg::options::="--force-confdef" -o DPkg::options::="--force-confold" \ #DOCS:
- --allow-downgrades --allow-remove-essential --allow-change-held-packages \ #DOCS:
- -fuy install grub-pc #DOCS:
sudo apt update -qq
# Now you can install all packages needed for Nominatim:
@@ -59,8 +53,10 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
# To be able to copy and paste instructions from this manual, export
# user name and home directory now like this:
#
+if [ "x$USERNAME" == "x" ]; then #DOCS:
export USERNAME=vagrant #DOCS: export USERNAME=nominatim
export USERHOME=/home/vagrant #DOCS: export USERHOME=/srv/nominatim
+fi #DOCS:
#
# **Never, ever run the installation as a root user.** You have been warned.
#
@@ -78,8 +74,11 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
#
# Restart the postgresql service after updating this config file.
- sudo systemctl restart postgresql
-
+if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
+ sudo pg_ctlcluster 12 main start
+else #DOCS:
+ sudo systemctl restart postgresql #DOCS:
+fi #DOCS:
#
# Finally, we need to add two postgres users: one for the user that does
# the import and another for the webserver which should access the database
@@ -131,11 +130,13 @@ fi #DOCS:
#
# The webserver should serve the php scripts from the website directory of your
# [project directory](../admin/Import.md#creating-the-project-directory).
-# Therefore set up a project directory and populate the website directory:
+# This directory needs to exist when being configured.
+# Therefore set up a project directory and create a website directory:
mkdir $USERHOME/nominatim-project
- cd $USERHOME/nominatim-project
- nominatim refresh --website
+ mkdir $USERHOME/nominatim-project/website
+
+# The import process will populate the directory later.
#
# Option 1: Using Apache
@@ -169,7 +170,11 @@ EOFAPACHECONF
#
sudo a2enconf nominatim
+if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
+ sudo apache2ctl start #DOCS:
+else #DOCS:
sudo systemctl restart apache2
+fi #DOCS:
# The Nominatim API is now available at `http://localhost/nominatim/`.
@@ -252,7 +257,12 @@ EOF_NGINX_CONF
# Enable the configuration and restart Nginx
#
+if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
+ sudo /usr/sbin/php-fpm7.3 --nodaemonize --fpm-config /etc/php/7.3/fpm/php-fpm.conf & #DOCS:
+ sudo /usr/sbin/nginx & #DOCS:
+else #DOCS:
sudo systemctl restart php7.4-fpm nginx
+fi #DOCS:
# The Nominatim API is now available at `http://localhost/`.