From: Matt Amos Date: Mon, 21 Oct 2019 13:22:59 +0000 (+0100) Subject: Merge remote-tracking branch 'openstreetmap/master' into mac-os-x-install X-Git-Tag: live~2540^2 X-Git-Url: https://git.openstreetmap.org/rails.git/commitdiff_plain/afe679349fb7139f5cff5830f45e4b8214732812?hp=5895ae62af60a964a7c028cc23813fb292c566aa Merge remote-tracking branch 'openstreetmap/master' into mac-os-x-install --- diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 9e27360ca..57f876bd9 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -183,6 +183,7 @@ Style/Documentation: Style/FormatStringToken: Exclude: - 'app/models/concerns/geo_record.rb' + - 'app/views/api/map/_bounds.xml.builder' - 'lib/bounding_box.rb' - 'test/controllers/api/map_controller_test.rb' - 'test/controllers/api/relations_controller_test.rb' diff --git a/.travis.yml b/.travis.yml index e4c81e4c1..eed4575e5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,6 @@ addons: postgresql: 9.5 apt: packages: - - postgresql-server-dev-9.5 - libarchive-dev services: - memcached @@ -16,14 +15,10 @@ env: global: - OSM_MEMCACHE_SERVERS="127.0.0.1" before_script: - - sed -e "/idle_in_transaction_session_timeout/d" -e 's/ IMMUTABLE / /' -e "s/AS '.*libpgosm.*',/AS 'libpgosm',/" -e "/^--/d" db/structure.sql > db/structure.expected + - sed -e 's/ IMMUTABLE / /' -e "/^--/d" db/structure.sql > db/structure.expected - psql -U postgres -c "CREATE DATABASE openstreetmap" - psql -U postgres -c "CREATE EXTENSION btree_gist" openstreetmap - - make -C db/functions libpgosm.so - - ln db/functions/libpgosm.so /tmp - - psql -U postgres -c "CREATE FUNCTION maptile_for_point(int8, int8, int4) RETURNS int4 AS '/tmp/libpgosm', 'maptile_for_point' LANGUAGE C STRICT" openstreetmap - - psql -U postgres -c "CREATE FUNCTION tile_for_point(int4, int4) RETURNS int8 AS '/tmp/libpgosm', 'tile_for_point' LANGUAGE C STRICT" openstreetmap - - psql -U postgres -c "CREATE FUNCTION xid_to_int4(xid) RETURNS int4 AS '/tmp/libpgosm', 'xid_to_int4' LANGUAGE C STRICT" openstreetmap + - psql -U postgres -f db/functions/functions.sql openstreetmap - cp config/travis.database.yml config/database.yml - cp config/example.storage.yml config/storage.yml - touch config/settings.local.yml diff --git a/CONFIGURE.md b/CONFIGURE.md index 91a64119e..a78162b67 100644 --- a/CONFIGURE.md +++ b/CONFIGURE.md @@ -131,3 +131,5 @@ If you want to deploy The Rails Port for production use, you'll need to make a f * The included version of the GPX importer is slow and/or completely inoperable. You should consider using [the high-speed GPX importer](https://git.openstreetmap.org/gpx-import.git/). * Make sure you generate the i18n files and precompile the production assets: `RAILS_ENV=production rake i18n:js:export assets:precompile` * Make sure the web server user as well as the rails user can read, write and create directories in `tmp/`. +* If you want to use diff replication then you will need to install the shared library special SQL functions for the `xid_to_int4` function, for which there is no pure SQL alternative. (See the bottom of [INSTALL.md](INSTALL.md)) +* If you expect to serve a lot of `/changes` API calls, then you might also want to install the shared library versions of the SQL functions. diff --git a/Gemfile.lock b/Gemfile.lock index 9e1e32109..5c6a91b26 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -40,7 +40,7 @@ GEM activemodel (= 5.2.3) activesupport (= 5.2.3) arel (>= 9.0) - activerecord-import (1.0.2) + activerecord-import (1.0.3) activerecord (>= 3.2) activestorage (5.2.3) actionpack (= 5.2.3) @@ -61,8 +61,8 @@ GEM autoprefixer-rails (8.6.5) execjs aws-eventstream (1.0.3) - aws-partitions (1.220.0) - aws-sdk-core (3.68.1) + aws-partitions (1.226.0) + aws-sdk-core (3.69.1) aws-eventstream (~> 1.0, >= 1.0.2) aws-partitions (~> 1.0) aws-sigv4 (~> 1.1) @@ -70,7 +70,7 @@ GEM aws-sdk-kms (1.24.0) aws-sdk-core (~> 3, >= 3.61.1) aws-sigv4 (~> 1.1) - aws-sdk-s3 (1.48.0) + aws-sdk-s3 (1.50.0) aws-sdk-core (~> 3, >= 3.61.1) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.1) @@ -131,7 +131,7 @@ GEM tins (~> 1.6) crack (0.4.3) safe_yaml (~> 1.0.0) - crass (1.0.4) + crass (1.0.5) dalli (2.7.10) debug_inspector (0.0.3) deep_merge (1.2.1) @@ -150,20 +150,20 @@ GEM dry-core (0.4.9) concurrent-ruby (~> 1.0) dry-equalizer (0.2.2) - dry-inflector (0.1.2) + dry-inflector (0.2.0) dry-initializer (3.0.1) dry-logic (1.0.3) concurrent-ruby (~> 1.0) dry-core (~> 0.2) dry-equalizer (~> 0.2) - dry-schema (1.3.4) + dry-schema (1.4.1) concurrent-ruby (~> 1.0) dry-configurable (~> 0.8, >= 0.8.3) dry-core (~> 0.4) dry-equalizer (~> 0.2) dry-initializer (~> 3.0) dry-logic (~> 1.0) - dry-types (~> 1.0) + dry-types (~> 1.2) dry-types (1.2.0) concurrent-ruby (~> 1.0) dry-container (~> 0.3) @@ -206,7 +206,7 @@ GEM http_accept_language (2.0.5) i18n (0.9.5) concurrent-ruby (~> 1.0) - i18n-js (3.3.0) + i18n-js (3.4.0) i18n (>= 0.6.6) image_optim (0.26.5) exifr (~> 1.2, >= 1.2.2) @@ -258,7 +258,7 @@ GEM mini_portile2 (2.4.0) minitest (5.12.2) msgpack (1.3.1) - multi_json (1.13.1) + multi_json (1.14.1) multi_xml (0.6.0) multipart-post (2.1.1) nio4r (2.5.2) @@ -371,7 +371,7 @@ GEM rinku (2.0.6) rotp (5.1.0) addressable (~> 2.5) - rubocop (0.75.0) + rubocop (0.75.1) jaro_winkler (~> 1.5.1) parallel (~> 1.10) parser (>= 2.6) @@ -383,7 +383,7 @@ GEM rubocop-rails (2.3.2) rack (>= 1.1) rubocop (>= 0.72.0) - ruby-openid (2.9.1) + ruby-openid (2.9.2) ruby-progressbar (1.10.1) safe_yaml (1.0.5) sanitize (5.1.0) diff --git a/INSTALL.md b/INSTALL.md index ceb4eccb0..e414a12e4 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -31,8 +31,8 @@ sudo apt-get update sudo apt-get install ruby2.5 libruby2.5 ruby2.5-dev bundler \ libmagickwand-dev libxml2-dev libxslt1-dev nodejs \ apache2 apache2-dev build-essential git-core phantomjs \ - postgresql postgresql-contrib libpq-dev postgresql-server-dev-all \ - libsasl2-dev imagemagick libffi-dev libgd-dev libarchive-dev libbz2-dev + postgresql postgresql-contrib libpq-dev libsasl2-dev \ + imagemagick libffi-dev libgd-dev libarchive-dev libbz2-dev sudo gem2.5 install bundler ``` @@ -46,7 +46,7 @@ For Fedora, you can install the minimum requirements with: sudo dnf install ruby ruby-devel rubygem-rdoc rubygem-bundler rubygems \ libxml2-devel js \ gcc gcc-c++ git \ - postgresql postgresql-server postgresql-contrib postgresql-devel \ + postgresql postgresql-server postgresql-contrib \ perl-podlators ImageMagick libffi-devel gd-devel libarchive-devel \ bzip2-devel nodejs-yarn ``` @@ -206,20 +206,10 @@ psql -d openstreetmap -c "CREATE EXTENSION btree_gist" ### PostgreSQL Functions -We need to install special functions into the PostgreSQL databases, and these are provided by a library that needs compiling first. +We need to install some special functions into the PostgreSQL database: ``` -cd db/functions -make libpgosm.so -cd ../.. -``` - -Then we create the functions within each database. We're using `pwd` to substitute in the current working directory, since PostgreSQL needs the full path. - -``` -psql -d openstreetmap -c "CREATE FUNCTION maptile_for_point(int8, int8, int4) RETURNS int4 AS '`pwd`/db/functions/libpgosm', 'maptile_for_point' LANGUAGE C STRICT" -psql -d openstreetmap -c "CREATE FUNCTION tile_for_point(int4, int4) RETURNS int8 AS '`pwd`/db/functions/libpgosm', 'tile_for_point' LANGUAGE C STRICT" -psql -d openstreetmap -c "CREATE FUNCTION xid_to_int4(xid) RETURNS int4 AS '`pwd`/db/functions/libpgosm', 'xid_to_int4' LANGUAGE C STRICT" +psql -d openstreetmap -f db/functions/functions.sql ``` ### Database structure @@ -258,7 +248,48 @@ Note that the OSM map tiles you see aren't created from your local database - th After installing this software, you may need to carry out some [configuration steps](CONFIGURE.md), depending on your tasks. -# Ruby development install and versions +# Installing compiled shared library database functions (optional) + +There are special database functions required by a (little-used) API call, the migrations and diff replication. The former two are provided as *either* pure SQL functions or a compiled shared library. The SQL versions are installed as part of the recommended install procedure above and the shared library versions are recommended only if you are running a production server making a lot of `/changes` API calls or need the diff replication functionality. + +If you aren't sure which you need, stick with the SQL versions. + +Before installing the functions, it's necessary to install the PostgreSQL server development packages. On Ubuntu this means: + +``` +sudo apt-get install postgresql-server-dev-all +``` + +On Fedora: + +``` +sudo dnf install postgresql-devel +``` + +The library then needs compiling. + +``` +cd db/functions +make libpgosm.so +cd ../.. +``` + +If you previously installed the SQL versions of these functions, we'll need to delete those before adding the new ones: + +``` +psql -d openstreetmap -c "DROP FUNCTION IF EXISTS maptile_for_point" +psql -d openstreetmap -c "DROP FUNCTION IF EXISTS tile_for_point" +``` + +Then we create the functions within each database. We're using `pwd` to substitute in the current working directory, since PostgreSQL needs the full path. + +``` +psql -d openstreetmap -c "CREATE FUNCTION maptile_for_point(int8, int8, int4) RETURNS int4 AS '`pwd`/db/functions/libpgosm', 'maptile_for_point' LANGUAGE C STRICT" +psql -d openstreetmap -c "CREATE FUNCTION tile_for_point(int4, int4) RETURNS int8 AS '`pwd`/db/functions/libpgosm', 'tile_for_point' LANGUAGE C STRICT" +psql -d openstreetmap -c "CREATE FUNCTION xid_to_int4(xid) RETURNS int4 AS '`pwd`/db/functions/libpgosm', 'xid_to_int4' LANGUAGE C STRICT" +``` + +# Ruby development install and versions (optional) For simplicity, this document explains how to install all the website dependencies as "system" dependencies. While this is simpler, and usually faster, you might want more control over the process or the ability to install multiple different versions of software alongside eachother. For many developers, [`rbenv`](https://github.com/rbenv/rbenv) is the easiest way to manage multiple different Ruby versions on the same computer - with the added advantage that the installs are all in your home directory, so you don't need administrator permissions. diff --git a/VAGRANT.md b/VAGRANT.md index 9dea61861..ff4cb5737 100644 --- a/VAGRANT.md +++ b/VAGRANT.md @@ -10,8 +10,7 @@ Other Linux distributions should have similar installation instructions using `d Installers are available for Mac OS X and Windows, please see the [Vagrant project download page](https://www.vagrantup.com/downloads.html) for more information. -Note than until there are suitable _xenial64_ [vagrant boxes](https://atlas.hashicorp.com/boxes/search?utf8=%E2%9C%93&sort=&provider=&q=xenial64) for other providers, -the only virtualization provider supported is virtualbox. You might need to install it and specify `--provider virtualbox` when setting up your environment. +We currently support three Vagrant providers, namely `virtualbox`, `lxc` and `libvirt`. You might need to install one of these, for example virtualbox, and specify the provider, e.g. `--provider virtualbox`, when setting up your environment. # Setting up openstreetmap-website diff --git a/db/functions/functions.sql b/db/functions/functions.sql new file mode 100644 index 000000000..5ed00ea63 --- /dev/null +++ b/db/functions/functions.sql @@ -0,0 +1,70 @@ +-------------------------------------------------------------------------------- +-- SQL versions of the C database functions. +-- +-- Pure pl/pgsql versions are *slower* than the C versions, and not recommended +-- for production use. However, they are significantly easier to install, and +-- require fewer dependencies. +-------------------------------------------------------------------------------- + +-- tile_for_point function returns a Morton-encoded integer representing a z16 +-- tile which contains the given (scaled_lon, scaled_lat) coordinate. Note that +-- these are passed into the function as (lat, lon) and should be scaled by +-- 10^7. +-- +-- The Morton encoding packs two dimensions down to one with fairly good +-- spatial locality, and can be used to index points without the need for a +-- proper 2D index. +CREATE OR REPLACE FUNCTION tile_for_point(scaled_lat int4, scaled_lon int4) + RETURNS int8 + AS $$ +DECLARE + x int8; -- quantized x from lon, + y int8; -- quantized y from lat, +BEGIN + x := round(((scaled_lon / 10000000.0) + 180.0) * 65535.0 / 360.0); + y := round(((scaled_lat / 10000000.0) + 90.0) * 65535.0 / 180.0); + + -- these bit-masks are special numbers used in the bit interleaving algorithm. + -- see https://graphics.stanford.edu/~seander/bithacks.html#InterleaveBMN + -- for the original algorithm and more details. + x := (x | (x << 8)) & 16711935; -- 0x00FF00FF + x := (x | (x << 4)) & 252645135; -- 0x0F0F0F0F + x := (x | (x << 2)) & 858993459; -- 0x33333333 + x := (x | (x << 1)) & 1431655765; -- 0x55555555 + + y := (y | (y << 8)) & 16711935; -- 0x00FF00FF + y := (y | (y << 4)) & 252645135; -- 0x0F0F0F0F + y := (y | (y << 2)) & 858993459; -- 0x33333333 + y := (y | (y << 1)) & 1431655765; -- 0x55555555 + + RETURN (x << 1) | y; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + + +-- maptile_for_point returns an integer representing the tile at the given zoom +-- which contains the point (scaled_lon, scaled_lat). Note that the arguments +-- are in the order (lat, lon), and should be scaled by 10^7. +-- +-- The maptile_for_point function is used only for grouping the results of the +-- (deprecated?) /changes API call. Please don't use it for anything else, as +-- it might go away in the future. +CREATE OR REPLACE FUNCTION maptile_for_point(scaled_lat int8, scaled_lon int8, zoom int4) + RETURNS int4 + AS $$ +DECLARE + lat CONSTANT DOUBLE PRECISION := scaled_lat / 10000000.0; + lon CONSTANT DOUBLE PRECISION := scaled_lon / 10000000.0; + zscale CONSTANT DOUBLE PRECISION := 2.0 ^ zoom; + pi CONSTANT DOUBLE PRECISION := 3.141592653589793; + r_per_d CONSTANT DOUBLE PRECISION := pi / 180.0; + x int4; + y int4; +BEGIN + -- straight port of the C code. see db/functions/maptile.c + x := floor((lon + 180.0) * zscale / 360.0); + y := floor((1.0 - ln(tan(lat * r_per_d) + 1.0 / cos(lat * r_per_d)) / pi) * zscale / 2.0); + + RETURN (x << zoom) | y; +END; +$$ LANGUAGE plpgsql IMMUTABLE; diff --git a/db/structure.sql b/db/structure.sql index 47f3cf7a0..03950502c 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -1,6 +1,5 @@ SET statement_timeout = 0; SET lock_timeout = 0; -SET idle_in_transaction_session_timeout = 0; SET client_encoding = 'UTF8'; SET standard_conforming_strings = on; SELECT pg_catalog.set_config('search_path', '', false); @@ -132,27 +131,57 @@ CREATE TYPE public.user_status_enum AS ENUM ( -- Name: maptile_for_point(bigint, bigint, integer); Type: FUNCTION; Schema: public; Owner: - -- -CREATE FUNCTION public.maptile_for_point(bigint, bigint, integer) RETURNS integer - LANGUAGE c STRICT - AS '$libdir/libpgosm.so', 'maptile_for_point'; +CREATE FUNCTION public.maptile_for_point(scaled_lat bigint, scaled_lon bigint, zoom integer) RETURNS integer + LANGUAGE plpgsql IMMUTABLE + AS $$ +DECLARE + lat CONSTANT DOUBLE PRECISION := scaled_lat / 10000000.0; + lon CONSTANT DOUBLE PRECISION := scaled_lon / 10000000.0; + zscale CONSTANT DOUBLE PRECISION := 2.0 ^ zoom; + pi CONSTANT DOUBLE PRECISION := 3.141592653589793; + r_per_d CONSTANT DOUBLE PRECISION := pi / 180.0; + x int4; + y int4; +BEGIN + -- straight port of the C code. see db/functions/maptile.c + x := floor((lon + 180.0) * zscale / 360.0); + y := floor((1.0 - ln(tan(lat * r_per_d) + 1.0 / cos(lat * r_per_d)) / pi) * zscale / 2.0); - --- --- Name: tile_for_point(integer, integer); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.tile_for_point(integer, integer) RETURNS bigint - LANGUAGE c STRICT - AS '$libdir/libpgosm.so', 'tile_for_point'; + RETURN (x << zoom) | y; +END; +$$; -- --- Name: xid_to_int4(xid); Type: FUNCTION; Schema: public; Owner: - +-- Name: tile_for_point(integer, integer); Type: FUNCTION; Schema: public; Owner: - -- -CREATE FUNCTION public.xid_to_int4(xid) RETURNS integer - LANGUAGE c IMMUTABLE STRICT - AS '$libdir/libpgosm.so', 'xid_to_int4'; +CREATE FUNCTION public.tile_for_point(scaled_lat integer, scaled_lon integer) RETURNS bigint + LANGUAGE plpgsql IMMUTABLE + AS $$ +DECLARE + x int8; -- quantized x from lon, + y int8; -- quantized y from lat, +BEGIN + x := round(((scaled_lon / 10000000.0) + 180.0) * 65535.0 / 360.0); + y := round(((scaled_lat / 10000000.0) + 90.0) * 65535.0 / 180.0); + + -- these bit-masks are special numbers used in the bit interleaving algorithm. + -- see https://graphics.stanford.edu/~seander/bithacks.html#InterleaveBMN + -- for the original algorithm and more details. + x := (x | (x << 8)) & 16711935; -- 0x00FF00FF + x := (x | (x << 4)) & 252645135; -- 0x0F0F0F0F + x := (x | (x << 2)) & 858993459; -- 0x33333333 + x := (x | (x << 1)) & 1431655765; -- 0x55555555 + + y := (y | (y << 8)) & 16711935; -- 0x00FF00FF + y := (y | (y << 4)) & 252645135; -- 0x0F0F0F0F + y := (y | (y << 2)) & 858993459; -- 0x33333333 + y := (y | (y << 1)) & 1431655765; -- 0x55555555 + + RETURN (x << 1) | y; +END; +$$; SET default_tablespace = ''; diff --git a/script/vagrant/setup/provision.sh b/script/vagrant/setup/provision.sh index c181376c9..5eb49b24e 100644 --- a/script/vagrant/setup/provision.sh +++ b/script/vagrant/setup/provision.sh @@ -19,7 +19,7 @@ apt-get upgrade -y apt-get install -y ruby2.5 libruby2.5 ruby2.5-dev \ libmagickwand-dev libxml2-dev libxslt1-dev nodejs \ apache2 apache2-dev build-essential git-core phantomjs \ - postgresql postgresql-contrib libpq-dev postgresql-server-dev-all \ + postgresql postgresql-contrib libpq-dev \ libsasl2-dev imagemagick libffi-dev libgd-dev libarchive-dev libbz2-dev gem2.5 install rake gem2.5 install --version "~> 1.16.2" bundler @@ -38,18 +38,29 @@ if [ "$db_user_exists" != "1" ]; then sudo -u vagrant psql -c "create extension btree_gist" openstreetmap sudo -u vagrant psql -c "create extension btree_gist" osm_test fi -# build and set up postgres extensions -pushd db/functions -sudo -u vagrant make -sudo -u vagrant psql openstreetmap -c "CREATE OR REPLACE FUNCTION maptile_for_point(int8, int8, int4) RETURNS int4 AS '/srv/openstreetmap-website/db/functions/libpgosm.so', 'maptile_for_point' LANGUAGE C STRICT" -sudo -u vagrant psql openstreetmap -c "CREATE OR REPLACE FUNCTION tile_for_point(int4, int4) RETURNS int8 AS '/srv/openstreetmap-website/db/functions/libpgosm.so', 'tile_for_point' LANGUAGE C STRICT" -sudo -u vagrant psql openstreetmap -c "CREATE OR REPLACE FUNCTION xid_to_int4(xid) RETURNS int4 AS '/srv/openstreetmap-website/db/functions/libpgosm.so', 'xid_to_int4' LANGUAGE C STRICT" -popd + + +# install PostgreSQL functions +sudo -u vagrant psql -d openstreetmap -f db/functions/functions.sql +################################################################################ +# *IF* you want a vagrant image which supports replication (or perhaps you're +# using this script to provision some other server and want replication), then +# uncomment the following lines (until popd) and comment out the one above +# (functions.sql). +################################################################################ +#pushd db/functions +#sudo -u vagrant make +#sudo -u vagrant psql openstreetmap -c "CREATE OR REPLACE FUNCTION maptile_for_point(int8, int8, int4) RETURNS int4 AS '/srv/openstreetmap-website/db/functions/libpgosm.so', 'maptile_for_point' LANGUAGE C ST#RICT" +#sudo -u vagrant psql openstreetmap -c "CREATE OR REPLACE FUNCTION tile_for_point(int4, int4) RETURNS int8 AS '/srv/openstreetmap-website/db/functions/libpgosm.so', 'tile_for_point' LANGUAGE C STRICT" +#sudo -u vagrant psql openstreetmap -c "CREATE OR REPLACE FUNCTION xid_to_int4(xid) RETURNS int4 AS '/srv/openstreetmap-website/db/functions/libpgosm.so', 'xid_to_int4' LANGUAGE C STRICT" +#popd + + # set up sample configs if [ ! -f config/database.yml ]; then sudo -u vagrant cp config/example.database.yml config/database.yml fi touch config/settings.local.yml # migrate the database to the latest version -sudo -u vagrant rake db:migrate +sudo -u vagrant bundle exec rake db:migrate popd