Style/FormatStringToken:
Exclude:
- 'app/models/concerns/geo_record.rb'
+ - 'app/views/api/map/_bounds.xml.builder'
- 'lib/bounding_box.rb'
- 'test/controllers/api/map_controller_test.rb'
- 'test/controllers/api/relations_controller_test.rb'
postgresql: 9.5
apt:
packages:
- - postgresql-server-dev-9.5
- libarchive-dev
services:
- memcached
global:
- OSM_MEMCACHE_SERVERS="127.0.0.1"
before_script:
- - sed -e "/idle_in_transaction_session_timeout/d" -e 's/ IMMUTABLE / /' -e "s/AS '.*libpgosm.*',/AS 'libpgosm',/" -e "/^--/d" db/structure.sql > db/structure.expected
+ - sed -e 's/ IMMUTABLE / /' -e "/^--/d" db/structure.sql > db/structure.expected
- psql -U postgres -c "CREATE DATABASE openstreetmap"
- psql -U postgres -c "CREATE EXTENSION btree_gist" openstreetmap
- - make -C db/functions libpgosm.so
- - ln db/functions/libpgosm.so /tmp
- - psql -U postgres -c "CREATE FUNCTION maptile_for_point(int8, int8, int4) RETURNS int4 AS '/tmp/libpgosm', 'maptile_for_point' LANGUAGE C STRICT" openstreetmap
- - psql -U postgres -c "CREATE FUNCTION tile_for_point(int4, int4) RETURNS int8 AS '/tmp/libpgosm', 'tile_for_point' LANGUAGE C STRICT" openstreetmap
- - psql -U postgres -c "CREATE FUNCTION xid_to_int4(xid) RETURNS int4 AS '/tmp/libpgosm', 'xid_to_int4' LANGUAGE C STRICT" openstreetmap
+ - psql -U postgres -f db/functions/functions.sql openstreetmap
- cp config/travis.database.yml config/database.yml
- cp config/example.storage.yml config/storage.yml
- touch config/settings.local.yml
* The included version of the GPX importer is slow and/or completely inoperable. You should consider using [the high-speed GPX importer](https://git.openstreetmap.org/gpx-import.git/).
* Make sure you generate the i18n files and precompile the production assets: `RAILS_ENV=production rake i18n:js:export assets:precompile`
* Make sure the web server user as well as the rails user can read, write and create directories in `tmp/`.
+* If you want to use diff replication then you will need to install the shared library special SQL functions for the `xid_to_int4` function, for which there is no pure SQL alternative. (See the bottom of [INSTALL.md](INSTALL.md))
+* If you expect to serve a lot of `/changes` API calls, then you might also want to install the shared library versions of the SQL functions.
activemodel (= 5.2.3)
activesupport (= 5.2.3)
arel (>= 9.0)
- activerecord-import (1.0.2)
+ activerecord-import (1.0.3)
activerecord (>= 3.2)
activestorage (5.2.3)
actionpack (= 5.2.3)
autoprefixer-rails (8.6.5)
execjs
aws-eventstream (1.0.3)
- aws-partitions (1.220.0)
- aws-sdk-core (3.68.1)
+ aws-partitions (1.226.0)
+ aws-sdk-core (3.69.1)
aws-eventstream (~> 1.0, >= 1.0.2)
aws-partitions (~> 1.0)
aws-sigv4 (~> 1.1)
aws-sdk-kms (1.24.0)
aws-sdk-core (~> 3, >= 3.61.1)
aws-sigv4 (~> 1.1)
- aws-sdk-s3 (1.48.0)
+ aws-sdk-s3 (1.50.0)
aws-sdk-core (~> 3, >= 3.61.1)
aws-sdk-kms (~> 1)
aws-sigv4 (~> 1.1)
tins (~> 1.6)
crack (0.4.3)
safe_yaml (~> 1.0.0)
- crass (1.0.4)
+ crass (1.0.5)
dalli (2.7.10)
debug_inspector (0.0.3)
deep_merge (1.2.1)
dry-core (0.4.9)
concurrent-ruby (~> 1.0)
dry-equalizer (0.2.2)
- dry-inflector (0.1.2)
+ dry-inflector (0.2.0)
dry-initializer (3.0.1)
dry-logic (1.0.3)
concurrent-ruby (~> 1.0)
dry-core (~> 0.2)
dry-equalizer (~> 0.2)
- dry-schema (1.3.4)
+ dry-schema (1.4.1)
concurrent-ruby (~> 1.0)
dry-configurable (~> 0.8, >= 0.8.3)
dry-core (~> 0.4)
dry-equalizer (~> 0.2)
dry-initializer (~> 3.0)
dry-logic (~> 1.0)
- dry-types (~> 1.0)
+ dry-types (~> 1.2)
dry-types (1.2.0)
concurrent-ruby (~> 1.0)
dry-container (~> 0.3)
http_accept_language (2.0.5)
i18n (0.9.5)
concurrent-ruby (~> 1.0)
- i18n-js (3.3.0)
+ i18n-js (3.4.0)
i18n (>= 0.6.6)
image_optim (0.26.5)
exifr (~> 1.2, >= 1.2.2)
mini_portile2 (2.4.0)
minitest (5.12.2)
msgpack (1.3.1)
- multi_json (1.13.1)
+ multi_json (1.14.1)
multi_xml (0.6.0)
multipart-post (2.1.1)
nio4r (2.5.2)
rinku (2.0.6)
rotp (5.1.0)
addressable (~> 2.5)
- rubocop (0.75.0)
+ rubocop (0.75.1)
jaro_winkler (~> 1.5.1)
parallel (~> 1.10)
parser (>= 2.6)
rubocop-rails (2.3.2)
rack (>= 1.1)
rubocop (>= 0.72.0)
- ruby-openid (2.9.1)
+ ruby-openid (2.9.2)
ruby-progressbar (1.10.1)
safe_yaml (1.0.5)
sanitize (5.1.0)
sudo apt-get install ruby2.5 libruby2.5 ruby2.5-dev bundler \
libmagickwand-dev libxml2-dev libxslt1-dev nodejs \
apache2 apache2-dev build-essential git-core phantomjs \
- postgresql postgresql-contrib libpq-dev postgresql-server-dev-all \
- libsasl2-dev imagemagick libffi-dev libgd-dev libarchive-dev libbz2-dev
+ postgresql postgresql-contrib libpq-dev libsasl2-dev \
+ imagemagick libffi-dev libgd-dev libarchive-dev libbz2-dev
sudo gem2.5 install bundler
```
sudo dnf install ruby ruby-devel rubygem-rdoc rubygem-bundler rubygems \
libxml2-devel js \
gcc gcc-c++ git \
- postgresql postgresql-server postgresql-contrib postgresql-devel \
+ postgresql postgresql-server postgresql-contrib \
perl-podlators ImageMagick libffi-devel gd-devel libarchive-devel \
bzip2-devel nodejs-yarn
```
### PostgreSQL Functions
-We need to install special functions into the PostgreSQL databases, and these are provided by a library that needs compiling first.
+We need to install some special functions into the PostgreSQL database:
```
-cd db/functions
-make libpgosm.so
-cd ../..
-```
-
-Then we create the functions within each database. We're using `pwd` to substitute in the current working directory, since PostgreSQL needs the full path.
-
-```
-psql -d openstreetmap -c "CREATE FUNCTION maptile_for_point(int8, int8, int4) RETURNS int4 AS '`pwd`/db/functions/libpgosm', 'maptile_for_point' LANGUAGE C STRICT"
-psql -d openstreetmap -c "CREATE FUNCTION tile_for_point(int4, int4) RETURNS int8 AS '`pwd`/db/functions/libpgosm', 'tile_for_point' LANGUAGE C STRICT"
-psql -d openstreetmap -c "CREATE FUNCTION xid_to_int4(xid) RETURNS int4 AS '`pwd`/db/functions/libpgosm', 'xid_to_int4' LANGUAGE C STRICT"
+psql -d openstreetmap -f db/functions/functions.sql
```
### Database structure
After installing this software, you may need to carry out some [configuration steps](CONFIGURE.md), depending on your tasks.
-# Ruby development install and versions<a name="rbenv"></a>
+# Installing compiled shared library database functions (optional)
+
+There are special database functions required by a (little-used) API call, the migrations and diff replication. The former two are provided as *either* pure SQL functions or a compiled shared library. The SQL versions are installed as part of the recommended install procedure above and the shared library versions are recommended only if you are running a production server making a lot of `/changes` API calls or need the diff replication functionality.
+
+If you aren't sure which you need, stick with the SQL versions.
+
+Before installing the functions, it's necessary to install the PostgreSQL server development packages. On Ubuntu this means:
+
+```
+sudo apt-get install postgresql-server-dev-all
+```
+
+On Fedora:
+
+```
+sudo dnf install postgresql-devel
+```
+
+The library then needs compiling.
+
+```
+cd db/functions
+make libpgosm.so
+cd ../..
+```
+
+If you previously installed the SQL versions of these functions, we'll need to delete those before adding the new ones:
+
+```
+psql -d openstreetmap -c "DROP FUNCTION IF EXISTS maptile_for_point"
+psql -d openstreetmap -c "DROP FUNCTION IF EXISTS tile_for_point"
+```
+
+Then we create the functions within each database. We're using `pwd` to substitute in the current working directory, since PostgreSQL needs the full path.
+
+```
+psql -d openstreetmap -c "CREATE FUNCTION maptile_for_point(int8, int8, int4) RETURNS int4 AS '`pwd`/db/functions/libpgosm', 'maptile_for_point' LANGUAGE C STRICT"
+psql -d openstreetmap -c "CREATE FUNCTION tile_for_point(int4, int4) RETURNS int8 AS '`pwd`/db/functions/libpgosm', 'tile_for_point' LANGUAGE C STRICT"
+psql -d openstreetmap -c "CREATE FUNCTION xid_to_int4(xid) RETURNS int4 AS '`pwd`/db/functions/libpgosm', 'xid_to_int4' LANGUAGE C STRICT"
+```
+
+# Ruby development install and versions<a name="rbenv"></a> (optional)
For simplicity, this document explains how to install all the website dependencies as "system" dependencies. While this is simpler, and usually faster, you might want more control over the process or the ability to install multiple different versions of software alongside eachother. For many developers, [`rbenv`](https://github.com/rbenv/rbenv) is the easiest way to manage multiple different Ruby versions on the same computer - with the added advantage that the installs are all in your home directory, so you don't need administrator permissions.
Installers are available for Mac OS X and Windows, please see the [Vagrant project download page](https://www.vagrantup.com/downloads.html) for more information.
-Note than until there are suitable _xenial64_ [vagrant boxes](https://atlas.hashicorp.com/boxes/search?utf8=%E2%9C%93&sort=&provider=&q=xenial64) for other providers,
-the only virtualization provider supported is virtualbox. You might need to install it and specify `--provider virtualbox` when setting up your environment.
+We currently support three Vagrant providers, namely `virtualbox`, `lxc` and `libvirt`. You might need to install one of these, for example virtualbox, and specify the provider, e.g. `--provider virtualbox`, when setting up your environment.
# Setting up openstreetmap-website
--- /dev/null
+--------------------------------------------------------------------------------
+-- SQL versions of the C database functions.
+--
+-- Pure pl/pgsql versions are *slower* than the C versions, and not recommended
+-- for production use. However, they are significantly easier to install, and
+-- require fewer dependencies.
+--------------------------------------------------------------------------------
+
+-- tile_for_point function returns a Morton-encoded integer representing a z16
+-- tile which contains the given (scaled_lon, scaled_lat) coordinate. Note that
+-- these are passed into the function as (lat, lon) and should be scaled by
+-- 10^7.
+--
+-- The Morton encoding packs two dimensions down to one with fairly good
+-- spatial locality, and can be used to index points without the need for a
+-- proper 2D index.
+CREATE OR REPLACE FUNCTION tile_for_point(scaled_lat int4, scaled_lon int4)
+ RETURNS int8
+ AS $$
+DECLARE
+ x int8; -- quantized x from lon,
+ y int8; -- quantized y from lat,
+BEGIN
+ x := round(((scaled_lon / 10000000.0) + 180.0) * 65535.0 / 360.0);
+ y := round(((scaled_lat / 10000000.0) + 90.0) * 65535.0 / 180.0);
+
+ -- these bit-masks are special numbers used in the bit interleaving algorithm.
+ -- see https://graphics.stanford.edu/~seander/bithacks.html#InterleaveBMN
+ -- for the original algorithm and more details.
+ x := (x | (x << 8)) & 16711935; -- 0x00FF00FF
+ x := (x | (x << 4)) & 252645135; -- 0x0F0F0F0F
+ x := (x | (x << 2)) & 858993459; -- 0x33333333
+ x := (x | (x << 1)) & 1431655765; -- 0x55555555
+
+ y := (y | (y << 8)) & 16711935; -- 0x00FF00FF
+ y := (y | (y << 4)) & 252645135; -- 0x0F0F0F0F
+ y := (y | (y << 2)) & 858993459; -- 0x33333333
+ y := (y | (y << 1)) & 1431655765; -- 0x55555555
+
+ RETURN (x << 1) | y;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+
+-- maptile_for_point returns an integer representing the tile at the given zoom
+-- which contains the point (scaled_lon, scaled_lat). Note that the arguments
+-- are in the order (lat, lon), and should be scaled by 10^7.
+--
+-- The maptile_for_point function is used only for grouping the results of the
+-- (deprecated?) /changes API call. Please don't use it for anything else, as
+-- it might go away in the future.
+CREATE OR REPLACE FUNCTION maptile_for_point(scaled_lat int8, scaled_lon int8, zoom int4)
+ RETURNS int4
+ AS $$
+DECLARE
+ lat CONSTANT DOUBLE PRECISION := scaled_lat / 10000000.0;
+ lon CONSTANT DOUBLE PRECISION := scaled_lon / 10000000.0;
+ zscale CONSTANT DOUBLE PRECISION := 2.0 ^ zoom;
+ pi CONSTANT DOUBLE PRECISION := 3.141592653589793;
+ r_per_d CONSTANT DOUBLE PRECISION := pi / 180.0;
+ x int4;
+ y int4;
+BEGIN
+ -- straight port of the C code. see db/functions/maptile.c
+ x := floor((lon + 180.0) * zscale / 360.0);
+ y := floor((1.0 - ln(tan(lat * r_per_d) + 1.0 / cos(lat * r_per_d)) / pi) * zscale / 2.0);
+
+ RETURN (x << zoom) | y;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
SET statement_timeout = 0;
SET lock_timeout = 0;
-SET idle_in_transaction_session_timeout = 0;
SET client_encoding = 'UTF8';
SET standard_conforming_strings = on;
SELECT pg_catalog.set_config('search_path', '', false);
-- Name: maptile_for_point(bigint, bigint, integer); Type: FUNCTION; Schema: public; Owner: -
--
-CREATE FUNCTION public.maptile_for_point(bigint, bigint, integer) RETURNS integer
- LANGUAGE c STRICT
- AS '$libdir/libpgosm.so', 'maptile_for_point';
+CREATE FUNCTION public.maptile_for_point(scaled_lat bigint, scaled_lon bigint, zoom integer) RETURNS integer
+ LANGUAGE plpgsql IMMUTABLE
+ AS $$
+DECLARE
+ lat CONSTANT DOUBLE PRECISION := scaled_lat / 10000000.0;
+ lon CONSTANT DOUBLE PRECISION := scaled_lon / 10000000.0;
+ zscale CONSTANT DOUBLE PRECISION := 2.0 ^ zoom;
+ pi CONSTANT DOUBLE PRECISION := 3.141592653589793;
+ r_per_d CONSTANT DOUBLE PRECISION := pi / 180.0;
+ x int4;
+ y int4;
+BEGIN
+ -- straight port of the C code. see db/functions/maptile.c
+ x := floor((lon + 180.0) * zscale / 360.0);
+ y := floor((1.0 - ln(tan(lat * r_per_d) + 1.0 / cos(lat * r_per_d)) / pi) * zscale / 2.0);
-
---
--- Name: tile_for_point(integer, integer); Type: FUNCTION; Schema: public; Owner: -
---
-
-CREATE FUNCTION public.tile_for_point(integer, integer) RETURNS bigint
- LANGUAGE c STRICT
- AS '$libdir/libpgosm.so', 'tile_for_point';
+ RETURN (x << zoom) | y;
+END;
+$$;
--
--- Name: xid_to_int4(xid); Type: FUNCTION; Schema: public; Owner: -
+-- Name: tile_for_point(integer, integer); Type: FUNCTION; Schema: public; Owner: -
--
-CREATE FUNCTION public.xid_to_int4(xid) RETURNS integer
- LANGUAGE c IMMUTABLE STRICT
- AS '$libdir/libpgosm.so', 'xid_to_int4';
+CREATE FUNCTION public.tile_for_point(scaled_lat integer, scaled_lon integer) RETURNS bigint
+ LANGUAGE plpgsql IMMUTABLE
+ AS $$
+DECLARE
+ x int8; -- quantized x from lon,
+ y int8; -- quantized y from lat,
+BEGIN
+ x := round(((scaled_lon / 10000000.0) + 180.0) * 65535.0 / 360.0);
+ y := round(((scaled_lat / 10000000.0) + 90.0) * 65535.0 / 180.0);
+
+ -- these bit-masks are special numbers used in the bit interleaving algorithm.
+ -- see https://graphics.stanford.edu/~seander/bithacks.html#InterleaveBMN
+ -- for the original algorithm and more details.
+ x := (x | (x << 8)) & 16711935; -- 0x00FF00FF
+ x := (x | (x << 4)) & 252645135; -- 0x0F0F0F0F
+ x := (x | (x << 2)) & 858993459; -- 0x33333333
+ x := (x | (x << 1)) & 1431655765; -- 0x55555555
+
+ y := (y | (y << 8)) & 16711935; -- 0x00FF00FF
+ y := (y | (y << 4)) & 252645135; -- 0x0F0F0F0F
+ y := (y | (y << 2)) & 858993459; -- 0x33333333
+ y := (y | (y << 1)) & 1431655765; -- 0x55555555
+
+ RETURN (x << 1) | y;
+END;
+$$;
SET default_tablespace = '';
apt-get install -y ruby2.5 libruby2.5 ruby2.5-dev \
libmagickwand-dev libxml2-dev libxslt1-dev nodejs \
apache2 apache2-dev build-essential git-core phantomjs \
- postgresql postgresql-contrib libpq-dev postgresql-server-dev-all \
+ postgresql postgresql-contrib libpq-dev \
libsasl2-dev imagemagick libffi-dev libgd-dev libarchive-dev libbz2-dev
gem2.5 install rake
gem2.5 install --version "~> 1.16.2" bundler
sudo -u vagrant psql -c "create extension btree_gist" openstreetmap
sudo -u vagrant psql -c "create extension btree_gist" osm_test
fi
-# build and set up postgres extensions
-pushd db/functions
-sudo -u vagrant make
-sudo -u vagrant psql openstreetmap -c "CREATE OR REPLACE FUNCTION maptile_for_point(int8, int8, int4) RETURNS int4 AS '/srv/openstreetmap-website/db/functions/libpgosm.so', 'maptile_for_point' LANGUAGE C STRICT"
-sudo -u vagrant psql openstreetmap -c "CREATE OR REPLACE FUNCTION tile_for_point(int4, int4) RETURNS int8 AS '/srv/openstreetmap-website/db/functions/libpgosm.so', 'tile_for_point' LANGUAGE C STRICT"
-sudo -u vagrant psql openstreetmap -c "CREATE OR REPLACE FUNCTION xid_to_int4(xid) RETURNS int4 AS '/srv/openstreetmap-website/db/functions/libpgosm.so', 'xid_to_int4' LANGUAGE C STRICT"
-popd
+
+
+# install PostgreSQL functions
+sudo -u vagrant psql -d openstreetmap -f db/functions/functions.sql
+################################################################################
+# *IF* you want a vagrant image which supports replication (or perhaps you're
+# using this script to provision some other server and want replication), then
+# uncomment the following lines (until popd) and comment out the one above
+# (functions.sql).
+################################################################################
+#pushd db/functions
+#sudo -u vagrant make
+#sudo -u vagrant psql openstreetmap -c "CREATE OR REPLACE FUNCTION maptile_for_point(int8, int8, int4) RETURNS int4 AS '/srv/openstreetmap-website/db/functions/libpgosm.so', 'maptile_for_point' LANGUAGE C ST#RICT"
+#sudo -u vagrant psql openstreetmap -c "CREATE OR REPLACE FUNCTION tile_for_point(int4, int4) RETURNS int8 AS '/srv/openstreetmap-website/db/functions/libpgosm.so', 'tile_for_point' LANGUAGE C STRICT"
+#sudo -u vagrant psql openstreetmap -c "CREATE OR REPLACE FUNCTION xid_to_int4(xid) RETURNS int4 AS '/srv/openstreetmap-website/db/functions/libpgosm.so', 'xid_to_int4' LANGUAGE C STRICT"
+#popd
+
+
# set up sample configs
if [ ! -f config/database.yml ]; then
sudo -u vagrant cp config/example.database.yml config/database.yml
fi
touch config/settings.local.yml
# migrate the database to the latest version
-sudo -u vagrant rake db:migrate
+sudo -u vagrant bundle exec rake db:migrate
popd
projects / rails.git / commitdiff