From: Tom Hughes Date: Thu, 26 Jan 2017 18:29:11 +0000 (+0000) Subject: Merge remote-tracking branch 'github/pull/102' X-Git-Url: https://git.openstreetmap.org/chef.git/commitdiff_plain/67c2aceb91955ff583c41807759d203a4478e4b1?hp=4f37c04f6ba42143e99ddf354dd4219563298ae0 Merge remote-tracking branch 'github/pull/102' --- diff --git a/cookbooks/chef/attributes/default.rb b/cookbooks/chef/attributes/default.rb index 137435300..f2b48c94e 100644 --- a/cookbooks/chef/attributes/default.rb +++ b/cookbooks/chef/attributes/default.rb @@ -5,4 +5,4 @@ default[:apt][:sources] = node[:apt][:sources] | ["opscode"] default[:chef][:server][:version] = "12.9.1-1" # Set the default client version -default[:chef][:client][:version] = "12.14.89-1" +default[:chef][:client][:version] = "12.17.44" diff --git a/cookbooks/chef/recipes/default.rb b/cookbooks/chef/recipes/default.rb index c6b5b0102..c06384be7 100644 --- a/cookbooks/chef/recipes/default.rb +++ b/cookbooks/chef/recipes/default.rb @@ -17,7 +17,8 @@ # limitations under the License. # -chef_package = "chef_#{node[:chef][:client][:version]}_amd64.deb" +chef_version = node[:chef][:client][:version] +chef_package = "chef_#{chef_version}-1_amd64.deb" directory "/var/cache/chef" do owner "root" @@ -35,7 +36,7 @@ Dir.glob("/var/cache/chef/chef_*.deb").each do |deb| end remote_file "/var/cache/chef/#{chef_package}" do - source "https://packages.chef.io/stable/ubuntu/12.04/#{chef_package}" + source "https://packages.chef.io/files/stable/chef/#{chef_version}/ubuntu/16.04/#{chef_package}" owner "root" group "root" mode 0o644 @@ -45,7 +46,7 @@ end dpkg_package "chef" do source "/var/cache/chef/#{chef_package}" - version node[:chef][:client][:version] + version "#{chef_version}-1" end directory "/etc/chef" do diff --git a/cookbooks/chef/templates/default/report.rb.erb b/cookbooks/chef/templates/default/report.rb.erb index 0317f9286..6b8ec5b9c 100644 --- a/cookbooks/chef/templates/default/report.rb.erb +++ b/cookbooks/chef/templates/default/report.rb.erb @@ -17,7 +17,7 @@ class Chef if failed? && !exception.is_a?(SystemExit) subject = "Chef run failed on #{node.name}" message = "#{run_status.formatted_exception}\n" - elsif elapsed_time > 300 + elsif elapsed_time > 600 subject = "Chef run took #{elapsed_time} on #{node.name}" message = "" end diff --git a/cookbooks/clamav/recipes/default.rb b/cookbooks/clamav/recipes/default.rb index 08c873ee2..fe92becb5 100644 --- a/cookbooks/clamav/recipes/default.rb +++ b/cookbooks/clamav/recipes/default.rb @@ -19,6 +19,14 @@ package "clamav-daemon" package "clamav-freshclam" +package "clamav-unofficial-sigs" + +template "/etc/clamav-unofficial-sigs.conf.d/50-chef.conf" do + source "clamav-unofficial-sigs.conf.erb" + owner "root" + group "root" + mode 0o644 +end service "clamav-daemon" do action [:enable, :start] diff --git a/cookbooks/clamav/templates/default/clamav-unofficial-sigs.conf.erb b/cookbooks/clamav/templates/default/clamav-unofficial-sigs.conf.erb new file mode 100644 index 000000000..f0c37f28e --- /dev/null +++ b/cookbooks/clamav/templates/default/clamav-unofficial-sigs.conf.erb @@ -0,0 +1,18 @@ +# DO NOT EDIT - This file is being maintained by Chef + +# Add Sanesecurity foxhole databases +ss_dbs=" + ${ss_dbs} + foxhole_all.cdb + foxhole_all.ndb + foxhole_filename.cdb + foxhole_generic.cdb + foxhole_js.cdb + foxhole_js.ndb +" + +# Disable SecuriteInfo databases as the domain no longer exists +si_dbs="" + +# Disable MalwarePatrol databases as they fail checksum validation +mbl_dbs="" diff --git a/cookbooks/dhcpd/templates/default/dhcpd.conf.erb b/cookbooks/dhcpd/templates/default/dhcpd.conf.erb index 37fed4268..6728745ba 100644 --- a/cookbooks/dhcpd/templates/default/dhcpd.conf.erb +++ b/cookbooks/dhcpd/templates/default/dhcpd.conf.erb @@ -164,6 +164,12 @@ host spike-03.oob.openstreetmap.org { fixed-address spike-03.oob.openstreetmap.org; } +host tiamat-22.oob.openstreetmap.org { + hardware ethernet 00:25:90:29:a8:01; + server-name "tiamat-22.oob.openstreetmap.org"; + fixed-address tiamat-22.oob.openstreetmap.org; +} + host thorn-01.oob.openstreetmap.org { hardware ethernet 00:19:bb:35:87:94; server-name "thorn-01.oob.openstreetmap.org"; diff --git a/cookbooks/exim/files/default/noreply/forum b/cookbooks/exim/files/default/noreply/forum new file mode 100644 index 000000000..c2f80aac6 --- /dev/null +++ b/cookbooks/exim/files/default/noreply/forum @@ -0,0 +1,9 @@ +This is an automated response to your email, which was sent to an +unattended address. + +If you are having technical problems with the forums then please +contact support@openstreetmap.org for assistance. + +Thank you, + +OpenStreetMap Administrators diff --git a/cookbooks/exim/templates/default/exim4.conf.erb b/cookbooks/exim/templates/default/exim4.conf.erb index 5766cd2f1..f37dad240 100644 --- a/cookbooks/exim/templates/default/exim4.conf.erb +++ b/cookbooks/exim/templates/default/exim4.conf.erb @@ -503,6 +503,12 @@ acl_check_data: message = This message scored $spam_score SpamAssassin points. <% end -%> + # Deny spammy messages with headers of the form: + # X-PHP-Originating-Script: :SendMail.class.php + # X-PHP-Originating-Script: :ExtendedMail.class.php + deny condition = ${if match {$h_X-PHP-Originating-Script:}{^[0-9]+:(Send|Extended)Mail\\.class\\.php\$}} + message = This message failed local spam checks. + # Accept the message. accept diff --git a/cookbooks/foundation/metadata.rb b/cookbooks/foundation/metadata.rb index f048278be..3a2bec765 100644 --- a/cookbooks/foundation/metadata.rb +++ b/cookbooks/foundation/metadata.rb @@ -5,4 +5,6 @@ license "Apache 2.0" description "Installs and configures foundation services" long_description IO.read(File.join(File.dirname(__FILE__), "README.md")) version "1.0.0" +depends "apache" +depends "git" depends "mediawiki" diff --git a/cookbooks/foundation/recipes/owg.rb b/cookbooks/foundation/recipes/owg.rb new file mode 100644 index 000000000..75d332063 --- /dev/null +++ b/cookbooks/foundation/recipes/owg.rb @@ -0,0 +1,62 @@ +# +# Cookbook Name:: foundation +# Recipe:: owg +# +# Copyright 2016, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "apache::ssl" +include_recipe "git" + +package "ruby" +package "ruby-dev" + +gem_package "bundler" + +git "/srv/operations.osmfoundation.org" do + action :sync + repository "git://github.com/openstreetmap/owg-website.git" + user "root" + group "root" + notifies :run, "execute[/srv/operations.osmfoundation.org/Gemfile]" +end + +directory "/srv/operations.osmfoundation.org/_site" do + mode 0o755 + owner "nobody" + group "nogroup" +end + +execute "/srv/operations.osmfoundation.org/Gemfile" do + action :nothing + command "bundle install" + cwd "/srv/operations.osmfoundation.org" + user "root" + group "root" + notifies :run, "execute[/srv/operations.osmfoundation.org]" +end + +execute "/srv/operations.osmfoundation.org" do + action :nothing + command "bundle exec jekyll build --trace --baseurl=https://operations.osmfoundation.org" + cwd "/srv/operations.osmfoundation.org" + user "nobody" + group "nogroup" +end + +apache_site "operations.osmfoundation.org" do + template "apache.owg.erb" + directory "/srv/operations.osmfoundation.org/_site" +end diff --git a/cookbooks/foundation/templates/default/apache.owg.erb b/cookbooks/foundation/templates/default/apache.owg.erb new file mode 100644 index 000000000..1cb0fe44c --- /dev/null +++ b/cookbooks/foundation/templates/default/apache.owg.erb @@ -0,0 +1,27 @@ +# DO NOT EDIT - This file is being maintained by Chef + + + ServerName <%= @name %> + ServerAdmin webmaster@openstreetmap.org + + CustomLog /var/log/apache2/<%= @name %>-access.log combined + ErrorLog /var/log/apache2/<%= @name %>-error.log + + Redirect permanent / https://<%= @name %>/ + + + + ServerName <%= @name %> + ServerAdmin webmaster@openstreetmap.org + + CustomLog /var/log/apache2/<%= @name %>-access.log combined + ErrorLog /var/log/apache2/<%= @name %>-error.log + + DocumentRoot <%= @directory %> + + SSLEngine on + + +> + Require all granted + diff --git a/cookbooks/gps-tile/files/default/html/index.html b/cookbooks/gps-tile/files/default/html/index.html new file mode 100644 index 000000000..f8482d3c4 --- /dev/null +++ b/cookbooks/gps-tile/files/default/html/index.html @@ -0,0 +1,15 @@ + + + + + gps.tile.openstreetmap.org + + + + + + + +
+ + diff --git a/cookbooks/gps-tile/files/default/html/map.css b/cookbooks/gps-tile/files/default/html/map.css new file mode 100644 index 000000000..0a22de636 --- /dev/null +++ b/cookbooks/gps-tile/files/default/html/map.css @@ -0,0 +1,7 @@ +#map { + position: absolute; + top: 0px; + bottom: 0px; + left: 0px; + right: 0px; +} diff --git a/cookbooks/gps-tile/files/default/html/map.js b/cookbooks/gps-tile/files/default/html/map.js new file mode 100644 index 000000000..af3b0d997 --- /dev/null +++ b/cookbooks/gps-tile/files/default/html/map.js @@ -0,0 +1,13 @@ +$(document).ready(function () { + // Create a map + var map = L.map("map"); + + // Add GPS tile layer + L.tileLayer("//gps-{s}.tile.openstreetmap.org/gps-lines/tile/{z}/{x}/{y}.png", { + attribution: "© OpenStreetMap and contributors, under an open license", + maxZoom: 18 + }).addTo(map); + + // SHow the whole world + map.fitWorld(); +}); diff --git a/cookbooks/hardware/recipes/default.rb b/cookbooks/hardware/recipes/default.rb index 5d1757b46..9dd3c0a6a 100644 --- a/cookbooks/hardware/recipes/default.rb +++ b/cookbooks/hardware/recipes/default.rb @@ -78,7 +78,7 @@ when "TYAN Computer Corporation" units << "0" when "Supermicro" case product - when "H8DGU", "X9SCD", "X7DBU", "X7DW3", "X9DR7/E-(J)LN4F", "X9DR3-F", "X9DRW", "SYS-2028U-TN24R4T+" + when "H8DGU", "X9SCD", "X7DBU", "X7DW3", "X9DR7/E-(J)LN4F", "X9DR3-F", "X9DRW", "SYS-2028U-TN24R4T+", "Super Server" units << "1" else units << "0" @@ -201,6 +201,13 @@ if node[:lsb][:release].to_f >= 12.10 end end +# Link Layer Discovery Protocol Daemon +package "lldpd" +service "lldpd" do + action [:start, :enable] + supports :status => true, :restart => true, :reload => true +end + tools_packages = [] status_packages = {} diff --git a/cookbooks/mediawiki/providers/extension.rb b/cookbooks/mediawiki/providers/extension.rb index ca0b6706e..e0eefa245 100644 --- a/cookbooks/mediawiki/providers/extension.rb +++ b/cookbooks/mediawiki/providers/extension.rb @@ -37,7 +37,9 @@ action :create do end else extension_repository = new_resource.repository || default_repository - extension_reference = if new_resource.tag + extension_reference = if new_resource.reference + new_resource.reference + elsif new_resource.tag "refs/tags/#{new_resource.tag}" else "REL#{extension_version}".tr(".", "_") diff --git a/cookbooks/mediawiki/providers/site.rb b/cookbooks/mediawiki/providers/site.rb index 2548e5c74..7c5792aa0 100644 --- a/cookbooks/mediawiki/providers/site.rb +++ b/cookbooks/mediawiki/providers/site.rb @@ -292,12 +292,13 @@ action :create do # MediaWiki Language Extension Bundle # FIXME: should automatically resolve tag - mw_lang_ext_bundle_tag = "2015.10" + mw_lang_ext_bundle_tag = "2016.10" mediawiki_extension "Babel" do site new_resource.name template "mw-ext-Babel.inc.php.erb" - tag mw_lang_ext_bundle_tag + # tag mw_lang_ext_bundle_tag + reference "740782459d34e756a932ea1ceddde432a1295197" update_site false end diff --git a/cookbooks/mediawiki/resources/extension.rb b/cookbooks/mediawiki/resources/extension.rb index b97fb7d33..c567ee507 100644 --- a/cookbooks/mediawiki/resources/extension.rb +++ b/cookbooks/mediawiki/resources/extension.rb @@ -28,6 +28,7 @@ attribute :variables, :kind_of => Hash, :default => {} attribute :version, :kind_of => String attribute :repository, :kind_of => String attribute :tag, :kind_of => String +attribute :reference, :kind_of => String attribute :update_site, :kind_of => [TrueClass, FalseClass], :default => true def after_created diff --git a/cookbooks/munin/files/default/rrddump.sh b/cookbooks/munin/files/default/rrddump.sh new file mode 100644 index 000000000..08942038e --- /dev/null +++ b/cookbooks/munin/files/default/rrddump.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +RRD_DIR=/var/lib/munin/openstreetmap +DIR=`mktemp -d` +DUMP_DIR=/srv/munin.openstreetmap.org/dumps +TARGET_TGZ=`date "+munin-data-%Y-%m-%d.tar.gz"` +KEEP_OLD_COUNT=3 + +function cleanup { + rm -rf "$DIR" +} + +trap cleanup EXIT + +set -e + +cd "$RRD_DIR" +for f in *.rrd; do + rrdtool dump "$f" "$DIR/${f}.xml" + touch -r "$f" "$DIR/${f}.xml" +done + +cd "$DIR" +find -name "*.xml" -print0 | tar zcf "dump.tar.gz" --null -T - + +# if we got here, then the file was created okay so we're okay to delete any +# old files. +find "${DUMP_DIR}" -name "munin-data-*.tar.gz" -print0 | \ + sort -z -r | \ + tail -z -n "+${KEEP_OLD_COUNT}" | \ + xargs --null rm -f + +mv dump.tar.gz "${DUMP_DIR}/${TARGET_TGZ}" diff --git a/cookbooks/munin/recipes/default.rb b/cookbooks/munin/recipes/default.rb index 1b4ae6d75..39085f0e2 100644 --- a/cookbooks/munin/recipes/default.rb +++ b/cookbooks/munin/recipes/default.rb @@ -162,7 +162,9 @@ end node[:network][:interfaces].each do |ifname, ifattr| if ifattr[:encapsulation] == "Ethernet" && ifattr[:state] == "up" - if node[:hardware] && node[:hardware][:network][ifname][:device] =~ /^virtio/ + if node[:hardware] && + node[:hardware][:network] && + node[:hardware][:network][ifname][:device] =~ /^virtio/ munin_plugin_conf "if_#{ifname}" do template "if.erb" variables :ifname => ifname diff --git a/cookbooks/munin/recipes/server.rb b/cookbooks/munin/recipes/server.rb index dcff86860..d4f8cb047 100644 --- a/cookbooks/munin/recipes/server.rb +++ b/cookbooks/munin/recipes/server.rb @@ -49,13 +49,13 @@ clients = search(:node, "recipes:munin\\:\\:default").sort_by { |n| n[:hostname] frontends = search(:node, "recipes:web\\:\\:frontend").reject { |n| Time.now - Time.at(n[:ohai_time]) > expiry_time }.map { |n| n[:hostname] }.sort # ~FC010 backends = search(:node, "recipes:web\\:\\:backend").reject { |n| Time.now - Time.at(n[:ohai_time]) > expiry_time }.map { |n| n[:hostname] }.sort # ~FC010 tilecaches = search(:node, "roles:tilecache").reject { |n| Time.now - Time.at(n[:ohai_time]) > expiry_time }.sort_by { |n| n[:hostname] }.map do |n| - { :name => n[:hostname], :interface => n.interfaces(:role => :external).first[:interface] } + { :name => n[:hostname], :interface => n.interfaces(:role => :external).first[:interface].tr(".", "_") } end renderers = search(:node, "roles:tile").reject { |n| Time.now - Time.at(n[:ohai_time]) > expiry_time }.sort_by { |n| n[:hostname] }.map do |n| - { :name => n[:hostname], :interface => n.interfaces(:role => :external).first[:interface] } + { :name => n[:hostname], :interface => n.interfaces(:role => :external).first[:interface].tr(".", "_") } end geocoders = search(:node, "roles:nominatim").reject { |n| Time.now - Time.at(n[:ohai_time]) > expiry_time }.sort_by { |n| n[:hostname] }.map do |n| - { :name => n[:hostname], :interface => n.interfaces(:role => :external).first[:interface] } + { :name => n[:hostname], :interface => n.interfaces(:role => :external).first[:interface].tr(".", "_") } end template "/etc/munin/munin.conf" do @@ -81,7 +81,13 @@ remote_directory "/srv/munin.openstreetmap.org" do files_owner "root" files_group "root" files_mode 0o644 - purge true +end + +# directory to put dumped files in +directory "/srv/munin.openstreetmap.org/dumps" do + owner "www-data" + group "www-data" + mode 0o755 end apache_site "munin.openstreetmap.org" do @@ -95,6 +101,21 @@ template "/etc/cron.daily/munin-backup" do mode 0o755 end +# simple shell script to dump RRD data to a file +cookbook_file "/usr/local/bin/rrddump" do + source "rrddump.sh" + owner "root" + group "root" + mode 0o755 +end + +template "/etc/cron.d/rrddump" do + source "rrddump.cron.erb" + owner "root" + group "root" + mode 0o755 +end + munin_plugin "munin_stats" munin_plugin "munin_update" munin_plugin "munin_rrdcached" diff --git a/cookbooks/munin/templates/default/apache.erb b/cookbooks/munin/templates/default/apache.erb index 15c5009e6..e59c33fb0 100644 --- a/cookbooks/munin/templates/default/apache.erb +++ b/cookbooks/munin/templates/default/apache.erb @@ -20,6 +20,7 @@ RewriteEngine on RewriteCond %{REQUEST_URI} !^/static/ + RewriteCond %{REQUEST_URI} !^/dumps/ RewriteRule ^(/.*\.html)?$ /munin-cgi/munin-cgi-html/$1 [PT] @@ -27,6 +28,10 @@ Require all granted + + Options +Indexes + + Require all granted diff --git a/cookbooks/munin/templates/default/rrddump.cron.erb b/cookbooks/munin/templates/default/rrddump.cron.erb new file mode 100644 index 000000000..33087c3fd --- /dev/null +++ b/cookbooks/munin/templates/default/rrddump.cron.erb @@ -0,0 +1,3 @@ +MAILTO=zerebubuth@gmail.com +# do the dump & cleanup in the early hours of the morning +43 3 * * * www-data nice /usr/local/bin/rrddump diff --git a/cookbooks/networking/templates/default/interfaces.erb b/cookbooks/networking/templates/default/interfaces.erb index 4d1f277e5..2344c1f9d 100644 --- a/cookbooks/networking/templates/default/interfaces.erb +++ b/cookbooks/networking/templates/default/interfaces.erb @@ -14,31 +14,34 @@ auto <%= interface[:bond][:slaves].sort.uniq.join(" ") %> <% end -%> iface <%= interface[:interface] %> <%= interface[:family] %> static - address <%= interface[:address] %> +<% if interface[:interface] =~ /\.\d+$/ -%> + vlan-raw-device <%= interface[:interface].split(".").first %> +<% end -%> + address <%= interface[:address] %> <% if interface[:family] == "inet" -%> - netmask <%= interface[:netmask] %> + netmask <%= interface[:netmask] %> <% elsif interface[:family] == "inet6" -%> - netmask <%= interface[:prefix] %> + netmask <%= interface[:prefix] %> <% end -%> <% if interface[:hwaddress] -%> hwaddress <%= interface[:hwaddress] %> <% end -%> <% if interface[:gateway] -%> <% if interface[:network].include?(interface[:gateway]) or IPAddr.new("fe80::/64").include?(interface[:gateway]) -%> - gateway <%= interface[:gateway] %> - metric <%= interface[:metric] %> + gateway <%= interface[:gateway] %> + metric <%= interface[:metric] %> <% else -%> - post-up /sbin/ip -f <%= interface[:family] %> route add <%= interface[:gateway] %> dev <%= interface[:interface] %> - post-up /sbin/ip -f <%= interface[:family] %> route add default metric <%= interface[:metric] %> via <%= interface[:gateway] %> - pre-down /sbin/ip -f <%= interface[:family] %> route del default metric <%= interface[:metric] %> via <%= interface[:gateway] %> - pre-down /sbin/ip -f <%= interface[:family] %> route del <%= interface[:gateway] %> dev <%= interface[:interface] %> + post-up /sbin/ip -f <%= interface[:family] %> route add <%= interface[:gateway] %> dev <%= interface[:interface] %> + post-up /sbin/ip -f <%= interface[:family] %> route add default metric <%= interface[:metric] %> via <%= interface[:gateway] %> + pre-down /sbin/ip -f <%= interface[:family] %> route del default metric <%= interface[:metric] %> via <%= interface[:gateway] %> + pre-down /sbin/ip -f <%= interface[:family] %> route del <%= interface[:gateway] %> dev <%= interface[:interface] %> <% end -%> <% end -%> <% if interface[:mtu] -%> - mtu <%= interface[:mtu] %> + mtu <%= interface[:mtu] %> <% end -%> <% if interface[:family] == "inet6" -%> - autoconf 0 + autoconf 0 <% end -%> <% if interface[:bond] -%> bond-mode <%= interface[:bond][:mode] || "active-backup" %> diff --git a/cookbooks/networking/templates/default/shorewall-zones.erb b/cookbooks/networking/templates/default/shorewall-zones.erb index 345ebd860..d1875b599 100644 --- a/cookbooks/networking/templates/default/shorewall-zones.erb +++ b/cookbooks/networking/templates/default/shorewall-zones.erb @@ -31,3 +31,5 @@ dh:osm <%= @type %> aws:osm <%= @type %> ds:osm <%= @type %> uz:osm <%= @type %> +ovh:osm <%= @type %> +ffr:osm <%= @type %> diff --git a/cookbooks/nginx/recipes/default.rb b/cookbooks/nginx/recipes/default.rb index 0dbc7a730..0c97546bd 100644 --- a/cookbooks/nginx/recipes/default.rb +++ b/cookbooks/nginx/recipes/default.rb @@ -43,7 +43,7 @@ directory "/var/cache/nginx/proxy-cache" do end service "nginx" do - action [:enable, :start] + action [:enable] # Do not start the service as config may be broken from failed chef run supports :status => true, :restart => true, :reload => true subscribes :restart, "template[/etc/nginx/nginx.conf]" end diff --git a/cookbooks/nginx/templates/default/nginx.conf.erb b/cookbooks/nginx/templates/default/nginx.conf.erb index 903afd51c..a895af303 100644 --- a/cookbooks/nginx/templates/default/nginx.conf.erb +++ b/cookbooks/nginx/templates/default/nginx.conf.erb @@ -1,7 +1,8 @@ # DO NOT EDIT - This file is being maintained by Chef user www-data; -worker_processes <%= node['cpu']['total'] %>; +worker_processes auto; +worker_cpu_affinity auto; worker_rlimit_nofile 65536; error_log /var/log/nginx/error.log warn; diff --git a/cookbooks/piwik/attributes/default.rb b/cookbooks/piwik/attributes/default.rb index cb08d5af9..513501c16 100644 --- a/cookbooks/piwik/attributes/default.rb +++ b/cookbooks/piwik/attributes/default.rb @@ -1,4 +1,4 @@ -default[:piwik][:version] = "2.17.1" +default[:piwik][:version] = "3.0.1" default[:piwik][:plugins] = %w( Actions API BulkTracking Contents CoreAdminHome CoreConsole CoreHome CorePluginsAdmin CoreUpdater CoreVisualizations CustomVariables diff --git a/cookbooks/tile/recipes/default.rb b/cookbooks/tile/recipes/default.rb index dd4a7a603..62f9a8858 100644 --- a/cookbooks/tile/recipes/default.rb +++ b/cookbooks/tile/recipes/default.rb @@ -24,6 +24,7 @@ include_recipe "postgresql" include_recipe "tools" blocks = data_bag_item("tile", "blocks") +web_passwords = data_bag_item("web", "passwords") apache_module "alias" apache_module "cgi" @@ -116,6 +117,9 @@ end package "python-cairo" package "python-mapnik" +package "python-setuptools" + +easy_install_package "pyotp" package "fonts-noto-cjk" package "fonts-noto-hinted" @@ -133,7 +137,7 @@ template "/srv/tile.openstreetmap.org/cgi-bin/export" do owner "tile" group "tile" mode 0o755 - variables :blocks => blocks + variables :blocks => blocks, :totp_key => web_passwords["totp_key"] end template "/srv/tile.openstreetmap.org/cgi-bin/debug" do @@ -315,7 +319,7 @@ node[:tile][:styles].each do |name, details| execute "#{style_directory}/project.mml" do action :nothing - command "carto project.mml > project.xml" + command "carto -a 3.0.0 project.mml > project.xml" cwd style_directory user "tile" group "tile" @@ -325,7 +329,10 @@ node[:tile][:styles].each do |name, details| end end +postgresql_version = node[:tile][:database][:cluster].split("/").first + package "postgis" +package "postgresql-#{postgresql_version}-postgis-2.3" postgresql_user "jburgess" do cluster node[:tile][:database][:cluster] diff --git a/cookbooks/tile/templates/default/apache.erb b/cookbooks/tile/templates/default/apache.erb index 4fcbf9b47..c41c923bf 100644 --- a/cookbooks/tile/templates/default/apache.erb +++ b/cookbooks/tile/templates/default/apache.erb @@ -22,7 +22,8 @@ <% end -%> # Setup logging - CustomLog /var/log/apache2/access.log combined + LogFormat "%a %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" combined_with_remoteip + CustomLog /var/log/apache2/access.log combined_with_remoteip ErrorLog /var/log/apache2/error.log BufferedLogs on diff --git a/cookbooks/tile/templates/default/export.erb b/cookbooks/tile/templates/default/export.erb index dfd076252..7d1b8c5bf 100644 --- a/cookbooks/tile/templates/default/export.erb +++ b/cookbooks/tile/templates/default/export.erb @@ -3,13 +3,15 @@ import cairo import cgi +import Cookie import mapnik import os +import pyotp +import resource import shutil +import signal import sys import tempfile -import resource -import signal # Limit maximum CPU time # The Postscript output format can sometimes take hours @@ -51,19 +53,38 @@ def output_error(message, status = "400 Bad Request"): print "" print "" +# Create TOTP token validator +totp = pyotp.TOTP('<%= @totp_key %>', interval = 3600) + # Parse CGI parameters form = cgi.FieldStorage() +# Import cookies +cookies = Cookie.SimpleCookie(os.environ.get('HTTP_COOKIE')) + # Make sure we have a user agent if not os.environ.has_key('HTTP_USER_AGENT'): os.environ['HTTP_USER_AGENT'] = 'NONE' +# Make sure we have a referer +if not os.environ.has_key('HTTP_REFERER'): + os.environ['HTTP_REFERER'] = 'NONE' + +# Look for TOTP token +if cookies.has_key('_osm_totp_token'): + token = cookies['_osm_totp_token'].value +else: + token = None + # Get the load average cputimes = [float(n) for n in open("/proc/stat").readline().rstrip().split()[1:-1]] idletime = cputimes[3] / sum(cputimes) # Process the request -if idletime < 0.2: +if not totp.verify(token, valid_window = 1): + # Abort if the request didn't have a valid TOTP token + output_error("Missing or invalid token") +elif idletime < 0.2: # Abort if the CPU idle time on the machine is too low output_error("The server is too busy at the moment. Please wait a few minutes before trying again.", "503 Service Unavailable") <% @blocks["user_agents"].each do |user_agent| -%> @@ -71,6 +92,11 @@ elif os.environ['HTTP_USER_AGENT'] == '<%= user_agent %>': # Block scraper output_error("The server is too busy at the moment. Please wait a few minutes before trying again.", "503 Service Unavailable") <% end -%> +<% @blocks["referers"].each do |referer| -%> +elif os.environ['HTTP_REFERER'] == '<%= referer %>': + # Block scraper + output_error("The server is too busy at the moment. Please wait a few minutes before trying again.", "503 Service Unavailable") +<% end -%> elif not form.has_key("bbox"): # No bounding box specified output_error("No bounding box specified") diff --git a/cookbooks/tilecache/recipes/default.rb b/cookbooks/tilecache/recipes/default.rb index b97ce12c8..df4e76f43 100644 --- a/cookbooks/tilecache/recipes/default.rb +++ b/cookbooks/tilecache/recipes/default.rb @@ -33,9 +33,14 @@ end package "xz-utils" package "openssl" +# oathtool for QoS token +package "oathtool" + tilecaches = search(:node, "roles:tilecache").sort_by { |n| n[:hostname] } tilerenders = search(:node, "roles:tile").sort_by { |n| n[:hostname] } +web_passwords = data_bag_item("web", "passwords") + tilecaches.each do |cache| cache.ipaddresses(:family => :inet, :role => :external).sort.each do |address| firewall_rule "accept-squid" do @@ -88,6 +93,27 @@ resolvers = node[:networking][:nameservers].map do |resolver| IPAddr.new(resolver).ipv6? ? "[#{resolver}]" : resolver end +template "/usr/local/bin/nginx_generate_tilecache_qos_map" do + source "nginx_generate_tilecache_qos_map.erb" + owner "root" + group "root" + mode 0o750 + variables :totp_key => web_passwords["totp_key"] +end + +template "/etc/cron.d/tilecache" do + source "cron.erb" + owner "root" + group "root" + mode 0o644 +end + +execute "execute_nginx_generate_tilecache_qos_map" do + command "/usr/local/bin/nginx_generate_tilecache_qos_map" + creates "/etc/nginx/conf.d/tile_qos_rates.map" + action :run +end + nginx_site "tile-ssl" do template "nginx_tile_ssl.conf.erb" variables :certificate => certificate, :resolvers => resolvers, :caches => tilecaches diff --git a/cookbooks/tilecache/templates/default/cron.erb b/cookbooks/tilecache/templates/default/cron.erb new file mode 100644 index 000000000..32fb06762 --- /dev/null +++ b/cookbooks/tilecache/templates/default/cron.erb @@ -0,0 +1,3 @@ +# DO NOT EDIT - This file is being maintained by Chef + +0 * * * * root /usr/local/bin/nginx_generate_tilecache_qos_map diff --git a/cookbooks/tilecache/templates/default/nginx_generate_tilecache_qos_map.erb b/cookbooks/tilecache/templates/default/nginx_generate_tilecache_qos_map.erb new file mode 100755 index 000000000..9c6816071 --- /dev/null +++ b/cookbooks/tilecache/templates/default/nginx_generate_tilecache_qos_map.erb @@ -0,0 +1,50 @@ +#!/bin/bash +# DO NOT EDIT - This file is being maintained by Chef +set -e + +NUM_TOKENS=4 # current + 4 +VALID_TOKEN=3600 # in seconds + +SECONDS_AGO=$((${NUM_TOKENS} * ${VALID_TOKEN})) +OLD_TIME=$(/bin/date -u "+%Y-%m-%dT %H:%M:%S %z" -d "${SECONDS_AGO} seconds ago") +QOS_TOKENS=($(/usr/bin/oathtool --totp --now="${OLD_TIME}" --window=${NUM_TOKENS} --time-step-size=${VALID_TOKEN}s -b "<%= @totp_key %>")) + +# ${qos_tokens[4]/[-1] } = OSM.org exclusive / current +# ${qos_tokens[3]/[-2] } = OSM.org exclusive / stale +# ${qos_tokens[2]/[-3] } = tile.openstreetmap.org default +# ${qos_tokens[1]/[-4] } = stale ~ 1 hour +# ${qos_tokens[0]} = expired + +# Test if number of tokens returned by oathtool is expected number +if [ "${#QOS_TOKENS[@]}" -ne "$((${NUM_TOKENS}+1))" ]; then + >&2 echo "ERROR: Unexpected number of tokens" + exit 1 +fi + +QOS_TOKEN_OSM=${QOS_TOKENS[-1]} # Cookie set by openstreetmap.org +QOS_TOKEN_OSM_STALE=${QOS_TOKENS[-2]} # Cookie set by openstreetmap.org stale +QOS_TOKEN_DEFAULT=${QOS_TOKENS[-3]} # Cookie presented by tile.openstreetmap.org to browsers +QOS_TOKEN_STALE=${QOS_TOKENS[-4]} # Cookie which has become stale and will be replaced + +if [ -z "$QOS_TOKEN_OSM" -o -z "$QOS_TOKEN_DEFAULT" -o -z "$QOS_TOKEN_STALE" ]; then + >&2 echo "ERROR: Unexpected blank token" + exit 2 +fi + +cat </etc/nginx/conf.d/tile_qos_rates.map +default 24576; # Default Rate (No QoS cookie) +"${QOS_TOKEN_STALE}" 24576; # Stale +"${QOS_TOKEN_DEFAULT}" 24576; # Default +"${QOS_TOKEN_OSM_STALE}" 32768; # Exclusive Stale +"${QOS_TOKEN_OSM}" 32768; # Exclusive +EOF + +cat </etc/nginx/conf.d/tile_qos_cookies.map +default 'qos_token=${QOS_TOKEN_DEFAULT}; Max-Age=${VALID_TOKEN}; Domain=openstreetmap.org; Path=/'; # Cookie Domain per RFC6265 +"${QOS_TOKEN_DEFAULT}" ''; # Do not Set-Cookie. # Default +"${QOS_TOKEN_OSM_STALE}" ''; # Do not Set-Cookie. # Exclusive Stale +"${QOS_TOKEN_OSM}" ''; # Do not Set-Cookie. # Exclusive +EOF + +# Check config, reload config and fail safe +/etc/init.d/nginx configtest 2>/dev/null && /bin/systemctl try-reload-or-restart nginx diff --git a/cookbooks/tilecache/templates/default/nginx_tile_ssl.conf.erb b/cookbooks/tilecache/templates/default/nginx_tile_ssl.conf.erb index 0ec51a10e..60d7d451f 100644 --- a/cookbooks/tilecache/templates/default/nginx_tile_ssl.conf.erb +++ b/cookbooks/tilecache/templates/default/nginx_tile_ssl.conf.erb @@ -1,3 +1,5 @@ +# DO NOT EDIT - This file is being maintained by Chef + upstream tile_cache_backend { server 127.0.0.1; <% @caches.each do |cache| -%> @@ -12,8 +14,38 @@ upstream tile_cache_backend { keepalive 32; } +# Rates table based on current cookie value +map $cookie_qos_token $limit_rate_qos { + include /etc/nginx/conf.d/tile_qos_rates.map; +} + +# Set-Cookie table based on current cookie value +map $cookie_qos_token $cookie_qos_token_set { + include /etc/nginx/conf.d/tile_qos_cookies.map; +} + +map $http_user_agent $approved_scraper { + default ''; # Not approved + '~^JOSM\/' 'JOSM'; + '~^Mozilla\/5\.0\ QGIS\/' 'QGIS'; +} + +# Limit Cache-Control header to only approved User-Agents +map $http_user_agent $limit_http_cache_control { + default ''; # Unset Header + '~^Mozilla\/5\.0\ QGIS\/' ''; # Unset Header + '~^Mozilla\/5\.0\ ' $http_cache_control; # Pass Header +} + +# Limit Pragma header to only approved User-Agents +map $http_user_agent $limit_http_pragma { + default ''; # Unset Header + '~^Mozilla\/5\.0\ QGIS\/' ''; # Unset Header + '~^Mozilla\/5\.0\ ' $http_pragma; # Pass Header +} + server { - listen 443 ssl http2 default_server; + listen 443 ssl fastopen=2048 http2 default_server; server_name localhost; proxy_buffers 8 64k; @@ -29,14 +61,41 @@ server { ssl_stapling on; ssl_dhparam /etc/ssl/certs/dhparam.pem; resolver <%= @resolvers.join(" ") %>; + resolver_timeout 5s; location / { proxy_pass http://tile_cache_backend; proxy_set_header X-Forwarded-For $remote_addr; proxy_http_version 1.1; - proxy_set_header Connection ""; + proxy_set_header Connection ''; + + proxy_connect_timeout 5s; + + # Do not pass cookies to backends. + proxy_set_header Cookie ''; + # Do not pass Accept-Encoding to backends. + proxy_set_header Accept-Encoding ''; + + # Do not allow setting cookies from backends due to caching. + proxy_ignore_headers Set-Cookie; + proxy_hide_header Set-Cookie; + + # Set a QoS cookie if none presented (uses nginx Map) + add_header Set-Cookie $cookie_qos_token_set; + + # QoS Traffic Rate see $limit_rate on http://nginx.org/en/docs/http/ngx_http_core_module.html + set $limit_rate $limit_rate_qos; + + # Allow Higher Traffic Rate from Approved User-Agents which do not support cookies (uses nginx Map) + if ($approved_scraper) { + set $limit_rate 32768; + } + + # Strip any ?query parameters from urls + set $args ''; - # Slow traffic slightly - limit_rate 24576; + # Allow cache purging headers only from select User-Agents (uses nginx Map) + proxy_set_header Cache-Control $limit_http_cache_control; + proxy_set_header Pragma $limit_http_pragma; } } diff --git a/cookbooks/tilecache/templates/default/squid.conf.erb b/cookbooks/tilecache/templates/default/squid.conf.erb index 9b41d81d0..b6ca8e419 100644 --- a/cookbooks/tilecache/templates/default/squid.conf.erb +++ b/cookbooks/tilecache/templates/default/squid.conf.erb @@ -5,6 +5,7 @@ acl osmtileScrapers browser ^$ acl osmtileScrapers browser ^MOBAC acl osmtileScrapers browser ^JTileDownloader acl osmtileScrapers browser ^Apache\-HttpClient +acl osmtileScrapers browser ^Go-http-client\/ acl osmtileScrapers browser ^Opera\/10\.00 acl osmtileScrapers browser ^shipxy01 #acl osmtileScrapers browser ^OsmAnd #Victor + Email. Whitelist for 1 week @@ -38,7 +39,7 @@ acl osmtileScrapers browser Firefox\/10\.0 acl osmtileScrapers browser Firefox\/11\.0 acl osmtileScrapers browser Firefox\/12\.0 acl osmtileScrapers browser Firefox\/13\.0 - + acl is_fake_browser browser Firefox\/3\.0 acl is_fake_browser browser Firefox\/4\.0 acl is_fake_browser browser Firefox\/5\.0 diff --git a/cookbooks/web/definitions/rails_port.rb b/cookbooks/web/definitions/rails_port.rb index 6773cf974..bbc3d60d9 100644 --- a/cookbooks/web/definitions/rails_port.rb +++ b/cookbooks/web/definitions/rails_port.rb @@ -187,6 +187,11 @@ define :rails_port, :action => [:create, :enable] do line.gsub!(/^( *)#github_auth_secret:.*$/, "\\1github_auth_secret: \"#{params[:github_auth_secret]}\"") end + if params[:wikipedia_auth_id] + line.gsub!(/^( *)#wikipedia_auth_id:.*$/, "\\1wikipedia_auth_id: \"#{params[:wikipedia_auth_id]}\"") + line.gsub!(/^( *)#wikipedia_auth_secret:.*$/, "\\1wikipedia_auth_secret: \"#{params[:wikipedia_auth_secret]}\"") + end + if params[:mapquest_key] line.gsub!(/^( *)#mapquest_key:.*$/, "\\1mapquest_key: \"#{params[:mapquest_key]}\"") end @@ -199,6 +204,10 @@ define :rails_port, :action => [:create, :enable] do line.gsub!(/^( *)#thunderforest_key:.*$/, "\\1thunderforest_key: \"#{params[:thunderforest_key]}\"") end + if params[:totp_key] + line.gsub!(/^( *)#totp_key:.*$/, "\\1totp_key: \"#{params[:totp_key]}\"") + end + line.gsub!(/^( *)require_terms_seen:.*$/, "\\1require_terms_seen: true") line.gsub!(/^( *)require_terms_agreed:.*$/, "\\1require_terms_agreed: true") @@ -269,20 +278,21 @@ define :rails_port, :action => [:create, :enable] do cwd "#{rails_directory}/lib/quad_tile" user rails_user group rails_group - not_if { File.exist?("#{rails_directory}/lib/quad_tile/Makefile") && File.mtime("#{rails_directory}/lib/quad_tile/Makefile") >= File.mtime("#{rails_directory}/lib/quad_tile/extconf.rb") } + not_if do + File.exist?("#{rails_directory}/lib/quad_tile/quad_tile_so.so") && + File.mtime("#{rails_directory}/lib/quad_tile/quad_tile_so.so") >= File.mtime("#{rails_directory}/lib/quad_tile/extconf.rb") && + File.mtime("#{rails_directory}/lib/quad_tile/quad_tile_so.so") >= File.mtime("#{rails_directory}/lib/quad_tile/quad_tile.c") && + File.mtime("#{rails_directory}/lib/quad_tile/quad_tile_so.so") >= File.mtime("#{rails_directory}/lib/quad_tile/quad_tile.h") + end + notifies :run, "execute[#{rails_directory}/lib/quad_tile/Makefile]" end execute "#{rails_directory}/lib/quad_tile/Makefile" do + action :nothing command "make" cwd "#{rails_directory}/lib/quad_tile" user rails_user group rails_group - not_if do - File.exist?("#{rails_directory}/lib/quad_tile/quad_tile_so.so") && - File.mtime("#{rails_directory}/lib/quad_tile/quad_tile_so.so") >= File.mtime("#{rails_directory}/lib/quad_tile/Makefile") && - File.mtime("#{rails_directory}/lib/quad_tile/quad_tile_so.so") >= File.mtime("#{rails_directory}/lib/quad_tile/quad_tile.c") && - File.mtime("#{rails_directory}/lib/quad_tile/quad_tile_so.so") >= File.mtime("#{rails_directory}/lib/quad_tile/quad_tile.h") - end notifies :run, "execute[#{rails_directory}]" end diff --git a/cookbooks/web/recipes/rails.rb b/cookbooks/web/recipes/rails.rb index 7945d3d03..90efbb421 100644 --- a/cookbooks/web/recipes/rails.rb +++ b/cookbooks/web/recipes/rails.rb @@ -76,8 +76,11 @@ rails_port "www.openstreetmap.org" do windowslive_auth_secret web_passwords["windowslive_auth_secret"] github_auth_id "acf7da34edee99e35499" github_auth_secret web_passwords["github_auth_secret"] + wikipedia_auth_id "e4fe0c2c5855d23ed7e1f1c0fa1f1c58" + wikipedia_auth_secret web_passwords["wikipedia_auth_secret"] mapzen_valhalla_key web_passwords["mapzen_valhalla_key"] thunderforest_key web_passwords["thunderforest_key"] + totp_key web_passwords["totp_key"] end package "libjson-xs-perl" diff --git a/cookbooks/web/templates/default/apache.frontend.erb b/cookbooks/web/templates/default/apache.frontend.erb index 526ec8dd7..ceaf4874b 100644 --- a/cookbooks/web/templates/default/apache.frontend.erb +++ b/cookbooks/web/templates/default/apache.frontend.erb @@ -6,7 +6,7 @@ # Basic server configuration # ServerName <%= node[:fqdn] %> - ServerAlias api.openstreetmap.org www.openstreetmap.org + ServerAlias api.openstreetmap.org www.openstreetmap.org 127.0.0.1 ServerAdmin webmaster@openstreetmap.org <% if port == 443 -%> @@ -281,65 +281,32 @@ <% end -%> + + ServerName openstreetmap.org.uk + ServerAlias www.openstreetmap.org.uk + ServerAlias openstreetmap.co.uk + ServerAlias www.openstreetmap.co.uk + + RedirectPermanent /events.ics http://calendar.openstreetmap.org.uk/events.ics + RedirectPermanent / http://www.openstreetmap.org/ + + ServerName openstreetmap.org - ServerAlias maps.openstreetmap.org mapz.openstreetmap.org - ServerAlias openstreetmap.com www.openstreetmap.com - ServerAlias maps.openstreetmap.com mapz.openstreetmap.com - ServerAlias openstreetmap.net www.openstreetmap.net - ServerAlias maps.openstreetmap.net mapz.openstreetmap.net - ServerAlias openstreetmap.ca www.openstreetmap.ca - ServerAlias maps.openstreetmap.ca mapz.openstreetmap.ca - ServerAlias openstreetmap.eu www.openstreetmap.eu - ServerAlias maps.openstreetmap.eu mapz.openstreetmap.eu - ServerAlias openstreetmap.pro www.openstreetmap.pro - ServerAlias maps.openstreetmap.pro mapz.openstreetmap.pro - ServerAlias openstreetmaps.org www.openstreetmaps.org - ServerAlias maps.openstreetmaps.org mapz.openstreetmaps.org - ServerAlias osm.org www.osm.org - ServerAlias maps.osm.org mapz.osm.org - ServerAlias openmaps.org www.openmaps.org - ServerAlias maps.openmaps.org mapz.openmaps.org - ServerAlias openstreetmap.io www.openstreetmap.io - ServerAlias maps.openstreetmap.io mapz.openstreetmap.io - ServerAlias osm.io www.osm.io - ServerAlias maps.osm.io mapz.osm.io - ServerAlias openworldmap.org www.openworldmap.org - ServerAlias maps.openworldmap.org mapz.openworldmap.org - ServerAlias freeosm.org www.freeosm.org - ServerAlias maps.freeosm.org mapz.freeosm.org - ServerAlias open-maps.org www.open-maps.org - ServerAlias maps.open-maps.org mapz.open-maps.org - ServerAlias open-maps.com www.open-maps.com - ServerAlias maps.open-maps.com mapz.open-maps.com - ServerAlias osmbugs.org www.osmbugs.org - ServerAlias maps.osmbugs.org mapz.osmbugs.org - - #Third Party Sites - ServerAlias openstreetmap.pm www.openstreetmap.pm + ServerAlias * RedirectPermanent / http://www.openstreetmap.org/ ServerName openstreetmap.org - ServerAlias maps.openstreetmap.org mapz.openstreetmap.org + ServerAlias * SSLEngine on RedirectPermanent / https://www.openstreetmap.org/ - - ServerName openstreetmap.org.uk - ServerAlias www.openstreetmap.org.uk - ServerAlias openstreetmap.co.uk - ServerAlias www.openstreetmap.co.uk - - RedirectPermanent /events.ics http://calendar.openstreetmap.org.uk/events.ics - RedirectPermanent / http://www.openstreetmap.org/ - - /rails/public> Require all granted diff --git a/roles/ascalon.rb b/roles/ascalon.rb deleted file mode 100644 index cffccad35..000000000 --- a/roles/ascalon.rb +++ /dev/null @@ -1,32 +0,0 @@ -name "ascalon" -description "Master role applied to ascalon" - -default_attributes( - :networking => { - :interfaces => { - :internal_ipv4 => { - :interface => "eth0", - :role => :internal, - :family => :inet, - :address => "10.0.0.18" - }, - :external_ipv4 => { - :interface => "eth1", - :role => :external, - :family => :inet, - :address => "128.40.45.193" - } - } - }, - :accounts => { - :users => { - :emacsen => { :status => :administrator } - } - } -) - -run_list( - "role[ucl-wolfson]", - "role[hp-g5]", - "role[roundup]" -) diff --git a/roles/backup.rb b/roles/backup.rb index 1a9f1911b..5b58145d0 100644 --- a/roles/backup.rb +++ b/roles/backup.rb @@ -19,8 +19,7 @@ default_attributes( :gid => "osmbackup", :transfer_logging => false, :hosts_allow => [ - "128.40.168.0/24", # ucl external (wates) - "128.40.45.192/27", # ucl external (wolfson) + "193.60.236.0/24", # ucl external "146.179.159.160/27", # ic internal "193.63.75.96/27", # ic external "2001:630:12:500::/64", # ic external diff --git a/roles/clifford.rb b/roles/clifford.rb index 700cb6253..247125739 100644 --- a/roles/clifford.rb +++ b/roles/clifford.rb @@ -2,26 +2,35 @@ name "clifford" description "Master role applied to clifford" default_attributes( + :exim => { + :rewrites => [ + { + :pattern => "www-data@openstreetmap.org", + :replacement => "forum@noreply.openstreetmap.org", + :flags => "F" + } + ] + }, :networking => { :interfaces => { :internal_ipv4 => { - :interface => "enp2s0f0", + :interface => "enp2s0f0.2801", :role => :internal, :family => :inet, :address => "10.0.0.17" }, :external_ipv4 => { - :interface => "enp2s0f1", + :interface => "enp2s0f0.2800", :role => :external, :family => :inet, - :address => "128.40.45.194" + :address => "193.60.236.11" } } } ) run_list( - "role[ucl-wolfson]", + "role[ucl]", "role[hp-dl360-g6]", "role[forum]" ) diff --git a/roles/draco.rb b/roles/draco.rb index 9bbaddd5c..dc28adf56 100644 --- a/roles/draco.rb +++ b/roles/draco.rb @@ -5,16 +5,16 @@ default_attributes( :networking => { :interfaces => { :internal_ipv4 => { - :interface => "eth0", + :interface => "eth0.2801", :role => :internal, :family => :inet, :address => "10.0.0.11" }, :external_ipv4 => { - :interface => "eth1", + :interface => "eth0.2800", :role => :external, :family => :inet, - :address => "128.40.45.195" + :address => "193.60.236.12" } } }, @@ -30,6 +30,6 @@ default_attributes( ) run_list( - "role[ucl-wolfson]", + "role[ucl]", "role[hp-g5]" ) diff --git a/roles/errol.rb b/roles/errol.rb index 69b0d8db4..e4db08b80 100644 --- a/roles/errol.rb +++ b/roles/errol.rb @@ -27,23 +27,23 @@ default_attributes( :networking => { :interfaces => { :internal_ipv4 => { - :interface => "eth0", + :interface => "eth0.2801", :role => :internal, :family => :inet, :address => "10.0.0.14" }, :external_ipv4 => { - :interface => "eth1", + :interface => "eth0.2800", :role => :external, :family => :inet, - :address => "128.40.45.196" + :address => "193.60.236.13" } } } ) run_list( - "role[ucl-wolfson]", + "role[ucl]", "role[tyan-s7010]", "role[dev]" ) diff --git a/roles/eustace.rb b/roles/eustace.rb index 0002388c3..4aa2fdd31 100644 --- a/roles/eustace.rb +++ b/roles/eustace.rb @@ -5,23 +5,23 @@ default_attributes( :networking => { :interfaces => { :internal_ipv4 => { - :interface => "eth0", + :interface => "eth0.2801", :role => :internal, :family => :inet, :address => "10.0.0.9" }, :external_ipv4 => { - :interface => "eth1", + :interface => "eth0.2800", :role => :external, :family => :inet, - :address => "128.40.45.197" + :address => "193.60.236.14" } } } ) run_list( - "role[ucl-wolfson]", + "role[ucl]", "role[hp-dl360-g6]", "role[piwik]" ) diff --git a/roles/fafnir.rb b/roles/fafnir.rb deleted file mode 100644 index 099fc38bb..000000000 --- a/roles/fafnir.rb +++ /dev/null @@ -1,70 +0,0 @@ -name "fafnir" -description "Master role applied to fafnir" - -default_attributes( - :db => { - :cluster => "9.1/main" - }, - :networking => { - :interfaces => { - :internal_ipv4 => { - :interface => "eth0", - :role => :internal, - :family => :inet, - :address => "172.31.10.210", - :hwaddress => "02:c1:c5:8b:5f:1d" - }, - :external_ipv4 => { - :role => :external, - :family => :inet, - :address => "52.50.86.69" - } - } - }, - :openvpn => { - :address => "10.0.16.4", - :tunnels => { - :aws2ic => { - :port => "1194", - :mode => "client", - :peer => { - :host => "ironbelly.openstreetmap.org", - :port => "1195" - } - }, - :aws2bm => { - :port => "1195", - :mode => "client", - :peer => { - :host => "grisu.openstreetmap.org", - :port => "1195" - } - } - } - }, - :postgresql => { - :settings => { - :defaults => { - :shared_buffers => "64GB", - :work_mem => "64MB", - :maintenance_work_mem => "1GB", - :effective_cache_size => "180GB" - } - } - }, - :sysctl => { - :postgres => { - :comment => "Increase shared memory for postgres", - :parameters => { - "kernel.shmmax" => 66 * 1024 * 1024 * 1024, - "kernel.shmall" => 66 * 1024 * 1024 * 1024 / 4096 - } - } - } -) - -run_list( - "role[aws]", - "role[db-slave]", - "recipe[openvpn]" -) diff --git a/roles/ffrl.rb b/roles/ffrl.rb new file mode 100644 index 000000000..7858e1b59 --- /dev/null +++ b/roles/ffrl.rb @@ -0,0 +1,28 @@ +name "ffrl" +description "Role applied to all servers at Freifunk Rheinland" + +default_attributes( + :hosted_by => "Freifunk Rheinland", + :location => "Berlin, Germany", + :networking => { + :nameservers => [ + "8.8.8.8", + "8.8.4.4" + ], + :roles => { + :external => { + :zone => "ffr" + } + } + } +) + +override_attributes( + :ntp => { + :servers => ["0.de.pool.ntp.org", "1.de.pool.ntp.org", "europe.pool.ntp.org"] + } +) + +run_list( + "role[de]" +) diff --git a/roles/foundation.rb b/roles/foundation.rb index 4926edeb2..338d1ab4c 100644 --- a/roles/foundation.rb +++ b/roles/foundation.rb @@ -8,7 +8,7 @@ default_attributes( :keepalive => false }, :apt => { - :sources => ["passenger"] + :sources => ["passenger", "brightbox-ruby-ng"] }, :elasticsearch => { :cluster => { @@ -30,5 +30,6 @@ run_list( "role[elasticsearch]", "recipe[foundation::wiki]", "recipe[foundation::board]", - "recipe[foundation::dwg]" + "recipe[foundation::dwg]", + "recipe[foundation::owg]" ) diff --git a/roles/gorynych.rb b/roles/gorynych.rb index 597951828..1d2343350 100644 --- a/roles/gorynych.rb +++ b/roles/gorynych.rb @@ -18,9 +18,9 @@ default_attributes( :interface => "eth1", :role => :external, :family => :inet, - :address => "130.193.62.73", - :prefix => "29", - :gateway => "130.193.62.78" + :address => "5.45.248.21", + :prefix => "30", + :gateway => "5.45.248.22" }, :external_ipv6 => { :interface => "eth1", diff --git a/roles/grindtooth.rb b/roles/grindtooth.rb index d6636294c..f24618d0e 100644 --- a/roles/grindtooth.rb +++ b/roles/grindtooth.rb @@ -5,23 +5,23 @@ default_attributes( :networking => { :interfaces => { :internal_ipv4 => { - :interface => "em1", + :interface => "em1.2801", :role => :internal, :family => :inet, :address => "10.0.0.19" }, :external_ipv4 => { - :interface => "em2", + :interface => "em1.2800", :role => :external, :family => :inet, - :address => "128.40.45.199" + :address => "193.60.236.15" } } } ) run_list( - "role[ucl-wolfson]", + "role[ucl]", "role[hp-dl360-g6]", "role[taginfo]" ) diff --git a/roles/ironbelly.rb b/roles/ironbelly.rb index 4eb5fbc45..3edf96241 100644 --- a/roles/ironbelly.rb +++ b/roles/ironbelly.rb @@ -107,8 +107,7 @@ default_attributes( :gid => "www-data", :transfer_logging => false, :hosts_allow => [ - "128.40.168.0/24", # ucl external (wates) - "128.40.45.192/27", # ucl external (wolfson) + "193.60.236.0/24", # ucl external "146.179.159.160/27", # ic internal "193.63.75.96/27", # ic external "2001:630:12:500::/64", # ic external diff --git a/roles/kalessin.rb b/roles/kalessin.rb new file mode 100644 index 000000000..93106dd42 --- /dev/null +++ b/roles/kalessin.rb @@ -0,0 +1,46 @@ +name "kalessin" +description "Master role applied to kalessin" + +default_attributes( + :networking => { + :interfaces => { + :external_ipv4 => { + :interface => "ens3", + :role => :external, + :family => :inet, + :address => "185.66.195.245", + :prefix => "28", + :gateway => "185.66.195.241" + }, + :external_ipv6 => { + :interface => "ens3", + :role => :external, + :family => :inet6, + :address => "2a03:2260:2000:1::5", + :prefix => "64", + :gateway => "2a03:2260:2000:1::1" + } + } + }, + :squid => { + :cache_mem => "12500 MB", + :cache_dir => "coss /store/squid/coss-01 128000 block-size=8192 max-size=262144 membufs=80" + }, + :tilecache => { + :tile_parent => "germany.render.openstreetmap.org", + :tile_siblings => [ + "konqi.openstreetmap.org", + "trogdor.openstreetmap.org", + "nepomuk.openstreetmap.org", + "ridgeback.openstreetmap.org", + "gorynych.openstreetmap.org", + "simurgh.openstreetmap.org" + ] + } +) + +run_list( + "role[ffrl]", + "role[geodns]", + "role[tilecache]" +) diff --git a/roles/noquiklos.rb b/roles/noquiklos.rb index 036255f4f..f42afa81b 100644 --- a/roles/noquiklos.rb +++ b/roles/noquiklos.rb @@ -5,23 +5,23 @@ default_attributes( :networking => { :interfaces => { :internal_ipv4 => { - :interface => "eth0", + :interface => "eth0.2801", :role => :internal, :family => :inet, :address => "10.0.0.13" }, :external_ipv4 => { - :interface => "eth1", + :interface => "eth0.2800", :role => :external, :family => :inet, - :address => "128.40.45.201" + :address => "193.60.236.16" } } } ) run_list( - "role[ucl-wolfson]", + "role[ucl]", "role[hp-dl360-g6]", "role[gps-tile]" ) diff --git a/roles/norbert.rb b/roles/norbert.rb deleted file mode 100644 index a96812200..000000000 --- a/roles/norbert.rb +++ /dev/null @@ -1,50 +0,0 @@ -name "norbert" -description "Master role applied to norbert" - -default_attributes( - :accounts => { - :users => { - :yellowbkpk => { :status => :administrator }, - :pnorman => { :status => :user } - } - }, - :exim => { - :aliases => { - :root => "yellowbkpk" - } - }, - :networking => { - :interfaces => { - :internal_ipv4 => { - :interface => "eth0", - :role => :internal, - :family => :inet, - :address => "10.0.0.5" - }, - :external_ipv4 => { - :interface => "eth1", - :role => :external, - :family => :inet, - :address => "128.40.45.202" - } - } - }, - :sysfs => { - :hdd_tune => { - :comment => "Tune the queue for improved performance", - :parameters => { - "block/cciss\!c0d0/queue/nr_requests" => "512", - "block/cciss\!c0d1/queue/nr_requests" => "512", - "block/cciss\!c0d0/queue/scheduler" => "noop", - "block/cciss\!c0d1/queue/scheduler" => "noop", - "block/sda/queue/nr_requests" => "512", - "block/sda/queue/scheduler" => "deadline" - } - } - } -) - -run_list( - "role[ucl-wolfson]", - "role[hp-g5]" -) diff --git a/roles/osuosl.rb b/roles/osuosl.rb index 2d7edd7cf..33f18e7ee 100644 --- a/roles/osuosl.rb +++ b/roles/osuosl.rb @@ -17,6 +17,10 @@ default_attributes( :inet => { :prefix => "28", :gateway => "140.211.167.97" + }, + :inet6 => { + :prefix => "64", + :gateway => "2605:bc80:3010:700::1" } } } diff --git a/roles/ovh.rb b/roles/ovh.rb new file mode 100644 index 000000000..506be9c9a --- /dev/null +++ b/roles/ovh.rb @@ -0,0 +1,27 @@ +name "ovh" +description "Role applied to all servers at OVH" + +default_attributes( + :hosted_by => "OVH", + :location => "Roubaix, France", + :networking => { + :nameservers => [ + "213.186.33.99" + ], + :roles => { + :external => { + :zone => "ovh" + } + } + } +) + +override_attributes( + :ntp => { + :servers => ["0.fr.pool.ntp.org", "1.fr.pool.ntp.org", "europe.pool.ntp.org"] + } +) + +run_list( + "role[fr]" +) diff --git a/roles/pummelzacken.rb b/roles/pummelzacken.rb index 9f63dd0c0..f5a573d47 100644 --- a/roles/pummelzacken.rb +++ b/roles/pummelzacken.rb @@ -5,16 +5,16 @@ default_attributes( :networking => { :interfaces => { :internal_ipv4 => { - :interface => "em1", + :interface => "em1.2801", :role => :internal, :family => :inet, :address => "10.0.0.20" }, :external_ipv4 => { - :interface => "em2", + :interface => "em1.2800", :role => :external, :family => :inet, - :address => "128.40.45.204" + :address => "193.60.236.18" } } }, @@ -52,6 +52,6 @@ default_attributes( ) run_list( - "role[ucl-wolfson]", + "role[ucl]", "role[nominatim]" ) diff --git a/roles/ridley.rb b/roles/ridley.rb index d3244ea88..7250ce893 100644 --- a/roles/ridley.rb +++ b/roles/ridley.rb @@ -33,13 +33,13 @@ default_attributes( :networking => { :interfaces => { :external_ipv4 => { - :interface => "eth0", + :interface => "eth0.2800", :role => :external, :family => :inet, - :address => "128.40.45.205" + :address => "193.60.236.19" }, :internal_ipv4 => { - :interface => "eth1", + :interface => "eth0.2801", :role => :internal, :family => :inet, :address => "10.0.0.3" @@ -87,7 +87,7 @@ default_attributes( ) run_list( - "role[ucl-wolfson]", + "role[ucl]", "role[hp-dl360-g6]", "role[gateway]", "role[foundation]", diff --git a/roles/sarel.rb b/roles/sarel.rb index 26f845b90..aab567bd8 100644 --- a/roles/sarel.rb +++ b/roles/sarel.rb @@ -5,23 +5,23 @@ default_attributes( :networking => { :interfaces => { :internal_ipv4 => { - :interface => "eth0", + :interface => "eth0.2801", :role => :internal, :family => :inet, :address => "10.0.0.12" }, :external_ipv4 => { - :interface => "eth1", + :interface => "eth0.2800", :role => :external, :family => :inet, - :address => "128.40.45.206" + :address => "193.60.236.20" } } } ) run_list( - "role[ucl-wolfson]", + "role[ucl]", "role[hp-g5]", "role[yournavigation]" ) diff --git a/roles/scorch.rb b/roles/scorch.rb new file mode 100644 index 000000000..c8f3b0aa0 --- /dev/null +++ b/roles/scorch.rb @@ -0,0 +1,89 @@ +name "scorch" +description "Master role applied to scorch" + +default_attributes( + :apt => { + :sources => ["postgresql"] + }, + :devices => { + :ssd_system => { + :comment => "Tune scheduler for system disk", + :type => "block", + :bus => "scsi", + :serial => "3600605b009bbf5601fc3206407a43546", + :attrs => { + "queue/scheduler" => "noop", + "queue/nr_requests" => "256", + "queue/read_ahead_kb" => "2048" + } + }, + :ssd_database => { + :comment => "Tune scheduler for database disk", + :type => "block", + :bus => "scsi", + :serial => "3600605b009bbf5601fd931c6dfac767f", + :attrs => { + "queue/scheduler" => "noop", + "queue/nr_requests" => "256", + "queue/read_ahead_kb" => "2048" + } + } + }, + :networking => { + :interfaces => { + :external_ipv4 => { + :interface => "eth0", + :role => :external, + :family => :inet, + :address => "176.31.235.79", + :prefix => "24", + :gateway => "176.31.235.254" + }, + :external_ipv6 => { + :interface => "eth0", + :role => :external, + :family => :inet6, + :address => "2001:41d0:2:fc4f::1", + :prefix => "64", + :gateway => "2001:41d0:2:fcff:ff:ff:ff:ff" + } + } + }, + :postgresql => { + :versions => ["9.5"], + :settings => { + :defaults => { + :shared_buffers => "8GB", + :maintenance_work_mem => "7144MB", + :effective_cache_size => "16GB" + } + } + }, + :sysctl => { + :postgres => { + :comment => "Increase shared memory for postgres", + :parameters => { + "kernel.shmmax" => 9 * 1024 * 1024 * 1024, + "kernel.shmall" => 9 * 1024 * 1024 * 1024 / 4096 + } + } + }, + :tile => { + :database => { + :cluster => "9.5/main" + }, + :node_file => "/store/database/nodes", + :styles => { + :default => { + :tile_directories => [ + { :name => "/store/tiles/default", :min_zoom => 0, :max_zoom => 19 } + ] + } + } + } +) + +run_list( + "role[ovh]", + "role[tile]" +) diff --git a/roles/stormfly-01.rb b/roles/stormfly-01.rb index f0492ad50..fc50adc39 100644 --- a/roles/stormfly-01.rb +++ b/roles/stormfly-01.rb @@ -9,6 +9,12 @@ default_attributes( :role => :external, :family => :inet, :address => "140.211.167.104" + }, + :external_ipv6 => { + :interface => "em1", + :role => :external, + :family => :inet6, + :address => "2605:bc80:3010:700::8cde:a768" } } } diff --git a/roles/stormfly-02.rb b/roles/stormfly-02.rb index b9b339262..76c62732a 100644 --- a/roles/stormfly-02.rb +++ b/roles/stormfly-02.rb @@ -9,6 +9,12 @@ default_attributes( :role => :external, :family => :inet, :address => "140.211.167.105" + }, + :external_ipv6 => { + :interface => "em1", + :role => :external, + :family => :inet6, + :address => "2605:bc80:3010:700::8cde:a769" } } }, diff --git a/roles/idris.rb b/roles/tiamat-00.rb similarity index 54% rename from roles/idris.rb rename to roles/tiamat-00.rb index 372aba8cb..5e0f10478 100644 --- a/roles/idris.rb +++ b/roles/tiamat-00.rb @@ -1,26 +1,25 @@ -name "idris" -description "Master role applied to idris" +name "tiamat-00" +description "Master role applied to tiamat-00" default_attributes( :networking => { :interfaces => { :internal_ipv4 => { - :interface => "eth0", + :interface => "enp1s0.2801", :role => :internal, :family => :inet, - :address => "10.0.0.4" + :address => "10.0.0.40" }, :external_ipv4 => { - :interface => "eth1", + :interface => "enp1s0.2800", :role => :external, :family => :inet, - :address => "128.40.45.200" + :address => "193.60.236.40" } } } ) run_list( - "role[ucl-wolfson]", - "role[hp-g5]" + "role[ucl]" ) diff --git a/roles/tiamat-01.rb b/roles/tiamat-01.rb new file mode 100644 index 000000000..2d3aff407 --- /dev/null +++ b/roles/tiamat-01.rb @@ -0,0 +1,25 @@ +name "tiamat-01" +description "Master role applied to tiamat-01" + +default_attributes( + :networking => { + :interfaces => { + :internal_ipv4 => { + :interface => "enp1s0.2801", + :role => :internal, + :family => :inet, + :address => "10.0.0.41" + }, + :external_ipv4 => { + :interface => "enp1s0.2800", + :role => :external, + :family => :inet, + :address => "193.60.236.41" + } + } + } +) + +run_list( + "role[ucl]" +) diff --git a/roles/tiamat-02.rb b/roles/tiamat-02.rb new file mode 100644 index 000000000..d8a2ed0c9 --- /dev/null +++ b/roles/tiamat-02.rb @@ -0,0 +1,25 @@ +name "tiamat-02" +description "Master role applied to tiamat-02" + +default_attributes( + :networking => { + :interfaces => { + :internal_ipv4 => { + :interface => "enp1s0.2801", + :role => :internal, + :family => :inet, + :address => "10.0.0.42" + }, + :external_ipv4 => { + :interface => "enp1s0.2800", + :role => :external, + :family => :inet, + :address => "193.60.236.42" + } + } + } +) + +run_list( + "role[ucl]" +) diff --git a/roles/tiamat-03.rb b/roles/tiamat-03.rb new file mode 100644 index 000000000..d550ad4ee --- /dev/null +++ b/roles/tiamat-03.rb @@ -0,0 +1,25 @@ +name "tiamat-03" +description "Master role applied to tiamat-03" + +default_attributes( + :networking => { + :interfaces => { + :internal_ipv4 => { + :interface => "enp1s0.2801", + :role => :internal, + :family => :inet, + :address => "10.0.0.43" + }, + :external_ipv4 => { + :interface => "enp1s0.2800", + :role => :external, + :family => :inet, + :address => "193.60.236.43" + } + } + } +) + +run_list( + "role[ucl]" +) diff --git a/roles/tiamat-10.rb b/roles/tiamat-10.rb new file mode 100644 index 000000000..2d8b6e557 --- /dev/null +++ b/roles/tiamat-10.rb @@ -0,0 +1,25 @@ +name "tiamat-10" +description "Master role applied to tiamat-10" + +default_attributes( + :networking => { + :interfaces => { + :internal_ipv4 => { + :interface => "enp1s0.2801", + :role => :internal, + :family => :inet, + :address => "10.0.0.44" + }, + :external_ipv4 => { + :interface => "enp1s0.2800", + :role => :external, + :family => :inet, + :address => "193.60.236.44" + } + } + } +) + +run_list( + "role[ucl]" +) diff --git a/roles/tiamat-11.rb b/roles/tiamat-11.rb new file mode 100644 index 000000000..b698d11c1 --- /dev/null +++ b/roles/tiamat-11.rb @@ -0,0 +1,25 @@ +name "tiamat-11" +description "Master role applied to tiamat-11" + +default_attributes( + :networking => { + :interfaces => { + :internal_ipv4 => { + :interface => "enp1s0.2801", + :role => :internal, + :family => :inet, + :address => "10.0.0.45" + }, + :external_ipv4 => { + :interface => "enp1s0.2800", + :role => :external, + :family => :inet, + :address => "193.60.236.45" + } + } + } +) + +run_list( + "role[ucl]" +) diff --git a/roles/tiamat-12.rb b/roles/tiamat-12.rb new file mode 100644 index 000000000..9dc371962 --- /dev/null +++ b/roles/tiamat-12.rb @@ -0,0 +1,25 @@ +name "tiamat-12" +description "Master role applied to tiamat-12" + +default_attributes( + :networking => { + :interfaces => { + :internal_ipv4 => { + :interface => "enp1s0.2801", + :role => :internal, + :family => :inet, + :address => "10.0.0.46" + }, + :external_ipv4 => { + :interface => "enp1s0.2800", + :role => :external, + :family => :inet, + :address => "193.60.236.46" + } + } + } +) + +run_list( + "role[ucl]" +) diff --git a/roles/tiamat-13.rb b/roles/tiamat-13.rb new file mode 100644 index 000000000..90b83797d --- /dev/null +++ b/roles/tiamat-13.rb @@ -0,0 +1,25 @@ +name "tiamat-13" +description "Master role applied to tiamat-13" + +default_attributes( + :networking => { + :interfaces => { + :internal_ipv4 => { + :interface => "enp1s0.2801", + :role => :internal, + :family => :inet, + :address => "10.0.0.47" + }, + :external_ipv4 => { + :interface => "enp1s0.2800", + :role => :external, + :family => :inet, + :address => "193.60.236.47" + } + } + } +) + +run_list( + "role[ucl]" +) diff --git a/roles/tiamat-20.rb b/roles/tiamat-20.rb new file mode 100644 index 000000000..48a33794f --- /dev/null +++ b/roles/tiamat-20.rb @@ -0,0 +1,25 @@ +name "tiamat-20" +description "Master role applied to tiamat-20" + +default_attributes( + :networking => { + :interfaces => { + :internal_ipv4 => { + :interface => "enp1s0.2801", + :role => :internal, + :family => :inet, + :address => "10.0.0.48" + }, + :external_ipv4 => { + :interface => "enp1s0.2800", + :role => :external, + :family => :inet, + :address => "193.60.236.48" + } + } + } +) + +run_list( + "role[ucl]" +) diff --git a/roles/tiamat-21.rb b/roles/tiamat-21.rb new file mode 100644 index 000000000..33badf508 --- /dev/null +++ b/roles/tiamat-21.rb @@ -0,0 +1,25 @@ +name "tiamat-21" +description "Master role applied to tiamat-21" + +default_attributes( + :networking => { + :interfaces => { + :internal_ipv4 => { + :interface => "enp1s0.2801", + :role => :internal, + :family => :inet, + :address => "10.0.0.49" + }, + :external_ipv4 => { + :interface => "enp1s0.2800", + :role => :external, + :family => :inet, + :address => "193.60.236.49" + } + } + } +) + +run_list( + "role[ucl]" +) diff --git a/roles/tiamat-22.rb b/roles/tiamat-22.rb new file mode 100644 index 000000000..f2359438b --- /dev/null +++ b/roles/tiamat-22.rb @@ -0,0 +1,25 @@ +name "tiamat-22" +description "Master role applied to tiamat-22" + +default_attributes( + :networking => { + :interfaces => { + :internal_ipv4 => { + :interface => "enp1s0.2801", + :role => :internal, + :family => :inet, + :address => "10.0.0.50" + }, + :external_ipv4 => { + :interface => "enp1s0.2800", + :role => :external, + :family => :inet, + :address => "193.60.236.50" + } + } + } +) + +run_list( + "role[ucl]" +) diff --git a/roles/tiamat-23.rb b/roles/tiamat-23.rb new file mode 100644 index 000000000..886e5f611 --- /dev/null +++ b/roles/tiamat-23.rb @@ -0,0 +1,25 @@ +name "tiamat-23" +description "Master role applied to tiamat-23" + +default_attributes( + :networking => { + :interfaces => { + :internal_ipv4 => { + :interface => "enp1s0.2801", + :role => :internal, + :family => :inet, + :address => "10.0.0.51" + }, + :external_ipv4 => { + :interface => "enp1s0.2800", + :role => :external, + :family => :inet, + :address => "193.60.236.51" + } + } + } +) + +run_list( + "role[ucl]" +) diff --git a/roles/tile.rb b/roles/tile.rb index 00aa1f82a..e0f940d9a 100644 --- a/roles/tile.rb +++ b/roles/tile.rb @@ -17,9 +17,9 @@ default_attributes( :server_limit => 60, :max_request_workers => 1200, :threads_per_child => 20, - :min_spare_threads => 30, - :max_spare_threads => 540, - :max_connections_per_child => 100000, + :min_spare_threads => 300, + :max_spare_threads => 1200, + :max_connections_per_child => 0, :async_request_worker_factor => 4 } }, @@ -90,7 +90,7 @@ default_attributes( :styles => { :default => { :repository => "git://github.com/gravitystorm/openstreetmap-carto.git", - :revision => "v2.44.1", + :revision => "v3.0.1", :max_zoom => 19 } } diff --git a/roles/tilecache.rb b/roles/tilecache.rb index 438f61992..8e6da21ee 100644 --- a/roles/tilecache.rb +++ b/roles/tilecache.rb @@ -12,6 +12,13 @@ default_attributes( :apt => { :sources => ["nginx"] }, + :munin => { + :plugins => { + :cpu => { + :user => { :warning => 200, :critical => 400 } + } + } + }, :sysctl => { :network_conntrack_time_wait => { :comment => "Only track completed connections for 30 seconds", @@ -25,6 +32,12 @@ default_attributes( "net.netfilter.nf_conntrack_max" => "131072" } }, + :kernel_tfo_listen_enable => { + :comment => "Enable TCP Fast Open for listening sockets", + :parameters => { + "net.ipv4.tcp_fastopen" => 3 + } + }, :squid_swappiness => { :comment => "Prefer not to swapout to free memory", :parameters => { diff --git a/roles/ucl-wates.rb b/roles/ucl-wates.rb deleted file mode 100644 index bba547e0f..000000000 --- a/roles/ucl-wates.rb +++ /dev/null @@ -1,41 +0,0 @@ -name "ucl-wates" -description "Role applied to all servers at UCL which are in Wates House" - -default_attributes( - :networking => { - :roles => { - :internal => { - :inet => { - :prefix => "20", - :gateway => "10.0.0.3" - } - }, - :external => { - :zone => "ucl", - :inet => { - :prefix => "24", - :gateway => "128.40.168.126" - } - } - } - }, - :sysctl => { - :sack => { - :comment => "Disable SACK as the UCL firewall breaks it", - :parameters => { - "net.ipv4.tcp_sack" => "0" - } - } - } -) - -override_attributes( - :networking => { - :nameservers => ["10.0.0.3", "8.8.8.8", "8.8.4.4"], - :search => ["ucl.openstreetmap.org", "openstreetmap.org"] - } -) - -run_list( - "role[ucl]" -) diff --git a/roles/ucl-wolfson.rb b/roles/ucl-wolfson.rb deleted file mode 100644 index 208185a37..000000000 --- a/roles/ucl-wolfson.rb +++ /dev/null @@ -1,33 +0,0 @@ -name "ucl-wolfson" -description "Role applied to all servers at UCL which are in Wolfson House" - -default_attributes( - :networking => { - :roles => { - :internal => { - :inet => { - :prefix => "20", - :gateway => "10.0.0.3" - } - }, - :external => { - :zone => "ucl", - :inet => { - :prefix => "27", - :gateway => "128.40.45.222" - } - } - } - } -) - -override_attributes( - :networking => { - :nameservers => ["10.0.0.3", "8.8.8.8", "8.8.4.4"], - :search => ["ucl.openstreetmap.org", "openstreetmap.org"] - } -) - -run_list( - "role[ucl]" -) diff --git a/roles/ucl.rb b/roles/ucl.rb index 2dd44ee70..27064ea23 100644 --- a/roles/ucl.rb +++ b/roles/ucl.rb @@ -2,13 +2,31 @@ name "ucl" description "Role applied to all servers at UCL" default_attributes( - :bind => { - :forwarders => ["144.82.100.1", "144.82.100.41"] - }, - :location => "London, England" + :location => "Slough, England", + :networking => { + :roles => { + :internal => { + :inet => { + :prefix => "20", + :gateway => "10.0.0.3" + } + }, + :external => { + :zone => "ucl", + :inet => { + :prefix => "24", + :gateway => "193.60.236.254" + } + } + } + } ) override_attributes( + :networking => { + :nameservers => ["10.0.0.3", "8.8.8.8", "8.8.4.4"], + :search => ["ucl.openstreetmap.org", "openstreetmap.org"] + }, :ntp => { :servers => ["ntp1.ucl.ac.uk", "ntp2.ucl.ac.uk"] } diff --git a/roles/urmel.rb b/roles/urmel.rb index ef6899a1c..dba68ec66 100644 --- a/roles/urmel.rb +++ b/roles/urmel.rb @@ -5,23 +5,23 @@ default_attributes( :networking => { :interfaces => { :internal_ipv4 => { - :interface => "eth0", + :interface => "eth0.2801", :role => :internal, :family => :inet, :address => "10.0.0.6" }, :external_ipv4 => { - :interface => "eth1", + :interface => "eth0.2800", :role => :external, :family => :inet, - :address => "128.40.45.207" + :address => "193.60.236.21" } } } ) run_list( - "role[ucl-wolfson]", + "role[ucl]", "role[hp-dl360-g6]", "role[munin]" ) diff --git a/roles/viserion.rb b/roles/viserion.rb index a233e908e..d945ef790 100644 --- a/roles/viserion.rb +++ b/roles/viserion.rb @@ -31,10 +31,10 @@ default_attributes( } }, :nameservers => [ - "161.53.2.66", - "2001:b68:ff:1::2", - "2001:b68:ff:2::2", - "2001:4860:4860::8888" + "8.8.8.8", + "8.8.4.4", + "2001:4860:4860::8888", + "2001:4860:4860::8844" ] }, :squid => { diff --git a/roles/yevaud.rb b/roles/yevaud.rb index 42a29fc10..3d259f674 100644 --- a/roles/yevaud.rb +++ b/roles/yevaud.rb @@ -66,11 +66,17 @@ default_attributes( }, :networking => { :interfaces => { + :internal_ipv4 => { + :interface => "eth0.2801", + :role => :internal, + :family => :inet, + :address => "10.0.0.15" + }, :external_ipv4 => { - :interface => "eth1", + :interface => "eth0.2800", :role => :external, :family => :inet, - :address => "128.40.45.208" + :address => "193.60.236.22" } } }, @@ -110,7 +116,7 @@ default_attributes( ) run_list( - "role[ucl-wolfson]", + "role[ucl]", "role[tyan-s7010]", "role[tile]" ) diff --git a/roles/zark.rb b/roles/zark.rb index abee7f5e7..45a6de4d2 100644 --- a/roles/zark.rb +++ b/roles/zark.rb @@ -5,16 +5,16 @@ default_attributes( :networking => { :interfaces => { :internal_ipv4 => { - :interface => "eth0", + :interface => "eth1.2801", :role => :internal, :family => :inet, :address => "10.0.0.8" }, :external_ipv4 => { - :interface => "eth1", + :interface => "eth1.2800", :role => :external, :family => :inet, - :address => "128.40.45.209" + :address => "193.60.236.23" } } }, @@ -31,6 +31,6 @@ default_attributes( ) run_list( - "role[ucl-wolfson]", + "role[ucl]", "role[owl]" )