cookbooks/planet/recipes/replication.rb

   1 #
   2 # Cookbook:: planet
   3 # Recipe:: dump
   4 #
   5 # Copyright:: 2013, OpenStreetMap Foundation
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #     https://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19
  20 require "yaml"
  21
  22 include_recipe "accounts"
  23 include_recipe "osmosis"
  24
  25 db_passwords = data_bag_item("db", "passwords")
  26
  27 package %w[
  28   postgresql-client
  29   ruby
  30   ruby-dev
  31   ruby-libxml
  32   make
  33   gcc
  34   libpq-dev
  35   osmdbt
  36 ]
  37
  38 gem_package "pg"
  39
  40 remote_directory "/opt/flush" do
  41   source "flush"
  42   owner "root"
  43   group "root"
  44   mode "755"
  45   files_owner "root"
  46   files_group "root"
  47   files_mode "755"
  48 end
  49
  50 execute "/opt/flush/Makefile" do
  51   action :nothing
  52   command "make"
  53   cwd "/opt/flush"
  54   user "root"
  55   group "root"
  56   subscribes :run, "remote_directory[/opt/flush]"
  57 end
  58
  59 remote_directory "/usr/local/bin" do
  60   source "replication-bin"
  61   owner "root"
  62   group "root"
  63   mode "755"
  64   files_owner "root"
  65   files_group "root"
  66   files_mode "755"
  67 end
  68
  69 template "/usr/local/bin/replicate-minute" do
  70   source "replicate-minute.erb"
  71   owner "root"
  72   group "root"
  73   mode "755"
  74 end
  75
  76 template "/usr/local/bin/users-agreed" do
  77   source "users-agreed.erb"
  78   owner "root"
  79   group "root"
  80   mode "755"
  81 end
  82
  83 template "/usr/local/bin/users-deleted" do
  84   source "users-deleted.erb"
  85   owner "root"
  86   group "root"
  87   mode "755"
  88 end
  89
  90 remote_directory "/store/planet/users_deleted" do
  91   source "users_deleted"
  92   owner "planet"
  93   group "planet"
  94   mode "755"
  95   files_owner "root"
  96   files_group "root"
  97   files_mode "644"
  98 end
  99
 100 remote_directory "/store/planet/replication" do
 101   source "replication-cgi"
 102   owner "root"
 103   group "root"
 104   mode "755"
 105   files_owner "root"
 106   files_group "root"
 107   files_mode "755"
 108 end
 109
 110 directory "/store/planet/replication/changesets" do
 111   owner "planet"
 112   group "planet"
 113   mode "755"
 114 end
 115
 116 directory "/store/planet/replication/day" do
 117   owner "planet"
 118   group "planet"
 119   mode "755"
 120 end
 121
 122 directory "/store/planet/replication/hour" do
 123   owner "planet"
 124   group "planet"
 125   mode "755"
 126 end
 127
 128 directory "/store/planet/replication/minute" do
 129   owner "planet"
 130   group "planet"
 131   mode "755"
 132 end
 133
 134 directory "/store/planet/replication/test" do
 135   owner "planet"
 136   group "planet"
 137   mode "755"
 138 end
 139
 140 directory "/store/planet/replication/test/minute" do
 141   owner "planet"
 142   group "planet"
 143   mode "755"
 144 end
 145
 146 directory "/store/replication" do
 147   owner "planet"
 148   group "planet"
 149   mode "755"
 150 end
 151
 152 directory "/store/replication/minute" do
 153   owner "planet"
 154   group "planet"
 155   mode "755"
 156 end
 157
 158 systemd_tmpfile "/run/replication" do
 159   type "d"
 160   owner "planet"
 161   group "planet"
 162   mode "755"
 163 end
 164
 165 directory "/etc/replication" do
 166   owner "root"
 167   group "root"
 168   mode "755"
 169 end
 170
 171 directory "/var/run/lock/changeset-replication/" do
 172   owner "planet"
 173   group "planet"
 174   mode "750"
 175 end
 176
 177 template "/etc/replication/auth.conf" do
 178   source "replication.auth.erb"
 179   user "root"
 180   group "planet"
 181   mode "640"
 182   variables :password => db_passwords["planetdiff"]
 183 end
 184
 185 osmdbt_config = {
 186   "database" => {
 187     "host" => node[:web][:database_host],
 188     "dbname" => "openstreetmap",
 189     "user" => "planetdiff",
 190     "password" => db_passwords["planetdiff"],
 191     "replication_slot" => "osmdbt"
 192   },
 193   "log_dir" => "/var/lib/replication/minute",
 194   "changes_dir" => "/store/planet/replication/test/minute",
 195   "tmp_dir" => "/store/replication/minute",
 196   "run_dir" => "/run/replication"
 197 }
 198
 199 file "/etc/replication/osmdbt-config.yaml" do
 200   user "root"
 201   group "planet"
 202   mode "640"
 203   content YAML.dump(osmdbt_config)
 204 end
 205
 206 systemd_service "replication-minutely" do
 207   description "Minutely replication"
 208   user "planet"
 209   working_directory "/etc/replication"
 210   exec_start "/usr/local/bin/replicate-minute"
 211   private_tmp true
 212   private_devices true
 213   protect_system "full"
 214   protect_home true
 215   restrict_address_families %w[AF_INET AF_INET6]
 216   no_new_privileges true
 217 end
 218
 219 template "/etc/replication/changesets.conf" do
 220   source "changesets.conf.erb"
 221   user "root"
 222   group "planet"
 223   mode "640"
 224   variables :password => db_passwords["planetdiff"]
 225 end
 226
 227 template "/etc/replication/users-agreed.conf" do
 228   source "users-agreed.conf.erb"
 229   user "planet"
 230   group "planet"
 231   mode "600"
 232   variables :password => db_passwords["planetdiff"]
 233 end
 234
 235 directory "/var/lib/replication" do
 236   owner "planet"
 237   group "planet"
 238   mode "755"
 239 end
 240
 241 directory "/var/lib/replication/minute" do
 242   owner "planet"
 243   group "planet"
 244   mode "755"
 245 end
 246
 247 directory "/var/lib/replication/hour" do
 248   owner "planet"
 249   group "planet"
 250   mode "755"
 251 end
 252
 253 template "/var/lib/replication/hour/configuration.txt" do
 254   source "replication.config.erb"
 255   owner "planet"
 256   group "planet"
 257   mode "644"
 258   variables :base => "minute", :interval => 3600
 259 end
 260
 261 link "/var/lib/replication/hour/data" do
 262   to "/store/planet/replication/hour"
 263 end
 264
 265 directory "/var/lib/replication/day" do
 266   owner "planet"
 267   group "planet"
 268   mode "755"
 269 end
 270
 271 template "/var/lib/replication/day/configuration.txt" do
 272   source "replication.config.erb"
 273   owner "planet"
 274   group "planet"
 275   mode "644"
 276   variables :base => "hour", :interval => 86400
 277 end
 278
 279 link "/var/lib/replication/day/data" do
 280   to "/store/planet/replication/day"
 281 end
 282
 283 if node[:planet][:replication] == "enabled"
 284   cron_d "users-agreed" do
 285     minute "0"
 286     hour "7"
 287     user "planet"
 288     command "/usr/local/bin/users-agreed"
 289     mailto "zerebubuth@gmail.com"
 290   end
 291
 292   cron_d "users-deleted" do
 293     minute "0"
 294     hour "17"
 295     user "planet"
 296     command "/usr/local/bin/users-deleted"
 297     mailto "zerebubuth@gmail.com"
 298   end
 299
 300   cron_d "replication-changesets" do
 301     user "planet"
 302     command "/usr/local/bin/replicate-changesets /etc/replication/changesets.conf"
 303     mailto "zerebubuth@gmail.com"
 304   end
 305
 306   systemd_timer "replication-minutely" do
 307     description "Minutely replication"
 308     on_boot_sec 60
 309     on_unit_active_sec 60
 310   end
 311
 312   cron_d "replication-minutely" do
 313     user "planet"
 314     command "/usr/local/bin/osmosis -q --replicate-apidb authFile=/etc/replication/auth.conf validateSchemaVersion=false --write-replication workingDirectory=/store/planet/replication/minute"
 315     mailto "brett@bretth.com"
 316     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 317   end
 318
 319   cron_d "replication-hourly" do
 320     minute "2,7,12,17"
 321     user "planet"
 322     command "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/hour"
 323     mailto "brett@bretth.com"
 324     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 325   end
 326
 327   cron_d "replication-daily" do
 328     minute "5,10,15,20"
 329     user "planet"
 330     command "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/day"
 331     mailto "brett@bretth.com"
 332     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 333   end
 334 else
 335   cron_d "users-agreed" do
 336     action :delete
 337   end
 338
 339   cron_d "users-deleted" do
 340     action :delete
 341   end
 342
 343   cron_d "replication-changesets" do
 344     action :delete
 345   end
 346
 347   cron_d "replication-minutely" do
 348     action :delete
 349   end
 350
 351   cron_d "replication-hourly" do
 352     action :delete
 353   end
 354
 355   cron_d "replication-daily" do
 356     action :delete
 357   end
 358 end