cookbooks/planet/recipes/replication.rb

   1 #
   2 # Cookbook:: planet
   3 # Recipe:: dump
   4 #
   5 # Copyright:: 2013, OpenStreetMap Foundation
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #     https://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19
  20 require "yaml"
  21
  22 include_recipe "accounts"
  23 include_recipe "apt"
  24 include_recipe "osmosis"
  25
  26 db_passwords = data_bag_item("db", "passwords")
  27
  28 package %w[
  29   postgresql-client
  30   ruby
  31   ruby-dev
  32   ruby-libxml
  33   make
  34   gcc
  35   libpq-dev
  36   osmdbt
  37 ]
  38
  39 gem_package "pg"
  40
  41 remote_directory "/opt/flush" do
  42   source "flush"
  43   owner "root"
  44   group "root"
  45   mode "755"
  46   files_owner "root"
  47   files_group "root"
  48   files_mode "755"
  49 end
  50
  51 execute "/opt/flush/Makefile" do
  52   action :nothing
  53   command "make"
  54   cwd "/opt/flush"
  55   user "root"
  56   group "root"
  57   subscribes :run, "remote_directory[/opt/flush]"
  58 end
  59
  60 remote_directory "/usr/local/bin" do
  61   source "replication-bin"
  62   owner "root"
  63   group "root"
  64   mode "755"
  65   files_owner "root"
  66   files_group "root"
  67   files_mode "755"
  68 end
  69
  70 template "/usr/local/bin/replicate-minute" do
  71   source "replicate-minute.erb"
  72   owner "root"
  73   group "root"
  74   mode "755"
  75 end
  76
  77 template "/usr/local/bin/users-agreed" do
  78   source "users-agreed.erb"
  79   owner "root"
  80   group "root"
  81   mode "755"
  82 end
  83
  84 template "/usr/local/bin/users-deleted" do
  85   source "users-deleted.erb"
  86   owner "root"
  87   group "root"
  88   mode "755"
  89 end
  90
  91 remote_directory "/store/planet/users_deleted" do
  92   source "users_deleted"
  93   owner "planet"
  94   group "planet"
  95   mode "755"
  96   files_owner "root"
  97   files_group "root"
  98   files_mode "644"
  99 end
 100
 101 remote_directory "/store/planet/replication" do
 102   source "replication-cgi"
 103   owner "root"
 104   group "root"
 105   mode "755"
 106   files_owner "root"
 107   files_group "root"
 108   files_mode "755"
 109 end
 110
 111 directory "/store/planet/replication/changesets" do
 112   owner "planet"
 113   group "planet"
 114   mode "755"
 115 end
 116
 117 directory "/store/planet/replication/day" do
 118   owner "planet"
 119   group "planet"
 120   mode "755"
 121 end
 122
 123 directory "/store/planet/replication/hour" do
 124   owner "planet"
 125   group "planet"
 126   mode "755"
 127 end
 128
 129 directory "/store/planet/replication/minute" do
 130   owner "planet"
 131   group "planet"
 132   mode "755"
 133 end
 134
 135 directory "/store/planet/replication/test" do
 136   owner "planet"
 137   group "planet"
 138   mode "755"
 139 end
 140
 141 directory "/store/planet/replication/test/day" do
 142   owner "planet"
 143   group "planet"
 144   mode "755"
 145 end
 146
 147 directory "/store/planet/replication/test/hour" do
 148   owner "planet"
 149   group "planet"
 150   mode "755"
 151 end
 152
 153 directory "/store/planet/replication/test/minute" do
 154   owner "planet"
 155   group "planet"
 156   mode "755"
 157 end
 158
 159 directory "/store/replication" do
 160   owner "planet"
 161   group "planet"
 162   mode "755"
 163 end
 164
 165 directory "/store/replication/minute" do
 166   owner "planet"
 167   group "planet"
 168   mode "755"
 169 end
 170
 171 systemd_tmpfile "/run/replication" do
 172   type "d"
 173   owner "planet"
 174   group "planet"
 175   mode "755"
 176 end
 177
 178 directory "/etc/replication" do
 179   owner "root"
 180   group "root"
 181   mode "755"
 182 end
 183
 184 directory "/var/run/lock/changeset-replication/" do
 185   owner "planet"
 186   group "planet"
 187   mode "750"
 188 end
 189
 190 directory "/var/lib/replication" do
 191   owner "planet"
 192   group "planet"
 193   mode "755"
 194 end
 195
 196 directory "/var/lib/replication/test" do
 197   owner "planet"
 198   group "planet"
 199   mode "755"
 200 end
 201
 202 template "/etc/replication/auth.conf" do
 203   source "replication.auth.erb"
 204   user "root"
 205   group "planet"
 206   mode "640"
 207   variables :password => db_passwords["planetdiff"]
 208 end
 209
 210 osmdbt_config = {
 211   "database" => {
 212     "host" => node[:web][:database_host],
 213     "dbname" => "openstreetmap",
 214     "user" => "planetdiff",
 215     "password" => db_passwords["planetdiff"],
 216     "replication_slot" => "osmdbt"
 217   },
 218   "log_dir" => "/var/lib/replication/minute",
 219   "changes_dir" => "/store/planet/replication/test/minute",
 220   "tmp_dir" => "/store/replication/minute",
 221   "run_dir" => "/run/replication"
 222 }
 223
 224 file "/etc/replication/osmdbt-config.yaml" do
 225   user "root"
 226   group "planet"
 227   mode "640"
 228   content YAML.dump(osmdbt_config)
 229 end
 230
 231 systemd_service "replication-minutely" do
 232   description "Minutely replication"
 233   user "planet"
 234   working_directory "/etc/replication"
 235   exec_start "/usr/local/bin/replicate-minute"
 236   private_tmp true
 237   private_devices true
 238   protect_system "full"
 239   protect_home true
 240   restrict_address_families %w[AF_INET AF_INET6]
 241   no_new_privileges true
 242 end
 243
 244 systemd_timer "replication-minutely" do
 245   description "Minutely replication"
 246   on_boot_sec 60
 247   on_unit_active_sec 60
 248   accuracy_sec 5
 249 end
 250
 251 directory "/var/lib/replication/test/hour" do
 252   owner "planet"
 253   group "planet"
 254   mode "755"
 255 end
 256
 257 template "/var/lib/replication/test/hour/configuration.txt" do
 258   source "replication.config.erb"
 259   owner "planet"
 260   group "planet"
 261   mode "644"
 262   variables :base => "test/minute", :interval => 3600
 263 end
 264
 265 link "/var/lib/replication/test/hour/data" do
 266   to "/store/planet/replication/test/hour"
 267 end
 268
 269 systemd_service "replication-hourly" do
 270   description "Hourly replication"
 271   user "planet"
 272   exec_start "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/test/hour"
 273   private_tmp true
 274   private_devices true
 275   protect_system "full"
 276   protect_home true
 277   restrict_address_families %w[AF_INET AF_INET6]
 278   no_new_privileges true
 279 end
 280
 281 systemd_timer "replication-hourly" do
 282   description "Daily replication"
 283   on_calendar "*-*-* *:02/15:00"
 284 end
 285
 286 directory "/var/lib/replication/test/day" do
 287   owner "planet"
 288   group "planet"
 289   mode "755"
 290 end
 291
 292 template "/var/lib/replication/test/day/configuration.txt" do
 293   source "replication.config.erb"
 294   owner "planet"
 295   group "planet"
 296   mode "644"
 297   variables :base => "test/hour", :interval => 86400
 298 end
 299
 300 link "/var/lib/replication/test/day/data" do
 301   to "/store/planet/replication/test/day"
 302 end
 303
 304 systemd_service "replication-daily" do
 305   description "Daily replication"
 306   user "planet"
 307   exec_start "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/test/day"
 308   private_tmp true
 309   private_devices true
 310   protect_system "full"
 311   protect_home true
 312   restrict_address_families %w[AF_INET AF_INET6]
 313   no_new_privileges true
 314 end
 315
 316 systemd_timer "replication-daily" do
 317   description "Daily replication"
 318   on_calendar "*-*-* *:02/15:00"
 319 end
 320
 321 template "/etc/replication/changesets.conf" do
 322   source "changesets.conf.erb"
 323   user "root"
 324   group "planet"
 325   mode "640"
 326   variables :password => db_passwords["planetdiff"]
 327 end
 328
 329 template "/etc/replication/users-agreed.conf" do
 330   source "users-agreed.conf.erb"
 331   user "planet"
 332   group "planet"
 333   mode "600"
 334   variables :password => db_passwords["planetdiff"]
 335 end
 336
 337 directory "/var/lib/replication/minute" do
 338   owner "planet"
 339   group "planet"
 340   mode "755"
 341 end
 342
 343 directory "/var/lib/replication/hour" do
 344   owner "planet"
 345   group "planet"
 346   mode "755"
 347 end
 348
 349 template "/var/lib/replication/hour/configuration.txt" do
 350   source "replication.config.erb"
 351   owner "planet"
 352   group "planet"
 353   mode "644"
 354   variables :base => "minute", :interval => 3600
 355 end
 356
 357 link "/var/lib/replication/hour/data" do
 358   to "/store/planet/replication/hour"
 359 end
 360
 361 directory "/var/lib/replication/day" do
 362   owner "planet"
 363   group "planet"
 364   mode "755"
 365 end
 366
 367 template "/var/lib/replication/day/configuration.txt" do
 368   source "replication.config.erb"
 369   owner "planet"
 370   group "planet"
 371   mode "644"
 372   variables :base => "hour", :interval => 86400
 373 end
 374
 375 link "/var/lib/replication/day/data" do
 376   to "/store/planet/replication/day"
 377 end
 378
 379 if node[:planet][:replication] == "enabled"
 380   cron_d "users-agreed" do
 381     minute "0"
 382     hour "7"
 383     user "planet"
 384     command "/usr/local/bin/users-agreed"
 385     mailto "zerebubuth@gmail.com"
 386   end
 387
 388   cron_d "users-deleted" do
 389     minute "0"
 390     hour "17"
 391     user "planet"
 392     command "/usr/local/bin/users-deleted"
 393     mailto "zerebubuth@gmail.com"
 394   end
 395
 396   cron_d "replication-changesets" do
 397     user "planet"
 398     command "/usr/local/bin/replicate-changesets /etc/replication/changesets.conf"
 399     mailto "zerebubuth@gmail.com"
 400   end
 401
 402   service "replication-minutely.timer" do
 403     action [:enable, :start]
 404   end
 405
 406   service "replication-hourly.timer" do
 407     action [:enable, :start]
 408   end
 409
 410   service "replication-daily.timer" do
 411     action [:enable, :start]
 412   end
 413
 414   cron_d "replication-minutely" do
 415     user "planet"
 416     command "/usr/local/bin/osmosis -q --replicate-apidb authFile=/etc/replication/auth.conf validateSchemaVersion=false --write-replication workingDirectory=/store/planet/replication/minute"
 417     mailto "brett@bretth.com"
 418     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 419   end
 420
 421   cron_d "replication-hourly" do
 422     minute "2,7,12,17"
 423     user "planet"
 424     command "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/hour"
 425     mailto "brett@bretth.com"
 426     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 427   end
 428
 429   cron_d "replication-daily" do
 430     minute "5,10,15,20"
 431     user "planet"
 432     command "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/day"
 433     mailto "brett@bretth.com"
 434     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 435   end
 436 else
 437   cron_d "users-agreed" do
 438     action :delete
 439   end
 440
 441   cron_d "users-deleted" do
 442     action :delete
 443   end
 444
 445   cron_d "replication-changesets" do
 446     action :delete
 447   end
 448
 449   service "replication-minutely.timer" do
 450     action [:stop, :disable]
 451   end
 452
 453   service "replication-hourly.timer" do
 454     action [:stop, :disable]
 455   end
 456
 457   service "replication-daily.timer" do
 458     action [:stop, :disable]
 459   end
 460
 461   cron_d "replication-minutely" do
 462     action :delete
 463   end
 464
 465   cron_d "replication-hourly" do
 466     action :delete
 467   end
 468
 469   cron_d "replication-daily" do
 470     action :delete
 471   end
 472 end