cookbooks/planet/recipes/replication.rb

   1 #
   2 # Cookbook:: planet
   3 # Recipe:: dump
   4 #
   5 # Copyright:: 2013, OpenStreetMap Foundation
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #     https://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19
  20 require "yaml"
  21
  22 include_recipe "accounts"
  23 include_recipe "osmosis"
  24
  25 db_passwords = data_bag_item("db", "passwords")
  26
  27 package %w[
  28   postgresql-client
  29   ruby
  30   ruby-dev
  31   ruby-libxml
  32   make
  33   gcc
  34   libpq-dev
  35   osmdbt
  36 ]
  37
  38 gem_package "pg"
  39
  40 remote_directory "/opt/flush" do
  41   source "flush"
  42   owner "root"
  43   group "root"
  44   mode "755"
  45   files_owner "root"
  46   files_group "root"
  47   files_mode "755"
  48 end
  49
  50 execute "/opt/flush/Makefile" do
  51   action :nothing
  52   command "make"
  53   cwd "/opt/flush"
  54   user "root"
  55   group "root"
  56   subscribes :run, "remote_directory[/opt/flush]"
  57 end
  58
  59 remote_directory "/usr/local/bin" do
  60   source "replication-bin"
  61   owner "root"
  62   group "root"
  63   mode "755"
  64   files_owner "root"
  65   files_group "root"
  66   files_mode "755"
  67 end
  68
  69 template "/usr/local/bin/replicate-minute" do
  70   source "replicate-minute.erb"
  71   owner "root"
  72   group "root"
  73   mode "755"
  74 end
  75
  76 template "/usr/local/bin/users-agreed" do
  77   source "users-agreed.erb"
  78   owner "root"
  79   group "root"
  80   mode "755"
  81 end
  82
  83 template "/usr/local/bin/users-deleted" do
  84   source "users-deleted.erb"
  85   owner "root"
  86   group "root"
  87   mode "755"
  88 end
  89
  90 remote_directory "/store/planet/users_deleted" do
  91   source "users_deleted"
  92   owner "planet"
  93   group "planet"
  94   mode "755"
  95   files_owner "root"
  96   files_group "root"
  97   files_mode "644"
  98 end
  99
 100 remote_directory "/store/planet/replication" do
 101   source "replication-cgi"
 102   owner "root"
 103   group "root"
 104   mode "755"
 105   files_owner "root"
 106   files_group "root"
 107   files_mode "755"
 108 end
 109
 110 directory "/store/planet/replication/changesets" do
 111   owner "planet"
 112   group "planet"
 113   mode "755"
 114 end
 115
 116 directory "/store/planet/replication/day" do
 117   owner "planet"
 118   group "planet"
 119   mode "755"
 120 end
 121
 122 directory "/store/planet/replication/hour" do
 123   owner "planet"
 124   group "planet"
 125   mode "755"
 126 end
 127
 128 directory "/store/planet/replication/minute" do
 129   owner "planet"
 130   group "planet"
 131   mode "755"
 132 end
 133
 134 directory "/store/planet/replication/test" do
 135   owner "planet"
 136   group "planet"
 137   mode "755"
 138 end
 139
 140 directory "/store/planet/replication/test/minute" do
 141   owner "planet"
 142   group "planet"
 143   mode "755"
 144 end
 145
 146 directory "/store/replication" do
 147   owner "planet"
 148   group "planet"
 149   mode "755"
 150 end
 151
 152 directory "/store/replication/minute" do
 153   owner "planet"
 154   group "planet"
 155   mode "755"
 156 end
 157
 158 systemd_tmpfile "/run/replication" do
 159   type "d"
 160   owner "planet"
 161   group "planet"
 162   mode "755"
 163 end
 164
 165 directory "/etc/replication" do
 166   owner "root"
 167   group "root"
 168   mode "755"
 169 end
 170
 171 directory "/var/run/lock/changeset-replication/" do
 172   owner "planet"
 173   group "planet"
 174   mode "750"
 175 end
 176
 177 template "/etc/replication/auth.conf" do
 178   source "replication.auth.erb"
 179   user "root"
 180   group "planet"
 181   mode "640"
 182   variables :password => db_passwords["planetdiff"]
 183 end
 184
 185 osmdbt_config = {
 186   "database" => {
 187     "host" => node[:web][:database_host],
 188     "dbname" => "openstreetmap",
 189     "user" => "planetdiff",
 190     "password" => db_passwords["planetdiff"],
 191     "replication_slot" => "osmdbt"
 192   },
 193   "log_dir" => "/var/lib/replication/minute",
 194   "changes_dir" => "/store/planet/replication/test/minute",
 195   "tmp_dir" => "/store/replication/minute",
 196   "run_dir" => "/run/replication"
 197 }
 198
 199 file "/etc/replication/osmdbt-config.yaml" do
 200   user "root"
 201   group "planet"
 202   mode "640"
 203   content YAML.dump(osmdbt_config)
 204 end
 205
 206 systemd_service "replication-minutely" do
 207   description "Minutely replication"
 208   user "planet"
 209   working_directory "/etc/replication"
 210   exec_start "/usr/local/bin/replicate-minute"
 211   private_tmp true
 212   private_devices true
 213   protect_system "full"
 214   protect_home true
 215   restrict_address_families %w[AF_INET AF_INET6]
 216   no_new_privileges true
 217 end
 218
 219 systemd_timer "replication-minutely" do
 220   description "Minutely replication"
 221   on_boot_sec 60
 222   on_unit_active_sec 60
 223   accuracy_sec 5
 224 end
 225
 226 template "/etc/replication/changesets.conf" do
 227   source "changesets.conf.erb"
 228   user "root"
 229   group "planet"
 230   mode "640"
 231   variables :password => db_passwords["planetdiff"]
 232 end
 233
 234 template "/etc/replication/users-agreed.conf" do
 235   source "users-agreed.conf.erb"
 236   user "planet"
 237   group "planet"
 238   mode "600"
 239   variables :password => db_passwords["planetdiff"]
 240 end
 241
 242 directory "/var/lib/replication" do
 243   owner "planet"
 244   group "planet"
 245   mode "755"
 246 end
 247
 248 directory "/var/lib/replication/minute" do
 249   owner "planet"
 250   group "planet"
 251   mode "755"
 252 end
 253
 254 directory "/var/lib/replication/hour" do
 255   owner "planet"
 256   group "planet"
 257   mode "755"
 258 end
 259
 260 template "/var/lib/replication/hour/configuration.txt" do
 261   source "replication.config.erb"
 262   owner "planet"
 263   group "planet"
 264   mode "644"
 265   variables :base => "minute", :interval => 3600
 266 end
 267
 268 link "/var/lib/replication/hour/data" do
 269   to "/store/planet/replication/hour"
 270 end
 271
 272 directory "/var/lib/replication/day" do
 273   owner "planet"
 274   group "planet"
 275   mode "755"
 276 end
 277
 278 template "/var/lib/replication/day/configuration.txt" do
 279   source "replication.config.erb"
 280   owner "planet"
 281   group "planet"
 282   mode "644"
 283   variables :base => "hour", :interval => 86400
 284 end
 285
 286 link "/var/lib/replication/day/data" do
 287   to "/store/planet/replication/day"
 288 end
 289
 290 if node[:planet][:replication] == "enabled"
 291   cron_d "users-agreed" do
 292     minute "0"
 293     hour "7"
 294     user "planet"
 295     command "/usr/local/bin/users-agreed"
 296     mailto "zerebubuth@gmail.com"
 297   end
 298
 299   cron_d "users-deleted" do
 300     minute "0"
 301     hour "17"
 302     user "planet"
 303     command "/usr/local/bin/users-deleted"
 304     mailto "zerebubuth@gmail.com"
 305   end
 306
 307   cron_d "replication-changesets" do
 308     user "planet"
 309     command "/usr/local/bin/replicate-changesets /etc/replication/changesets.conf"
 310     mailto "zerebubuth@gmail.com"
 311   end
 312
 313   service "replication-minutely.timer" do
 314     action [:enable, :start]
 315   end
 316
 317   cron_d "replication-minutely" do
 318     user "planet"
 319     command "/usr/local/bin/osmosis -q --replicate-apidb authFile=/etc/replication/auth.conf validateSchemaVersion=false --write-replication workingDirectory=/store/planet/replication/minute"
 320     mailto "brett@bretth.com"
 321     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 322   end
 323
 324   cron_d "replication-hourly" do
 325     minute "2,7,12,17"
 326     user "planet"
 327     command "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/hour"
 328     mailto "brett@bretth.com"
 329     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 330   end
 331
 332   cron_d "replication-daily" do
 333     minute "5,10,15,20"
 334     user "planet"
 335     command "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/day"
 336     mailto "brett@bretth.com"
 337     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 338   end
 339 else
 340   cron_d "users-agreed" do
 341     action :delete
 342   end
 343
 344   cron_d "users-deleted" do
 345     action :delete
 346   end
 347
 348   cron_d "replication-changesets" do
 349     action :delete
 350   end
 351
 352   service "replication-minutely.timer" do
 353     action [:stop, :disable]
 354   end
 355
 356   cron_d "replication-minutely" do
 357     action :delete
 358   end
 359
 360   cron_d "replication-hourly" do
 361     action :delete
 362   end
 363
 364   cron_d "replication-daily" do
 365     action :delete
 366   end
 367 end