cookbooks/planet/recipes/replication.rb

   1 #
   2 # Cookbook:: planet
   3 # Recipe:: dump
   4 #
   5 # Copyright:: 2013, OpenStreetMap Foundation
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #     https://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19
  20 require "yaml"
  21
  22 include_recipe "accounts"
  23 include_recipe "osmosis"
  24
  25 db_passwords = data_bag_item("db", "passwords")
  26
  27 package %w[
  28   postgresql-client
  29   ruby
  30   ruby-dev
  31   ruby-libxml
  32   make
  33   gcc
  34   libpq-dev
  35   osmdbt
  36 ]
  37
  38 gem_package "pg"
  39
  40 remote_directory "/opt/flush" do
  41   source "flush"
  42   owner "root"
  43   group "root"
  44   mode "755"
  45   files_owner "root"
  46   files_group "root"
  47   files_mode "755"
  48 end
  49
  50 execute "/opt/flush/Makefile" do
  51   action :nothing
  52   command "make"
  53   cwd "/opt/flush"
  54   user "root"
  55   group "root"
  56   subscribes :run, "remote_directory[/opt/flush]"
  57 end
  58
  59 remote_directory "/usr/local/bin" do
  60   source "replication-bin"
  61   owner "root"
  62   group "root"
  63   mode "755"
  64   files_owner "root"
  65   files_group "root"
  66   files_mode "755"
  67 end
  68
  69 template "/usr/local/bin/users-agreed" do
  70   source "users-agreed.erb"
  71   owner "root"
  72   group "root"
  73   mode "755"
  74 end
  75
  76 template "/usr/local/bin/users-deleted" do
  77   source "users-deleted.erb"
  78   owner "root"
  79   group "root"
  80   mode "755"
  81 end
  82
  83 remote_directory "/store/planet/users_deleted" do
  84   source "users_deleted"
  85   owner "planet"
  86   group "planet"
  87   mode "755"
  88   files_owner "root"
  89   files_group "root"
  90   files_mode "644"
  91 end
  92
  93 remote_directory "/store/planet/replication" do
  94   source "replication-cgi"
  95   owner "root"
  96   group "root"
  97   mode "755"
  98   files_owner "root"
  99   files_group "root"
 100   files_mode "755"
 101 end
 102
 103 directory "/store/planet/replication/changesets" do
 104   owner "planet"
 105   group "planet"
 106   mode "755"
 107 end
 108
 109 directory "/store/planet/replication/day" do
 110   owner "planet"
 111   group "planet"
 112   mode "755"
 113 end
 114
 115 directory "/store/planet/replication/hour" do
 116   owner "planet"
 117   group "planet"
 118   mode "755"
 119 end
 120
 121 directory "/store/planet/replication/minute" do
 122   owner "planet"
 123   group "planet"
 124   mode "755"
 125 end
 126
 127 directory "/store/planet/replication/test" do
 128   owner "planet"
 129   group "planet"
 130   mode "755"
 131 end
 132
 133 directory "/store/planet/replication/test/minute" do
 134   owner "planet"
 135   group "planet"
 136   mode "755"
 137 end
 138
 139 systemd_tmpfile "/run/replication" do
 140   type "d"
 141   owner "planet"
 142   group "planet"
 143   mode "755"
 144 end
 145
 146 directory "/etc/replication" do
 147   owner "root"
 148   group "root"
 149   mode "755"
 150 end
 151
 152 directory "/var/run/lock/changeset-replication/" do
 153   owner "planet"
 154   group "planet"
 155   mode "750"
 156 end
 157
 158 template "/etc/replication/auth.conf" do
 159   source "replication.auth.erb"
 160   user "root"
 161   group "planet"
 162   mode "640"
 163   variables :password => db_passwords["planetdiff"]
 164 end
 165
 166 osmdbt_config = {
 167   "database" => {
 168     "host" => node[:web][:database_host],
 169     "dbname" => "openstreetmap",
 170     "user" => "planetdiff",
 171     "password" => db_passwords["planetdiff"],
 172     "replication_slot" => "osmdbt"
 173   },
 174   "log_dir" => "/var/lib/replication/minute",
 175   "changes_dir" => "/store/planet/replication/test/minute",
 176   "tmp_dir" => "/tmp",
 177   "run_dir" => "/run/replication"
 178 }
 179
 180 file "/etc/replication/osmdbt-config.yaml" do
 181   user "root"
 182   group "planet"
 183   mode "640"
 184   content YAML.dump(osmdbt_config)
 185 end
 186
 187 template "/etc/replication/changesets.conf" do
 188   source "changesets.conf.erb"
 189   user "root"
 190   group "planet"
 191   mode "640"
 192   variables :password => db_passwords["planetdiff"]
 193 end
 194
 195 template "/etc/replication/users-agreed.conf" do
 196   source "users-agreed.conf.erb"
 197   user "planet"
 198   group "planet"
 199   mode "600"
 200   variables :password => db_passwords["planetdiff"]
 201 end
 202
 203 directory "/var/lib/replication" do
 204   owner "planet"
 205   group "planet"
 206   mode "755"
 207 end
 208
 209 directory "/var/lib/replication/minute" do
 210   owner "planet"
 211   group "planet"
 212   mode "755"
 213 end
 214
 215 directory "/var/lib/replication/hour" do
 216   owner "planet"
 217   group "planet"
 218   mode "755"
 219 end
 220
 221 template "/var/lib/replication/hour/configuration.txt" do
 222   source "replication.config.erb"
 223   owner "planet"
 224   group "planet"
 225   mode "644"
 226   variables :base => "minute", :interval => 3600
 227 end
 228
 229 link "/var/lib/replication/hour/data" do
 230   to "/store/planet/replication/hour"
 231 end
 232
 233 directory "/var/lib/replication/day" do
 234   owner "planet"
 235   group "planet"
 236   mode "755"
 237 end
 238
 239 template "/var/lib/replication/day/configuration.txt" do
 240   source "replication.config.erb"
 241   owner "planet"
 242   group "planet"
 243   mode "644"
 244   variables :base => "hour", :interval => 86400
 245 end
 246
 247 link "/var/lib/replication/day/data" do
 248   to "/store/planet/replication/day"
 249 end
 250
 251 if node[:planet][:replication] == "enabled"
 252   cron_d "users-agreed" do
 253     minute "0"
 254     hour "7"
 255     user "planet"
 256     command "/usr/local/bin/users-agreed"
 257     mailto "zerebubuth@gmail.com"
 258   end
 259
 260   cron_d "users-deleted" do
 261     minute "0"
 262     hour "17"
 263     user "planet"
 264     command "/usr/local/bin/users-deleted"
 265     mailto "zerebubuth@gmail.com"
 266   end
 267
 268   cron_d "replication-changesets" do
 269     user "planet"
 270     command "/usr/local/bin/replicate-changesets /etc/replication/changesets.conf"
 271     mailto "zerebubuth@gmail.com"
 272   end
 273
 274   cron_d "replication-minutely" do
 275     user "planet"
 276     command "/usr/local/bin/osmosis -q --replicate-apidb authFile=/etc/replication/auth.conf validateSchemaVersion=false --write-replication workingDirectory=/store/planet/replication/minute"
 277     mailto "brett@bretth.com"
 278     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 279   end
 280
 281   cron_d "replication-hourly" do
 282     minute "2,7,12,17"
 283     user "planet"
 284     command "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/hour"
 285     mailto "brett@bretth.com"
 286     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 287   end
 288
 289   cron_d "replication-daily" do
 290     minute "5,10,15,20"
 291     user "planet"
 292     command "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/day"
 293     mailto "brett@bretth.com"
 294     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 295   end
 296 else
 297   cron_d "users-agreed" do
 298     action :delete
 299   end
 300
 301   cron_d "users-deleted" do
 302     action :delete
 303   end
 304
 305   cron_d "replication-changesets" do
 306     action :delete
 307   end
 308
 309   cron_d "replication-minutely" do
 310     action :delete
 311   end
 312
 313   cron_d "replication-hourly" do
 314     action :delete
 315   end
 316
 317   cron_d "replication-daily" do
 318     action :delete
 319   end
 320 end