]> git.openstreetmap.org Git - chef.git/blob - cookbooks/planet/recipes/replication.rb
Configure osmdbt based minutely replication service
[chef.git] / cookbooks / planet / recipes / replication.rb
1 #
2 # Cookbook:: planet
3 # Recipe:: dump
4 #
5 # Copyright:: 2013, OpenStreetMap Foundation
6 #
7 # Licensed under the Apache License, Version 2.0 (the "License");
8 # you may not use this file except in compliance with the License.
9 # You may obtain a copy of the License at
10 #
11 #     https://www.apache.org/licenses/LICENSE-2.0
12 #
13 # Unless required by applicable law or agreed to in writing, software
14 # distributed under the License is distributed on an "AS IS" BASIS,
15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
18 #
19
20 require "yaml"
21
22 include_recipe "accounts"
23 include_recipe "osmosis"
24
25 db_passwords = data_bag_item("db", "passwords")
26
27 package %w[
28   postgresql-client
29   ruby
30   ruby-dev
31   ruby-libxml
32   make
33   gcc
34   libpq-dev
35   osmdbt
36 ]
37
38 gem_package "pg"
39
40 remote_directory "/opt/flush" do
41   source "flush"
42   owner "root"
43   group "root"
44   mode "755"
45   files_owner "root"
46   files_group "root"
47   files_mode "755"
48 end
49
50 execute "/opt/flush/Makefile" do
51   action :nothing
52   command "make"
53   cwd "/opt/flush"
54   user "root"
55   group "root"
56   subscribes :run, "remote_directory[/opt/flush]"
57 end
58
59 remote_directory "/usr/local/bin" do
60   source "replication-bin"
61   owner "root"
62   group "root"
63   mode "755"
64   files_owner "root"
65   files_group "root"
66   files_mode "755"
67 end
68
69 template "/usr/local/bin/replicate-minute" do
70   source "replicate-minute.erb"
71   owner "root"
72   group "root"
73   mode "755"
74 end
75
76 template "/usr/local/bin/users-agreed" do
77   source "users-agreed.erb"
78   owner "root"
79   group "root"
80   mode "755"
81 end
82
83 template "/usr/local/bin/users-deleted" do
84   source "users-deleted.erb"
85   owner "root"
86   group "root"
87   mode "755"
88 end
89
90 remote_directory "/store/planet/users_deleted" do
91   source "users_deleted"
92   owner "planet"
93   group "planet"
94   mode "755"
95   files_owner "root"
96   files_group "root"
97   files_mode "644"
98 end
99
100 remote_directory "/store/planet/replication" do
101   source "replication-cgi"
102   owner "root"
103   group "root"
104   mode "755"
105   files_owner "root"
106   files_group "root"
107   files_mode "755"
108 end
109
110 directory "/store/planet/replication/changesets" do
111   owner "planet"
112   group "planet"
113   mode "755"
114 end
115
116 directory "/store/planet/replication/day" do
117   owner "planet"
118   group "planet"
119   mode "755"
120 end
121
122 directory "/store/planet/replication/hour" do
123   owner "planet"
124   group "planet"
125   mode "755"
126 end
127
128 directory "/store/planet/replication/minute" do
129   owner "planet"
130   group "planet"
131   mode "755"
132 end
133
134 directory "/store/planet/replication/test" do
135   owner "planet"
136   group "planet"
137   mode "755"
138 end
139
140 directory "/store/planet/replication/test/minute" do
141   owner "planet"
142   group "planet"
143   mode "755"
144 end
145
146 systemd_tmpfile "/run/replication" do
147   type "d"
148   owner "planet"
149   group "planet"
150   mode "755"
151 end
152
153 directory "/etc/replication" do
154   owner "root"
155   group "root"
156   mode "755"
157 end
158
159 directory "/var/run/lock/changeset-replication/" do
160   owner "planet"
161   group "planet"
162   mode "750"
163 end
164
165 template "/etc/replication/auth.conf" do
166   source "replication.auth.erb"
167   user "root"
168   group "planet"
169   mode "640"
170   variables :password => db_passwords["planetdiff"]
171 end
172
173 osmdbt_config = {
174   "database" => {
175     "host" => node[:web][:database_host],
176     "dbname" => "openstreetmap",
177     "user" => "planetdiff",
178     "password" => db_passwords["planetdiff"],
179     "replication_slot" => "osmdbt"
180   },
181   "log_dir" => "/var/lib/replication/minute",
182   "changes_dir" => "/store/planet/replication/test/minute",
183   "tmp_dir" => "/tmp",
184   "run_dir" => "/run/replication"
185 }
186
187 file "/etc/replication/osmdbt-config.yaml" do
188   user "root"
189   group "planet"
190   mode "640"
191   content YAML.dump(osmdbt_config)
192 end
193
194 systemd_service "replication-minutely" do
195   description "Minutely replication"
196   user "planet"
197   working_directory "/etc/replication"
198   exec_start "/usr/local/bin/replicate-minute"
199   private_tmp true
200   private_devices true
201   protect_system "full"
202   protect_home true
203   restrict_address_families %w[AF_INET AF_INET6]
204   no_new_privileges true
205 end
206
207 template "/etc/replication/changesets.conf" do
208   source "changesets.conf.erb"
209   user "root"
210   group "planet"
211   mode "640"
212   variables :password => db_passwords["planetdiff"]
213 end
214
215 template "/etc/replication/users-agreed.conf" do
216   source "users-agreed.conf.erb"
217   user "planet"
218   group "planet"
219   mode "600"
220   variables :password => db_passwords["planetdiff"]
221 end
222
223 directory "/var/lib/replication" do
224   owner "planet"
225   group "planet"
226   mode "755"
227 end
228
229 directory "/var/lib/replication/minute" do
230   owner "planet"
231   group "planet"
232   mode "755"
233 end
234
235 directory "/var/lib/replication/hour" do
236   owner "planet"
237   group "planet"
238   mode "755"
239 end
240
241 template "/var/lib/replication/hour/configuration.txt" do
242   source "replication.config.erb"
243   owner "planet"
244   group "planet"
245   mode "644"
246   variables :base => "minute", :interval => 3600
247 end
248
249 link "/var/lib/replication/hour/data" do
250   to "/store/planet/replication/hour"
251 end
252
253 directory "/var/lib/replication/day" do
254   owner "planet"
255   group "planet"
256   mode "755"
257 end
258
259 template "/var/lib/replication/day/configuration.txt" do
260   source "replication.config.erb"
261   owner "planet"
262   group "planet"
263   mode "644"
264   variables :base => "hour", :interval => 86400
265 end
266
267 link "/var/lib/replication/day/data" do
268   to "/store/planet/replication/day"
269 end
270
271 if node[:planet][:replication] == "enabled"
272   cron_d "users-agreed" do
273     minute "0"
274     hour "7"
275     user "planet"
276     command "/usr/local/bin/users-agreed"
277     mailto "zerebubuth@gmail.com"
278   end
279
280   cron_d "users-deleted" do
281     minute "0"
282     hour "17"
283     user "planet"
284     command "/usr/local/bin/users-deleted"
285     mailto "zerebubuth@gmail.com"
286   end
287
288   cron_d "replication-changesets" do
289     user "planet"
290     command "/usr/local/bin/replicate-changesets /etc/replication/changesets.conf"
291     mailto "zerebubuth@gmail.com"
292   end
293
294   systemd_timer "replication-minutely" do
295     description "Minutely replication"
296     on_boot_sec 60
297     on_unit_active_sec 60
298   end
299
300   cron_d "replication-minutely" do
301     user "planet"
302     command "/usr/local/bin/osmosis -q --replicate-apidb authFile=/etc/replication/auth.conf validateSchemaVersion=false --write-replication workingDirectory=/store/planet/replication/minute"
303     mailto "brett@bretth.com"
304     environment "LD_PRELOAD" => "/opt/flush/flush.so"
305   end
306
307   cron_d "replication-hourly" do
308     minute "2,7,12,17"
309     user "planet"
310     command "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/hour"
311     mailto "brett@bretth.com"
312     environment "LD_PRELOAD" => "/opt/flush/flush.so"
313   end
314
315   cron_d "replication-daily" do
316     minute "5,10,15,20"
317     user "planet"
318     command "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/day"
319     mailto "brett@bretth.com"
320     environment "LD_PRELOAD" => "/opt/flush/flush.so"
321   end
322 else
323   cron_d "users-agreed" do
324     action :delete
325   end
326
327   cron_d "users-deleted" do
328     action :delete
329   end
330
331   cron_d "replication-changesets" do
332     action :delete
333   end
334
335   cron_d "replication-minutely" do
336     action :delete
337   end
338
339   cron_d "replication-hourly" do
340     action :delete
341   end
342
343   cron_d "replication-daily" do
344     action :delete
345   end
346 end