]> git.openstreetmap.org Git - chef.git/blob - cookbooks/planet/recipes/replication.rb
Install libc6-dev so we can compile pg extension
[chef.git] / cookbooks / planet / recipes / replication.rb
1 #
2 # Cookbook:: planet
3 # Recipe:: dump
4 #
5 # Copyright:: 2013, OpenStreetMap Foundation
6 #
7 # Licensed under the Apache License, Version 2.0 (the "License");
8 # you may not use this file except in compliance with the License.
9 # You may obtain a copy of the License at
10 #
11 #     https://www.apache.org/licenses/LICENSE-2.0
12 #
13 # Unless required by applicable law or agreed to in writing, software
14 # distributed under the License is distributed on an "AS IS" BASIS,
15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
18 #
19
20 require "yaml"
21
22 include_recipe "accounts"
23 include_recipe "apt"
24 include_recipe "osmosis"
25
26 db_passwords = data_bag_item("db", "passwords")
27
28 package %w[
29   postgresql-client
30   ruby
31   ruby-dev
32   ruby-libxml
33   make
34   gcc
35   libc6-dev
36   libpq-dev
37   osmdbt
38 ]
39
40 gem_package "pg"
41
42 remote_directory "/opt/flush" do
43   source "flush"
44   owner "root"
45   group "root"
46   mode "755"
47   files_owner "root"
48   files_group "root"
49   files_mode "755"
50 end
51
52 execute "/opt/flush/Makefile" do
53   action :nothing
54   command "make"
55   cwd "/opt/flush"
56   user "root"
57   group "root"
58   subscribes :run, "remote_directory[/opt/flush]"
59 end
60
61 remote_directory "/usr/local/bin" do
62   source "replication-bin"
63   owner "root"
64   group "root"
65   mode "755"
66   files_owner "root"
67   files_group "root"
68   files_mode "755"
69 end
70
71 template "/usr/local/bin/replicate-minute" do
72   source "replicate-minute.erb"
73   owner "root"
74   group "root"
75   mode "755"
76 end
77
78 template "/usr/local/bin/users-agreed" do
79   source "users-agreed.erb"
80   owner "root"
81   group "root"
82   mode "755"
83 end
84
85 template "/usr/local/bin/users-deleted" do
86   source "users-deleted.erb"
87   owner "root"
88   group "root"
89   mode "755"
90 end
91
92 remote_directory "/store/planet/users_deleted" do
93   source "users_deleted"
94   owner "planet"
95   group "planet"
96   mode "755"
97   files_owner "root"
98   files_group "root"
99   files_mode "644"
100 end
101
102 remote_directory "/store/planet/replication" do
103   source "replication-cgi"
104   owner "root"
105   group "root"
106   mode "755"
107   files_owner "root"
108   files_group "root"
109   files_mode "755"
110 end
111
112 directory "/store/planet/replication/changesets" do
113   owner "planet"
114   group "planet"
115   mode "755"
116 end
117
118 directory "/store/planet/replication/day" do
119   owner "planet"
120   group "planet"
121   mode "755"
122 end
123
124 directory "/store/planet/replication/hour" do
125   owner "planet"
126   group "planet"
127   mode "755"
128 end
129
130 directory "/store/planet/replication/minute" do
131   owner "planet"
132   group "planet"
133   mode "755"
134 end
135
136 directory "/store/planet/replication/test" do
137   owner "planet"
138   group "planet"
139   mode "755"
140 end
141
142 directory "/store/planet/replication/test/day" do
143   owner "planet"
144   group "planet"
145   mode "755"
146 end
147
148 directory "/store/planet/replication/test/hour" do
149   owner "planet"
150   group "planet"
151   mode "755"
152 end
153
154 directory "/store/planet/replication/test/minute" do
155   owner "planet"
156   group "planet"
157   mode "755"
158 end
159
160 directory "/store/replication" do
161   owner "planet"
162   group "planet"
163   mode "755"
164 end
165
166 directory "/store/replication/minute" do
167   owner "planet"
168   group "planet"
169   mode "755"
170 end
171
172 systemd_tmpfile "/run/replication" do
173   type "d"
174   owner "planet"
175   group "planet"
176   mode "755"
177 end
178
179 directory "/etc/replication" do
180   owner "root"
181   group "root"
182   mode "755"
183 end
184
185 directory "/var/run/lock/changeset-replication/" do
186   owner "planet"
187   group "planet"
188   mode "750"
189 end
190
191 directory "/var/lib/replication" do
192   owner "planet"
193   group "planet"
194   mode "755"
195 end
196
197 directory "/var/lib/replication/test" do
198   owner "planet"
199   group "planet"
200   mode "755"
201 end
202
203 template "/etc/replication/auth.conf" do
204   source "replication.auth.erb"
205   user "root"
206   group "planet"
207   mode "640"
208   variables :password => db_passwords["planetdiff"]
209 end
210
211 osmdbt_config = {
212   "database" => {
213     "host" => node[:web][:database_host],
214     "dbname" => "openstreetmap",
215     "user" => "planetdiff",
216     "password" => db_passwords["planetdiff"],
217     "replication_slot" => "osmdbt"
218   },
219   "log_dir" => "/var/lib/replication/minute",
220   "changes_dir" => "/store/planet/replication/test/minute",
221   "tmp_dir" => "/store/replication/minute",
222   "run_dir" => "/run/replication"
223 }
224
225 file "/etc/replication/osmdbt-config.yaml" do
226   user "root"
227   group "planet"
228   mode "640"
229   content YAML.dump(osmdbt_config)
230 end
231
232 systemd_service "replication-minutely" do
233   description "Minutely replication"
234   user "planet"
235   working_directory "/etc/replication"
236   exec_start "/usr/local/bin/replicate-minute"
237   private_tmp true
238   private_devices true
239   protect_system "full"
240   protect_home true
241   restrict_address_families %w[AF_INET AF_INET6]
242   no_new_privileges true
243 end
244
245 systemd_timer "replication-minutely" do
246   description "Minutely replication"
247   on_boot_sec 60
248   on_unit_active_sec 60
249   accuracy_sec 5
250 end
251
252 directory "/var/lib/replication/test/hour" do
253   owner "planet"
254   group "planet"
255   mode "755"
256 end
257
258 template "/var/lib/replication/test/hour/configuration.txt" do
259   source "replication.config.erb"
260   owner "planet"
261   group "planet"
262   mode "644"
263   variables :base => "test/minute", :interval => 3600
264 end
265
266 link "/var/lib/replication/test/hour/data" do
267   to "/store/planet/replication/test/hour"
268 end
269
270 systemd_service "replication-hourly" do
271   description "Hourly replication"
272   user "planet"
273   exec_start "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/test/hour"
274   private_tmp true
275   private_devices true
276   protect_system "full"
277   protect_home true
278   restrict_address_families %w[AF_INET AF_INET6]
279   no_new_privileges true
280 end
281
282 systemd_timer "replication-hourly" do
283   description "Daily replication"
284   on_calendar "*-*-* *:02/15:00"
285 end
286
287 directory "/var/lib/replication/test/day" do
288   owner "planet"
289   group "planet"
290   mode "755"
291 end
292
293 template "/var/lib/replication/test/day/configuration.txt" do
294   source "replication.config.erb"
295   owner "planet"
296   group "planet"
297   mode "644"
298   variables :base => "test/hour", :interval => 86400
299 end
300
301 link "/var/lib/replication/test/day/data" do
302   to "/store/planet/replication/test/day"
303 end
304
305 systemd_service "replication-daily" do
306   description "Daily replication"
307   user "planet"
308   exec_start "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/test/day"
309   private_tmp true
310   private_devices true
311   protect_system "full"
312   protect_home true
313   restrict_address_families %w[AF_INET AF_INET6]
314   no_new_privileges true
315 end
316
317 systemd_timer "replication-daily" do
318   description "Daily replication"
319   on_calendar "*-*-* *:02/15:00"
320 end
321
322 template "/etc/replication/changesets.conf" do
323   source "changesets.conf.erb"
324   user "root"
325   group "planet"
326   mode "640"
327   variables :password => db_passwords["planetdiff"]
328 end
329
330 template "/etc/replication/users-agreed.conf" do
331   source "users-agreed.conf.erb"
332   user "planet"
333   group "planet"
334   mode "600"
335   variables :password => db_passwords["planetdiff"]
336 end
337
338 directory "/var/lib/replication/minute" do
339   owner "planet"
340   group "planet"
341   mode "755"
342 end
343
344 directory "/var/lib/replication/hour" do
345   owner "planet"
346   group "planet"
347   mode "755"
348 end
349
350 template "/var/lib/replication/hour/configuration.txt" do
351   source "replication.config.erb"
352   owner "planet"
353   group "planet"
354   mode "644"
355   variables :base => "minute", :interval => 3600
356 end
357
358 link "/var/lib/replication/hour/data" do
359   to "/store/planet/replication/hour"
360 end
361
362 directory "/var/lib/replication/day" do
363   owner "planet"
364   group "planet"
365   mode "755"
366 end
367
368 template "/var/lib/replication/day/configuration.txt" do
369   source "replication.config.erb"
370   owner "planet"
371   group "planet"
372   mode "644"
373   variables :base => "hour", :interval => 86400
374 end
375
376 link "/var/lib/replication/day/data" do
377   to "/store/planet/replication/day"
378 end
379
380 if node[:planet][:replication] == "enabled"
381   cron_d "users-agreed" do
382     minute "0"
383     hour "7"
384     user "planet"
385     command "/usr/local/bin/users-agreed"
386     mailto "zerebubuth@gmail.com"
387   end
388
389   cron_d "users-deleted" do
390     minute "0"
391     hour "17"
392     user "planet"
393     command "/usr/local/bin/users-deleted"
394     mailto "zerebubuth@gmail.com"
395   end
396
397   cron_d "replication-changesets" do
398     user "planet"
399     command "/usr/local/bin/replicate-changesets /etc/replication/changesets.conf"
400     mailto "zerebubuth@gmail.com"
401   end
402
403   service "replication-minutely.timer" do
404     action [:enable, :start]
405   end
406
407   service "replication-hourly.timer" do
408     action [:enable, :start]
409   end
410
411   service "replication-daily.timer" do
412     action [:enable, :start]
413   end
414
415   cron_d "replication-minutely" do
416     user "planet"
417     command "/usr/local/bin/osmosis -q --replicate-apidb authFile=/etc/replication/auth.conf validateSchemaVersion=false --write-replication workingDirectory=/store/planet/replication/minute"
418     mailto "brett@bretth.com"
419     environment "LD_PRELOAD" => "/opt/flush/flush.so"
420   end
421
422   cron_d "replication-hourly" do
423     minute "2,7,12,17"
424     user "planet"
425     command "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/hour"
426     mailto "brett@bretth.com"
427     environment "LD_PRELOAD" => "/opt/flush/flush.so"
428   end
429
430   cron_d "replication-daily" do
431     minute "5,10,15,20"
432     user "planet"
433     command "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/day"
434     mailto "brett@bretth.com"
435     environment "LD_PRELOAD" => "/opt/flush/flush.so"
436   end
437 else
438   cron_d "users-agreed" do
439     action :delete
440   end
441
442   cron_d "users-deleted" do
443     action :delete
444   end
445
446   cron_d "replication-changesets" do
447     action :delete
448   end
449
450   service "replication-minutely.timer" do
451     action [:stop, :disable]
452   end
453
454   service "replication-hourly.timer" do
455     action [:stop, :disable]
456   end
457
458   service "replication-daily.timer" do
459     action [:stop, :disable]
460   end
461
462   cron_d "replication-minutely" do
463     action :delete
464   end
465
466   cron_d "replication-hourly" do
467     action :delete
468   end
469
470   cron_d "replication-daily" do
471     action :delete
472   end
473 end