From: Tigerfell Date: Tue, 9 Mar 2021 18:29:53 +0000 (+0100) Subject: Merge branch 'patch-2' of https://github.com/Tigerfell/chef into pr257 X-Git-Url: https://git.openstreetmap.org/chef.git/commitdiff_plain/ab47927c016597f8602b81dcdc168ad5c335389d?hp=a5a3b8b5d676e4e7c46f5124325d2ca799bee76f Merge branch 'patch-2' of https://github.com/Tigerfell/chef into pr257 --- diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..c0d0101a4 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,11 @@ +# EditorConfig is awesome: https://EditorConfig.org + +root = true + +[*] +charset = utf-8 +indent_style = space +trim_trailing_whitespace = true +insert_final_newline = true +end_of_line = lf +indent_size = 2 diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..9f77688ac --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "bundler" + directory: "/" + schedule: + interval: "daily" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" diff --git a/.github/workflows/cookstyle.yml b/.github/workflows/cookstyle.yml new file mode 100644 index 000000000..56603f598 --- /dev/null +++ b/.github/workflows/cookstyle.yml @@ -0,0 +1,27 @@ +name: Cookstyle +on: + - push + - pull_request +jobs: + cookstyle: + name: Cookstyle + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v2 + - name: Setup ruby + uses: actions/setup-ruby@v1.1.3 + - name: Cache gems + uses: actions/cache@v2.1.4 + with: + path: vendor/bundle + key: bundle-${{ runner.OS }}-${{ hashFiles('Gemfile.lock') }} + restore-keys: | + bundle-${{ runner.OS }}- + - name: Install gems + run: | + gem install bundler + bundle config set deployment true + bundle install --jobs 4 --retry 3 + - name: Run cookstyle + run: bundle exec cookstyle --format fuubar diff --git a/.github/workflows/test-kitchen.yml b/.github/workflows/test-kitchen.yml new file mode 100644 index 000000000..297d0bbfe --- /dev/null +++ b/.github/workflows/test-kitchen.yml @@ -0,0 +1,135 @@ +name: Test Kitchen +on: + - push + - pull_request +jobs: + kitchen: + name: Test Kitchen + runs-on: ubuntu-latest + strategy: + matrix: + suite: + - accounts + - apache + - apt + - backup + - bind + - blog + - blogs + - civicrm + - clamav + - db-backup + - db-base + - db-master + - db-slave + - devices + - dhcpd + - dmca + - dns + - docker + - donate + - elasticsearch + - exim + - fail2ban + - forum + - ftp + - geodns + - geoipupdate + - git + - git-server + - git-web + - gps-tile + - hardware + - hot + - incron + - irc + - kibana + - letsencrypt + - logstash + - logstash-forwarder + - mail + - mailman + - memcached + - munin + - munin-plugins + - munin-server + - mysql + - networking + - nginx + - nodejs + - nominatim + - ntp + - openssh + - osmosis + - osqa + - otrs + - passenger + - php + - php-apache + - php-fpm + - piwik + - planet + - planet-current + - planet-dump + - planet-notes + - planet-replication + - postgresql + - prometheus + - prometheus-server + - python + - rsyncd + - serverinfo + - snmpd + - spamassassin + - squid + - ssl + - stateofthemap + - subversion + - supybot + - switch2osm + - sysctl + - sysfs + - taginfo + - tile + - tilecache + - tilelog + - tools + - trac + - web-backend + - web-cgimap + - web-frontend + - web-rails + - wordpress + - wiki + os: + - ubuntu-2004 + include: + - suite: trac + os: ubuntu-1804 + exclude: + - suite: trac + os: ubuntu-2004 + fail-fast: false + steps: + - name: Check out code + uses: actions/checkout@v2 + - name: Setup ruby + uses: actions/setup-ruby@v1.1.3 + - name: Cache gems + uses: actions/cache@v2.1.4 + with: + path: vendor/bundle + key: bundle-${{ runner.OS }}-${{ hashFiles('Gemfile.lock') }} + restore-keys: | + bundle-${{ runner.OS }}- + - name: Disable apparmor for mysqld + run: | + sudo apt-get install apparmor-utils + sudo aa-disable /usr/sbin/mysqld + - name: Install gems + run: | + gem install bundler + bundle config set deployment true + bundle install --jobs 4 --retry 3 + - name: Run kitchen test ${{ matrix.suite }}-${{ matrix.os }} + run: bundle exec kitchen test ${{ matrix.suite }}-${{ matrix.os }} diff --git a/.gitignore b/.gitignore index 82d1b5caa..b31363aa4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ +.DS_Store .kitchen/ .kitchen.local.yml diff --git a/.kitchen.provision.rb b/.kitchen.provision.rb deleted file mode 100644 index 8398bd549..000000000 --- a/.kitchen.provision.rb +++ /dev/null @@ -1,5 +0,0 @@ -Vagrant.configure(2) do |config| - config.vm.provision "shell", :inline => <<-SHELL - apt-get update -y - SHELL -end diff --git a/.kitchen.yml b/.kitchen.yml index 1b49363e6..58a63e3b9 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -1,16 +1,37 @@ --- driver: - name: vagrant - provision: true - vagrantfiles: - - .kitchen.provision.rb + name: dokken + chef_version: 16 + env: + - container=dokken + +transport: + name: dokken provisioner: - name: chef_zero + name: dokken + chef_license: accept data_bags_path: test/data_bags +verifier: + root_path: /opt/verifier + sudo: false + platforms: - - name: ubuntu-16.04 + - name: ubuntu-18.04 + driver: + image: dokken/ubuntu-18.04 + privileged: true + pid_one_command: /bin/systemd + intermediate_instructions: + - RUN /usr/bin/apt-get update -y + - name: ubuntu-20.04 + driver: + image: dokken/ubuntu-20.04 + privileged: true + pid_one_command: /bin/systemd + intermediate_instructions: + - RUN /usr/bin/apt-get update -y suites: - name: accounts @@ -19,35 +40,165 @@ suites: - name: apache run_list: - recipe[apache::default] - - name: apache-ssl - run_list: - - recipe[apache::ssl] - name: apt run_list: - recipe[apt::default] + - name: backup + run_list: + - recipe[backup::default] - name: bind run_list: - recipe[bind::default] + attributes: + bind: + clients: none + - name: blog + run_list: + - recipe[blog::default] - name: blogs run_list: - - recipe[accounts::default] - recipe[blogs::default] + - name: civicrm + run_list: + - recipe[civicrm::default] + - name: clamav + run_list: + - recipe[clamav::default] + - name: db-backup + run_list: + - recipe[db::backup] + - name: db-base + run_list: + - recipe[db::base] + - name: db-master + run_list: + - recipe[db::master] + - name: db-slave + run_list: + - recipe[db::slave] + - name: devices + run_list: + - recipe[devices::default] + - name: dhcpd + run_list: + - recipe[dhcpd::default] + attributes: + dhcpd: + first_address: 172.18.100.1 + last_address: 172.18.100.254 + networking: + interfaces: + test: + interface: eth0 + role: internal + address: 172.18.0.2 + prefix: 16 + gateway: 172.18.0.1 + roles: + external: + zone: test + - name: dmca + run_list: + - recipe[dmca::default] + - name: dns + run_list: + - recipe[dns::default] + - name: docker + run_list: + - recipe[docker::default] + - name: donate + run_list: + - recipe[donate::default] + - name: elasticsearch + run_list: + - recipe[elasticsearch::default] + - name: exim + run_list: + - recipe[exim::default] + - name: fail2ban + run_list: + - recipe[fail2ban::default] - name: forum run_list: - - recipe[accounts::default] - - role[forum] + - recipe[forum::default] + - name: ftp + run_list: + - recipe[ftp::default] + - name: geodns + run_list: + - recipe[geodns::default] + attributes: + networking: + interfaces: + test: + role: external + address: 172.18.0.2 + - name: geoipupdate + run_list: + - recipe[geoipupdate::default] + - name: git + run_list: + - recipe[git::default] + - name: git-server + run_list: + - recipe[git::server] + - name: git-web + run_list: + - recipe[git::web] + - name: gps-tile + run_list: + - recipe[gps-tile::default] + - name: hardware + run_list: + - recipe[hardware::default] + - name: hot + run_list: + - recipe[hot::default] + - name: incron + run_list: + - recipe[incron::default] + - name: irc + run_list: + - recipe[irc::default] + - name: kibana + run_list: + - recipe[kibana::default] - name: letsencrypt run_list: - - recipe[accounts::default] - - recipe[apt::default] - - role[letsencrypt] + - recipe[letsencrypt::default] attributes: apt: sources: - openstreetmap + - name: logstash + run_list: + - recipe[logstash::default] + - name: logstash-forwarder + run_list: + - recipe[logstash::forwarder] + attributes: + logstash: + forwarder: + filebeat.prospectors: + - input_type: log + paths: + - /var/log/apache2/access.log + type: apache + - name: mail + run_list: + - role[mail] + - name: mailman + run_list: + - recipe[mailman::default] + - name: memcached + run_list: + - recipe[memcached::default] - name: munin run_list: - recipe[munin::default] + - name: munin-plugins + run_list: + - recipe[munin::plugins] - name: munin-server run_list: - recipe[munin::server] @@ -57,14 +208,167 @@ suites: - name: networking run_list: - recipe[networking::default] + - name: nginx + run_list: + - recipe[nginx::default] + - name: nodejs + run_list: + - recipe[nodejs::default] + - name: nominatim + run_list: + - recipe[nominatim::default] + - name: ntp + run_list: + - recipe[ntp::default] + - name: openssh + run_list: + - recipe[openssh::default] + - name: osmosis + run_list: + - recipe[osmosis::default] + - name: osqa + run_list: + - recipe[osqa::default] - name: otrs run_list: - - recipe[accounts::default] - - recipe[chef::default] - - role[otrs] + - recipe[otrs::default] + - name: passenger + run_list: + - recipe[passenger::default] + - name: php + run_list: + - recipe[php::default] + - name: php-apache + run_list: + - recipe[php::apache] + - name: php-fpm + run_list: + - recipe[php::fpm] + - name: piwik + run_list: + - recipe[piwik::default] + - name: planet + run_list: + - recipe[planet::default] + - name: planet-current + run_list: + - recipe[planet::current] + - name: planet-dump + run_list: + - recipe[planet::dump] + - name: planet-notes + run_list: + - recipe[planet::notes] + attributes: + web: + readonly_database_host: readonly + - name: planet-replication + run_list: + - recipe[planet::replication] + attributes: + web: + readonly_database_host: readonly + - name: postgresql + run_list: + - recipe[postgresql::default] + attributes: + postgresql: + versions: + - 10 + - name: prometheus + run_list: + - recipe[prometheus::default] + attributes: + networking: + interfaces: + test: + interface: eth0 + role: internal + address: 172.18.0.2 + prefix: 16 + gateway: 172.18.0.1 + - name: prometheus-server + run_list: + - recipe[prometheus::server] - name: python run_list: - recipe[python::default] + - name: rsyncd + run_list: + - recipe[rsyncd::default] + - name: serverinfo + run_list: + - recipe[serverinfo::default] + - name: snmpd + run_list: + - recipe[snmpd::default] + - name: spamassassin + run_list: + - recipe[spamassassin::default] + - name: squid + run_list: + - recipe[squid::default] + - name: ssl + run_list: + - recipe[ssl::default] + - name: stateofthemap + run_list: + - recipe[stateofthemap::default] + - name: subversion + run_list: + - recipe[subversion::default] + - name: supybot + run_list: + - recipe[supybot::default] + - name: switch2osm + run_list: + - recipe[switch2osm::default] + - name: sysctl + run_list: + - recipe[sysctl::default] + - name: sysfs + run_list: + - recipe[sysfs::default] + - name: taginfo + run_list: + - recipe[taginfo::default] + attributes: + taginfo: + sites: + - name: taginfo.example.com + - name: tile + run_list: + - recipe[tile::default] + - name: tilecache + run_list: + - recipe[tilecache::default] + - name: tilelog + run_list: + - recipe[tilelog::default] + - name: timescaledb + run_list: + - recipe[timescaledb::default] - name: tools run_list: - recipe[tools::default] + - name: trac + run_list: + - recipe[trac::default] + - name: web-backend + run_list: + - recipe[web::backend] + - name: web-cgimap + run_list: + - recipe[web::cgimap] + - name: web-frontend + run_list: + - recipe[web::frontend] + - name: web-rails + run_list: + - recipe[web::rails] + - name: wiki + run_list: + - recipe[wiki::default] + - name: wordpress + run_list: + - recipe[wordpress::default] diff --git a/.mailmap b/.mailmap index 90f166d5b..15718e519 100644 --- a/.mailmap +++ b/.mailmap @@ -1,3 +1,3 @@ -Grant Slater +Grant Slater Guillaume Rischard Jochen Topf diff --git a/.rubocop.yml b/.rubocop.yml index 5db1a3104..80bd36dad 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,17 +1,23 @@ inherit_from: .rubocop_todo.yml AllCops: - TargetRubyVersion: 2.3 + TargetRubyVersion: 2.7 + +ChefModernize/IncludingAptDefaultRecipe: + Enabled: false Layout/ExtraSpacing: AllowForAlignment: true -Layout/IndentHeredoc: - EnforcedStyle: squiggly +Layout/HeredocIndentation: + Enabled: true -Naming/UncommunicativeMethodParamName: +Naming/MethodParameterName: Enabled: false +Style/CommandLiteral: + EnforcedStyle: percent_x + Style/HashSyntax: EnforcedStyle: hash_rockets diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 945610536..bdb397acf 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -1,38 +1,19 @@ # This configuration was generated by # `rubocop --auto-gen-config` -# on 2019-09-09 18:59:09 +0100 using RuboCop version 0.72.0. +# on 2020-07-30 08:39:06 UTC using RuboCop version 0.88.0. # The point is for the user to remove these configuration records # one by one as the offenses are removed from the code base. # Note that changes in the inspected code, or installation of new # versions of RuboCop, may require this file to be generated again. -# Offense count: 14 -ChefCorrectness/NodeNormal: - Exclude: - - '**/metadata.rb' - - 'cookbooks/dev/recipes/default.rb' - - 'cookbooks/mediawiki/resources/site.rb' - - 'cookbooks/networking/recipes/default.rb' - - 'cookbooks/openvpn/recipes/default.rb' - - 'cookbooks/web/recipes/backend.rb' - - 'cookbooks/wordpress/resources/site.rb' - -# Offense count: 12 -ChefCorrectness/NodeNormalUnless: - Exclude: - - '**/metadata.rb' - - 'cookbooks/mailman/recipes/default.rb' - - 'cookbooks/mediawiki/resources/site.rb' - - 'cookbooks/wordpress/resources/site.rb' - -# Offense count: 947 +# Offense count: 1038 # Cop supports --auto-correct. # Configuration parameters: . # SupportedStyles: strings, symbols ChefStyle/AttributeKeys: EnforcedStyle: symbols -# Offense count: 466 -# Cop supports --auto-correct. -ChefStyle/FileMode: - Enabled: false +# Offense count: 3 +# Configuration parameters: CountBlocks. +Metrics/BlockNesting: + Max: 4 diff --git a/.ruby-version b/.ruby-version index 7bf4b6a8a..860487ca1 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -2.4.6 +2.7.1 diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 87e77d6c7..000000000 --- a/.travis.yml +++ /dev/null @@ -1,5 +0,0 @@ -sudo: false -language: ruby -cache: bundler -script: - - bundle exec cookstyle -f fuubar diff --git a/Dockerfile b/Dockerfile index 01e194f4a..620c6a2ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,24 @@ -FROM ruby:2.6 +# Basic Dockerfile to run cookstyle linting +# run: docker build -t chef-test . +FROM ruby:2.7-alpine as build +# Add Gem build requirements +RUN apk add --no-cache build-base + +# Create app directory WORKDIR /app -RUN apt-get update \ - && apt-get install -y --no-install-recommends libssl-dev +# Add Gemfile and Gemfile.lock +ADD Gemfile* ./ -ADD Gemfile* /app/ -RUN gem install bundler --no-document \ +# Install Gems +RUN gem install bundler \ && bundle config build.nokogiri --use-system-libraries \ - && bundle install --jobs $(nproc) --retry 5 + && bundle config --global jobs $(nproc) \ + && bundle install -ADD . /app/ +# Add repo +ADD . . +# Run linting RUN bundle exec cookstyle -f fuubar diff --git a/Gemfile b/Gemfile index ddf1c6b1c..f04be603c 100644 --- a/Gemfile +++ b/Gemfile @@ -1,6 +1,6 @@ source "https://rubygems.org" gem "cookstyle" -gem "kitchen-vagrant" +gem "kitchen-dokken" gem "serverspec" gem "test-kitchen" diff --git a/Gemfile.lock b/Gemfile.lock index 973d415ee..65ffe696d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,145 +1,157 @@ GEM remote: https://rubygems.org/ specs: - ast (2.4.0) - bcrypt_pbkdf (1.0.1) - builder (3.2.3) - cookstyle (5.9.3) - rubocop (= 0.72.0) - diff-lcs (1.3) + ast (2.4.2) + bcrypt_pbkdf (1.1.0) + builder (3.2.4) + chef-utils (16.10.17) + cookstyle (7.9.0) + rubocop (= 1.11.0) + diff-lcs (1.4.4) + docker-api (2.0.0) + excon (>= 0.47.0) + multi_json ed25519 (1.2.4) - equatable (0.6.1) - erubis (2.7.0) - ffi (1.11.1) - gssapi (1.3.0) + erubi (1.10.0) + excon (0.79.0) + ffi (1.14.2) + gssapi (1.3.1) ffi (>= 1.0.1) gyoku (1.3.1) builder (>= 2.1.2) httpclient (2.8.3) - jaro_winkler (1.5.3) - kitchen-vagrant (1.6.0) - test-kitchen (>= 1.4, < 3) - license-acceptance (1.0.13) + kitchen-dokken (2.12.1) + docker-api (>= 1.33, < 3) + lockfile (~> 2.1) + test-kitchen (>= 1.15, < 3) + license-acceptance (2.1.13) pastel (~> 0.7) - tomlrb (~> 1.2) - tty-box (~> 0.3) - tty-prompt (~> 0.18) + tomlrb (>= 1.2, < 3.0) + tty-box (~> 0.6) + tty-prompt (~> 0.20) little-plugger (1.1.4) - logging (2.2.2) + lockfile (2.1.3) + logging (2.3.0) little-plugger (~> 1.1) - multi_json (~> 1.10) - mixlib-install (3.11.21) + multi_json (~> 1.14) + mixlib-install (3.12.7) mixlib-shellout mixlib-versioning thor - mixlib-shellout (3.0.7) - mixlib-versioning (1.2.7) - multi_json (1.14.1) - necromancer (0.5.0) - net-scp (2.0.0) - net-ssh (>= 2.6.5, < 6.0.0) - net-ssh (5.2.0) + mixlib-shellout (3.2.5) + chef-utils + mixlib-versioning (1.2.12) + multi_json (1.15.0) + net-scp (3.0.0) + net-ssh (>= 2.6.5, < 7.0.0) + net-ssh (6.1.0) net-ssh-gateway (2.0.0) net-ssh (>= 4.0.0) net-telnet (0.1.1) nori (2.6.0) - parallel (1.18.0) - parser (2.6.5.0) - ast (~> 2.4.0) - pastel (0.7.3) - equatable (~> 0.6) + parallel (1.20.1) + parser (3.0.0.0) + ast (~> 2.4.1) + pastel (0.8.0) tty-color (~> 0.5) rainbow (3.0.0) + regexp_parser (2.1.1) + rexml (3.2.4) rspec (3.9.0) rspec-core (~> 3.9.0) rspec-expectations (~> 3.9.0) rspec-mocks (~> 3.9.0) - rspec-core (3.9.0) - rspec-support (~> 3.9.0) - rspec-expectations (3.9.0) + rspec-core (3.9.2) + rspec-support (~> 3.9.3) + rspec-expectations (3.9.2) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.9.0) rspec-its (1.3.0) rspec-core (>= 3.0.0) rspec-expectations (>= 3.0.0) - rspec-mocks (3.9.0) + rspec-mocks (3.9.1) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.9.0) - rspec-support (3.9.0) - rubocop (0.72.0) - jaro_winkler (~> 1.5.1) + rspec-support (3.9.3) + rubocop (1.11.0) parallel (~> 1.10) - parser (>= 2.6) + parser (>= 3.0.0.0) rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml + rubocop-ast (>= 1.2.0, < 2.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 1.4.0, < 1.7) - ruby-progressbar (1.10.1) - rubyntlm (0.6.2) - rubyzip (1.3.0) + unicode-display_width (>= 1.4.0, < 3.0) + rubocop-ast (1.4.1) + parser (>= 2.7.1.5) + ruby-progressbar (1.11.0) + rubyntlm (0.6.3) + rubyzip (2.3.0) serverspec (2.41.5) multi_json rspec (~> 3.0) rspec-its specinfra (~> 2.72) sfl (2.3) - specinfra (2.82.3) + specinfra (2.82.19) net-scp net-ssh (>= 2.7) net-telnet (= 0.1.1) sfl - strings (0.1.6) - strings-ansi (~> 0.1) + strings (0.2.0) + strings-ansi (~> 0.2) unicode-display_width (~> 1.5) unicode_utils (~> 1.4) - strings-ansi (0.1.0) - test-kitchen (2.3.3) + strings-ansi (0.2.0) + test-kitchen (2.11.1) bcrypt_pbkdf (~> 1.0) + chef-utils (>= 16.4.35) ed25519 (~> 1.2) - license-acceptance (~> 1.0, >= 1.0.11) + license-acceptance (>= 1.0.11, < 3.0) mixlib-install (~> 3.6) mixlib-shellout (>= 1.2, < 4.0) - net-scp (>= 1.1, < 3.0) - net-ssh (>= 2.9, < 6.0) + net-scp (>= 1.1, < 4.0) + net-ssh (>= 2.9, < 7.0) net-ssh-gateway (>= 1.2, < 3.0) - thor (~> 0.19) + thor (>= 0.19, < 2.0) winrm (~> 2.0) winrm-elevated (~> 1.0) winrm-fs (~> 1.1) - thor (0.20.3) - tomlrb (1.2.8) - tty-box (0.5.0) - pastel (~> 0.7.2) - strings (~> 0.1.6) + thor (1.1.0) + tomlrb (2.0.1) + tty-box (0.7.0) + pastel (~> 0.8) + strings (~> 0.2.0) tty-cursor (~> 0.7) - tty-color (0.5.0) - tty-cursor (0.7.0) - tty-prompt (0.19.0) - necromancer (~> 0.5.0) - pastel (~> 0.7.0) - tty-reader (~> 0.6.0) - tty-reader (0.6.0) + tty-color (0.6.0) + tty-cursor (0.7.1) + tty-prompt (0.23.0) + pastel (~> 0.8) + tty-reader (~> 0.8) + tty-reader (0.9.0) tty-cursor (~> 0.7) - tty-screen (~> 0.7) - wisper (~> 2.0.0) - tty-screen (0.7.0) - unicode-display_width (1.6.0) + tty-screen (~> 0.8) + wisper (~> 2.0) + tty-screen (0.8.1) + unicode-display_width (1.7.0) unicode_utils (1.4.0) - winrm (2.3.2) + winrm (2.3.6) builder (>= 2.1.2) - erubis (~> 2.7) + erubi (~> 1.8) gssapi (~> 1.2) gyoku (~> 1.0) httpclient (~> 2.2, >= 2.2.0.2) logging (>= 1.6.1, < 3.0) nori (~> 2.0) - rubyntlm (~> 0.6.0, >= 0.6.1) - winrm-elevated (1.1.1) + rubyntlm (~> 0.6.0, >= 0.6.3) + winrm-elevated (1.2.3) + erubi (~> 1.8) winrm (~> 2.0) winrm-fs (~> 1.0) - winrm-fs (1.3.2) - erubis (~> 2.7) + winrm-fs (1.3.5) + erubi (~> 1.8) logging (>= 1.6.1, < 3.0) - rubyzip (~> 1.1) + rubyzip (~> 2.0) winrm (~> 2.0) wisper (2.0.1) @@ -148,9 +160,9 @@ PLATFORMS DEPENDENCIES cookstyle - kitchen-vagrant + kitchen-dokken serverspec test-kitchen BUNDLED WITH - 1.17.2 + 2.1.4 diff --git a/README.md b/README.md index 8707c7b46..65b41d609 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # OpenStreetMap chef cookbooks -[![Build Status](https://travis-ci.org/openstreetmap/chef.svg?branch=master)](https://travis-ci.org/openstreetmap/chef) +[![Cookstyle](https://github.com/openstreetmap/chef/workflows/Cookstyle/badge.svg?branch=master&event=push)](https://github.com/openstreetmap/chef/actions?query=workflow%3ACookstyle%20branch%3Amaster%20event%3Apush) +[![Test Kitchen](https://github.com/openstreetmap/chef/workflows/Test%20Kitchen/badge.svg?branch=master&event=push)](https://github.com/openstreetmap/chef/actions?query=workflow%3A%22Test+Kitchen%22%20branch%3Amaster%20event%3Apush) This repository manages the configuration of all the servers run by the OpenStreetMap Foundation's Operations Working Group. We use diff --git a/cookbooks/accounts/attributes/default.rb b/cookbooks/accounts/attributes/default.rb index 8535ee403..e529fe2fd 100644 --- a/cookbooks/accounts/attributes/default.rb +++ b/cookbooks/accounts/attributes/default.rb @@ -4,3 +4,4 @@ default[:accounts][:manage_home] = true default[:accounts][:groups] = {} default[:accounts][:users] = {} +default[:accounts][:administrators] = [] diff --git a/cookbooks/accounts/files/default/dmlu/.ssh/authorized_keys b/cookbooks/accounts/files/default/dmlu/.ssh/authorized_keys new file mode 100644 index 000000000..2170df582 --- /dev/null +++ b/cookbooks/accounts/files/default/dmlu/.ssh/authorized_keys @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAqX90M2sP8AYuO7pdH8yHwu6T2RtuLFG7vzgVTiDSaEq8XvgRdaRRIECXbotwDONaOUgjlFHLo7bKc6aj1sWq21rP6FO+nL0TrzpirdspmIeCDDzkGzCRJSIIOLOzVCY8E8+JLqosrpIzScnLmDFopVpvhzBDIVF9LzYaKZc8s+kiB74Q28qKFKAzBi4mfxxkSUXSIQ3UA/CchRUef6UtSJjl04b4vnnnCG+DVO4HzCurwZq/47VIe1xwAIFavTuHniBd/iaoc2tGtclD9Je5T1ja5NaWoh3pf8hPxNpTZmGc3fex3xkHkF1x5FMu2sscgB3R0fdRaNZ9vPqu4WyY2Q== dmlu diff --git a/cookbooks/accounts/files/default/grant/.gitconfig b/cookbooks/accounts/files/default/grant/.gitconfig new file mode 100644 index 000000000..4e9b2107c --- /dev/null +++ b/cookbooks/accounts/files/default/grant/.gitconfig @@ -0,0 +1,9 @@ +[user] + email = git@firefishy.com + name = Grant Slater +[push] + default = simple +[core] + editor = vim +[diff] + algorithm = histogram diff --git a/cookbooks/accounts/recipes/default.rb b/cookbooks/accounts/recipes/default.rb index 7eb760f3c..b395aeca6 100644 --- a/cookbooks/accounts/recipes/default.rb +++ b/cookbooks/accounts/recipes/default.rb @@ -19,7 +19,7 @@ package "zsh" -administrators = [] +administrators = node[:accounts][:administrators].to_a search(:accounts, "*:*").each do |account| name = account["id"] @@ -39,9 +39,15 @@ search(:accounts, "*:*").each do |account| user_shell = details[:shell] || account["shell"] || node[:accounts][:shell] end + available_users = if node[:etc] + node[:etc][:passwd].keys + else + [] + end + group name.to_s do gid account["uid"].to_i - members group_members & node[:etc][:passwd].keys + members group_members & available_users end user name.to_s do @@ -58,18 +64,16 @@ search(:accounts, "*:*").each do |account| source name.to_s owner name.to_s group name.to_s - mode 0o755 + mode "755" files_owner name.to_s files_group name.to_s - files_mode 0o644 + files_mode "644" only_if do - begin - cookbook = run_context.cookbook_collection[cookbook_name] - files = cookbook.relative_filenames_in_preferred_directory(node, :files, name.to_s) - !files.empty? - rescue Chef::Exceptions::FileNotFound - false - end + cookbook = run_context.cookbook_collection[cookbook_name] + files = cookbook.relative_filenames_in_preferred_directory(node, :files, name.to_s) + !files.empty? + rescue Chef::Exceptions::FileNotFound + false end end diff --git a/cookbooks/apache/attributes/default.rb b/cookbooks/apache/attributes/default.rb index cff5e5b83..080ca4e3a 100644 --- a/cookbooks/apache/attributes/default.rb +++ b/cookbooks/apache/attributes/default.rb @@ -1,4 +1,4 @@ -default[:apache][:mpm] = "worker" +default[:apache][:mpm] = "event" default[:apache][:timeout] = 300 diff --git a/cookbooks/apache/metadata.rb b/cookbooks/apache/metadata.rb index 06b231a1c..f6176ee8b 100644 --- a/cookbooks/apache/metadata.rb +++ b/cookbooks/apache/metadata.rb @@ -7,4 +7,5 @@ description "Installs and configures apache" version "1.0.0" supports "ubuntu" depends "munin" +depends "prometheus" depends "ssl" diff --git a/cookbooks/apache/recipes/default.rb b/cookbooks/apache/recipes/default.rb index ffa03c34c..d443f1e40 100644 --- a/cookbooks/apache/recipes/default.rb +++ b/cookbooks/apache/recipes/default.rb @@ -17,6 +17,8 @@ # limitations under the License. # +include_recipe "munin" +include_recipe "prometheus" include_recipe "ssl" package %w[ @@ -49,11 +51,13 @@ template "/etc/apache2/ports.conf" do source "ports.conf.erb" owner "root" group "root" - mode 0o644 + mode "644" end service "apache2" do action [:enable, :start] + retries 2 + retry_delay 10 supports :status => true, :restart => true, :reload => true end @@ -91,3 +95,9 @@ end munin_plugin "apache_accesses" munin_plugin "apache_processes" munin_plugin "apache_volume" + +prometheus_exporter "apache" do + port 9117 + listen_switch "telemetry.address" + options "--scrape_uri=http://localhost/server-status?auto" +end diff --git a/cookbooks/apache/resources/conf.rb b/cookbooks/apache/resources/conf.rb index 0ac19b0c7..12daf87ab 100644 --- a/cookbooks/apache/resources/conf.rb +++ b/cookbooks/apache/resources/conf.rb @@ -19,9 +19,9 @@ default_action [:create, :enable] -property :conf, :kind_of => String, :name_attribute => true +property :conf, :kind_of => String, :name_property => true property :cookbook, :kind_of => String -property :template, :kind_of => String, :required => true +property :template, :kind_of => String, :required => [:create] property :variables, :kind_of => Hash, :default => {} property :reload_apache, :kind_of => [TrueClass, FalseClass], :default => true @@ -48,7 +48,7 @@ action_class do source new_resource.template owner "root" group "root" - mode 0o644 + mode "644" variables new_resource.variables end end diff --git a/cookbooks/apache/resources/module.rb b/cookbooks/apache/resources/module.rb index 0ac0f8f01..6658d52a6 100644 --- a/cookbooks/apache/resources/module.rb +++ b/cookbooks/apache/resources/module.rb @@ -19,7 +19,7 @@ default_action [:install, :enable] -property :module, :kind_of => String, :name_attribute => true +property :module, :kind_of => String, :name_property => true property :package, :kind_of => String property :conf, :kind_of => String property :variables, :kind_of => Hash, :default => {} @@ -33,7 +33,7 @@ action :install do source new_resource.conf owner "root" group "root" - mode 0o644 + mode "644" variables new_resource.variables end end diff --git a/cookbooks/apache/resources/site.rb b/cookbooks/apache/resources/site.rb index 5b5230425..e039e3ac5 100644 --- a/cookbooks/apache/resources/site.rb +++ b/cookbooks/apache/resources/site.rb @@ -19,10 +19,10 @@ default_action [:create, :enable] -property :site, :kind_of => String, :name_attribute => true +property :site, :kind_of => String, :name_property => true property :directory, :kind_of => String property :cookbook, :kind_of => String -property :template, :kind_of => String, :required => true +property :template, :kind_of => String, :required => [:create] property :variables, :kind_of => Hash, :default => {} property :reload_apache, :kind_of => [TrueClass, FalseClass], :default => true @@ -32,7 +32,7 @@ action :create do source new_resource.template owner "root" group "root" - mode 0o644 + mode "644" variables new_resource.variables.merge(:name => new_resource.site, :directory => site_directory) end end diff --git a/cookbooks/apache/templates/default/httpd.conf.erb b/cookbooks/apache/templates/default/httpd.conf.erb index 81c8f6e9f..62533efae 100644 --- a/cookbooks/apache/templates/default/httpd.conf.erb +++ b/cookbooks/apache/templates/default/httpd.conf.erb @@ -50,6 +50,9 @@ MaxConnectionsPerChild <%= node[:apache][:event][:max_connections_per_child] %> <% if node[:apache][:event][:async_request_worker_factor] -%> AsyncRequestWorkerFactor <%= node[:apache][:event][:async_request_worker_factor] %> <% end -%> +<% if node[:apache][:event][:listen_cores_buckets_ratio] -%> +ListenCoresBucketsRatio <%= node[:apache][:event][:listen_cores_buckets_ratio] %> +<% end -%> <% end -%> # Default to UTF-8 diff --git a/cookbooks/apache/templates/default/ssl.erb b/cookbooks/apache/templates/default/ssl.erb index 80735c9cb..ccfef6048 100644 --- a/cookbooks/apache/templates/default/ssl.erb +++ b/cookbooks/apache/templates/default/ssl.erb @@ -13,4 +13,8 @@ SSLStaplingFakeTryLater off SSLStaplingCache shmcb:${APACHE_RUN_DIR}/ssl_ocspcache(512000) Header always set Strict-Transport-Security "<%= node[:ssl][:strict_transport_security] %>" "expr=%{HTTPS} == 'on'" -Header always set Expect-CT "max-age=0, report-uri=\"https://openstreetmap.report-uri.com/r/d/ct/reportOnly\"" "expr=%{HTTPS} == 'on' +<% if node[:ssl][:ct_report_uri] -%> +Header always set Expect-CT "max-age=0, report-uri=\"<%= node[:ssl][:ct_report_uri] %>\"" "expr=%{HTTPS} == 'on'" +<% else -%> +Header always set Expect-CT "max-age=0" "expr=%{HTTPS} == 'on'" +<% end -%> diff --git a/cookbooks/apt/attributes/default.rb b/cookbooks/apt/attributes/default.rb index f75ed9e34..a949a1c02 100644 --- a/cookbooks/apt/attributes/default.rb +++ b/cookbooks/apt/attributes/default.rb @@ -1,4 +1,3 @@ -default_unless[:apt][:sources] = [] - +default[:apt][:sources] = [ "openstreetmap" ] default[:apt][:unattended_upgrades][:enable] = true default[:apt][:unattended_upgrades][:remove_unused_dependencies] = true diff --git a/cookbooks/apt/recipes/default.rb b/cookbooks/apt/recipes/default.rb index 9aa2f7a82..6ab6afb7e 100644 --- a/cookbooks/apt/recipes/default.rb +++ b/cookbooks/apt/recipes/default.rb @@ -32,7 +32,7 @@ template "/etc/apt/preferences.d/99-chef" do source "preferences.erb" owner "root" group "root" - mode 0o644 + mode "644" end apt_update "/etc/apt/sources.list" do @@ -49,7 +49,7 @@ template "/etc/apt/sources.list" do source "sources.list.erb" owner "root" group "root" - mode 0o644 + mode "644" variables :archive_host => archive_host, :codename => node[:lsb][:codename] notifies :update, "apt_update[/etc/apt/sources.list]", :immediately end @@ -73,6 +73,16 @@ apt_repository "ubuntugis-unstable" do uri "ppa:ubuntugis/ubuntugis-unstable" end +apt_repository "git-core" do + action repository_actions["git-core"] + uri "ppa:git-core/ppa" +end + +apt_repository "maxmind" do + action repository_actions["maxmind"] + uri "ppa:maxmind/ppa" +end + apt_repository "openstreetmap" do action repository_actions["openstreetmap"] uri "ppa:osmadmins/ppa" @@ -96,7 +106,7 @@ end apt_repository "management-component-pack" do action repository_actions["management-component-pack"] uri "https://downloads.linux.hpe.com/SDR/repo/mcp" - distribution "#{node[:lsb][:codename]}/current-gen9" + distribution "bionic/current-gen9" components ["non-free"] key "C208ADDE26C2B797" end @@ -153,7 +163,28 @@ apt_repository "mediawiki" do uri "https://releases.wikimedia.org/debian" distribution "jessie-mediawiki" components ["main"] - key "90E9F83F22250DD7" + key "AF380A3036A03444" +end + +apt_repository "docker" do + action repository_actions["docker"] + uri "https://download.docker.com/linux/ubuntu" + arch "amd64" + components ["stable"] + key "https://download.docker.com/linux/ubuntu/gpg" +end + +apt_repository "grafana" do + action repository_actions["grafana"] + uri "https://packages.grafana.com/enterprise/deb" + distribution "stable" + components ["main"] + key "https://packages.grafana.com/gpg.key" +end + +apt_repository "timescaledb" do + action repository_actions["timescaledb"] + uri "ppa:timescale/timescaledb-ppa" end package "unattended-upgrades" @@ -168,7 +199,7 @@ if Dir.exist?("/usr/share/unattended-upgrades") file "/etc/apt/apt.conf.d/20auto-upgrades" do user "root" group "root" - mode 0o644 + mode "644" content auto_upgrades end end @@ -177,5 +208,5 @@ template "/etc/apt/apt.conf.d/60chef" do source "apt.conf.erb" owner "root" group "root" - mode 0o644 + mode "644" end diff --git a/cookbooks/apt/templates/default/apt.conf.erb b/cookbooks/apt/templates/default/apt.conf.erb index b0552bbc1..730678e58 100644 --- a/cookbooks/apt/templates/default/apt.conf.erb +++ b/cookbooks/apt/templates/default/apt.conf.erb @@ -3,3 +3,7 @@ // Do automatic removal of new unused dependencies after the upgrade // (equivalent to apt-get autoremove) Unattended-Upgrade::Remove-Unused-Dependencies "<%= node[:apt][:unattended_upgrades][:remove_unused_dependencies] ? 'true' : 'false' %>"; + +// Don't install recommended packages as we don't want to get +// new postgres versions automatically +APT::Install-Recommends "false"; diff --git a/cookbooks/apt/templates/default/sources.list.erb b/cookbooks/apt/templates/default/sources.list.erb index e572f7ced..d3d52ef7b 100644 --- a/cookbooks/apt/templates/default/sources.list.erb +++ b/cookbooks/apt/templates/default/sources.list.erb @@ -1,51 +1,49 @@ # DO NOT EDIT - This file is being maintained by Chef deb http://<%= @archive_host %>/ubuntu/ <%= @codename %> main restricted -deb-src http://<%= @archive_host %>/ubuntu/ <%= @codename %> main restricted +# deb-src http://<%= @archive_host %>/ubuntu/ <%= @codename %> main restricted ## Major bug fix updates produced after the final release of the ## distribution. deb http://<%= @archive_host %>/ubuntu/ <%= @codename %>-updates main restricted -deb-src http://<%= @archive_host %>/ubuntu/ <%= @codename %>-updates main restricted +# deb-src http://<%= @archive_host %>/ubuntu/ <%= @codename %>-updates main restricted ## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu ## team. Also, please note that software in universe WILL NOT receive any ## review or updates from the Ubuntu security team. deb http://<%= @archive_host %>/ubuntu/ <%= @codename %> universe -deb-src http://<%= @archive_host %>/ubuntu/ <%= @codename %> universe +# deb-src http://<%= @archive_host %>/ubuntu/ <%= @codename %> universe deb http://<%= @archive_host %>/ubuntu/ <%= @codename %>-updates universe -deb-src http://<%= @archive_host %>/ubuntu/ <%= @codename %>-updates universe +# deb-src http://<%= @archive_host %>/ubuntu/ <%= @codename %>-updates universe -## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu -## team, and may not be under a free licence. Please satisfy yourself as to -## your rights to use the software. Also, please note that software in +## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu +## team, and may not be under a free licence. Please satisfy yourself as to +## your rights to use the software. Also, please note that software in ## multiverse WILL NOT receive any review or updates from the Ubuntu ## security team. deb http://<%= @archive_host %>/ubuntu/ <%= @codename %> multiverse -deb-src http://<%= @archive_host %>/ubuntu/ <%= @codename %> multiverse +# deb-src http://<%= @archive_host %>/ubuntu/ <%= @codename %> multiverse deb http://<%= @archive_host %>/ubuntu/ <%= @codename %>-updates multiverse -deb-src http://<%= @archive_host %>/ubuntu/ <%= @codename %>-updates multiverse +# deb-src http://<%= @archive_host %>/ubuntu/ <%= @codename %>-updates multiverse -## Uncomment the following two lines to add software from the 'backports' -## repository. ## N.B. software from this repository may not have been tested as ## extensively as that contained in the main release, although it includes ## newer versions of some applications which may provide useful features. ## Also, please note that software in backports WILL NOT receive any review ## or updates from the Ubuntu security team. deb http://<%= @archive_host %>/ubuntu/ <%= @codename %>-backports main restricted universe multiverse -deb-src http://<%= @archive_host %>/ubuntu/ <%= @codename %>-backports main restricted universe multiverse +# deb-src http://<%= @archive_host %>/ubuntu/ <%= @codename %>-backports main restricted universe multiverse ## Uncomment the following two lines to add software from Canonical's -## 'partner' repository. This software is not part of Ubuntu, but is -## offered by Canonical and the respective vendors as a service to Ubuntu -## users. +## 'partner' repository. +## This software is not part of Ubuntu, but is offered by Canonical and the +## respective vendors as a service to Ubuntu users. # deb http://archive.canonical.com/ubuntu <%= @codename %> partner # deb-src http://archive.canonical.com/ubuntu <%= @codename %> partner -deb http://security.ubuntu.com/ubuntu <%= @codename %>-security main restricted -deb-src http://security.ubuntu.com/ubuntu <%= @codename %>-security main restricted -deb http://security.ubuntu.com/ubuntu <%= @codename %>-security universe -deb-src http://security.ubuntu.com/ubuntu <%= @codename %>-security universe -deb http://security.ubuntu.com/ubuntu <%= @codename %>-security multiverse -deb-src http://security.ubuntu.com/ubuntu <%= @codename %>-security multiverse +deb http://security.ubuntu.com/ubuntu/ <%= @codename %>-security main restricted +# deb-src http://security.ubuntu.com/ubuntu/ <%= @codename %>-security main restricted +deb http://security.ubuntu.com/ubuntu/ <%= @codename %>-security universe +# deb-src http://security.ubuntu.com/ubuntu/ <%= @codename %>-security universe +deb http://security.ubuntu.com/ubuntu/ <%= @codename %>-security multiverse +# deb-src http://security.ubuntu.com/ubuntu/ <%= @codename %>-security multiverse diff --git a/cookbooks/backup/attributes/default.rb b/cookbooks/backup/attributes/default.rb index c0b8de57f..2773238ab 100644 --- a/cookbooks/backup/attributes/default.rb +++ b/cookbooks/backup/attributes/default.rb @@ -1 +1,3 @@ default[:stats][:sites] = [] + +default[:accounts][:users][:osmbackup][:status] = :role diff --git a/cookbooks/backup/metadata.rb b/cookbooks/backup/metadata.rb index c4f1bdabd..d5baa1664 100644 --- a/cookbooks/backup/metadata.rb +++ b/cookbooks/backup/metadata.rb @@ -6,3 +6,4 @@ description "Installs and configures backup.openstreetmap.org" version "1.0.0" supports "ubuntu" +depends "accounts" diff --git a/cookbooks/backup/recipes/default.rb b/cookbooks/backup/recipes/default.rb index 0204f56f2..a123a19ab 100644 --- a/cookbooks/backup/recipes/default.rb +++ b/cookbooks/backup/recipes/default.rb @@ -17,6 +17,8 @@ # limitations under the License. # +include_recipe "accounts" + package %w[ perl libdate-calc-perl @@ -25,18 +27,19 @@ package %w[ directory "/store/backup" do owner "osmbackup" group "osmbackup" - mode 0o2755 + mode "2755" + recursive true end cookbook_file "/usr/local/bin/expire-backups" do owner "root" group "root" - mode 0o755 + mode "755" end template "/etc/cron.daily/expire-backups" do source "expire.cron.erb" owner "root" group "root" - mode 0o755 + mode "755" end diff --git a/cookbooks/backup/templates/default/expire.cron.erb b/cookbooks/backup/templates/default/expire.cron.erb index 9f80b24c0..dc4f7e15e 100644 --- a/cookbooks/backup/templates/default/expire.cron.erb +++ b/cookbooks/backup/templates/default/expire.cron.erb @@ -2,7 +2,7 @@ # DO NOT EDIT - This file is being maintained by Chef -for prefix in chef-server chef-repository chef-git forum git lists munin osm-blog osm-donate osmf-crm osmf-ledgersmb wiki-wiki.osmfoundation.org osqa otrs sotm svn switch2osm trac wiki-board.osmfoundation.org wiki-dwg.osmfoundation.org wiki-wiki.openstreetmap.org +for prefix in chef-server chef-repository chef-git forum git lists munin osm-blog osm-donate osmf-crm osmf-ledgersmb wiki-wiki.osmfoundation.org osqa otrs prometheus sotm svn switch2osm trac wiki-board.osmfoundation.org wiki-dwg.osmfoundation.org wiki-mwg.osmfoundation.org wiki-wiki.openstreetmap.org do /usr/local/bin/expire-backups --days=3 --weeks=3 --months=3 /store/backup $prefix done diff --git a/cookbooks/bind/recipes/default.rb b/cookbooks/bind/recipes/default.rb index 087ef5802..77c777e1a 100644 --- a/cookbooks/bind/recipes/default.rb +++ b/cookbooks/bind/recipes/default.rb @@ -31,34 +31,39 @@ end.flatten package "bind9" -service "bind9" do +service_name = if node[:lsb][:release].to_f < 20.04 + "bind9" + else + "named" + end + +service service_name do action [:enable, :start] - supports :status => true, :restart => true, :reload => true end template "/etc/bind/named.conf.local" do source "named.local.erb" owner "root" group "root" - mode 0o644 - notifies :restart, "service[bind9]" + mode "644" + notifies :restart, "service[#{service_name}]" end template "/etc/bind/named.conf.options" do source "named.options.erb" owner "root" group "root" - mode 0o644 + mode "644" variables :ipv4_clients => ipv4_clients, :ipv6_clients => ipv6_clients - notifies :restart, "service[bind9]" + notifies :restart, "service[#{service_name}]" end template "/etc/bind/db.10" do source "db.10.erb" owner "root" group "root" - mode 0o644 - notifies :reload, "service[bind9]" + mode "644" + notifies :reload, "service[#{service_name}]" end firewall_rule "accept-dns-udp" do diff --git a/cookbooks/blog/recipes/default.rb b/cookbooks/blog/recipes/default.rb index 789658448..2f80e2b13 100644 --- a/cookbooks/blog/recipes/default.rb +++ b/cookbooks/blog/recipes/default.rb @@ -24,7 +24,7 @@ passwords = data_bag_item("blog", "passwords") directory "/srv/blog.openstreetmap.org" do owner "wordpress" group "wordpress" - mode 0o755 + mode "755" end wordpress_site "blog.openstreetmap.org" do @@ -38,12 +38,13 @@ wordpress_site "blog.openstreetmap.org" do urls "/casts" => "/srv/blog.openstreetmap.org/casts", "/images" => "/srv/blog.openstreetmap.org/images", "/static" => "/srv/blog.openstreetmap.org/static" + fpm_prometheus_port 11401 end wordpress_theme "blog.openstreetmap.org-osmblog-wp-theme" do theme "osmblog-wp-theme" site "blog.openstreetmap.org" - repository "git://github.com/harry-wood/osmblog-wp-theme.git" + repository "https://github.com/harry-wood/osmblog-wp-theme.git" end wordpress_plugin "blog.openstreetmap.org-google-analytics-for-wordpress" do @@ -65,6 +66,7 @@ wordpress_plugin "blog.openstreetmap.org-sitepress-multilingual-cms" do plugin "sitepress-multilingual-cms" site "blog.openstreetmap.org" repository "https://git.openstreetmap.org/private/sitepress-multilingual-cms.git" + not_if { ENV["TEST_KITCHEN"] } end wordpress_plugin "blog.openstreetmap.org-wordpress-importer" do @@ -74,7 +76,7 @@ end git "/srv/blog.openstreetmap.org/casts" do action :sync - repository "git://github.com/openstreetmap/opengeodata-podcasts.git" + repository "https://github.com/openstreetmap/opengeodata-podcasts.git" depth 1 user "wordpress" group "wordpress" @@ -82,7 +84,7 @@ end git "/srv/blog.openstreetmap.org/images" do action :sync - repository "git://github.com/openstreetmap/opengeodata-images.git" + repository "https://github.com/openstreetmap/opengeodata-images.git" depth 1 user "wordpress" group "wordpress" @@ -90,7 +92,7 @@ end git "/srv/blog.openstreetmap.org/static" do action :sync - repository "git://github.com/openstreetmap/opengeodata-static.git" + repository "https://github.com/openstreetmap/opengeodata-static.git" depth 1 user "wordpress" group "wordpress" @@ -110,6 +112,6 @@ template "/etc/cron.daily/blog-backup" do source "backup.cron.erb" owner "root" group "root" - mode 0o750 + mode "750" variables :passwords => passwords end diff --git a/cookbooks/blog/templates/default/backup.cron.erb b/cookbooks/blog/templates/default/backup.cron.erb index 557440ff4..bc6159666 100644 --- a/cookbooks/blog/templates/default/backup.cron.erb +++ b/cookbooks/blog/templates/default/backup.cron.erb @@ -10,13 +10,12 @@ mkdir $T/osm-blog-$D echo '[mysqldump]' > $T/mysqldump.opts echo 'user=osm-blog-user' >> $T/mysqldump.opts echo 'password=<%= @passwords["osm-blog-user"] %>' >> $T/mysqldump.opts -mysqldump --defaults-file=$T/mysqldump.opts --opt osm-blog > $T/osm-blog-$D/osm-blog.sql +mysqldump --defaults-file=$T/mysqldump.opts --opt --no-tablespaces osm-blog > $T/osm-blog-$D/osm-blog.sql ln -s /srv/blog.openstreetmap.org $T/osm-blog-$D/www -export GZIP="--rsyncable -9" export RSYNC_RSH="ssh -ax" -nice tar --create --gzip --dereference --directory=$T --warning=no-file-changed --file=$T/$B osm-blog-$D +nice tar --create --dereference --directory=$T --warning=no-file-changed osm-blog-$D | nice gzip --rsyncable -9 > $T/$B nice rsync --preallocate --fuzzy $T/$B backup::backup rm -rf $T diff --git a/cookbooks/blogs/metadata.rb b/cookbooks/blogs/metadata.rb index 6f2b7dfc6..19d88bcfe 100644 --- a/cookbooks/blogs/metadata.rb +++ b/cookbooks/blogs/metadata.rb @@ -6,5 +6,6 @@ description "Configures server-info web site" version "1.0.0" supports "ubuntu" +depends "accounts" depends "apache" depends "git" diff --git a/cookbooks/blogs/recipes/default.rb b/cookbooks/blogs/recipes/default.rb index d08fc482c..165befb5e 100644 --- a/cookbooks/blogs/recipes/default.rb +++ b/cookbooks/blogs/recipes/default.rb @@ -17,45 +17,59 @@ # limitations under the License. # +include_recipe "accounts" include_recipe "apache" include_recipe "git" -package %w[ - ruby - ruby-dev +ruby_version = if node[:lsb][:release].to_f < 20.04 + "2.5" + else + "2.7" + end + +package %W[ + ruby#{ruby_version} + ruby#{ruby_version}-dev make gcc + g++ libsqlite3-dev ] -gem_package "bundler" +gem_package "bundler#{ruby_version}" do + package_name "bundler" + version "~> 2.1.4" + gem_binary "gem#{ruby_version}" + options "--format-executable" +end directory "/srv/blogs.openstreetmap.org" do owner "blogs" group "blogs" - mode 0o755 + mode "755" end git "/srv/blogs.openstreetmap.org" do action :sync - repository "git://github.com/gravitystorm/blogs.osm.org.git" + repository "https://github.com/gravitystorm/blogs.osm.org.git" + depth 1 user "blogs" group "blogs" - notifies :run, "execute[/srv/blogs.openstreetmap.org/Gemfile]", :immediate + notifies :run, "execute[/srv/blogs.openstreetmap.org/Gemfile]", :immediately end execute "/srv/blogs.openstreetmap.org/Gemfile" do action :nothing - command "bundle install" + command "bundle#{ruby_version} install --deployment" cwd "/srv/blogs.openstreetmap.org" - user "root" - group "root" - notifies :run, "execute[/srv/blogs.openstreetmap.org]", :immediate + user "blogs" + group "blogs" + notifies :run, "execute[/srv/blogs.openstreetmap.org]", :immediately end execute "/srv/blogs.openstreetmap.org" do action :nothing - command "bundle exec /usr/local/bin/pluto build -t osm -o build" + command "bundle#{ruby_version} exec pluto build -t osm -o build" cwd "/srv/blogs.openstreetmap.org" user "blogs" group "blogs" @@ -72,9 +86,17 @@ apache_site "blogs.openstreetmap.org" do variables :aliases => ["blogs.osm.org"] end -template "/etc/cron.d/blogs" do - source "cron.erb" +template "/usr/local/bin/blogs-update" do + source "blogs-update.erb" owner "root" group "root" - mode "0644" + mode "0755" + variables :ruby_version => ruby_version +end + +cron_d "blogs" do + minute "*/30" + user "blogs" + command "/usr/local/bin/blogs-update" + mailto "admins@openstreetmap.org" end diff --git a/cookbooks/blogs/templates/default/blogs-update.erb b/cookbooks/blogs/templates/default/blogs-update.erb new file mode 100644 index 000000000..ef880cdf8 --- /dev/null +++ b/cookbooks/blogs/templates/default/blogs-update.erb @@ -0,0 +1,18 @@ +#!/bin/sh + +cd /srv/blogs.openstreetmap.org + +/usr/local/bin/bundle<%= @ruby_version %> exec pluto \ + --quieter \ + --config=/srv/blogs.openstreetmap.org build \ + --dbpath=/srv/blogs.openstreetmap.org \ + --template=osm \ + --output=/srv/blogs.openstreetmap.org/build \ + /srv/blogs.openstreetmap.org/planet.ini > log.$$ 2>&1 + +if [ $? -ne 0 ] +then + cat log.$$ +fi + +rm -f log.$$ diff --git a/cookbooks/blogs/templates/default/cron.erb b/cookbooks/blogs/templates/default/cron.erb deleted file mode 100644 index c2d6dfb16..000000000 --- a/cookbooks/blogs/templates/default/cron.erb +++ /dev/null @@ -1,5 +0,0 @@ -# DO NOT EDIT - This file is being maintained by Chef - -MAILTO=admins@openstreetmap.org - -*/30 * * * * blogs cd /srv/blogs.openstreetmap.org; bundle exec /usr/local/bin/pluto --quieter --config=/srv/blogs.openstreetmap.org build --dbpath=/srv/blogs.openstreetmap.org --template=osm --output=/srv/blogs.openstreetmap.org/build /srv/blogs.openstreetmap.org/planet.ini > /dev/null diff --git a/cookbooks/cgiirc/README.md b/cookbooks/cgiirc/README.md deleted file mode 100644 index 49eeb3d54..000000000 --- a/cookbooks/cgiirc/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# Cgiirc Cookbook - -This cookbook installs and configures the `cgiirc` program that powers the -web interface to the OSM IRC channels, found at irc.openstreetmap.org diff --git a/cookbooks/cgiirc/templates/default/cgiirc.config.erb b/cookbooks/cgiirc/templates/default/cgiirc.config.erb deleted file mode 100644 index 794b4cb01..000000000 --- a/cookbooks/cgiirc/templates/default/cgiirc.config.erb +++ /dev/null @@ -1,24 +0,0 @@ -# DO NOT EDIT - This file is being maintained by Chef - -# CGI:IRC configuration file. -# -# Check /usr/share/doc/cgiirc/examples/cgiirc.config.full.gz -# for more details. - -# Configure defaults -default_server = irc.oftc.net -default_port = 6667 -default_channel = #osm,#osm-dev,#osm-ewg,#osm-cwg,#osm-ar,#osm-asia,#osm-au,#osm-br,#osm-bw,#osm-by,#osm-ca,#osm-ch,#osm-cz,#osm-de,#osm-dk,#osm-es,#osm-fi,#osm-fr,#osm-gb,#osm-gr,#osm-gsoc,#osm-ht,#osm-ie,#osm-it,#osm-ke,#osm-latam,#osm-local,#osm-lv,#osm-nl,#osm-no,#osm-nominatim,#osm-pl,#osm-pt,#osm-ru,#osm.se,#osm-strategic,#osm-ua,#osm-us,#osm-za,#osm-zh,#osmf-gm,#osrm,#openrailwaymap,#hot -default_name = CGI:IRC User -default_nick = CGI??? - -# Path to images -image_path = /images - -# Paths to CGI scripts -script_nph = nph-irc.cgi -script_form = client-perl.cgi -script_login = irc.cgi - -# Access control file -ip_access_file = ipaccess diff --git a/cookbooks/cgiirc/templates/default/ipaccess.erb b/cookbooks/cgiirc/templates/default/ipaccess.erb deleted file mode 100644 index ba2851cf2..000000000 --- a/cookbooks/cgiirc/templates/default/ipaccess.erb +++ /dev/null @@ -1,16 +0,0 @@ -# DO NOT EDIT - This file is being maintained by Chef - -# CGI:IRC ipaccess file. (For CGI:IRC versions from 0.5.3). -# -# Check /usr/share/doc/cgiirc/examples/ipaccess.example -# for more details. - -<% @blocks.each do |name,addresses| -%> -# Block <%= name %> -<% addresses.each do |address| -%> -<%= address %> 0 -<% end -%> - -<% end -%> -# Allow everybody. -0.0.0.0/0 diff --git a/cookbooks/chef/attributes/default.rb b/cookbooks/chef/attributes/default.rb index 2cac140f1..41b03e63a 100644 --- a/cookbooks/chef/attributes/default.rb +++ b/cookbooks/chef/attributes/default.rb @@ -5,4 +5,4 @@ default[:apt][:sources] = node[:apt][:sources] | ["opscode"] default[:chef][:server][:version] = "12.17.33" # Set the default client version -default[:chef][:client][:version] = "14.12.9" +default[:chef][:client][:version] = "16.8.14" diff --git a/cookbooks/chef/libraries/git.rb b/cookbooks/chef/libraries/git.rb new file mode 100644 index 000000000..176613d0a --- /dev/null +++ b/cookbooks/chef/libraries/git.rb @@ -0,0 +1,13 @@ +module OpenStreetMap + module Provider + module Git + def git(*args, **run_opts) + args.push("--force") if args.first == "fetch" && args.last == "--tags" + + super(args, **run_opts) + end + end + end +end + +Chef::Provider::Git.prepend(OpenStreetMap::Provider::Git) diff --git a/cookbooks/chef/libraries/persistent_token.rb b/cookbooks/chef/libraries/persistent_token.rb new file mode 100644 index 000000000..8299e8550 --- /dev/null +++ b/cookbooks/chef/libraries/persistent_token.rb @@ -0,0 +1,20 @@ +require "digest" + +class Chef + module Mixin + module PersistentToken + def persistent_token(*args) + sha256 = Digest::SHA256.new + sha256.update(node[:machine_id]) + args.each do |arg| + sha256.update(arg) + end + sha256.hexdigest + end + end + end + + class Recipe + include Chef::Mixin::PersistentToken + end +end diff --git a/cookbooks/chef/libraries/subversion.rb b/cookbooks/chef/libraries/subversion.rb index cbcce026a..8e9f1e3d2 100644 --- a/cookbooks/chef/libraries/subversion.rb +++ b/cookbooks/chef/libraries/subversion.rb @@ -44,6 +44,19 @@ class Chef command = scm(:info) shell_out!(command, run_options(:cwd => cwd, :returns => [0, 1])).stdout end + + def revision_int + @revision_int ||= begin + if new_resource.revision =~ /^\d+$/ + new_resource.revision + else + command = scm(:info, new_resource.repository, new_resource.svn_info_args, authentication, "-r#{new_resource.revision}") + svn_info = shell_out!(command, run_options(:returns => [0, 1])).stdout + + extract_revision_info(svn_info) + end + end + end end end end diff --git a/cookbooks/chef/recipes/default.rb b/cookbooks/chef/recipes/default.rb index f2f0101c0..61ac42415 100644 --- a/cookbooks/chef/recipes/default.rb +++ b/cookbooks/chef/recipes/default.rb @@ -17,17 +17,18 @@ # limitations under the License. # +cache_dir = Chef::Config[:file_cache_path] + chef_version = node[:chef][:client][:version] chef_package = "chef_#{chef_version}-1_amd64.deb" directory "/var/cache/chef" do - owner "root" - group "root" - mode 0o755 + action :delete + recursive true end -Dir.glob("/var/cache/chef/chef_*.deb").each do |deb| - next if deb == "/var/cache/chef/#{chef_package}" +Dir.glob("#{cache_dir}/chef_*.deb").each do |deb| + next if deb == "#{cache_dir}/#{chef_package}" file deb do action :delete @@ -35,91 +36,96 @@ Dir.glob("/var/cache/chef/chef_*.deb").each do |deb| end end -remote_file "/var/cache/chef/#{chef_package}" do +remote_file "#{cache_dir}/#{chef_package}" do source "https://packages.chef.io/files/stable/chef/#{chef_version}/ubuntu/#{node[:lsb][:release]}/#{chef_package}" owner "root" group "root" - mode 0o644 + mode "644" backup false ignore_failure true end dpkg_package "chef" do - source "/var/cache/chef/#{chef_package}" + source "#{cache_dir}/#{chef_package}" version "#{chef_version}-1" end directory "/etc/chef" do owner "root" group "root" - mode 0o755 + mode "755" end template "/etc/chef/client.rb" do source "client.rb.erb" owner "root" group "root" - mode 0o640 + mode "640" end file "/etc/chef/client.pem" do owner "root" group "root" - mode 0o400 + mode "400" end template "/etc/chef/report.rb" do source "report.rb.erb" owner "root" group "root" - mode 0o644 + mode "644" end template "/etc/logrotate.d/chef" do source "logrotate.erb" owner "root" group "root" - mode 0o644 + mode "644" end directory "/etc/chef/trusted_certs" do owner "root" group "root" - mode 0o755 + mode "755" end template "/etc/chef/trusted_certs/verisign.pem" do source "verisign.pem.erb" owner "root" group "root" - mode 0o644 + mode "644" end directory node[:ohai][:plugin_dir] do owner "root" group "root" - mode 0o755 + mode "755" end directory "/var/log/chef" do owner "root" group "root" - mode 0o755 + mode "755" end systemd_service "chef-client" do + description "Chef client" + exec_start "/usr/bin/chef-client" +end + +systemd_timer "chef-client" do description "Chef client" after "network.target" - exec_start "/usr/bin/chef-client -i 1800 -s 20" - restart "on-failure" + on_active_sec 60 + on_unit_inactive_sec 25 * 60 + randomized_delay_sec 10 * 60 end -service "chef-client" do +service "chef-client.timer" do action [:enable, :start] - restart_command "systemctl kill --signal=TERM chef-client.service" - supports :status => true, :restart => true, :reload => true - subscribes :restart, "dpkg_package[chef]" - subscribes :restart, "template[/etc/init/chef-client.conf]" - subscribes :restart, "template[/etc/chef/client.rb]" - subscribes :restart, "template[/etc/chef/report.rb]" +end + +service "chef-client.service" do + action :disable + subscribes :stop, "service[chef-client.timer]" end diff --git a/cookbooks/chef/recipes/repository.rb b/cookbooks/chef/recipes/repository.rb index 28d0b2f39..d5d8d2bc2 100644 --- a/cookbooks/chef/recipes/repository.rb +++ b/cookbooks/chef/recipes/repository.rb @@ -21,10 +21,14 @@ include_recipe "git" keys = data_bag_item("chef", "keys") +chef_gem "bundler" do + version ">= 2.1.4" +end + directory "/var/lib/chef" do owner "chefrepo" group "chefrepo" - mode 0o2775 + mode "2775" end %w[public private].each do |repository| @@ -41,28 +45,28 @@ end directory "/var/lib/chef/#{repository}/.chef" do owner "chefrepo" group "chefrepo" - mode 0o2775 + mode "2775" end file "/var/lib/chef/#{repository}/.chef/client.pem" do content keys["git"].join("\n") owner "chefrepo" group "chefrepo" - mode 0o660 + mode "660" end cookbook_file "/var/lib/chef/#{repository}/.chef/knife.rb" do source "knife.rb" owner "chefrepo" group "chefrepo" - mode 0o660 + mode "660" end template "#{repository_directory}/hooks/post-receive" do source "post-receive.erb" owner "chefrepo" group "chefrepo" - mode 0o750 + mode "750" variables :repository => repository end end diff --git a/cookbooks/chef/recipes/server.rb b/cookbooks/chef/recipes/server.rb index eb2a33ae4..cd4968b73 100644 --- a/cookbooks/chef/recipes/server.rb +++ b/cookbooks/chef/recipes/server.rb @@ -18,26 +18,23 @@ # include_recipe "apache" +include_recipe "munin" +# cache_dir = Chef::Config[:file_cache_path] +# # chef_version = node[:chef][:server][:version] # chef_package = "chef-server-core_#{chef_version}-1_amd64.deb" # -# directory "/var/cache/chef" do -# owner "root" -# group "root" -# mode 0755 -# end +# Dir.glob("#{cache_dir}/chef-server-core_*.deb").each do |deb| +# next if deb == "#{cache_dir}/#{chef_package}" # -# Dir.glob("/var/cache/chef/chef-server-core_*.deb").each do |deb| -# next if deb == "/var/cache/chef/#{chef_package}" - # file deb do # action :delete # backup false # end # end # -# remote_file "/var/cache/chef/#{chef_package}" do +# remote_file "#{cache_dir}/#{chef_package}" do # source "https://packages.chef.io/files/stable/chef-server/#{chef_version}/ubuntu/16.04/#{chef_package}" # owner "root" # group "root" @@ -46,7 +43,7 @@ include_recipe "apache" # end # # dpkg_package "chef-server-core" do -# source "/var/cache/chef/#{chef_package}" +# source "#{cache_dir}/#{chef_package}" # version "#{chef_version}-1" # notifies :run, "execute[chef-server-reconfigure]" # end @@ -55,7 +52,7 @@ template "/etc/opscode/chef-server.rb" do source "server.rb.erb" owner "root" group "root" - mode 0o640 + mode "640" notifies :run, "execute[chef-server-reconfigure]" end @@ -101,7 +98,7 @@ template "/etc/cron.daily/chef-server-backup" do source "server-backup.cron.erb" owner "root" group "root" - mode 0o755 + mode "755" end munin_plugin "chef_status" diff --git a/cookbooks/chef/templates/default/chef-client.conf.erb b/cookbooks/chef/templates/default/chef-client.conf.erb deleted file mode 100644 index 9414e05c9..000000000 --- a/cookbooks/chef/templates/default/chef-client.conf.erb +++ /dev/null @@ -1,16 +0,0 @@ -# chef-client -# -# Startup script for chef-client - -description "starts up chef-client in daemon mode" - -start on (net-device-up - and local-filesystems - and runlevel [2345]) -stop on runlevel [!2345] - -script - exec /usr/bin/chef-client -i 1800 -s 20 -end script - -respawn diff --git a/cookbooks/chef/templates/default/client.rb.erb b/cookbooks/chef/templates/default/client.rb.erb index 5d1f46393..796b276c5 100644 --- a/cookbooks/chef/templates/default/client.rb.erb +++ b/cookbooks/chef/templates/default/client.rb.erb @@ -9,6 +9,10 @@ Encoding.default_external = Encoding::UTF_8 +# Accept the license + +chef_license "accept" + # Log at level info log_level :info @@ -33,6 +37,10 @@ chef_server_url "https://chef.openstreetmap.org/organizations/openstreetmap" ohai.plugin_path << "<%= node[:ohai][:plugin_dir] %>" +# Enable some optional ohai plugins + +ohai.optional_plugins = %w[Passwd] + # Load supporting code for report handlers require "/etc/chef/report" diff --git a/cookbooks/chef/templates/default/report.rb.erb b/cookbooks/chef/templates/default/report.rb.erb index 20e32ef4f..4265d0660 100644 --- a/cookbooks/chef/templates/default/report.rb.erb +++ b/cookbooks/chef/templates/default/report.rb.erb @@ -31,6 +31,10 @@ class Chef require "mail" + Mail.defaults do + delivery_method :exim, :location => "/usr/sbin/exim" + end + Mail.deliver do to to_address from from_address diff --git a/cookbooks/chef/templates/default/server-backup.cron.erb b/cookbooks/chef/templates/default/server-backup.cron.erb index 048bc99e9..04e6c0e0c 100644 --- a/cookbooks/chef/templates/default/server-backup.cron.erb +++ b/cookbooks/chef/templates/default/server-backup.cron.erb @@ -10,9 +10,7 @@ chmod g+rwx $T $T/chef-server-$D sudo -u opscode-pgsql /opt/opscode/embedded/bin/pg_dumpall --file=$T/chef-server-$D/chef.dmp --clean ln -s /var/opt/opscode/bookshelf/data $T/chef-server-$D/bookshelf -export GZIP="--rsyncable -9" - -nice tar --create --gzip --dereference --directory=$T --file=$T/$B chef-server-$D +nice tar --create --dereference --directory=$T chef-server-$D | nice gzip --rsyncable -9 > $T/$B nice rsync --preallocate --fuzzy $T/$B backup::backup rm -rf $T diff --git a/cookbooks/civicrm/attributes/default.rb b/cookbooks/civicrm/attributes/default.rb index d273ae4bd..f0143f6f7 100644 --- a/cookbooks/civicrm/attributes/default.rb +++ b/cookbooks/civicrm/attributes/default.rb @@ -1,37 +1,37 @@ -default[:civicrm][:version] = "5.14.0" +default[:civicrm][:version] = "5.34.0" default[:civicrm][:extensions][:cividiscount][:name] = "org.civicrm.module.cividiscount" -default[:civicrm][:extensions][:cividiscount][:repository] = "git://github.com/dlobo/org.civicrm.module.cividiscount.git" -default[:civicrm][:extensions][:cividiscount][:revision] = "3.7" +default[:civicrm][:extensions][:cividiscount][:repository] = "https://github.com/dlobo/org.civicrm.module.cividiscount.git" +default[:civicrm][:extensions][:cividiscount][:revision] = "3.8.2" default[:civicrm][:extensions][:osm][:name] = "de.systopia.osm" -default[:civicrm][:extensions][:osm][:repository] = "git://github.com/systopia/de.systopia.osm.git" +default[:civicrm][:extensions][:osm][:repository] = "https://github.com/systopia/de.systopia.osm.git" default[:civicrm][:extensions][:osm][:revision] = "1.2.1" default[:civicrm][:extensions][:emailapi][:name] = "org.civicoop.emailapi" -default[:civicrm][:extensions][:emailapi][:repository] = "git://github.com/CiviCooP/org.civicoop.emailapi.git" +default[:civicrm][:extensions][:emailapi][:repository] = "https://github.com/CiviCooP/org.civicoop.emailapi.git" default[:civicrm][:extensions][:emailapi][:revision] = "1.19" default[:civicrm][:extensions][:civiruleshttppost][:name] = "org.civicoop.civiruleshttppost" -default[:civicrm][:extensions][:civiruleshttppost][:repository] = "git://github.com/CiviCooP/org.civicoop.civiruleshttppost.git" +default[:civicrm][:extensions][:civiruleshttppost][:repository] = "https://github.com/CiviCooP/org.civicoop.civiruleshttppost.git" default[:civicrm][:extensions][:civiruleshttppost][:revision] = "e2c7de5f0fee319b9fca8adb1d1e122202bd2bec" default[:civicrm][:extensions][:civirules][:name] = "org.civicoop.civirules" default[:civicrm][:extensions][:civirules][:repository] = "https://lab.civicrm.org/extensions/civirules.git" -default[:civicrm][:extensions][:civirules][:revision] = "2.7" - -default[:civicrm][:extensions][:stripe][:name] = "com.drastikbydesign.stripe" -default[:civicrm][:extensions][:stripe][:repository] = "git://github.com/drastik/com.drastikbydesign.stripe.git" -default[:civicrm][:extensions][:stripe][:revision] = "4.7.3" +default[:civicrm][:extensions][:civirules][:revision] = "2.22" default[:civicrm][:extensions][:mailchimp][:name] = "uk.co.vedaconsulting.mailchimp" -default[:civicrm][:extensions][:mailchimp][:repository] = "git://github.com/veda-consulting/uk.co.vedaconsulting.mailchimp.git" -default[:civicrm][:extensions][:mailchimp][:revision] = "v2.0.1" +default[:civicrm][:extensions][:mailchimp][:repository] = "https://github.com/veda-consulting/uk.co.vedaconsulting.mailchimp.git" +default[:civicrm][:extensions][:mailchimp][:revision] = "124083b29ab28246883bc83f207498a85f01ecde" default[:civicrm][:extensions][:username][:name] = "org.openstreetmap.username" -default[:civicrm][:extensions][:username][:repository] = "git://github.com/grischard/org.openstreetmap.username.git" +default[:civicrm][:extensions][:username][:repository] = "https://github.com/grischard/org.openstreetmap.username.git" default[:civicrm][:extensions][:username][:revision] = "master" default[:civicrm][:extensions][:donotsendreportemail][:name] = "org.civicrm.donotsendreportemail" -default[:civicrm][:extensions][:donotsendreportemail][:repository] = "git://github.com/pradpnayak/org.civicrm.donotsendreportemail.git" +default[:civicrm][:extensions][:donotsendreportemail][:repository] = "https://github.com/pradpnayak/org.civicrm.donotsendreportemail.git" default[:civicrm][:extensions][:donotsendreportemail][:revision] = "3b31c2e0c62183872c7ecd244395fb8dcfbd5dbb" + +default[:civicrm][:extensions][:shoreditch][:name] = "org.civicrm.shoreditch" +default[:civicrm][:extensions][:shoreditch][:repository] = "https://github.com/civicrm/org.civicrm.shoreditch.git" +default[:civicrm][:extensions][:shoreditch][:revision] = "1.0.0-beta.1" diff --git a/cookbooks/civicrm/recipes/default.rb b/cookbooks/civicrm/recipes/default.rb index 46c17d504..a8881fc0d 100644 --- a/cookbooks/civicrm/recipes/default.rb +++ b/cookbooks/civicrm/recipes/default.rb @@ -20,7 +20,16 @@ include_recipe "wordpress" include_recipe "mysql" -package "wkhtmltopdf" +package %w[ + php-xml + php-curl + rsync + unzip + wkhtmltopdf + php-bcmath +] + +cache_dir = Chef::Config[:file_cache_path] passwords = data_bag_item("civicrm", "passwords") @@ -40,11 +49,12 @@ wordpress_site "join.osmfoundation.org" do database_name "civicrm" database_user "civicrm" database_password database_password + fpm_prometheus_port 11301 end wordpress_theme "osmblog-wp-theme" do site "join.osmfoundation.org" - repository "git://github.com/harry-wood/osmblog-wp-theme.git" + repository "https://github.com/harry-wood/osmblog-wp-theme.git" end wordpress_plugin "registration-honeypot" do @@ -54,55 +64,60 @@ end wordpress_plugin "sitepress-multilingual-cms" do site "join.osmfoundation.org" repository "https://git.openstreetmap.org/private/sitepress-multilingual-cms.git" + not_if { ENV["TEST_KITCHEN"] } end wordpress_plugin "contact-form-7" do site "join.osmfoundation.org" end +wordpress_plugin "civicrm-admin-utilities" do + site "join.osmfoundation.org" +end + civicrm_version = node[:civicrm][:version] civicrm_directory = "/srv/join.osmfoundation.org/wp-content/plugins/civicrm" directory "/opt/civicrm-#{civicrm_version}" do owner "wordpress" group "wordpress" - mode 0o755 + mode "755" end -remote_file "/var/cache/chef/civicrm-#{civicrm_version}-wordpress.zip" do +remote_file "#{cache_dir}/civicrm-#{civicrm_version}-wordpress.zip" do action :create_if_missing source "https://download.civicrm.org/civicrm-#{civicrm_version}-wordpress.zip" owner "wordpress" group "wordpress" - mode 0o644 + mode "644" backup false end -remote_file "/var/cache/chef/civicrm-#{civicrm_version}-l10n.tar.gz" do +remote_file "#{cache_dir}/civicrm-#{civicrm_version}-l10n.tar.gz" do action :create_if_missing source "https://download.civicrm.org/civicrm-#{civicrm_version}-l10n.tar.gz" owner "wordpress" group "wordpress" - mode 0o644 + mode "644" backup false end -execute "/var/cache/chef/civicrm-#{civicrm_version}-wordpress.zip" do +execute "#{cache_dir}/civicrm-#{civicrm_version}-wordpress.zip" do action :nothing - command "unzip -qq /var/cache/chef/civicrm-#{civicrm_version}-wordpress.zip" + command "unzip -o -qq #{cache_dir}/civicrm-#{civicrm_version}-wordpress.zip" cwd "/opt/civicrm-#{civicrm_version}" user "wordpress" group "wordpress" - subscribes :run, "remote_file[/var/cache/chef/civicrm-#{civicrm_version}-wordpress.zip]", :immediately + subscribes :run, "remote_file[#{cache_dir}/civicrm-#{civicrm_version}-wordpress.zip]", :immediately end -execute "/var/cache/chef/civicrm-#{civicrm_version}-l10n.tar.gz" do +execute "#{cache_dir}/civicrm-#{civicrm_version}-l10n.tar.gz" do action :nothing - command "tar -zxf /var/cache/chef/civicrm-#{civicrm_version}-l10n.tar.gz" + command "tar -zxf #{cache_dir}/civicrm-#{civicrm_version}-l10n.tar.gz" cwd "/opt/civicrm-#{civicrm_version}/civicrm" user "wordpress" group "wordpress" - subscribes :run, "remote_file[/var/cache/chef/civicrm-#{civicrm_version}-l10n.tar.gz]", :immediately + subscribes :run, "remote_file[#{cache_dir}/civicrm-#{civicrm_version}-l10n.tar.gz]", :immediately end execute "/opt/civicrm-#{civicrm_version}/civicrm" do @@ -110,14 +125,14 @@ execute "/opt/civicrm-#{civicrm_version}/civicrm" do command "rsync --archive --delete /opt/civicrm-#{civicrm_version}/civicrm/ #{civicrm_directory}" user "wordpress" group "wordpress" - subscribes :run, "execute[/var/cache/chef/civicrm-#{civicrm_version}-wordpress.zip]", :immediately - subscribes :run, "execute[/var/cache/chef/civicrm-#{civicrm_version}-l10n.tar.gz]", :immediately + subscribes :run, "execute[#{cache_dir}/civicrm-#{civicrm_version}-wordpress.zip]", :immediately + subscribes :run, "execute[#{cache_dir}/civicrm-#{civicrm_version}-l10n.tar.gz]", :immediately end -directory "/srv/join.osmfoundation.org/wp-content/plugins/files" do +directory "/srv/join.osmfoundation.org/wp-content/uploads" do owner "www-data" group "www-data" - mode 0o755 + mode "755" end extensions_directory = "/srv/join.osmfoundation.org/wp-content/plugins/civicrm-extensions" @@ -125,7 +140,7 @@ extensions_directory = "/srv/join.osmfoundation.org/wp-content/plugins/civicrm-e directory extensions_directory do owner "wordpress" group "wordpress" - mode 0o755 + mode "755" end node[:civicrm][:extensions].each_value do |details| @@ -149,10 +164,9 @@ settings = edit_file "#{civicrm_directory}/civicrm/templates/CRM/common/civicrm. line.gsub!(/%%dbHost%%/, "localhost") line.gsub!(/%%dbName%%/, "civicrm") line.gsub!(/%%crmRoot%%/, "#{civicrm_directory}/civicrm/") - line.gsub!(/%%templateCompileDir%%/, "/srv/join.osmfoundation.org/wp-content/plugins/files/civicrm/templates_c/") + line.gsub!(/%%templateCompileDir%%/, "/srv/join.osmfoundation.org/wp-content/uploads/civicrm/templates_c/") line.gsub!(/%%baseURL%%/, "http://join.osmfoundation.org/") line.gsub!(/%%siteKey%%/, site_key) - line.gsub!(%r{// *(.*'ext_repo_url'.*)$}, "\\1") line.gsub!(%r{// *define\('CIVICRM_CMSDIR', '/path/to/install/root/'\);}, "define('CIVICRM_CMSDIR', '/srv/join.osmfoundation.org');") line @@ -161,22 +175,21 @@ end file "#{civicrm_directory}/civicrm.settings.php" do owner "wordpress" group "wordpress" - mode 0o644 + mode "644" content settings end -template "/etc/cron.d/osmf-crm" do - source "cron.erb" - owner "root" - group "root" - mode 0o600 - variables :directory => civicrm_directory, :passwords => passwords +cron_d "osmf-crm" do + minute "*/15" + user "www-data" + command "php #{civicrm_directory}/civicrm/bin/cli.php -s join.osmfoundation.org -u batch -p \"#{passwords['batch']}\" -e Job -a execute 2>&1 | egrep -v '^PHP (Deprecated|Warning):'" + mailto "admins@openstreetmap.org" end template "/etc/cron.daily/osmf-crm-backup" do source "backup.cron.erb" owner "root" group "root" - mode 0o750 + mode "750" variables :passwords => passwords end diff --git a/cookbooks/civicrm/templates/default/backup.cron.erb b/cookbooks/civicrm/templates/default/backup.cron.erb index 61bb4cd45..29fc2ba1b 100644 --- a/cookbooks/civicrm/templates/default/backup.cron.erb +++ b/cookbooks/civicrm/templates/default/backup.cron.erb @@ -10,13 +10,12 @@ mkdir $T/osmf-crm-$D echo '[mysqldump]' > $T/mysqldump.opts echo 'user=civicrm' >> $T/mysqldump.opts echo 'password=<%= @passwords["database"] %>' >> $T/mysqldump.opts -mysqldump --defaults-file=$T/mysqldump.opts --opt --skip-lock-tables civicrm > $T/osmf-crm-$D/civicrm.sql +mysqldump --defaults-file=$T/mysqldump.opts --opt --skip-lock-tables --no-tablespaces civicrm > $T/osmf-crm-$D/civicrm.sql ln -s /srv/join.osmfoundation.org $T/osmf-crm-$D/www -export GZIP="--rsyncable -9" export RSYNC_RSH="ssh -ax" -nice tar --create --gzip --dereference --directory=$T --file=$T/$B osmf-crm-$D +nice tar --create --dereference --directory=$T osmf-crm-$D | nice gzip --rsyncable -9 > $T/$B nice rsync --preallocate --fuzzy $T/$B backup::backup rm -rf $T diff --git a/cookbooks/civicrm/templates/default/cron.erb b/cookbooks/civicrm/templates/default/cron.erb deleted file mode 100644 index 22c363169..000000000 --- a/cookbooks/civicrm/templates/default/cron.erb +++ /dev/null @@ -1,5 +0,0 @@ -# DO NOT EDIT - This file is being maintained by Chef - -MAILTO=admins@openstreetmap.org - -*/15 * * * * www-data php <%= @directory %>/civicrm/bin/cli.php -s join.osmfoundation.org -u batch -p "<%= @passwords["batch"] %>" -e Job -a execute 2>&1 | egrep -v '^PHP (Deprecated|Warning):' diff --git a/cookbooks/clamav/recipes/default.rb b/cookbooks/clamav/recipes/default.rb index 8e07f4fb7..f6dd24a5c 100644 --- a/cookbooks/clamav/recipes/default.rb +++ b/cookbooks/clamav/recipes/default.rb @@ -27,7 +27,17 @@ template "/etc/clamav-unofficial-sigs.conf.d/50-chef.conf" do source "clamav-unofficial-sigs.conf.erb" owner "root" group "root" - mode 0o644 + mode "644" +end + +execute "freshclam" do + command "/usr/bin/freshclam" + user "clamav" + group "clamav" + not_if do + ::File.exist?("/var/lib/clamav/daily.cld") || + ::File.exist?("/var/lib/clamav/daily.cvd") + end end service "clamav-daemon" do diff --git a/cookbooks/db/attributes/default.rb b/cookbooks/db/attributes/default.rb new file mode 100644 index 000000000..1b674e3ad --- /dev/null +++ b/cookbooks/db/attributes/default.rb @@ -0,0 +1,3 @@ +default[:db][:cluster] = "9.5/main" + +default[:postgresql][:versions] |= ["9.5"] diff --git a/cookbooks/db/metadata.rb b/cookbooks/db/metadata.rb index 092949cb0..aadfe47db 100644 --- a/cookbooks/db/metadata.rb +++ b/cookbooks/db/metadata.rb @@ -6,7 +6,8 @@ description "Installs and configures database servers" version "1.0.0" supports "ubuntu" -depends "postgresql" -depends "web" +depends "accounts" depends "git" +depends "postgresql" depends "python" +depends "web" diff --git a/cookbooks/db/recipes/backup.rb b/cookbooks/db/recipes/backup.rb index 5a814392a..0fe004d20 100644 --- a/cookbooks/db/recipes/backup.rb +++ b/cookbooks/db/recipes/backup.rb @@ -21,12 +21,14 @@ template "/usr/local/bin/backup-db" do source "backup-db.erb" owner "root" group "root" - mode 0o755 + mode "755" end -template "/etc/cron.d/backup-db" do - source "backup.cron.erb" - owner "root" - group "root" - mode 0o644 +cron_d "backup-db" do + minute "00" + hour "02" + weekday "1" + user "osmbackup" + command "/usr/local/bin/backup-db" + mailto "admins@openstreetmap.org" end diff --git a/cookbooks/db/recipes/base.rb b/cookbooks/db/recipes/base.rb index ee9d0089f..617392701 100644 --- a/cookbooks/db/recipes/base.rb +++ b/cookbooks/db/recipes/base.rb @@ -17,13 +17,19 @@ # limitations under the License. # -include_recipe "postgresql" +include_recipe "accounts" include_recipe "git" +include_recipe "postgresql" include_recipe "python" passwords = data_bag_item("db", "passwords") wal_secrets = data_bag_item("db", "wal-secrets") +ruby_version = node[:passenger][:ruby_version] +db_version = node[:db][:cluster].split("/").first +pg_config = "/usr/lib/postgresql/#{db_version}/bin/pg_config" +function_directory = "/srv/www.openstreetmap.org/rails/db/functions/#{db_version}" + postgresql_munin "openstreetmap" do cluster node[:db][:cluster] database "openstreetmap" @@ -31,16 +37,17 @@ end directory "/srv/www.openstreetmap.org" do group "rails" - mode 0o2775 + mode "2775" end rails_port "www.openstreetmap.org" do - ruby "2.5" + ruby ruby_version directory "/srv/www.openstreetmap.org/rails" user "rails" group "rails" repository "https://git.openstreetmap.org/public/rails.git" revision "live" + build_assets false database_host "localhost" database_name "openstreetmap" database_username "openstreetmap" @@ -48,19 +55,15 @@ rails_port "www.openstreetmap.org" do gpx_dir "/store/rails/gpx" end -db_version = node[:db][:cluster].split("/").first -pg_config = "/usr/lib/postgresql/#{db_version}/bin/pg_config" -function_directory = "/srv/www.openstreetmap.org/rails/db/functions/#{db_version}" - directory function_directory do owner "rails" group "rails" - mode 0o755 + mode "755" end execute function_directory do action :nothing - command "make PG_CONFIG=#{pg_config} DESTDIR=#{function_directory}" + command "make BUNDLE=bundle#{ruby_version} PG_CONFIG=#{pg_config} DESTDIR=#{function_directory}" cwd "/srv/www.openstreetmap.org/rails/db/functions" user "rails" group "rails" @@ -74,6 +77,58 @@ link "/usr/lib/postgresql/#{db_version}/lib/libpgosm.so" do group "root" end +package %w[ + cmake + libosmium2-dev + libprotozero-dev + libboost-filesystem-dev + libboost-program-options-dev + libbz2-dev + zlib1g-dev + libexpat1-dev + libyaml-cpp-dev + libpqxx-dev +] + +git "/opt/osmdbt" do + action :sync + repository "https://github.com/openstreetmap/osmdbt.git" + revision "v0.2" + depth 1 + user "root" + group "root" +end + +directory "/opt/osmdbt/build-#{db_version}" do + owner "root" + group "root" + mode "755" +end + +execute "/opt/osmdbt/CMakeLists.txt" do + action :nothing + command "cmake -DPG_CONFIG=/usr/lib/postgresql/#{db_version}/bin/pg_config .." + cwd "/opt/osmdbt/build-#{db_version}" + user "root" + group "root" + subscribes :run, "git[/opt/osmdbt]" +end + +execute "/opt/osmdbt/build-#{db_version}/postgresql-plugin/Makefile" do + action :nothing + command "make" + cwd "/opt/osmdbt/build-#{db_version}/postgresql-plugin" + user "root" + group "root" + subscribes :run, "execute[/opt/osmdbt/CMakeLists.txt]" +end + +link "/usr/lib/postgresql/#{db_version}/lib/osm-logical.so" do + to "/opt/osmdbt/build-#{db_version}/postgresql-plugin/osm-logical.so" + owner "root" + group "root" +end + package "lzop" python_package "wal-e" do @@ -88,6 +143,6 @@ template "/usr/local/bin/openstreetmap-wal-e" do source "wal-e.erb" owner "root" group "postgres" - mode 0o750 + mode "750" variables :s3_key => wal_secrets["s3_key"] end diff --git a/cookbooks/db/recipes/master.rb b/cookbooks/db/recipes/master.rb index 5e51ca030..f6641fd18 100644 --- a/cookbooks/db/recipes/master.rb +++ b/cookbooks/db/recipes/master.rb @@ -49,6 +49,7 @@ end postgresql_user "planetdiff" do cluster node[:db][:cluster] password passwords["planetdiff"] + replication true end postgresql_user "backup" do diff --git a/cookbooks/db/templates/default/backup.cron.erb b/cookbooks/db/templates/default/backup.cron.erb deleted file mode 100644 index b7249c7bc..000000000 --- a/cookbooks/db/templates/default/backup.cron.erb +++ /dev/null @@ -1,5 +0,0 @@ -# DO NOT EDIT - This file is being maintained by Chef - -MAILTO=admins@openstreetmap.org - -00 02 * * 1 osmbackup /usr/local/bin/backup-db diff --git a/cookbooks/db/templates/default/wal-e.erb b/cookbooks/db/templates/default/wal-e.erb index 93f314498..b4c13bd1d 100644 --- a/cookbooks/db/templates/default/wal-e.erb +++ b/cookbooks/db/templates/default/wal-e.erb @@ -7,4 +7,4 @@ export AWS_ACCESS_KEY_ID="AKIAIQX2LTDOBIW4CZUQ" export AWS_SECRET_ACCESS_KEY="<%= @s3_key %>" export AWS_REGION="eu-west-2" -exec /usr/local/bin/wal-e "$@" +exec /usr/local/bin/wal-e "$@" < /dev/null diff --git a/cookbooks/dev/files/default/ooc/index.html b/cookbooks/dev/files/default/ooc/index.html index e7208b0ae..d6aa06212 100644 --- a/cookbooks/dev/files/default/ooc/index.html +++ b/cookbooks/dev/files/default/ooc/index.html @@ -6,7 +6,7 @@ - + diff --git a/cookbooks/dev/metadata.rb b/cookbooks/dev/metadata.rb index bcd716e41..457e46dd0 100644 --- a/cookbooks/dev/metadata.rb +++ b/cookbooks/dev/metadata.rb @@ -8,10 +8,14 @@ version "1.0.0" supports "ubuntu" depends "apache" depends "passenger" +depends "chef" +depends "geoipupdate" depends "git" depends "memcached" +depends "munin" depends "mysql" depends "nodejs" +depends "php" depends "postgresql" depends "python" depends "tools" diff --git a/cookbooks/dev/recipes/default.rb b/cookbooks/dev/recipes/default.rb index a5729160f..e4a977f8a 100644 --- a/cookbooks/dev/recipes/default.rb +++ b/cookbooks/dev/recipes/default.rb @@ -22,20 +22,21 @@ require "securerandom" include_recipe "apache" include_recipe "passenger" +include_recipe "geoipupdate" include_recipe "git" include_recipe "memcached" +include_recipe "munin" include_recipe "mysql" include_recipe "nodejs" +include_recipe "php::fpm" include_recipe "postgresql" include_recipe "python" package %w[ - php php-cgi php-cli php-curl php-db - php-fpm php-imagick php-mysql php-pear @@ -43,14 +44,13 @@ package %w[ php-sqlite3 pngcrush pngquant - python - python-argparse - python-beautifulsoup - python-cheetah - python-dateutil - python-magic - python-psycopg2 - python-gdal + python3 + python3-bs4 + python3-cheetah + python3-dateutil + python3-magic + python3-psycopg2 + python3-gdal g++ gcc make @@ -74,7 +74,9 @@ package %w[ nodejs_package "svgo" -python_package "geojson" +python_package "geojson" do + python_version "3" +end apache_module "env" apache_module "expires" @@ -84,38 +86,35 @@ apache_module "proxy_fcgi" apache_module "rewrite" apache_module "suexec" apache_module "userdir" -apache_module "wsgi" - -package "apache2-suexec-pristine" -service "php7.2-fpm" do - action [:enable, :start] +apache_module "wsgi" do + package "libapache2-mod-wsgi-py3" end -template "/etc/php/7.2/fpm/pool.d/default.conf" do - source "fpm-default.conf.erb" - owner "root" - group "root" - mode 0o644 - notifies :reload, "service[php7.2-fpm]" +package "apache2-suexec-pristine" + +php_fpm "default" do + pm_max_children 10 + pm_start_servers 4 + pm_min_spare_servers 2 + pm_max_spare_servers 6 end -file "/etc/php/7.2/fpm/pool.d/www.conf" do +php_fpm "www" do action :delete - notifies :reload, "service[php7.2-fpm]" end directory "/srv/dev.openstreetmap.org" do owner "root" group "root" - mode 0o755 + mode "755" end template "/srv/dev.openstreetmap.org/index.html" do source "dev.html.erb" owner "root" group "root" - mode 0o644 + mode "644" end ssl_certificate "dev.openstreetmap.org" do @@ -133,7 +132,7 @@ template "/etc/phppgadmin/config.inc.php" do source "phppgadmin.conf.erb" owner "root" group "root" - mode 0o644 + mode "644" end file "/etc/apache2/conf.d/phppgadmin" do @@ -159,15 +158,27 @@ search(:accounts, "*:*").each do |account| next unless File.directory?("#{user_home}/public_html") - port = 7000 + account["uid"].to_i - - template "/etc/php/7.2/fpm/pool.d/#{name}.conf" do - source "fpm.conf.erb" - owner "root" - group "root" - mode 0o644 - variables :user => name, :port => port - notifies :reload, "service[php7.2-fpm]" + php_fpm name do + user name + group name + pm_max_children 10 + pm_start_servers 4 + pm_min_spare_servers 2 + pm_max_spare_servers 6 + pm_max_requests 10000 + request_terminate_timeout 1800 + environment "HOSTNAME" => "$HOSTNAME", + "PATH" => "/usr/local/bin:/usr/bin:/bin", + "TMP" => "/tmp", + "TMPDIR" => "/tmp", + "TEMP" => "/tmp" + php_values "max_execution_time" => "300", + "memory_limit" => "128M", + "post_max_size" => "32M", + "upload_max_filesize" => "32M" + php_admin_values "sendmail_path" => "/usr/sbin/sendmail -t -i -f #{name}@errol.openstreetmap.org", + "open_basedir" => "/home/#{name}/:/tmp/:/usr/share/php/" + php_flags "display_errors" => "on" end ssl_certificate "#{name}.dev.openstreetmap.org" do @@ -178,21 +189,21 @@ search(:accounts, "*:*").each do |account| apache_site "#{name}.dev.openstreetmap.org" do template "apache.user.erb" directory "#{user_home}/public_html" - variables :user => name, :port => port + variables :user => name end template "/etc/sudoers.d/#{name}" do source "sudoers.user.erb" owner "root" group "root" - mode 0o440 + mode "440" variables :user => name end end -if node[:postgresql][:clusters][:"9.5/main"] +if node[:postgresql][:clusters][:"12/main"] postgresql_user "apis" do - cluster "9.5/main" + cluster "12/main" end template "/usr/local/bin/cleanup-rails-assets" do @@ -200,7 +211,7 @@ if node[:postgresql][:clusters][:"9.5/main"] source "cleanup-assets.erb" owner "root" group "root" - mode 0o755 + mode "755" end ruby_version = node[:passenger][:ruby_version] @@ -247,17 +258,15 @@ if node[:postgresql][:clusters][:"9.5/main"] if details[:repository] site_aliases = details[:aliases] || [] - secret_key_base = details[:secret_key_base] || SecureRandom.base64(96) - - node.normal[:dev][:rails][name][:secret_key_base] = secret_key_base + secret_key_base = persistent_token("dev", "rails", name, "secret_key_base") postgresql_database database_name do - cluster "9.5/main" + cluster "12/main" owner "apis" end postgresql_extension "#{database_name}_btree_gist" do - cluster "9.5/main" + cluster "12/main" database database_name extension "btree_gist" end @@ -265,31 +274,31 @@ if node[:postgresql][:clusters][:"9.5/main"] directory site_directory do owner "apis" group "apis" - mode 0o755 + mode "755" end directory log_directory do owner "apis" group "apis" - mode 0o755 + mode "755" end directory gpx_directory do owner "apis" group "apis" - mode 0o755 + mode "755" end directory "#{gpx_directory}/traces" do owner "apis" group "apis" - mode 0o755 + mode "755" end directory "#{gpx_directory}/images" do owner "apis" group "apis" - mode 0o755 + mode "755" end rails_port site_name do @@ -299,9 +308,10 @@ if node[:postgresql][:clusters][:"9.5/main"] group "apis" repository details[:repository] revision details[:revision] - database_port node[:postgresql][:clusters][:"9.5/main"][:port] + database_port node[:postgresql][:clusters][:"12/main"][:port] database_name database_name database_username "apis" + email_from "OpenStreetMap " gpx_dir gpx_directory log_path "#{log_directory}/rails.log" memcache_servers ["127.0.0.1"] @@ -314,7 +324,7 @@ if node[:postgresql][:clusters][:"9.5/main"] source "rails.setup.rb.erb" owner "apis" group "apis" - mode 0o644 + mode "644" variables :site => site_name notifies :restart, "rails_port[#{site_name}]" end @@ -368,9 +378,9 @@ if node[:postgresql][:clusters][:"9.5/main"] source "cgimap.environment.erb" owner "root" group "root" - mode 0o640 + mode "640" variables :cgimap_port => cgimap_port, - :database_port => node[:postgresql][:clusters][:"9.5/main"][:port], + :database_port => node[:postgresql][:clusters][:"12/main"][:port], :database_name => database_name, :log_directory => log_directory notifies :restart, "service[cgimap@#{name}]" @@ -399,7 +409,7 @@ if node[:postgresql][:clusters][:"9.5/main"] source "logrotate.apis.erb" owner "root" group "root" - mode 0o644 + mode "644" variables :name => name, :log_directory => log_directory, :rails_directory => rails_directory @@ -434,24 +444,22 @@ if node[:postgresql][:clusters][:"9.5/main"] postgresql_database database_name do action :drop - cluster "9.5/main" + cluster "12/main" end - - node.normal[:dev][:rails].delete(name) end end directory "/srv/apis.dev.openstreetmap.org" do owner "apis" group "apis" - mode 0o755 + mode "755" end template "/srv/apis.dev.openstreetmap.org/index.html" do source "apis.html.erb" owner "apis" group "apis" - mode 0o644 + mode "644" end ssl_certificate "apis.dev.openstreetmap.org" do @@ -474,17 +482,17 @@ end directory "/srv/ooc.openstreetmap.org" do owner "root" group "root" - mode 0o755 + mode "755" end remote_directory "/srv/ooc.openstreetmap.org/html" do source "ooc" owner "root" group "root" - mode 0o755 + mode "755" files_owner "root" files_group "root" - files_mode 0o644 + files_mode "644" end ssl_certificate "ooc.openstreetmap.org" do diff --git a/cookbooks/dev/templates/default/apache.phppgadmin.erb b/cookbooks/dev/templates/default/apache.phppgadmin.erb index ec74a54da..2965946e2 100644 --- a/cookbooks/dev/templates/default/apache.phppgadmin.erb +++ b/cookbooks/dev/templates/default/apache.phppgadmin.erb @@ -16,9 +16,9 @@ # Remove Proxy request header to mitigate https://httpoxy.org/ RequestHeader unset Proxy early - ProxyPassMatch ^/(.*\.php(/.*)?)$ fcgi://127.0.0.1:7000/usr/share/phppgadmin - ProxyPassMatch ^/(.*\.phpx(/.*)?)$ fcgi://127.0.0.1:7000/usr/share/phppgadmin - ProxyPassMatch ^/(.*\.phpj(/.*)?)$ fcgi://127.0.0.1:7000/usr/share/phppgadmin + ProxyPassMatch ^/(.*\.php(/.*)?)$ unix:/run/php/default.sock|fcgi://127.0.0.1 + ProxyPassMatch ^/(.*\.phpx(/.*)?)$ unix:/run/php/default.sock|fcgi://127.0.0.1 + ProxyPassMatch ^/(.*\.phpj(/.*)?)$ unix:/run/php/default.sock|fcgi://127.0.0.1 diff --git a/cookbooks/dev/templates/default/apache.rails.erb b/cookbooks/dev/templates/default/apache.rails.erb index 3c7eca7e4..cf2a56613 100644 --- a/cookbooks/dev/templates/default/apache.rails.erb +++ b/cookbooks/dev/templates/default/apache.rails.erb @@ -41,15 +41,15 @@ # Pass supported calls to cgimap RewriteEngine on - RewriteRule ^/api/0\.6/map(\.json)?$ fcgi://127.0.0.1:<%= @cgimap_port %>$0 [P] + RewriteRule ^/api/0\.6/map(\.json|\.xml)?$ fcgi://127.0.0.1:<%= @cgimap_port %>$0 [P] RewriteCond %{REQUEST_METHOD} ^(HEAD|GET)$ - RewriteRule ^/api/0\.6/(node|way|relation|changeset)/[0-9]+(\.json)?$ fcgi://127.0.0.1:<%= @cgimap_port %>$0 [P] - RewriteRule ^/api/0\.6/(node|way|relation)/[0-9]+/history(\.json)?$ fcgi://127.0.0.1:<%= @cgimap_port %>$0 [P] - RewriteRule ^/api/0\.6/(node|way|relation)/[0-9]+/relations(\.json)?$ fcgi://127.0.0.1:<%= @cgimap_port %>$0 [P] - RewriteRule ^/api/0\.6/node/[0-9]+/ways(\.json)?$ fcgi://127.0.0.1:<%= @cgimap_port %>$0 [P] - RewriteRule ^/api/0\.6/(way|relation)/[0-9]+/full(\.json)?$ fcgi://127.0.0.1:<%= @cgimap_port %>$0 [P] - RewriteRule ^/api/0\.6/(nodes|ways|relations)(\.json)?$ fcgi://127.0.0.1:<%= @cgimap_port %>$0 [P] - RewriteRule ^/api/0\.6/changeset/[0-9]+/(upload|download)(\.json)?$ fcgi://127.0.0.1:<%= @cgimap_port %>$0 [P] + RewriteRule ^/api/0\.6/(node|way|relation|changeset)/[0-9]+(\.json|\.xml)?$ fcgi://127.0.0.1:<%= @cgimap_port %>$0 [P] + RewriteRule ^/api/0\.6/(node|way|relation)/[0-9]+/history(\.json|\.xml)?$ fcgi://127.0.0.1:<%= @cgimap_port %>$0 [P] + RewriteRule ^/api/0\.6/(node|way|relation)/[0-9]+/relations(\.json|\.xml)?$ fcgi://127.0.0.1:<%= @cgimap_port %>$0 [P] + RewriteRule ^/api/0\.6/node/[0-9]+/ways(\.json|\.xml)?$ fcgi://127.0.0.1:<%= @cgimap_port %>$0 [P] + RewriteRule ^/api/0\.6/(way|relation)/[0-9]+/full(\.json|\.xml)?$ fcgi://127.0.0.1:<%= @cgimap_port %>$0 [P] + RewriteRule ^/api/0\.6/(nodes|ways|relations)(\.json|\.xml)?$ fcgi://127.0.0.1:<%= @cgimap_port %>$0 [P] + RewriteRule ^/api/0\.6/changeset/[0-9]+/(upload|download)(\.json|\.xml)?$ fcgi://127.0.0.1:<%= @cgimap_port %>$0 [P] <% end -%> diff --git a/cookbooks/dev/templates/default/apache.user.erb b/cookbooks/dev/templates/default/apache.user.erb index a6d4d8866..258f120d3 100644 --- a/cookbooks/dev/templates/default/apache.user.erb +++ b/cookbooks/dev/templates/default/apache.user.erb @@ -1,6 +1,6 @@ # DO NOT EDIT - This file is being maintained by Chef -WSGIDaemonProcess <%= @user %>.dev.openstreetmap.org user=<%= @user %> inactivity-timeout=600 +WSGIDaemonProcess <%= @user %>.dev.openstreetmap.org user=<%= @user %> processes=2 threads=8 restart-interval=3600 inactivity-timeout=600 graceful-timeout=60 maximum-requests=2000 ServerName <%= @user %>.dev.openstreetmap.org @@ -29,8 +29,9 @@ WSGIDaemonProcess <%= @user %>.dev.openstreetmap.org user=<%= @user %> inactivit RewriteCond <%= @directory %>%{REQUEST_FILENAME} -f RewriteRule ^/cgi-bin/(.*)$ /~<%= @user %>/cgi-bin/$1 [PT,L] - RewriteCond <%= @directory %>%{REQUEST_FILENAME} -f - RewriteRule ^/(.*\.ph(p|ps|p3|tml)(/.*)?)$ fcgi://127.0.0.1:<%= @port %><%= @directory %>/$1 [P] + + SetHandler "proxy:unix:/run/php/<%= @user %>.sock|fcgi://127.0.0.1" + diff --git a/cookbooks/dev/templates/default/fpm-default.conf.erb b/cookbooks/dev/templates/default/fpm-default.conf.erb deleted file mode 100644 index fe7ee4c40..000000000 --- a/cookbooks/dev/templates/default/fpm-default.conf.erb +++ /dev/null @@ -1,384 +0,0 @@ -; Start a new pool named 'default'. -; the variable $pool can we used in any directive and will be replaced by the -; pool name ('default' here) -[default] - -; Per pool prefix -; It only applies on the following directives: -; - 'slowlog' -; - 'listen' (unixsocket) -; - 'chroot' -; - 'chdir' -; - 'php_values' -; - 'php_admin_values' -; When not set, the global prefix (or /usr) applies instead. -; Note: This directive can also be relative to the global prefix. -; Default Value: none -;prefix = /path/to/pools/$pool - -; Unix user/group of processes -; Note: The user is mandatory. If the group is not set, the default user's group -; will be used. -user = www-data -group = www-data - -; The address on which to accept FastCGI requests. -; Valid syntaxes are: -; 'ip.add.re.ss:port' - to listen on a TCP socket to a specific address on -; a specific port; -; 'port' - to listen on a TCP socket to all addresses on a -; specific port; -; '/path/to/unix/socket' - to listen on a unix socket. -; Note: This value is mandatory. -listen = 127.0.0.1:7000 - -; Set listen(2) backlog. A value of '-1' means unlimited. -; Default Value: 128 (-1 on FreeBSD and OpenBSD) -;listen.backlog = -1 - -; Set permissions for unix socket, if one is used. In Linux, read/write -; permissions must be set in order to allow connections from a web server. Many -; BSD-derived systems allow connections regardless of permissions. -; Default Values: user and group are set as the running user -; mode is set to 0666 -;listen.owner = www-data -;listen.group = www-data -;listen.mode = 0660 - -; List of ipv4 addresses of FastCGI clients which are allowed to connect. -; Equivalent to the FCGI_WEB_SERVER_ADDRS environment variable in the original -; PHP FCGI (5.2.2+). Makes sense only with a tcp listening socket. Each address -; must be separated by a comma. If this value is left blank, connections will be -; accepted from any ip address. -; Default Value: any -;listen.allowed_clients = 127.0.0.1 - -; Choose how the process manager will control the number of child processes. -; Possible Values: -; static - a fixed number (pm.max_children) of child processes; -; dynamic - the number of child processes are set dynamically based on the -; following directives. With this process management, there will be -; always at least 1 children. -; pm.max_children - the maximum number of children that can -; be alive at the same time. -; pm.start_servers - the number of children created on startup. -; pm.min_spare_servers - the minimum number of children in 'idle' -; state (waiting to process). If the number -; of 'idle' processes is less than this -; number then some children will be created. -; pm.max_spare_servers - the maximum number of children in 'idle' -; state (waiting to process). If the number -; of 'idle' processes is greater than this -; number then some children will be killed. -; ondemand - no children are created at startup. Children will be forked when -; new requests will connect. The following parameter are used: -; pm.max_children - the maximum number of children that -; can be alive at the same time. -; pm.process_idle_timeout - The number of seconds after which -; an idle process will be killed. -; Note: This value is mandatory. -pm = dynamic - -; The number of child processes to be created when pm is set to 'static' and the -; maximum number of child processes when pm is set to 'dynamic' or 'ondemand'. -; This value sets the limit on the number of simultaneous requests that will be -; served. Equivalent to the ApacheMaxClients directive with mpm_prefork. -; Equivalent to the PHP_FCGI_CHILDREN environment variable in the original PHP -; CGI. The below defaults are based on a server without much resources. Don't -; forget to tweak pm.* to fit your needs. -; Note: Used when pm is set to 'static', 'dynamic' or 'ondemand' -; Note: This value is mandatory. -pm.max_children = 10 - -; The number of child processes created on startup. -; Note: Used only when pm is set to 'dynamic' -; Default Value: min_spare_servers + (max_spare_servers - min_spare_servers) / 2 -pm.start_servers = 4 - -; The desired minimum number of idle server processes. -; Note: Used only when pm is set to 'dynamic' -; Note: Mandatory when pm is set to 'dynamic' -pm.min_spare_servers = 2 - -; The desired maximum number of idle server processes. -; Note: Used only when pm is set to 'dynamic' -; Note: Mandatory when pm is set to 'dynamic' -pm.max_spare_servers = 6 - -; The number of seconds after which an idle process will be killed. -; Note: Used only when pm is set to 'ondemand' -; Default Value: 10s -;pm.process_idle_timeout = 10s; - -; The number of requests each child process should execute before respawning. -; This can be useful to work around memory leaks in 3rd party libraries. For -; endless request processing specify '0'. Equivalent to PHP_FCGI_MAX_REQUESTS. -; Default Value: 0 -;pm.max_requests = 500 - -; The URI to view the FPM status page. If this value is not set, no URI will be -; recognized as a status page. It shows the following informations: -; pool - the name of the pool; -; process manager - static, dynamic or ondemand; -; start time - the date and time FPM has started; -; start since - number of seconds since FPM has started; -; accepted conn - the number of request accepted by the pool; -; listen queue - the number of request in the queue of pending -; connections (see backlog in listen(2)); -; max listen queue - the maximum number of requests in the queue -; of pending connections since FPM has started; -; listen queue len - the size of the socket queue of pending connections; -; idle processes - the number of idle processes; -; active processes - the number of active processes; -; total processes - the number of idle + active processes; -; max active processes - the maximum number of active processes since FPM -; has started; -; max children reached - number of times, the process limit has been reached, -; when pm tries to start more children (works only for -; pm 'dynamic' and 'ondemand'); -; Value are updated in real time. -; Example output: -; pool: www -; process manager: static -; start time: 01/Jul/2011:17:53:49 +0200 -; start since: 62636 -; accepted conn: 190460 -; listen queue: 0 -; max listen queue: 1 -; listen queue len: 42 -; idle processes: 4 -; active processes: 11 -; total processes: 15 -; max active processes: 12 -; max children reached: 0 -; -; By default the status page output is formatted as text/plain. Passing either -; 'html', 'xml' or 'json' in the query string will return the corresponding -; output syntax. Example: -; http://www.foo.bar/status -; http://www.foo.bar/status?json -; http://www.foo.bar/status?html -; http://www.foo.bar/status?xml -; -; By default the status page only outputs short status. Passing 'full' in the -; query string will also return status for each pool process. -; Example: -; http://www.foo.bar/status?full -; http://www.foo.bar/status?json&full -; http://www.foo.bar/status?html&full -; http://www.foo.bar/status?xml&full -; The Full status returns for each process: -; pid - the PID of the process; -; state - the state of the process (Idle, Running, ...); -; start time - the date and time the process has started; -; start since - the number of seconds since the process has started; -; requests - the number of requests the process has served; -; request duration - the duration in µs of the requests; -; request method - the request method (GET, POST, ...); -; request URI - the request URI with the query string; -; content length - the content length of the request (only with POST); -; user - the user (PHP_AUTH_USER) (or '-' if not set); -; script - the main script called (or '-' if not set); -; last request cpu - the %cpu the last request consumed -; it's always 0 if the process is not in Idle state -; because CPU calculation is done when the request -; processing has terminated; -; last request memory - the max amount of memory the last request consumed -; it's always 0 if the process is not in Idle state -; because memory calculation is done when the request -; processing has terminated; -; If the process is in Idle state, then informations are related to the -; last request the process has served. Otherwise informations are related to -; the current request being served. -; Example output: -; ************************ -; pid: 31330 -; state: Running -; start time: 01/Jul/2011:17:53:49 +0200 -; start since: 63087 -; requests: 12808 -; request duration: 1250261 -; request method: GET -; request URI: /test_mem.php?N=10000 -; content length: 0 -; user: - -; script: /home/fat/web/docs/php/test_mem.php -; last request cpu: 0.00 -; last request memory: 0 -; -; Note: There is a real-time FPM status monitoring sample web page available -; It's available in: ${prefix}/share/fpm/status.html -; -; Note: The value must start with a leading slash (/). The value can be -; anything, but it may not be a good idea to use the .php extension or it -; may conflict with a real PHP file. -; Default Value: not set -;pm.status_path = /status - -; The ping URI to call the monitoring page of FPM. If this value is not set, no -; URI will be recognized as a ping page. This could be used to test from outside -; that FPM is alive and responding, or to -; - create a graph of FPM availability (rrd or such); -; - remove a server from a group if it is not responding (load balancing); -; - trigger alerts for the operating team (24/7). -; Note: The value must start with a leading slash (/). The value can be -; anything, but it may not be a good idea to use the .php extension or it -; may conflict with a real PHP file. -; Default Value: not set -;ping.path = /ping - -; This directive may be used to customize the response of a ping request. The -; response is formatted as text/plain with a 200 response code. -; Default Value: pong -;ping.response = pong - -; The access log file -; Default: not set -;access.log = log/$pool.access.log - -; The access log format. -; The following syntax is allowed -; %%: the '%' character -; %C: %CPU used by the request -; it can accept the following format: -; - %{user}C for user CPU only -; - %{system}C for system CPU only -; - %{total}C for user + system CPU (default) -; %d: time taken to serve the request -; it can accept the following format: -; - %{seconds}d (default) -; - %{miliseconds}d -; - %{mili}d -; - %{microseconds}d -; - %{micro}d -; %e: an environment variable (same as $_ENV or $_SERVER) -; it must be associated with embraces to specify the name of the env -; variable. Some exemples: -; - server specifics like: %{REQUEST_METHOD}e or %{SERVER_PROTOCOL}e -; - HTTP headers like: %{HTTP_HOST}e or %{HTTP_USER_AGENT}e -; %f: script filename -; %l: content-length of the request (for POST request only) -; %m: request method -; %M: peak of memory allocated by PHP -; it can accept the following format: -; - %{bytes}M (default) -; - %{kilobytes}M -; - %{kilo}M -; - %{megabytes}M -; - %{mega}M -; %n: pool name -; %o: ouput header -; it must be associated with embraces to specify the name of the header: -; - %{Content-Type}o -; - %{X-Powered-By}o -; - %{Transfert-Encoding}o -; - .... -; %p: PID of the child that serviced the request -; %P: PID of the parent of the child that serviced the request -; %q: the query string -; %Q: the '?' character if query string exists -; %r: the request URI (without the query string, see %q and %Q) -; %R: remote IP address -; %s: status (response code) -; %t: server time the request was received -; it can accept a strftime(3) format: -; %d/%b/%Y:%H:%M:%S %z (default) -; %T: time the log has been written (the request has finished) -; it can accept a strftime(3) format: -; %d/%b/%Y:%H:%M:%S %z (default) -; %u: remote user -; -; Default: "%R - %u %t \"%m %r\" %s" -;access.format = %R - %u %t "%m %r%Q%q" %s %f %{mili}d %{kilo}M %C%% - -; The log file for slow requests -; Default Value: not set -; Note: slowlog is mandatory if request_slowlog_timeout is set -;slowlog = log/$pool.log.slow - -; The timeout for serving a single request after which a PHP backtrace will be -; dumped to the 'slowlog' file. A value of '0s' means 'off'. -; Available units: s(econds)(default), m(inutes), h(ours), or d(ays) -; Default Value: 0 -;request_slowlog_timeout = 0 - -; The timeout for serving a single request after which the worker process will -; be killed. This option should be used when the 'max_execution_time' ini option -; does not stop script execution for some reason. A value of '0' means 'off'. -; Available units: s(econds)(default), m(inutes), h(ours), or d(ays) -; Default Value: 0 -;request_terminate_timeout = 0 - -; Set open file descriptor rlimit. -; Default Value: system defined value -;rlimit_files = 1024 - -; Set max core size rlimit. -; Possible Values: 'unlimited' or an integer greater or equal to 0 -; Default Value: system defined value -;rlimit_core = 0 - -; Chroot to this directory at the start. This value must be defined as an -; absolute path. When this value is not set, chroot is not used. -; Note: you can prefix with '$prefix' to chroot to the pool prefix or one -; of its subdirectories. If the pool prefix is not set, the global prefix -; will be used instead. -; Note: chrooting is a great security feature and should be used whenever -; possible. However, all PHP paths will be relative to the chroot -; (error_log, sessions.save_path, ...). -; Default Value: not set -;chroot = - -; Chdir to this directory at the start. -; Note: relative path can be used. -; Default Value: current directory or / when chroot -chdir = / - -; Redirect worker stdout and stderr into main error log. If not set, stdout and -; stderr will be redirected to /dev/null according to FastCGI specs. -; Note: on highloaded environement, this can cause some delay in the page -; process time (several ms). -; Default Value: no -;catch_workers_output = yes - -; Limits the extensions of the main script FPM will allow to parse. This can -; prevent configuration mistakes on the web server side. You should only limit -; FPM to .php extensions to prevent malicious users to use other extensions to -; exectute php code. -; Note: set an empty value to allow all extensions. -; Default Value: .php -;security.limit_extensions = .php .php3 .php4 .php5 - -; Pass environment variables like LD_LIBRARY_PATH. All $VARIABLEs are taken from -; the current environment. -; Default Value: clean env -;env[HOSTNAME] = $HOSTNAME -;env[PATH] = /usr/local/bin:/usr/bin:/bin -;env[TMP] = /tmp -;env[TMPDIR] = /tmp -;env[TEMP] = /tmp - -; Additional php.ini defines, specific to this pool of workers. These settings -; overwrite the values previously defined in the php.ini. The directives are the -; same as the PHP SAPI: -; php_value/php_flag - you can set classic ini defines which can -; be overwritten from PHP call 'ini_set'. -; php_admin_value/php_admin_flag - these directives won't be overwritten by -; PHP call 'ini_set' -; For php_*flag, valid values are on, off, 1, 0, true, false, yes or no. - -; Defining 'extension' will load the corresponding shared extension from -; extension_dir. Defining 'disable_functions' or 'disable_classes' will not -; overwrite previously defined php.ini values, but will append the new value -; instead. - -; Note: path INI options can be relative and will be expanded with the prefix -; (pool, global or /usr) - -; Default Value: nothing is defined by default except the values in php.ini and -; specified at startup with the -d argument -;php_admin_value[sendmail_path] = /usr/sbin/sendmail -t -i -f www@my.domain.com -;php_flag[display_errors] = off -;php_admin_value[error_log] = /var/log/fpm-php.www.log -;php_admin_flag[log_errors] = on -;php_admin_value[memory_limit] = 32M diff --git a/cookbooks/dev/templates/default/fpm.conf.erb b/cookbooks/dev/templates/default/fpm.conf.erb deleted file mode 100644 index 523d8ca85..000000000 --- a/cookbooks/dev/templates/default/fpm.conf.erb +++ /dev/null @@ -1,47 +0,0 @@ -; DO NOT EDIT - This file is being maintained by Chef - -[<%= @user %>] -listen = 127.0.0.1:<%= @port %> -listen.backlog = 256 - -user = <%= @user %> -group = <%= @user %> - -pm = dynamic -pm.max_children = 10 -pm.start_servers = 4 -pm.min_spare_servers = 2 -pm.max_spare_servers = 6 -pm.max_requests = 10000 - -request_terminate_timeout = 1800 - -;pm.status_path = /status -;ping.path = /ping -;ping.response = pong - -;request_terminate_timeout = 0 -;request_slowlog_timeout = 0 -;slowlog = /var/log/php5-fpm.log.slow -;rlimit_files = 1024 -;rlimit_core = 0 -;chroot = -;chdir = /var/www -;catch_workers_output = yes - -env[HOSTNAME] = $HOSTNAME -env[PATH] = /usr/local/bin:/usr/bin:/bin -env[TMP] = /tmp -env[TMPDIR] = /tmp -env[TEMP] = /tmp - -php_admin_value[sendmail_path] = /usr/sbin/sendmail -t -i -f <%= @user %>@errol.openstreetmap.org -php_admin_value[open_basedir] = /home/<%= @user %>/:/tmp/:/usr/share/php/ -php_flag[display_errors] = on -;php_admin_value[error_log] = /var/log/fpm-php.www.log -;php_admin_flag[log_errors] = on -php_value[max_execution_time] = 300 -php_value[memory_limit] = 128M -php_value[post_max_size] = 32M -php_value[upload_max_filesize] = 32M - diff --git a/cookbooks/devices/recipes/default.rb b/cookbooks/devices/recipes/default.rb index 0b8ba5a51..8572dd232 100644 --- a/cookbooks/devices/recipes/default.rb +++ b/cookbooks/devices/recipes/default.rb @@ -20,18 +20,18 @@ cookbook_file "/usr/local/bin/fixeep-82574_83.sh" do owner "root" group "root" - mode 0o755 + mode "755" end execute "udevadm-trigger" do action :nothing - command "/sbin/udevadm trigger --action=add" + command "/bin/udevadm trigger --action=add" end template "/etc/udev/rules.d/99-chef.rules" do source "udev.rules.erb" owner "root" group "root" - mode 0o644 + mode "644" notifies :run, "execute[udevadm-trigger]" end diff --git a/cookbooks/devices/templates/default/udev.rules.erb b/cookbooks/devices/templates/default/udev.rules.erb index bc6f67eb4..2c687566c 100644 --- a/cookbooks/devices/templates/default/udev.rules.erb +++ b/cookbooks/devices/templates/default/udev.rules.erb @@ -1,10 +1,16 @@ # DO NOT EDIT - This file is being maintained by Chef # HP Smart Array configuration -ACTION=="add", SUBSYSTEM=="block", ENV{ID_VENDOR}=="HP", ENV{ID_MODEL}=="LOGICAL_VOLUME", ATTR{queue/scheduler}="noop" -ACTION=="add", SUBSYSTEM=="block", ENV{ID_VENDOR}=="HP", ENV{ID_MODEL}=="LOGICAL_VOLUME", ATTR{queue/nr_requests}="512" -ACTION=="add", SUBSYSTEM=="block", ENV{ID_VENDOR}=="HP", ENV{ID_MODEL}=="LOGICAL_VOLUME", ATTR{queue/read_ahead_kb}="4096" -ACTION=="add", SUBSYSTEM=="block", ENV{ID_VENDOR}=="HP", ENV{ID_MODEL}=="LOGICAL_VOLUME", ATTR{queue/rq_affinity}="2" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_VENDOR}=="HP", ENV{ID_MODEL}=="LOGICAL_VOLUME", ATTR{queue/scheduler}="noop" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_VENDOR}=="HP", ENV{ID_MODEL}=="LOGICAL_VOLUME", ATTR{queue/nr_requests}="512" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_VENDOR}=="HP", ENV{ID_MODEL}=="LOGICAL_VOLUME", ATTR{queue/read_ahead_kb}="4096" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_VENDOR}=="HP", ENV{ID_MODEL}=="LOGICAL_VOLUME", ATTR{queue/rq_affinity}="2" + +# Dell PERC H710 tune +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_VENDOR}=="DELL", ENV{ID_MODEL}=="PERC_H710", ATTR{queue/read_ahead_kb}="4096" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_VENDOR}=="DELL", ENV{ID_MODEL}=="PERC_H710", ATTR{queue/scheduler}="deadline" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_VENDOR}=="DELL", ENV{ID_MODEL}=="PERC_H710", ATTR{queue/nr_requests}="512" + <% node[:devices].each do |name,device| -%> # <%= device[:comment] %> @@ -26,14 +32,66 @@ ACTION=="add", SUBSYSTEM=="block", ENV{ID_BUS}=="<%= device[:bus] %>", ENV{ID_SE <% end -%> <% end -%> +# Ethernet controller: Realtek Semiconductor Co., Ltd. RTL8111/8168/8411 PCI Express Gigabit Ethernet Controller +SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x10ec", ATTRS{device}=="0x8168", RUN+="/sbin/ethtool -K $name tso off gso off" + +# Ethernet controller: Mellanox Technologies MT27710 Family [ConnectX-4 Lx] +SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x15b3", ATTRS{device}=="0x1015", RUN+="/sbin/ethtool -G $name rx 8192 tx 8192" + +# Ethernet controller: Broadcom Inc. and subsidiaries NetXtreme II BCM5709 Gigabit Ethernet +# Hewlett-Packard Company NC382i Integrated Multi-port PCI Express Gigabit Server Adapter +SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x14e4", ATTRS{device}=="0x1639", RUN+="/sbin/ethtool -G $name rx 2040" +# Ethernet controller: Broadcom Inc. and subsidiaries NetXtreme II BCM5708 Gigabit Ethernet +SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x14e4", ATTRS{device}=="0x164c", RUN+="/sbin/ethtool -G $name rx 2040" +# Ethernet controller: Broadcom Inc. and subsidiaries NetXtreme BCM5719 Gigabit Ethernet PCIe +# Hewlett-Packard Company Ethernet 1Gb 4-port 331i Adapter +SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x14e4", ATTRS{device}=="0x1657", RUN+="/sbin/ethtool -G $name rx 2047" +# Ethernet controller: Broadcom Inc. and subsidiaries NetXtreme BCM5721 Gigabit Ethernet PCI Express +SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x14e4", ATTRS{device}=="0x1659", RUN+="/sbin/ethtool -G $name rx 511" +# Ethernet controller: Broadcom Inc. and subsidiaries NetXtreme BCM5720 2-port Gigabit Ethernet PCIe +SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x14e4", ATTRS{device}=="0x165f", RUN+="/sbin/ethtool -G $name rx 2047" + +# Ethernet controller: Intel Corporation 82571EB/82571GB Gigabit Ethernet Controller D0/D1 (copper applications) +SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x105e", RUN+="/sbin/ethtool -G $name rx 4096 tx 4096" +# Ethernet controller: Intel Corporation 80003ES2LAN Gigabit Ethernet Controller (Copper) +SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x1096", RUN+="/sbin/ethtool -G $name rx 4096 tx 4096" +# Ethernet controller: Intel Corporation 82575EB Gigabit Network Connection +SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x10a7", RUN+="/sbin/ethtool -G $name rx 4096 tx 4096" # Disable scatter-gather offload for HP NC362i network controllers SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x10c9", ATTRS{subsystem_vendor}=="0x103c", ATTRS{subsystem_device}=="0x323f", RUN+="/sbin/ethtool -K $name gso off tso off sg off gro off" - +SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x10c9", ATTRS{subsystem_vendor}=="0x103c", ATTRS{subsystem_device}=="0x323f", RUN+="/sbin/ethtool -G $name rx 4096 tx 4096" +# Ethernet controller: Intel Corporation 82574L Gigabit Network Connection +SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x10d3", RUN+="/sbin/ethtool -G $name rx 4096 tx 4096" # Fix Power Saving Bug on Intel 82574L and Intel 82583 network controllers SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x10d3", RUN+="/usr/local/bin/fixeep-82574_83.sh $name" SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x10f6", RUN+="/usr/local/bin/fixeep-82574_83.sh $name" SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x150c", RUN+="/usr/local/bin/fixeep-82574_83.sh $name" +# Ethernet controller: Intel Corporation 82599ES 10-Gigabit SFI/SFP+ Network Connection +SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x10fb", RUN+="/sbin/ethtool -G $name rx 4096 tx 4096" +# Ethernet controller: Intel Corporation 82580 Gigabit Network Connection +SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x150e", RUN+="/sbin/ethtool -G $name rx 4096 tx 4096" +# Ethernet controller: Intel Corporation I350 Gigabit Network Connection +SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x1521", RUN+="/sbin/ethtool -G $name rx 4096 tx 4096" +# Ethernet controller: Intel Corporation Ethernet Controller 10-Gigabit X540-AT2 +SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x1528", RUN+="/sbin/ethtool -G $name rx 4096 tx 4096" +# Ethernet controller: Intel Corporation I210 Gigabit Network Connection +SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x1533", RUN+="/sbin/ethtool -G $name rx 4096 tx 4096" +# Ethernet controller: Intel Corporation I211 Gigabit Network Connection +SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x1539", RUN+="/sbin/ethtool -G $name rx 4096 tx 4096" +# Ethernet controller: Intel Corporation I210 Gigabit Network Connection (rev 03) +SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x157b", RUN+="/sbin/ethtool -G $name rx 4096 tx 4096" +# Ethernet controller: Intel Corporation Ethernet Controller 10G X550T +SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x1563", RUN+="/sbin/ethtool -G $name rx 4096 tx 4096" +# Ethernet controller: Intel Corporation Ethernet Controller X710 for 10GBASE-T +SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x1586", RUN+="/sbin/ethtool -G $name rx 4096 tx 4096" +# Ethernet controller: Intel Corporation Ethernet Controller X710 for 10GBASE-T +SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x15ff", RUN+="/sbin/ethtool -G $name rx 4096 tx 4096" +# Ethernet controller: Intel Corporation Ethernet Connection I354 (rev 03) +SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x1f41", RUN+="/sbin/ethtool -G $name rx 4096 tx 4096" +# Ethernet controller: Intel Corporation Ethernet Connection X722 for 10GBASE-T +SUBSYSTEM=="net", ACTION=="add", ATTRS{vendor}=="0x8086", ATTRS{device}=="0x37d2", RUN+="/sbin/ethtool -G $name rx 4096 tx 4096" + # Workaround unreliable Western Digital WD RE3/RE4 disks (ATA only) # Set sufficent Linux subsystem timeout and fix severe NCQ performance issue ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="WDC_WD5002ABYS-02B1B0", ATTR{device/timeout}="90", ATTR{device/queue_depth}="1", ATTR{queue/nr_requests}="256" @@ -53,8 +111,47 @@ ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="HGST_HTS725050A7E630", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,100,100 $env{DEVNAME}" ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="HGST_HTE721010A9E630", RUN+="/usr/sbin/smartctl -q errorsonly -l scterc,100,100 $env{DEVNAME}" +# Add SSD optimisation +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="OCZ-VERTEX3", ATTR{queue/read_ahead_kb}="4096" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="OCZ-VERTEX3", ATTR{queue/scheduler}="noop" + +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_840_PRO_*", ATTR{queue/read_ahead_kb}="4096" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_840_PRO_*", ATTR{queue/scheduler}="noop" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_840_PRO_*", ATTR{queue/read_ahead_kb}="256" + +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_850_PRO_*", ATTR{queue/read_ahead_kb}="4096" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_850_PRO_*", ATTR{queue/scheduler}="noop" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_850_PRO_*", ATTR{queue/read_ahead_kb}="256" + +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_860_PRO_*", ATTR{queue/read_ahead_kb}="4096" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_860_PRO_*", ATTR{queue/scheduler}="noop" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="Samsung_SSD_860_PRO_*", ATTR{queue/read_ahead_kb}="256" + +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="ST240FN0021", ATTR{queue/read_ahead_kb}="4096" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="ST240FN0021", ATTR{queue/scheduler}="noop" + +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="SuperMicro_SSD", ATTR{queue/read_ahead_kb}="4096" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="SuperMicro_SSD", ATTR{queue/scheduler}="noop" + +# Delete failed disk in cmok +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_BUS}=="ata", ENV{ID_MODEL}=="ST_M13FQBL", ENV{ID_SERIAL}=="ST_M13FQBL_QNR_BFW", ATTR{device/delete}="1" + # Enable request merging for NVME devices ACTION=="add", SUBSYSTEM=="block", DRIVERS=="nvme", ATTR{queue/nomerges}="1" +# VMware Virtual disk configuration +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_VENDOR}=="VMware", ENV{ID_MODEL}=="Virtual_disk", ATTR{queue/read_ahead_kb}="4096" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_VENDOR}=="VMware", ENV{ID_MODEL}=="Virtual_disk", ATTR{queue/scheduler}="noop" + # Increase readahead on virtio block devices -ACTION=="add", SUBSYSTEM=="block", DRIVERS=="virtio_blk", ATTR{queue/read_ahead_kb}="512" +ACTION=="add", SUBSYSTEM=="block", DRIVERS=="virtio_blk", ATTR{queue/read_ahead_kb}="4096" +# Note virtio sets scheduler to none automatically + +# QEMU Disk IO tune +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_MODEL}=="QEMU_HARDDISK", ATTR{queue/read_ahead_kb}="4096" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{ID_MODEL}=="QEMU_HARDDISK", ATTR{queue/scheduler}="noop" +# Vendor is sometimes missing + +# Increase default MD raid5/raid6 strip cache + group_thread_cnt +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{MD_LEVEL}=="raid5", ATTR{md/stripe_cache_size}="8192", ATTR{md/group_thread_cnt}="4" +ACTION=="add", SUBSYSTEM=="block", ENV{DEVTYPE}=="disk", ENV{MD_LEVEL}=="raid6", ATTR{md/stripe_cache_size}="8192", ATTR{md/group_thread_cnt}="4" diff --git a/cookbooks/dhcpd/metadata.rb b/cookbooks/dhcpd/metadata.rb index cc693c7df..2e239b157 100644 --- a/cookbooks/dhcpd/metadata.rb +++ b/cookbooks/dhcpd/metadata.rb @@ -7,3 +7,4 @@ description "Configures dhcpd" version "1.0.0" supports "ubuntu" depends "networking" +depends "ntp" diff --git a/cookbooks/dhcpd/recipes/default.rb b/cookbooks/dhcpd/recipes/default.rb index 18343a67d..a8f0dd6f7 100644 --- a/cookbooks/dhcpd/recipes/default.rb +++ b/cookbooks/dhcpd/recipes/default.rb @@ -27,7 +27,7 @@ template "/etc/dhcp/dhcpd.conf" do source "dhcpd.conf.erb" owner "root" group "root" - mode 0o644 + mode "644" variables :domain => domain end diff --git a/cookbooks/dhcpd/templates/default/dhcpd.conf.erb b/cookbooks/dhcpd/templates/default/dhcpd.conf.erb index 78397dff7..af2496dec 100644 --- a/cookbooks/dhcpd/templates/default/dhcpd.conf.erb +++ b/cookbooks/dhcpd/templates/default/dhcpd.conf.erb @@ -129,6 +129,18 @@ host sarel.oob.openstreetmap.org { fixed-address sarel.oob.openstreetmap.org; } +host snap-01.oob.openstreetmap.org { + hardware ethernet 3c:ec:ef:17:a6:72; + server-name "snap-01.oob.openstreetmap.org"; + fixed-address snap-01.oob.openstreetmap.org; +} + +host snap-02.oob.openstreetmap.org { + hardware ethernet 3c:ec:ef:17:a5:37; + server-name "snap-02.oob.openstreetmap.org"; + fixed-address snap-02.oob.openstreetmap.org; +} + host smaug.oob.openstreetmap.org { hardware ethernet 00:30:48:9d:57:ff; server-name "smaug.oob.openstreetmap.org"; diff --git a/cookbooks/dmca/files/default/html/index.php b/cookbooks/dmca/files/default/html/index.php index 4b283d3b3..47e1afb06 100644 --- a/cookbooks/dmca/files/default/html/index.php +++ b/cookbooks/dmca/files/default/html/index.php @@ -10,7 +10,10 @@ function process_data ($values) { $email_body = 'OpenStreetMap - Claim of Copyright Infringement form:'."\n\n"; $email_body .= 'Automated Email - Form Posted.'."\n\n"; $email_body .= print_r($values, true); - mail('dmca@osmfoundation.org','OSM Claim of Copyright Infringement', $email_body, 'From: OSMF Copyright Form ', '-fdmca@osmfoundation.org'); + $reply_address = $values['name_first'].' '.$values['name_last'].' <'.$values['email'].'>'; + $email_body .= 'Formatted address: '.$reply_address."\n\n"; + $email_header = 'From: OSMF Copyright Form ' . "\r\n" . 'Content-Type: text/plain; charset="utf-8"'; + mail('dmca@osmfoundation.org','OSM Claim of Copyright Infringement', $email_body, $email_header, '-fdmca@osmfoundation.org'); } ?> @@ -37,7 +40,8 @@ $form->addElement('static', null, '

To file a copyright infringement notificat

  • A statement that the complaining party has a good faith belief that use of the material in the manner complained of is not authorized by the copyright owner, its agent, or the law.
  • A statement that the information in the notification is accurate and, under penalty of perjury, that the complaining party is authorized to act on behalf of the owner of an exclusive right that is allegedly infringed.

    To expedite our ability to process your request, such written notice should be sent to our designated agent via our online copyright complaint form below.

    -

    This form is only for cases where you believe that material on OpenStreetMap\'s websites or in its geodata database infringes your copyright or that of your clients. For example, you claim someone has copied material from a map belonging to you. If you have come here for another reason, Go here.

    '); +

    This form is only for cases where you believe that material on OpenStreetMap\'s websites or in its geodata database infringes your copyright or that of your clients. For example, you claim someone has copied material from a map belonging to you.

    +

    If you have come here for reporting map inaccuracies, privacy issues or another reason, Read here.

    '); $form->addElement('text', 'url', 'URL of Allegedly Infringing Material', array('size' => 50, 'maxlength' => 255)); $form->addRule('url', 'Please enter URL of Allegedly Infringing Material', 'required', null, 'client'); diff --git a/cookbooks/dmca/metadata.rb b/cookbooks/dmca/metadata.rb index 6b12e9822..987656295 100644 --- a/cookbooks/dmca/metadata.rb +++ b/cookbooks/dmca/metadata.rb @@ -7,3 +7,4 @@ description "Configure DMCA form" version "1.0.0" supports "ubuntu" depends "apache" +depends "php" diff --git a/cookbooks/dmca/recipes/default.rb b/cookbooks/dmca/recipes/default.rb index 652ae8cb8..f7f9cfadb 100644 --- a/cookbooks/dmca/recipes/default.rb +++ b/cookbooks/dmca/recipes/default.rb @@ -18,23 +18,25 @@ # include_recipe "apache" +include_recipe "php::fpm" -apache_module "php7.2" +apache_module "proxy" +apache_module "proxy_fcgi" directory "/srv/dmca.openstreetmap.org" do owner "root" group "root" - mode 0o755 + mode "755" end remote_directory "/srv/dmca.openstreetmap.org/html" do source "html" owner "root" group "root" - mode 0o755 + mode "755" files_owner "root" files_group "root" - files_mode 0o644 + files_mode "644" end ssl_certificate "dmca.openstreetmap.org" do @@ -42,6 +44,12 @@ ssl_certificate "dmca.openstreetmap.org" do notifies :reload, "service[apache2]" end +php_fpm "dmca.openstreetmap.org" do + php_admin_values "open_basedir" => "/srv/dmca.openstreetmap.org/html/:/usr/share/php/:/tmp/", + "disable_functions" => "exec,shell_exec,system,passthru,popen,proc_open" + prometheus_port 11201 +end + apache_site "dmca.openstreetmap.org" do template "apache.erb" directory "/srv/dmca.openstreetmap.org" diff --git a/cookbooks/dmca/templates/default/apache.erb b/cookbooks/dmca/templates/default/apache.erb index bf53e2d2e..672a1be3e 100644 --- a/cookbooks/dmca/templates/default/apache.erb +++ b/cookbooks/dmca/templates/default/apache.erb @@ -51,4 +51,8 @@ /html> Require all granted + + + SetHandler "proxy:unix:/run/php/<%= @name %>.sock|fcgi://127.0.0.1" + diff --git a/cookbooks/dns/attributes/default.rb b/cookbooks/dns/attributes/default.rb new file mode 100644 index 000000000..5bc8153d6 --- /dev/null +++ b/cookbooks/dns/attributes/default.rb @@ -0,0 +1,3 @@ +default[:dns][:repository] = "/var/lib/git/public/dns.git" + +default[:accounts][:users][:git][:status] = :role diff --git a/cookbooks/dns/metadata.rb b/cookbooks/dns/metadata.rb index ee177a46b..fb1120681 100644 --- a/cookbooks/dns/metadata.rb +++ b/cookbooks/dns/metadata.rb @@ -6,5 +6,6 @@ description "Configure DNS management" version "1.0.0" supports "ubuntu" -depends "git" +depends "accounts" depends "apache" +depends "git" diff --git a/cookbooks/dns/recipes/default.rb b/cookbooks/dns/recipes/default.rb index 1896e99b8..eaeaf5623 100644 --- a/cookbooks/dns/recipes/default.rb +++ b/cookbooks/dns/recipes/default.rb @@ -17,8 +17,9 @@ # limitations under the License. # -include_recipe "git" +include_recipe "accounts" include_recipe "apache" +include_recipe "git" geoservers = search(:node, "roles:geodns").collect(&:name).sort @@ -29,27 +30,37 @@ package %w[ parallel rsync perl + libdigest-sha-perl + libjson-xs-perl + libwww-perl libxml-treebuilder-perl libxml-writer-perl - libyaml-perl - libwww-perl - libjson-xs-perl + libyaml-libyaml-perl + lockfile-progs ] +remote_file "/usr/local/bin/dnscontrol" do + action :create + source "https://github.com/StackExchange/dnscontrol/releases/download/v3.5.0/dnscontrol-Linux" + owner "root" + group "root" + mode "755" +end + directory "/srv/dns.openstreetmap.org" do owner "root" group "root" - mode 0o755 + mode "755" end remote_directory "/srv/dns.openstreetmap.org/html" do source "html" owner "root" group "root" - mode 0o755 + mode "755" files_owner "root" files_group "root" - files_mode 0o644 + files_mode "644" end zones = [] @@ -61,7 +72,7 @@ Dir.glob("/var/lib/dns/json/*.json").each do |kmlfile| source "zone.html.erb" owner "root" group "root" - mode 0o644 + mode "644" variables :zone => zone end @@ -72,7 +83,7 @@ template "/srv/dns.openstreetmap.org/html/index.html" do source "index.html.erb" owner "root" group "root" - mode 0o644 + mode "644" variables :zones => zones end @@ -91,7 +102,7 @@ template "/usr/local/bin/dns-update" do source "dns-update.erb" owner "root" group "git" - mode 0o750 + mode "750" variables :passwords => passwords, :geoservers => geoservers end @@ -105,28 +116,37 @@ end directory "/var/lib/dns" do owner "git" group "git" - mode 0o2775 + mode "2775" notifies :run, "execute[dns-update]" end +template "/var/lib/dns/creds.json" do + source "creds.json.erb" + owner "git" + group "git" + mode "440" + variables :passwords => passwords +end + cookbook_file "#{node[:dns][:repository]}/hooks/post-receive" do source "post-receive" owner "git" group "git" - mode 0o750 + mode "750" + only_if { ::Dir.exist?("#{node[:dns][:repository]}/hooks") } end template "/usr/local/bin/dns-check" do source "dns-check.erb" owner "root" group "git" - mode 0o750 + mode "750" variables :passwords => passwords, :geoservers => geoservers end -template "/etc/cron.d/dns" do - source "cron.erb" - owner "root" - group "root" - mode 0o644 +cron_d "dns" do + minute "*/3" + user "git" + command "/usr/local/bin/dns-check" + mailto "admins@openstreetmap.org" end diff --git a/cookbooks/dns/templates/default/creds.json.erb b/cookbooks/dns/templates/default/creds.json.erb new file mode 100644 index 000000000..980c0fb38 --- /dev/null +++ b/cookbooks/dns/templates/default/creds.json.erb @@ -0,0 +1,11 @@ +{ + "gandi_v5": { + "apikey": "<%= @passwords["gandi"] %>", + "sharing_id": "7028b616-ba65-11e7-8343-00163ec31f40" + }, + "cloudflare": { + "accountid": "049c95aba02c95fc1e78a9d255282e0f", + "accountname": "OpenStreetMap", + "apitoken": "<%= @passwords["cloudflare"] %>" + } +} diff --git a/cookbooks/dns/templates/default/dns-check.erb b/cookbooks/dns/templates/default/dns-check.erb index ca38dddc8..09f061b67 100755 --- a/cookbooks/dns/templates/default/dns-check.erb +++ b/cookbooks/dns/templates/default/dns-check.erb @@ -1,10 +1,12 @@ #!/bin/sh -export RSYNC_PASSWORD="<%= @passwords["rsync"] %>" export GEODNS_SERVERS="<%= @geoservers.join(" ") %>" -export PINGDOM_USERNAME="pingdom@openstreetmap.org" -export PINGDOM_PASSWORD="<%= @passwords["pingdom"] %>" export STATUSCAKE_USERNAME="OpenStreetMap" export STATUSCAKE_APIKEY="<%= @passwords["statuscake"] %>" +export PATH=/usr/local/bin:$PATH -make --quiet --jobs --directory=/var/lib/dns --assume-new=lib/countries.xml update > /dev/null +lockfile-create --use-pid /var/lib/dns/update || exit 1 + +make --quiet --jobs --directory=/var/lib/dns update_primary > /dev/null + +lockfile-remove /var/lib/dns/update diff --git a/cookbooks/dns/templates/default/dns-update.erb b/cookbooks/dns/templates/default/dns-update.erb index 401c5c74f..5353d69d4 100755 --- a/cookbooks/dns/templates/default/dns-update.erb +++ b/cookbooks/dns/templates/default/dns-update.erb @@ -4,15 +4,15 @@ umask 0002 -export RSYNC_PASSWORD="<%= @passwords["rsync"] %>" export GEODNS_SERVERS="<%= @geoservers.join(" ") %>" -export PINGDOM_USERNAME="pingdom@openstreetmap.org" -export PINGDOM_PASSWORD="<%= @passwords["pingdom"] %>" export STATUSCAKE_USERNAME="OpenStreetMap" export STATUSCAKE_APIKEY="<%= @passwords["statuscake"] %>" +export PATH=/usr/local/bin:$PATH cd /var/lib/dns +lockfile-create --use-pid update || exit 1 + if [ ! -d .git ] then git clone /var/lib/git/public/dns.git /var/lib/dns @@ -21,3 +21,5 @@ fi git pull -q make --jobs update + +lockfile-remove update diff --git a/cookbooks/dns/templates/default/zone.html.erb b/cookbooks/dns/templates/default/zone.html.erb index 3bf01af60..fb34d94bd 100644 --- a/cookbooks/dns/templates/default/zone.html.erb +++ b/cookbooks/dns/templates/default/zone.html.erb @@ -6,7 +6,7 @@ - + diff --git a/cookbooks/docker/attributes/default.rb b/cookbooks/docker/attributes/default.rb new file mode 100644 index 000000000..f71e326f3 --- /dev/null +++ b/cookbooks/docker/attributes/default.rb @@ -0,0 +1,2 @@ +# Add the docker APT source +default[:apt][:sources] = node[:apt][:sources] | ["docker"] diff --git a/cookbooks/openvpn/metadata.rb b/cookbooks/docker/metadata.rb similarity index 61% rename from cookbooks/openvpn/metadata.rb rename to cookbooks/docker/metadata.rb index 15bee3b88..110cc3202 100644 --- a/cookbooks/openvpn/metadata.rb +++ b/cookbooks/docker/metadata.rb @@ -1,8 +1,9 @@ -name "openvpn" +name "docker" maintainer "OpenStreetMap Administrators" maintainer_email "admins@openstreetmap.org" license "Apache-2.0" -description "Installs and configures OpenVPN" +description "Installs and configures the docker daemon" version "1.0.0" supports "ubuntu" +depends "apt" diff --git a/cookbooks/docker/recipes/default.rb b/cookbooks/docker/recipes/default.rb new file mode 100644 index 000000000..23891f110 --- /dev/null +++ b/cookbooks/docker/recipes/default.rb @@ -0,0 +1,45 @@ +# +# Cookbook:: docker +# Recipe:: default +# +# Copyright:: 2020, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "apt" + +package %w[ + docker-ce + docker-ce-cli + containerd.io +] + +directory "/etc/docker" do + owner "root" + group "root" + mode "755" +end + +template "/etc/docker/daemon.json" do + source "daemon.json.erb" + owner "root" + group "root" + mode "644" +end + +service "docker" do + action [:enable, :start] + subscribes :restart, "template[/etc/docker/daemon.json]" + not_if { ENV["TEST_KITCHEN"] } +end diff --git a/cookbooks/docker/templates/default/daemon.json.erb b/cookbooks/docker/templates/default/daemon.json.erb new file mode 100644 index 000000000..5d18abccd --- /dev/null +++ b/cookbooks/docker/templates/default/daemon.json.erb @@ -0,0 +1,8 @@ +{ + "exec-opts": ["native.cgroupdriver=systemd"], + "log-driver": "json-file", + "log-opts": { + "max-size": "100m" + }, + "storage-driver": "overlay2" +} diff --git a/cookbooks/donate/attributes/default.rb b/cookbooks/donate/attributes/default.rb new file mode 100644 index 000000000..c6c3785d6 --- /dev/null +++ b/cookbooks/donate/attributes/default.rb @@ -0,0 +1,2 @@ +# Enable the "donate" role +default[:accounts][:users][:donate][:status] = :role diff --git a/cookbooks/donate/metadata.rb b/cookbooks/donate/metadata.rb index 0882bddf8..4c4649b1a 100644 --- a/cookbooks/donate/metadata.rb +++ b/cookbooks/donate/metadata.rb @@ -6,6 +6,8 @@ description "Installs and configures Donate Site" version "1.0.0" supports "ubuntu" +depends "accounts" depends "apache" -depends "mysql" depends "git" +depends "mysql" +depends "php" diff --git a/cookbooks/donate/recipes/default.rb b/cookbooks/donate/recipes/default.rb index 1b3ede310..7389db3cf 100644 --- a/cookbooks/donate/recipes/default.rb +++ b/cookbooks/donate/recipes/default.rb @@ -17,21 +17,22 @@ # limitations under the License. # +include_recipe "accounts" include_recipe "apache" -include_recipe "mysql" include_recipe "git" +include_recipe "mysql" +include_recipe "php::fpm" package %w[ - php php-cli php-curl php-mysql php-gd ] -apache_module "php7.2" - apache_module "headers" +apache_module "proxy" +apache_module "proxy_fcgi" passwords = data_bag_item("donate", "passwords") @@ -48,12 +49,13 @@ end directory "/srv/donate.openstreetmap.org" do owner "donate" group "donate" - mode 0o755 + mode "755" end git "/srv/donate.openstreetmap.org" do action :sync - repository "git://github.com/osmfoundation/donation-drive.git" + repository "https://github.com/osmfoundation/donation-drive.git" + depth 1 user "donate" group "donate" end @@ -61,14 +63,14 @@ end directory "/srv/donate.openstreetmap.org/data" do owner "donate" group "donate" - mode 0o755 + mode "755" end template "/srv/donate.openstreetmap.org/scripts/db-connect.inc.php" do source "db-connect.inc.php.erb" owner "root" group "donate" - mode 0o644 + mode "644" variables :passwords => passwords end @@ -78,22 +80,26 @@ ssl_certificate "donate.openstreetmap.org" do notifies :reload, "service[apache2]" end +php_fpm "donate.openstreetmap.org" do + php_admin_values "open_basedir" => "/srv/donate.openstreetmap.org/:/usr/share/php/:/tmp/", + "disable_functions" => "exec,shell_exec,system,passthru,popen,proc_open" + prometheus_port 11101 +end + apache_site "donate.openstreetmap.org" do template "apache.erb" end -template "/etc/cron.d/osmf-donate" do - source "cron.erb" - owner "root" - group "root" - mode 0o600 - variables :passwords => passwords +cron_d "osmf-donate" do + minute "*/2" + user "donate" + command "cd /srv/donate.openstreetmap.org/scripts/; /usr/bin/php /srv/donate.openstreetmap.org/scripts/update_csv_donate2016.php" end template "/etc/cron.daily/osmf-donate-backup" do source "backup.cron.erb" owner "root" group "root" - mode 0o750 + mode "750" variables :passwords => passwords end diff --git a/cookbooks/donate/templates/default/apache.erb b/cookbooks/donate/templates/default/apache.erb index 9404ad13a..329dc12bd 100644 --- a/cookbooks/donate/templates/default/apache.erb +++ b/cookbooks/donate/templates/default/apache.erb @@ -27,9 +27,6 @@ DocumentRoot /srv/donate.openstreetmap.org - php_admin_value open_basedir /srv/donate.openstreetmap.org/:/usr/share/php/:/tmp/ - php_admin_value disable_functions "exec,shell_exec,system,passthru,popen,proc_open" - # Alias Dynamic Content to data folder to avoid serving dummy git content Alias /donors-eur.csv /srv/donate.openstreetmap.org/data/donors-eur.csv Alias /donors.csv /srv/donate.openstreetmap.org/data/donors.csv @@ -42,7 +39,11 @@ Redirect permanent /server2015 https://donate.openstreetmap.org/ - Require all granted + Require all granted + + + SetHandler "proxy:unix:/run/php/donate.openstreetmap.org.sock|fcgi://127.0.0.1" + diff --git a/cookbooks/donate/templates/default/backup.cron.erb b/cookbooks/donate/templates/default/backup.cron.erb index 92b66cf00..3782762c3 100644 --- a/cookbooks/donate/templates/default/backup.cron.erb +++ b/cookbooks/donate/templates/default/backup.cron.erb @@ -10,13 +10,12 @@ mkdir $T/osm-donate-$D echo '[mysqldump]' > $T/mysqldump.opts echo 'user=donate' >> $T/mysqldump.opts echo 'password=<%= @passwords["database"] %>' >> $T/mysqldump.opts -mysqldump --defaults-file=$T/mysqldump.opts --opt donate > $T/osm-donate-$D/osm-donate.sql +mysqldump --defaults-file=$T/mysqldump.opts --opt --no-tablespaces donate > $T/osm-donate-$D/osm-donate.sql ln -s /srv/donate.openstreetmap.org $T/osm-donate-$D/www -export GZIP="--rsyncable -9" export RSYNC_RSH="ssh -ax" -nice tar --create --gzip --dereference --directory=$T --file=$T/$B osm-donate-$D +nice tar --create --dereference --directory=$T osm-donate-$D | nice gzip --rsyncable -9 > $T/$B nice rsync --preallocate --fuzzy $T/$B backup::backup rm -rf $T diff --git a/cookbooks/donate/templates/default/cron.erb b/cookbooks/donate/templates/default/cron.erb deleted file mode 100644 index 42d6c770f..000000000 --- a/cookbooks/donate/templates/default/cron.erb +++ /dev/null @@ -1,2 +0,0 @@ -# DO NOT EDIT - This file is being maintained by Chef -*/2 * * * * donate cd /srv/donate.openstreetmap.org/scripts/; /usr/bin/php /srv/donate.openstreetmap.org/scripts/update_csv_donate2016.php diff --git a/cookbooks/elasticsearch/attributes/default.rb b/cookbooks/elasticsearch/attributes/default.rb index a2a643412..1de57c968 100644 --- a/cookbooks/elasticsearch/attributes/default.rb +++ b/cookbooks/elasticsearch/attributes/default.rb @@ -1,4 +1,5 @@ -default[:elasticsearch][:version] = "1.7" +default[:elasticsearch][:version] = "6.x" +default[:elasticsearch][:cluster][:name] = "default" default[:elasticsearch][:cluster][:routing][:allocation][:disk][:watermark][:low] = "85%" default[:elasticsearch][:cluster][:routing][:allocation][:disk][:watermark][:high] = "90%" default[:elasticsearch][:cluster][:routing][:allocation][:disk][:watermark][:flood_stage] = "95%" diff --git a/cookbooks/elasticsearch/metadata.rb b/cookbooks/elasticsearch/metadata.rb index 9bab57ae2..37f38ee1a 100644 --- a/cookbooks/elasticsearch/metadata.rb +++ b/cookbooks/elasticsearch/metadata.rb @@ -6,3 +6,4 @@ description "Installs and configures a elasticsearch server" version "1.0.0" supports "ubuntu" +depends "apt" diff --git a/cookbooks/elasticsearch/recipes/default.rb b/cookbooks/elasticsearch/recipes/default.rb index d2ba1d31b..cd7e8ff01 100644 --- a/cookbooks/elasticsearch/recipes/default.rb +++ b/cookbooks/elasticsearch/recipes/default.rb @@ -17,16 +17,16 @@ # limitations under the License. # -package %w[ - default-jre-headless - elasticsearch -] +include_recipe "apt" + +package "default-jre-headless" +package "elasticsearch" template "/etc/elasticsearch/elasticsearch.yml" do source "elasticsearch.yml.erb" user "root" group "root" - mode 0o644 + mode "644" notifies :restart, "service[elasticsearch]" end diff --git a/cookbooks/exim/attributes/default.rb b/cookbooks/exim/attributes/default.rb index 77b8d8716..21e999109 100644 --- a/cookbooks/exim/attributes/default.rb +++ b/cookbooks/exim/attributes/default.rb @@ -3,6 +3,7 @@ default[:exim][:relay_to_domains] = [] default[:exim][:relay_from_hosts] = ["127.0.0.1", "::1"] default[:exim][:daemon_smtp_ports] = [25] default[:exim][:trusted_users] = [] +default[:exim][:queue_run_max] = 1 default[:exim][:smarthost_name] = nil default[:exim][:smarthost_via] = "mail.openstreetmap.org:26" default[:exim][:routes] = {} diff --git a/cookbooks/exim/metadata.rb b/cookbooks/exim/metadata.rb index 754ee82de..cd9da2c45 100644 --- a/cookbooks/exim/metadata.rb +++ b/cookbooks/exim/metadata.rb @@ -6,6 +6,9 @@ description "Installs and configures exim" version "1.0.0" supports "ubuntu" +depends "accounts" +depends "apache" +depends "munin" depends "networking" +depends "prometheus" depends "ssl" -depends "apache" diff --git a/cookbooks/exim/recipes/default.rb b/cookbooks/exim/recipes/default.rb index 94133183a..8a1fd5efb 100644 --- a/cookbooks/exim/recipes/default.rb +++ b/cookbooks/exim/recipes/default.rb @@ -17,7 +17,9 @@ # limitations under the License. # +include_recipe "munin" include_recipe "networking" +include_recipe "prometheus" package %w[ exim4 @@ -25,7 +27,9 @@ package %w[ ssl-cert ] -package "exim4-daemon-heavy" if File.exist?("/var/run/clamav/clamd.ctl") +package "exim4-daemon-heavy" do + only_if { ::File.exist?("/var/run/clamav/clamd.ctl") } +end group "ssl-cert" do action :modify @@ -50,7 +54,7 @@ else key_file "/etc/ssl/private/exim.key" owner "root" group "ssl-cert" - mode 0o640 + mode "640" org "OpenStreetMap" email "postmaster@openstreetmap.org" common_name node[:fqdn] @@ -73,22 +77,105 @@ end relay_from_hosts = node[:exim][:relay_from_hosts] if node[:exim][:smarthost_name] + search(:node, "roles:gateway") do |gateway| + allowed_ips = gateway.interfaces(:role => :internal).map do |interface| + "#{interface[:network]}/#{interface[:prefix]}" + end + + node.default[:networking][:wireguard][:peers] << { + :public_key => gateway[:networking][:wireguard][:public_key], + :allowed_ips => allowed_ips, + :endpoint => "#{gateway.name}:51820" + } + end + search(:node, "exim_smarthost_via:#{node[:exim][:smarthost_name]}\\:*").each do |host| relay_from_hosts |= host.ipaddresses(:role => :external) end + + domains = node[:exim][:certificate_names].select { |c| c =~ /^a\.mx\./ }.collect { |c| c.sub(/^a\.mx./, "") } + primary_domain = domains.first + + directory "/srv/mta-sts.#{primary_domain}" do + owner "root" + group "root" + mode "755" + end + + domains.each do |domain| + template "/srv/mta-sts.#{primary_domain}/#{domain}.txt" do + source "mta-sts.erb" + owner "root" + group "root" + mode "644" + variables :domain => domain + end + end + + ssl_certificate "mta-sts.#{primary_domain}" do + domains domains.collect { |d| "mta-sts.#{d}" } + notifies :reload, "service[apache2]" + end + + apache_site "mta-sts.#{primary_domain}" do + template "apache-mta-sts.erb" + directory "/srv/mta-sts.#{primary_domain}" + variables :domains => domains + end end file "/etc/exim4/blocked-senders" do owner "root" group "Debian-exim" - mode 0o644 + mode "644" +end + +if node[:exim][:dkim_selectors] + keys = data_bag_item("exim", "dkim") + + template "/etc/exim4/dkim-domains" do + owner "root" + source "dkim-domains.erb" + group "Debian-exim" + mode "644" + end + + template "/etc/exim4/dkim-selectors" do + owner "root" + source "dkim-selectors.erb" + group "Debian-exim" + mode "644" + end + + directory "/etc/exim4/dkim-keys" do + owner "root" + group "Debian-exim" + mode "755" + end + + node[:exim][:dkim_selectors].each do |domain, _selector| + file "/etc/exim4/dkim-keys/#{domain}" do + content keys[domain].join("\n") + owner "root" + group "Debian-exim" + mode "640" + end + end +end + +template "/etc/default/exim4" do + source "default.erb" + owner "root" + group "root" + mode "044" + notifies :restart, "service[exim4]" end template "/etc/exim4/exim4.conf" do source "exim4.conf.erb" owner "root" group "Debian-exim" - mode 0o644 + mode "644" variables :relay_to_domains => relay_to_domains.sort, :relay_from_hosts => relay_from_hosts.sort notifies :restart, "service[exim4]" @@ -115,23 +202,27 @@ template "/etc/aliases" do source "aliases.erb" owner "root" group "root" - mode 0o644 + mode "644" end remote_directory "/etc/exim4/noreply" do source "noreply" owner "root" group "Debian-exim" - mode 0o755 + mode "755" files_owner "root" files_group "Debian-exim" - files_mode 0o755 + files_mode "755" purge true end munin_plugin "exim_mailqueue" munin_plugin "exim_mailstats" +prometheus_exporter "exim" do + port 9636 +end + if node[:exim][:smarthost_name] node[:exim][:daemon_smtp_ports].each do |port| firewall_rule "accept-inbound-smtp-#{port}" do diff --git a/cookbooks/exim/templates/default/apache-mta-sts.erb b/cookbooks/exim/templates/default/apache-mta-sts.erb new file mode 100644 index 000000000..82b787fab --- /dev/null +++ b/cookbooks/exim/templates/default/apache-mta-sts.erb @@ -0,0 +1,34 @@ +# DO NOT EDIT - This file is being maintained by Chef + + + ServerName <%= @name %> +<% @domains.drop(1).each do |domain| -%> + ServerAlias mta-sts.<%= domain %> +<% end -%> + ServerAdmin webmaster@openstreetmap.org + + CustomLog /var/log/apache2/<%= @name %>-access.log combined + ErrorLog /var/log/apache2/<%= @name %>-error.log + + RedirectPermanent /.well-known/acme-challenge/ http://acme.openstreetmap.org/.well-known/acme-challenge/ + +<% @domains.each do |domain| -%> + + + ServerName mta-sts.<%= domain %> + ServerAdmin webmaster@openstreetmap.org + + SSLEngine on + SSLCertificateFile /etc/ssl/certs/<%= @name %>.pem + SSLCertificateKeyFile /etc/ssl/private/<%= @name %>.key + + CustomLog /var/log/apache2/<%= @name %>-access.log combined + ErrorLog /var/log/apache2/<%= @name %>-error.log + + Alias /.well-known/mta-sts.txt <%= @directory %>/<%= domain %>.txt + +<% end -%> + +> + Require all granted + diff --git a/cookbooks/exim/templates/default/default.erb b/cookbooks/exim/templates/default/default.erb new file mode 100644 index 000000000..2fb3dcc17 --- /dev/null +++ b/cookbooks/exim/templates/default/default.erb @@ -0,0 +1,23 @@ +# DO NOT EDIT - This file is being maintained by Chef + +# 'combined' - one daemon running queue and listening on SMTP port +# 'no' - no daemon running the queue +# 'separate' - two separate daemons +# 'ppp' - only run queue with /etc/ppp/ip-up.d/exim4. +# 'nodaemon' - no daemon is started at all. +# 'queueonly' - only a queue running daemon is started, no SMTP listener. +# setting this to 'no' will also disable queueruns from /etc/ppp/ip-up.d/exim4 +QUEUERUNNER='combined' +# how often should we run the queue +QUEUEINTERVAL='30s' +# options common to quez-runner and listening daemon +COMMONOPTIONS='' +# more options for the daemon/process running the queue (applies to the one +# started in /etc/ppp/ip-up.d/exim4, too. +QUEUERUNNEROPTIONS='' +# special flags given to exim directly after the -q. See exim(8) +QFLAGS='' +# options for daemon listening on port 25 +SMTPLISTENEROPTIONS='' +# only warn once about each error +E4BCD_WATCH_PANICLOG='once' diff --git a/cookbooks/exim/templates/default/dkim-domains.erb b/cookbooks/exim/templates/default/dkim-domains.erb new file mode 100644 index 000000000..992ac39c7 --- /dev/null +++ b/cookbooks/exim/templates/default/dkim-domains.erb @@ -0,0 +1,3 @@ +<% node[:exim][:dkim_selectors].each do |domain, _selector| -%> +*.<%= domain %>: <%= domain %> +<% end -%> diff --git a/cookbooks/exim/templates/default/dkim-selectors.erb b/cookbooks/exim/templates/default/dkim-selectors.erb new file mode 100644 index 000000000..13078faad --- /dev/null +++ b/cookbooks/exim/templates/default/dkim-selectors.erb @@ -0,0 +1,3 @@ +<% node[:exim][:dkim_selectors].each do |domain, selector| -%> +<%= domain %>: <%= selector %> +<% end -%> diff --git a/cookbooks/exim/templates/default/exim4.conf.erb b/cookbooks/exim/templates/default/exim4.conf.erb index 0a4f1d04f..2d1d96e12 100644 --- a/cookbooks/exim/templates/default/exim4.conf.erb +++ b/cookbooks/exim/templates/default/exim4.conf.erb @@ -42,6 +42,7 @@ ###################################################################### # MAIN CONFIGURATION SETTINGS # ###################################################################### +# # Specify your host's canonical name here. This should normally be the fully # qualified "official" name of your host. If this option is not set, the @@ -93,12 +94,13 @@ hostlist relay_from_hosts = <; <%= @relay_from_hosts.join(" ; ") %> # to any other host on the Internet. Such a setting commonly refers to a # complete local network as well as the localhost. For example: # -# hostlist relay_from_hosts = 127.0.0.1 : 192.168.0.0/16 +# hostlist relay_from_hosts = <; 127.0.0.1 ; ::1 ; 192.168.0.0/16 # # The "/16" is a bit mask (CIDR notation), not a number of hosts. Note that you # have to include 127.0.0.1 if you want to allow processes on your host to send # SMTP mail by using the loopback address. A number of MUAs use this method of -# sending mail. +# sending mail. Often, connections are made to "localhost", which might be ::1 +# on IPv6-enabled hosts. Do not forget CIDR for your IPv6 networks. # All three of these lists may contain many different kinds of item, including # wildcarded names, regular expressions, and file lookups. See the reference @@ -144,11 +146,7 @@ spamd_address = 127.0.0.1 783 # Allow any client to use TLS. -tls_advertise_hosts = <; !127.0.0.1 ; !::1 - -# Configured TLS cipher selection. - -tls_require_ciphers = <%= node[:ssl][:gnutls_ciphers] %>:%SERVER_PRECEDENCE +tls_advertise_hosts = * # Specify the location of the Exim server's TLS certificate and private key. # The private key must not be encrypted (password protected). You can put @@ -164,6 +162,10 @@ tls_certificate = /etc/ssl/certs/exim.pem tls_privatekey = /etc/ssl/private/exim.key <% end -%> +# Configure TLS cipher selection. + +tls_require_ciphers = <%= node[:ssl][:gnutls_ciphers] %>:%SERVER_PRECEDENCE + # In order to support roaming users who wish to send email from anywhere, # you may want to make Exim listen on other ports as well as port 25, in # case these users need to send email from a network that blocks port 25. @@ -233,18 +235,26 @@ never_users = root host_lookup = * -# The settings below, which are actually the same as the defaults in the -# code, cause Exim to make RFC 1413 (ident) callbacks for all incoming SMTP -# calls. You can limit the hosts to which these calls are made, and/or change -# the timeout that is used. If you set the timeout to zero, all RFC 1413 calls -# are disabled. RFC 1413 calls are cheap and can provide useful information -# for tracing problem messages, but some hosts and firewalls have problems -# with them. This can result in a timeout instead of an immediate refused -# connection, leading to delays on starting up SMTP sessions. (The default was -# reduced from 30s to 5s for release 4.61.) +# The settings below cause Exim to make RFC 1413 (ident) callbacks +# for all incoming SMTP calls. You can limit the hosts to which these +# calls are made, and/or change the timeout that is used. If you set +# the timeout to zero, all RFC 1413 calls are disabled. RFC 1413 calls +# are cheap and can provide useful information for tracing problem +# messages, but some hosts and firewalls have problems with them. +# This can result in a timeout instead of an immediate refused +# connection, leading to delays on starting up SMTP sessions. +# (The default was reduced from 30s to 5s for release 4.61. and to +# disabled for release 4.86) +# +#rfc1413_hosts = * +#rfc1413_query_timeout = 5s + -rfc1413_hosts = * -rfc1413_query_timeout = 5s +# Enable an efficiency feature. We advertise the feature; clients +# may request to use it. For multi-recipient mails we then can +# reject or accept per-user after the message is received. +# +prdr_enable = true # By default, Exim expects all envelope addresses to be fully qualified, that @@ -260,6 +270,12 @@ rfc1413_query_timeout = 5s # and/or qualify_recipient (see above). +# Unless you run a high-volume site you probably want more logging +# detail than the default. Adjust to suit. + +log_selector = +all -skip_delivery + + # If you want Exim to support the "percent hack" for certain domains, # uncomment the following line and provide a list of domains. The "percent # hack" is the feature by which mail addressed to x%y@z (where z is one of @@ -304,10 +320,37 @@ timeout_frozen_after = 7d # split_spool_directory = true -# Log just about everything we can log so that we have the best -# possible chance of knowing what's going on. +# If you're in a part of the world where ASCII is not sufficient for most +# text, then you're probably familiar with RFC2047 message header extensions. +# By default, Exim adheres to the specification, including a limit of 76 +# characters to a line, with encoded words fitting within a line. +# If you wish to use decoded headers in message filters in such a way +# that successful decoding of malformed messages matters, you may wish to +# configure Exim to be more lenient. +# +# check_rfc2047_length = false +# +# In particular, the Exim maintainers have had multiple reports of problems +# from Russian administrators of issues until they disable this check, +# because of some popular, yet buggy, mail composition software. + + +# If you wish to be strictly RFC compliant, or if you know you'll be +# exchanging email with systems that are not 8-bit clean, then you may +# wish to disable advertising 8BITMIME. Uncomment this option to do so. + +# accept_8bitmime = false -log_selector = +all -skip_delivery + +# Exim does not make use of environment variables itself. However, +# libraries that Exim uses (e.g. LDAP) depend on specific environment settings. +# There are two lists: keep_environment for the variables we trust, and +# add_environment for variables we want to set to a specific value. +# Note that TZ is handled separately by the timezone runtime option +# and TIMEZONE_DEFAULT buildtime option. + +# keep_environment = ^LDAP +# add_environment = PATH=/usr/bin::/bin # Define trusted users. @@ -315,10 +358,10 @@ log_selector = +all -skip_delivery trusted_users = <%= node[:exim][:trusted_users].join(" : ") %> -# Don't keep any environment when starting programs - this is the -# default but setting it stops exim warning +# Do all deliveries via a limited set of queues. -keep_environment = +queue_only = true +queue_run_max = <%= node[:exim][:queue_run_max] %> @@ -384,7 +427,7 @@ acl_check_rcpt: deny message = Restricted characters in address domains = !+local_domains - local_parts = ^[./|] : ^.*[@!] : ^.*/\\.\\./ + local_parts = ^[./|] : ^.*[@%!] : ^.*/\\.\\./ ############################################################################# # Block bounces to selected addresses @@ -513,9 +556,13 @@ acl_check_data: !hosts = +relay_from_hosts message = This message failed local spam checks. - # Accept the message. + # Accept messages from relay hosts without modification. - accept + accept hosts = +relay_from_hosts + + # Accept messages from external hosts and add authentication results. + + accept add_header = :at_start:${authresults {$primary_hostname}} @@ -641,7 +688,7 @@ smarthost: dnslookup: driver = dnslookup domains = ! +local_domains - transport = remote_smtp + transport = signed_smtp same_domain_copy_routing = yes ignore_target_hosts = 0.0.0.0 : 127.0.0.0/8 no_more @@ -669,6 +716,20 @@ remote_smtp: tls_require_ciphers = <%= node[:ssl][:gnutls_ciphers] %>:%LATEST_RECORD_VERSION +# This transport is used for delivering DKIM signed messages over SMTP connections. + +signed_smtp: + driver = smtp + dkim_domain = ${lookup{${domain:$h_from:}}partial-lsearch{/etc/exim4/dkim-domains}{$value}} + dkim_selector = ${lookup{$dkim_domain}lsearch{/etc/exim4/dkim-selectors}{$value}} + dkim_private_key = /etc/exim4/dkim-keys/${dkim_domain} + dkim_identity = ${lc:${address:$h_from:}} + dkim_timestamps = 1209600 + multi_domain = false + hosts_try_dane = + tls_require_ciphers = <%= node[:ssl][:gnutls_ciphers] %>:%LATEST_RECORD_VERSION + + # This transport is used for handling pipe deliveries generated by alias or # .forward files. If the pipe generates any standard output, it is returned # to the sender of the message as a delivery error. Set return_fail_output @@ -718,7 +779,7 @@ noreply: to = $sender_address subject = Re: $header_subject: headers = MIME-Version: 1.0\nContent-Type: text/plain; charset=utf-8 - file = /etc/exim4/noreply/$local_part + file = /etc/exim4/noreply/$local_part_data user = Debian-exim group = Debian-exim diff --git a/cookbooks/exim/templates/default/mta-sts.erb b/cookbooks/exim/templates/default/mta-sts.erb new file mode 100644 index 000000000..bd237b1d1 --- /dev/null +++ b/cookbooks/exim/templates/default/mta-sts.erb @@ -0,0 +1,4 @@ +version: STSv1 +mode: enforce +mx: *.mx.<%= @domain %> +max_age: 31536000 diff --git a/cookbooks/fail2ban/metadata.rb b/cookbooks/fail2ban/metadata.rb index 6ff596b43..93f21f14d 100644 --- a/cookbooks/fail2ban/metadata.rb +++ b/cookbooks/fail2ban/metadata.rb @@ -6,3 +6,4 @@ description "Configures fail2ban" version "1.0.0" supports "ubuntu" +depends "munin" diff --git a/cookbooks/fail2ban/recipes/default.rb b/cookbooks/fail2ban/recipes/default.rb index a002fe131..e0e8e2372 100644 --- a/cookbooks/fail2ban/recipes/default.rb +++ b/cookbooks/fail2ban/recipes/default.rb @@ -17,13 +17,23 @@ # limitations under the License. # +include_recipe "munin" + package "fail2ban" template "/etc/fail2ban/jail.d/00-default.conf" do source "jail.default.erb" owner "root" group "root" - mode 0o644 + mode "644" + notifies :restart, "service[fail2ban]" +end + +template "/etc/fail2ban/paths-overrides.local" do + source "paths-overrides.local.erb" + owner "root" + group "root" + mode "644" notifies :restart, "service[fail2ban]" end diff --git a/cookbooks/fail2ban/resources/filter.rb b/cookbooks/fail2ban/resources/filter.rb index d37ba4bb2..3cd173eee 100644 --- a/cookbooks/fail2ban/resources/filter.rb +++ b/cookbooks/fail2ban/resources/filter.rb @@ -19,7 +19,7 @@ default_action :create -property :filter, :kind_of => String, :name_attribute => true +property :filter, :kind_of => String, :name_property => true property :source, :kind_of => String property :failregex, :kind_of => [String, Array] property :ignoreregex, :kind_of => [String, Array] @@ -30,7 +30,7 @@ action :create do source new_resource.source owner "root" group "root" - mode 0o644 + mode "644" end else template "/etc/fail2ban/filter.d/#{new_resource.filter}.conf" do @@ -38,7 +38,7 @@ action :create do source "filter.erb" owner "root" group "root" - mode 0o644 + mode "644" variables :failregex => new_resource.failregex, :ignoreregex => new_resource.ignoreregex end diff --git a/cookbooks/fail2ban/resources/jail.rb b/cookbooks/fail2ban/resources/jail.rb index c07914d2c..6ff5307e3 100644 --- a/cookbooks/fail2ban/resources/jail.rb +++ b/cookbooks/fail2ban/resources/jail.rb @@ -19,7 +19,7 @@ default_action :create -property :jail, :kind_of => String, :name_attribute => true +property :jail, :kind_of => String, :name_property => true property :filter, :kind_of => String property :logpath, :kind_of => String property :protocol, :kind_of => String @@ -33,7 +33,7 @@ action :create do source "jail.erb" owner "root" group "root" - mode 0o644 + mode "644" variables :name => new_resource.jail, :filter => new_resource.filter, :logpath => new_resource.logpath, diff --git a/cookbooks/dns/templates/default/cron.erb b/cookbooks/fail2ban/templates/default/paths-overrides.local.erb similarity index 57% rename from cookbooks/dns/templates/default/cron.erb rename to cookbooks/fail2ban/templates/default/paths-overrides.local.erb index 5ab9b3af7..7fbd19a79 100644 --- a/cookbooks/dns/templates/default/cron.erb +++ b/cookbooks/fail2ban/templates/default/paths-overrides.local.erb @@ -1,3 +1,4 @@ # DO NOT EDIT - This file is being maintained by Chef -*/3 * * * * git /usr/local/bin/dns-check +[DEFAULT] +sshd_backend = systemd diff --git a/cookbooks/forum/attributes/default.rb b/cookbooks/forum/attributes/default.rb new file mode 100644 index 000000000..9f4114f80 --- /dev/null +++ b/cookbooks/forum/attributes/default.rb @@ -0,0 +1,2 @@ +# Enable the "forum" role +default[:accounts][:users][:forum][:status] = :role diff --git a/cookbooks/forum/metadata.rb b/cookbooks/forum/metadata.rb index 21edbfe64..6721b1ca1 100644 --- a/cookbooks/forum/metadata.rb +++ b/cookbooks/forum/metadata.rb @@ -6,6 +6,8 @@ description "Installs and configures a roundup server" version "1.0.0" supports "ubuntu" +depends "accounts" depends "apache" depends "git" depends "mysql" +depends "php" diff --git a/cookbooks/forum/recipes/default.rb b/cookbooks/forum/recipes/default.rb index e75e5a321..728408a0c 100644 --- a/cookbooks/forum/recipes/default.rb +++ b/cookbooks/forum/recipes/default.rb @@ -16,21 +16,26 @@ # See the License for the specific language governing permissions and # limitations under the License. # + +include_recipe "accounts" include_recipe "apache" include_recipe "git" include_recipe "mysql" +include_recipe "php::fpm" + +cache_dir = Chef::Config[:file_cache_path] passwords = data_bag_item("forum", "passwords") package %w[ - php php-cli php-mysql php-xml php-apcu + unzip ] -apache_module "php7.2" +apache_module "env" apache_module "rewrite" ssl_certificate "forum.openstreetmap.org" do @@ -38,6 +43,12 @@ ssl_certificate "forum.openstreetmap.org" do notifies :reload, "service[apache2]" end +php_fpm "forum.openstreetmap.org" do + php_admin_values "open_basedir" => "/srv/forum.openstreetmap.org/html/:/usr/share/php/:/tmp/", + "disable_functions" => "exec,shell_exec,system,passthru,popen,proc_open" + prometheus_port 9253 +end + apache_site "forum.openstreetmap.org" do template "apache.erb" end @@ -45,7 +56,7 @@ end directory "/srv/forum.openstreetmap.org" do owner "forum" group "forum" - mode 0o755 + mode "755" end git "/srv/forum.openstreetmap.org/html/" do @@ -58,41 +69,41 @@ git "/srv/forum.openstreetmap.org/html/" do notifies :reload, "service[apache2]" end -remote_file "/var/cache/chef/air3_v0.8.zip" do +remote_file "#{cache_dir}/air3_v0.8.zip" do action :create_if_missing source "https://fluxbb.org/resources/styles/air3/releases/0.8/air3_v0.8.zip" owner "root" group "root" - mode 0o644 + mode "644" backup false end -execute "/var/cache/chef/air3_v0.8.zip" do +execute "#{cache_dir}/air3_v0.8.zip" do action :nothing - command "unzip -o -qq /var/cache/chef/air3_v0.8.zip Air3.css Air3/*" + command "unzip -o -qq #{cache_dir}/air3_v0.8.zip Air3.css 'Air3/*'" cwd "/srv/forum.openstreetmap.org/html/style" user "forum" group "forum" - subscribes :run, "remote_file[/var/cache/chef/air3_v0.8.zip]", :immediately + subscribes :run, "remote_file[#{cache_dir}/air3_v0.8.zip]", :immediately end directory "/srv/forum.openstreetmap.org/html/cache/" do owner "www-data" group "www-data" - mode 0o755 + mode "755" end directory "/srv/forum.openstreetmap.org/html/img/avatars/" do owner "www-data" group "www-data" - mode 0o755 + mode "755" end template "/srv/forum.openstreetmap.org/html/config.php" do source "config.php.erb" owner "forum" group "www-data" - mode 0o440 + mode "440" variables :passwords => passwords end @@ -108,6 +119,6 @@ template "/etc/cron.daily/forum-backup" do source "backup.cron.erb" owner "root" group "root" - mode 0o750 + mode "750" variables :passwords => passwords end diff --git a/cookbooks/forum/templates/default/apache.erb b/cookbooks/forum/templates/default/apache.erb index e84f1e35b..d51786b26 100644 --- a/cookbooks/forum/templates/default/apache.erb +++ b/cookbooks/forum/templates/default/apache.erb @@ -39,10 +39,9 @@ DocumentRoot /srv/forum.openstreetmap.org/html - php_admin_value open_basedir /srv/forum.openstreetmap.org/html/:/usr/share/php/:/tmp/ - php_admin_value disable_functions "exec,shell_exec,system,passthru,popen,proc_open" - php_value upload_max_filesize 70M - php_value post_max_size 100M + + SetHandler "proxy:unix:/run/php/forum.openstreetmap.org.sock|fcgi://127.0.0.1" + @@ -55,5 +54,7 @@ - php_admin_flag engine off + + SetHandler None + diff --git a/cookbooks/forum/templates/default/backup.cron.erb b/cookbooks/forum/templates/default/backup.cron.erb index c3475b876..517f71634 100644 --- a/cookbooks/forum/templates/default/backup.cron.erb +++ b/cookbooks/forum/templates/default/backup.cron.erb @@ -10,13 +10,12 @@ mkdir $T/forum-$D echo '[mysqldump]' > $T/mysqldump.opts echo 'user=forum' >> $T/mysqldump.opts echo 'password=<%= @passwords["database"] %>' >> $T/mysqldump.opts -mysqldump --defaults-file=$T/mysqldump.opts --opt forum > $T/forum-$D/forum.sql +mysqldump --defaults-file=$T/mysqldump.opts --opt --no-tablespaces forum > $T/forum-$D/forum.sql ln -s /srv/forum.openstreetmap.org $T/forum-$D/www -export GZIP="--rsyncable -9" export RSYNC_RSH="ssh -ax" -nice tar --create --gzip --dereference --directory=$T --file=$T/$B forum-$D +nice tar --create --dereference --directory=$T forum-$D | nice gzip --rsyncable -9 > $T/$B nice rsync --preallocate --fuzzy $T/$B backup::backup rm -rf $T diff --git a/cookbooks/foundation/recipes/board.rb b/cookbooks/foundation/recipes/board.rb index d951a2a0d..84d6b24b1 100644 --- a/cookbooks/foundation/recipes/board.rb +++ b/cookbooks/foundation/recipes/board.rb @@ -25,6 +25,7 @@ mediawiki_site "board.osmfoundation.org" do sitename "OSMF Board Wiki" metanamespace "OSMFBoard" directory "/srv/board.osmfoundation.org" + fpm_prometheus_port 11004 database_name "board-wiki" database_user "board-wikiuser" database_password passwords["board"]["database"] @@ -36,10 +37,11 @@ mediawiki_site "board.osmfoundation.org" do private_site true recaptcha_public_key "6LflIQATAAAAAMXyDWpba-FgipVzE-aGF4HIR59N" recaptcha_private_key passwords["board"]["recaptcha"] + version "1.34" end cookbook_file "/srv/board.osmfoundation.org/Wiki.png" do owner node[:mediawiki][:user] group node[:mediawiki][:group] - mode 0o644 + mode "644" end diff --git a/cookbooks/foundation/recipes/dwg.rb b/cookbooks/foundation/recipes/dwg.rb index 13238888e..2eff0db56 100644 --- a/cookbooks/foundation/recipes/dwg.rb +++ b/cookbooks/foundation/recipes/dwg.rb @@ -25,6 +25,7 @@ mediawiki_site "dwg.osmfoundation.org" do sitename "OSMF Data Working Group Wiki" metanamespace "OSMFDWG" directory "/srv/dwg.osmfoundation.org" + fpm_prometheus_port 11002 database_name "dwg-wiki" database_user "dwg-wikiuser" database_password passwords["dwg"]["database"] @@ -36,10 +37,11 @@ mediawiki_site "dwg.osmfoundation.org" do private_site true recaptcha_public_key "6LflIQATAAAAAMXyDWpba-FgipVzE-aGF4HIR59N" recaptcha_private_key passwords["dwg"]["recaptcha"] + version "1.34" end cookbook_file "/srv/dwg.osmfoundation.org/Wiki.png" do owner node[:mediawiki][:user] group node[:mediawiki][:group] - mode 0o644 + mode "644" end diff --git a/cookbooks/foundation/recipes/mwg.rb b/cookbooks/foundation/recipes/mwg.rb index d347528c8..60bb76fd9 100644 --- a/cookbooks/foundation/recipes/mwg.rb +++ b/cookbooks/foundation/recipes/mwg.rb @@ -25,6 +25,7 @@ mediawiki_site "mwg.osmfoundation.org" do sitename "OSMF Membership Working Group Wiki" metanamespace "OSMFMWG" directory "/srv/mwg.osmfoundation.org" + fpm_prometheus_port 11003 database_name "mwg_wiki" database_user "mwg_wikiuser" database_password passwords["mwg"]["database"] @@ -36,10 +37,11 @@ mediawiki_site "mwg.osmfoundation.org" do private_site true recaptcha_public_key "6LflIQATAAAAAMXyDWpba-FgipVzE-aGF4HIR59N" recaptcha_private_key passwords["mwg"]["recaptcha"] + version "1.34" end cookbook_file "/srv/mwg.osmfoundation.org/Wiki.png" do owner node[:mediawiki][:user] group node[:mediawiki][:group] - mode 0o644 + mode "644" end diff --git a/cookbooks/foundation/recipes/owg.rb b/cookbooks/foundation/recipes/owg.rb index b60d02bac..31d41aa1f 100644 --- a/cookbooks/foundation/recipes/owg.rb +++ b/cookbooks/foundation/recipes/owg.rb @@ -21,9 +21,14 @@ include_recipe "apache" include_recipe "git" package %w[ + gcc + g++ + make ruby ruby-dev + libssl-dev zlib1g-dev + pkg-config ] gem_package "bundler" do @@ -32,14 +37,15 @@ end git "/srv/operations.osmfoundation.org" do action :sync - repository "git://github.com/openstreetmap/owg-website.git" + repository "https://github.com/openstreetmap/owg-website.git" + depth 1 user "root" group "root" notifies :run, "execute[/srv/operations.osmfoundation.org/Gemfile]" end directory "/srv/operations.osmfoundation.org/_site" do - mode 0o755 + mode "755" owner "nobody" group "nogroup" end @@ -47,7 +53,7 @@ end # Workaround https://github.com/jekyll/jekyll/issues/7804 # by creating a .jekyll-cache folder directory "/srv/operations.osmfoundation.org/.jekyll-cache" do - mode 0o755 + mode "755" owner "nobody" group "nogroup" end @@ -62,8 +68,7 @@ execute "/srv/operations.osmfoundation.org/Gemfile" do end execute "/srv/operations.osmfoundation.org" do - action :nothing - command "bundle exec jekyll build --trace --baseurl=https://operations.osmfoundation.org" + command "bundle exec jekyll build --trace" cwd "/srv/operations.osmfoundation.org" user "nobody" group "nogroup" diff --git a/cookbooks/foundation/recipes/wiki.rb b/cookbooks/foundation/recipes/wiki.rb index b05c2ad26..e635b536a 100644 --- a/cookbooks/foundation/recipes/wiki.rb +++ b/cookbooks/foundation/recipes/wiki.rb @@ -26,6 +26,11 @@ mediawiki_site "wiki.osmfoundation.org" do "foundation.openstreetmap.org", "foundation.osm.org"] sitename "OpenStreetMap Foundation" directory "/srv/wiki.osmfoundation.org" + fpm_max_children 20 + fpm_start_servers 5 + fpm_min_spare_servers 5 + fpm_max_spare_servers 10 + fpm_prometheus_port 11001 database_name "osmf-wiki" database_user "osmf-wikiuser" database_password passwords["wiki"]["database"] @@ -39,17 +44,18 @@ mediawiki_site "wiki.osmfoundation.org" do recaptcha_public_key "6LflIQATAAAAAMXyDWpba-FgipVzE-aGF4HIR59N" recaptcha_private_key passwords["wiki"]["recaptcha"] extra_file_extensions ["mp3"] + version "1.34" end mediawiki_skin "osmf" do site "wiki.osmfoundation.org" - repository "git://github.com/openstreetmap/mediawiki-skins-osmf.git" + repository "https://github.com/openstreetmap/mediawiki-skins-osmf.git" revision "master" end mediawiki_skin "OSMFoundation" do site "wiki.osmfoundation.org" - repository "git://github.com/osmfoundation/osmf-mediawiki-skin.git" + repository "https://github.com/osmfoundation/osmf-mediawiki-skin.git" revision "master" legacy false end @@ -57,5 +63,5 @@ end cookbook_file "/srv/wiki.osmfoundation.org/Wiki.png" do owner node[:mediawiki][:user] group node[:mediawiki][:group] - mode 0o644 + mode "644" end diff --git a/cookbooks/ftp/recipes/default.rb b/cookbooks/ftp/recipes/default.rb index d2c42100c..28d69a751 100644 --- a/cookbooks/ftp/recipes/default.rb +++ b/cookbooks/ftp/recipes/default.rb @@ -26,14 +26,14 @@ template "/etc/vsftpd.conf" do source "vsftpd.conf.erb" owner "root" group "root" - mode 0o644 + mode "644" end template "/etc/pam.d/vsftpd" do source "pam-vsftpd.erb" owner "root" group "root" - mode 0o644 + mode "644" end service "vsftpd" do diff --git a/cookbooks/geodns/metadata.rb b/cookbooks/geodns/metadata.rb index 4ce6925bb..b912c7442 100644 --- a/cookbooks/geodns/metadata.rb +++ b/cookbooks/geodns/metadata.rb @@ -6,4 +6,6 @@ description "Installs and configures a geographic DNS server" version "1.0.0" supports "ubuntu" +depends "geoipupdate" +depends "networking" depends "systemd" diff --git a/cookbooks/geodns/recipes/default.rb b/cookbooks/geodns/recipes/default.rb index f24f1b8de..43643c2f9 100644 --- a/cookbooks/geodns/recipes/default.rb +++ b/cookbooks/geodns/recipes/default.rb @@ -17,29 +17,36 @@ # limitations under the License. # +include_recipe "geoipupdate" + package %w[ - geoipupdate gdnsd ] -execute "geoipdate" do - command "geoipupdate" - user "root" - group "root" - not_if { ::File.exist?("/var/lib/GeoIP/GeoLite2-Country.mmdb") } -end - directory "/etc/gdnsd/config.d" do owner "nobody" group "nogroup" - mode 0o755 + mode "755" +end + +%w[tile nominatim].each do |zone| + %w[map resource weighted].each do |type| + template "/etc/gdnsd/config.d/#{zone}.#{type}" do + action :create_if_missing + source "zone.#{type}.erb" + owner "nobody" + group "nogroup" + mode "644" + variables :zone => zone + end + end end template "/etc/gdnsd/config" do source "config.erb" owner "root" group "root" - mode 0o644 + mode "644" notifies :restart, "service[gdnsd]" end @@ -47,7 +54,7 @@ template "/etc/gdnsd/zones/geo.openstreetmap.org" do source "geo.erb" owner "root" group "root" - mode 0o644 + mode "644" notifies :restart, "service[gdnsd]" end diff --git a/cookbooks/geodns/templates/default/config.erb b/cookbooks/geodns/templates/default/config.erb index 38c17dc10..6ceafc737 100644 --- a/cookbooks/geodns/templates/default/config.erb +++ b/cookbooks/geodns/templates/default/config.erb @@ -1,4 +1,5 @@ options => { + include_optional_ns = true listen => [ <%= node.ipaddresses(:role => :external).join(", ") %> ] } @@ -6,9 +7,15 @@ plugins => { geoip => { maps => { $include{config.d/tile.map} + $include{config.d/nominatim.map} } resources => { $include{config.d/tile.resource} + $include{config.d/nominatim.resource} } + }, + weighted => { + $include{config.d/tile.weighted} + $include{config.d/nominatim.weighted} } } diff --git a/cookbooks/geodns/templates/default/geo.erb b/cookbooks/geodns/templates/default/geo.erb index 7f33a3760..9e3064218 100644 --- a/cookbooks/geodns/templates/default/geo.erb +++ b/cookbooks/geodns/templates/default/geo.erb @@ -1,18 +1,19 @@ $TTL 86400 -@ SOA a.ns.openstreetmap.org. hostmaster.openstreetmap.org. ( - 1 ; serial - 86400 ; refresh - 172800 ; retry - 604800 ; expire - 3600 ; ncache +@ SOA saphira.openstreetmap.org. hostmaster.openstreetmap.org. ( + 3 ; serial + 86400 ; refresh + 7200 ; retry + 604800 ; expire + 3600 ; ncache ) -@ 86400 NS a.ns.openstreetmap.org. -@ 86400 NS b.ns.openstreetmap.org. -@ 86400 NS c.ns.openstreetmap.org. -@ 86400 NS d.ns.openstreetmap.org. -@ 86400 NS e.ns.openstreetmap.org. -@ 86400 NS f.ns.openstreetmap.org. +@ 86400 NS balerion.openstreetmap.org. +@ 86400 NS chrysophylax.openstreetmap.org. +@ 86400 NS katie.openstreetmap.org. +@ 86400 NS saphira.openstreetmap.org. +@ 86400 NS stormfly-04.openstreetmap.org. +@ 86400 NS ridgeback.openstreetmap.org. -tile 600 DYNC geoip!tile +tile 300 DYNC geoip!tile +nominatim 300 DYNC geoip!nominatim diff --git a/cookbooks/geodns/templates/default/zone.map.erb b/cookbooks/geodns/templates/default/zone.map.erb new file mode 100644 index 000000000..25bf3ad1a --- /dev/null +++ b/cookbooks/geodns/templates/default/zone.map.erb @@ -0,0 +1,3 @@ +<%= @zone %> => { + datacenters => [dummy] +} diff --git a/cookbooks/geodns/templates/default/zone.resource.erb b/cookbooks/geodns/templates/default/zone.resource.erb new file mode 100644 index 000000000..3c1426bb7 --- /dev/null +++ b/cookbooks/geodns/templates/default/zone.resource.erb @@ -0,0 +1,6 @@ +<%= @zone %> => { + map => <%= @zone %> + dcmap => { + dummy => dummy.example.com. + } +} diff --git a/cookbooks/geodns/templates/default/zone.weighted.erb b/cookbooks/geodns/templates/default/zone.weighted.erb new file mode 100644 index 000000000..e69de29bb diff --git a/cookbooks/geoipupdate/README.md b/cookbooks/geoipupdate/README.md new file mode 100644 index 000000000..5eb996611 --- /dev/null +++ b/cookbooks/geoipupdate/README.md @@ -0,0 +1,4 @@ +# geoipupdate Cookbook + +This cookbook installs and configures geoipupdate, a tool to keep MaxMind +GeoIP databases up to date. diff --git a/cookbooks/geoipupdate/attributes/default.rb b/cookbooks/geoipupdate/attributes/default.rb new file mode 100644 index 000000000..3165aa115 --- /dev/null +++ b/cookbooks/geoipupdate/attributes/default.rb @@ -0,0 +1,4 @@ +default[:geoipupdate][:account] = "149244" +default[:geoipupdate][:editions] = %w[GeoLite2-ASN GeoLite2-City GeoLite2-Country] + +default[:apt][:sources] |= ["maxmind"] diff --git a/cookbooks/geoipupdate/metadata.rb b/cookbooks/geoipupdate/metadata.rb new file mode 100644 index 000000000..ccc09f626 --- /dev/null +++ b/cookbooks/geoipupdate/metadata.rb @@ -0,0 +1,10 @@ +name "geoipupdate" +maintainer "OpenStreetMap Administrators" +maintainer_email "admins@openstreetmap.org" +license "Apache-2.0" +description "Installs and configures geoipupdate" + +version "1.0.0" +supports "ubuntu" +depends "apt" +depends "systemd" diff --git a/cookbooks/geoipupdate/recipes/default.rb b/cookbooks/geoipupdate/recipes/default.rb new file mode 100644 index 000000000..fedae0011 --- /dev/null +++ b/cookbooks/geoipupdate/recipes/default.rb @@ -0,0 +1,61 @@ +# +# Cookbook:: geoipdate +# Recipe:: default +# +# Copyright:: 2020, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "apt" + +license_keys = data_bag_item("geoipupdate", "license-keys") + +package "geoipupdate" + +template "/etc/GeoIP.conf" do + source "GeoIP.conf.erb" + owner "root" + group "root" + mode "644" + variables :license_keys => license_keys +end + +execute "geoipupdate" do + command "geoipupdate" + user "root" + group "root" + not_if { ENV.key?("TEST_KITCHEN") || node[:geoipupdate][:editions].all? { |edition| ::File.exist?("/usr/share/GeoIP/#{edition}.mmdb") } } +end + +systemd_service "geoipupdate" do + description "Update GeoIP databases" + user "root" + exec_start "/usr/bin/geoipupdate" + private_tmp true + private_devices true + protect_system "strict" + protect_home true + read_write_paths "/usr/share/GeoIP" +end + +systemd_timer "geoipupdate" do + description "Update GeoIP databases" + on_boot_sec "15m" + on_unit_active_sec "7d" + randomized_delay_sec "4h" +end + +service "geoipupdate.timer" do + action [:enable, :start] +end diff --git a/cookbooks/geoipupdate/templates/default/GeoIP.conf.erb b/cookbooks/geoipupdate/templates/default/GeoIP.conf.erb new file mode 100644 index 000000000..9caa330ad --- /dev/null +++ b/cookbooks/geoipupdate/templates/default/GeoIP.conf.erb @@ -0,0 +1,5 @@ +# DO NOT EDIT - This file is being maintained by Chef + +AccountID <%= node[:geoipupdate][:account] %> +LicenseKey <%= @license_keys[node[:geoipupdate][:account]] %> +EditionIDs <%= Array(node[:geoipupdate][:editions]).join(" ") %> diff --git a/cookbooks/git/attributes/default.rb b/cookbooks/git/attributes/default.rb index 2d89a83de..7a4209f5d 100644 --- a/cookbooks/git/attributes/default.rb +++ b/cookbooks/git/attributes/default.rb @@ -1,5 +1,9 @@ +default[:git][:host] = "git.openstreetmap.org" default[:git][:directory] = "/var/lib/git" default[:git][:public_user] = "git" default[:git][:public_group] = "git" default[:git][:private_user] = "git" default[:git][:private_group] = "git" +default[:git][:private_nodes] = "fqdn:*" + +default[:apt][:sources] |= ["git-core"] diff --git a/cookbooks/git/metadata.rb b/cookbooks/git/metadata.rb index 003590d11..b86267bd2 100644 --- a/cookbooks/git/metadata.rb +++ b/cookbooks/git/metadata.rb @@ -6,5 +6,7 @@ description "Installs and configures git" version "1.0.0" supports "ubuntu" +depends "accounts" depends "apache" +depends "apt" depends "networking" diff --git a/cookbooks/git/recipes/default.rb b/cookbooks/git/recipes/default.rb index 3f93f651c..49d43ae82 100644 --- a/cookbooks/git/recipes/default.rb +++ b/cookbooks/git/recipes/default.rb @@ -17,4 +17,6 @@ # limitations under the License. # +include_recipe "apt" + package "git" diff --git a/cookbooks/git/recipes/server.rb b/cookbooks/git/recipes/server.rb index 4a98818e3..9b4e06a45 100644 --- a/cookbooks/git/recipes/server.rb +++ b/cookbooks/git/recipes/server.rb @@ -17,6 +17,10 @@ # limitations under the License. # +node.default[:accounts][:users][:git][:status] = :role + +include_recipe "accounts" +include_recipe "apt" include_recipe "networking" git_directory = node[:git][:directory] @@ -24,19 +28,19 @@ git_directory = node[:git][:directory] directory git_directory do owner "root" group "root" - mode 0o775 + mode "775" end directory "#{git_directory}/public" do owner node[:git][:public_user] group node[:git][:public_group] - mode 0o2775 + mode "2775" end directory "#{git_directory}/private" do owner node[:git][:private_user] group node[:git][:private_group] - mode 0o2775 + mode "2775" end Dir.glob("#{git_directory}/*/*.git").each do |repository| @@ -44,7 +48,7 @@ Dir.glob("#{git_directory}/*/*.git").each do |repository| source "post-update.erb" owner "root" group node[:git][:group] - mode 0o755 + mode "755" end end @@ -52,5 +56,5 @@ template "/etc/cron.daily/git-backup" do source "backup.cron.erb" owner "root" group "root" - mode 0o755 + mode "755" end diff --git a/cookbooks/git/recipes/web.rb b/cookbooks/git/recipes/web.rb index 468074e2c..7e1f38cc2 100644 --- a/cookbooks/git/recipes/web.rb +++ b/cookbooks/git/recipes/web.rb @@ -29,20 +29,20 @@ template "/etc/gitweb.conf" do source "gitweb.conf.erb" owner "root" group "root" - mode 0o644 + mode "644" end directory "/srv/#{git_site}" do owner "root" group "root" - mode 0o755 + mode "755" end template "/srv/#{git_site}/robots.txt" do source "robots.txt.erb" owner "root" group "root" - mode 0o644 + mode "644" end ssl_certificate git_site do diff --git a/cookbooks/git/templates/default/apache.erb b/cookbooks/git/templates/default/apache.erb index 9ec36f7d8..30685fad9 100644 --- a/cookbooks/git/templates/default/apache.erb +++ b/cookbooks/git/templates/default/apache.erb @@ -46,6 +46,12 @@ SetEnv GIT_PROJECT_ROOT /var/lib/git SetEnv GIT_HTTP_EXPORT_ALL + SetEnv GIT_HTTP_MAX_REQUEST_BUFFER 100M + + SetEnvIf Git-Protocol ".*" GIT_PROTOCOL=$0 + + # KeepaliveTimeout longer than git config uploadpack.keepalive 5 second default + KeepAliveTimeout 20 ScriptAlias /public /usr/lib/git-core/git-http-backend/public ScriptAlias /private /usr/lib/git-core/git-http-backend/private @@ -56,10 +62,12 @@ Require all granted +<% unless @private_allowed.empty? -%> Require ip <%= @private_allowed.sort.join(" ") %> +<% end -%> Require all denied diff --git a/cookbooks/git/templates/default/backup.cron.erb b/cookbooks/git/templates/default/backup.cron.erb index 5a9d01284..63882720f 100644 --- a/cookbooks/git/templates/default/backup.cron.erb +++ b/cookbooks/git/templates/default/backup.cron.erb @@ -8,10 +8,9 @@ B=git-$D.tar.gz ln -s /var/lib/git $T/git-$D -export GZIP="--rsyncable -9" export RSYNC_RSH="ssh -ax" -nice tar --create --gzip --dereference --directory=$T --file=$T/$B git-$D +nice tar --create --dereference --directory=$T git-$D | nice gzip --rsyncable -9 > $T/$B nice rsync --preallocate --fuzzy $T/$B backup::backup rm -rf $T diff --git a/cookbooks/gps-tile/attributes/default.rb b/cookbooks/gps-tile/attributes/default.rb new file mode 100644 index 000000000..61ab02a8b --- /dev/null +++ b/cookbooks/gps-tile/attributes/default.rb @@ -0,0 +1 @@ +default[:accounts][:users][:gpstile][:status] = :role diff --git a/cookbooks/gps-tile/files/default/html/index.html b/cookbooks/gps-tile/files/default/html/index.html index e5818349d..cb62f1a17 100644 --- a/cookbooks/gps-tile/files/default/html/index.html +++ b/cookbooks/gps-tile/files/default/html/index.html @@ -6,7 +6,7 @@ - + diff --git a/cookbooks/gps-tile/metadata.rb b/cookbooks/gps-tile/metadata.rb index 78acecfa5..c23644f44 100644 --- a/cookbooks/gps-tile/metadata.rb +++ b/cookbooks/gps-tile/metadata.rb @@ -6,5 +6,8 @@ description "Configures a GPS tile server" version "1.0.0" supports "ubuntu" +depends "accounts" depends "apache" +depends "git" +depends "memcached" depends "systemd" diff --git a/cookbooks/gps-tile/recipes/default.rb b/cookbooks/gps-tile/recipes/default.rb index 5de9d5573..21a515034 100644 --- a/cookbooks/gps-tile/recipes/default.rb +++ b/cookbooks/gps-tile/recipes/default.rb @@ -17,7 +17,10 @@ # limitations under the License. # +include_recipe "accounts" include_recipe "apache" +include_recipe "git" +include_recipe "memcached" package %w[ make @@ -35,20 +38,21 @@ package %w[ directory "/srv/gps-tile.openstreetmap.org" do owner "gpstile" group "gpstile" - mode 0o755 + mode "755" end git "/srv/gps-tile.openstreetmap.org/import" do action :sync - repository "git://github.com/ericfischer/gpx-import.git" + repository "https://github.com/ericfischer/gpx-import.git" revision "live" + depth 1 user "gpstile" group "gpstile" end execute "/srv/gps-tile.openstreetmap.org/import/src/Makefile" do action :nothing - command "make" + command "make DB=none LDFLAGS=-lm" cwd "/srv/gps-tile.openstreetmap.org/import/src" user "gpstile" group "gpstile" @@ -57,8 +61,9 @@ end git "/srv/gps-tile.openstreetmap.org/datamaps" do action :sync - repository "git://github.com/ericfischer/datamaps.git" + repository "https://github.com/ericfischer/datamaps.git" revision "live" + depth 1 user "gpstile" group "gpstile" end @@ -74,8 +79,9 @@ end git "/srv/gps-tile.openstreetmap.org/updater" do action :sync - repository "git://github.com/ericfischer/gpx-updater.git" + repository "https://github.com/ericfischer/gpx-updater.git" revision "live" + depth 1 user "gpstile" group "gpstile" end @@ -105,10 +111,10 @@ remote_directory "/srv/gps-tile.openstreetmap.org/html" do source "html" owner "gpstile" group "gpstile" - mode 0o755 + mode "755" files_owner "gpstile" files_group "gpstile" - files_mode 0o644 + files_mode "644" end apache_module "headers" diff --git a/cookbooks/hardware/attributes/default.rb b/cookbooks/hardware/attributes/default.rb index 2619fd549..bbe44348e 100644 --- a/cookbooks/hardware/attributes/default.rb +++ b/cookbooks/hardware/attributes/default.rb @@ -1,6 +1,8 @@ default[:hardware][:modules] = %w[lp] default[:hardware][:grub][:cmdline] = %w[nomodeset] default[:hardware][:sensors] = {} +default[:hardware][:hwmon] = {} +default[:hardware][:ipmi][:excluded_sensors] = [] if node[:dmi] && node[:dmi][:system] case node[:dmi][:system][:manufacturer] @@ -38,7 +40,7 @@ elsif node[:kernel][:modules].include?("i6300esb") default[:hardware][:watchdog] = "none" end -if File.exist?("sys/devices/system/cpu/cpu0/cpufreq/scaling_governor") && +if File.exist?("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor") && File.read("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor").chomp == "ondemand" default[:sysfs][:cpufreq_ondemand][:comment] = "Tune the ondemand CPU frequency governor" default[:sysfs][:cpufreq_ondemand][:parameters][:"devices/system/cpu/cpufreq/ondemand/up_threshold"] = "25" diff --git a/cookbooks/hardware/metadata.rb b/cookbooks/hardware/metadata.rb index b632a23d9..35dd20ed8 100644 --- a/cookbooks/hardware/metadata.rb +++ b/cookbooks/hardware/metadata.rb @@ -7,8 +7,10 @@ description "Configures hardware" version "1.0.0" supports "ubuntu" depends "apt" +depends "chef" depends "git" depends "munin" +depends "prometheus" depends "ohai" depends "tools" depends "sysfs" diff --git a/cookbooks/hardware/recipes/default.rb b/cookbooks/hardware/recipes/default.rb index 454386459..ebdf64bb1 100644 --- a/cookbooks/hardware/recipes/default.rb +++ b/cookbooks/hardware/recipes/default.rb @@ -17,8 +17,12 @@ # limitations under the License. # -include_recipe "tools" +include_recipe "apt" +include_recipe "git" include_recipe "munin" +include_recipe "prometheus" +include_recipe "sysfs" +include_recipe "tools" ohai_plugin "hardware" do template "ohai.rb.erb" @@ -47,7 +51,7 @@ end units = [] -if node[:roles].include?("bytemark") || node[:roles].include?("exonetric") +if node[:roles].include?("bytemark") || node[:roles].include?("exonetric") || node[:roles].include?("prgmr") units << "0" end @@ -83,14 +87,29 @@ when "TYAN" when "TYAN Computer Corporation" units << "0" when "Supermicro" - case product - when "H8DGU", "X9SCD", "X7DBU", "X7DW3", "X9DR7/E-(J)LN4F", "X9DR3-F", "X9DRW", "SYS-1028U-TN10RT+", "SYS-2028U-TN24R4T+", "SYS-1029P-WTRT", "Super Server" - units << "1" - else - units << "0" - end + units << "1" when "IBM" units << "0" +when "VMware, Inc." + package "open-vm-tools" + + # Remove timeSync plugin completely + # https://github.com/vmware/open-vm-tools/issues/302 + file "/usr/lib/open-vm-tools/plugins/vmsvc/libtimeSync.so" do + action :delete + notifies :restart, "service[open-vm-tools]" + end + + # Attempt to tell Host we are not interested in timeSync + execute "vmware-toolbox-cmd-timesync-disable" do + command "/usr/bin/vmware-toolbox-cmd timesync disable" + ignore_failure true + end + + service "open-vm-tools" do + action [:enable, :start] + supports :status => true, :restart => true + end end units.sort.uniq.each do |unit| @@ -122,13 +141,14 @@ if File.exist?("/etc/default/grub") execute "update-grub" do action :nothing command "/usr/sbin/update-grub" + not_if { ENV["TEST_KITCHEN"] } end template "/etc/default/grub" do source "grub.erb" owner "root" group "root" - mode 0o644 + mode "644" variables :units => units, :entry => grub_entry notifies :run, "execute[update-grub]" end @@ -145,7 +165,7 @@ template "/etc/initramfs-tools/conf.d/mdadm" do source "initramfs-mdadm.erb" owner "root" group "root" - mode 0o644 + mode "644" notifies :run, "execute[update-initramfs]" end @@ -154,21 +174,29 @@ service "haveged" do action [:enable, :start] end -package "ipmitool" if node[:kernel][:modules].include?("ipmi_si") +if node[:kernel][:modules].include?("ipmi_si") + package "ipmitool" + package "freeipmi-tools" -package "irqbalance" + template "/etc/prometheus/ipmi_local.yml" do + source "ipmi_local.yml.erb" + owner "root" + group "root" + mode "644" + end -template "/etc/default/irqbalance" do - source "irqbalance.erb" - owner "root" - group "root" - mode 0o644 + prometheus_exporter "ipmi" do + port 9290 + options "--config.file=/etc/prometheus/ipmi_local.yml" + subscribes :restart, "template[/etc/prometheus/ipmi_local.yml]" + end end +package "irqbalance" + service "irqbalance" do action [:start, :enable] supports :status => false, :restart => true, :reload => false - subscribes :restart, "template[/etc/default/irqbalance]" end # Link Layer Discovery Protocol Daemon @@ -181,42 +209,48 @@ end tools_packages = [] status_packages = {} -node[:kernel][:modules].each_key do |modname| - case modname - when "cciss" - tools_packages << "ssacli" - status_packages["cciss-vol-status"] ||= [] - when "hpsa" - tools_packages << "ssacli" - status_packages["cciss-vol-status"] ||= [] - when "mptsas" - tools_packages << "lsiutil" - status_packages["mpt-status"] ||= [] - when "mpt2sas", "mpt3sas" - tools_packages << "sas2ircu" - status_packages["sas2ircu-status"] ||= [] - when "megaraid_mm" - tools_packages << "megactl" - status_packages["megaraid-status"] ||= [] - when "megaraid_sas" - tools_packages << "megacli" - status_packages["megaclisas-status"] ||= [] - when "aacraid" - tools_packages << "arcconf" - status_packages["aacraid-status"] ||= [] - when "arcmsr" - tools_packages << "areca" +if node[:virtualization][:role] != "guest" || + (node[:virtualization][:system] != "lxc" && + node[:virtualization][:system] != "lxd" && + node[:virtualization][:system] != "openvz") + + node[:kernel][:modules].each_key do |modname| + case modname + when "cciss" + tools_packages << "ssacli" + status_packages["cciss-vol-status"] ||= [] + when "hpsa" + tools_packages << "ssacli" + status_packages["cciss-vol-status"] ||= [] + when "mptsas" + tools_packages << "lsiutil" + status_packages["mpt-status"] ||= [] + when "mpt2sas", "mpt3sas" + tools_packages << "sas2ircu" + status_packages["sas2ircu-status"] ||= [] + when "megaraid_mm" + tools_packages << "megactl" + status_packages["megaraid-status"] ||= [] + when "megaraid_sas" + tools_packages << "megacli" + status_packages["megaclisas-status"] ||= [] + when "aacraid" + tools_packages << "arcconf" + status_packages["aacraid-status"] ||= [] + when "arcmsr" + tools_packages << "areca" + end end -end -node[:block_device].each do |name, attributes| - next unless attributes[:vendor] == "HP" && attributes[:model] == "LOGICAL VOLUME" + node[:block_device].each do |name, attributes| + next unless attributes[:vendor] == "HP" && attributes[:model] == "LOGICAL VOLUME" - if name =~ /^cciss!(c[0-9]+)d[0-9]+$/ - status_packages["cciss-vol-status"] |= ["cciss/#{Regexp.last_match[1]}d0"] - else - Dir.glob("/sys/block/#{name}/device/scsi_generic/*").each do |sg| - status_packages["cciss-vol-status"] |= [File.basename(sg)] + if name =~ /^cciss!(c[0-9]+)d[0-9]+$/ + status_packages["cciss-vol-status"] |= ["cciss/#{Regexp.last_match[1]}d0"] + else + Dir.glob("/sys/block/#{name}/device/scsi_generic/*").each do |sg| + status_packages["cciss-vol-status"] |= [File.basename(sg)] + end end end end @@ -237,8 +271,10 @@ if tools_packages.include?("areca") git "/opt/areca" do action :sync repository "https://git.openstreetmap.org/private/areca.git" + depth 1 user "root" group "root" + not_if { ENV["TEST_KITCHEN"] } end else directory "/opt/areca" do @@ -252,7 +288,7 @@ if status_packages.include?("cciss-vol-status") source "cciss-vol-statusd.erb" owner "root" group "root" - mode 0o755 + mode "755" notifies :restart, "service[cciss-vol-statusd]" end @@ -265,6 +301,14 @@ if status_packages.include?("cciss-vol-status") no_new_privileges true notifies :restart, "service[cciss-vol-statusd]" end +else + systemd_service "cciss-vol-statusd" do + action :delete + end + + template "/usr/local/bin/cciss-vol-statusd" do + action :delete + end end %w[cciss-vol-status mpt-status sas2ircu-status megaraid-status megaclisas-status aacraid-status].each do |status_package| @@ -275,7 +319,7 @@ end source "raid.default.erb" owner "root" group "root" - mode 0o644 + mode "644" variables :devices => status_packages[status_package] end @@ -301,38 +345,43 @@ disks = if node[:hardware][:disk] [] end -# intel_ssds = disks.select { |d| d[:vendor] == "INTEL" && d[:model] =~ /^SSD/ } -# -# nvmes = if node[:hardware][:pci] -# node[:hardware][:pci].values.select { |pci| pci[:driver] == "nvme" } -# else -# [] -# end -# -# intel_nvmes = nvmes.select { |pci| pci[:vendor_name] == "Intel Corporation" } -# -# if !intel_ssds.empty? || !intel_nvmes.empty? -# package "unzip" -# -# intel_ssd_tool_version = "3.0.21" -# -# remote_file "#{Chef::Config[:file_cache_path]}/Intel_SSD_Data_Center_Tool_#{intel_ssd_tool_version}_Linux.zip" do -# source "https://downloadmirror.intel.com/29115/eng/Intel_SSD_Data_Center_Tool_#{intel_ssd_tool_version}_Linux.zip" -# end -# -# execute "#{Chef::Config[:file_cache_path]}/Intel_SSD_Data_Center_Tool_#{intel_ssd_tool_version}_Linux.zip" do -# command "unzip Intel_SSD_Data_Center_Tool_#{intel_ssd_tool_version}_Linux.zip isdct_#{intel_ssd_tool_version}-1_amd64.deb" -# cwd Chef::Config[:file_cache_path] -# user "root" -# group "root" -# not_if { File.exist?("#{Chef::Config[:file_cache_path]}/isdct_#{intel_ssd_tool_version}-1_amd64.deb") } -# end -# -# dpkg_package "isdct" do -# version "#{intel_ssd_tool_version}-1" -# source "#{Chef::Config[:file_cache_path]}/isdct_#{intel_ssd_tool_version}-1_amd64.deb" -# end -# end +intel_ssds = disks.select { |d| d[:vendor] == "INTEL" && d[:model] =~ /^SSD/ } + +nvmes = if node[:hardware][:pci] + node[:hardware][:pci].values.select { |pci| pci[:driver] == "nvme" } + else + [] + end + +intel_nvmes = nvmes.select { |pci| pci[:vendor_name] == "Intel Corporation" } + +if !intel_ssds.empty? || !intel_nvmes.empty? + package "unzip" + + intel_mas_tool_version = "1.6" + intel_mas_package_version = "#{intel_mas_tool_version}.122-0" + + remote_file "#{Chef::Config[:file_cache_path]}/Intel_MAS_CLI_Tool_#{intel_mas_tool_version}_Linux.zip" do + source "https://downloadmirror.intel.com/30259/eng/Intel%C2%AE_MAS_CLI_Tool_Linux_#{intel_mas_tool_version}.zip" + end + + execute "#{Chef::Config[:file_cache_path]}/Intel_MAS_CLI_Tool_#{intel_mas_tool_version}_Linux.zip" do + command "unzip Intel_MAS_CLI_Tool_#{intel_mas_tool_version}_Linux.zip intelmas_#{intel_mas_package_version}_amd64.deb" + cwd Chef::Config[:file_cache_path] + user "root" + group "root" + not_if { ::File.exist?("#{Chef::Config[:file_cache_path]}/intelmas_#{intel_mas_package_version}_amd64.deb") } + end + + dpkg_package "intelmas" do + version "#{intel_mas_package_version}" + source "#{Chef::Config[:file_cache_path]}/intelmas_#{intel_mas_package_version}_amd64.deb" + end + + dpkg_package "isdct" do + action :purge + end +end disks = disks.map do |disk| next if disk[:state] == "spun_down" || %w[unconfigured failed].any?(disk[:status]) @@ -352,6 +401,13 @@ disks = disks.map do |disk| elsif smart =~ %r{^.*,(\d+)/(\d+)$} munin = "#{device}-#{Regexp.last_match(1)}:#{Regexp.last_match(2)}" end + elsif disk[:device] + device = disk[:device].sub("/dev/", "") + smart = disk[:smart_device] + + if smart =~ /^.*,(\d+),(\d+),(\d+)$/ + munin = "#{device}-#{Regexp.last_match(1)}:#{Regexp.last_match(2)}:#{Regexp.last_match(3)}" + end end elsif disk[:device] =~ %r{^/dev/(nvme\d+)n\d+$} device = Regexp.last_match(1) @@ -376,18 +432,25 @@ disks = disks.compact.uniq if disks.count.positive? package "smartmontools" + template "/etc/cron.daily/update-smart-drivedb" do + source "update-smart-drivedb.erb" + owner "root" + group "root" + mode "755" + end + template "/usr/local/bin/smartd-mailer" do source "smartd-mailer.erb" owner "root" group "root" - mode 0o755 + mode "755" end template "/etc/smartd.conf" do source "smartd.conf.erb" owner "root" group "root" - mode 0o644 + mode "644" variables :disks => disks end @@ -395,15 +458,27 @@ if disks.count.positive? source "smartmontools.erb" owner "root" group "root" - mode 0o644 + mode "644" end - service "smartd" do + service "smartmontools" do action [:enable, :start] subscribes :reload, "template[/etc/smartd.conf]" subscribes :restart, "template[/etc/default/smartmontools]" end + template "/etc/prometheus/collectors/smart.devices" do + source "smart.devices.erb" + owner "root" + group "root" + mode "644" + variables :disks => disks + end + + prometheus_collector "smart" do + interval "15m" + end + # Don't try and do munin monitoring of disks behind # an Areca controller as they only allow one thing to # talk to the controller at a time and smartd will @@ -455,7 +530,7 @@ if File.exist?("/etc/mdadm/mdadm.conf") file "/etc/mdadm/mdadm.conf" do owner "root" group "root" - mode 0o644 + mode "644" content mdadm_conf end @@ -469,7 +544,7 @@ template "/etc/modules" do source "modules.erb" owner "root" group "root" - mode 0o644 + mode "644" end service "kmod" do @@ -484,7 +559,7 @@ if node[:hardware][:watchdog] source "watchdog.erb" owner "root" group "root" - mode 0o644 + mode "644" variables :module => node[:hardware][:watchdog] end @@ -531,16 +606,24 @@ unless Dir.glob("/sys/class/hwmon/hwmon*").empty? source "sensors.conf.erb" owner "root" group "root" - mode 0o644 + mode "644" notifies :run, "execute[/etc/sensors.d/chef.conf]" end end if node[:hardware][:shm_size] + execute "remount-dev-shm" do + action :nothing + command "/bin/mount -o remount /dev/shm" + user "root" + group "root" + end + mount "/dev/shm" do - action [:mount, :enable] + action :enable device "tmpfs" fstype "tmpfs" options "rw,nosuid,nodev,size=#{node[:hardware][:shm_size]}" + notifies :run, "execute[remount-dev-shm]" end end diff --git a/cookbooks/hardware/templates/default/ipmi_local.yml.erb b/cookbooks/hardware/templates/default/ipmi_local.yml.erb new file mode 100644 index 000000000..4250a62eb --- /dev/null +++ b/cookbooks/hardware/templates/default/ipmi_local.yml.erb @@ -0,0 +1,13 @@ +# DO NOT EDIT - This file is being maintained by Chef + +modules: + default: + collectors: + - bmc + - ipmi + - dcmi + - chassis + exclude_sensor_ids: +<% node[:hardware][:ipmi][:excluded_sensors].each do |sensor| -%> + - <%= sensor %> +<% end -%> diff --git a/cookbooks/hardware/templates/default/irqbalance.erb b/cookbooks/hardware/templates/default/irqbalance.erb deleted file mode 100644 index 4a10bd20d..000000000 --- a/cookbooks/hardware/templates/default/irqbalance.erb +++ /dev/null @@ -1,11 +0,0 @@ -# DO NOT EDIT - This file is being maintained by Chef -# Configuration for the irqbalance daemon - -# Should irqbalance be enabled? -ENABLED="1" -# Balance the IRQs only once? -ONESHOT="0" - -# irqbalance maintainer recommends ignore hint policy -# http://sourceforge.net/p/e1000/bugs/394/?page=1 -DOPTIONS="--hintpolicy=ignore" diff --git a/cookbooks/hardware/templates/default/ohai.rb.erb b/cookbooks/hardware/templates/default/ohai.rb.erb index 82194acf2..c3308eb8c 100644 --- a/cookbooks/hardware/templates/default/ohai.rb.erb +++ b/cookbooks/hardware/templates/default/ohai.rb.erb @@ -287,7 +287,7 @@ Ohai.plugin(:Hardware) do disk = nil IO.popen(%w(ssacli controller all show config detail)).each do |line| - if line =~ /^Smart Array (\S+) / + if line =~ /^Smart (?:Array|HBA) (\S+) / controller = { :id => devices[:controllers].count, :model => Regexp.last_match(1), @@ -303,6 +303,7 @@ Ohai.plugin(:Hardware) do disk = nil elsif controller && line =~ /^ (\S.*):\s+(.*)$/ case Regexp.last_match(1) + when "Slot" then controller[:slot] = Regexp.last_match(2) when "Serial Number" then controller[:serial_number] = Regexp.last_match(2) when "Hardware Revision" then controller[:hardware_version] = Regexp.last_match(2) when "Firmware Version" then controller[:firmware_version] = Regexp.last_match(2) @@ -320,15 +321,12 @@ Ohai.plugin(:Hardware) do controller[:arrays] << array[:id] disk = nil - elsif controller && line =~ /^ physicaldrive (\S+) / - disks << Regexp.last_match(1) elsif array && line =~ /^ physicaldrive (\S+)$/ disk = { :id => devices[:disks].count, :controller => controller[:id], :arrays => [array[:id]], - :location => Regexp.last_match(1), - :smart_device => "cciss,#{disks.find_index(Regexp.last_match(1))}" + :location => Regexp.last_match(1) } devices[:disks] << disk @@ -355,6 +353,14 @@ Ohai.plugin(:Hardware) do end controllers.each do |controller| + slot = controller[:slot] + + IO.popen(%W(ssacli controller slot=#{slot} pd all show status)).each do |line| + if line =~ /^ physicaldrive (\S+) / + disks << Regexp.last_match(1) + end + end + if device = Dir.glob("/sys/bus/pci/devices/#{controller[:pci_slot]}/cciss*").first controller[:device] = File.basename(device).sub(/^cciss(\d+)$/, "/dev/cciss/c\\1d0") elsif device = Dir.glob("/sys/bus/pci/devices/#{controller[:pci_slot]}/host*/target*:3:0/*:3:0:0/scsi_generic/sg*").first @@ -363,6 +369,10 @@ Ohai.plugin(:Hardware) do controller[:device] = "/dev/#{File.basename(device)}" end end + + devices[:disks].each do |disk| + disk[:smart_device] = "cciss,#{disks.find_index(disk[:location])}" + end end def find_megaraid_disks(devices) @@ -766,10 +776,11 @@ Ohai.plugin(:Hardware) do array[:disks].map! do |location| disk = disks.find { |disk| disk[:location] == location } + controller_number = controller[:number] - 1 device_number = disk[:device_number] - device = Dir.glob("#{host}/device/target*:1:#{device_number}/*:1:#{device_number}:0/scsi_generic/*").first disk[:device] = "/dev/#{File.basename(device)}" + disk[:smart_device] = "aacraid,#{controller_number},0,#{device_number}" disk[:arrays] << array[:id] disk[:id] diff --git a/cookbooks/hardware/templates/default/smart.devices.erb b/cookbooks/hardware/templates/default/smart.devices.erb new file mode 100644 index 000000000..36498d74e --- /dev/null +++ b/cookbooks/hardware/templates/default/smart.devices.erb @@ -0,0 +1,9 @@ +<% @disks.each do |disk| -%> +<% if disk[:smart] -%> +/dev/<%= disk[:device] %>|<%= disk[:smart] %> +<% elsif disk[:device] =~ /nvme/ -%> +/dev/<%= disk[:device] %>|nvme +<% else -%> +/dev/<%= disk[:device] %>|sat +<% end -%> +<% end -%> diff --git a/cookbooks/hardware/templates/default/update-smart-drivedb.erb b/cookbooks/hardware/templates/default/update-smart-drivedb.erb new file mode 100644 index 000000000..5093be813 --- /dev/null +++ b/cookbooks/hardware/templates/default/update-smart-drivedb.erb @@ -0,0 +1,10 @@ +#!/bin/bash +# DO NOT EDIT - This file is being maintained by Chef + +set -eu + +if [ -x /usr/sbin/update-smart-drivedb ]; then + /usr/sbin/update-smart-drivedb -u github > /dev/null +fi + +exit 0 diff --git a/cookbooks/imagery/files/default/os-openmap-local-palette.txt b/cookbooks/imagery/files/default/os-openmap-local-palette.txt new file mode 100644 index 000000000..deb83599e --- /dev/null +++ b/cookbooks/imagery/files/default/os-openmap-local-palette.txt @@ -0,0 +1,256 @@ +0,0,0 +8,7,7 +24,24,24 +40,40,40 +52,52,52 +85,15,45 +49,83,63 +73,66,53 +61,61,72 +89,88,72 +80,80,80 +87,87,87 +112,83,88 +108,105,87 +87,88,104 +84,111,108 +100,100,100 +105,104,104 +120,120,118 +215,18,54 +148,105,35 +157,113,51 +216,120,19 +136,24,72 +189,38,104 +212,28,64 +216,47,79 +145,120,84 +142,96,102 +138,122,104 +138,120,120 +165,111,122 +217,69,74 +227,85,116 +63,143,54 +8,166,34 +0,166,41 +9,179,51 +55,148,82 +43,179,82 +76,132,88 +76,153,84 +97,170,90 +95,139,107 +121,159,121 +106,160,110 +124,184,117 +55,195,91 +54,200,88 +82,199,111 +152,152,8 +214,133,34 +225,151,59 +200,201,8 +243,243,0 +248,248,8 +174,140,87 +144,157,102 +136,136,120 +153,136,120 +136,158,120 +152,152,120 +162,136,104 +183,151,106 +187,184,104 +240,162,67 +201,155,102 +221,165,123 +226,173,102 +233,184,127 +146,204,109 +255,192,115 +248,200,120 +73,81,199 +255,60,140 +192,115,130 +235,101,128 +229,111,139 +0,159,184 +40,152,172 +69,135,144 +127,154,155 +92,167,180 +25,194,164 +120,214,142 +9,151,200 +0,154,207 +0,169,202 +14,170,200 +6,177,202 +24,161,216 +24,184,217 +45,168,210 +50,185,213 +0,191,230 +41,184,232 +75,184,213 +39,200,232 +81,201,215 +109,199,214 +115,216,212 +80,213,232 +85,216,248 +113,214,236 +117,232,248 +136,136,136 +153,136,136 +133,153,136 +152,152,136 +128,136,153 +152,136,152 +154,154,144 +152,152,152 +168,152,136 +185,150,136 +168,136,152 +185,135,153 +168,152,152 +184,152,152 +151,168,150 +157,191,153 +168,168,136 +184,168,136 +184,184,136 +168,168,152 +184,168,152 +184,184,152 +151,152,170 +171,152,171 +145,172,172 +167,167,167 +184,168,168 +168,188,168 +178,184,173 +184,184,168 +168,168,184 +185,167,184 +166,185,185 +183,184,183 +210,141,149 +255,135,158 +247,136,152 +200,168,132 +200,185,136 +216,191,128 +200,168,152 +199,184,152 +217,184,152 +210,145,169 +248,137,168 +232,154,167 +208,184,168 +242,166,184 +248,185,184 +141,193,137 +164,217,148 +189,220,165 +188,219,184 +158,225,175 +208,204,152 +242,200,145 +246,233,151 +200,200,168 +215,199,168 +216,216,168 +200,200,184 +216,200,184 +216,216,184 +232,200,168 +248,195,168 +232,216,168 +248,216,165 +232,200,184 +231,215,183 +248,216,184 +208,235,184 +232,232,168 +248,232,168 +228,248,168 +248,248,168 +232,232,184 +250,223,191 +252,225,189 +248,232,184 +232,248,184 +255,243,181 +247,247,184 +184,184,200 +248,182,201 +143,214,209 +181,207,204 +149,232,214 +179,232,204 +142,216,234 +169,216,226 +146,232,247 +176,232,242 +199,199,199 +216,200,200 +200,216,200 +216,216,200 +200,200,216 +217,200,216 +200,216,216 +216,216,216 +243,212,207 +200,232,200 +216,232,200 +200,248,200 +208,239,207 +209,241,205 +216,248,200 +200,232,216 +216,232,216 +200,248,216 +215,247,222 +216,247,216 +232,232,200 +251,232,201 +232,248,200 +248,248,200 +232,232,216 +248,232,216 +252,235,217 +232,248,216 +248,248,216 +216,216,232 +232,216,232 +247,215,229 +200,232,232 +216,232,232 +199,248,232 +216,248,232 +194,229,240 +200,232,248 +216,232,248 +199,248,248 +213,244,248 +212,244,250 +216,247,247 +230,235,228 +229,231,231 +232,232,232 +234,234,234 +248,232,232 +231,247,231 +247,247,232 +232,232,248 +248,232,248 +231,247,247 +252,252,252 +249,249,247 +254,254,254 +252,252,254 +255,255,255 +255,255,255 diff --git a/cookbooks/imagery/recipes/default.rb b/cookbooks/imagery/recipes/default.rb index 13a7f915b..63b33b70f 100644 --- a/cookbooks/imagery/recipes/default.rb +++ b/cookbooks/imagery/recipes/default.rb @@ -21,10 +21,8 @@ include_recipe "nginx" include_recipe "git" # Imagery gdal Requirements -package %w[ - gdal-bin - python-gdal -] +package "gdal-bin" +# python-gdal - disable while broken in gis unstable repo # Imagery MapServer + Mapcache Requirements package %w[ @@ -51,26 +49,26 @@ package %w[ directory "/srv/imagery/mapserver" do owner "root" group "root" - mode 0o755 + mode "755" recursive true end directory "/srv/imagery/common" do owner "root" group "root" - mode 0o755 + mode "755" recursive true end directory "/srv/imagery/common/ostn02-ntv2-data" do owner "root" group "root" - mode 0o755 + mode "755" end remote_file "#{Chef::Config[:file_cache_path]}/ostn02-ntv2-data.zip" do source "https://www.ordnancesurvey.co.uk/docs/gps/ostn02-ntv2-data.zip" - not_if { File.exist?("/srv/imagery/common/ostn02-ntv2-data/OSTN02_NTv2.gsb") } + not_if { ::File.exist?("/srv/imagery/common/ostn02-ntv2-data/OSTN02_NTv2.gsb") } end execute "unzip-ostn02-ntv2-data" do @@ -78,13 +76,12 @@ execute "unzip-ostn02-ntv2-data" do cwd "/srv/imagery/common/ostn02-ntv2-data" user "root" group "root" - not_if { File.exist?("/srv/imagery/common/ostn02-ntv2-data/OSTN02_NTv2.gsb") } + not_if { ::File.exist?("/srv/imagery/common/ostn02-ntv2-data/OSTN02_NTv2.gsb") } end nginx_site "default" do template "nginx_default.conf.erb" directory "/srv/imagery/default" - restart_nginx false end systemd_tmpfile "/run/mapserver-fastcgi" do diff --git a/cookbooks/imagery/recipes/gb_os_sv.rb b/cookbooks/imagery/recipes/gb_os_sv.rb index 878ee0fe0..b9af18645 100644 --- a/cookbooks/imagery/recipes/gb_os_sv.rb +++ b/cookbooks/imagery/recipes/gb_os_sv.rb @@ -26,6 +26,13 @@ cookbook_file "/srv/imagery/common/ossv-palette.txt" do mode "0644" end +cookbook_file "/srv/imagery/common/os-openmap-local-palette.txt" do + source "os-openmap-local-palette.txt" + owner "root" + group "root" + mode "0644" +end + cookbook_file "/srv/imagery/common/osstvw_process" do source "osstvw_process" owner "root" @@ -181,7 +188,6 @@ end imagery_layer "gb_os_sv_2016_04" do site "os.openstreetmap.org" title "April 2016" - default_layer true projection "EPSG:27700" source "/data/imagery/gb/os-sv/ossv-2016-04-combined.vrt" copyright "Contains Ordnance Survey data © Crown copyright and database right 2016" @@ -1049,3 +1055,76 @@ imagery_layer "gb_os_sv_diff_2015_11_2016_04" do url_aliases ["/sv-diff-2015-11-2016-04"] overlay true end + +imagery_layer "gb_os_om_local_2020_04" do + site "os.openstreetmap.org" + title "OS OpenMap Local - April 2020" + projection "EPSG:27700" + source "/data/imagery/gb/openmap-local/2020-04/os-openmap-local-2020-04-combined-sea-average-zstd22.vrt" + copyright "Contains OS data © Crown copyright and database right 2020" + revision 2 + background_colour "213 244 248" # OS OpenMap Local Water Blue + extension "os_om_local_png" + url_aliases ["/om-local-2020-04", "/om-local"] + default_layer true +end + +imagery_layer "gb_os_om_local_2016_10" do + site "os.openstreetmap.org" + title "OS OpenMap Local - October 2016" + projection "EPSG:27700" + source "/data/imagery/gb/openmap-local/2016-10/os-openmap-local-2016-10.vrt" + copyright "Contains OS data © Crown copyright and database right 2016" + background_colour "213 244 248" # OS OpenMap Local Water Blue + extension "os_om_local_png" +end + +imagery_layer "gb_os_om_local_2017_04" do + site "os.openstreetmap.org" + title "OS OpenMap Local - April 2017" + projection "EPSG:27700" + source "/data/imagery/gb/openmap-local/2017-04/os-openmap-local-2017-04.vrt" + copyright "Contains OS data © Crown copyright and database right 2017" + background_colour "213 244 248" # OS OpenMap Local Water Blue + extension "os_om_local_png" +end + +imagery_layer "gb_os_om_local_2017_10" do + site "os.openstreetmap.org" + title "OS OpenMap Local - October 2017" + projection "EPSG:27700" + source "/data/imagery/gb/openmap-local/2017-10/os-openmap-local-2017-10.vrt" + copyright "Contains OS data © Crown copyright and database right 2017" + background_colour "213 244 248" # OS OpenMap Local Water Blue + extension "os_om_local_png" +end + +imagery_layer "gb_os_om_local_2018_04" do + site "os.openstreetmap.org" + title "OS OpenMap Local - April 2018" + projection "EPSG:27700" + source "/data/imagery/gb/openmap-local/2018-04/os-openmap-local-2018-04.vrt" + copyright "Contains OS data © Crown copyright and database right 2018" + background_colour "213 244 248" # OS OpenMap Local Water Blue + extension "os_om_local_png" +end + +imagery_layer "gb_os_om_local_2018_05" do + site "os.openstreetmap.org" + title "OS OpenMap Local - May 2018" + projection "EPSG:27700" + source "/data/imagery/gb/openmap-local/2018-05/os-openmap-local-2018-05.vrt" + copyright "Contains OS data © Crown copyright and database right 2018" + background_colour "213 244 248" # OS OpenMap Local Water Blue + extension "os_om_local_png" +end + +imagery_layer "gb_os_om_local_2019_04" do + site "os.openstreetmap.org" + title "OS OpenMap Local - April 2019" + projection "EPSG:27700" + source "/data/imagery/gb/openmap-local/2019-04/os-openmap-local-2019-04.vrt" + copyright "Contains OS data © Crown copyright and database right 2019" + background_colour "213 244 248" # OS OpenMap Local Water Blue + extension "os_om_local_png" +end diff --git a/cookbooks/imagery/recipes/lu_lidar_hillshade.rb b/cookbooks/imagery/recipes/lu_lidar_hillshade.rb new file mode 100644 index 000000000..c80742946 --- /dev/null +++ b/cookbooks/imagery/recipes/lu_lidar_hillshade.rb @@ -0,0 +1,60 @@ +# +# Cookbook:: imagery +# Recipe:: lu_lidar_hillshade +# +# Copyright:: 2016, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "imagery" + +imagery_site "lidar-hillshade-2019.openstreetmap.lu" do + title "OpenStreetMap - Lidar Hillshade 2019" + bbox [[49.38, 5.64], [50.2, 6.64]] +end + +# Delete borken layers like this + +imagery_layer "classy_mappers_delight_lidar_hillshade_2019" do + site "lidar-hillshade-2019.openstreetmap.lu" + action :delete +end + +imagery_layer "mappers_delight_lidar_dem_2019" do + site "lidar-hillshade-2019.openstreetmap.lu" + projection "EPSG:3857" + source "/data/imagery/lu/lidar-hillshade/dem-3857.tif" + max_zoom 20 + title "OpenStreetMap.lu Mapper's Delight 2019 Lidar DEM" + copyright 'Lidar data 2019 Administration du Cadastre et de la Topographie Luxembourg, DEM Guillaume Rischard, CC0' +end + +imagery_layer "mappers_delight_lidar_hillshade_2019_reprojected" do + site "lidar-hillshade-2019.openstreetmap.lu" + default_layer true + projection "EPSG:3857" + source "/data/imagery/lu/lidar-hillshade/lu_hillshade_2019-3857.tif" + max_zoom 20 + title "OpenStreetMap.lu Mapper's Delight 2019 Lidar Hillshading" + copyright 'Lidar data 2019 Administration du Cadastre et de la Topographie Luxembourg, DEM and hillshading Guillaume Rischard, CC0' +end + +imagery_layer "mappers_delight_lidar_hillshade_2019_withunclassified" do + site "lidar-hillshade-2019.openstreetmap.lu" + projection "EPSG:3857" + source "/data/imagery/lu/lidar-hillshade/classy-hillshade.tif" + max_zoom 20 + title "OpenStreetMap.lu Mapper's Delight 2019 Lidar Hillshading with unclassified points" + copyright 'Lidar data 2019 Administration du Cadastre et de la Topographie Luxembourg, DEM and hillshading Guillaume Rischard, CC0' +end diff --git a/cookbooks/imagery/recipes/lu_ngl_dtm.rb b/cookbooks/imagery/recipes/lu_ngl_dtm.rb new file mode 100644 index 000000000..528c408e0 --- /dev/null +++ b/cookbooks/imagery/recipes/lu_ngl_dtm.rb @@ -0,0 +1,53 @@ +# +# Cookbook:: imagery +# Recipe:: lu_ngl_dtm +# +# Copyright:: 2016, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "imagery" + +imagery_site "ana-dtm-2017.openstreetmap.lu" do + title "OpenStreetMap - ANA DTM 2017" + bbox [[49.38, 5.64], [50.2, 6.64]] +end + +imagery_layer "ana_dtm_2017" do + site "ana-dtm-2017.openstreetmap.lu" + projection "EPSG:3857" + source "/data/imagery/lu/LUREF_NGL/lu_color_relief-epsg3857-compress.tif" + max_zoom 21 + title "DTM" + copyright '© 2017 Administration de la Navigation Aérienne Luxembourg, CC0' +end + +imagery_layer "ana_dtm_2017_hillshading" do + site "ana-dtm-2017.openstreetmap.lu" + projection "EPSG:3857" + source "/data/imagery/lu/LUREF_NGL/lu_hillshade_2017-epsg-3857-compress.tif" + max_zoom 21 + title "DTM Hillshading (single light source)" + copyright '© 2017 Administration de la Navigation Aérienne Luxembourg, CC0' +end + +imagery_layer "ana_dtm_2017_hillshading_multi" do + site "ana-dtm-2017.openstreetmap.lu" + default_layer true + projection "EPSG:3857" + source "/data/imagery/lu/LUREF_NGL/ANA_LUREF_NGL_DTM_hillshade_multi_epsg3857.tif" + max_zoom 21 + title "DTM Hillshading (multidirectional)" + copyright 'DEM 2017 Administration de la Navigation Aérienne Luxembourg, hillshading David Morais Ferreira & Guillaume Rischard CC0' +end diff --git a/cookbooks/imagery/resources/layer.rb b/cookbooks/imagery/resources/layer.rb index cddd95847..6affe4b80 100644 --- a/cookbooks/imagery/resources/layer.rb +++ b/cookbooks/imagery/resources/layer.rb @@ -23,8 +23,8 @@ default_action :create property :layer, String, :name_property => true property :site, String, :required => true -property :source, String, :required => true -property :root_layer, [TrueClass, FalseClass], :default => false +property :source, String, :required => [:create] +property :root_layer, [true, false], :default => false property :title, String property :copyright, String, :default => "Copyright" property :projection, String, :default => "EPSG:3857" @@ -37,14 +37,14 @@ property :extension, String, :default => "png" property :max_zoom, Integer, :default => 18 property :url_aliases, [String, Array], :default => [] property :revision, Integer, :default => 0 -property :overlay, [TrueClass, FalseClass], :default => false -property :default_layer, [TrueClass, FalseClass], :default => false +property :overlay, [true, false], :default => false +property :default_layer, [true, false], :default => false action :create do file "/srv/imagery/layers/#{new_resource.site}/#{new_resource.layer}.yml" do owner "root" group "root" - mode 0o644 + mode "644" content YAML.dump(:name => new_resource.layer, :title => new_resource.title || new_resource.layer, :url => "//{s}.#{new_resource.site}/layer/#{new_resource.layer}/{z}/{x}/{y}.png", @@ -59,24 +59,14 @@ action :create do source "mapserver.map.erb" owner "root" group "root" - mode 0o644 + mode "644" variables new_resource.to_hash end - # Disable legacy service - service "mapserv-fcgi-#{new_resource.layer}" do - action [:stop, :disable] - end - - # Remove legacy service - systemd_service "mapserv-fcgi-#{new_resource.layer}" do - action :delete - end - directory "/srv/imagery/nginx/#{new_resource.site}" do owner "root" group "root" - mode 0o755 + mode "755" recursive true end @@ -85,7 +75,7 @@ action :create do source "nginx_imagery_layer_fragment.conf.erb" owner "root" group "root" - mode 0o644 + mode "644" variables new_resource.to_hash end end @@ -95,18 +85,10 @@ action :delete do action :delete end - service "mapserv-fcgi-layer-#{new_resource.layer}" do - action [:stop, :disable] - end - file "/srv/imagery/mapserver/layer-#{new_resource.layer}.map" do action :delete end - systemd_service "mapserv-fcgi-#{new_resource.layer}" do - action :delete - end - file "/srv/imagery/nginx/#{new_resource.site}/layer-#{new_resource.layer}.conf" do action :delete end @@ -114,6 +96,5 @@ end def after_created notifies :create, "imagery_site[#{site}]" - notifies :reload, "service[nginx]" - # notifies :restart, "service[mapserv-fcgi-#{site}]" + notifies :restart, "service[nginx]" end diff --git a/cookbooks/imagery/resources/site.rb b/cookbooks/imagery/resources/site.rb index d48380384..c30c26d95 100644 --- a/cookbooks/imagery/resources/site.rb +++ b/cookbooks/imagery/resources/site.rb @@ -22,28 +22,28 @@ require "yaml" default_action :create property :site, String, :name_property => true -property :title, String, :required => true +property :title, String, :required => [:create] property :aliases, [String, Array], :default => [] -property :bbox, Array, :required => true +property :bbox, Array, :required => [:create] action :create do directory "/srv/#{new_resource.site}" do user "root" group "root" - mode 0o755 + mode "755" end directory "/srv/imagery/layers/#{new_resource.site}" do user "root" group "root" - mode 0o755 + mode "755" recursive true end directory "/srv/imagery/overlays/#{new_resource.site}" do user "root" group "root" - mode 0o755 + mode "755" recursive true end @@ -51,7 +51,7 @@ action :create do source "index.html.erb" user "root" group "root" - mode 0o644 + mode "644" variables :title => new_resource.title end @@ -59,28 +59,28 @@ action :create do source "robots.txt" user "root" group "root" - mode 0o644 + mode "644" end cookbook_file "/srv/#{new_resource.site}/imagery.css" do source "imagery.css" user "root" group "root" - mode 0o644 + mode "644" end cookbook_file "/srv/#{new_resource.site}/clientaccesspolicy.xml" do source "clientaccesspolicy.xml" user "root" group "root" - mode 0o644 + mode "644" end cookbook_file "/srv/#{new_resource.site}/crossdomain.xml" do source "crossdomain.xml" user "root" group "root" - mode 0o644 + mode "644" end layers = Dir.glob("/srv/imagery/layers/#{new_resource.site}/*.yml").collect do |path| @@ -91,7 +91,7 @@ action :create do source "imagery.js.erb" user "root" group "root" - mode 0o644 + mode "644" variables :bbox => new_resource.bbox, :layers => layers end @@ -106,8 +106,9 @@ action :create do "MS_ERRORFILE" => "stderr", "GDAL_CACHEMAX" => "512" limit_nofile 16384 - limit_cpu 60 - memory_max "4G" + limit_cpu 180 + memory_high "512M" + memory_max "1G" user "imagery" group "imagery" exec_start_pre "/bin/rm -f /run/mapserver-fastcgi/layer-#{new_resource.site}-#{index}.socket" @@ -137,19 +138,20 @@ action :create do nginx_site new_resource.site do template "nginx_imagery.conf.erb" directory "/srv/imagery/#{new_resource.site}" - restart_nginx false variables new_resource.to_hash end end action :delete do - service "mapserv-fcgi-#{new_resource.site}" do - provider Chef::Provider::Service::Systemd - action [:stop, :disable] - end + %w[0 1 2 3 4 5 6 7].each do |index| + service "mapserv-fcgi-#{new_resource.site}-#{index}" do + provider Chef::Provider::Service::Systemd + action [:stop, :disable] + end - systemd_service "mapserv-fcgi-#{new_resource.site}" do - action :delete + systemd_service "mapserv-fcgi-#{new_resource.site}-#{index}" do + action :delete + end end nginx_site new_resource.site do diff --git a/cookbooks/imagery/templates/default/imagery.js.erb b/cookbooks/imagery/templates/default/imagery.js.erb index 0f277444b..bee7747ab 100644 --- a/cookbooks/imagery/templates/default/imagery.js.erb +++ b/cookbooks/imagery/templates/default/imagery.js.erb @@ -1,12 +1,14 @@ function createMap(divName) { // Create a map - var map = L.map(divName).fitBounds(<%= @bbox.to_json %>); + var map = L.map(divName, { + worldCopyJump: true + }).fitBounds(<%= @bbox.to_json %>); // Create a layer switcher var layers = L.control.layers(null, null, {collapsed:false}).addTo(map); // Add OpenStreetMap layer - layers.addBaseLayer(L.tileLayer("//{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", { + layers.addBaseLayer(L.tileLayer("https://tile.openstreetmap.org/{z}/{x}/{y}.png", { attribution: "© OpenStreetMap and contributors, under an open license", maxZoom: 19 }), "OpenStreetMap"); diff --git a/cookbooks/imagery/templates/default/index.html.erb b/cookbooks/imagery/templates/default/index.html.erb index 148543937..c6975cb78 100644 --- a/cookbooks/imagery/templates/default/index.html.erb +++ b/cookbooks/imagery/templates/default/index.html.erb @@ -4,11 +4,11 @@ <%= @title %> - - - - - + + + + +