Planet redirect: add pbf+full history

[chef.git] / cookbooks / planet / templates / default / planet-mirror-redirect-update.erb

diff --git a/cookbooks/planet/templates/default/planet-mirror-redirect-update.erb b/cookbooks/planet/templates/default/planet-mirror-redirect-update.erb
index c673998e66916e40997ebb694ecb03f00fdf7bee..33abc665149e5988dd1ddd92a1c2ea0f21a48c89 100644 (file)
--- a/cookbooks/planet/templates/default/planet-mirror-redirect-update.erb
+++ b/cookbooks/planet/templates/default/planet-mirror-redirect-update.erb
@@ -8,10 +8,35 @@
        Modifies .htaccess
 */
 $_YEAR = date('Y');
-$_PLANET_FOLDER = '<%= node[:planet][:dump][:xml_directory] %>/'.$_YEAR.'/';
-$_PLANET_REGEX = "/^(planet|changesets)\-\d{6}(\-nolt)?\.osm\.(bz2|gz)$/";
-$_MIRROR = 'http://ftp.heanet.ie/mirrors/openstreetmap.org/planet/'.$_YEAR.'/';
-$_PLANET_HTACCESS = realpath($_PLANET_FOLDER.'../..').'/.htaccess';
+
+$_MIRROR = 'https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/';
+
+$_FOLDERS = [
+              "planet_bz2" => [
+                          'FOLDER'     => '<%= node[:planet][:dump][:xml_directory] %>/'.$_YEAR.'/',
+                          'REGEX'      => '/^(planet|changesets|discussions)\-\d{6}(\-nolt)?\.osm\.(bz2|gz)$/',
+                          'DIR_PREFIX' => 'planet/'.$_YEAR.'/'
+                          ],
+
+              "history_bz2" => [
+                          'FOLDER' => '<%= node[:planet][:dump][:xml_history_directory] %>/'.$_YEAR.'/',
+                          'REGEX'  => '/^(history)\-\d{6}(\-nolt)?\.osm\.(bz2|gz)$/',
+                          'DIR_PREFIX' => 'planet/full-history/'.$_YEAR.'/'
+                          ],
+              "planet_pbf" => [
+                          'FOLDER'     => '<%= node[:planet][:dump][:pbf_directory] %>/',
+                          'REGEX'      => '/^(planet)\-\d{6}(\-nolt)?\.osm\.pbf$/',
+                          'DIR_PREFIX' => 'pbf/'
+                          ],
+
+              "history_pbf" => [
+                          'FOLDER' => '<%= node[:planet][:dump][:pbf_history_directory] %>/',
+                          'REGEX'  => '/^(history)\-\d{6}(\-nolt)?\.osm\.pbf$/',
+                          'DIR_PREFIX' => 'pbf/full-history/'
+                          ]
+            ];
+
+$_PLANET_HTACCESS = realpath('<%= node[:planet][:dump][:xml_directory] %>/..').'/.htaccess';
 
 function _MIRROR_FILE_SIZE($url) {
        $ch = @curl_init();
@@ -22,34 +47,40 @@ function _MIRROR_FILE_SIZE($url) {
        $curl_result = curl_getinfo($ch);
        if ($curl_result['http_code']!='200') return FALSE;
        return ($curl_result['download_content_length']);
-}                      
+}
 
 if (!is_writable($_PLANET_HTACCESS)) die('File '.$_PLANET_HTACCESS.' is not writable by current user.'."\n");
 
-if (is_dir($_PLANET_FOLDER)) {
-       $htaccess_contents = file_get_contents($_PLANET_HTACCESS);
-       $htaccess_handle = fopen($_PLANET_HTACCESS, 'a');
-       if ($dh = opendir($_PLANET_FOLDER)) {
-               while (($file = readdir($dh)) !== false ) {
-                       if (preg_match($_PLANET_REGEX,$file)) {
-                               $file_slashed = 'planet/'.$_YEAR.'/'.str_replace(array('.','-'), array('\.','\-'), $file);
-                               if (strpos($htaccess_contents,$file_slashed) === false) {
-                                       //PHP IS BRAINDEAD filesize borked >4GB
-                                       //$file_size = $file_stats['size'];
-                                       $file_size = trim(`stat -c%s $_PLANET_FOLDER$file`);
-                                       sleep(rand(2,5));
-                                       $file_mirror_size = _MIRROR_FILE_SIZE($_MIRROR.$file);
-                                       if ($file_size==$file_mirror_size) {
-                                                       echo 'Adding: '.$file."\n";
-                                                       fwrite($htaccess_handle,        'RewriteRule'."\t".
-                                                                                       '^('.$file_slashed.')$'."\t".
-                                                                                       'http://ftp.heanet.ie/mirrors/openstreetmap.org/$1'."\t".
-                                                                                       '[R,L]'."\n");
-                                       }
-                               }
-                       }
-               }
-               closedir($dh);
-               fclose($htaccess_handle);
-       }
+$htaccess_contents = file_get_contents($_PLANET_HTACCESS);
+$htaccess_handle = fopen($_PLANET_HTACCESS, 'a');
+
+foreach ($_FOLDERS as $FOLDER) {
+
+  $_PLANET_FOLDER = $FOLDER["FOLDER"];
+  $_PLANET_REGEX  = $FOLDER["REGEX"];
+
+  if (!(is_dir($_PLANET_FOLDER))) {
+    continue;
+  }
+
+  if ($dh = opendir($_PLANET_FOLDER)) {
+    while (($file = readdir($dh)) !== false ) {
+      if (preg_match($_PLANET_REGEX, $file)) {
+        $file_slashed = $FOLDER['DIR_PREFIX'].str_replace(array('.','-'), array('\.','\-'), $file);
+        if (strpos($htaccess_contents, $file_slashed) === false) {
+          $file_size = filesize($_PLANET_FOLDER.$file);
+          sleep(rand(2,5));
+          $file_mirror_size = _MIRROR_FILE_SIZE($_MIRROR.$FOLDER['DIR_PREFIX'].$file);
+          if ($file_size==$file_mirror_size) {
+            echo 'Adding: '.$file."\n";
+            fwrite($htaccess_handle, 'RewriteRule'."\t". '^('.$file_slashed.')$'."\t".$_MIRROR.'$1'."\t".'[R,L]'."\n");
+          }
+        }
+      }
+    }
+    closedir($dh);
+  }
 }
+
+fclose($htaccess_handle);
+