From c2b0461cee07adc79a479348c7cea0dae59ddf2b Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Mon, 25 Oct 2021 12:50:26 +0100 Subject: [PATCH] Convert gdns map build to be request based --- Makefile | 4 +- bandwidth/nominatim.openstreetmap.yml | 242 -------------------------- bin/mkcountries | 10 +- bin/mkgeo | 44 ++--- requests/nominatim.openstreetmap.yml | 236 +++++++++++++++++++++++++ src/nominatim.openstreetmap.yml | 6 +- 6 files changed, 268 insertions(+), 274 deletions(-) delete mode 100644 bandwidth/nominatim.openstreetmap.yml create mode 100644 requests/nominatim.openstreetmap.yml diff --git a/Makefile b/Makefile index 0328866..ee2fd7b 100644 --- a/Makefile +++ b/Makefile @@ -30,5 +30,5 @@ clean: lib/countries.xml: curl -s -o $@ http://api.geonames.org/countryInfo?username=demo -origins/nominatim.openstreetmap.yml: bin/mkcountries lib/countries.xml bandwidth/nominatim.openstreetmap.yml - bin/mkcountries bandwidth/nominatim.openstreetmap.yml origins/nominatim.openstreetmap.yml +origins/nominatim.openstreetmap.yml: bin/mkcountries lib/countries.xml requests/nominatim.openstreetmap.yml + bin/mkcountries requests/nominatim.openstreetmap.yml origins/nominatim.openstreetmap.yml diff --git a/bandwidth/nominatim.openstreetmap.yml b/bandwidth/nominatim.openstreetmap.yml deleted file mode 100644 index 32beae0..0000000 --- a/bandwidth/nominatim.openstreetmap.yml +++ /dev/null @@ -1,242 +0,0 @@ ---- -AD: 2.86888888889 -AE: 23.3488888889 -AF: 1.31777777778 -AG: 2.78 -AI: 0.0733333333333 -AL: 7.23777777778 -AM: 11.0422222222 -AO: 6.88888888889 -AQ: 0.0311111111111 -AR: 528.111111111 -AS: 4.29333333333 -AT: 312.84 -AU: 216.437777778 -AW: 2.46444444444 -AX: 0.0377777777778 -AZ: 5.50888888889 -BA: 6.07333333333 -BB: 2.85333333333 -BD: 36.64 -BE: 235.76 -BF: 5.08 -BG: 23.8266666667 -BH: 4.57555555556 -BI: 0.84 -BJ: 0.391111111111 -BL: 0.135555555556 -BM: 0.368888888889 -BN: 1.78888888889 -BO: 24.3533333333 -BQ: 0.0488888888889 -BR: 808.902222222 -BS: 0.446666666667 -BT: 9.26888888889 -BW: 19.9311111111 -BY: 73.8688888889 -BZ: 0.817777777778 -CA: 500.788888889 -CC: 0.104444444444 -CD: 0.78 -CF: 0.0777777777778 -CG: 0.811111111111 -CH: 180.373333333 -CI: 35.5088888889 -CK: 0.0533333333333 -CL: 167.984444444 -CM: 11.9977777778 -CN: 134.986666667 -CO: 369.002222222 -CR: 4.63555555556 -CU: 2.92666666667 -CV: 0.137777777778 -CW: 0.32 -CX: 0.00444444444444 -CY: 24.6577777778 -CZ: 87.84 -DE: 2324.72222222 -DJ: 0.464444444444 -DK: 43.8622222222 -DM: 0.0555555555556 -DO: 19.7577777778 -DZ: 16.6888888889 -EC: 92.8555555556 -EE: 43.9266666667 -EG: 86.5244444444 -EH: 0.00444444444444 -ER: 0.00888888888889 -ES: 279.497777778 -ET: 3 -FI: 49.3488888889 -FJ: 0.506666666667 -FK: 2.38444444444 -FM: 0.0377777777778 -FO: 25.5711111111 -FR: 784.886666667 -GA: 0.526666666667 -GB: 504.966666667 -GD: 0.0977777777778 -GE: 14.6311111111 -GF: 0.322222222222 -GG: 0.491111111111 -GH: 9.01555555556 -GI: 0.215555555556 -GL: 3.12 -GM: 0.537777777778 -GN: 1.00444444444 -GP: 0.895555555556 -GQ: 0.117777777778 -GR: 163.475555556 -GT: 5.70222222222 -GU: 0.326666666667 -GW: 0.0422222222222 -GY: 1.5 -HK: 59.1088888889 -HN: 22.2111111111 -HR: 133.737777778 -HT: 0.686666666667 -HU: 111.706666667 -ID: 381.413333333 -IE: 3421.39333333 -IL: 215.791111111 -IM: 0.342222222222 -IN: 484.255555556 -IO: 0.0288888888889 -IQ: 18.8822222222 -IR: 295.331111111 -IS: 22.9177777778 -IT: 362.433333333 -JE: 0.551111111111 -JM: 1.8 -JO: 3.26 -JP: 48.7711111111 -KE: 47.2155555556 -KG: 9.83333333333 -KH: 12.4244444444 -KI: 0.0266666666667 -KM: 0.0466666666667 -KN: 0.275555555556 -KR: 64.56 -KW: 13.0622222222 -KY: 2.92222222222 -KZ: 806.228888889 -LA: 8.74888888889 -LB: 1.72222222222 -LC: 0.142222222222 -LI: 0.124444444444 -LK: 35.6422222222 -LR: 0.697777777778 -LS: 0.126666666667 -LT: 64.9866666667 -LU: 19.9866666667 -LV: 14.3955555556 -LY: 1.91333333333 -MA: 109.646666667 -MC: 0.504444444444 -MD: 29.9288888889 -ME: 0.575555555556 -MF: 0.0644444444444 -MG: 2.29111111111 -MH: 0.0288888888889 -MK: 2.22444444444 -ML: 2.43777777778 -MM: 6.92888888889 -MN: 18.56 -MO: 0.748888888889 -MP: 0.08 -MQ: 4.79555555556 -MR: 0.526666666667 -MS: 0.0111111111111 -MT: 13.6088888889 -MU: 6.8 -MV: 0.982222222222 -MW: 0.902222222222 -MX: 256.54 -MY: 80.1244444444 -MZ: 15.3066666667 -NA: 8.03555555556 -NC: 0.924444444444 -NE: 0.451111111111 -NF: 0.00222222222222 -NG: 37.3733333333 -NI: 6.52444444444 -NL: 396.193333333 -NO: 27.7666666667 -NP: 2.64666666667 -NR: 3.38666666667 -NZ: 20.3733333333 -OM: 3.52888888889 -PA: 5.06888888889 -PE: 58.9888888889 -PF: 0.208888888889 -PG: 0.222222222222 -PH: 121.284444444 -PK: 32.86 -PL: 743.211111111 -PM: 0.0133333333333 -PR: 5.56 -PS: 13.9244444444 -PT: 57.5111111111 -PW: 0.02 -PY: 400.531111111 -QA: 7.32666666667 -RE: 20.6755555556 -RO: 76.48 -RS: 57.4933333333 -RU: 1054.97777778 -RW: 0.673333333333 -SA: 70.8666666667 -SB: 0.115555555556 -SC: 1.24444444444 -SD: 58.4177777778 -SE: 98.8555555556 -SG: 221.015555556 -SH: 0.0155555555556 -SI: 106.275555556 -SK: 27.84 -SL: 9.67555555556 -SM: 8.70444444444 -SN: 5.27777777778 -SO: 1.10444444444 -SR: 3.95111111111 -SS: 0.351111111111 -ST: 0.0377777777778 -SV: 3.54888888889 -SX: 0.135555555556 -SY: 40.98 -SZ: 0.64 -TC: 0.04 -TD: 0.0777777777778 -TG: 1.98666666667 -TH: 407.406666667 -TJ: 5.63333333333 -TK: 0.00444444444444 -TL: 0.142222222222 -TM: 1.22222222222 -TN: 11.4555555556 -TO: 0.02 -TR: 216.353333333 -TT: 10.3533333333 -TV: 2.75555555556 -TW: 58.3933333333 -TZ: 12.3488888889 -UA: 141.504444444 -UG: 5.98666666667 -US: 3539.80666667 -UY: 19.1933333333 -UZ: 63.5244444444 -VA: 0.02 -VC: 0.0444444444444 -VE: 55.3066666667 -VG: 0.0711111111111 -VI: 0.173333333333 -VN: 125.317777778 -VU: 0.104444444444 -WF: 0.00888888888889 -WS: 0.06 -XK: 0.113333333333 -YE: 2.95111111111 -YT: 1.34 -ZA: 249.057777778 -ZM: 3.03111111111 -ZW: 6 diff --git a/bin/mkcountries b/bin/mkcountries index 42863a7..ca8427a 100755 --- a/bin/mkcountries +++ b/bin/mkcountries @@ -7,7 +7,7 @@ use XML::TreeBuilder; use YAML; # Get arguments -my $bandwidthfile = shift @ARGV; +my $requestsfile = shift @ARGV; my $originsfile = shift @ARGV; # Initialise origins @@ -19,8 +19,8 @@ my $countries = XML::TreeBuilder->new; # Parse the country database $countries->parsefile("lib/countries.xml"); -# Load the per-country bandwidth details -my $bandwidth = YAML::LoadFile($bandwidthfile); +# Load the per-country requests details +my $requests = YAML::LoadFile($requestsfile); # Fill in country table and work out which clusters each can use foreach my $country ($countries->look_down("_tag" => "country")) @@ -28,7 +28,7 @@ foreach my $country ($countries->look_down("_tag" => "country")) my $code = $country->look_down("_tag" => "countryCode")->as_text; my $name = $country->look_down("_tag" => "countryName")->as_text; my $population = $country->look_down("_tag" => "population")->as_text; - my $bandwidth = $bandwidth->{$code} || 0; + my $requests = $requests->{$code} || 0; my $continent = $country->look_down("_tag" => "continent")->as_text; my $west = $country->look_down("_tag" => "west")->as_text; my $north = $country->look_down("_tag" => "north")->as_text; @@ -40,7 +40,7 @@ foreach my $country ($countries->look_down("_tag" => "country")) $origins->{$code} = { code => $code, name => $name, country => $code, continent => $continent, - bandwidth => $bandwidth, lat => $lat, lon => $lon + requests => $requests, lat => $lat, lon => $lon }; } diff --git a/bin/mkgeo b/bin/mkgeo index 7373c2d..94c600a 100755 --- a/bin/mkgeo +++ b/bin/mkgeo @@ -25,12 +25,12 @@ while (my($name,$cluster) = each %$clusters) { if ($cluster->{servers}) { - $cluster->{bandwidth} = 0; + $cluster->{requests} = 0; foreach my $server (@{$cluster->{servers}}) { $server->{cluster} = $cluster; - $cluster->{bandwidth} = $cluster->{bandwidth} + $server->{bandwidth}; + $cluster->{requests} = $cluster->{requests} + $server->{requests}; push @servers, $server; } @@ -40,7 +40,7 @@ while (my($name,$cluster) = each %$clusters) my $server = { cluster => $cluster, statuscake => $cluster->{statuscake}, - bandwidth => $cluster->{bandwidth}, + requests => $cluster->{requests}, cname => $cluster->{cname}, ipv4 => $cluster->{ipv4}, ipv6 => $cluster->{ipv6} @@ -133,7 +133,7 @@ foreach my $server (@servers) } else { - $server->{cluster}->{bandwidth} = $server->{cluster}->{bandwidth} - $server->{bandwidth}; + $server->{cluster}->{requests} = $server->{cluster}->{requests} - $server->{requests}; } } @@ -143,8 +143,8 @@ my $targetorigins = {}; # Initialise cluster details while (my($name,$cluster) = each %$clusters) { - $cluster->{bandwidth_limit} = $cluster->{bandwidth} * 1024 * 1024; - $cluster->{bandwidth_used} = 0; + $cluster->{requests_limit} = $cluster->{requests}; + $cluster->{requests_used} = 0; next if $cluster->{global}; @@ -153,7 +153,7 @@ while (my($name,$cluster) = each %$clusters) name => $cluster->{name}, lat => $cluster->{lat}, lon => $cluster->{lon}, - bandwidth => 0 + requests => 0 }; } @@ -192,7 +192,7 @@ foreach my $origin (values %$origins) allocate_clusters(@mappings); # If we failed to allocate every origin then loop, increasing -# the bandwidth for each cluster by a little and retrying until +# the requests for each cluster by a little and retrying until # we manage to allocate everything while (grep { !exists($_->{cluster}) } values %$origins) { @@ -202,11 +202,11 @@ while (grep { !exists($_->{cluster}) } values %$origins) delete $origin->{cluster}; } - # Reset bandwidth usage for clusters and increase limits by 10% + # Reset requests usage for clusters and increase limits by 10% foreach my $cluster (values %$clusters) { - $cluster->{bandwidth_used} = 0; - $cluster->{bandwidth_limit} = $cluster->{bandwidth_limit} * 1.1; + $cluster->{requests_used} = 0; + $cluster->{requests_limit} = $cluster->{requests_limit} * 1.1; } # Try the allocate again @@ -265,7 +265,7 @@ foreach my $origin (sort { $a->{name} cmp $b->{name} } values %$origins) next if $cluster->{global}; - $targetorigins->{$cluster->{name}}->{bandwidth} += $origin->{bandwidth}; + $targetorigins->{$cluster->{name}}->{requests} += $origin->{requests}; } # Skip default records if we don't need them @@ -367,15 +367,15 @@ if (defined($gdnsname)) if ($server->{status} eq "up") { my $number = sprintf("%02d", $index + 1); - my $bandwidth = $server->{bandwidth}; + my $requests = $server->{requests}; if (my $cname = $server->{cname}) { - $gdnsweightedfile->print(" ${name}-${number} = [ ${cname}., ${bandwidth} ]\n"); + $gdnsweightedfile->print(" ${name}-${number} = [ ${cname}., ${requests} ]\n"); } else { - $gdnsweightedfile->print(" ${name}-${number} = [ ${name}-${number}.${zone}.openstreetmap.org., ${bandwidth} ]\n"); + $gdnsweightedfile->print(" ${name}-${number} = [ ${name}-${number}.${zone}.openstreetmap.org., ${requests} ]\n"); } } } @@ -497,7 +497,7 @@ sub allocate_clusters my @mappings = sort { compare_mappings($a, $b) } @_; # Loop over the mappings, trying to assign each origin to the - # nearest cluster, but subject to the bandwidth limits + # nearest cluster, but subject to the request limits while (my $mapping = shift @mappings) { my @group; @@ -509,16 +509,16 @@ sub allocate_clusters push @group, shift @mappings; } - for my $mapping (sort compare_bandwidth @group) + for my $mapping (sort compare_requests @group) { my $origin = $mapping->{origin}; my $cluster = $mapping->{cluster}; if (!exists($origin->{cluster}) && - $cluster->{bandwidth_used} + $origin->{bandwidth} <= $cluster->{bandwidth_limit}) + $cluster->{requests_used} + $origin->{requests} <= $cluster->{requests_limit}) { $origin->{cluster} = $cluster; - $cluster->{bandwidth_used} = $cluster->{bandwidth_used} + $origin->{bandwidth}; + $cluster->{requests_used} = $cluster->{requests_used} + $origin->{requests}; } } } @@ -541,10 +541,10 @@ sub compare_mappings # # Compare two mappings to decide which to try first # -sub compare_bandwidth +sub compare_requests { - my $a_used = ( $a->{cluster}->{bandwidth_used} * 100.0 ) / ( $a->{cluster}->{bandwidth_limit} * 1.0 ); - my $b_used = ( $b->{cluster}->{bandwidth_used} * 100.0 ) / ( $b->{cluster}->{bandwidth_limit} * 1.0 ); + my $a_used = ( $a->{cluster}->{requests_used} * 100.0 ) / ( $a->{cluster}->{requests_limit} * 1.0 ); + my $b_used = ( $b->{cluster}->{requests_used} * 100.0 ) / ( $b->{cluster}->{requests_limit} * 1.0 ); return $a_used <=> $b_used; } diff --git a/requests/nominatim.openstreetmap.yml b/requests/nominatim.openstreetmap.yml new file mode 100644 index 0000000..931a653 --- /dev/null +++ b/requests/nominatim.openstreetmap.yml @@ -0,0 +1,236 @@ +--- +AD: 0.0302777777778 +AE: 1.98777777778 +AF: 0.0408333333333 +AG: 0.00222222222222 +AI: 0.249166666667 +AL: 2.22527777778 +AM: 1.59555555556 +AO: 0.583055555556 +AQ: 0.0025 +AR: 10.4358333333 +AS: 0.000555555555556 +AT: 15.6136111111 +AU: 18.2058333333 +AW: 0.0358333333333 +AX: 0.0152777777778 +AZ: 1.45888888889 +BA: 0.412222222222 +BB: 0.0313888888889 +BD: 2.07638888889 +BE: 5.54 +BF: 0.0527777777778 +BG: 2.63083333333 +BH: 0.121944444444 +BI: 0.0691666666667 +BJ: 0.257777777778 +BL: 0.00666666666667 +BM: 0.0594444444444 +BN: 0.106388888889 +BO: 2.04444444444 +BQ: 0.00416666666667 +BR: 64.7872222222 +BS: 0.0113888888889 +BT: 0.0211111111111 +BW: 0.0675 +BY: 3.45055555556 +BZ: 0.015 +CA: 18.0144444444 +CD: 0.364722222222 +CF: 0.005 +CG: 0.0661111111111 +CH: 11.8022222222 +CI: 0.66 +CK: 0.000555555555556 +CL: 5.82055555556 +CM: 1.11805555556 +CN: 24.0813888889 +CO: 29.0861111111 +CR: 0.450555555556 +CU: 0.0975 +CV: 0.0138888888889 +CW: 0.0155555555556 +CY: 0.251944444444 +CZ: 6.90222222222 +DE: 229.34 +DJ: 0.0125 +DK: 3.63361111111 +DM: 0.00194444444444 +DO: 3.98111111111 +DZ: 0.991666666667 +EC: 3.18333333333 +EE: 3.37527777778 +EG: 4.04583333333 +ES: 23.5105555556 +ET: 0.326666666667 +FI: 4.62611111111 +FJ: 0.0161111111111 +FK: 0.000833333333333 +FM: 0.000833333333333 +FO: 0.613333333333 +FR: 49.7969444444 +GA: 0.0636111111111 +GB: 30.8966666667 +GD: 0.00138888888889 +GE: 0.204166666667 +GF: 0.0297222222222 +GG: 0.00527777777778 +GH: 1.145 +GI: 0.00888888888889 +GL: 0.0880555555556 +GM: 0.0172222222222 +GN: 0.0661111111111 +GP: 0.0627777777778 +GQ: 0.000833333333333 +GR: 21.7488888889 +GT: 15.895 +GU: 0.0144444444444 +GW: 0.000833333333333 +GY: 0.0122222222222 +HK: 1.82722222222 +HN: 0.228333333333 +HR: 8.52138888889 +HT: 0.00666666666667 +HU: 4.32694444444 +ID: 38.6758333333 +IE: 30.0491666667 +IL: 6.955 +IM: 0.142777777778 +IN: 45.3866666667 +IO: 0.000555555555556 +IQ: 0.7525 +IR: 4.71722222222 +IS: 0.1725 +IT: 31.4088888889 +JE: 0.0513888888889 +JM: 0.130833333333 +JO: 0.599166666667 +JP: 11.1275 +KE: 4.11861111111 +KG: 0.218055555556 +KH: 0.379722222222 +KI: 0.000277777777778 +KM: 0.00388888888889 +KN: 0.000555555555556 +KR: 3.85888888889 +KW: 0.819166666667 +KY: 0.00416666666667 +KZ: 2.39666666667 +LA: 0.366944444444 +LB: 0.231944444444 +LC: 0.01 +LI: 0.0236111111111 +LK: 1.51722222222 +LR: 0.0355555555556 +LS: 0.00305555555556 +LT: 6.14444444444 +LU: 2.09 +LV: 2.19222222222 +LY: 0.0883333333333 +MA: 5.09111111111 +MC: 0.0163888888889 +MD: 3.43583333333 +ME: 0.145833333333 +MF: 0.00361111111111 +MG: 0.525277777778 +MH: 0.00305555555556 +MK: 0.112222222222 +ML: 0.0725 +MM: 0.0772222222222 +MN: 1.35861111111 +MO: 0.0475 +MP: 0.00194444444444 +MQ: 0.0322222222222 +MR: 0.0458333333333 +MS: 0.0025 +MT: 0.314166666667 +MU: 0.234166666667 +MV: 0.0244444444444 +MW: 0.0447222222222 +MX: 30.385 +MY: 6.13722222222 +MZ: 0.246666666667 +NA: 0.486666666667 +NC: 0.0458333333333 +NE: 0.173611111111 +NF: 0.000277777777778 +NG: 6.42138888889 +NI: 0.0344444444444 +NL: 30.8016666667 +NO: 4.5425 +NP: 1.77666666667 +NZ: 2.23888888889 +OM: 0.176388888889 +PA: 0.261111111111 +PE: 1.65388888889 +PF: 0.285555555556 +PG: 0.055 +PH: 4.54111111111 +PK: 1.99972222222 +PL: 53.8841666667 +PM: 0.000833333333333 +PR: 0.4075 +PS: 1.21805555556 +PT: 2.37777777778 +PW: 0.000277777777778 +PY: 4.65277777778 +QA: 0.277777777778 +RE: 0.497222222222 +RO: 4.84888888889 +RS: 1.51027777778 +RU: 79.0533333333 +RW: 0.0455555555556 +SA: 3.76305555556 +SB: 0.194444444444 +SC: 1.6425 +SD: 0.899722222222 +SE: 6.68861111111 +SG: 13.1025 +SI: 0.849166666667 +SK: 2.11944444444 +SL: 0.163333333333 +SM: 0.0205555555556 +SN: 0.185277777778 +SO: 0.146944444444 +SR: 0.0136111111111 +SS: 0.00777777777778 +ST: 0.00138888888889 +SV: 0.171666666667 +SX: 0.00444444444444 +SY: 0.728888888889 +SZ: 0.0116666666667 +TC: 0.0025 +TD: 0.00388888888889 +TG: 0.0719444444444 +TH: 5.42694444444 +TJ: 0.534444444444 +TL: 0.0172222222222 +TM: 0.0577777777778 +TN: 1.01861111111 +TO: 0.000277777777778 +TR: 14.9716666667 +TT: 1.00166666667 +TV: 0.00583333333333 +TW: 5.9775 +TZ: 0.296944444444 +UA: 37.3986111111 +UG: 0.272222222222 +UM: 0.000833333333333 +US: 324.626944444 +UY: 0.799166666667 +UZ: 7.335 +VA: 0.00555555555556 +VC: 0.000833333333333 +VE: 4.03694444444 +VG: 0.00111111111111 +VI: 0.00944444444444 +VN: 15.1244444444 +VU: 0.00277777777778 +WF: 0.0183333333333 +WS: 0.00111111111111 +XK: 0.00166666666667 +YE: 0.0991666666667 +YT: 0.00527777777778 +ZA: 8.20611111111 +ZM: 0.110277777778 +ZW: 0.420277777778 diff --git a/src/nominatim.openstreetmap.yml b/src/nominatim.openstreetmap.yml index 746dbda..6a2f90a 100644 --- a/src/nominatim.openstreetmap.yml +++ b/src/nominatim.openstreetmap.yml @@ -5,7 +5,7 @@ amsterdam: - 2217359 - 2217360 colour: "#8dd3c7" - bandwidth: 0.01 + requests: 800 ipv4: 130.117.76.9 ipv6: 2001:978:2:2c::172:9 default: "xx" @@ -17,7 +17,7 @@ slough: - 2217363 - 2217364 colour: "#bebada" - bandwidth: 0.002 + requests: 200 ipv4: 193.60.236.18 default: "xx" @@ -28,7 +28,7 @@ corvallis: - 5769055 - 5769056 colour: "#ffffb3" - bandwidth: 0.01 + requests: 800 ipv4: 140.211.167.100 ipv6: 2605:bc80:3010:700::8cd3:a764 default: "xx" -- 2.45.1