From: Tom Hughes Date: Sun, 4 May 2008 11:26:36 +0000 (+0000) Subject: Merge 7649:7673 from trunk. X-Git-Tag: live~7601^2~371 X-Git-Url: https://git.openstreetmap.org/rails.git/commitdiff_plain/cf304fe60f05f48251a480d19ce3ea9440f8a10e?hp=02efd7ba4a736c09a45f6fb4ac329650498d9431 Merge 7649:7673 from trunk. --- diff --git a/app/controllers/changeset_controller.rb b/app/controllers/changeset_controller.rb new file mode 100644 index 000000000..1041c7469 --- /dev/null +++ b/app/controllers/changeset_controller.rb @@ -0,0 +1,26 @@ +# The ChangesetController is the RESTful interface to Changeset objects + +class ChangesetController < ApplicationController + require 'xml/libxml' + + before_filter :authorize, :only => [:create, :update, :delete] + before_filter :check_write_availability, :only => [:create, :update, :delete] + before_filter :check_read_availability, :except => [:create, :update, :delete] + + # Create a changeset from XML. + def create + if request.put? + cs = Changeset.from_xml(request.raw_post, true) + + if cs + cs.user_id = @user.id + cs.save_with_tags! + render :text => cs.id.to_s, :content_type => "text/plain" + else + render :nothing => true, :status => :bad_request + end + else + render :nothing => true, :status => :method_not_allowed + end + end +end diff --git a/app/controllers/changeset_tag_controller.rb b/app/controllers/changeset_tag_controller.rb new file mode 100644 index 000000000..3e8db3fc2 --- /dev/null +++ b/app/controllers/changeset_tag_controller.rb @@ -0,0 +1,9 @@ +class ChangesetTagController < ApplicationController + layout 'site' + + def search + @tags = ChangesetTag.find(:all, :limit => 11, :conditions => ["match(v) against (?)", params[:query][:query].to_s] ) + end + + +end diff --git a/app/controllers/node_controller.rb b/app/controllers/node_controller.rb index edc3675e5..9f8f4a38b 100644 --- a/app/controllers/node_controller.rb +++ b/app/controllers/node_controller.rb @@ -15,6 +15,7 @@ class NodeController < ApplicationController node = Node.from_xml(request.raw_post, true) if node + node.version = 0 node.user_id = @user.id node.visible = true node.save_with_history! diff --git a/app/controllers/relation_controller.rb b/app/controllers/relation_controller.rb index 2b1ba6c75..f4e938176 100644 --- a/app/controllers/relation_controller.rb +++ b/app/controllers/relation_controller.rb @@ -15,6 +15,7 @@ class RelationController < ApplicationController if !relation.preconditions_ok? render :text => "", :status => :precondition_failed else + relation.version = 0 relation.user_id = @user.id relation.save_with_history! diff --git a/app/controllers/way_controller.rb b/app/controllers/way_controller.rb index 3b6491cf0..a7f74e50c 100644 --- a/app/controllers/way_controller.rb +++ b/app/controllers/way_controller.rb @@ -15,6 +15,7 @@ class WayController < ApplicationController if !way.preconditions_ok? render :text => "", :status => :precondition_failed else + way.version = 0 way.user_id = @user.id way.save_with_history! diff --git a/app/models/changeset.rb b/app/models/changeset.rb new file mode 100644 index 000000000..85621dce2 --- /dev/null +++ b/app/models/changeset.rb @@ -0,0 +1,73 @@ +class Changeset < ActiveRecord::Base + require 'xml/libxml' + + belongs_to :user + + has_many :changeset_tags, :foreign_key => 'id' + + def self.from_xml(xml, create=false) + begin + p = XML::Parser.new + p.string = xml + doc = p.parse + + cs = Changeset.new + + doc.find('//osm/changeset').each do |pt| + if create + cs.created_at = Time.now + end + + pt.find('tag').each do |tag| + cs.add_tag_keyval(tag['k'], tag['v']) + end + end + rescue Exception => ex + print "noes "+ ex.to_s + "\n" + cs = nil + end + + return cs + end + + def tags + unless @tags + @tags = {} + self.changeset_tags.each do |tag| + @tags[tag.k] = tag.v + end + end + @tags + end + + def tags=(t) + @tags = t + end + + def add_tag_keyval(k, v) + @tags = Hash.new unless @tags + @tags[k] = v + end + + def save_with_tags! + t = Time.now + + Changeset.transaction do + # fixme update modified_at time? + self.save! + end + + ChangesetTag.transaction do + tags = self.tags + ChangesetTag.delete_all(['id = ?', self.id]) + + tags.each do |k,v| + tag = ChangesetTag.new + tag.k = k + tag.v = v + tag.id = self.id + tag.save! + end + end + end +end diff --git a/app/models/changeset_tag.rb b/app/models/changeset_tag.rb new file mode 100644 index 000000000..6298fbe77 --- /dev/null +++ b/app/models/changeset_tag.rb @@ -0,0 +1,5 @@ +class ChangesetTag < ActiveRecord::Base + + belongs_to :changeset, :foreign_key => 'id' + +end diff --git a/app/models/node.rb b/app/models/node.rb index cc646b768..5e5e7a0b2 100644 --- a/app/models/node.rb +++ b/app/models/node.rb @@ -1,5 +1,3 @@ -# The node model represents a current existing node, that is, the latest version. Use OldNode for historical nodes. - class Node < GeoRecord require 'xml/libxml' @@ -10,16 +8,21 @@ class Node < GeoRecord validates_numericality_of :latitude, :longitude validate :validate_position - has_many :ways, :through => :way_nodes has_many :old_nodes, :foreign_key => :id has_many :way_nodes + has_many :node_tags, :foreign_key => :id belongs_to :user - # Sanity check the latitude and longitude and add an error if it's broken def validate_position errors.add_to_base("Node is not in the world") unless in_world? end + def in_world? + return false if self.lat < -90 or self.lat > 90 + return false if self.lon < -180 or self.lon > 180 + return true + end + # # Search for nodes matching tags within bounding_box # @@ -80,10 +83,9 @@ class Node < GeoRecord tags = [] pt.find('tag').each do |tag| - tags << [tag['k'],tag['v']] + node.add_tag_key_val(tag['k'],tag['v']) end - node.tags = Tags.join(tags) end rescue node = nil @@ -92,24 +94,37 @@ class Node < GeoRecord return node end - # Save this node with the appropriate OldNode object to represent it's history. def save_with_history! + t = Time.now Node.transaction do - self.timestamp = Time.now + self.version += 1 + self.timestamp = t self.save! + + # Create a NodeTag + tags = self.tags + NodeTag.delete_all(['id = ?', self.id]) + tags.each do |k,v| + tag = NodeTag.new + tag.k = k + tag.v = v + tag.id = self.id + tag.save! + end + + # Create an OldNode old_node = OldNode.from_node(self) - old_node.save! + old_node.timestamp = t + old_node.save_with_dependencies! end end - # Turn this Node in to a complete OSM XML object with wrapper def to_xml doc = OSM::API.new.get_xml_doc doc.root << to_xml_node() return doc end - # Turn this Node in to an XML Node without the wrapper. def to_xml_node(user_display_name_cache = nil) el1 = XML::Node.new 'node' el1['id'] = self.id.to_s @@ -128,7 +143,7 @@ class Node < GeoRecord el1['user'] = user_display_name_cache[self.user_id] unless user_display_name_cache[self.user_id].nil? - Tags.split(self.tags) do |k,v| + self.tags.each do |k,v| el2 = XML::Node.new('tag') el2['k'] = k.to_s el2['v'] = v.to_s @@ -137,10 +152,10 @@ class Node < GeoRecord el1['visible'] = self.visible.to_s el1['timestamp'] = self.timestamp.xmlschema + el1['version'] = self.version.to_s return el1 end - # Return the node's tags as a Hash of keys and their values def tags_as_hash hash = {} Tags.split(self.tags) do |k,v| @@ -148,4 +163,26 @@ class Node < GeoRecord end hash end + + def tags + unless @tags + @tags = {} + self.node_tags.each do |tag| + @tags[tag.k] = tag.v + end + end + @tags + end + + def tags=(t) + @tags = t + end + + def add_tag_key_val(k,v) + @tags = Hash.new unless @tags + @tags[k] = v + end + + + end diff --git a/app/models/node_tag.rb b/app/models/node_tag.rb new file mode 100644 index 000000000..9795ff493 --- /dev/null +++ b/app/models/node_tag.rb @@ -0,0 +1,5 @@ +class NodeTag < ActiveRecord::Base + set_table_name 'current_node_tags' + + belongs_to :node, :foreign_key => 'id' +end diff --git a/app/models/old_node.rb b/app/models/old_node.rb index 2c3e93b20..247a69923 100644 --- a/app/models/old_node.rb +++ b/app/models/old_node.rb @@ -27,6 +27,7 @@ class OldNode < GeoRecord old_node.timestamp = node.timestamp old_node.user_id = node.user_id old_node.id = node.id + old_node.version = node.version return old_node end @@ -37,7 +38,7 @@ class OldNode < GeoRecord el1['lon'] = self.lon.to_s el1['user'] = self.user.display_name if self.user.data_public? - Tags.split(self.tags) do |k,v| + self.tags.each do |k,v| el2 = XML::Node.new('tag') el2['k'] = k.to_s el2['v'] = v.to_s @@ -46,6 +47,41 @@ class OldNode < GeoRecord el1['visible'] = self.visible.to_s el1['timestamp'] = self.timestamp.xmlschema + el1['version'] = self.version.to_s return el1 end + + def save_with_dependencies! + save! + #not sure whats going on here + clear_aggregation_cache + clear_association_cache + #ok from here + @attributes.update(OldNode.find(:first, :conditions => ['id = ? AND timestamp = ?', self.id, self.timestamp]).instance_variable_get('@attributes')) + + self.tags.each do |k,v| + tag = OldNodeTag.new + tag.k = k + tag.v = v + tag.id = self.id + tag.version = self.version + tag.save! + end + end + + def tags + unless @tags + @tags = Hash.new + OldNodeTag.find(:all, :conditions => ["id = ? AND version = ?", self.id, self.version]).each do |tag| + @tags[tag.k] = tag.v + end + end + @tags = Hash.new unless @tags + @tags + end + + def tags=(t) + @tags = t + end + end diff --git a/app/models/old_node_tag.rb b/app/models/old_node_tag.rb new file mode 100644 index 000000000..26a6c92b4 --- /dev/null +++ b/app/models/old_node_tag.rb @@ -0,0 +1,7 @@ +class OldNodeTag < ActiveRecord::Base + belongs_to :user + + set_table_name 'node_tags' + + +end diff --git a/app/models/old_relation.rb b/app/models/old_relation.rb index 6da7814c2..03d5aebff 100644 --- a/app/models/old_relation.rb +++ b/app/models/old_relation.rb @@ -9,6 +9,7 @@ class OldRelation < ActiveRecord::Base old_relation.user_id = relation.user_id old_relation.timestamp = relation.timestamp old_relation.id = relation.id + old_relation.version = relation.version old_relation.members = relation.members old_relation.tags = relation.tags return old_relation @@ -91,6 +92,7 @@ class OldRelation < ActiveRecord::Base el1['visible'] = self.visible.to_s el1['timestamp'] = self.timestamp.xmlschema el1['user'] = self.user.display_name if self.user.data_public? + el1['version'] = self.version.to_s self.old_members.each do |member| e = XML::Node.new 'member' diff --git a/app/models/old_way.rb b/app/models/old_way.rb index a2b165e42..136e64793 100644 --- a/app/models/old_way.rb +++ b/app/models/old_way.rb @@ -9,6 +9,7 @@ class OldWay < ActiveRecord::Base old_way.user_id = way.user_id old_way.timestamp = way.timestamp old_way.id = way.id + old_way.version = way.version old_way.nds = way.nds old_way.tags = way.tags return old_way @@ -94,6 +95,7 @@ class OldWay < ActiveRecord::Base el1['visible'] = self.visible.to_s el1['timestamp'] = self.timestamp.xmlschema el1['user'] = self.user.display_name if self.user.data_public? + el1['version'] = self.version.to_s self.old_nodes.each do |nd| # FIXME need to make sure they come back in the right order e = XML::Node.new 'nd' diff --git a/app/models/relation.rb b/app/models/relation.rb index 61344bdfb..ec8c92c92 100644 --- a/app/models/relation.rb +++ b/app/models/relation.rb @@ -58,6 +58,7 @@ class Relation < ActiveRecord::Base el1['id'] = self.id.to_s el1['visible'] = self.visible.to_s el1['timestamp'] = self.timestamp.xmlschema + el1['version'] = self.version.to_s user_display_name_cache = {} if user_display_name_cache.nil? @@ -167,13 +168,12 @@ class Relation < ActiveRecord::Base def save_with_history! Relation.transaction do t = Time.now + self.version += 1 self.timestamp = t self.save! tags = self.tags - RelationTag.delete_all(['id = ?', self.id]) - tags.each do |k,v| tag = RelationTag.new tag.k = k @@ -183,9 +183,7 @@ class Relation < ActiveRecord::Base end members = self.members - RelationMember.delete_all(['id = ?', self.id]) - members.each do |n| mem = RelationMember.new mem.id = self.id diff --git a/app/models/way.rb b/app/models/way.rb index f1dc76eb4..104c550a9 100644 --- a/app/models/way.rb +++ b/app/models/way.rb @@ -70,6 +70,7 @@ class Way < ActiveRecord::Base el1['id'] = self.id.to_s el1['visible'] = self.visible.to_s el1['timestamp'] = self.timestamp.xmlschema + el1['version'] = self.version.to_s user_display_name_cache = {} if user_display_name_cache.nil? @@ -158,15 +159,12 @@ class Way < ActiveRecord::Base t = Time.now Way.transaction do + self.version += 1 self.timestamp = t self.save! - end - WayTag.transaction do tags = self.tags - WayTag.delete_all(['id = ?', self.id]) - tags.each do |k,v| tag = WayTag.new tag.k = k @@ -174,13 +172,9 @@ class Way < ActiveRecord::Base tag.id = self.id tag.save! end - end - WayNode.transaction do nds = self.nds - WayNode.delete_all(['id = ?', self.id]) - sequence = 1 nds.each do |n| nd = WayNode.new @@ -189,11 +183,11 @@ class Way < ActiveRecord::Base nd.save! sequence += 1 end - end - old_way = OldWay.from_way(self) - old_way.timestamp = t - old_way.save_with_dependencies! + old_way = OldWay.from_way(self) + old_way.timestamp = t + old_way.save_with_dependencies! + end end def preconditions_ok? diff --git a/config/database.yml b/config/database.yml index b884f3b93..c0b5dd98c 100644 --- a/config/database.yml +++ b/config/database.yml @@ -12,9 +12,9 @@ # http://dev.mysql.com/doc/refman/5.0/en/old-client.html development: adapter: mysql - database: openstreetmap - username: openstreetmap - password: openstreetmap + database: osm + username: osm + password: osm host: localhost # Warning: The database defined as 'test' will be erased and @@ -29,8 +29,8 @@ test: production: adapter: mysql - database: openstreetmap - username: openstreetmap - password: openstreetmap - host: db.openstreetmap.org + database: osm + username: osm + password: osm + host: localhost diff --git a/config/environment.rb b/config/environment.rb index 02d99f20c..45c9ed4d6 100644 --- a/config/environment.rb +++ b/config/environment.rb @@ -6,13 +6,13 @@ ENV['RAILS_ENV'] ||= 'production' # Specifies gem version of Rails to use when vendor/rails is not present # DO NOT BUMP THIS TO 2.0.2 AS THE LIVE SERVERS CAN'T RUN THAT -RAILS_GEM_VERSION = '2.0.1' unless defined? RAILS_GEM_VERSION +RAILS_GEM_VERSION = '2.0.2' unless defined? RAILS_GEM_VERSION # Set the server URL SERVER_URL = ENV['OSM_SERVER_URL'] || 'www.openstreetmap.org' # Application constants needed for routes.rb - must go before Initializer call -API_VERSION = ENV['OSM_API_VERSION'] || '0.5' +API_VERSION = ENV['OSM_API_VERSION'] || '0.6' # Set to :readonly to put the API in read-only mode or :offline to # take it completely offline diff --git a/config/routes.rb b/config/routes.rb index 06f1583ff..e4d68f87a 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,6 +1,8 @@ ActionController::Routing::Routes.draw do |map| # API + map.connect "api/#{API_VERSION}/changeset/create", :controller => 'changeset', :action => 'create' + map.connect "api/#{API_VERSION}/node/create", :controller => 'node', :action => 'create' map.connect "api/#{API_VERSION}/node/:id/ways", :controller => 'way', :action => 'ways_for_node', :id => /\d+/ map.connect "api/#{API_VERSION}/node/:id/relations", :controller => 'relation', :action => 'relations_for_node', :id => /\d+/ diff --git a/db/migrate/013_populate_node_tags_and_remove.rb b/db/migrate/013_populate_node_tags_and_remove.rb new file mode 100644 index 000000000..29a91c70b --- /dev/null +++ b/db/migrate/013_populate_node_tags_and_remove.rb @@ -0,0 +1,62 @@ +class PopulateNodeTagsAndRemove < ActiveRecord::Migration + def self.up + have_nodes = select_value("SELECT count(*) FROM current_nodes").to_i != 0 + + if have_nodes + prefix = File.join Dir.tmpdir, "013_populate_node_tags_and_remove.#{$$}." + + cmd = "db/migrate/013_populate_node_tags_and_remove_helper" + src = "#{cmd}.c" + if not File.exists? cmd or File.mtime(cmd) < File.mtime(src) then + system 'cc -O3 -Wall `mysql_config --cflags --libs` ' + + "#{src} -o #{cmd}" or fail + end + + conn_opts = ActiveRecord::Base.connection. + instance_eval { @connection_options } + args = conn_opts.map { |arg| arg.to_s } + [prefix] + fail "#{cmd} failed" unless system cmd, *args + + tempfiles = ['nodes', 'node_tags', + 'current_nodes', 'current_node_tags']. + map { |base| prefix + base } + nodes, node_tags, current_nodes, current_node_tags = tempfiles + end + + execute "TRUNCATE nodes" + remove_column :nodes, :tags + remove_column :current_nodes, :tags + + add_column :nodes, :version, :bigint, :limit => 20, :null => false + + create_table :current_node_tags, innodb_table do |t| + t.column :id, :bigint, :limit => 64, :null => false + t.column :k, :string, :default => "", :null => false + t.column :v, :string, :default => "", :null => false + end + + create_table :node_tags, innodb_table do |t| + t.column :id, :bigint, :limit => 64, :null => false + t.column :version, :bigint, :limit => 20, :null => false + t.column :k, :string, :default => "", :null => false + t.column :v, :string, :default => "", :null => false + end + + # now get the data back + csvopts = "FIELDS TERMINATED BY ',' ENCLOSED BY '\"' ESCAPED BY '\"' LINES TERMINATED BY '\\n'" + + if have_nodes + execute "LOAD DATA INFILE '#{nodes}' INTO TABLE nodes #{csvopts} (id, latitude, longitude, user_id, visible, timestamp, tile, version)"; + execute "LOAD DATA INFILE '#{node_tags}' INTO TABLE node_tags #{csvopts} (id, version, k, v)" + execute "LOAD DATA INFILE '#{current_node_tags}' INTO TABLE current_node_tags #{csvopts} (id, k, v)" + end + + tempfiles.each { |fn| File.unlink fn } if have_nodes + end + + def self.down + raise IrreversibleMigration.new +# add_column :nodes, "tags", :text, :default => "", :null => false +# add_column :current_nodes, "tags", :text, :default => "", :null => false + end +end diff --git a/db/migrate/013_populate_node_tags_and_remove_helper.c b/db/migrate/013_populate_node_tags_and_remove_helper.c new file mode 100644 index 000000000..b1868ef24 --- /dev/null +++ b/db/migrate/013_populate_node_tags_and_remove_helper.c @@ -0,0 +1,234 @@ +#include +#include +#include +#include +#include + +static void exit_mysql_err(MYSQL *mysql) { + const char *err = mysql_error(mysql); + if (err) { + fprintf(stderr, "013_populate_node_tags_and_remove_helper: MySQL error: %s\n", err); + } else { + fprintf(stderr, "013_populate_node_tags_and_remove_helper: MySQL error\n"); + } + abort(); + exit(EXIT_FAILURE); +} + +static void write_csv_col(FILE *f, const char *str, char end) { + char *out = (char *) malloc(2 * strlen(str) + 4); + char *o = out; + size_t len; + + *(o++) = '\"'; + for (; *str; str++) { + if (*str == '\0') { + break; + } else if (*str == '\"') { + *(o++) = '\"'; + *(o++) = '\"'; + } else { + *(o++) = *str; + } + } + *(o++) = '\"'; + *(o++) = end; + *(o++) = '\0'; + + len = strlen(out); + if (fwrite(out, len, 1, f) != 1) { + perror("fwrite"); + exit(EXIT_FAILURE); + } + + free(out); +} + +static void unescape(char *str) { + char *i = str, *o = str; + + while (*i) { + if (*i == '\\') { + i++; + switch (*i++) { + case 's': *o++ = ';'; break; + case 'e': *o++ = '='; break; + case '\\': *o++ = '\\'; break; + } + } else { + *o++ = *i++; + } + } +} + +static int read_node_tags(char **tags, char **k, char **v) { + if (!**tags) return 0; + char *i = strchr(*tags, ';'); + if (!i) i = *tags + strlen(*tags); + char *j = strchr(*tags, '='); + *k = *tags; + if (j && j < i) { + *v = j + 1; + } else { + *v = i; + } + *tags = *i ? i + 1 : i; + *i = '\0'; + if (j) *j = '\0'; + + unescape(*k); + unescape(*v); + + return 1; +} + +struct data { + MYSQL *mysql; + size_t version_size; + uint32_t *version; +}; + +static void proc_nodes(struct data *d, const char *tbl, FILE *out, FILE *out_tags, int hist) { + MYSQL_RES *res; + MYSQL_ROW row; + char query[256]; + + snprintf(query, sizeof(query), "SELECT id, latitude, longitude, " + "user_id, visible, tags, timestamp, tile FROM %s", tbl); + if (mysql_query(d->mysql, query)) + exit_mysql_err(d->mysql); + + res = mysql_use_result(d->mysql); + if (!res) exit_mysql_err(d->mysql); + + while ((row = mysql_fetch_row(res))) { + unsigned long id = strtoul(row[0], NULL, 10); + uint32_t version; + + if (id > d->version_size) { + fprintf(stderr, "preallocated nodes size exceeded"); + abort(); + } + + if (hist) { + version = ++(d->version[id]); + + fprintf(out, "\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%u\"\n", + row[0], row[1], row[2], row[3], row[4], row[6], row[7], version); + } else { + /*fprintf(out, "\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\"\n", + row[0], row[1], row[2], row[3], row[4], row[6], row[7]);*/ + } + + char *tags_it = row[5], *k, *v; + while (read_node_tags(&tags_it, &k, &v)) { + if (hist) { + fprintf(out_tags, "\"%s\",\"%u\",", row[0], version); + } else { + fprintf(out_tags, "\"%s\",", row[0]); + } + + write_csv_col(out_tags, k, ','); + write_csv_col(out_tags, v, '\n'); + } + } + if (mysql_errno(d->mysql)) exit_mysql_err(d->mysql); + + mysql_free_result(res); +} + +static size_t select_size(MYSQL *mysql, const char *q) { + MYSQL_RES *res; + MYSQL_ROW row; + size_t ret; + + if (mysql_query(mysql, q)) + exit_mysql_err(mysql); + + res = mysql_store_result(mysql); + if (!res) exit_mysql_err(mysql); + + row = mysql_fetch_row(res); + if (!row) exit_mysql_err(mysql); + + if (row[0]) { + ret = strtoul(row[0], NULL, 10); + } else { + ret = 0; + } + + mysql_free_result(res); + + return ret; +} + +static MYSQL *connect_to_mysql(char **argv) { + MYSQL *mysql = mysql_init(NULL); + if (!mysql) exit_mysql_err(mysql); + + if (!mysql_real_connect(mysql, argv[1], argv[2], argv[3], argv[4], + argv[5][0] ? atoi(argv[5]) : 0, argv[6][0] ? argv[6] : NULL, 0)) + exit_mysql_err(mysql); + + if (mysql_set_character_set(mysql, "utf8")) + exit_mysql_err(mysql); + + return mysql; +} + +static void open_file(FILE **f, char *fn) { + *f = fopen(fn, "w+"); + if (!*f) { + perror("fopen"); + exit(EXIT_FAILURE); + } +} + +int main(int argc, char **argv) { + size_t prefix_len; + FILE *current_nodes, *current_node_tags, *nodes, *node_tags; + char *tempfn; + struct data data, *d = &data; + + if (argc != 8) { + printf("Usage: 013_populate_node_tags_and_remove_helper host user passwd database port socket prefix\n"); + exit(EXIT_FAILURE); + } + + d->mysql = connect_to_mysql(argv); + + d->version_size = 1 + select_size(d->mysql, "SELECT max(id) FROM current_nodes"); + d->version = malloc(sizeof(uint32_t) * d->version_size); + + prefix_len = strlen(argv[7]); + tempfn = (char *) malloc(prefix_len + 16); + strcpy(tempfn, argv[7]); + + strcpy(tempfn + prefix_len, "current_nodes"); + open_file(¤t_nodes, tempfn); + + strcpy(tempfn + prefix_len, "current_node_tags"); + open_file(¤t_node_tags, tempfn); + + strcpy(tempfn + prefix_len, "nodes"); + open_file(&nodes, tempfn); + + strcpy(tempfn + prefix_len, "node_tags"); + open_file(&node_tags, tempfn); + + free(tempfn); + + proc_nodes(d, "nodes", nodes, node_tags, 1); + proc_nodes(d, "current_nodes", current_nodes, current_node_tags, 0); + + free(d->version); + + mysql_close(d->mysql); + + fclose(current_nodes); + fclose(current_node_tags); + fclose(nodes); + fclose(node_tags); + + exit(EXIT_SUCCESS); +} diff --git a/db/migrate/014_move_to_innodb.rb b/db/migrate/014_move_to_innodb.rb new file mode 100644 index 000000000..da6442dab --- /dev/null +++ b/db/migrate/014_move_to_innodb.rb @@ -0,0 +1,30 @@ +class MoveToInnodb < ActiveRecord::Migration + @@conv_tables = ['nodes', 'ways', 'way_tags', 'way_nodes', + 'current_way_nodes', 'current_way_tags', 'relation_members', + 'relations', 'relation_tags', 'current_relation_tags'] + + @@ver_tbl = ['nodes', 'ways', 'relations'] + + def self.up + execute 'DROP INDEX current_way_tags_v_idx ON current_way_tags' + execute 'DROP INDEX current_relation_tags_v_idx ON current_relation_tags' + + @@ver_tbl.each { |tbl| + change_column tbl, "version", :bigint, :limit => 20, :null => false + } + + @@conv_tables.each { |tbl| + execute "ALTER TABLE #{tbl} ENGINE = InnoDB" + } + + @@ver_tbl.each { |tbl| + add_column "current_#{tbl}", "version", :bigint, :limit => 20, :null => false + execute "UPDATE current_#{tbl} SET version = " + + "(SELECT max(version) FROM #{tbl} WHERE #{tbl}.id = current_#{tbl}.id)" + } + end + + def self.down + raise IrreversibleMigration.new + end +end diff --git a/db/migrate/015_key_constraints.rb b/db/migrate/015_key_constraints.rb new file mode 100644 index 000000000..40f98be02 --- /dev/null +++ b/db/migrate/015_key_constraints.rb @@ -0,0 +1,50 @@ +class KeyConstraints < ActiveRecord::Migration + def self.up + # Primary keys + add_primary_key :current_node_tags, [:id, :k] + add_primary_key :current_way_tags, [:id, :k] + add_primary_key :current_relation_tags, [:id, :k] + + add_primary_key :node_tags, [:id, :version, :k] + add_primary_key :way_tags, [:id, :version, :k] + add_primary_key :relation_tags, [:id, :version, :k] + + add_primary_key :nodes, [:id, :version] + + # Remove indexes superseded by primary keys + remove_index :current_way_tags, :name => :current_way_tags_id_idx + remove_index :current_relation_tags, :name => :current_relation_tags_id_idx + + remove_index :way_tags, :name => :way_tags_id_version_idx + remove_index :relation_tags, :name => :relation_tags_id_version_idx + + remove_index :nodes, :name => :nodes_uid_idx + + # Foreign keys (between ways, way_tags, way_nodes, etc.) + add_foreign_key :current_node_tags, [:id], :current_nodes + add_foreign_key :node_tags, [:id, :version], :nodes + + add_foreign_key :current_way_tags, [:id], :current_ways + add_foreign_key :current_way_nodes, [:id], :current_ways + add_foreign_key :way_tags, [:id, :version], :ways + add_foreign_key :way_nodes, [:id, :version], :ways + + add_foreign_key :current_relation_tags, [:id], :current_relations + add_foreign_key :current_relation_members, [:id], :current_relations + add_foreign_key :relation_tags, [:id, :version], :relations + add_foreign_key :relation_members, [:id, :version], :relations + + # Foreign keys (between different types of primitives) + add_foreign_key :current_way_nodes, [:node_id], :current_nodes, [:id] + + # FIXME: We don't have foreign keys for relation members since the id + # might point to a different table depending on the `type' column. + # We'd probably need different current_relation_member_nodes, + # current_relation_member_ways and current_relation_member_relations + # tables for this to work cleanly. + end + + def self.down + raise IrreversibleMigration.new + end +end diff --git a/db/migrate/016_add_changesets.rb b/db/migrate/016_add_changesets.rb new file mode 100644 index 000000000..40455ec68 --- /dev/null +++ b/db/migrate/016_add_changesets.rb @@ -0,0 +1,32 @@ +class AddChangesets < ActiveRecord::Migration + def self.up + create_table "changesets", innodb_table do |t| + t.column "id", :bigint, :limit => 20, :null => false + t.column "user_id", :bigint, :limit => 20, :null => false + t.column "created_at", :datetime, :null => false + t.column "open", :boolean, :null => false, :default => true + t.column "min_lat", :integer, :null => true + t.column "max_lat", :integer, :null => true + t.column "min_lon", :integer, :null => true + t.column "max_lon", :integer, :null => true + end + + add_primary_key "changesets", ["id"] + # FIXME add indexes? + + change_column "changesets", "id", :bigint, :limit => 20, :null => false, :options => "AUTO_INCREMENT" + + create_table "changeset_tags", innodb_table do |t| + t.column "id", :bigint, :limit => 64, :null => false + t.column "k", :string, :default => "", :null => false + t.column "v", :string, :default => "", :null => false + end + + add_index "changeset_tags", ["id"], :name => "changeset_tags_id_idx" + end + + def self.down + drop_table "changesets" + drop_table "changeset_tags" + end +end diff --git a/lib/migrate.rb b/lib/migrate.rb index 1d32d175d..26e95a496 100644 --- a/lib/migrate.rb +++ b/lib/migrate.rb @@ -1,6 +1,10 @@ module ActiveRecord module ConnectionAdapters module SchemaStatements + def quote_column_names(column_name) + Array(column_name).map { |e| quote_column_name(e) }.join(", ") + end + def add_primary_key(table_name, column_name, options = {}) column_names = Array(column_name) quoted_column_names = column_names.map { |e| quote_column_name(e) }.join(", ") @@ -11,6 +15,12 @@ module ActiveRecord execute "ALTER TABLE #{table_name} DROP PRIMARY KEY" end + def add_foreign_key(table_name, column_name, reftbl, refcol = nil) + execute "ALTER TABLE #{table_name} ADD " + + "FOREIGN KEY (#{quote_column_names(column_name)}) " + + "REFERENCES #{reftbl} (#{quote_column_names(refcol || column_name)})" + end + alias_method :old_options_include_default?, :options_include_default? def options_include_default?(options) diff --git a/lib/tasks/populate_node_tags.rake b/lib/tasks/populate_node_tags.rake deleted file mode 100644 index 86747cfe4..000000000 --- a/lib/tasks/populate_node_tags.rake +++ /dev/null @@ -1,42 +0,0 @@ -namespace 'db' do - desc 'Populate the node_tags table' - task :node_tags do - require File.dirname(__FILE__) + '/../../config/environment' - - node_count = Node.count - limit = 1000 #the number of nodes to grab in one go - offset = 0 - - while offset < node_count - Node.find(:all, :limit => limit, :offset => offset).each do |node| - seq_id = 1 - node.tags.split(';').each do |tag| - nt = NodeTag.new - nt.id = node.id - nt.k = tag.split('=')[0] || '' - nt.v = tag.split('=')[1] || '' - nt.sequence_id = seq_id - nt.save! || raise - seq_id += 1 - end - - version = 1 #version refers to one set of histories - node.old_nodes.find(:all, :order => 'timestamp asc').each do |old_node| - sequence_id = 1 #sequence_id refers to the sequence of node tags within a history - old_node.tags.split(';').each do |tag| - ont = OldNodeTag.new - ont.id = node.id #the id of the node tag - ont.k = tag.split('=')[0] || '' - ont.v = tag.split('=')[1] || '' - ont.version = version - ont.sequence_id = sequence_id - ont.save! || raise - sequence_id += 1 - end - version += 1 - end - end - offset += limit - end - end -end