From: Gabriel Ebner Date: Thu, 20 Sep 2007 14:37:29 +0000 (+0000) Subject: Merge rails_port as of r4613 & fix tests. X-Git-Tag: live~8129^2~18 X-Git-Url: https://git.openstreetmap.org/rails.git/commitdiff_plain/cfbdd3f7e1c688e2c875ded9fd847fcc1c3a4caf Merge rails_port as of r4613 & fix tests. --- cfbdd3f7e1c688e2c875ded9fd847fcc1c3a4caf diff --cc app/controllers/relation_controller.rb index efd5bf9dc,000000000..d00e9e077 mode 100644,000000..100644 --- a/app/controllers/relation_controller.rb +++ b/app/controllers/relation_controller.rb @@@ -1,227 -1,0 +1,218 @@@ +class RelationController < ApplicationController + require 'xml/libxml' + + before_filter :authorize, :only => [:create, :update, :delete] + before_filter :check_availability, :only => [:create, :update, :delete] + + after_filter :compress_output + + def create + if request.put? + relation = Relation.from_xml(request.raw_post, true) + + if relation + if !relation.preconditions_ok? + render :nothing => true, :status => :precondition_failed + else + relation.user_id = @user.id ++ relation.save_with_history! + - if relation.save_with_history - render :text => relation.id.to_s, :content_type => "text/plain" - else - render :text => "save error", :status => :internal_server_error - end ++ render :text => relation.id.to_s, :content_type => "text/plain" + end + else + render :nothing => true, :status => :bad_request + end + else + render :nothing => true, :status => :method_not_allowed + end + end + + def read + begin + relation = Relation.find(params[:id]) + + if relation.visible + render :text => relation.to_xml.to_s, :content_type => "text/xml" + else + render :nothing => true, :status => :gone + end + rescue ActiveRecord::RecordNotFound + render :nothing => true, :status => :not_found + rescue + render :nothing => true, :status => :internal_server_error + end + end + + def update + begin + relation = Relation.find(params[:id]) + + if relation.visible + new_relation = Relation.from_xml(request.raw_post) + + if new_relation and new_relation.id == relation.id + if !new_relation.preconditions_ok? + render :nothing => true, :status => :precondition_failed + else + relation.user_id = @user.id + relation.tags = new_relation.tags + relation.members = new_relation.members + relation.visible = true ++ relation.save_with_history! + - if relation.save_with_history - render :nothing => true - else - render :nothing => true, :status => :internal_server_error - end ++ render :nothing => true + end + else + render :nothing => true, :status => :bad_request + end + else + render :nothing => true, :status => :gone + end + rescue ActiveRecord::RecordNotFound + render :nothing => true, :status => :not_found + rescue + render :nothing => true, :status => :internal_server_error + end + end + + def delete +#XXX check if member somewhere! + begin + relation = Relation.find(params[:id]) + + if relation.visible + if RelationMember.find(:first, :joins => "INNER JOIN current_relations ON current_relations.id=current_relation_members.id", :conditions => [ "visible = 1 AND member_type='relation' and member_id=?", params[:id]]) + render :nothing => true, :status => :precondition_failed + else + relation.user_id = @user.id + relation.tags = [] + relation.members = [] + relation.visible = false ++ relation.save_with_history! + - if relation.save_with_history - render :nothing => true - else - render :nothing => true, :status => :internal_server_error - end ++ render :nothing => true + end + else + render :nothing => true, :status => :gone + end + rescue ActiveRecord::RecordNotFound + render :nothing => true, :status => :not_found + rescue + render :nothing => true, :status => :internal_server_error + end + end + + # ----------------------------------------------------------------- + # full + # + # input parameters: id + # + # returns XML representation of one relation object plus all its + # members, plus all nodes part of member ways + # ----------------------------------------------------------------- + def full + begin + relation = Relation.find(params[:id]) + + if relation.visible + + # first collect nodes, ways, and relations referenced by this relation. + + ways = Way.find_by_sql("select w.* from current_ways w,current_relation_members rm where "+ + "rm.member_type='way' and rm.member_id=w.id and rm.id=#{relation.id}"); + nodes = Node.find_by_sql("select n.* from current_nodes n,current_relation_members rm where "+ + "rm.member_type='node' and rm.member_id=n.id and rm.id=#{relation.id}"); + # note query is built to exclude self just in case. + relations = Relation.find_by_sql("select r.* from current_relations r,current_relation_members rm where "+ + "rm.member_type='relation' and rm.member_id=r.id and rm.id=#{relation.id} and r.id<>rm.id"); + + # now additionally collect nodes referenced by ways. Note how we recursively + # evaluate ways but NOT relations. + + node_ids = nodes.collect {|node| node.id } + way_node_ids = ways.collect { |way| + way.way_nodes.collect { |way_node| way_node.node_id } + } + way_node_ids.flatten! + way_node_ids.uniq + way_node_ids -= node_ids + nodes += Node.find(way_node_ids) + + # create XML. + doc = OSM::API.new.get_xml_doc + user_display_name_cache = {} + + nodes.each do |node| + if node.visible? # should be unnecessary if data is consistent. + doc.root << node.to_xml_node(user_display_name_cache) + end + end + ways.each do |way| + if way.visible? # should be unnecessary if data is consistent. + doc.root << way.to_xml_node(user_display_name_cache) + end + end + relations.each do |rel| + if rel.visible? # should be unnecessary if data is consistent. + doc.root << rel.to_xml_node(user_display_name_cache) + end + end + # finally add self and output + doc.root << relation.to_xml_node(user_display_name_cache) + render :text => doc.to_s, :content_type => "text/xml" + + else + + render :nothing => true, :status => :gone + end + + rescue ActiveRecord::RecordNotFound + render :nothing => true, :status => :not_found + + rescue + render :nothing => true, :status => :internal_server_error + end + end + + def relations + ids = params['relations'].split(',').collect { |w| w.to_i } + + if ids.length > 0 + doc = OSM::API.new.get_xml_doc + + Relation.find(ids).each do |relation| + doc.root << relation.to_xml_node + end + + render :text => doc.to_s, :content_type => "text/xml" + else + render :nothing => true, :status => :bad_request + end + end + + def relations_for_way + relations_for_object("way") + end + def relations_for_node + relations_for_object("node") + end + def relations_for_relation + relations_for_object("relation") + end + + def relations_for_object(objtype) + relationids = RelationMember.find(:all, :conditions => ['member_type=? and member_id=?', objtype, params[:id]]).collect { |ws| ws.id }.uniq + + if relationids.length > 0 + doc = OSM::API.new.get_xml_doc + + Relation.find(relationids).each do |relation| + doc.root << relation.to_xml_node + end + + render :text => doc.to_s, :content_type => "text/xml" + else + render :nothing => true, :status => :not_found + end + end +end diff --cc app/controllers/way_controller.rb index 2ef960c6e,935ff3a72..2e0623df4 --- a/app/controllers/way_controller.rb +++ b/app/controllers/way_controller.rb @@@ -59,14 -54,11 +54,11 @@@ class WayController < ApplicationContro else way.user_id = @user.id way.tags = new_way.tags - way.segs = new_way.segs + way.nds = new_way.nds way.visible = true + way.save_with_history! - if way.save_with_history - render :nothing => true - else - render :nothing => true, :status => :internal_server_error - end + render :nothing => true end else render :nothing => true, :status => :bad_request @@@ -86,27 -76,18 +76,24 @@@ way = Way.find(params[:id]) if way.visible - way.user_id = @user.id - way.tags = [] - way.segs = [] - way.visible = false - way.save_with_history! + if RelationMember.find(:first, :joins => "INNER JOIN current_relations ON current_relations.id=current_relation_members.id", :conditions => [ "visible = 1 AND member_type='way' and member_id=?", params[:id]]) + render :nothing => true, :status => :precondition_failed + else + way.user_id = @user.id + way.tags = [] + way.nds = [] + way.visible = false ++ way.save_with_history! - if way.save_with_history - render :nothing => true - else - render :nothing => true, :status => :internal_server_error - end - render :nothing => true ++ render :nothing => true + end else render :nothing => true, :status => :gone end rescue ActiveRecord::RecordNotFound render :nothing => true, :status => :not_found - rescue - render :nothing => true, :status => :internal_server_error ++ rescue => ex ++ puts ex end end diff --cc app/models/relation.rb index f74a149ec,000000000..a5d463ffb mode 100644,000000..100644 --- a/app/models/relation.rb +++ b/app/models/relation.rb @@@ -1,213 -1,0 +1,207 @@@ +class Relation < ActiveRecord::Base + require 'xml/libxml' + + belongs_to :user + + has_many :relation_members, :foreign_key => 'id' + has_many :relation_tags, :foreign_key => 'id' + + has_many :old_relations, :foreign_key => 'id', :order => 'version' + + set_table_name 'current_relations' + + def self.from_xml(xml, create=false) + begin + p = XML::Parser.new + p.string = xml + doc = p.parse + + relation = Relation.new + + doc.find('//osm/relation').each do |pt| + if !create and pt['id'] != '0' + relation.id = pt['id'].to_i + end + + if create + relation.timestamp = Time.now + relation.visible = true + else + if pt['timestamp'] + relation.timestamp = Time.parse(pt['timestamp']) + end + end + + pt.find('tag').each do |tag| + relation.add_tag_keyval(tag['k'], tag['v']) + end + + pt.find('member').each do |member| + relation.add_member(member['type'], member['ref'], member['role']) + end + end + rescue + relation = nil + end + + return relation + end + + def to_xml + doc = OSM::API.new.get_xml_doc + doc.root << to_xml_node() + return doc + end + + def to_xml_node(user_display_name_cache = nil) + el1 = XML::Node.new 'relation' + el1['id'] = self.id.to_s + el1['visible'] = self.visible.to_s + el1['timestamp'] = self.timestamp.xmlschema + + user_display_name_cache = {} if user_display_name_cache.nil? + + if user_display_name_cache and user_display_name_cache.key?(self.user_id) + # use the cache if available + elsif self.user.data_public? + user_display_name_cache[self.user_id] = self.user.display_name + else + user_display_name_cache[self.user_id] = nil + end + + el1['user'] = user_display_name_cache[self.user_id] unless user_display_name_cache[self.user_id].nil? + + self.relation_members.each do |member| + p=0 + #if visible_members + # # if there is a list of visible members then use that to weed out deleted segments + # if visible_members[member.member_type][member.member_id] + # p=1 + # end + #else + # otherwise, manually go to the db to check things + if member.member.visible? + p=1 + end + #end + if p + e = XML::Node.new 'member' + e['type'] = member.member_type + e['ref'] = member.member_id.to_s + e['role'] = member.member_role + el1 << e + end + end + + self.relation_tags.each do |tag| + e = XML::Node.new 'tag' + e['k'] = tag.k + e['v'] = tag.v + el1 << e + end + return el1 + end + + # FIXME is this really needed? + def members + unless @members + @members = Array.new + self.relation_members.each do |member| + @members += [[member.member_type,member.member_id,member.member_role]] + end + end + @members + end + + def tags + unless @tags + @tags = Hash.new + self.relation_tags.each do |tag| + @tags[tag.k] = tag.v + end + end + @tags + end + + def members=(m) + @members = m + end + + def tags=(t) + @tags = t + end + + def add_member(type,id,role) + @members = Array.new unless @members + @members += [[type,id,role]] + end + + def add_tag_keyval(k, v) + @tags = Hash.new unless @tags + @tags[k] = v + end + - def save_with_history - begin - Relation.transaction do - t = Time.now - self.timestamp = t - self.save! - - tags = self.tags - - RelationTag.delete_all(['id = ?', self.id]) - - tags.each do |k,v| - tag = RelationTag.new - tag.k = k - tag.v = v - tag.id = self.id - tag.save! - end - - members = self.members ++ def save_with_history! ++ Relation.transaction do ++ t = Time.now ++ self.timestamp = t ++ self.save! ++ ++ tags = self.tags ++ ++ RelationTag.delete_all(['id = ?', self.id]) ++ ++ tags.each do |k,v| ++ tag = RelationTag.new ++ tag.k = k ++ tag.v = v ++ tag.id = self.id ++ tag.save! ++ end + - RelationMember.delete_all(['id = ?', self.id]) ++ members = self.members + - members.each do |n| - mem = RelationMember.new - mem.id = self.id - mem.member_type = n[0]; - mem.member_id = n[1]; - mem.member_role = n[2]; - mem.save! - end ++ RelationMember.delete_all(['id = ?', self.id]) + - old_relation = OldRelation.from_relation(self) - old_relation.timestamp = t - old_relation.save_with_dependencies! ++ members.each do |n| ++ mem = RelationMember.new ++ mem.id = self.id ++ mem.member_type = n[0]; ++ mem.member_id = n[1]; ++ mem.member_role = n[2]; ++ mem.save! + end + - return true - rescue Exception => ex - return nil ++ old_relation = OldRelation.from_relation(self) ++ old_relation.timestamp = t ++ old_relation.save_with_dependencies! + end + end + + def preconditions_ok? + self.members.each do |m| + if (m[0] == "node") + n = Node.find(:first, :conditions => ["id = ?", m[1]]) + unless n and n.visible + return false + end + elsif (m[0] == "way") + w = Way.find(:first, :conditions => ["id = ?", m[1]]) + unless w and w.visible and w.preconditions_ok? + return false + end + elsif (m[0] == "relation") + e = Relation.find(:first, :conditions => ["id = ?", m[1]]) + unless e and e.visible and e.preconditions_ok? + return false + end + else + return false + end + end + return true + rescue + return false + end + +end diff --cc app/models/way.rb index b564a3317,c65119235..e0a445074 --- a/app/models/way.rb +++ b/app/models/way.rb @@@ -142,49 -142,47 +142,47 @@@ class Way < ActiveRecord::Bas @tags[k] = v end - def save_with_history - begin - Way.transaction do - t = Time.now - self.timestamp = t - self.save! + def save_with_history! + t = Time.now - tags = self.tags + Way.transaction do + self.timestamp = t + self.save! + end - WayTag.delete_all(['id = ?', self.id]) + WayTag.transaction do + tags = self.tags - tags.each do |k,v| - tag = WayTag.new - tag.k = k - tag.v = v - tag.id = self.id - tag.save! - end + WayTag.delete_all(['id = ?', self.id]) - nds = self.nds + tags.each do |k,v| + tag = WayTag.new + tag.k = k + tag.v = v + tag.id = self.id + tag.save! + end + end - WayNode.delete_all(['id = ?', self.id]) - WaySegment.transaction do - segs = self.segs ++ WayNode.transaction do ++ nds = self.nds - i = 1 - nds.each do |n| - nd = WayNode.new - nd.id = self.id - nd.node_id = n - nd.sequence_id = i - nd.save! - i += 1 - end - WaySegment.delete_all(['id = ?', self.id]) ++ WayNode.delete_all(['id = ?', self.id]) - old_way = OldWay.from_way(self) - old_way.timestamp = t - old_way.save_with_dependencies! + i = 1 - segs.each do |n| - seg = WaySegment.new - seg.id = self.id - seg.segment_id = n - seg.sequence_id = i - seg.save! ++ nds.each do |n| ++ nd = WayNode.new ++ nd.id = self.id ++ nd.node_id = n ++ nd.sequence_id = i ++ nd.save! + i += 1 end - - return true - rescue => ex - puts ex - return nil end + + old_way = OldWay.from_way(self) + old_way.timestamp = t + old_way.save_with_dependencies! end def preconditions_ok? diff --cc db/migrate/006_add_relations.rb index a30642e32,000000000..a30642e32 mode 100644,000000..100644 --- a/db/migrate/006_add_relations.rb +++ b/db/migrate/006_add_relations.rb diff --cc db/migrate/007_remove_segments.rb index de9557cc9,000000000..492beda63 mode 100644,000000..100644 --- a/db/migrate/007_remove_segments.rb +++ b/db/migrate/007_remove_segments.rb @@@ -1,89 -1,0 +1,89 @@@ +require 'lib/migrate' + +class RemoveSegments < ActiveRecord::Migration + def self.up + have_segs = select_value("SELECT count(*) FROM current_segments").to_i != 0 + + if have_segs - prefix = File.join Dir.tmpdir, "006_remove_segments.#{$$}." ++ prefix = File.join Dir.tmpdir, "007_remove_segments.#{$$}." + - cmd = "db/migrate/006_remove_segments_helper" ++ cmd = "db/migrate/007_remove_segments_helper" + src = "#{cmd}.cc" + if not File.exists? cmd or File.mtime(cmd) < File.mtime(src) then + system 'c++ -O3 -Wall `mysql_config --cflags --libs` ' + + "#{src} -o #{cmd}" or fail + end + + conn_opts = ActiveRecord::Base.connection. + instance_eval { @connection_options } + args = conn_opts.map { |arg| arg.to_s } + [prefix] + fail "#{cmd} failed" unless system cmd, *args + + tempfiles = ['ways', 'way_nodes', 'way_tags', + 'relations', 'relation_members', 'relation_tags']. + map { |base| prefix + base } + ways, way_nodes, way_tags, + relations, relation_members, relation_tags = tempfiles + end + + drop_table :segments + drop_table :way_segments + create_table :way_nodes, myisam_table do |t| + t.column :id, :bigint, :limit => 64, :null => false + t.column :node_id, :bigint, :limit => 64, :null => false + t.column :version, :bigint, :limit => 20, :null => false + t.column :sequence_id, :bigint, :limit => 11, :null => false + end + add_primary_key :way_nodes, [:id, :version, :sequence_id] + + drop_table :current_segments + drop_table :current_way_segments + create_table :current_way_nodes, innodb_table do |t| + t.column :id, :bigint, :limit => 64, :null => false + t.column :node_id, :bigint, :limit => 64, :null => false + t.column :sequence_id, :bigint, :limit => 11, :null => false + end + add_primary_key :current_way_nodes, [:id, :sequence_id] + + execute "TRUNCATE way_tags" + execute "TRUNCATE ways" + execute "TRUNCATE current_way_tags" + execute "TRUNCATE current_ways" + + # now get the data back + csvopts = "FIELDS TERMINATED BY ',' ENCLOSED BY '\"' ESCAPED BY '\"' LINES TERMINATED BY '\\n'" + + tempfiles.each { |fn| File.chmod 0644, fn } if have_segs + + if have_segs + execute "LOAD DATA LOCAL INFILE '#{ways}' INTO TABLE ways #{csvopts} (id, user_id, timestamp) SET visible = 1, version = 1" + execute "LOAD DATA LOCAL INFILE '#{way_nodes}' INTO TABLE way_nodes #{csvopts} (id, node_id, sequence_id) SET version = 1" + execute "LOAD DATA LOCAL INFILE '#{way_tags}' INTO TABLE way_tags #{csvopts} (id, k, v) SET version = 1" + + execute "INSERT INTO current_ways SELECT id, user_id, timestamp, visible FROM ways" + execute "INSERT INTO current_way_nodes SELECT id, node_id, sequence_id FROM way_nodes" + execute "INSERT INTO current_way_tags SELECT id, k, v FROM way_tags" + end + + # and then readd the index + add_index :current_way_nodes, [:node_id], :name => "current_way_nodes_node_idx" + + if have_segs + execute "LOAD DATA LOCAL INFILE '#{relations}' INTO TABLE relations #{csvopts} (id, user_id, timestamp) SET visible = 1, version = 1" + execute "LOAD DATA LOCAL INFILE '#{relation_members}' INTO TABLE relation_members #{csvopts} (id, member_type, member_id, member_role) SET version = 1" + execute "LOAD DATA LOCAL INFILE '#{relation_tags}' INTO TABLE relation_tags #{csvopts} (id, k, v) SET version = 1" + + # FIXME: This will only work if there were no relations before the + # migration! + execute "INSERT INTO current_relations SELECT id, user_id, timestamp, visible FROM relations" + execute "INSERT INTO current_relation_members SELECT id, member_type, member_id, member_role FROM relation_members" + execute "INSERT INTO current_relation_tags SELECT id, k, v FROM relation_tags" + end + + tempfiles.each { |fn| File.unlink fn } if have_segs + end + + def self.down + raise IrreversibleMigration.new + end +end diff --cc db/migrate/007_remove_segments_helper.cc index dd2176a04,000000000..2234a3bb5 mode 100644,000000..100644 --- a/db/migrate/007_remove_segments_helper.cc +++ b/db/migrate/007_remove_segments_helper.cc @@@ -1,591 -1,0 +1,591 @@@ +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __amd64__ + +#define F_U64 "%lu" +#define F_U32 "%u" + +#else + +#define F_U64 "%Lu" +#define F_U32 "%u" + +#endif + +using namespace std; + +template +static T parse(const char *str) { + istringstream in(str); + T t; + in >> t; + return t; +} + +static void exit_mysql_err(MYSQL *mysql) { + const char *err = mysql_error(mysql); + if (err) { - fprintf(stderr, "005_remove_segments_helper: MySQL error: %s\n", err); ++ fprintf(stderr, "007_remove_segments_helper: MySQL error: %s\n", err); + } else { - fprintf(stderr, "005_remove_segments_helper: MySQL error\n"); ++ fprintf(stderr, "007_remove_segments_helper: MySQL error\n"); + } + abort(); + exit(EXIT_FAILURE); +} + +static void exit_stmt_err(MYSQL_STMT *stmt) { + const char *err = mysql_stmt_error(stmt); + if (err) { - fprintf(stderr, "005_remove_segments_helper: MySQL stmt error: %s\n", err); ++ fprintf(stderr, "007_remove_segments_helper: MySQL stmt error: %s\n", err); + } else { - fprintf(stderr, "005_remove_segments_helper: MySQL stmt error\n"); ++ fprintf(stderr, "007_remove_segments_helper: MySQL stmt error\n"); + } + abort(); + exit(EXIT_FAILURE); +} + +struct segment { + uint32_t from, to; +}; + +struct data { + MYSQL *mysql, *mysql2; + + uint64_t seg_maxid, way_maxid; + uint64_t new_way_id; + uint64_t new_relation_id; + + size_t segs_len; + struct segment *segs; + unsigned char *rem_segs; + + FILE *ways, *way_nodes, *way_tags, + *relations, *relation_members, *relation_tags; +}; + +static uint64_t select_u64(MYSQL *mysql, const char *q) { + MYSQL_RES *res; + MYSQL_ROW row; + uint64_t ret; + + if (mysql_query(mysql, q)) + exit_mysql_err(mysql); + + res = mysql_store_result(mysql); + if (!res) exit_mysql_err(mysql); + + row = mysql_fetch_row(res); + if (!row) exit_mysql_err(mysql); + + if (row[0]) { + ret = parse(row[0]); + } else { + ret = 0; + } + + mysql_free_result(res); + + return ret; +} + +static void find_maxids(struct data *d) { + d->seg_maxid = select_u64(d->mysql, "SELECT max(id) FROM current_segments"); + d->segs_len = d->seg_maxid + 1; + d->way_maxid = select_u64(d->mysql, "SELECT max(id) FROM current_ways"); + d->new_way_id = d->way_maxid + 1; + d->new_relation_id = select_u64(d->mysql, "SELECT max(id) FROM current_relations") + 1; +} + +static void populate_segs(struct data *d) { + MYSQL_RES *res; + MYSQL_ROW row; + size_t id; + + d->segs = (segment *) malloc(sizeof(struct segment) * d->segs_len); + memset(d->segs, 0, sizeof(struct segment) * d->segs_len); + + d->rem_segs = (unsigned char *) malloc(d->segs_len); + memset(d->rem_segs, 0, d->segs_len); + + if (mysql_query(d->mysql, "SELECT id, node_a, node_b " + "FROM current_segments WHERE visible")) + exit_mysql_err(d->mysql); + + res = mysql_use_result(d->mysql); + if (!res) exit_mysql_err(d->mysql); + + while ((row = mysql_fetch_row(res))) { + id = parse(row[0]); + if (id >= d->segs_len) continue; + d->segs[id].from = parse(row[1]); + d->segs[id].to = parse(row[2]); + d->rem_segs[id] = 1; + } + if (mysql_errno(d->mysql)) exit_mysql_err(d->mysql); + + mysql_free_result(res); +} + +static void write_csv_col(FILE *f, const char *str, char end) { + char *out = (char *) malloc(2 * strlen(str) + 4); + char *o = out; + size_t len; + + *(o++) = '\"'; + for (; *str; str++) { + if (*str == '\0') { + break; + } else if (*str == '\"') { + *(o++) = '\"'; + *(o++) = '\"'; + } else { + *(o++) = *str; + } + } + *(o++) = '\"'; + *(o++) = end; + *(o++) = '\0'; + + len = strlen(out); + if (fwrite(out, len, 1, f) != 1) { + perror("fwrite"); + exit(EXIT_FAILURE); + } + + free(out); +} + +static void convert_ways(struct data *d) { + MYSQL_RES *res; + MYSQL_ROW row; + MYSQL_STMT *load_segs, *load_tags; + const char + load_segs_stmt[] = "SELECT segment_id FROM current_way_segments " + "WHERE id = ? ORDER BY sequence_id", + load_tags_stmt[] = "SELECT k, v FROM current_way_tags WHERE id = ?"; + char *k, *v; + const size_t max_tag_len = 1 << 16; + long long mysql_id, mysql_seg_id; + unsigned long res_len; + my_bool res_error; + MYSQL_BIND bind[1], seg_bind[1], tag_bind[2]; + + /* F***ing libmysql only support fixed size buffers for string results of + * prepared statements. So allocate 65k for the tag key and the tag value + * and hope it'll suffice. */ + k = (char *) malloc(max_tag_len); + v = (char *) malloc(max_tag_len); + + load_segs = mysql_stmt_init(d->mysql2); + if (!load_segs) exit_mysql_err(d->mysql2); + if (mysql_stmt_prepare(load_segs, load_segs_stmt, sizeof(load_segs_stmt))) + exit_stmt_err(load_segs); + + memset(bind, 0, sizeof(bind)); + bind[0].buffer_type = MYSQL_TYPE_LONGLONG; + bind[0].buffer = (char *) &mysql_id; + bind[0].is_null = 0; + bind[0].length = 0; + if (mysql_stmt_bind_param(load_segs, bind)) + exit_stmt_err(load_segs); + + memset(bind, 0, sizeof(seg_bind)); + seg_bind[0].buffer_type = MYSQL_TYPE_LONGLONG; + seg_bind[0].buffer = (char *) &mysql_seg_id; + seg_bind[0].is_null = 0; + seg_bind[0].length = 0; + seg_bind[0].error = &res_error; + if (mysql_stmt_bind_result(load_segs, seg_bind)) + exit_stmt_err(load_segs); + + load_tags = mysql_stmt_init(d->mysql2); + if (!load_tags) exit_mysql_err(d->mysql2); + if (mysql_stmt_prepare(load_tags, load_tags_stmt, sizeof(load_tags_stmt))) + exit_stmt_err(load_tags); + + memset(bind, 0, sizeof(bind)); + bind[0].buffer_type = MYSQL_TYPE_LONGLONG; + bind[0].buffer = (char *) &mysql_id; + bind[0].is_null = 0; + bind[0].length = 0; + + if (mysql_stmt_bind_param(load_tags, bind)) + exit_stmt_err(load_tags); + + memset(bind, 0, sizeof(tag_bind)); + tag_bind[0].buffer_type = MYSQL_TYPE_STRING; + tag_bind[0].buffer = k; + tag_bind[0].is_null = 0; + tag_bind[0].length = &res_len; + tag_bind[0].error = &res_error; + tag_bind[0].buffer_length = max_tag_len; + tag_bind[1].buffer_type = MYSQL_TYPE_STRING; + tag_bind[1].buffer = v; + tag_bind[1].is_null = 0; + tag_bind[1].length = &res_len; + tag_bind[1].error = &res_error; + tag_bind[1].buffer_length = max_tag_len; + if (mysql_stmt_bind_result(load_tags, tag_bind)) + exit_stmt_err(load_tags); + + if (mysql_query(d->mysql, "SELECT id, user_id, timestamp " + "FROM current_ways WHERE visible")) + exit_mysql_err(d->mysql); + + res = mysql_use_result(d->mysql); + if (!res) exit_mysql_err(d->mysql); + + while ((row = mysql_fetch_row(res))) { + uint64_t id; + const char *user_id, *timestamp; + + id = parse(row[0]); + user_id = row[1]; + timestamp = row[2]; + + mysql_id = (long long) id; + + if (mysql_stmt_execute(load_segs)) + exit_stmt_err(load_segs); + + if (mysql_stmt_store_result(load_segs)) + exit_stmt_err(load_segs); + + list segs; + while (!mysql_stmt_fetch(load_segs)) { + if (((uint64_t) mysql_seg_id) >= d->segs_len) continue; + segs.push_back(d->segs[mysql_seg_id]); + d->rem_segs[mysql_seg_id] = 0; + } + + list > node_lists; + while (segs.size()) { + list node_list; + node_list.push_back(segs.front().from); + node_list.push_back(segs.front().to); + segs.pop_front(); + while (true) { + bool found = false; + for (list::iterator it = segs.begin(); + it != segs.end(); ) { + if (it->from == node_list.back()) { + node_list.push_back(it->to); + segs.erase(it++); + found = true; + } else if (it->to == node_list.front()) { + node_list.insert(node_list.begin(), it->from); + segs.erase(it++); + found = true; + } else { + ++it; + } + } + if (!found) break; + } + node_lists.push_back(node_list); + } + + vector ids; ids.reserve(node_lists.size()); + bool orig_id_used = false; + for (list >::iterator it = node_lists.begin(); + it != node_lists.end(); ++it) { + uint64_t way_id; + int sid; + if (orig_id_used) { + way_id = d->new_way_id++; + } else { + way_id = id; + orig_id_used = true; + } + ids.push_back(way_id); + + fprintf(d->ways, "\"" F_U64 "\",", way_id); + write_csv_col(d->ways, user_id, ','); + write_csv_col(d->ways, timestamp, '\n'); + + sid = 1; + for (list::iterator nit = it->begin(); + nit != it->end(); ++nit) { + fprintf(d->way_nodes, "\"" F_U64 "\",\"" F_U32 "\",\"%i\"\n", way_id, *nit, sid++); + } + } + + if (mysql_stmt_execute(load_tags)) + exit_stmt_err(load_tags); + + if (mysql_stmt_store_result(load_tags)) + exit_stmt_err(load_tags); + + bool multiple_parts = ids.size() > 1, + create_multipolygon = false; + + while (!mysql_stmt_fetch(load_tags)) { + if (multiple_parts && !create_multipolygon) { + if (!strcmp(k, "natural")) { + if (strcmp(v, "coastline")) { + create_multipolygon = true; + } + } else if (!strcmp(k, "waterway")) { + if (!strcmp(v, "riverbank")) { + create_multipolygon = true; + } + } else if (!strcmp(k, "leisure") || !strcmp(k, "landuse") + || !strcmp(k, "sport") || !strcmp(k, "amenity") + || !strcmp(k, "tourism") || !strcmp(k, "building")) { + create_multipolygon = true; + } + } + + for (vector::iterator it = ids.begin(); + it != ids.end(); ++it) { + fprintf(d->way_tags, "\"" F_U64 "\",", *it); + write_csv_col(d->way_tags, k, ','); + write_csv_col(d->way_tags, v, '\n'); + } + } + + if (multiple_parts && create_multipolygon) { + uint64_t ent_id = d->new_relation_id++; + + fprintf(d->relations, "\"" F_U64 "\",", ent_id); + write_csv_col(d->relations, user_id, ','); + write_csv_col(d->relations, timestamp, '\n'); + + fprintf(d->relation_tags, + "\"" F_U64 "\",\"type\",\"multipolygon\"\n", ent_id); + + for (vector::iterator it = ids.begin(); + it != ids.end(); ++it) { + fprintf(d->relation_members, + "\"" F_U64 "\",\"way\",\"" F_U64 "\",\"\"\n", ent_id, *it); + } + } + } + if (mysql_errno(d->mysql)) exit_stmt_err(load_tags); + + mysql_stmt_close(load_segs); + mysql_stmt_close(load_tags); + + mysql_free_result(res); + free(k); + free(v); +} + +static int read_seg_tags(char **tags, char **k, char **v) { + if (!**tags) return 0; + char *i = strchr(*tags, ';'); + if (!i) i = *tags + strlen(*tags); + char *j = strchr(*tags, '='); + *k = *tags; + if (j && j < i) { + *v = j + 1; + } else { + *v = i; + } + *tags = *i ? i + 1 : i; + *i = '\0'; + if (j) *j = '\0'; + return 1; +} + +static void mark_tagged_segs(struct data *d) { + MYSQL_RES *res; + MYSQL_ROW row; + + if (mysql_query(d->mysql, "SELECT id, tags FROM current_segments " + "WHERE visible && tags != '' && tags != 'created_by=JOSM'")) + exit_mysql_err(d->mysql); + + res = mysql_use_result(d->mysql); + if (!res) exit_mysql_err(d->mysql); + + while ((row = mysql_fetch_row(res))) { + size_t id = parse(row[0]); + if (d->rem_segs[id]) continue; + char *tags_it = row[1], *k, *v; + while (read_seg_tags(&tags_it, &k, &v)) { + if (strcmp(k, "created_by") && + strcmp(k, "tiger:county") && + strcmp(k, "tiger:upload_uuid") && + strcmp(k, "converted_by") && + (strcmp(k, "natural") || strcmp(v, "coastline")) && + (strcmp(k, "source") || strcmp(v, "PGS"))) { + d->rem_segs[id] = 1; + break; + } + } + } + + mysql_free_result(res); +} + +static void convert_remaining_segs(struct data *d) { + MYSQL_STMT *load_seg; + MYSQL_BIND args[1], res[3]; + const size_t max_tag_len = 1 << 16; + char *tags, timestamp[100]; + char *k, *v; + int user_id; + long long mysql_id; + unsigned long res_len; + my_bool res_error; + const char load_seg_stmt[] = + "SELECT user_id, tags, CAST(timestamp AS CHAR) FROM current_segments " + "WHERE visible && id = ?"; + + tags = (char *) malloc(max_tag_len); + + load_seg = mysql_stmt_init(d->mysql); + if (!load_seg) exit_mysql_err(d->mysql); + if (mysql_stmt_prepare(load_seg, load_seg_stmt, sizeof(load_seg_stmt))) + exit_stmt_err(load_seg); + + memset(args, 0, sizeof(args)); + args[0].buffer_type = MYSQL_TYPE_LONGLONG; + args[0].buffer = (char *) &mysql_id; + args[0].is_null = 0; + args[0].length = 0; + if (mysql_stmt_bind_param(load_seg, args)) + exit_stmt_err(load_seg); + + memset(res, 0, sizeof(res)); + res[0].buffer_type = MYSQL_TYPE_LONG; + res[0].buffer = (char *) &user_id; + res[0].is_null = 0; + res[0].length = 0; + res[0].error = &res_error; + res[1].buffer_type = MYSQL_TYPE_STRING; + res[1].buffer = tags; + res[1].is_null = 0; + res[1].length = &res_len; + res[1].error = &res_error; + res[1].buffer_length = max_tag_len; + res[2].buffer_type = MYSQL_TYPE_STRING; + res[2].buffer = timestamp; + res[2].is_null = 0; + res[2].length = &res_len; + res[2].error = &res_error; + res[2].buffer_length = sizeof(timestamp); + if (mysql_stmt_bind_result(load_seg, res)) + exit_stmt_err(load_seg); + + for (size_t seg_id = 0; seg_id < d->segs_len; seg_id++) { + if (!d->rem_segs[seg_id]) continue; + segment seg = d->segs[seg_id]; + + mysql_id = seg_id; + if (mysql_stmt_execute(load_seg)) exit_stmt_err(load_seg); + if (mysql_stmt_store_result(load_seg)) exit_stmt_err(load_seg); + + while (!mysql_stmt_fetch(load_seg)) { + uint64_t way_id = d->new_way_id++; + + fprintf(d->ways, "\"" F_U64 "\",\"%i\",", way_id, user_id); + write_csv_col(d->ways, timestamp, '\n'); + + fprintf(d->way_nodes, "\"" F_U64 "\",\"" F_U32 "\",\"%i\"\n", way_id, seg.from, 1); + fprintf(d->way_nodes, "\"" F_U64 "\",\"" F_U32 "\",\"%i\"\n", way_id, seg.to, 2); + + char *tags_it = tags; + while (read_seg_tags(&tags_it, &k, &v)) { + fprintf(d->way_tags, "\"" F_U64 "\",", way_id); + write_csv_col(d->way_tags, k, ','); + write_csv_col(d->way_tags, v, '\n'); + } + } + } + + mysql_stmt_close(load_seg); + + free(tags); +} + +static MYSQL *connect_to_mysql(char **argv) { + MYSQL *mysql = mysql_init(NULL); + if (!mysql) exit_mysql_err(mysql); + + if (!mysql_real_connect(mysql, argv[1], argv[2], argv[3], argv[4], + argv[5][0] ? atoi(argv[5]) : 0, argv[6][0] ? argv[6] : NULL, 0)) + exit_mysql_err(mysql); + + if (mysql_set_character_set(mysql, "utf8")) + exit_mysql_err(mysql); + + return mysql; +} + +static void open_file(FILE **f, char *fn) { + *f = fopen(fn, "w+"); + if (!*f) { + perror("fopen"); + exit(EXIT_FAILURE); + } +} + +int main(int argc, char **argv) { + struct data data; + struct data *d = &data; + size_t prefix_len; + char *tempfn; + + if (argc != 8) { - printf("Usage: 006_remove_segments_helper host user passwd database port socket prefix\n"); ++ printf("Usage: 007_remove_segments_helper host user passwd database port socket prefix\n"); + exit(EXIT_FAILURE); + } + + d->mysql = connect_to_mysql(argv); + d->mysql2 = connect_to_mysql(argv); + + prefix_len = strlen(argv[7]); + tempfn = (char *) malloc(prefix_len + 15); + strcpy(tempfn, argv[7]); + + strcpy(tempfn + prefix_len, "ways"); + open_file(&d->ways, tempfn); + + strcpy(tempfn + prefix_len, "way_nodes"); + open_file(&d->way_nodes, tempfn); + + strcpy(tempfn + prefix_len, "way_tags"); + open_file(&d->way_tags, tempfn); + + strcpy(tempfn + prefix_len, "relations"); + open_file(&d->relations, tempfn); + + strcpy(tempfn + prefix_len, "relation_members"); + open_file(&d->relation_members, tempfn); + + strcpy(tempfn + prefix_len, "relation_tags"); + open_file(&d->relation_tags, tempfn); + + free(tempfn); + + find_maxids(d); + populate_segs(d); + convert_ways(d); + mark_tagged_segs(d); + convert_remaining_segs(d); + + mysql_close(d->mysql); + mysql_close(d->mysql2); + + fclose(d->ways); + fclose(d->way_nodes); + fclose(d->way_tags); + + fclose(d->relations); + fclose(d->relation_members); + fclose(d->relation_tags); + + free(d->segs); + free(d->rem_segs); + + exit(EXIT_SUCCESS); +} diff --cc test/fixtures/current_relation_members.yml index 7921ac145,000000000..bddc8a0dd mode 100644,000000..100644 --- a/test/fixtures/current_relation_members.yml +++ b/test/fixtures/current_relation_members.yml @@@ -1,5 -1,0 +1,23 @@@ +t1: ++ id: 1 ++ member_role: "some" ++ member_type: "way" ++ member_id: 3 ++ ++t2: + id: 1 + member_role: "some" + member_type: "node" ++ member_id: 5 ++ ++t3: ++ id: 1 ++ member_role: "some" ++ member_type: "relation" + member_id: 3 ++ ++t4: ++ id: 3 ++ member_role: "some" ++ member_type: "node" ++ member_id: 5 diff --cc test/fixtures/current_relation_tags.yml index ba795b698,000000000..aaf06a397 mode 100644,000000..100644 --- a/test/fixtures/current_relation_tags.yml +++ b/test/fixtures/current_relation_tags.yml @@@ -1,9 -1,0 +1,14 @@@ +t1: + id: 1 + k: test + v: yes + +t2: + id: 2 + k: test + v: yes ++ ++t2: ++ id: 3 ++ k: test ++ v: yes diff --cc test/fixtures/current_relations.yml index 67c8ddb95,000000000..c1f77d428 mode 100644,000000..100644 --- a/test/fixtures/current_relations.yml +++ b/test/fixtures/current_relations.yml @@@ -1,11 -1,0 +1,17 @@@ +visible_relation: + id: 1 + user_id: 1 + timestamp: 2007-01-01 00:00:00 + visible: 1 + +invisible_relation: + id: 2 + user_id: 1 + timestamp: 2007-01-01 00:00:00 + visible: 0 ++ ++used_relation: ++ id: 3 ++ user_id: 1 ++ timestamp: 2007-01-01 00:00:00 ++ visible: 1 diff --cc test/fixtures/relation_members.yml index f424e6a51,000000000..27e8b533a mode 100644,000000..100644 --- a/test/fixtures/relation_members.yml +++ b/test/fixtures/relation_members.yml @@@ -1,6 -1,0 +1,24 @@@ +t1: ++ id: 1 ++ member_role: "some" ++ member_type: "way" ++ member_id: 3 ++ version: 1 ++t2: + id: 1 + member_role: "some" + member_type: "node" ++ member_id: 5 ++ version: 1 ++t3: ++ id: 1 ++ member_role: "some" ++ member_type: "relation" + member_id: 3 + version: 1 ++t4: ++ id: 3 ++ member_role: "some" ++ member_type: "node" ++ member_id: 5 ++ version: 1 diff --cc test/fixtures/relation_tags.yml index d1c69a6d4,000000000..39f4bd5de mode 100644,000000..100644 --- a/test/fixtures/relation_tags.yml +++ b/test/fixtures/relation_tags.yml @@@ -1,11 -1,0 +1,17 @@@ +t1: + id: 1 + k: test + v: yes + version: 1 + +t2: + id: 2 + k: test + v: yes + version: 1 ++ ++t3: ++ id: 3 ++ k: test ++ v: yes ++ version: 1 diff --cc test/fixtures/relations.yml index f1e4026ff,000000000..cf1d1ff56 mode 100644,000000..100644 --- a/test/fixtures/relations.yml +++ b/test/fixtures/relations.yml @@@ -1,13 -1,0 +1,20 @@@ +visible_relation: + id: 1 + user_id: 1 + timestamp: 2007-01-01 00:00:00 + visible: 1 + version: 1 + +invisible_relation: + id: 2 + user_id: 1 + timestamp: 2007-01-01 00:00:00 + visible: 0 + version: 1 ++ ++used_relation: ++ id: 3 ++ user_id: 1 ++ timestamp: 2007-01-01 00:00:00 ++ visible: 1 ++ version: 1 diff --cc test/functional/relation_controller_test.rb index 2893ba956,000000000..8f8b72770 mode 100644,000000..100644 --- a/test/functional/relation_controller_test.rb +++ b/test/functional/relation_controller_test.rb @@@ -1,209 -1,0 +1,209 @@@ +require File.dirname(__FILE__) + '/../test_helper' +require 'relation_controller' + +# Re-raise errors caught by the controller. +class RelationController; def rescue_action(e) raise e end; end + +class RelationControllerTest < Test::Unit::TestCase + api_fixtures + fixtures :relations, :current_relations, :relation_members, :current_relation_members, :relation_tags, :current_relation_tags + set_fixture_class :current_relations => :Relation + set_fixture_class :relations => :OldRelation + + def setup + @controller = RelationController.new + @request = ActionController::TestRequest.new + @response = ActionController::TestResponse.new + end + + def basic_authorization(user, pass) + @request.env["HTTP_AUTHORIZATION"] = "Basic %s" % Base64.encode64("#{user}:#{pass}") + end + + def content(c) + @request.env["RAW_POST_DATA"] = c + end + + # ------------------------------------- + # Test reading relations. + # ------------------------------------- + + def test_read + # check that a visible relation is returned properly + get :read, :id => current_relations(:visible_relation).id + assert_response :success + + # check that an invisible relation is not returned + get :read, :id => current_relations(:invisible_relation).id + assert_response :gone + + # check chat a non-existent relation is not returned + get :read, :id => 0 + assert_response :not_found + + # check the "relations for node" mode - get :relations_for_node, :id => current_nodes(:used_node_1).id ++ get :relations_for_node, :id => current_nodes(:node_used_by_relationship).id + assert_response :success + # FIXME check whether this contains the stuff we want! + if $VERBOSE + print @response.body + end + + # check the "relations for way" mode + get :relations_for_way, :id => current_ways(:used_way).id + assert_response :success + # FIXME check whether this contains the stuff we want! + if $VERBOSE + print @response.body + end + + + # check the "relations for relation" mode - get :relations_for_node, :id => current_relations(:used_relation).id ++ get :relations_for_relation, :id => current_relations(:used_relation).id + assert_response :success + # FIXME check whether this contains the stuff we want! + if $VERBOSE + print @response.body + end + + # check the "full" mode - get :full, :id => current_relations(:relation_using_all).id ++ get :full, :id => current_relations(:visible_relation).id + assert_response :success + # FIXME check whether this contains the stuff we want! + if $VERBOSE + print @response.body + end + end + + # ------------------------------------- + # Test simple relation creation. + # ------------------------------------- + + def test_create + basic_authorization "test@openstreetmap.org", "test" + + # create an relation without members + content "" + put :create + # hope for success + assert_response :success, + "relation upload did not return success status" + # read id of created relation and search for it + relationid = @response.body + checkrelation = Relation.find(relationid) + assert_not_nil checkrelation, + "uploaded relation not found in data base after upload" + # compare values + assert_equal checkrelation.members.length, 0, + "saved relation contains members but should not" + assert_equal checkrelation.tags.length, 1, + "saved relation does not contain exactly one tag" + assert_equal users(:normal_user).id, checkrelation.user_id, + "saved relation does not belong to user that created it" + assert_equal true, checkrelation.visible, + "saved relation is not visible" + # ok the relation is there but can we also retrieve it? + get :read, :id => relationid + assert_response :success + + + # create an relation with a node as member + nid = current_nodes(:used_node_1).id + content "" + + "" + put :create + # hope for success + assert_response :success, + "relation upload did not return success status" + # read id of created relation and search for it + relationid = @response.body + checkrelation = Relation.find(relationid) + assert_not_nil checkrelation, + "uploaded relation not found in data base after upload" + # compare values + assert_equal checkrelation.members.length, 1, + "saved relation does not contain exactly one member" + assert_equal checkrelation.tags.length, 1, + "saved relation does not contain exactly one tag" + assert_equal users(:normal_user).id, checkrelation.user_id, + "saved relation does not belong to user that created it" + assert_equal true, checkrelation.visible, + "saved relation is not visible" + # ok the relation is there but can we also retrieve it? + + get :read, :id => relationid + assert_response :success + + # create an relation with a way and a node as members + nid = current_nodes(:used_node_1).id + wid = current_ways(:used_way).id + content "" + + "" + + "" + put :create + # hope for success + assert_response :success, + "relation upload did not return success status" + # read id of created relation and search for it + relationid = @response.body + checkrelation = Relation.find(relationid) + assert_not_nil checkrelation, + "uploaded relation not found in data base after upload" + # compare values + assert_equal checkrelation.members.length, 2, + "saved relation does not have exactly two members" + assert_equal checkrelation.tags.length, 1, + "saved relation does not contain exactly one tag" + assert_equal users(:normal_user).id, checkrelation.user_id, + "saved relation does not belong to user that created it" + assert_equal true, checkrelation.visible, + "saved relation is not visible" + # ok the relation is there but can we also retrieve it? + get :read, :id => relationid + assert_response :success + + end + + # ------------------------------------- + # Test creating some invalid relations. + # ------------------------------------- + + def test_create_invalid + basic_authorization "test@openstreetmap.org", "test" + + # create a relation with non-existing node as member + content "" + put :create + # expect failure + assert_response :precondition_failed, + "relation upload with invalid node did not return 'precondition failed'" + end + + # ------------------------------------- + # Test deleting relations. + # ------------------------------------- + + def test_delete + return true + + # first try to delete relation without auth + delete :delete, :id => current_relations(:visible_relation).id + assert_response :unauthorized + + # now set auth + basic_authorization("test@openstreetmap.org", "test"); + + # this should work + delete :delete, :id => current_relations(:visible_relation).id + assert_response :success + + # this won't work since the relation is already deleted + delete :delete, :id => current_relations(:invisible_relation).id + assert_response :gone + + # this won't work since the relation never existed + delete :delete, :id => 0 + assert_response :not_found + end + +end diff --cc test/unit/node_test.rb index b3e834370,b3e834370..95321b5cf --- a/test/unit/node_test.rb +++ b/test/unit/node_test.rb @@@ -11,7 -11,7 +11,7 @@@ class NodeTest < Test::Unit::TestCas :user_id => users(:normal_user).id, :visible => 1, :tags => "") -- assert node_template.save_with_history ++ assert node_template.save_with_history! node = Node.find(node_template.id) assert_not_nil node @@@ -44,7 -44,7 +44,7 @@@ node_template.latitude = 12.3456 node_template.longitude = 65.4321 node_template.tags = "updated=yes" -- assert node_template.save_with_history ++ assert node_template.save_with_history! node = Node.find(node_template.id) assert_not_nil node @@@ -76,7 -76,7 +76,7 @@@ assert_not_nil old_node_template node_template.visible = 0 -- assert node_template.save_with_history ++ assert node_template.save_with_history! node = Node.find(node_template.id) assert_not_nil node