From: Shaun McDonald Date: Thu, 25 Sep 2008 15:06:05 +0000 (+0000) Subject: Doing a resync from mainline 8633:10895. There was one simple to resolve conflict... X-Git-Tag: live~7601^2~305 X-Git-Url: https://git.openstreetmap.org/rails.git/commitdiff_plain/d9e070e06956801aba2378c1b79b4d9f13ae12ee?hp=-c Doing a resync from mainline 8633:10895. There was one simple to resolve conflict on app/models/node.rb. Also moving the migrations for API0.6 to new sequence numbers since there was some new migrations added to mainline, where the migration numbers would conflict if not moved. --- d9e070e06956801aba2378c1b79b4d9f13ae12ee diff --combined app/controllers/api_controller.rb index 05dfb0133,6b36b41ae..2f040a92b --- a/app/controllers/api_controller.rb +++ b/app/controllers/api_controller.rb @@@ -120,12 -120,20 +120,20 @@@ class ApiController < ApplicationContro return end if node_ids.length == 0 - render :text => "", :content_type => "text/xml" + render :text => "", :content_type => "text/xml" return end doc = OSM::API.new.get_xml_doc + # add bounds + bounds = XML::Node.new 'bounds' + bounds['minlat'] = min_lat.to_s + bounds['minlon'] = min_lon.to_s + bounds['maxlat'] = max_lat.to_s + bounds['maxlon'] = max_lon.to_s + doc.root << bounds + # get ways # find which ways are needed ways = Array.new @@@ -168,15 -176,15 +176,15 @@@ end end - relations = visible_nodes.values.collect { |node| node.containing_relations.visible }.flatten + - way_ids.collect { |id| Way.find(id).containing_relations.visible }.flatten + relations = Relation.find_for_nodes(visible_nodes.keys, :conditions => "visible = 1") + + Relation.find_for_ways(way_ids, :conditions => "visible = 1") # we do not normally return the "other" partners referenced by an relation, # e.g. if we return a way A that is referenced by relation X, and there's # another way B also referenced, that is not returned. But we do make # an exception for cases where an relation references another *relation*; # in that case we return that as well (but we don't go recursive here) - relations += relations.collect { |relation| relation.containing_relations.visible }.flatten + relations += Relation.find_for_relations(relations.collect { |r| r.id }, :conditions => "visible = 1") # this "uniq" may be slightly inefficient; it may be better to first collect and output # all node-related relations, then find the *not yet covered* way-related ones etc. @@@ -246,8 -254,8 +254,8 @@@ api = XML::Node.new 'api' version = XML::Node.new 'version' - version['minimum'] = '0.5'; - version['maximum'] = '0.5'; + version['minimum'] = "#{API_VERSION}"; + version['maximum'] = "#{API_VERSION}"; api << version area = XML::Node.new 'area' area['maximum'] = MAX_REQUEST_AREA.to_s; diff --combined app/controllers/application.rb index 68359585e,ce13a6aa3..1c27cb4d5 --- a/app/controllers/application.rb +++ b/app/controllers/application.rb @@@ -8,7 -8,7 +8,7 @@@ class ApplicationController < ActionCon def authorize_web if session[:user] - @user = User.find(session[:user]) + @user = User.find(session[:user], :conditions => "visible = 1") elsif session[:token] @user = User.authenticate(:token => session[:token]) session[:user] = @user.id @@@ -71,7 -71,7 +71,7 @@@ # phrase from that, we can also put the error message into the status # message. For now, rails won't let us) def report_error(message) - render :nothing => true, :status => :bad_request + render :text => message, :status => :bad_request # Todo: some sort of escaping of problem characters in the message response.headers['Error'] = message end @@@ -82,8 -82,6 +82,8 @@@ privat def get_auth_data if request.env.has_key? 'X-HTTP_AUTHORIZATION' # where mod_rewrite might have put it authdata = request.env['X-HTTP_AUTHORIZATION'].to_s.split + elsif request.env.has_key? 'REDIRECT_X_HTTP_AUTHORIZATION' # mod_fcgi + authdata = request.env['REDIRECT_X_HTTP_AUTHORIZATION'].to_s.split elsif request.env.has_key? 'HTTP_AUTHORIZATION' # regular location authdata = request.env['HTTP_AUTHORIZATION'].to_s.split end diff --combined app/models/node.rb index c8770922d,cec755f47..677023179 --- a/app/models/node.rb +++ b/app/models/node.rb @@@ -17,8 -17,9 +17,11 @@@ class Node < ActiveRecord::Bas has_many :way_nodes has_many :ways, :through => :way_nodes + has_many :node_tags, :foreign_key => :id + + has_many :old_way_nodes + has_many :ways_via_history, :class_name=> "Way", :through => :old_way_nodes, :source => :way + has_many :containing_relation_members, :class_name => "RelationMember", :as => :member has_many :containing_relations, :class_name => "Relation", :through => :containing_relation_members, :source => :relation, :extend => ObjectFinder @@@ -59,109 -60,64 +62,109 @@@ p = XML::Parser.new p.string = xml doc = p.parse - - node = Node.new doc.find('//osm/node').each do |pt| - node.lat = pt['lat'].to_f - node.lon = pt['lon'].to_f + return Node.from_xml_node(pt, create) + end + rescue + return nil + end + end - return nil unless node.in_world? + def self.from_xml_node(pt, create=false) + node = Node.new + + node.version = pt['version'] + node.lat = pt['lat'].to_f + node.lon = pt['lon'].to_f - unless create - if pt['id'] != '0' - node.id = pt['id'].to_i - end - end + return nil unless node.in_world? - node.visible = pt['visible'] and pt['visible'] == 'true' + unless create + if pt['id'] != '0' + node.id = pt['id'].to_i + end + end - if create - node.timestamp = Time.now - else - if pt['timestamp'] - node.timestamp = Time.parse(pt['timestamp']) - end - end + node.visible = pt['visible'] and pt['visible'] == 'true' - tags = [] + if create + node.timestamp = Time.now + else + if pt['timestamp'] + node.timestamp = Time.parse(pt['timestamp']) + end + end - pt.find('tag').each do |tag| - tags << [tag['k'],tag['v']] - end + tags = [] - node.tags = Tags.join(tags) - end - rescue - node = nil + pt.find('tag').each do |tag| + node.add_tag_key_val(tag['k'],tag['v']) end return node end - # Save this node with the appropriate OldNode object to represent it's history. def save_with_history! + t = Time.now Node.transaction do - self.timestamp = Time.now + self.version += 1 + self.timestamp = t self.save! + + # Create a NodeTag + tags = self.tags + NodeTag.delete_all(['id = ?', self.id]) + tags.each do |k,v| + tag = NodeTag.new + tag.k = k + tag.v = v + tag.id = self.id + tag.save! + end + + # Create an OldNode old_node = OldNode.from_node(self) - old_node.save! + old_node.timestamp = t + old_node.save_with_dependencies! + end + end + + def delete_with_history(user) + if self.visible + if WayNode.find(:first, :joins => "INNER JOIN current_ways ON current_ways.id = current_way_nodes.id", :conditions => [ "current_ways.visible = 1 AND current_way_nodes.node_id = ?", self.id ]) + raise OSM::APIPreconditionFailedError.new + elsif RelationMember.find(:first, :joins => "INNER JOIN current_relations ON current_relations.id=current_relation_members.id", :conditions => [ "visible = 1 AND member_type='node' and member_id=?", self.id]) + raise OSM::APIPreconditionFailedError.new + else + self.user_id = user.id + self.visible = 0 + save_with_history! + end + else + raise OSM::APIAlreadyDeletedError.new + end + end + + def update_from(new_node, user) + if new_node.version != version + raise OSM::APIVersionMismatchError.new(new_node.version, version) end + + self.user_id = user.id + self.latitude = new_node.latitude + self.longitude = new_node.longitude + self.tags = new_node.tags + self.visible = true + save_with_history! end - # Turn this Node in to a complete OSM XML object with wrapper def to_xml doc = OSM::API.new.get_xml_doc doc.root << to_xml_node() return doc end - # Turn this Node in to an XML Node without the wrapper. def to_xml_node(user_display_name_cache = nil) el1 = XML::Node.new 'node' el1['id'] = self.id.to_s @@@ -180,7 -136,7 +183,7 @@@ el1['user'] = user_display_name_cache[self.user_id] unless user_display_name_cache[self.user_id].nil? - Tags.split(self.tags) do |k,v| + self.tags.each do |k,v| el2 = XML::Node.new('tag') el2['k'] = k.to_s el2['v'] = v.to_s @@@ -189,33 -145,15 +192,33 @@@ el1['visible'] = self.visible.to_s el1['timestamp'] = self.timestamp.xmlschema + el1['version'] = self.version.to_s return el1 end - # Return the node's tags as a Hash of keys and their values def tags_as_hash - hash = {} - Tags.split(self.tags) do |k,v| - hash[k] = v + return tags + end + + def tags + unless @tags + @tags = {} + self.node_tags.each do |tag| + @tags[tag.k] = tag.v + end end - hash + @tags + end + + def tags=(t) + @tags = t + end + + def add_tag_key_val(k,v) + @tags = Hash.new unless @tags + @tags[k] = v end + + + end diff --combined app/models/old_way.rb index edf66aac3,63265d6bf..3c88c4673 --- a/app/models/old_way.rb +++ b/app/models/old_way.rb @@@ -9,7 -9,6 +9,7 @@@ class OldWay < ActiveRecord::Bas old_way.user_id = way.user_id old_way.timestamp = way.timestamp old_way.id = way.id + old_way.version = way.version old_way.nds = way.nds old_way.tags = way.tags return old_way @@@ -95,7 -94,6 +95,7 @@@ el1['visible'] = self.visible.to_s el1['timestamp'] = self.timestamp.xmlschema el1['user'] = self.user.display_name if self.user.data_public? + el1['version'] = self.version.to_s self.old_nodes.each do |nd| # FIXME need to make sure they come back in the right order e = XML::Node.new 'nd' @@@ -112,6 -110,35 +112,35 @@@ return el1 end + # Read full version of old way + # For get_nodes_undelete, uses same nodes, even if they've moved since + # For get_nodes_revert, allocates new ids + # Currently returns Potlatch-style array + + def get_nodes_undelete + points = [] + self.nds.each do |n| + node=Node.find(n) + points << [node.lon, node.lat, n, node.visible ? 1 : 0, node.tags_as_hash] + end + points + end + + def get_nodes_revert + points=[] + self.nds.each do |n| + oldnode=OldNode.find(:first, :conditions=>['id=? AND timestamp<=?',n,self.timestamp], :order=>"timestamp DESC") + curnode=Node.find(n) + id=n; v=curnode.visible ? 1 : 0 + if oldnode.lat!=curnode.lat or oldnode.lon!=curnode.lon or oldnode.tags!=curnode.tags then + # node has changed: if it's in other ways, give it a new id + if curnode.ways-[self.id] then id=-1; v=nil end + end + points << [oldnode.lon, oldnode.lat, id, v, oldnode.tags_as_hash] + end + points + end + # Temporary method to match interface to nodes def tags_as_hash return self.tags diff --combined app/models/relation.rb index eb3b06a13,c8516b58a..e46da5ade --- a/app/models/relation.rb +++ b/app/models/relation.rb @@@ -19,38 -19,32 +19,38 @@@ class Relation < ActiveRecord::Bas p.string = xml doc = p.parse - relation = Relation.new - doc.find('//osm/relation').each do |pt| - if !create and pt['id'] != '0' - relation.id = pt['id'].to_i - end + return Relation.from_xml_node(pt, create) + end + rescue + return nil + end + end - if create - relation.timestamp = Time.now - relation.visible = true - else - if pt['timestamp'] - relation.timestamp = Time.parse(pt['timestamp']) - end - end + def self.from_xml_node(pt, create=false) + relation = Relation.new - pt.find('tag').each do |tag| - relation.add_tag_keyval(tag['k'], tag['v']) - end + if !create and pt['id'] != '0' + relation.id = pt['id'].to_i + end - pt.find('member').each do |member| - relation.add_member(member['type'], member['ref'], member['role']) - end + relation.version = pt['version'] + + if create + relation.timestamp = Time.now + relation.visible = true + else + if pt['timestamp'] + relation.timestamp = Time.parse(pt['timestamp']) end - rescue - relation = nil + end + + pt.find('tag').each do |tag| + relation.add_tag_keyval(tag['k'], tag['v']) + end + + pt.find('member').each do |member| + relation.add_member(member['type'], member['ref'], member['role']) end return relation @@@ -67,7 -61,6 +67,7 @@@ el1['id'] = self.id.to_s el1['visible'] = self.visible.to_s el1['timestamp'] = self.timestamp.xmlschema + el1['version'] = self.version.to_s user_display_name_cache = {} if user_display_name_cache.nil? @@@ -112,6 -105,36 +112,36 @@@ return el1 end + def self.find_for_nodes(ids, options = {}) + if ids.empty? + return [] + else + self.with_scope(:find => { :joins => "INNER JOIN current_relation_members ON current_relation_members.id = current_relations.id", :conditions => "current_relation_members.member_type = 'node' AND current_relation_members.member_id IN (#{ids.join(',')})" }) do + return self.find(:all, options) + end + end + end + + def self.find_for_ways(ids, options = {}) + if ids.empty? + return [] + else + self.with_scope(:find => { :joins => "INNER JOIN current_relation_members ON current_relation_members.id = current_relations.id", :conditions => "current_relation_members.member_type = 'way' AND current_relation_members.member_id IN (#{ids.join(',')})" }) do + return self.find(:all, options) + end + end + end + + def self.find_for_relations(ids, options = {}) + if ids.empty? + return [] + else + self.with_scope(:find => { :joins => "INNER JOIN current_relation_members ON current_relation_members.id = current_relations.id", :conditions => "current_relation_members.member_type = 'relation' AND current_relation_members.member_id IN (#{ids.join(',')})" }) do + return self.find(:all, options) + end + end + end + # FIXME is this really needed? def members unless @members @@@ -154,12 -177,13 +184,12 @@@ def save_with_history! Relation.transaction do t = Time.now + self.version += 1 self.timestamp = t self.save! tags = self.tags - RelationTag.delete_all(['id = ?', self.id]) - tags.each do |k,v| tag = RelationTag.new tag.k = k @@@ -169,7 -193,9 +199,7 @@@ end members = self.members - RelationMember.delete_all(['id = ?', self.id]) - members.each do |n| mem = RelationMember.new mem.id = self.id @@@ -185,36 -211,6 +215,36 @@@ end end + def delete_with_history(user) + if self.visible + if RelationMember.find(:first, :joins => "INNER JOIN current_relations ON current_relations.id=current_relation_members.id", :conditions => [ "visible = 1 AND member_type='relation' and member_id=?", self.id ]) + raise OSM::APIPreconditionFailedError.new + else + self.user_id = user.id + self.tags = [] + self.members = [] + self.visible = false + save_with_history! + end + else + raise OSM::APIAlreadyDeletedError.new + end + end + + def update_from(new_relation, user) + if !new_relation.preconditions_ok? + raise OSM::APIPreconditionFailedError.new + elsif new_relation.version != version + raise OSM::APIVersionMismatchError.new(new_relation.version, version) + else + self.user_id = user.id + self.tags = new_relation.tags + self.members = new_relation.members + self.visible = true + save_with_history! + end + end + def preconditions_ok? # These are hastables that store an id in the index of all # the nodes/way/relations that have already been added. diff --combined app/models/way.rb index 34afc6585,958944200..3bc8bcebe --- a/app/models/way.rb +++ b/app/models/way.rb @@@ -21,38 -21,32 +21,38 @@@ class Way < ActiveRecord::Bas p.string = xml doc = p.parse - way = Way.new - doc.find('//osm/way').each do |pt| - if !create and pt['id'] != '0' - way.id = pt['id'].to_i - end + return Way.from_xml_node(pt, create) + end + rescue + return nil + end + end - if create - way.timestamp = Time.now - way.visible = true - else - if pt['timestamp'] - way.timestamp = Time.parse(pt['timestamp']) - end - end + def self.from_xml_node(pt, create=false) + way = Way.new - pt.find('tag').each do |tag| - way.add_tag_keyval(tag['k'], tag['v']) - end + if !create and pt['id'] != '0' + way.id = pt['id'].to_i + end + + way.version = pt['version'] - pt.find('nd').each do |nd| - way.add_nd_num(nd['ref']) - end + if create + way.timestamp = Time.now + way.visible = true + else + if pt['timestamp'] + way.timestamp = Time.parse(pt['timestamp']) end - rescue - way = nil + end + + pt.find('tag').each do |tag| + way.add_tag_keyval(tag['k'], tag['v']) + end + + pt.find('nd').each do |nd| + way.add_nd_num(nd['ref']) end return way @@@ -80,7 -74,6 +80,7 @@@ el1['id'] = self.id.to_s el1['visible'] = self.visible.to_s el1['timestamp'] = self.timestamp.xmlschema + el1['version'] = self.version.to_s user_display_name_cache = {} if user_display_name_cache.nil? @@@ -169,12 -162,15 +169,12 @@@ t = Time.now Way.transaction do + self.version += 1 self.timestamp = t self.save! - end - WayTag.transaction do tags = self.tags - WayTag.delete_all(['id = ?', self.id]) - tags.each do |k,v| tag = WayTag.new tag.k = k @@@ -182,9 -178,13 +182,9 @@@ tag.id = self.id tag.save! end - end - WayNode.transaction do nds = self.nds - WayNode.delete_all(['id = ?', self.id]) - sequence = 1 nds.each do |n| nd = WayNode.new @@@ -193,25 -193,11 +193,25 @@@ nd.save! sequence += 1 end + + old_way = OldWay.from_way(self) + old_way.timestamp = t + old_way.save_with_dependencies! end + end - old_way = OldWay.from_way(self) - old_way.timestamp = t - old_way.save_with_dependencies! + def update_from(new_way, user) + if !new_way.preconditions_ok? + raise OSM::APIPreconditionFailedError.new + elsif new_way.version != version + raise OSM::APIVersionMismatchError.new(new_way.version, version) + else + self.user_id = user.id + self.tags = new_way.tags + self.nds = new_way.nds + self.visible = true + save_with_history! + end end def preconditions_ok? @@@ -225,13 -211,12 +225,13 @@@ return true end - # Delete the way and it's relations, but don't really delete it - set its visibility to false and update the history etc to maintain wiki-like functionality. - def delete_with_relations_and_history(user) + def delete_with_history(user) if self.visible # FIXME # this should actually delete the relations, # not just throw a PreconditionFailed if it's a member of a relation!! + + # FIXME: this should probably renamed to delete_with_history if RelationMember.find(:first, :joins => "INNER JOIN current_relations ON current_relations.id=current_relation_members.id", :conditions => [ "visible = 1 AND member_type='way' and member_id=?", self.id]) raise OSM::APIPreconditionFailedError @@@ -249,20 -234,9 +249,11 @@@ end # delete a way and it's nodes that aren't part of other ways, with history + + # FIXME: merge the potlatch code to delete the relations def delete_with_relations_and_nodes_and_history(user) - - node_ids = self.nodes.collect {|node| node.id } - node_ids_not_to_delete = [] - way_nodes = WayNode.find(:all, :conditions => "node_id in (#{node_ids.join(',')}) and id != #{self.id}") - - node_ids_not_to_delete = way_nodes.collect {|way_node| way_node.node_id} - - node_ids_to_delete = node_ids - node_ids_not_to_delete - # delete the nodes not used by other ways - node_ids_to_delete.each do |node_id| + self.unshared_node_ids.each do |node_id| n = Node.find(node_id) n.user_id = user.id n.visible = false @@@ -271,8 -245,19 +262,19 @@@ self.user_id = user.id - self.delete_with_relations_and_history(user) + self.delete_with_history(user) + end + + # Find nodes that belong to this way only + def unshared_node_ids + node_ids = self.nodes.collect { |node| node.id } + + unless node_ids.empty? + way_nodes = WayNode.find(:all, :conditions => "node_id in (#{node_ids.join(',')}) and id != #{self.id}") + node_ids = node_ids - way_nodes.collect { |way_node| way_node.node_id } + end + return node_ids end # Temporary method to match interface to nodes diff --combined config/environment.rb index fb7573d2a,e6af619eb..08c433788 --- a/config/environment.rb +++ b/config/environment.rb @@@ -11,7 -11,7 +11,7 @@@ RAILS_GEM_VERSION = '2.0.2' unless defi SERVER_URL = ENV['OSM_SERVER_URL'] || 'www.openstreetmap.org' # Application constants needed for routes.rb - must go before Initializer call -API_VERSION = ENV['OSM_API_VERSION'] || '0.5' +API_VERSION = ENV['OSM_API_VERSION'] || '0.6' # Set application status - possible settings are: # @@@ -74,8 -74,3 +74,3 @@@ Rails::Initializer.run do |config # Make Active Record use UTC-base instead of local time # config.active_record.default_timezone = :utc end - - # This has to be after the above block for some reason (doesnt pull in /lib/osm.rb?) - POTLATCH_PRESETS = Potlatch::Potlatch.get_presets() - - diff --combined config/routes.rb index 592178474,f040e1c1d..a45ad1e0b --- a/config/routes.rb +++ b/config/routes.rb @@@ -1,11 -1,6 +1,11 @@@ ActionController::Routing::Routes.draw do |map| # API + map.connect "api/#{API_VERSION}/changeset/create", :controller => 'changeset', :action => 'create' + map.connect "api/#{API_VERSION}/changeset/upload", :controller => 'changeset', :action => 'upload' + map.connect "api/#{API_VERSION}/changeset/:id", :controller => 'changeset', :action => 'read', :id => /\d+/ + map.connect "api/#{API_VERSION}/changeset/:id/close", :controller => 'changeset', :action => 'close', :id =>/\d+/ + map.connect "api/#{API_VERSION}/node/create", :controller => 'node', :action => 'create' map.connect "api/#{API_VERSION}/node/:id/ways", :controller => 'way', :action => 'ways_for_node', :id => /\d+/ map.connect "api/#{API_VERSION}/node/:id/relations", :controller => 'relation', :action => 'relations_for_node', :id => /\d+/ @@@ -59,7 -54,6 +59,7 @@@ # Potlatch API + map.connect "api/0.5/amf", :controller =>'amf', :action =>'talk' map.connect "api/#{API_VERSION}/amf", :controller =>'amf', :action =>'talk' map.connect "api/#{API_VERSION}/swf/trackpoints", :controller =>'swf', :action =>'trackpoints' @@@ -84,9 -78,11 +84,11 @@@ map.connect '/user/new', :controller => 'user', :action => 'new' map.connect '/user/save', :controller => 'user', :action => 'save' map.connect '/user/confirm', :controller => 'user', :action => 'confirm' + map.connect '/user/confirm-email', :controller => 'user', :action => 'confirm_email' map.connect '/user/go_public', :controller => 'user', :action => 'go_public' map.connect '/user/reset-password', :controller => 'user', :action => 'reset_password' map.connect '/user/upload-image', :controller => 'user', :action => 'upload_image' + map.connect '/user/delete-image', :controller => 'user', :action => 'delete_image' map.connect '/user/forgot-password', :controller => 'user', :action => 'lost_password' map.connect '/index.html', :controller => 'site', :action => 'index' @@@ -133,7 -129,8 +135,8 @@@ map.connect '/user/:display_name/diary/:id', :controller => 'diary_entry', :action => 'view', :id => /\d+/ map.connect '/user/:display_name/diary/:id/newcomment', :controller => 'diary_entry', :action => 'comment', :id => /\d+/ map.connect '/user/:display_name/diary/rss', :controller => 'diary_entry', :action => 'rss' - map.connect '/user/:display_name/diary/newpost', :controller => 'diary_entry', :action => 'new' + map.connect '/user/:display_name/diary/new', :controller => 'diary_entry', :action => 'new' + map.connect '/user/:display_name/diary/:id/edit', :controller => 'diary_entry', :action => 'edit', :id => /\d+/ map.connect '/user/:display_name/account', :controller => 'user', :action => 'account' map.connect '/user/:display_name/set_home', :controller => 'user', :action => 'set_home' map.connect '/diary', :controller => 'diary_entry', :action => 'list' diff --combined db/migrate/016_add_timestamp_indexes.rb index c6b3bc7c2,000000000..c6b3bc7c2 mode 100644,000000..100644 --- a/db/migrate/016_add_timestamp_indexes.rb +++ b/db/migrate/016_add_timestamp_indexes.rb @@@ -1,11 -1,0 +1,11 @@@ +class AddTimestampIndexes < ActiveRecord::Migration + def self.up + add_index :current_ways, :timestamp, :name => :current_ways_timestamp_idx + add_index :current_relations, :timestamp, :name => :current_relations_timestamp_idx + end + + def self.down + remove_index :current_ways, :name => :current_ways_timestamp_idx + remove_index :current_relations, :name => :current_relations_timestamp_idx + end +end diff --combined db/migrate/017_populate_node_tags_and_remove.rb index 29a91c70b,000000000..29a91c70b mode 100644,000000..100644 --- a/db/migrate/017_populate_node_tags_and_remove.rb +++ b/db/migrate/017_populate_node_tags_and_remove.rb @@@ -1,62 -1,0 +1,62 @@@ +class PopulateNodeTagsAndRemove < ActiveRecord::Migration + def self.up + have_nodes = select_value("SELECT count(*) FROM current_nodes").to_i != 0 + + if have_nodes + prefix = File.join Dir.tmpdir, "013_populate_node_tags_and_remove.#{$$}." + + cmd = "db/migrate/013_populate_node_tags_and_remove_helper" + src = "#{cmd}.c" + if not File.exists? cmd or File.mtime(cmd) < File.mtime(src) then + system 'cc -O3 -Wall `mysql_config --cflags --libs` ' + + "#{src} -o #{cmd}" or fail + end + + conn_opts = ActiveRecord::Base.connection. + instance_eval { @connection_options } + args = conn_opts.map { |arg| arg.to_s } + [prefix] + fail "#{cmd} failed" unless system cmd, *args + + tempfiles = ['nodes', 'node_tags', + 'current_nodes', 'current_node_tags']. + map { |base| prefix + base } + nodes, node_tags, current_nodes, current_node_tags = tempfiles + end + + execute "TRUNCATE nodes" + remove_column :nodes, :tags + remove_column :current_nodes, :tags + + add_column :nodes, :version, :bigint, :limit => 20, :null => false + + create_table :current_node_tags, innodb_table do |t| + t.column :id, :bigint, :limit => 64, :null => false + t.column :k, :string, :default => "", :null => false + t.column :v, :string, :default => "", :null => false + end + + create_table :node_tags, innodb_table do |t| + t.column :id, :bigint, :limit => 64, :null => false + t.column :version, :bigint, :limit => 20, :null => false + t.column :k, :string, :default => "", :null => false + t.column :v, :string, :default => "", :null => false + end + + # now get the data back + csvopts = "FIELDS TERMINATED BY ',' ENCLOSED BY '\"' ESCAPED BY '\"' LINES TERMINATED BY '\\n'" + + if have_nodes + execute "LOAD DATA INFILE '#{nodes}' INTO TABLE nodes #{csvopts} (id, latitude, longitude, user_id, visible, timestamp, tile, version)"; + execute "LOAD DATA INFILE '#{node_tags}' INTO TABLE node_tags #{csvopts} (id, version, k, v)" + execute "LOAD DATA INFILE '#{current_node_tags}' INTO TABLE current_node_tags #{csvopts} (id, k, v)" + end + + tempfiles.each { |fn| File.unlink fn } if have_nodes + end + + def self.down + raise IrreversibleMigration.new +# add_column :nodes, "tags", :text, :default => "", :null => false +# add_column :current_nodes, "tags", :text, :default => "", :null => false + end +end diff --combined db/migrate/017_populate_node_tags_and_remove_helper.c index 5a0fbb6cd,000000000..5a0fbb6cd mode 100644,000000..100644 --- a/db/migrate/017_populate_node_tags_and_remove_helper.c +++ b/db/migrate/017_populate_node_tags_and_remove_helper.c @@@ -1,241 -1,0 +1,241 @@@ +#include +#include +#include +#include +#include + +static void exit_mysql_err(MYSQL *mysql) { + const char *err = mysql_error(mysql); + if (err) { + fprintf(stderr, "013_populate_node_tags_and_remove_helper: MySQL error: %s\n", err); + } else { + fprintf(stderr, "013_populate_node_tags_and_remove_helper: MySQL error\n"); + } + abort(); + exit(EXIT_FAILURE); +} + +static void write_csv_col(FILE *f, const char *str, char end) { + char *out = (char *) malloc(2 * strlen(str) + 4); + char *o = out; + size_t len; + + *(o++) = '\"'; + for (; *str; str++) { + if (*str == '\0') { + break; + } else if (*str == '\"') { + *(o++) = '\"'; + *(o++) = '\"'; + } else { + *(o++) = *str; + } + } + *(o++) = '\"'; + *(o++) = end; + *(o++) = '\0'; + + len = strlen(out); + if (fwrite(out, len, 1, f) != 1) { + perror("fwrite"); + exit(EXIT_FAILURE); + } + + free(out); +} + +static void unescape(char *str) { + char *i = str, *o = str, tmp; + + while (*i) { + if (*i == '\\') { + i++; + switch (tmp = *i++) { + case 's': *o++ = ';'; break; + case 'e': *o++ = '='; break; + case '\\': *o++ = '\\'; break; + default: *o++ = tmp; break; + } + } else { + *o++ = *i++; + } + } +} + +static int read_node_tags(char **tags, char **k, char **v) { + if (!**tags) return 0; + char *i = strchr(*tags, ';'); + if (!i) i = *tags + strlen(*tags); + char *j = strchr(*tags, '='); + *k = *tags; + if (j && j < i) { + *v = j + 1; + } else { + *v = i; + } + *tags = *i ? i + 1 : i; + *i = '\0'; + if (j) *j = '\0'; + + unescape(*k); + unescape(*v); + + return 1; +} + +struct data { + MYSQL *mysql; + size_t version_size; + uint16_t *version; +}; + +static void proc_nodes(struct data *d, const char *tbl, FILE *out, FILE *out_tags, int hist) { + MYSQL_RES *res; + MYSQL_ROW row; + char query[256]; + + snprintf(query, sizeof(query), "SELECT id, latitude, longitude, " + "user_id, visible, tags, timestamp, tile FROM %s", tbl); + if (mysql_query(d->mysql, query)) + exit_mysql_err(d->mysql); + + res = mysql_use_result(d->mysql); + if (!res) exit_mysql_err(d->mysql); + + while ((row = mysql_fetch_row(res))) { + unsigned long id = strtoul(row[0], NULL, 10); + uint32_t version; + + if (id >= d->version_size) { + fprintf(stderr, "preallocated nodes size exceeded"); + abort(); + } + + if (hist) { + version = ++(d->version[id]); + + fprintf(out, "\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%u\"\n", + row[0], row[1], row[2], row[3], row[4], row[6], row[7], version); + } else { + /*fprintf(out, "\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\"\n", + row[0], row[1], row[2], row[3], row[4], row[6], row[7]);*/ + } + + char *tags_it = row[5], *k, *v; + while (read_node_tags(&tags_it, &k, &v)) { + if (hist) { + fprintf(out_tags, "\"%s\",\"%u\",", row[0], version); + } else { + fprintf(out_tags, "\"%s\",", row[0]); + } + + write_csv_col(out_tags, k, ','); + write_csv_col(out_tags, v, '\n'); + } + } + if (mysql_errno(d->mysql)) exit_mysql_err(d->mysql); + + mysql_free_result(res); +} + +static size_t select_size(MYSQL *mysql, const char *q) { + MYSQL_RES *res; + MYSQL_ROW row; + size_t ret; + + if (mysql_query(mysql, q)) + exit_mysql_err(mysql); + + res = mysql_store_result(mysql); + if (!res) exit_mysql_err(mysql); + + row = mysql_fetch_row(res); + if (!row) exit_mysql_err(mysql); + + if (row[0]) { + ret = strtoul(row[0], NULL, 10); + } else { + ret = 0; + } + + mysql_free_result(res); + + return ret; +} + +static MYSQL *connect_to_mysql(char **argv) { + MYSQL *mysql = mysql_init(NULL); + if (!mysql) exit_mysql_err(mysql); + + if (!mysql_real_connect(mysql, argv[1], argv[2], argv[3], argv[4], + argv[5][0] ? atoi(argv[5]) : 0, argv[6][0] ? argv[6] : NULL, 0)) + exit_mysql_err(mysql); + + if (mysql_set_character_set(mysql, "utf8")) + exit_mysql_err(mysql); + + return mysql; +} + +static void open_file(FILE **f, char *fn) { + *f = fopen(fn, "w+"); + if (!*f) { + perror("fopen"); + exit(EXIT_FAILURE); + } +} + +int main(int argc, char **argv) { + size_t prefix_len; + FILE *current_nodes, *current_node_tags, *nodes, *node_tags; + char *tempfn; + struct data data, *d = &data; + + if (argc != 8) { + printf("Usage: 013_populate_node_tags_and_remove_helper host user passwd database port socket prefix\n"); + exit(EXIT_FAILURE); + } + + d->mysql = connect_to_mysql(argv); + + d->version_size = 1 + select_size(d->mysql, "SELECT max(id) FROM current_nodes"); + d->version = (uint16_t *) malloc(sizeof(uint16_t) * d->version_size); + if (!d->version) { + perror("malloc"); + abort(); + exit(EXIT_FAILURE); + } + memset(d->version, 0, sizeof(uint16_t) * d->version_size); + + prefix_len = strlen(argv[7]); + tempfn = (char *) malloc(prefix_len + 32); + strcpy(tempfn, argv[7]); + + strcpy(tempfn + prefix_len, "current_nodes"); + open_file(¤t_nodes, tempfn); + + strcpy(tempfn + prefix_len, "current_node_tags"); + open_file(¤t_node_tags, tempfn); + + strcpy(tempfn + prefix_len, "nodes"); + open_file(&nodes, tempfn); + + strcpy(tempfn + prefix_len, "node_tags"); + open_file(&node_tags, tempfn); + + free(tempfn); + + proc_nodes(d, "nodes", nodes, node_tags, 1); + proc_nodes(d, "current_nodes", current_nodes, current_node_tags, 0); + + free(d->version); + + mysql_close(d->mysql); + + fclose(current_nodes); + fclose(current_node_tags); + fclose(nodes); + fclose(node_tags); + + exit(EXIT_SUCCESS); +} diff --combined db/migrate/018_move_to_innodb.rb index c551b0ef8,000000000..c551b0ef8 mode 100644,000000..100644 --- a/db/migrate/018_move_to_innodb.rb +++ b/db/migrate/018_move_to_innodb.rb @@@ -1,30 -1,0 +1,30 @@@ +class MoveToInnodb < ActiveRecord::Migration + @@conv_tables = ['nodes', 'ways', 'way_tags', 'way_nodes', + 'current_way_tags', 'relation_members', + 'relations', 'relation_tags', 'current_relation_tags'] + + @@ver_tbl = ['nodes', 'ways', 'relations'] + + def self.up + execute 'DROP INDEX current_way_tags_v_idx ON current_way_tags' + execute 'DROP INDEX current_relation_tags_v_idx ON current_relation_tags' + + @@ver_tbl.each { |tbl| + change_column tbl, "version", :bigint, :limit => 20, :null => false + } + + @@conv_tables.each { |tbl| + execute "ALTER TABLE #{tbl} ENGINE = InnoDB" + } + + @@ver_tbl.each { |tbl| + add_column "current_#{tbl}", "version", :bigint, :limit => 20, :null => false + execute "UPDATE current_#{tbl} SET version = " + + "(SELECT max(version) FROM #{tbl} WHERE #{tbl}.id = current_#{tbl}.id)" + } + end + + def self.down + raise IrreversibleMigration.new + end +end diff --combined db/migrate/019_key_constraints.rb index 40f98be02,000000000..40f98be02 mode 100644,000000..100644 --- a/db/migrate/019_key_constraints.rb +++ b/db/migrate/019_key_constraints.rb @@@ -1,50 -1,0 +1,50 @@@ +class KeyConstraints < ActiveRecord::Migration + def self.up + # Primary keys + add_primary_key :current_node_tags, [:id, :k] + add_primary_key :current_way_tags, [:id, :k] + add_primary_key :current_relation_tags, [:id, :k] + + add_primary_key :node_tags, [:id, :version, :k] + add_primary_key :way_tags, [:id, :version, :k] + add_primary_key :relation_tags, [:id, :version, :k] + + add_primary_key :nodes, [:id, :version] + + # Remove indexes superseded by primary keys + remove_index :current_way_tags, :name => :current_way_tags_id_idx + remove_index :current_relation_tags, :name => :current_relation_tags_id_idx + + remove_index :way_tags, :name => :way_tags_id_version_idx + remove_index :relation_tags, :name => :relation_tags_id_version_idx + + remove_index :nodes, :name => :nodes_uid_idx + + # Foreign keys (between ways, way_tags, way_nodes, etc.) + add_foreign_key :current_node_tags, [:id], :current_nodes + add_foreign_key :node_tags, [:id, :version], :nodes + + add_foreign_key :current_way_tags, [:id], :current_ways + add_foreign_key :current_way_nodes, [:id], :current_ways + add_foreign_key :way_tags, [:id, :version], :ways + add_foreign_key :way_nodes, [:id, :version], :ways + + add_foreign_key :current_relation_tags, [:id], :current_relations + add_foreign_key :current_relation_members, [:id], :current_relations + add_foreign_key :relation_tags, [:id, :version], :relations + add_foreign_key :relation_members, [:id, :version], :relations + + # Foreign keys (between different types of primitives) + add_foreign_key :current_way_nodes, [:node_id], :current_nodes, [:id] + + # FIXME: We don't have foreign keys for relation members since the id + # might point to a different table depending on the `type' column. + # We'd probably need different current_relation_member_nodes, + # current_relation_member_ways and current_relation_member_relations + # tables for this to work cleanly. + end + + def self.down + raise IrreversibleMigration.new + end +end diff --combined db/migrate/020_add_changesets.rb index 40455ec68,000000000..40455ec68 mode 100644,000000..100644 --- a/db/migrate/020_add_changesets.rb +++ b/db/migrate/020_add_changesets.rb @@@ -1,32 -1,0 +1,32 @@@ +class AddChangesets < ActiveRecord::Migration + def self.up + create_table "changesets", innodb_table do |t| + t.column "id", :bigint, :limit => 20, :null => false + t.column "user_id", :bigint, :limit => 20, :null => false + t.column "created_at", :datetime, :null => false + t.column "open", :boolean, :null => false, :default => true + t.column "min_lat", :integer, :null => true + t.column "max_lat", :integer, :null => true + t.column "min_lon", :integer, :null => true + t.column "max_lon", :integer, :null => true + end + + add_primary_key "changesets", ["id"] + # FIXME add indexes? + + change_column "changesets", "id", :bigint, :limit => 20, :null => false, :options => "AUTO_INCREMENT" + + create_table "changeset_tags", innodb_table do |t| + t.column "id", :bigint, :limit => 64, :null => false + t.column "k", :string, :default => "", :null => false + t.column "v", :string, :default => "", :null => false + end + + add_index "changeset_tags", ["id"], :name => "changeset_tags_id_idx" + end + + def self.down + drop_table "changesets" + drop_table "changeset_tags" + end +end diff --combined lib/geo_record.rb index 286ce69e1,f1a923c42..2740eab0c --- a/lib/geo_record.rb +++ b/lib/geo_record.rb @@@ -1,9 -1,4 +1,9 @@@ module GeoRecord + # This scaling factor is used to convert between the float lat/lon that is + # returned by the API, and the integer lat/lon equivalent that is stored in + # the database. + SCALE = 10000000 + def self.included(base) base.extend(ClassMethods) end @@@ -25,32 -20,23 +25,23 @@@ end def lat=(l) - self.latitude = (l * 10000000).round + self.latitude = (l * SCALE).round end def lon=(l) - self.longitude = (l * 10000000).round + self.longitude = (l * SCALE).round end # Return WGS84 latitude def lat - return self.latitude.to_f / 10000000 + return self.latitude.to_f / SCALE end # Return WGS84 longitude def lon - return self.longitude.to_f / 10000000 + return self.longitude.to_f / SCALE end - # Potlatch projections - def lon_potlatch(baselong,masterscale) - (self.lon-baselong)*masterscale - end - - def lat_potlatch(basey,masterscale) - -(lat2y(self.lat)-basey)*masterscale - end - private def lat2y(a) diff --combined lib/osm.rb index c038ab2d5,9c271607d..a64aa8c48 --- a/lib/osm.rb +++ b/lib/osm.rb @@@ -10,9 -10,6 +10,9 @@@ module OS # The base class for API Errors. class APIError < RuntimeError + def render_opts + { :text => "", :status => :internal_server_error } + end end # Raised when an API object is not found. @@@ -21,30 -18,10 +21,30 @@@ # Raised when a precondition to an API action fails sanity check. class APIPreconditionFailedError < APIError + def render_opts + { :text => "", :status => :precondition_failed } + end end # Raised when to delete an already-deleted object. class APIAlreadyDeletedError < APIError + def render_opts + { :text => "", :status => :gone } + end + end + + # Raised when the provided version is not equal to the latest in the db. + class APIVersionMismatchError < APIError + def initialize(provided, latest) + @provided, @latest = provided, latest + end + + attr_reader :provided, :latest + + def render_opts + { :text => "Version mismatch: Provided " + provided.to_s + + ", server had: " + latest.to_s, :status => :bad_request } + end end # Helper methods for going to/from mercator and lat/lng. @@@ -93,199 -70,6 +93,6 @@@ end end - - # This piece of magic reads a GPX with SAX and spits out - # lat/lng and stuff - # - # This would print every latitude value: - # - # gpx = OSM::GPXImporter.new('somefile.gpx') - # gpx.points {|p| puts p['latitude']} - class GPXImporter - # FIXME swap REXML for libXML - attr_reader :possible_points - attr_reader :actual_points - attr_reader :tracksegs - - def initialize(file) - @file = file - end - - def points - @possible_points = 0 - @actual_points = 0 - @tracksegs = 0 - - lat = -1 - lon = -1 - ele = -1 - date = DateTime.now(); - gotlatlon = false - gotele = false - gotdate = false - - @file.rewind - - parser = REXML::Parsers::SAX2Parser.new(@file) - - parser.listen( :start_element, %w{ trkpt }) do |uri,localname,qname,attributes| - lat = attributes['lat'].to_f - lon = attributes['lon'].to_f - gotlatlon = true - gotele = false - gotdate = false - @possible_points += 1 - end - - parser.listen( :characters, %w{ ele } ) do |text| - ele = text - gotele = true - end - - parser.listen( :characters, %w{ time } ) do |text| - if text && text != '' - begin - date = DateTime.parse(text) - gotdate = true - rescue - end - end - end - - parser.listen( :end_element, %w{ trkseg } ) do |uri, localname, qname| - @tracksegs += 1 - end - - parser.listen( :end_element, %w{ trkpt } ) do |uri,localname,qname| - if gotlatlon && gotdate - ele = '0' unless gotele - if lat < 90 && lat > -90 && lon > -180 && lon < 180 - @actual_points += 1 - yield Hash['latitude' => lat, 'longitude' => lon, 'timestamp' => date, 'altitude' => ele, 'segment' => @tracksegs] - end - end - gotlatlon = false - gotele = false - gotdate = false - end - - parser.parse - end - - def get_picture(min_lat, min_lon, max_lat, max_lon, num_points) - #puts "getting picfor bbox #{min_lat},#{min_lon} - #{max_lat},#{max_lon}" - frames = 10 - width = 250 - height = 250 - proj = OSM::Mercator.new(min_lat, min_lon, max_lat, max_lon, width, height) - - linegc = Magick::Draw.new - linegc.stroke_linejoin('miter') - linegc.stroke_width(1) - linegc.stroke('#BBBBBB') - linegc.fill('#BBBBBB') - - highlightgc = Magick::Draw.new - highlightgc.stroke_linejoin('miter') - highlightgc.stroke_width(3) - highlightgc.stroke('#000000') - highlightgc.fill('#000000') - - images = [] - - frames.times do - image = Magick::Image.new(width, height) do |image| - image.background_color = 'white' - image.format = 'GIF' - end - - images << image - end - - oldpx = 0.0 - oldpy = 0.0 - - first = true - - m = 0 - mm = 0 - points do |p| - px = proj.x(p['longitude']) - py = proj.y(p['latitude']) - - if m > 0 - frames.times do |n| - if n == mm - gc = highlightgc.dup - else - gc = linegc.dup - end - - gc.line(px, py, oldpx, oldpy) - - gc.draw(images[n]) - end - end - - m += 1 - if m > num_points.to_f / frames.to_f * (mm+1) - mm += 1 - end - - oldpy = py - oldpx = px - end - - il = Magick::ImageList.new - - images.each do |f| - il << f - end - - il.delay = 50 - il.format = 'GIF' - - return il.to_blob - end - - def get_icon(min_lat, min_lon, max_lat, max_lon) - #puts "getting icon for bbox #{min_lat},#{min_lon} - #{max_lat},#{max_lon}" - width = 50 - height = 50 - proj = OSM::Mercator.new(min_lat, min_lon, max_lat, max_lon, width, height) - - gc = Magick::Draw.new - gc.stroke_linejoin('miter') - gc.stroke_width(1) - gc.stroke('#000000') - gc.fill('#000000') - - image = Magick::Image.new(width, height) do |image| - image.background_color = 'white' - image.format = 'GIF' - end - - oldpx = 0.0 - oldpy = 0.0 - - first = true - - points do |p| - px = proj.x(p['longitude']) - py = proj.y(p['latitude']) - - gc.dup.line(px, py, oldpx, oldpy).draw(image) unless first - - first = false - oldpy = py - oldpx = px - end - - return image.to_blob - end - - end - class GreatCircle include Math