From dc2a95903722644f6109244cec9a2d1e195fef0e Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Thu, 23 Oct 2008 15:14:17 +0000 Subject: [PATCH 1/1] Added tests for changeset upload code. Refactored diff reading code and put it into /lib. Changed the route of a changeset upload to explicitly refer to the changeset it applies to (i.e: resource). --- app/controllers/changeset_controller.rb | 114 ++----- app/models/node.rb | 16 +- app/models/relation.rb | 18 + app/models/way.rb | 17 + config/routes.rb | 3 +- lib/diff_reader.rb | 167 ++++++++++ lib/osm.rb | 26 ++ test/functional/changeset_controller_test.rb | 329 ++++++++++++++++++- 8 files changed, 595 insertions(+), 95 deletions(-) create mode 100644 lib/diff_reader.rb diff --git a/app/controllers/changeset_controller.rb b/app/controllers/changeset_controller.rb index 8151d3a67..a24f61e28 100644 --- a/app/controllers/changeset_controller.rb +++ b/app/controllers/changeset_controller.rb @@ -2,6 +2,7 @@ class ChangesetController < ApplicationController require 'xml/libxml' + require 'diff_reader' before_filter :authorize, :only => [:create, :update, :delete, :upload] before_filter :check_write_availability, :only => [:create, :update, :delete, :upload] @@ -70,102 +71,37 @@ class ChangesetController < ApplicationController end end + ## + # Upload a diff in a single transaction. + # + # This means that each change within the diff must succeed, i.e: that + # each version number mentioned is still current. Otherwise the entire + # transaction *must* be rolled back. + # + # Furthermore, each element in the diff can only reference the current + # changeset. + # + # Returns: ??? the new document? updated diffs? def upload - unless request.put? + # only allow POST requests, as the upload method is most definitely + # not idempotent, as several uploads with placeholder IDs will have + # different side-effects. + # see http://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html#sec9.1.2 + unless request.post? render :nothing => true, :status => :method_not_allowed return end - p = XML::Reader.new request.raw_post - - node_ids, way_ids, rel_ids = {}, {}, {} - ids = {"node"=>node_ids, "way"=>way_ids, "relation"=>rel_ids} - - models = {"node"=>Node, "way"=>Way, "relation"=>Relation} - - # FIXME shouldn't this be done through the - # res = OSM::API.new.get_xml_doc - # as everything else is? - res = XML::Document.new - res.encoding = 'UTF-8' - root = XML::Node.new 'osm' - root['version'] = API_VERSION - root['creator'] = 'OpenStreetMap.org' - res.root = root - - root << XML::Node.new_comment(" Warning: this is a 0.6 result document, " + - "not a normal OSM file. ") - + changeset = Changeset.find(params[:id]) + + diff_reader = DiffReader.new(request.raw_post, changeset) Changeset.transaction do - while p.read == 1 - break if p.node_type == 15 # end element - next unless p.node_type == 1 # element - - case p.name - when 'create': - while p.read == 1 - break if p.node_type == 15 # end element - next unless p.node_type == 1 # element - - model = models[p.name] - next if model.nil? - - elem = XML::Node.new p.name - nd = p.expand; p.next - osm = model.from_xml_node(nd, true) - elem['old_id'] = nd['id'] - - case nd.name - when 'way': - fix_way(osm, node_ids) - raise OSM::APIPreconditionFailedError.new if !osm.preconditions_ok? - when 'relation': - fix_rel(osm, ids) - raise OSM::APIPreconditionFailedError.new if !osm.preconditions_ok? - end - - create_prim ids[nd.name], osm, nd - elem['new_id'] = osm.id.to_s - elem['new_version'] = osm.version.to_s - root << elem - end - when 'modify': - while p.read == 1 - break if p.node_type == 15 # end element - next unless p.node_type == 1 # element - - model = models[p.name] - next if model.nil? - - elem = XML::Node.new p.name - new_osm = model.from_xml_node(p.expand); p.next - osm = model.find(new_osm.id) - osm.update_from new_osm, @user - elem['old_id'] = elem['new_id'] = osm.id.to_s - elem['new_version'] = osm.version.to_s - root << elem - end - when 'delete': - while p.read == 1 - break if p.node_type == 15 # end element - next unless p.node_type == 1 # element - - model = models[p.name] - next if model.nil? - - elem = XML::Node.new p.name - osm = model.find(p.expand['id']); p.next - osm.delete_with_history(@user) - elem['old_id'] = elem['new_id'] = osm.id.to_s - elem['new_version'] = osm.version.to_s - root << elem - end - end - end + result = diff_reader.commit + render :text => result.to_s, :content_type => "text/xml" end - - render :text => res.to_s, :content_type => "text/xml" - + + rescue ActiveRecord::RecordNotFound + render :nothing => true, :status => :not_found rescue OSM::APIError => ex render ex.render_opts end diff --git a/app/models/node.rb b/app/models/node.rb index c2a61906b..67efeca2c 100644 --- a/app/models/node.rb +++ b/app/models/node.rb @@ -75,20 +75,25 @@ class Node < ActiveRecord::Base def self.from_xml_node(pt, create=false) node = Node.new - node.version = pt['version'].to_i node.lat = pt['lat'].to_f node.lon = pt['lon'].to_f node.changeset_id = pt['changeset'].to_i return nil unless node.in_world? + # version must be present unless creating + return nil unless create or not pt['version'].nil? + node.version = pt['version'].to_i + unless create if pt['id'] != '0' node.id = pt['id'].to_i end end - node.visible = pt['visible'] and pt['visible'] == 'true' + # visible if it says it is, or as the default if the attribute + # is missing. + node.visible = pt['visible'].nil? or pt['visible'] == 'true' if create node.timestamp = Time.now @@ -235,4 +240,11 @@ class Node < ActiveRecord::Base @tags[k] = v end + ## + # dummy method to make the interfaces of node, way and relation + # more consistent. + def fix_placeholders!(id_map) + # nodes don't refer to anything, so there is nothing to do here + end + end diff --git a/app/models/relation.rb b/app/models/relation.rb index db4dd52a6..81f178997 100644 --- a/app/models/relation.rb +++ b/app/models/relation.rb @@ -320,4 +320,22 @@ class Relation < ActiveRecord::Base def tags_as_hash return self.tags end + + ## + # if any members are referenced by placeholder IDs (i.e: negative) then + # this calling this method will fix them using the map from placeholders + # to IDs +id_map+. + def fix_placeholders!(id_map) + self.members.map! do |type, id, role| + old_id = id.to_i + if old_id < 0 + new_id = id_map[type.to_sym][old_id] + raise "invalid placeholder" if new_id.nil? + [type, new_id, role] + else + [type, id, role] + end + end + end + end diff --git a/app/models/way.rb b/app/models/way.rb index 1304d8a18..b413ccb28 100644 --- a/app/models/way.rb +++ b/app/models/way.rb @@ -299,4 +299,21 @@ class Way < ActiveRecord::Base def tags_as_hash return self.tags end + + ## + # if any referenced nodes are placeholder IDs (i.e: are negative) then + # this calling this method will fix them using the map from placeholders + # to IDs +id_map+. + def fix_placeholders!(id_map) + self.nds.map! do |node_id| + if node_id < 0 + new_id = id_map[:node][node_id] + raise "invalid placeholder for #{node_id.inspect}: #{new_id.inspect}" if new_id.nil? + new_id + else + node_id + end + end + end + end diff --git a/config/routes.rb b/config/routes.rb index e49cdf74e..39e2a1e74 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -4,7 +4,7 @@ ActionController::Routing::Routes.draw do |map| map.connect "api/capabilities", :controller => 'api', :action => 'capabilities' map.connect "api/#{API_VERSION}/changeset/create", :controller => 'changeset', :action => 'create' - map.connect "api/#{API_VERSION}/changeset/upload", :controller => 'changeset', :action => 'upload' + map.connect "api/#{API_VERSION}/changeset/:id/upload", :controller => 'changeset', :action => 'upload', :id => /\d+/ map.connect "api/#{API_VERSION}/changeset/:id", :controller => 'changeset', :action => 'read', :id => /\d+/ map.connect "api/#{API_VERSION}/changeset/:id/close", :controller => 'changeset', :action => 'close', :id =>/\d+/ @@ -22,6 +22,7 @@ ActionController::Routing::Routes.draw do |map| map.connect "api/#{API_VERSION}/way/:id/history", :controller => 'old_way', :action => 'history', :id => /\d+/ map.connect "api/#{API_VERSION}/way/:id/full", :controller => 'way', :action => 'full', :id => /\d+/ map.connect "api/#{API_VERSION}/way/:id/relations", :controller => 'relation', :action => 'relations_for_way', :id => /\d+/ + map.connect "api/#{API_VERSION}/way/:id/:version", :controller => 'old_way', :action => 'version', :id => /\d+/, :version => /\d+/ map.connect "api/#{API_VERSION}/way/:id", :controller => 'way', :action => 'read', :id => /\d+/, :conditions => { :method => :get } map.connect "api/#{API_VERSION}/way/:id", :controller => 'way', :action => 'update', :id => /\d+/, :conditions => { :method => :put } map.connect "api/#{API_VERSION}/way/:id", :controller => 'way', :action => 'delete', :id => /\d+/, :conditions => { :method => :delete } diff --git a/lib/diff_reader.rb b/lib/diff_reader.rb new file mode 100644 index 000000000..eca6d438c --- /dev/null +++ b/lib/diff_reader.rb @@ -0,0 +1,167 @@ +## +# DiffReader reads OSM diffs and applies them to the database. +# +# Uses the streaming LibXML "Reader" interface to cut down on memory +# usage, so hopefully we can process fairly large diffs. +class DiffReader + include ConsistencyValidations + + # maps each element type to the model class which handles it + MODELS = { + "node" => Node, + "way" => Way, + "relation" => Relation + } + + ## + # Construct a diff reader by giving it a bunch of XML +data+ to parse + # in OsmChange format. All diffs must be limited to a single changeset + # given in +changeset+. + def initialize(data, changeset) + @reader = XML::Reader.new data + @changeset = changeset + end + + ## + # An element-block mapping for using the LibXML reader interface. + # + # Since a lot of LibXML reader usage is boilerplate iteration through + # elements, it would be better to DRY and do this in a block. This + # could also help with error handling...? + def with_element + # skip the first element, which is our opening element of the block + @reader.read + # loop over all elements. + # NOTE: XML::Reader#read returns 0 for EOF and -1 for error. + while @reader.read == 1 + break if @reader.node_type == 15 # end element + next unless @reader.node_type == 1 # element + yield @reader.name + end + end + + ## + # An element-block mapping for using the LibXML reader interface. + # + # Since a lot of LibXML reader usage is boilerplate iteration through + # elements, it would be better to DRY and do this in a block. This + # could also help with error handling...? + def with_model + with_element do |model_name| + model = MODELS[model_name] + raise "Unexpected element type #{model_name}, " + + "expected node, way, relation." if model.nil? + yield model, @reader.expand + @reader.next + end + end + + ## + # Checks a few invariants. Others are checked in the model methods + # such as save_ and delete_with_history. + def check(model, xml, new) + raise OSM::APIBadXMLError.new(model, xml) if new.nil? + unless new.changeset_id == @changeset.id + raise OSM::APIChangesetMismatchError.new(new.changeset_id, @changeset.id) + end + end + + ## + # Consume the XML diff and try to commit it to the database. This code + # is *not* transactional, so code which calls it should ensure that the + # appropriate transaction block is in place. + # + # On a failure to meet preconditions (e.g: optimistic locking fails) + # an exception subclassing OSM::APIError will be thrown. + def commit + + node_ids, way_ids, rel_ids = {}, {}, {} + ids = { :node => node_ids, :way => way_ids, :relation => rel_ids} + + result = OSM::API.new.get_xml_doc + + # loop at the top level, within the element (although we + # don't actually check this...) + with_element do |action_name| + if action_name == 'create' + # create a new element. this code is agnostic of the element type + # because all the elements support the methods that we're using. + with_model do |model, xml| + new = model.from_xml_node(xml, true) + check(model, xml, new) + + # when this element is saved it will get a new ID, so we save it + # to produce the mapping which is sent to other elements. + placeholder_id = xml['id'].to_i + raise OSM::APIBadXMLError.new(model, xml) if placeholder_id.nil? + + # some elements may have placeholders for other elements in the + # diff, so we must fix these before saving the element. + new.fix_placeholders!(ids) + + # set the initial version to zero and save (which increments it) + new.version = 0 + new.save_with_history! + + # save placeholder => allocated ID map + ids[model.to_s.downcase.to_sym][placeholder_id] = new.id + + # add the result to the document we're building for return. + xml_result = XML::Node.new model.to_s.downcase + xml_result["old_id"] = placeholder_id.to_s + xml_result["new_id"] = new.id.to_s + xml_result["new_version"] = new.version.to_s + result.root << xml_result + end + + elsif action_name == 'modify' + # modify an existing element. again, this code doesn't directly deal + # with types, but uses duck typing to handle them transparently. + with_model do |model, xml| + # get the new element from the XML payload + new = model.from_xml_node(xml, false) + check(model, xml, new) + + # and the old one from the database + old = model.find(new.id) + + new.fix_placeholders!(ids) + old.update_from(new, @changeset.user) + + xml_result = XML::Node.new model.to_s.downcase + xml_result["old_id"] = old.id.to_s + xml_result["new_id"] = new.id.to_s + xml_result["new_version"] = new.version.to_s + result.root << xml_result + end + + elsif action_name == 'delete' + # delete action. this takes a payload in API 0.6, so we need to do + # most of the same checks that are done for the modify. + with_model do |model, xml| + new = model.from_xml_node(xml, false) + check(model, xml, new) + + old = model.find(new.id) + + # can a delete have placeholders under any circumstances? + # if a way is modified, then deleted is that a valid diff? + new.fix_placeholders!(ids) + old.delete_with_history!(new, @changeset.user) + + xml_result = XML::Node.new model.to_s.downcase + xml_result["old_id"] = old.id.to_s + result.root << xml_result + end + + else + # no other actions to choose from, so it must be the users fault! + raise "Unknown action #{action_name}, choices are create, modify, delete." + end + end + + # return the XML document to be rendered back to the client + return result + end + +end diff --git a/lib/osm.rb b/lib/osm.rb index e9d3c9464..246fedf54 100644 --- a/lib/osm.rb +++ b/lib/osm.rb @@ -54,6 +54,32 @@ module OSM end end + # Raised when a diff is uploaded containing many changeset IDs which don't match + # the changeset ID that the diff was uploaded to. + class APIChangesetMismatchError < APIError + def initialize(provided, allowed) + @provided, @allowed = provided, allowed + end + + def render_opts + { :text => "Changeset mismatch: Provided #{@provided} but only " + + "#{@allowed} is allowed.", :status => :conflict } + end + end + + # Raised when bad XML is encountered which stops things parsing as + # they should. + class APIBadXMLError < APIError + def initialize(model, xml) + @model, @xml = model, xml + end + + def render_opts + { :text => "Cannot parse valid #{@model} from xml string #{@xml}", + :status => :bad_request } + end + end + # Raised when the provided version is not equal to the latest in the db. class APIVersionMismatchError < APIError def initialize(provided, latest) diff --git a/test/functional/changeset_controller_test.rb b/test/functional/changeset_controller_test.rb index 1e050a71e..946d139d8 100644 --- a/test/functional/changeset_controller_test.rb +++ b/test/functional/changeset_controller_test.rb @@ -44,6 +44,7 @@ class ChangesetController; def rescue_action(e) raise e end; end basic_authorization "test@openstreetmap.org", "test" content "" put :create + assert_response :bad_request, "creating a invalid changeset should fail" end def test_read @@ -53,9 +54,331 @@ class ChangesetController; def rescue_action(e) raise e end; end def test_close end - - def test_upload + + ## + # upload something simple, but valid and check that it can + # be read back ok. + def test_upload_simple_valid + basic_authorization "test@openstreetmap.org", "test" + + # simple diff to change a node, way and relation by removing + # their tags + diff = < + + + + + + + + + + + + + + +EOF + + # upload it + content diff + post :upload, :id => 1 + assert_response :success, + "can't upload a simple valid diff to changeset: #{@response.body}" + + # check that the changes made it into the database + assert_equal 0, Node.find(1).tags.size, "node 1 should now have no tags" + assert_equal 0, Way.find(1).tags.size, "way 1 should now have no tags" + assert_equal 0, Relation.find(1).tags.size, "relation 1 should now have no tags" + end + ## + # upload something which creates new objects using placeholders + def test_upload_create_valid + basic_authorization "test@openstreetmap.org", "test" + + # simple diff to create a node way and relation using placeholders + diff = < + + + + + + + + + + + + + + + + + +EOF + + # upload it + content diff + post :upload, :id => 1 + assert_response :success, + "can't upload a simple valid creation to changeset: #{@response.body}" + + # check the returned payload + assert_select "osm[version=#{API_VERSION}][generator=\"OpenStreetMap server\"]", 1 + assert_select "osm>node", 1 + assert_select "osm>way", 1 + assert_select "osm>relation", 1 + + # inspect the response to find out what the new element IDs are + doc = XML::Parser.string(@response.body).parse + new_node_id = doc.find("//osm/node").first["new_id"].to_i + new_way_id = doc.find("//osm/way").first["new_id"].to_i + new_rel_id = doc.find("//osm/relation").first["new_id"].to_i + + # check the old IDs are all present and negative one + assert_equal -1, doc.find("//osm/node").first["old_id"].to_i + assert_equal -1, doc.find("//osm/way").first["old_id"].to_i + assert_equal -1, doc.find("//osm/relation").first["old_id"].to_i + + # check the versions are present and equal one + assert_equal 1, doc.find("//osm/node").first["new_version"].to_i + assert_equal 1, doc.find("//osm/way").first["new_version"].to_i + assert_equal 1, doc.find("//osm/relation").first["new_version"].to_i + + # check that the changes made it into the database + assert_equal 2, Node.find(new_node_id).tags.size, "new node should have two tags" + assert_equal 0, Way.find(new_way_id).tags.size, "new way should have no tags" + assert_equal 0, Relation.find(new_rel_id).tags.size, "new relation should have no tags" end - + + ## + # test a complex delete where we delete elements which rely on eachother + # in the same transaction. + def test_upload_delete + basic_authorization "test@openstreetmap.org", "test" + + diff = XML::Document.new + diff.root = XML::Node.new "osmChange" + delete = XML::Node.new "delete" + diff.root << delete + delete << current_relations(:visible_relation).to_xml_node + delete << current_relations(:used_relation).to_xml_node + delete << current_ways(:used_way).to_xml_node + delete << current_nodes(:node_used_by_relationship).to_xml_node + + # upload it + content diff + post :upload, :id => 1 + assert_response :success, + "can't upload a deletion diff to changeset: #{@response.body}" + + # check that everything was deleted + assert_equal false, Node.find(current_nodes(:node_used_by_relationship).id).visible + assert_equal false, Way.find(current_ways(:used_way).id).visible + assert_equal false, Relation.find(current_relations(:visible_relation).id).visible + assert_equal false, Relation.find(current_relations(:used_relation).id).visible + end + + ## + # test that deleting stuff in a transaction doesn't bypass the checks + # to ensure that used elements are not deleted. + def test_upload_delete_invalid + basic_authorization "test@openstreetmap.org", "test" + + diff = XML::Document.new + diff.root = XML::Node.new "osmChange" + delete = XML::Node.new "delete" + diff.root << delete + delete << current_relations(:visible_relation).to_xml_node + delete << current_ways(:used_way).to_xml_node + delete << current_nodes(:node_used_by_relationship).to_xml_node + + # upload it + content diff + post :upload, :id => 1 + assert_response :precondition_failed, + "shouldn't be able to upload a invalid deletion diff: #{@response.body}" + + # check that nothing was, in fact, deleted + assert_equal true, Node.find(current_nodes(:node_used_by_relationship).id).visible + assert_equal true, Way.find(current_ways(:used_way).id).visible + assert_equal true, Relation.find(current_relations(:visible_relation).id).visible + end + + ## + # upload something which creates new objects and inserts them into + # existing containers using placeholders. + def test_upload_complex + basic_authorization "test@openstreetmap.org", "test" + + # simple diff to create a node way and relation using placeholders + diff = < + + + + + + + + + + + + + + + + + + +EOF + + # upload it + content diff + post :upload, :id => 1 + assert_response :success, + "can't upload a complex diff to changeset: #{@response.body}" + + # check the returned payload + assert_select "osm[version=#{API_VERSION}][generator=\"OpenStreetMap server\"]", 1 + assert_select "osm>node", 1 + assert_select "osm>way", 1 + assert_select "osm>relation", 1 + + # inspect the response to find out what the new element IDs are + doc = XML::Parser.string(@response.body).parse + new_node_id = doc.find("//osm/node").first["new_id"].to_i + + # check that the changes made it into the database + assert_equal 2, Node.find(new_node_id).tags.size, "new node should have two tags" + assert_equal [new_node_id, 3], Way.find(1).nds, "way nodes should match" + Relation.find(1).members.each do |type,id,role| + if type == 'node' + assert_equal new_node_id, id, "relation should contain new node" + end + end + end + + ## + # create a diff which references several changesets, which should cause + # a rollback and none of the diff gets committed + def test_upload_invalid_changesets + basic_authorization "test@openstreetmap.org", "test" + + # simple diff to create a node way and relation using placeholders + diff = < + + + + + + + + + + + + + + + + + + + + +EOF + # cache the objects before uploading them + node = current_nodes(:visible_node) + way = current_ways(:visible_way) + rel = current_relations(:visible_relation) + + # upload it + content diff + post :upload, :id => 1 + assert_response :conflict, + "uploading a diff with multiple changsets should have failed" + + # check that objects are unmodified + assert_nodes_are_equal(node, Node.find(1)) + assert_ways_are_equal(way, Way.find(1)) + end + + ## + # upload multiple versions of the same element in the same diff. + def test_upload_multiple_valid + basic_authorization "test@openstreetmap.org", "test" + + # change the location of a node multiple times, each time referencing + # the last version. doesn't this depend on version numbers being + # sequential? + diff = < + + + + + + + + + + + +EOF + + # upload it + content diff + post :upload, :id => 1 + assert_response :success, + "can't upload multiple versions of an element in a diff: #{@response.body}" + end + + ## + # upload multiple versions of the same element in the same diff, but + # keep the version numbers the same. + def test_upload_multiple_duplicate + basic_authorization "test@openstreetmap.org", "test" + + diff = < + + + + + +EOF + + # upload it + content diff + post :upload, :id => 1 + assert_response :conflict, + "shouldn't be able to upload the same element twice in a diff: #{@response.body}" + end + + ## + # try to upload some elements without specifying the version + def test_upload_missing_version + basic_authorization "test@openstreetmap.org", "test" + + diff = < + + + + +EOF + + # upload it + content diff + post :upload, :id => 1 + assert_response :bad_request, + "shouldn't be able to upload an element without version: #{@response.body}" + end + end -- 2.39.5