Run .shp importer in chunks to prevent timeouts
authorRichard Fairhurst <richard@systemeD.net>
Thu, 19 Jul 2012 12:07:04 +0000 (13:07 +0100)
committerRichard Fairhurst <richard@systemeD.net>
Thu, 19 Jul 2012 12:07:04 +0000 (13:07 +0100)
net/systemeD/potlatch2/utils/GpxImporter.as
net/systemeD/potlatch2/utils/Importer.as
net/systemeD/potlatch2/utils/KmlImporter.as
net/systemeD/potlatch2/utils/OsmImporter.as
net/systemeD/potlatch2/utils/ShpImporter.as

index 6342b82..ae0de92 100644 (file)
@@ -1,9 +1,7 @@
 package net.systemeD.potlatch2.utils {
 
        import net.systemeD.halcyon.Map;
-       import net.systemeD.halcyon.connection.Connection;
-       import net.systemeD.halcyon.connection.Node;
-       import net.systemeD.halcyon.connection.Way;
+       import net.systemeD.halcyon.connection.*;
        import net.systemeD.potlatch2.tools.Simplify;
 
     /**
@@ -16,7 +14,9 @@ package net.systemeD.potlatch2.utils {
                        super(connection,map,callback,simplify,options);
                }
 
-               override protected function doImport(push:Function): void {
+               override protected function doImport(): void {
+                       var action:CompositeUndoableAction = new CompositeUndoableAction("Import GPX "+connection.name);
+
                        var file:XML = new XML(files[0]);
                        for each (var ns:Namespace in file.namespaceDeclarations()) {
                                if (ns.uri.match(/^http:\/\/www\.topografix\.com\/GPX\/1\/[01]$/)) {
@@ -28,10 +28,10 @@ package net.systemeD.potlatch2.utils {
                                var way:Way;
                 var nodestring:Array = [];
                 for each (var trkpt:XML in trkseg.trkpt) {
-                                       nodestring.push(connection.createNode({}, trkpt.@lat, trkpt.@lon, push));
+                                       nodestring.push(connection.createNode({}, trkpt.@lat, trkpt.@lon, action.push));
                                }
                 if (nodestring.length > 0) {
-                                       way = connection.createWay({}, nodestring, push);
+                                       way = connection.createWay({}, nodestring, action.push);
                                        if (simplify) { Simplify.simplify(way, map, false); }
                                }
                        }
@@ -41,10 +41,12 @@ package net.systemeD.potlatch2.utils {
                                for each (var tag:XML in wpt.children()) {
                                        tags[tag.name().localName]=tag.toString().substr(0,255);
                                }
-                               var node:Node = connection.createNode(tags, wpt.@lat, wpt.@lon, push);
+                               var node:Node = connection.createNode(tags, wpt.@lat, wpt.@lon, action.push);
                        }
 
                        default xml namespace = new Namespace("");
+                       action.doAction();
+                       finish();
                }
        }
 }
index 73d3481..10a3496 100644 (file)
@@ -68,20 +68,17 @@ package net.systemeD.potlatch2.utils {
                                        files[i]=zip.getInput(zip.entries[i]);
                                        filesloaded++;
                                }
-                               runImporter();
+                               doImport();
                        } else {
                                // Standard file
                                files[filenum]=rawData;
                                filesloaded++;
                                trace("loaded file "+filenum+" ("+filesloaded+"/"+filenames.length+")"); 
-                               if (filesloaded==filenames.length) { runImporter(); }
+                               if (filesloaded==filenames.length) { doImport(); }
                        }
                }
 
-               private function runImporter():void {
-                       var action:CompositeUndoableAction = new CompositeUndoableAction("Import layer "+connection.name);
-                       doImport(action.push);
-                       action.doAction(); // just do it, don't add to undo stack
+               protected function finish():void {
                        connection.registerPOINodes();
                        if (callback!=null) { callback(connection,options,true); }
                }
@@ -93,7 +90,7 @@ package net.systemeD.potlatch2.utils {
                        return null;
                }
                
-               protected function doImport(push:Function):void {
+               protected function doImport():void {
                }
 
                protected function securityErrorHandler( event:SecurityErrorEvent ):void { callback(connection,options,false,"You don't have permission to open that file."); }
index f5ebe5c..ea846d1 100644 (file)
@@ -1,11 +1,7 @@
 package net.systemeD.potlatch2.utils {
 
     import net.systemeD.halcyon.Map;
-    import net.systemeD.halcyon.connection.Connection;
-    import net.systemeD.halcyon.connection.Node;
-    import net.systemeD.halcyon.connection.Way;
-    import net.systemeD.halcyon.connection.Relation;
-    import net.systemeD.halcyon.connection.RelationMember;
+    import net.systemeD.halcyon.connection.*;
     import net.systemeD.potlatch2.tools.Simplify;
 
     /**
@@ -17,7 +13,8 @@ package net.systemeD.potlatch2.utils {
             super(connection, map, callback, simplify, options);
         }
 
-        override protected function doImport(push:Function): void {
+        override protected function doImport(): void {
+            var action:CompositeUndoableAction = new CompositeUndoableAction("Import KML "+connection.name);
             var kml:XML = new XML(files[0]);
 
             for each (var ns:Namespace in kml.namespaceDeclarations()) {
@@ -40,15 +37,15 @@ package net.systemeD.potlatch2.utils {
                 }
 
                 for each (var point:XML in placemark.Point) {
-                    importNode(point.coordinates, tags, push);
+                    importNode(point.coordinates, tags, action.push);
                 }
 
                 for each (var linestring:XML in placemark.LineString) {
-                    importWay(linestring.coordinates, tags, false, push);
+                    importWay(linestring.coordinates, tags, false, action.push);
                 }
 
                 for each (var linearring:XML in placemark.LinearRing) {
-                    importWay(linearring.coordinates, tags, true, push);
+                    importWay(linearring.coordinates, tags, true, action.push);
                 }
 
                 for each (var polygon:XML in placemark.Polygon) {
@@ -56,23 +53,25 @@ package net.systemeD.potlatch2.utils {
                         var members:Array = [];
                         var way:Way;
 
-                        way = importWay(polygon.outerBoundaryIs.LinearRing.coordinates, {}, true, push);
+                        way = importWay(polygon.outerBoundaryIs.LinearRing.coordinates, {}, true, action.push);
                         members.push(new RelationMember(way, "outer"));
 
                         for each (var inner:XML in polygon.innerBoundaryIs) {
-                            way = importWay(inner.LinearRing.coordinates, {}, true, push);
+                            way = importWay(inner.LinearRing.coordinates, {}, true, action.push);
                             members.push(new RelationMember(way, "inner"));
                         }
 
                         tags["type"] = "multipolygon";
 
-                        connection.createRelation(tags, members, push);
+                        connection.createRelation(tags, members, action.push);
                     } else {
-                        importWay(polygon.outerBoundaryIs.LinearRing.coordinates, tags, true, push);
+                        importWay(polygon.outerBoundaryIs.LinearRing.coordinates, tags, true, action.push);
                     }
                 }
             }
-                       default xml namespace = new Namespace("");
+            default xml namespace = new Namespace("");
+            action.doAction();
+            finish();
         }
 
         private function importNode(coordinates:String, tags:Object, push:Function): Node {
index fac2a27..42c0e3a 100644 (file)
@@ -10,7 +10,9 @@ package net.systemeD.potlatch2.utils {
                        super(connection,map,callback,simplify,options);
                }
 
-               override protected function doImport(push:Function):void {
+               override protected function doImport():void {
+                       var action:CompositeUndoableAction = new CompositeUndoableAction("Import OSM "+connection.name);
+
                        var data:XML;
                        var oldid:Number;
                        var tags:Object;
@@ -25,14 +27,14 @@ package net.systemeD.potlatch2.utils {
 
                                for each(data in map.node) {
                                        oldid = Number(data.@id);
-                                       nodemap[oldid] = connection.createNode(parseTags(data.tag), Number(data.@lat), Number(data.@lon), push);
+                                       nodemap[oldid] = connection.createNode(parseTags(data.tag), Number(data.@lat), Number(data.@lon), action.push);
                                }
 
                                for each(data in map.way) {
                                        oldid = Number(data.@id);
                                        var nodes:Array = [];
                                        for each(var nd:XML in data.nd) { nodes.push(nodemap[Number(nd.@ref)]); }
-                                       waymap[oldid] = connection.createWay(parseTags(data.tag), nodes, push);
+                                       waymap[oldid] = connection.createWay(parseTags(data.tag), nodes, action.push);
                                }
             
                                for each(data in map.relation) {
@@ -50,10 +52,11 @@ package net.systemeD.potlatch2.utils {
                                                }
                                                if (member!=null) { members.push(new RelationMember(member,role)); }
                                        }
-                                       relationmap[oldid] = connection.createRelation(parseTags(data.tag), members, push);
+                                       relationmap[oldid] = connection.createRelation(parseTags(data.tag), members, action.push);
                                }
                        }
-                       
+                       action.doAction();
+                       finish();
         }
 
         private function parseTags(tagElements:XMLList):Object {
index f789ab7..aaab440 100644 (file)
@@ -4,87 +4,110 @@ package net.systemeD.potlatch2.utils {
        import org.vanrijkom.dbf.*;
        import com.gradoservice.proj4as.*;
        import net.systemeD.halcyon.Map;
-       import net.systemeD.halcyon.connection.Connection;
-       import net.systemeD.halcyon.connection.Node;
-       import net.systemeD.halcyon.connection.Way;
+       import net.systemeD.halcyon.connection.*;
        import net.systemeD.potlatch2.tools.Simplify;
-       import flash.utils.ByteArray;
+       import flash.utils.*;
 
        public class ShpImporter extends Importer {
 
                private var projection:String;
+               
+               private var timeout:uint;
+               private var position:uint=0;
+
+               private var polyArray:Array;
+               private var shpFile:ByteArray;
+               private var dbfFile:ByteArray;
+               private var shp:ShpHeader;
+               private var dbf:DbfHeader;
+               private var proj:Proj4as;
+               private var toProj:ProjProjection;
+               private var fromProj:ProjProjection;
+
+               private static var MAX_ITEMS:uint=10000;                // maximum number of shapes to process in each pass
 
                public function ShpImporter(connection:Connection, map:Map, callback:Function=null, simplify:Boolean=false, options:Object=null) {
                        if (options['projection']) this.projection=options['projection'];
                        super(connection,map,callback,simplify,options);
                }
 
-               override protected function doImport(push:Function): void {
+               override protected function doImport():void {
                        // we load .shp as files[0], .shx as files[1], .dbf as files[2]
-                       var shpFile:ByteArray=getFileByName(/.shp$/);
-                       var dbfFile:ByteArray=getFileByName(/.dbf$/);
-                       var shp:ShpHeader=new ShpHeader(shpFile);
-                       var dbf:DbfHeader=new DbfHeader(dbfFile);
+                       shpFile=getFileByName(/.shp$/); shp=new ShpHeader(shpFile);
+                       dbfFile=getFileByName(/.dbf$/); dbf=new DbfHeader(dbfFile);
+                       if (shp.shapeType!=ShpType.SHAPE_POLYGON && shp.shapeType!=ShpType.SHAPE_POLYLINE) { return; }
 
                        if (projection) {
-                               var proj:Proj4as=new Proj4as();
-                               var toProj:ProjProjection=new ProjProjection('EPSG:4326');
-                               var fromProj:ProjProjection=new ProjProjection('EPSG:27700');
+                               proj=new Proj4as();
+                               toProj=new ProjProjection('EPSG:4326');
+                               fromProj=new ProjProjection('EPSG:27700');
                        }
 
+                       polyArray = ShpTools.readRecords(shpFile);
+                       timeout=setTimeout(runProcess,50);
+                       trace("Begin processing");
+               }
+               
+               private function runProcess():void {
                        var nodemap:Object={};
                        var key:String, v:String;
+                       clearTimeout(timeout);
+                       var action:CompositeUndoableAction = new CompositeUndoableAction("Import layer "+connection.name);
 
-                       if (shp.shapeType==ShpType.SHAPE_POLYGON || shp.shapeType==ShpType.SHAPE_POLYLINE) {
-
-                               // Loop through all polylines in the shape
-                               var polyArray:Array = ShpTools.readRecords(shpFile);
-                               for (var i:uint=0; i<polyArray.length; i++) {
-
-                                       // Get attributes and create a tags hash
-                                       // (note that dr.values is a Dictionary)
-                                       var dr:DbfRecord = DbfTools.getRecord(dbfFile, dbf, i);
-                                       var tags:Object={};
-                                       for (key in dr.values) {
-                                               v=dr.values[key];
-                                               while (v.substr(v.length-1,1)==" ") v=v.substr(0,v.length-1);
-                                               while (v.substr(0,1)==" ") v=v.substr(1);
-                                               if (v!='') tags[key.toLowerCase()]=v;
-                                       }
-
-                                       // Do each ring in turn, then each point in the ring
-                                       var way:Way;
-                                       var node:Node;
-                                       var x:Number, y:Number;
-                                       for (var j:int=0; j < polyArray[i].shape.rings.length; j++) {
-                                               var nodestring:Array=[];
-                                               var points:Array = polyArray[i].shape.rings[j];
-                                               if (points!=null) {
-                                                       for (var k:int=0; k < points.length; k++) {
-                                                               var p:ShpPoint = ShpPoint(points[k]);
-                                                               
-                                                               if (projection) {
-                                                                       var r:ProjPoint = new ProjPoint(p.x,p.y,0);
-                                                                       r=proj.transform(fromProj,toProj,r);
-                                                                       x=r.x; y=r.y;
-                                                               } else {
-                                                                       x=p.x; y=p.y;
-                                                               }
+                       trace("Starting at position "+position);
+                       for (var i:uint=position; i<Math.min(position+MAX_ITEMS,polyArray.length); i++) {
+                               // Get attributes and create a tags hash
+                               // (note that dr.values is a Dictionary)
+                               var dr:DbfRecord = DbfTools.getRecord(dbfFile, dbf, i);
+                               var tags:Object={};
+                               for (key in dr.values) {
+                                       v=dr.values[key];
+                                       while (v.substr(v.length-1,1)==" ") v=v.substr(0,v.length-1);
+                                       while (v.substr(0,1)==" ") v=v.substr(1);
+                                       if (v!='') tags[key.toLowerCase()]=v;
+                               }
 
-                                                               key=x+","+y;
-                                                               if (nodemap[key]) { node=nodemap[key]; }
-                                                               else { node=connection.createNode({}, y, x, push); nodemap[key]=node; }
-                                                               nodestring.push(node);
+                               // Do each ring in turn, then each point in the ring
+                               var way:Way;
+                               var node:Node;
+                               var x:Number, y:Number;
+                               for (var j:int=0; j < polyArray[i].shape.rings.length; j++) {
+                                       var nodestring:Array=[];
+                                       var points:Array = polyArray[i].shape.rings[j];
+                                       if (points!=null) {
+                                               for (var k:int=0; k < points.length; k++) {
+                                                       var p:ShpPoint = ShpPoint(points[k]);
+                                                       
+                                                       if (projection) {
+                                                               var r:ProjPoint = new ProjPoint(p.x,p.y,0);
+                                                               r=proj.transform(fromProj,toProj,r);
+                                                               x=r.x; y=r.y;
+                                                       } else {
+                                                               x=p.x; y=p.y;
                                                        }
+
+                                                       key=x+","+y;
+                                                       if (nodemap[key]) { node=nodemap[key]; }
+                                                       else { node=connection.createNode({}, y, x, action.push); nodemap[key]=node; }
+                                                       nodestring.push(node);
                                                }
-                                               if (nodestring.length>0) {
-                                                       way=connection.createWay(tags, nodestring, push);
-                                                       if (simplify) { Simplify.simplify(way, map, false); }
-                                               }
+                                       }
+                                       if (nodestring.length>0) {
+                                               way=connection.createWay(tags, nodestring, action.push);
+                                               if (simplify) { Simplify.simplify(way, map, false); }
                                        }
                                }
                        }
-               }
 
+                       // Set next iteration to run after a short delay
+                       action.doAction();
+                       if (i<polyArray.length) {
+                               position=i;
+                               timeout=setTimeout(runProcess,50);
+                       } else {
+                               trace("Finished");
+                               finish();
+                       }
+               }
        }
 }