2 # Tiger road data to OSM conversion script
3 # Creates Karlsruhe-style address ways beside the main way
4 # based on the Massachusetts GIS script by christopher schmidt
7 # On very tight curves, a loop may be generated in the address way.
8 # It would be nice if the ends of the address ways were not pulled back from dead ends
11 # Ways that include these mtfccs should not be uploaded
14 # H3013 Braided Stream
15 # H3020 Canal, Ditch or Aqueduct
16 # L4130 Point-to-Point Line
17 # L4140 Property/Parcel Line (Including PLSS)
18 # P0001 Nonvisible Linear Legal/Statistical Boundary
19 # P0002 Perennial Shoreline
20 # P0003 Intermittent Shoreline
21 # P0004 Other non-visible bounding Edge (e.g., Census water boundary, boundary of an areal feature)
22 ignoremtfcc = [ "H1100", "H3010", "H3013", "H3020", "L4130", "L4140", "P0001", "P0002", "P0003", "P0004" ]
24 # Sets the distance that the address ways should be from the main way, in feet.
27 # Sets the distance that the ends of the address ways should be pulled back from the ends of the main way, in feet
30 import sys, os.path, json
38 # https://www.census.gov/geo/reference/codes/cou.html
39 # tiger_county_fips.json was generated from the following:
40 # wget https://www2.census.gov/geo/docs/reference/codes/files/national_county.txt
41 # cat national_county.txt | perl -F, -naE'($F[0] ne 'AS') && $F[3] =~ s/ ((city|City|County|District|Borough|City and Borough|Municipio|Municipality|Parish|Island|Census Area)(?:, |\Z))+//; say qq( "$F[1]$F[2]": "$F[3], $F[0]",)'
42 json_fh = open(os.path.dirname(sys.argv[0]) + "/tiger_county_fips.json")
43 county_fips_data = json.load(json_fh)
45 def parse_shp_for_geom_and_tags( filename ):
48 dr = ogr.GetDriverByName("ESRI Shapefile")
49 poDS = dr.Open( filename )
54 poLayer = poDS.GetLayer( 0 )
57 layerDefinition = poLayer.GetLayerDefn()
58 for i in range(layerDefinition.GetFieldCount()):
59 fieldNameList.append(layerDefinition.GetFieldDefn(i).GetName())
60 # sys.stderr.write(",".join(fieldNameList))
62 poLayer.ResetReading()
66 poFeature = poLayer.GetNextFeature()
71 tags["tiger:way_id"] = int( poFeature.GetField("TLID") )
73 # FEATURE IDENTIFICATION
74 mtfcc = poFeature.GetField("MTFCC");
77 if mtfcc == "L4010": #Pipeline
78 tags["man_made"] = "pipeline"
79 if mtfcc == "L4020": #Powerline
80 tags["power"] = "line"
81 if mtfcc == "L4031": #Aerial Tramway/Ski Lift
82 tags["aerialway"] = "cable_car"
83 if mtfcc == "L4110": #Fence Line
84 tags["barrier"] = "fence"
85 if mtfcc == "L4125": #Cliff/Escarpment
86 tags["natural"] = "cliff"
87 if mtfcc == "L4165": #Ferry Crossing
88 tags["route"] = "ferry"
89 if mtfcc == "R1011": #Railroad Feature (Main, Spur, or Yard)
90 tags["railway"] = "rail"
91 ttyp = poFeature.GetField("TTYP")
94 tags["service"] = "spur"
96 tags["service"] = "yard"
97 tags["tiger:ttyp"] = ttyp
98 if mtfcc == "R1051": #Carline, Streetcar Track, Monorail, Other Mass Transit Rail)
99 tags["railway"] = "light_rail"
100 if mtfcc == "R1052": #Cog Rail Line, Incline Rail Line, Tram
101 tags["railway"] = "incline"
103 tags["highway"] = "primary"
105 tags["highway"] = "secondary"
107 tags["highway"] = "residential"
109 tags["highway"] = "track"
110 if mtfcc == "S1630": #Ramp
111 tags["highway"] = "motorway_link"
112 if mtfcc == "S1640": #Service Drive usually along a limited access highway
113 tags["highway"] = "service"
114 if mtfcc == "S1710": #Walkway/Pedestrian Trail
115 tags["highway"] = "path"
117 tags["highway"] = "steps"
118 if mtfcc == "S1730": #Alley
119 tags["highway"] = "service"
120 tags["service"] = "alley"
121 if mtfcc == "S1740": #Private Road for service vehicles (logging, oil, fields, ranches, etc.)
122 tags["highway"] = "service"
123 tags["access"] = "private"
124 if mtfcc == "S1750": #Private Driveway
125 tags["highway"] = "service"
126 tags["access"] = "private"
127 tags["service"] = "driveway"
128 if mtfcc == "S1780": #Parking Lot Road
129 tags["highway"] = "service"
130 tags["service"] = "parking_aisle"
131 if mtfcc == "S1820": #Bike Path or Trail
132 tags["highway"] = "cycleway"
133 if mtfcc == "S1830": #Bridle Path
134 tags["highway"] = "bridleway"
135 tags["tiger:mtfcc"] = mtfcc
138 if poFeature.GetField("FULLNAME"):
139 #capitalizes the first letter of each word
140 name = poFeature.GetField( "FULLNAME" )
143 #Attempt to guess highway grade
144 if name[0:2] == "I-":
145 tags["highway"] = "motorway"
146 if name[0:3] == "US ":
147 tags["highway"] = "primary"
148 if name[0:3] == "US-":
149 tags["highway"] = "primary"
150 if name[0:3] == "Hwy":
151 if tags["highway"] != "primary":
152 tags["highway"] = "secondary"
154 # TIGER 2017 no longer contains this field
155 if 'DIVROAD' in fieldNameList:
156 divroad = poFeature.GetField("DIVROAD")
158 if divroad == "Y" and "highway" in tags and tags["highway"] == "residential":
159 tags["highway"] = "tertiary"
160 tags["tiger:separated"] = divroad
162 statefp = poFeature.GetField("STATEFP")
163 countyfp = poFeature.GetField("COUNTYFP")
164 if (statefp != None) and (countyfp != None):
165 county_name = county_fips_data.get(statefp + '' + countyfp)
167 tags["tiger:county"] = county_name.encode("utf-8")
169 # tlid = poFeature.GetField("TLID")
171 # tags["tiger:tlid"] = tlid
173 lfromadd = poFeature.GetField("LFROMADD")
175 tags["tiger:lfromadd"] = lfromadd
177 rfromadd = poFeature.GetField("RFROMADD")
179 tags["tiger:rfromadd"] = rfromadd
181 ltoadd = poFeature.GetField("LTOADD")
183 tags["tiger:ltoadd"] = ltoadd
185 rtoadd = poFeature.GetField("RTOADD")
187 tags["tiger:rtoadd"] = rtoadd
189 zipl = poFeature.GetField("ZIPL")
191 tags["tiger:zip_left"] = zipl
193 zipr = poFeature.GetField("ZIPR")
195 tags["tiger:zip_right"] = zipr
197 if mtfcc not in ignoremtfcc:
198 # COPY DOWN THE GEOMETRY
201 rawgeom = poFeature.GetGeometryRef()
202 for i in range( rawgeom.GetPointCount() ):
203 geom.append( (rawgeom.GetX(i), rawgeom.GetY(i)) )
205 ret.append( (geom, tags) )
206 poFeature = poLayer.GetNextFeature()
211 # ====================================
212 # to do read .prj file for this data
213 # Change the Projcs_wkt to match your datas prj file.
214 # ====================================
216 """GEOGCS["GCS_North_American_1983",
217 DATUM["D_North_American_1983",
218 SPHEROID["GRS_1980",6378137,298.257222101]],
219 PRIMEM["Greenwich",0],
220 UNIT["Degree",0.017453292519943295]]"""
222 from_proj = osr.SpatialReference()
223 from_proj.ImportFromWkt( projcs_wkt )
226 to_proj = osr.SpatialReference()
227 to_proj.SetWellKnownGeogCS( "EPSG:4326" )
229 tr = osr.CoordinateTransformation( from_proj, to_proj )
232 def length(segment, nodelist):
233 '''Returns the length (in feet) of a segment'''
236 lat_feet = 364613 #The approximate number of feet in one degree of latitude
237 for point in segment:
238 pointid, (lat, lon) = nodelist[ round_point( point ) ]
242 #The approximate number of feet in one degree of longitute
243 lrad = math.radians(lat)
244 lon_feet = 365527.822 * math.cos(lrad) - 306.75853 * math.cos(3 * lrad) + 0.3937 * math.cos(5 * lrad)
245 distance += math.sqrt(((lat - previous[0])*lat_feet)**2 + ((lon - previous[1])*lon_feet)**2)
246 previous = (lat, lon)
249 def addressways(waylist, nodelist, first_id):
251 lat_feet = 364613 #The approximate number of feet in one degree of latitude
252 distance = float(address_distance)
255 for waykey, segments in waylist.items():
256 waykey = dict(waykey)
259 for segment in segments:
264 # Don't pull back the ends of very short ways too much
265 seglength = length(segment, nodelist)
266 if seglength < float(address_pullback) * 3.0:
267 pullback = seglength / 3.0
269 pullback = float(address_pullback)
270 if "tiger:lfromadd" in waykey:
271 lfromadd = waykey["tiger:lfromadd"]
274 if "tiger:ltoadd" in waykey:
275 ltoadd = waykey["tiger:ltoadd"]
278 if "tiger:rfromadd" in waykey:
279 rfromadd = waykey["tiger:rfromadd"]
282 if "tiger:rtoadd" in waykey:
283 rtoadd = waykey["tiger:rtoadd"]
286 if rfromadd != None and rtoadd != None:
290 if lfromadd != None and ltoadd != None:
296 firstpointid, firstpoint = nodelist[ round_point( segment[0] ) ]
298 finalpointid, finalpoint = nodelist[ round_point( segment[len(segment) - 1] ) ]
299 for point in segment:
300 pointid, (lat, lon) = nodelist[ round_point( point ) ]
302 #The approximate number of feet in one degree of longitute
303 lrad = math.radians(lat)
304 lon_feet = 365527.822 * math.cos(lrad) - 306.75853 * math.cos(3 * lrad) + 0.3937 * math.cos(5 * lrad)
306 #Calculate the points of the offset ways
307 if lastpoint != None:
308 #Skip points too close to start
309 if math.sqrt((lat * lat_feet - firstpoint[0] * lat_feet)**2 + (lon * lon_feet - firstpoint[1] * lon_feet)**2) < pullback:
310 #Preserve very short ways (but will be rendered backwards)
311 if pointid != finalpointid:
313 #Skip points too close to end
314 if math.sqrt((lat * lat_feet - finalpoint[0] * lat_feet)**2 + (lon * lon_feet - finalpoint[1] * lon_feet)**2) < pullback:
315 #Preserve very short ways (but will be rendered backwards)
316 if (pointid != firstpointid) and (pointid != finalpointid):
319 X = (lon - lastpoint[1]) * lon_feet
320 Y = (lat - lastpoint[0]) * lat_feet
322 theta = math.pi/2 - math.atan( X / Y)
323 Xp = math.sin(theta) * distance
324 Yp = math.cos(theta) * distance
339 dX = - (Yp * (pullback / distance)) / lon_feet #Pull back the first point
340 dY = (Xp * (pullback / distance)) / lat_feet
342 lpoint = (lastpoint[0] + (Yp / lat_feet) - dY, lastpoint[1] + (Xp / lon_feet) - dX)
343 lsegment.append( (id, lpoint) )
346 rpoint = (lastpoint[0] - (Yp / lat_feet) - dY, lastpoint[1] - (Xp / lon_feet) - dX)
347 rsegment.append( (id, rpoint) )
353 theta = abs(math.atan(delta[0] / delta[1]))
357 theta = theta - abs(math.atan(Yp / Xp))
358 else: theta = theta - math.pi / 2
359 r = 1 + abs(math.tan(theta/2))
361 lpoint = (lastpoint[0] + (Yp + delta[0]) * r / (lat_feet * 2), lastpoint[1] + (Xp + delta[1]) * r / (lon_feet * 2))
362 lsegment.append( (id, lpoint) )
365 rpoint = (lastpoint[0] - (Yp + delta[0]) * r / (lat_feet * 2), lastpoint[1] - (Xp + delta[1]) * r / (lon_feet * 2))
367 rsegment.append( (id, rpoint) )
372 lastpoint = (lat, lon)
375 #Add in the last node
376 dX = - (Yp * (pullback / distance)) / lon_feet
377 dY = (Xp * (pullback / distance)) / lat_feet
379 lpoint = (lastpoint[0] + (Yp + delta[0]) / (lat_feet * 2) + dY, lastpoint[1] + (Xp + delta[1]) / (lon_feet * 2) + dX )
380 lsegment.append( (id, lpoint) )
383 rpoint = (lastpoint[0] - Yp / lat_feet + dY, lastpoint[1] - Xp / lon_feet + dX)
384 rsegment.append( (id, rpoint) )
387 #Generate the tags for ways and nodes
392 if "tiger:zip_right" in waykey:
393 zipr = waykey["tiger:zip_right"]
394 if "tiger:zip_left" in waykey:
395 zipl = waykey["tiger:zip_left"]
397 name = waykey["name"]
398 if "tiger:county" in waykey:
399 county = waykey["tiger:county"]
400 if "tiger:separated" in waykey: # No longer set in Tiger-2017
401 separated = waykey["tiger:separated"]
405 #Write the nodes of the offset ways
408 for i, point in rsegment:
409 rlinestring.append( "%f %f" % (point[1], point[0]) )
412 for i, point in lsegment:
413 llinestring.append( "%f %f" % (point[1], point[0]) )
415 rsegments.append( rsegment )
417 lsegments.append( lsegment )
418 rtofromint = right #Do the addresses convert to integers?
419 ltofromint = left #Do the addresses convert to integers?
421 try: rfromint = int(rfromadd)
423 print("Non integer address: %s" % rfromadd)
425 try: rtoint = int(rtoadd)
427 print("Non integer address: %s" % rtoadd)
430 try: lfromint = int(lfromadd)
432 print("Non integer address: %s" % lfromadd)
434 try: ltoint = int(ltoadd)
436 print("Non integer address: %s" % ltoadd)
441 interpolationtype = "all";
443 if (rfromint % 2) == 0 and (rtoint % 2) == 0:
444 if separated == "Y": #Doesn't matter if there is another side
445 interpolationtype = "even";
446 elif ltofromint and (lfromint % 2) == 1 and (ltoint % 2) == 1:
447 interpolationtype = "even";
448 elif (rfromint % 2) == 1 and (rtoint % 2) == 1:
449 if separated == "Y": #Doesn't matter if there is another side
450 interpolationtype = "odd";
451 elif ltofromint and (lfromint % 2) == 0 and (ltoint % 2) == 0:
452 interpolationtype = "odd";
454 ret.append( "SELECT tiger_line_import(ST_GeomFromText('LINESTRING(%s)',4326), %s, %s, %s, %s, %s, %s);" %
455 ( ",".join(rlinestring), sql_quote(rfromadd), sql_quote(rtoadd), sql_quote(interpolationtype), sql_quote(name), sql_quote(county), sql_quote(zipr) ) )
460 interpolationtype = "all";
462 if (lfromint % 2) == 0 and (ltoint % 2) == 0:
464 interpolationtype = "even";
465 elif rtofromint and (rfromint % 2) == 1 and (rtoint % 2) == 1:
466 interpolationtype = "even";
467 elif (lfromint % 2) == 1 and (ltoint % 2) == 1:
469 interpolationtype = "odd";
470 elif rtofromint and (rfromint %2 ) == 0 and (rtoint % 2) == 0:
471 interpolationtype = "odd";
473 ret.append( "SELECT tiger_line_import(ST_GeomFromText('LINESTRING(%s)',4326), %s, %s, %s, %s, %s, %s);" %
474 ( ",".join(llinestring), sql_quote(lfromadd), sql_quote(ltoadd), sql_quote(interpolationtype), sql_quote(name), sql_quote(county), sql_quote(zipl) ) )
478 def sql_quote( string ):
479 return "'" + string.replace("'", "''") + "'"
481 def unproject( point ):
482 pt = tr.TransformPoint( point[0], point[1] )
483 return (pt[1], pt[0])
485 def round_point( point, accuracy=8 ):
486 return tuple( [ round(x,accuracy) for x in point ] )
488 def compile_nodelist( parsed_gisdata, first_id=1 ):
492 for geom, tags in parsed_gisdata:
497 r_point = round_point( point )
498 if r_point not in nodelist:
499 nodelist[ r_point ] = (i, unproject( point ))
504 def adjacent( left, right ):
505 left_left = round_point(left[0])
506 left_right = round_point(left[-1])
507 right_left = round_point(right[0])
508 right_right = round_point(right[-1])
510 return ( left_left == right_left or
511 left_left == right_right or
512 left_right == right_left or
513 left_right == right_right )
515 def glom( left, right ):
517 right = list( right )
519 left_left = round_point(left[0])
520 left_right = round_point(left[-1])
521 right_left = round_point(right[0])
522 right_right = round_point(right[-1])
524 if left_left == right_left:
526 return left[0:-1] + right
528 if left_left == right_right:
529 return right[0:-1] + left
531 if left_right == right_left:
532 return left[0:-1] + right
534 if left_right == right_right:
536 return left[0:-1] + right
538 raise 'segments are not adjacent'
540 def glom_once( segments ):
544 unsorted = list( segments )
547 while len( unsorted ) > 0:
550 for i in range(0, n):
557 # Sorted and unsorted lists have no adjacent segments
558 if len( unsorted ) == n:
563 def glom_all( segments ):
567 while unsorted != []:
568 chunk, unsorted = glom_once( unsorted )
569 chunks.append( chunk )
575 def compile_waylist( parsed_gisdata ):
578 #Group by tiger:way_id
579 for geom, tags in parsed_gisdata:
580 way_key = tags.copy()
581 way_key = ( way_key['tiger:way_id'], tuple( [(k,v) for k,v in way_key.items()] ) )
583 if way_key not in waylist:
584 waylist[way_key] = []
586 waylist[way_key].append( geom )
589 for (way_id, way_key), segments in waylist.items():
590 ret[way_key] = glom_all( segments )
594 def shape_to_sql( shp_filename, sql_filename ):
596 print("parsing shpfile %s" % shp_filename)
597 parsed_features = parse_shp_for_geom_and_tags( shp_filename )
599 print("compiling nodelist")
600 i, nodelist = compile_nodelist( parsed_features )
602 print("compiling waylist")
603 waylist = compile_waylist( parsed_features )
605 print("preparing address ways")
606 sql_lines = addressways(waylist, nodelist, i)
608 print("writing %s" % sql_filename)
609 fp = open( sql_filename, "w" )
610 fp.write( "\n".join( sql_lines ) )
613 if __name__ == '__main__':
615 if len(sys.argv) < 3:
616 print("%s input.shp output.sql" % sys.argv[0])
618 shp_filename = sys.argv[1]
619 sql_filename = sys.argv[2]
620 shape_to_sql(shp_filename, sql_filename)