secondo/bin/Scripts/OrderedRelationGraphFromFullOSMImportWithAreaSelection.SEC

###########################################################################
#
# Construction of a Road Network from OpenStreetMap Data in OrderedRelation
# Graph Representation
#
# This variant allows one to select a small subarea and construct the road 
# network for it only. Roads intersecting the subarea are constructed 
# completely, including parts outside the area.
#
# Uses NestedRelationAlgebra
#
# -> adapt the filename of the osm-file to be imported.
#
# running times and result sizes refer to arnsberg-regbez.osm, using 
# GlobalMemory = 1 GB and selected subarea for Hombruch
#
# second set of running times and sizes refers to data for california and
# subarea Greater San Francisco (Bay area down to San Jose, see CityArea below)
# using 2 GB global memory.
#
###########################################################################

# Database hombruch

query fullosmimport('california-latest.osm', "All")

# 234 secs

# file arnsberg-regbez.osm opened successfully
# relation AllNodes with 5008465 tuples stored
# relation AllNodeTags with 389898 tuples stored
# relation AllWays with 6419349 tuples stored
# relation AllWayTags with 2193352 tuples stored
# relation AllRelations with 355347 tuples stored
# relation AllRelationTags with 52448 tuples stored

# file california-latest.osm opened successfully
# relation AllNodes with 44131572 tuples stored
# relation AllNodeTags with 8469252 tuples stored
# relation AllWays with 49706123 tuples stored
# relation AllWayTags with 20363092 tuples stored
# relation AllRelations with 164141 tuples stored
# relation AllRelationTags with 81925 tuples stored
# Total runtime ...   Times (elapsed / cpu): 35:19min (2118.79sec) /1630.55sec = 1.29943

# 35:19min (2118.79sec)

# define area of interest (corresponds to Greater San Francisco)

let CityArea = [const rect value (-122.57724906420854 -121.73656307839723 37.30134709102425 38.01641067353821)]


# get the WayIds of ways having nodes inside the city area

let WayIds = AllNodes feed filter[bbox(makepoint(.Lon, .Lat)) inside CityArea] project[NodeId]
  AllWays feed itHashJoin[NodeId, NodeRef]
  project[WayId] sort rdup consume

# 1:07 mins
# result size 6165

# SF
# 7:48min (468.026sec)
# 203591


let CityWays = WayIds feed {w}
  AllWays feed itHashJoin[WayId_w, WayId]
  project[WayId, NodeCounter, NodeRef]
  sortby[WayId, NodeCounter]
  consume

# 12.69 secs
# result size 40065

# SF
# 2:12min (132.248sec)
# 2318612

let CityWayTags = WayIds feed {w}
  AllWayTags
  feed itHashJoin[WayId_w, WayIdInTag]
  project[WayIdInTag, WayTagKey, WayTagValue]
  sortby[WayIdInTag]
  consume

# 7.48 secs

# 1:32min (91.6926sec)
# 1165198


# get the nodes inside the city area as well as those belonging to ways having nodes inside the 
# city area

let CityNodes =  AllNodes feed filter[bbox(makepoint(.Lon, .Lat)) inside CityArea]
  CityWays feed
  AllNodes feed
  itHashJoin[NodeRef, NodeId]
  project[NodeId, Lat, Lon]
  concat
  sort rdup
  consume

# 43.15 secs
# result size 29721

# SF
# 7:46min (466.44sec)
# 1962728

let CityNodeTags = CityNodes feed {c}
  AllNodeTags feed 
  itHashJoin[NodeId_c, NodeIdInTag]
  project[NodeIdInTag, NodeTagKey, NodeTagValue]
  sortby[NodeIdInTag]
  consume
  
# 1.82 secs

# 40.0677sec
# 191472

let CityNodesNew = CityNodes feed 
  extend[Easting: .Lon * 1000000, Northing: .Lat * 1000000]
  extend[Box: rectangle2(.Easting, .Easting, .Northing, .Northing)]
  sortby[Box]
  projectextend[NodeId;  Pos: makepoint(.Lon, .Lat)] 
  addcounter[NodeIdNew, 1] 
  consume

# 1.29 secs

# 1:23min (83.3309sec)
# 1962728

let NodesTagged = CityNodesNew feed
  CityNodeTags feed
  itHashJoin[NodeId, NodeIdInTag]
  sortby[NodeIdNew]
  nest[NodeIdNew; NodeTags]
  consume

# 0.56 secs

# query nodes(CityNodesNew feed addid bulkloadrtree[Pos]) consume


let Ways =
  CityNodesNew feed   
  CityWays feed itHashJoin[NodeId, NodeRef] sortby[WayId, NodeCounter] nest[WayId; NodeList] 
    extend[Curve  : .NodeList afeed projecttransformstream[Pos] collect_line[TRUE]] 
  CityWayTags feed nest[WayIdInTag; WayInfo] itHashJoin[WayId, WayIdInTag]
  extend[Box: bbox(.Curve) scale[1000000.0]]
  sortby[Box]
  remove[Box]
  consume

# 6.74 secs

# 4:37min (277.212sec)
# 201840


derive Ways_Curve_rtree = Ways feed addid projectextend[TID; Box: bbox(.Curve)] bulkloadrtree[Box]

# 0.3 secs

# 2.92359sec



# query Ways feed addid projectextend[TID; Box: bbox(.Curve)] bulkloadrtree[Box]

# query nodes(Ways feed addid projectextend[TID; Box: bbox(.Curve)] bulkloadrtree[Box]) consume



let Roads = Ways feed filter[.WayInfo afeed filter[.WayTagKey = "highway"] count > 0] consume

# 0.91 secs

# 42.3311sec

derive Roads_Curve_rtree = Roads feed addid projectextend[TID; Box: bbox(.Curve)] bulkloadrtree[Box]

# 3.56 secs

# 1.72803sec

# compute Nodes as the union of start points, end points and intersections of roads

let Nodes = 
  CityWays feed 
  CityWays feed {h2}
  itHashJoin[NodeRef, NodeRef_h2]
  filter[.WayId # .WayId_h2]
  CityNodesNew feed  
  itHashJoin[NodeRef, NodeId]
  Roads feed {r1} itHashJoin[WayId, WayId_r1]
  Roads feed {r2} itHashJoin[WayId_h2, WayId_r2]
  project[WayId, NodeCounter, NodeIdNew, Pos]
Roads feed 
  projectextend[WayId; Node: .NodeList afeed filter[.NodeCounter = 0] aconsume]
  unnest[Node]
  project[WayId, NodeCounter, NodeIdNew, Pos]
  concat
Roads feed 
  extend[HighNodeNo: (.NodeList afeed count) - 1]
  projectextend[WayId; Node:  fun(t: TUPLE) 
    attr(t, NodeList) afeed filter[.NodeCounter = attr(t, HighNodeNo)] aconsume]
  unnest[Node]
  project[WayId, NodeCounter, NodeIdNew, Pos]
  concat
  sortby[WayId, NodeCounter]
  rdup
  consume

# 1.45 secs

# 1:28min (87.9793sec)
# 394532



let EdgesUp =
  Nodes feed nest[WayId; SectionNodes]
    projectextend[WayId; Sections: .SectionNodes afeed
      extend_last[Source: ..NodeIdNew::0, Target: .NodeIdNew::0, 
        SourcePos: ..Pos::[const point value undef], 
        TargetPos: .Pos::[const point value undef],
        SourceNodeCounter: ..NodeCounter::0,
        TargetNodeCounter: .NodeCounter::0]
      filter[.Source # 0] 
      project[Source, Target, SourcePos, TargetPos, 
        SourceNodeCounter, TargetNodeCounter]
      aconsume]
  Roads feed {r}
  itHashJoin[WayId, WayId_r]
  projectextend[WayId; Sections: fun(t:TUPLE)
    attr(t, Sections) afeed 
    extend[
      Curve: fun(u: TUPLE)
      attr(t, NodeList_r) afeed 
        filter[.NodeCounter_r between[attr(u, SourceNodeCounter),
          attr(u, TargetNodeCounter)] ]
        projecttransformstream[Pos_r]
        collect_sline[TRUE],
      RoadName: attr(t, WayInfo_r) afeed filter[.WayTagKey_r = "name"] extract 
       [WayTagValue_r],
      RoadType: attr(t, WayInfo_r) afeed filter[.WayTagKey_r = "highway"] extract 
       [WayTagValue_r],
      MaxSpeed: attr(t, WayInfo_r) afeed filter[.WayTagKey_r = "maxspeed"] extract 
       [WayTagValue_r]  
    ]
    aconsume ]
  unnest[Sections]
  consume

# 1.77 secs

# 3:11min (191.229sec)
# 277793



let EdgesDown =
  Nodes feed nest[WayId; SectionNodes]
    projectextend[WayId; Sections: .SectionNodes afeed sortby[NodeCounter desc]
      extend_last[Source: ..NodeIdNew::0, Target: .NodeIdNew::0, 
        SourcePos: ..Pos::[const point value undef], 
        TargetPos: .Pos::[const point value undef],
        SourceNodeCounter: ..NodeCounter::0,
        TargetNodeCounter: .NodeCounter::0]
      filter[.Source # 0] 
      project[Source, Target, SourcePos, TargetPos, 
        SourceNodeCounter, TargetNodeCounter]
      aconsume]
  Roads feed 
    filter[.WayInfo afeed filter[.WayTagKey = "oneway"] 
      filter[(.WayTagValue = "yes")] count = 0] {r}
  itHashJoin[WayId, WayId_r]
  projectextend[WayId; Sections: fun(t:TUPLE)
    attr(t, Sections) afeed extend[Curve: fun(u: TUPLE)
      attr(t, NodeList_r) afeed sortby[NodeCounter_r desc]
        filter[.NodeCounter_r between[attr(u, TargetNodeCounter),
          attr(u, SourceNodeCounter)] ]
        projecttransformstream[Pos_r]
        collect_sline[TRUE],
      RoadName: attr(t, WayInfo_r) afeed filter[.WayTagKey_r = "name"] extract 
       [WayTagValue_r],
      RoadType: attr(t, WayInfo_r) afeed filter[.WayTagKey_r = "highway"] extract 
       [WayTagValue_r],
      MaxSpeed: attr(t, WayInfo_r) afeed filter[.WayTagKey_r = "maxspeed"] extract 
       [WayTagValue_r]  
    ]
    aconsume ]
  unnest[Sections]
  consume

# 2.00 secs

# 3:29min (209.484sec)
# 233879


let Edges = EdgesUp feed EdgesDown feed concat 
  projectextend[Source, Target, SourcePos, TargetPos, SourceNodeCounter, TargetNodeCounter, Curve, RoadName, 
    RoadType, MaxSpeed; WayId: .WayId] 
  oconsume[Source, Target]

# 0.47 secs

# 28.9857sec
# 511672




let EdgeIndex = Edges feed projectextend[Source, Target, Curve, SourcePos; Box: bbox(.Curve)] 
  filter[isdefined(.Box)] 
  extend[Box2: bbox(.SourcePos) scale[1000000.0]]
  sortby[Box2]
  project[Source, Target, Curve, Box]
  consume

# 0.46 secs

# 33.4277sec
# 510869


# query nodes(EdgeIndex feed addid bulkloadrtree[Box]) consume




derive EdgeIndex_Box_rtree = EdgeIndex feed addid bulkloadrtree[Box]

# 0.24 secs

# 4.19004sec


# Queries
#
# Get edges from a restricted area:

# let hombruch = [const rect value (7.419 7.457 51.462 51.484)]

# query EdgeIndex_Box_rtree EdgeIndex windowintersects[hombruch] remove[Box] 
# loopsel[Edges orange[.Source, .Target; .Source, .Target]] consume

 

# query Ways_Curve_rtree windowintersectsS[hombruch] Ways gettuples remove[NodeList] consume

 


# Runtime for Scripts/OrderedRelationGraphFromFullOSMImport.SEC: Times (elapsed / cpu): 
# 5:08min (308.31sec)

# Run time for Greater San Francisco:
# Runtime for Scripts/OrderedRelationGraphFromFullOSMImportWithAreaSelection.SEC: Times (elapsed / 
# cpu): 71:41min (4300.77sec) /3622.82sec = 1.18713