################################################################################
### Adjusting imported OSM data to the standard geofabrik format (3 / 7)
################################################################################
### Description:
### - This script is only needed, if the shape files used for import were
### created manually. It adjusts the imported OSM data to the generic format
### provided by shape files from the provider geofabrik.
### Please, see tu_shp_import.sh for further details.
###
### Preconditions:
### - existing open database
### - streetsTmp-relation
### streetsTmp: rel{geoData: sline, ID: int, part: int, attrib: string}
###
### Postconditions:
### - streetsTmp-relation
### streetsTmp: rel{geoData: sline, osm_id: int, name: string, ref: string,
###                 type: string, oneway: int, bridge: int, maxspeed: int,
###                 tunnel: bool, layer: int}
### - pointsTmp-relation
### pointsTmp: rel{osm_id: int, timestamp: int, name: string, geoData: point,
###                type: string}
###
### Author:
### - Thomas Uchdorf, thomas.uchdorf(at)fernuni-hagen.de
################################################################################

# --- Working on streets
# Concatenating split up attrib-strings
let StreetsWholeAttrib =
   StreetsTmp feed 
   sortby [ID,Part]
   extend [Tags: trim(.Attrib)]
   remove [Attrib] 
   groupby [ID;  
      AllTags: group feed  
      aggregateB [Tags;  
         fun(A: text,B: text) 
         (A + B); A 
      ]]
   extend [HelpId: .ID]
   remove [ID]
   streetsTmp feed
   filter [.Part = 0]
   remove [Part, Attrib]
   hashjoin [HelpId,ID,99997]
   extend [Attrib: .AllTags]
   remove [AllTags]
   consume;

# Deleting temporary relation
delete StreetsTmp;

# Ignoring unwanted data on railways, barriers and so on
let InterestingStreets =
   StreetsWholeAttrib feed
   filter [.Attrib contains "highway"]
   consume;

# Extending tags that are kept in one attribute
let StreetTags =
   InterestingStreets feed
   extendstream [Abc: tokenize(.attrib,"|")]
   consume;

# Creating temporary relations with data on the relevant tags
let RoadClassTags =
   StreetTags feed
   sortby [ID]
   groupby [ID; 
      Cnt: group count, 
      RoadClassTmp: group feed
         aggregateB [Abc; 
            fun(A: text,B: text)
               ifthenelse(A contains "highway = ",
                  trim(A),
                  ifthenelse(B contains "highway = ",
                     trim(B),
                     'UNKNOWN')); 
               'EMPTY']]
   projectextend [; 
      OsmId: .ID,
      RroadClass: replace(.RoadClassTmp,'highway = ',"")]
   consume;

# TODO add special treatment for one-ways in the opposite direction of the road
#      (oneway = -1 or oneway = reverse) 
let OneWayTags =
   InterestingStreets feed
   projectextend [; 
      OsmId: .ID,
      OneWay:
         ifthenelse((.Attrib contains "oneway = yes") or
            (.Attrib contains "oneway = 1") or
            (.Attrib contains "oneway = true"),
            1,
            ifthenelse((.Attrib contains "oneway = -1") or
               (.Attrib contains "oneway = reverse"),-1,0))]   
   consume;

let NameTags =
   StreetTags feed
   sortby [ID]
   groupby [ID; 
      Cnt: group count, 
      NameTmp: group feed
         aggregateB [Abc; 
            fun(A: text,B: text)
               ifthenelse(A contains "name = ",
                  trim(A),
                  ifthenelse(B contains "name = ",
                     trim(B),
                     'UNKNOWN')); 
               'UNKNOWN']]
   projectextend [; 
      OsmId: .ID,
      Name: replace(.NameTmp,'name = ',"")]
   consume;

let LayerTags =
   StreetTags feed
   sortby [ID]
   groupby [ID; 
      Cnt: group count, 
      LayerTmp: group feed
         aggregateB [Abc; 
            fun(A: text,B: text)
               ifthenelse(A contains "layer = ",
                  trim(A),
                  ifthenelse(B contains "layer = ",
                     trim(B),
                     'UNKNOWN')); 
               'UNKNOWN']]
   projectextend [; 
      OsmId: .ID,
      Layer: replace(.LayerTmp,'layer = ',"")]
   consume;

let TunnelTags =
   InterestingStreets feed
   projectextend [; OsmId: .ID, Tunnel: (.Attrib contains "tunnel = yes")]   
   consume;

let BridgeTags =
   InterestingStreets feed
   projectextend [; 
      OsmId: .ID,
      Bridge: ifthenelse(.Attrib contains "bridge = yes", 1, 0)]   
   consume;

let RefTags =
   StreetTags feed
   sortby [ID]
   groupby [ID; 
      Cnt: group count, 
      RefTmp: group feed
         aggregateB [Abc; 
            fun(A: text,B: text)
               ifthenelse(A contains "ref = ",
                  trim(A),
                  ifthenelse(A contains "ref = ",
                     trim(B),
                     'UNKNOWN')); 
               'UNKNOWN']]
   projectextend [; 
      OsmId: .ID,
      Ref: replace(.RefTmp,'ref = ',"")]
   consume;

let MaxSpeedTags =
   StreetTags feed
   sortby [ID]
   groupby [ID; 
      Cnt: group count,
      MaxspeedTmp: group feed
         aggregateB [Abc; 
            fun(A: text,B: text)
               ifthenelse(A contains "maxspeed = ",
                  trim(A),
                  ifthenelse(B contains "maxspeed = ",
                     trim(B),
                     'UNKNOWN')); 
               'UNKNOWN']]
   projectextend [; 
      OsmId: .ID,
      Maxspeed: replace(.MaxspeedTmp,'maxspeed = ',"")]
   consume;

# Assembling the street data
let StreetsTmp =
   (((((((RoadClassTags feed
          OneWayTags feed {a}
          hashjoin [OsmId, OsmId_a, 999997])
         NameTags feed {b}
         hashjoin [OsmId, OsmId_b, 999997])
        LayerTags feed {c}
        hashjoin [OsmId, OsmId_c, 999997])
       TunnelTags feed {d}
       hashjoin [OsmId, OsmId_d, 999997])
      BridgeTags feed {e}
      hashjoin [OsmId, OsmId_e, 999997])
     RefTags feed {f}
     hashjoin [OsmId, OsmId_f, 999997])
    MaxSpeedTags feed {g}
    hashjoin [OsmId, OsmId_g, 999997])
   InterestingStreets feed
   hashjoin [OsmId, ID, 999997]
   projectextend [GeoData;
      Osm_id: .OsmId,
      type: tostring(.RoadClass),
      Oneway: .OneWay_a,
      Name: tostring(.Name_b),
      Layer: ifthenelse(.Layer_c contains "UNKNOWN",
                0,
                toObject(.Layer_c,1)),
      Tunnel: .Tunnel_d,
      Bridge: .Bridge_e,
      Ref: tostring(.Ref_f),
      Maxspeed: ifthenelse(.Maxspeed_g contains "UNKNOWN",
                   0,
                   toObject(.Maxspeed_g,1))]
   consume;

# Deleting temporary relation
delete InterestingStreets;

# Deleting tag-relations
delete RoadClassTags;
delete OneWayTags;
delete NameTags;
delete LayerTags;
delete TunnelTags;
delete BridgeTags;
delete RefTags;
delete MaxSpeedTags;
delete StreetTags;

# --- Processing POIs
# Concatenating split up attrib-strings
let PointsWholeAttrib =
   PointsTmp feed
   sortby [ID,Part]
   extend [Tags: trim(.Attrib)]
   remove [Attrib]
   groupby [ID;  
      AllTags: group feed  
      aggregateB [Tags;
         fun(A: text,B: text)
         (A + B); A
      ]] 
   consume;

# Extending tags that are kept in one attribute
let PointTags =
   PointsWholeAttrib feed
   extendstream [Tags: tokenize(.AllTags,"|")]
   consume;

# Creating temporary relations with data on the relevant tags
let NameTags =
   PointTags feed
   sortby [ID]
   groupby [ID; 
      Cnt: group count,
      NameTmp: group feed
         aggregateB [Tags;
            fun(A: text,B: text)
               ifthenelse(A contains "name = ",
                  trim(A),
                  ifthenelse(B contains "name = ",
                     trim(B),
                     'UNKNOWN')); 
               'UNKNOWN']]
   projectextend [; 
      OsmId: .ID,
      Name: replace(.NameTmp,'name = ',"")]
   consume;

let AmenityTags =
   PointTags feed
   sortby [ID]
   groupby [ID;
      Cnt: group feed count,
      AmenityTmp: group feed
         aggregateB [Tags;
            fun(A: text,B: text)
               ifthenelse(A contains "amenity = ",
                  trim(A),
                  ifthenelse(B contains "amenity = ",
                     trim(B),
                     'UNKNOWN')); 
               'UNKNOWN']]
   projectextend [; 
      OsmId: .ID,
      Type: replace(.AmenityTmp,'amenity = ',"")]
   consume;

# Renaming pointsTmp
let PointsHelp =
   PointsTmp feed
   consume;
delete PointsTmp;

# Restoring pointsTmp and inserting missing attributes from the temporary
# tag-relations
let PointsTmp =
   (PointsHelp feed
    filter [.Part = 0]
    remove [Part, Attrib]
    NameTags feed
    smouterjoin [ID, OsmId]
    filter [isdefined(.ID)])
   remove [OsmId]
   AmenityTags feed
   smouterjoin [ID, OsmId]
   filter [isdefined(.ID)]
   remove [ID]
   extend [Osm_id: .OsmId, Timestamp: 0]
   remove [OsmId]
   filter [not((isdefined(.Name) and (.Name = 'UNKNOWN'))
      and (isdefined(.Type) and (.Type = 'UNKNOWN')))]
   consume;

# Deleting temporary relations
delete PointsHelp;
delete PointsWholeAttrib;

# Deleting tag-relations
delete PointTags;
delete NameTags;
delete AmenityTags;