secondo/bin/Scripts3/DynamicGridPartitioningPD.sec

/*
//paragraph [10] title: [{\Large \bf ]  [}]
//[Contents] [\tableofcontents]
//[ue] [\"{u}]



[10] Dynamic Grid Partitioning


[Contents] 

1 Overview

This script is for partitioning spatial data with a dynamic grid.

It is a commented script suitable for viewing with ~pdview~. To be run with 

----    @%Scripts/DynamicGridPartitioningPD.sec or
	@&Scripts/DynamicGridPartitioningPD.sec
----

2 Preparations

Preparations: 

  * include the Distributed2Algebra in makefile.algebras and recompile secondo

  * get the desired shape-file from download.geofabrik.de

  * create and open a database

  * import relations ~Buildings~, ~Railways~ and ~Waterways~ using the script ~importGermanyOsm.psec~

  * restore Workers in relation ~Worker~

  * start the remoteMonitors for the workers

*/

#restore Worker from Workerfile

/*
3 Create a Box/Rectangle for all objects

Create a bounding box for all objects to be processed. 

HINT: In this example only relations ~Buildings~, ~Railways~ and ~Waterways~ are considered.

*/

let World = 
Buildings feed projectextend[; Box: bbox(.GeoData)]
Railways feed projectextend[; Box: bbox(.GeoData)] concat
Waterways feed projectextend[; Box: bbox(.GeoData)] concat
transformstream collect_box[TRUE]

/*
Create Borders

*/

let Left = minD(World, 1)

let Right = maxD(World, 1)

let Bottom = minD(World, 2)

let Top = maxD(World, 2)


/*
4 Determine rows and colums

For the partitioning rows and columns have to be determined. In case of 6 rows and 8 columns, there will be 48 cells.

*/

let NRows = 5

let NColumns = 10

/*
5 Determine Sample

Determine sample of one relation with approximately 5000 tuple. For relation ~Sample~ only the attribute Box is needed.

*/

let Sample = Buildings feedNth[(Buildings count) div 5000, FALSE] 
projectextend[; Box: bbox(.GeoData)] consume

/*
6 Create Fraction

*/

let Fraction = (2 * (Sample count)) div NRows

/*
7 Create VPartition

Create partition of y-axis. Determine for each sample the bottom and top border of its boxes, sort them an take every kth (fraction) element. Let them be the borders of the y-axis.

*/

let VPartition = 
  Sample feed projectextend[; Y: minD(.Box, 2)]
  Sample feed projectextend[; Y: maxD(.Box, 2)]
  concat sort
  nth[Fraction, TRUE] head[NRows - 1]
  consume;

/*
8 Create Stripes

Create the first (lower) stripe. The bottom of the first stripe is the bottom of the computed rectangle in section 3, top-value is the first value of ~VPartition~.

*/

let Stripe1 = rectangle2(Left, Right, Bottom, VPartition feed extract[Y])

/*
Create the remaining stripes.
Operator extendlast: Computes new attributes from the current tuple and the previous one in the stream. 

*/

let Stripes = VPartition feed 
  extend_last[Stripe: rectangle2(Left, Right, ..Y, .Y)::Stripe1]
  consume

/*
Create last stripe and insert into relation ~Stripes~.

*/

let LastY = VPartition feed tail[1] extract[Y]

query Stripes inserttuple[LastY, rectangle2(Left, Right, LastY, Top)] consume

/*
9 Create Fraction2

*/

let Fraction2 = (2 * (Sample count)) div (NRows * NColumns)

/*
10 Create Fields

Each sample is assigned to its corresponding stripe. After that, with Fraction2 the vertical borders are computed.

*/

let Fields = Stripes feed Sample feed itSpatialJoin[Stripe, Box] 
  sortby[Stripe]
  nest[Stripe; Boxes]
  extend[Bounds: fun(t: TUPLE)
    attr(t, Boxes) afeed projectextend[; X: minD(.Box, 1)] 
    attr(t, Boxes) afeed projectextend[; X: maxD(.Box, 1)]
    concat sort 
    nth[Fraction2, TRUE] head[NColumns - 1]
    aconsume]
  remove[Boxes]
  extend[LeftField: rectangle2(minD(.Stripe, 1), .Bounds afeed extract[X], 
    minD(.Stripe, 2), maxD(.Stripe, 2))]
  extend[RightField: rectangle2(.Bounds afeed tail[1] extract[X], 
    maxD(.Stripe, 1), minD(.Stripe, 2), maxD(.Stripe, 2))]
  extend[FieldsX: fun(t2: TUPLE)
    attr(t2, LeftField) feed namedtransformstream[Field]
    attr(t2, Bounds) afeed 
      extend_last[Field: rectangle2(..X, .X, minD(attr(t2, Stripe), 2), 
        maxD(attr(t2, Stripe), 2))::
        [const rect value undef]]
      filter[isdefined(.Field)] remove[X] concat
    attr(t2, RightField) feed namedtransformstream[Field] concat
    aconsume]
  project[FieldsX]
  unnest[FieldsX]
  addcounter[N, 0]
  consume

/*
Extend elements withs their MBR

*/

let BuildingsEx = Buildings feed extend[Box: bbox(.GeoData)] consume

let RailwaysEx = Railways feed extend[Box: bbox(.GeoData)] consume

let WaterwaysEx = Waterways feed extend[Box: bbox(.GeoData)] consume

/*
11 Assign elements to corresponding fields

*/

let BuildingsField = BuildingsEx feed Fields feed 
itSpatialJoin[Box, Field] 
consume

let RailwaysField = RailwaysEx feed Fields feed  
itSpatialJoin[Box, Field] 
consume

let WaterwaysField = WaterwaysEx feed Fields feed
itSpatialJoin[Box, Field] 
consume

/*
12 Distribute relations 
Create one slot for each field.

*/

let SizeSlots = Fields feed max[N]

let BuildingsB1 = BuildingsField feed 
dfdistribute2["BuildingsB1", N, SizeSlots, Worker]

let RailwaysB1 = RailwaysField feed 
dfdistribute2["RailwaysB1", N, SizeSlots, Worker]

let WaterwaysB1 = WaterwaysField feed 
dfdistribute2["WaterwaysB1", N, SizeSlots, Worker]

/*
12 Evaluation and histogram

This section is for evaluation of the grid.

*/

#let Eval = BuildingsField feed sortby[N] groupby[N; Cnt:group count] consume

#number of used cells
#query BuildingsField feed sortby[N] max[N]
#query Eval feed max[N]

#number of duplicates
#let duplicates = BuildingsField count - Buildings count
#query duplicates

#smallest number of elements assigned to a cell
#query BuildingsField feed sortby[N] groupby[N; Cnt:group count] min[Cnt]
#query Eval feed min[Cnt]

#highest number of elements assigned to a cell
#query BuildingsField feed sortby[N] groupby[N; Cnt:group count] max[Cnt]
#query Eval feed max[Cnt]

#average number of elements assigned to a cell
#query BuildingsField feed sortby[N] groupby[N; Cnt:group count] avg[Cnt]
#query Eval feed avg[Cnt]

/*
Create a histogram over groupsizes

*/

#let Dia = realstream(0.0, 0.0 + Eval count, 1.0)
#set_histogram1d

#query Eval feed projectextendstream[; P : fun( t : TUPLE) 
#intstream(0,attr(t,Cnt)) replaceElem[attr(t,N)  + 0.0] ] 
#create_histogram1d[P, Dia]