along/secondo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
dfd1e745480fabd88139d2804acb90d5b6dc055e
secondo/Optimizer/Costs2019/cConst.psec
along dfd1e74548 firs commit
2026-01-23 17:03:45 +08:00

2700 lines
78 KiB
Plaintext
Raw Blame History

/*
----
This file is part of SECONDO.
Copyright (C) 2004, University in Hagen, Department of Computer Science,
Database Systems for New Applications.
SECONDO is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
SECONDO is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with SECONDO; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
----
//paragraph [10] title: [{\Large \bf ] [}]
//characters [1] formula: [$] [$]
//[toc] [\tableofcontents]
//[newpage] [\newpage]
[10] Automatic Evaluation of Cost Constants - cConst.psec
[toc]
[newpage]
1 Evaluating Cost Constants
The cost constants will be evaluated with the datasets obtained from
http://download.geofabrik.de/europe/germany/nordrhein-westfalen-latest-free.shp.zip.
These must be imported first e.g. through the script ``nrwImportShapeNew.psec`` into the database nrw.
This script is designed to use the database nrw where the relation Cconst will be created to store the constants.
Each query will be executed three times, then the results(avg ElapsedTime) of these three executions will be transferred to the relation Cconst.
If necessary, further calculations to determine the constants will be executed.
When the constants are fully determined they will be written in the costs2014.pl file at the end of the script.
In the relation Cconst all queries that are just auxiliary ones are stored with IDs higher than 100.
This means that all IDs under 100 are the ones where the constants are stored.
The relation that is used throughout the script is Roads, with variations Roads20 and Roads60. These contain all roads with Maxspeed = 20 and 60 respectively.
Also the relation Waterways and Buildings are used.
If a query has a ratio between ElapsedTime and CpuTime that is greater than 1.5 it will be ignored for evaluating the runtime of this query.
The script must be run without error detection, because the commentary produces errors in SECONDO.
Preparations:
download and import nrw OSMdata.
Delete the existing constants in costs2014
schema of the relation that is created to store the queries and resulting constants:
----
Cconst [ ID:int Query:string ElapsedTime:real CPUTime:real
NoResTuple:int NoCompTuple:int Constant:real ]
----
1.1 Basic constants
First a few basic objects are defined, such as the new-line object for separating the constants while writing them back to costs2014.pl.
The ratio is defined and the relation Cconst itself.
A timestamp is directly written to costs2014.pl to mark the start of the constant evaluation.
*/
close database;
open database nrw;
delete Cconst;
# Definition of relation Cconst for storing the evaluated constants.
let Cconst = [const rel(tuple([
Id: string,
Query: text,
ElapsedTime: real,
CPUTime:real,
NoResTuple:int,
NoCompTuple:int,
Constant:real]))
value () ];
let nl = "
";
let ratio = 1.4;
let c1 = "----------------------------------";
let c2 = "Automatic Cost Constant Evaluation";
let c3 = "Started at: " + attr2text(now());
query writeFile(nl+nl+"/*"+nl+c1+c2+c1+nl+c3+nl+
nl+"*/"+nl+nl, 'costs2014.pl', TRUE);
delete c1;
delete c2;
delete c3;
/*
1.1.1 Feedproject
For the evaluation of the feedproject predicate we need to determine three constants, which are defined ``per tuple``, ``per attribute`` and ``per byte``.
Evaluation ``per attribute`` is done through three queries each with a different number of attributes (one, two and three).
Ratio is ignored and CpuTime instead of ElapsedTime is used, as tests showed that when given attributes are projected the runtime seems to be randomly off.
So, to get reliable values the CpuTime is used.
----
query Roads feedproject[Code] count
query Roads feedproject[Code,Layer] count
query Roads feedproject[Code,Layer,Maxspeed] count
----
*/
query Roads feedproject[Code] count;
query Roads feedproject[Code] count;
query Roads feedproject[Code] count;
query Cconst inserttuple[
"101",
'query Roads feedproject[Code] count',
SEC2COMMANDS feed tail[3] avg[ElapsedTime],
SEC2COMMANDS feed tail[3] avg[CpuTime],
Roads feedproject[Code] count,
Roads feedproject[Code] count,
0.0] consume;
query Roads feedproject[Code,Layer] count;
query Roads feedproject[Code,Layer] count;
query Roads feedproject[Code,Layer] count;
query Cconst inserttuple[
"102",
'query Roads feedproject[Code,Layer] count',
SEC2COMMANDS feed tail[3] avg[ElapsedTime],
SEC2COMMANDS feed tail[3] avg[CpuTime],
Roads feedproject[Code] count,
Roads feedproject[Code] count,
0.0] consume;
query Roads feedproject[Code,Layer,Maxspeed] count;
query Roads feedproject[Code,Layer,Maxspeed] count;
query Roads feedproject[Code,Layer,Maxspeed] count;
query Cconst inserttuple[
"103",
'query Roads feedproject[Code,Layer,Maxspeed] count',
SEC2COMMANDS feed tail[3] avg[ElapsedTime],
SEC2COMMANDS feed tail[3] avg[CpuTime],
Roads feedproject[Code] count,
Roads feedproject[Code] count,
0.0] consume;
/*
The query is evaluated with one, two or three attributes respectively.
The attributes are chosen with the same datatype and roughly the same size.
Now the gaps between one and two attributes and two and three attributes will be determined.
After that, the average of these two attributes defines the constant "per attribute".
*/
query Cconst inserttuple[
"10",
'query Roads feedproject[X] count (Per Attribute)',
0.0,
0.0,
Roads feedproject[Code] count,
Roads feedproject[Code] count,
round
(
(
(
(
(
(Cconst feed filter[(.Id = "102")] sum[CPUTime]
- Cconst feed filter[(.Id = "101")] sum[CPUTime])
+
(Cconst feed filter[(.Id = "103")] sum[CPUTime]
- Cconst feed filter[(.Id = "102")] sum[CPUTime])
)
/2)
/ Roads feedproject[Code] count)
* 1000)
,6)
]consume;
/*
Evaluation ``per Tuple``
simply substracts the costs for one attribute from the overall runtime divided through the number of tuples.
----
query Roads feedproject[Maxspeed] count
----
*/
query Roads feedproject[Maxspeed] count;
query Roads feedproject[Maxspeed] count;
query Roads feedproject[Maxspeed] count;
query Cconst inserttuple[
"11",
'query Roads feedproject[Maxspeed] count (per Tuple)',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
Roads feedproject[Code] count,
Roads feedproject[Code] count,
(round
(
(
(
(SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime])
/ Roads feedproject[Code] count
)
*1000
)
,6)
- (Cconst feed filter[(.Id = "10")] sum [Constant]))
] consume;
/*
Evaluation ``per byte'':
to evaluate this factor the script queries the relation Roads with two different attributes and subtracts the runtime as well as their sizes to determine the constant.
----
query Roads feedproject [Maxspeed] count
query Roads feedproject [GeoData] count
----
*/
query Roads feedproject [Maxspeed] count;
query Roads feedproject [Maxspeed] count;
query Roads feedproject [Maxspeed] count;
query Cconst inserttuple[
``104",
'query Roads feedproject [Maxspeed] count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
Roads feedproject[Code] count,
Roads feedproject[Code] count,
0.0] consume;
query Roads feedproject [GeoData] count;
query Roads feedproject [GeoData] count;
query Roads feedproject [GeoData] count;
query Cconst inserttuple[
"105",
'query Roads feedproject [GeoData] count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
Roads feedproject[Code] count,
Roads feedproject[Code] count,
0.0] consume;
query Cconst inserttuple[
"12",
'query Roads feedproject [X] count (per Byte)',
0.0,
0.0,
Roads feedproject[Code] count,
Roads feedproject[Code] count,
round
(
(
(
(Cconst feed filter[(.Id = "105")] sum[ElapsedTime]
- Cconst feed filter[(.Id = "104")] sum[ElapsedTime])
/
((Roads feed extend[Memory: .GeoData memattrsize] avg[Memory]
- Roads feed extend[Memory: .Maxspeed memattrsize] avg[Memory])
+ 136)
)
/ Roads feedproject[Code] count
)*1000
,6)]consume;
/*
1.1.2 Feed
The relation Road is used again.
Three attributes with the same datatype and size are selected and the differences determine the ``per attribute`` constant.
We also need to determine the constant ``per tuple``.
Definition ``per attribute`` is, as in feedproject, done using CpuTime.
----
query Roads feed project [Tunnel] count
query Roads feed
project [Tunnel, Bridge, Oneway, Ref, Fclass] count
----
*/
query Roads feed project [Tunnel] count;
query Roads feed project [Tunnel] count;
query Roads feed project [Tunnel] count;
query Cconst inserttuple[
"106",
'query Roads feed project [Tunnel] count',
SEC2COMMANDS feed tail[3] avg[ElapsedTime],
SEC2COMMANDS feed tail[3] avg[CpuTime],
Roads feedproject[Code] count,
Roads feedproject[Code] count,
0.0] consume;
query Roads feed
project [Tunnel, Bridge, Oneway, Ref, Fclass] count;
query Roads feed
project [Tunnel, Bridge, Oneway, Ref, Fclass] count;
query Roads feed
project [Tunnel, Bridge, Oneway, Ref, Fclass] count;
query Cconst inserttuple[
"107",
'query Roads feed
project [Tunnel, Bridge, Oneway, Ref, Fclass] count',
SEC2COMMANDS feed tail[3] avg[ElapsedTime],
SEC2COMMANDS feed tail[3] avg[CpuTime],
Roads feedproject[Code] count,
Roads feedproject[Code] count,
0.0] consume;
query Cconst inserttuple[
"13",
'query Roads feed project [X] count (per Attribute)',
0.0,
0.0,
Roads feedproject[Code] count,
Roads feedproject[Code] count,
round
(
(
(
((Cconst feed filter[(.Id = "107")] sum[CPUTime]
- Cconst feed filter[(.Id = "106")] sum[CPUTime])/4)
/ Roads feedproject[Code] count
)* 1000
)
,6)]consume;
/*
For the ``per tuple`` constant one query takes 100000 tuples and another one 1000000 tuples.
The time difference between these queries is then divided by the tuples difference (900000).
----
query Roads feed head[100000] count
query Roads feed head[1000000] count
----
*/
query Roads feed head[100000] count;
query Roads feed head[100000] count;
query Roads feed head[100000] count;
query Cconst inserttuple[
"108",
'query Roads feed head[100000] count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
100000,
100000,
0.0] consume;
query Roads feed head[1000000] count;
query Roads feed head[1000000] count;
query Roads feed head[1000000] count;
query Cconst inserttuple[
"109",
'query Roads feed head[1000000] count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
1000000,
1000000,
0.0] consume;
query Cconst inserttuple[
"14",
'query Roads feed head [X] count (per Tuple)',
0.0,
0.0,
0,
0,
round
(
round(
(
(Cconst feed filter[(.Id = "109")] sum[ElapsedTime]
- Cconst feed filter[(.Id = "108")] sum[ElapsedTime])
/900000),6)
* 1000,6)]consume;
/*
1.1.3 Consume
A simple relation is created.
Important is that the let operator generates two queries. One is for the creation of the new relation and another one is updating infos of that relation.
So the last six entries in SEC2COMMANDS must be retrieved and then filtered on the queries that contain the keyword 'let'.
----
let x = Waterways feed consume
----
*/
query Waterways feed count;
query Waterways feed count;
query Waterways feed count;
query Cconst inserttuple[
"110",
'query Waterways feed count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
Waterways feed count,
Waterways feed count,
0.0] consume;
let x = Waterways feed consume;
let y = Waterways feed consume;
let z = Waterways feed consume;
query Cconst inserttuple[
"111",
'let x = Waterways feed consume',
SEC2COMMANDS feed tail[6]
filter[(.CmdStr contains "let ")] avg[ElapsedTime],
SEC2COMMANDS feed tail[6]
filter[(.CmdStr contains "let ")] avg[CpuTime],
Waterways feed count,
Waterways feed count,
0.0] consume;
delete x;
delete y;
delete z;
query Cconst inserttuple[
"15",
'let x = Waterways feed consume',
0.0,
0.0,
Waterways feed count,
Waterways feed count,
round
(
(
(
(Cconst feed filter[(.Id = "111")] sum[ElapsedTime]
- Cconst feed filter[(.Id = "110")] sum[ElapsedTime])
/Waterways feed count
)
* 1000)
,6)]consume;
/*
1.1.4 Attribute Constant
This constant is used throughout all join operators. It is used to determine the cost for producing the results per tuple and attribute.
Two queries will be evaluated with two and four attributes respectively. The first query has two attributes and produces 10 million result tuples.
The second query has four attributes and produces also 10 million result tuples.
The difference between these two queries, devided by 2 and then divided by 10 Mio, is the constant per attribute for each result tuple.
In the script the runtime is divided by 20000 because 10 Mio tuples for 2 additional attributes would be a factor of 20000000.
After that the result is multiplied by 1000 to convert the time to milliseconds.
So, ((Runtime Difference total) / 20000000) * 1000 is the same as ((Runtime Difference total)/ 20000).
----
query Roads feedproject [Maxspeed] {r}
Roads feedproject [Maxspeed] {w}
hashjoin[Maxspeed_r, Maxspeed_w] head[10000000] count
query Roads feedproject [Maxspeed,Layer] {r}
Roads feedproject [Maxspeed, Layer] {w}
hashjoin[Maxspeed_r, Maxspeed_w] head[10000000] count
----
*/
query Roads feedproject [Maxspeed] {r} Roads feedproject [Maxspeed] {w}
hashjoin[Maxspeed_r, Maxspeed_w] head[10000000] count;
query Roads feedproject [Maxspeed] {r} Roads feedproject [Maxspeed] {w}
hashjoin[Maxspeed_r, Maxspeed_w] head[10000000] count;
query Roads feedproject [Maxspeed] {r} Roads feedproject [Maxspeed] {w}
hashjoin[Maxspeed_r, Maxspeed_w] head[10000000] count;
query Cconst inserttuple[
"112",
'query Roads feedproject [Maxspeed] {r} Roads feedproject [Maxspeed] {w}
hashjoin[Maxspeed_r, Maxspeed_w] head[10000000] count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
10000000,
10000000,
0.0] consume;
query Roads feedproject [Maxspeed,Layer] {r}
Roads feedproject [Maxspeed, Layer] {w}
hashjoin[Maxspeed_r, Maxspeed_w] head[10000000] count;
query Roads feedproject [Maxspeed,Layer] {r}
Roads feedproject [Maxspeed, Layer] {w}
hashjoin[Maxspeed_r, Maxspeed_w] head[10000000] count;
query Roads feedproject [Maxspeed,Layer] {r}
Roads feedproject [Maxspeed, Layer] {w}
hashjoin[Maxspeed_r, Maxspeed_w] head[10000000] count;
query Cconst inserttuple[
"113",
'query Roads feedproject [Maxspeed, Layer] {r}
Roads feedproject [Maxspeed,Layer] {w}
hashjoin[Maxspeed_r, Maxspeed_w] head[10000000] count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
10000000,
10000000,
0.0] consume;
query Cconst inserttuple[
"16",
'query Roads feedproject [X,Y,...] {r} Roads feed project [X,Y,...] {w}
hashjoin[X_r, X_w] head[10000000] count',
0.0,
0.0,
0,
0,
round
(
(
((Cconst feed filter[(.Id = "113")] sum[ElapsedTime])
- (Cconst feed filter[(.Id = "112")] sum[ElapsedTime]))
/ 20000
)
,6)]consume;
/*
1.1.5 Filter
The filter is evaluated by a query with and without a filter predicate. The time difference is the constant for the filter predicate.
Roads is used again.
The cost function contains a threshold value of 0.2 ms, corresponding to the processing of 2000 tuples (the standard sample size) within 400 ms.
This is evaluated in the original costs2014.pl, using a test with an expensive query to determine this threshold (GeoData intersects X).
This threshold is left untouched by the script as it is built in the cost function directly.
----
query Roads feed filter [(.Maxspeed > 50)] count
query Roads feed count
----
*/
query Roads feed filter [(.Maxspeed > 50)] count;
query Roads feed filter [(.Maxspeed > 50)] count;
query Roads feed filter [(.Maxspeed > 50)] count;
query Cconst inserttuple[
"114",
'query Roads feed filter [(.Maxspeed > 50)] count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
Roads feed filter [(.Maxspeed > 50)] count,
Roads feedproject[Code] count,
0.0] consume;
query Roads feed count;
query Roads feed count;
query Roads feed count;
query Cconst inserttuple[
"115",
'query Roads feed count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
Roads feedproject[Code] count,
Roads feedproject[Code] count,
0.0] consume;
query Cconst inserttuple[
"17",
'query Roads feed filter [(.X,.Y,...)] count',
0.0,
0.0,
0,
0,
round
(
(
(
(Cconst feed filter[(.Id = "114")] sum[ElapsedTime]
- Cconst feed filter[(.Id = "115")] sum[ElapsedTime])
/ Roads feedproject[Code] count
)
* 1000)
,6)]consume;
/*
1.1.6 Rename
The costs are linear in the number of tuples, but do not depend on the number of attributes or tuple size as just one pointer for the tuple is passed to the next operator.
To determine the costs one query is executed without and one with a rename.
The query number 115 (Roads feed count) has already been executed and will be used here to define the running time without rename.
----
query Roads feed {r} count
----
*/
query Roads feed {r} count;
query Roads feed {r} count;
query Roads feed {r} count;
query Cconst inserttuple[
"116",
'query Roads feed {r} count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
Roads feedproject[Code] count,
Roads feedproject[Code] count,
0.0] consume;
query Cconst inserttuple[
"18",
'query Roads feed {r} count',
0.0,
0.0,
0,
0,
round
(
(
(
(Cconst feed filter[(.Id = "116")] sum[CPUTime]
- Cconst feed filter[(.Id = "115")] sum[ElapsedTime])
/ Roads feedproject[Code] count
)
* 1000)
,6)]consume;
/*
1.1.7 Extend
The cost function is extended by a second constant which is ``per attribute``. These constant factors for the costs increase if more than one attribute is extended.
The cost constants that will be determined are the cost per tuple and the cost per attribute.
``per Tuple`` is a simple query on the relation Roads with one extended attribute.
``per Attribute`` is the difference between the runtime with two and three extended attributes respectively (usage of CpuTime).
Again query 115 (Roads feed count) is used as a reference for the base time of query Roads feed count.
----
query Roads feed extend[Code2: .Code*10] count
query Roads feed extend[Code2: .Code*10, Code3: .Code*100] count
query Roads feed extend[Code2: .Code*10, Code3: .Code*100, Code4: .Code*1000] count
----
*/
query Roads feed extend[Code2: .Code*10] count;
query Roads feed extend[Code2: .Code*10] count;
query Roads feed extend[Code2: .Code*10] count;
query Cconst inserttuple[
"117",
'query Roads feed extend[Code2: .Code*10] count',
SEC2COMMANDS feed tail[3] avg[ElapsedTime],
SEC2COMMANDS feed tail[3] avg[CpuTime],
Roads feedproject[Code] count,
Roads feedproject[Code] count,
0.0] consume;
query Cconst inserttuple[
"19",
'query Roads feed extend[.....] count (per Tuple)',
0.0,
0.0,
0,
0,
round
(
(
(
(Cconst feed filter[(.Id = "117")] sum[ElapsedTime]
- Cconst feed filter[(.Id = "115")] sum[ElapsedTime])
/ Roads feedproject[Code] count
)
* 1000)
,6)]consume;
query Roads feed extend[Code2: .Code*10, Code3: .Code*100] count;
query Roads feed extend[Code2: .Code*10, Code3: .Code*100] count;
query Roads feed extend[Code2: .Code*10, Code3: .Code*100] count;
query Cconst inserttuple[
"118",
'query Roads feed extend[Code2: .Code*10, Code3: .Code*100] count',
SEC2COMMANDS feed tail[3] avg[ElapsedTime],
SEC2COMMANDS feed tail[3] avg[CpuTime],
Roads feedproject[Code] count,
Roads feedproject[Code] count,
0.0] consume;
query Roads feed
extend[Code2: .Code*10, Code3: .Code*100, Code4: .Code*1000] count;
query Roads feed
extend[Code2: .Code*10, Code3: .Code*100, Code4: .Code*1000] count;
query Roads feed
extend[Code2: .Code*10, Code3: .Code*100, Code4: .Code*1000] count;
query Cconst inserttuple[
"119",
'query Roads feed
extend[Code2: .Code*10, Code3: .Code*100, Code4: .Code*1000] count',
SEC2COMMANDS feed tail[3] avg[ElapsedTime],
SEC2COMMANDS feed tail[3] avg[CpuTime],
Roads feedproject[Code] count,
Roads feedproject[Code] count,
0.0] consume;
query Cconst inserttuple[
"20",
'query Roads feed extend[.....] count (per Attr)',
0.0,
0.0,
0,
0,
round
(
(
(
(Cconst feed filter[(.Id = "119")] sum[CPUTime]
- Cconst feed filter[(.Id = "118")] sum[CPUTime])
/ Roads feedproject[Code] count
)
* 1000)
,6)]consume;
/*
1.1.8 Remove
Here one attribute is removed from the stream through a simple query on the relation Roads.
----
query Roads feed remove[Code] count
----
*/
query Roads feed remove[Code] count;
query Roads feed remove[Code] count;
query Roads feed remove[Code] count;
query Cconst inserttuple[
"120",
'query Roads feed remove[Code] count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
Roads feedproject[Code] count,
Roads feedproject[Code] count,
0.0] consume;
query Cconst inserttuple[
"21",
'query Roads feed remove[.....] count',
0.0,
0.0,
0,
0,
round
(
(
(Cconst feed filter[(.Id = "120")] sum[ElapsedTime]
- Cconst feed filter[(.Id = "115")] sum[ElapsedTime])
/ Roads feedproject[Code] count
)
* 1000,6)]consume;
/*
1.1.9 Project
Project is similiar to the evaluation of remove.
Query 115 is used again to determine the time for Roads feed count.
---- query Roads feed project[Code] count
----
*/
query Roads feed project[Code] count;
query Roads feed project[Code] count;
query Roads feed project[Code] count;
query Cconst inserttuple[
"121",
'query Roads feed project[Code] count',
SEC2COMMANDS feed tail[3] avg[ElapsedTime],
SEC2COMMANDS feed tail[3] avg[CpuTime],
Roads feedproject[Code] count,
Roads feedproject[Code] count,
0.0] consume;
query Cconst inserttuple[
"22",
'query Roads feed project[...] count',
0.0,
0.0,
0,
0,
round
(
(
(Cconst feed filter[(.Id = "121")] sum[CPUTime]
- Cconst feed filter[(.Id = "115")] sum[CPUTime])
/ Roads feedproject[Code] count
)
* 1000,6)]consume;
/*
1.1.10 Product
The cartesian product of its two input streams, hashes the second stream and performs the product then.
It contains of two variables, hashing and joining.
The constant is evaluated with the relation Roads as the second stream (for the hashing) and an empty relation which is called Leer.
Two queries, one with 100k tuples and one with 600k tules, are executed. The time difference divided by 500k additional tuples is the constant per (hash-)tuple.
For the product cost itself a query with two tuples from Roads is joined with 600k tuples. The join then contains 1.2 million tuples.
The runtime of that query is reduced by the hashphase so that the product cost is isolated.
----
query Leer feed {w} Roads feed head[100000] product count
query Leer feed {w} Roads feed head[600000] product count
query Roads feed head[2] {w} Roads feed head[600000] product count
----
*/
let Leer = Roads feed filter[(.Code = 1)] consume;
query Leer feed {w} Roads feed head[100000] product count;
query Leer feed {w} Roads feed head[100000] product count;
query Leer feed {w} Roads feed head[100000] product count;
query Cconst inserttuple[
"122",
'query Leer feed {w} Roads feed head[100000] product count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
100000,
100000,
0.0] consume;
query Leer feed {w} Roads feed head[600000] product count;
query Leer feed {w} Roads feed head[600000] product count;
query Leer feed {w} Roads feed head[600000] product count;
query Cconst inserttuple[
"123",
'query Leer feed {w} Roads feed head[600000] product count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
600000,
0,
0.0] consume;
query Cconst inserttuple[
"23",
'query Leer feed {w} Roads feed product count (per Hashtupel)',
0.0,
0.0,
0,
0,
round
(
(
(
((Cconst feed filter[(.Id = "123")] sum[ElapsedTime])
- (Cconst feed filter[(.Id = "122")] sum[ElapsedTime]))
/ 500000
)
* 1000)
,6)]consume;
query Roads feed head[2] {w} Roads feed head[600000] product count;
query Roads feed head[2] {w} Roads feed head[600000] product count;
query Roads feed head[2] {w} Roads feed head[600000] product count;
query Cconst inserttuple[
"124",
'query Leer feed head[2] {w} Roads feed head[600000] product count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
1200000,
1200000,
0.0] consume;
query Cconst inserttuple[
"24",
'query Roads feed {w} Roads feed product count (per Resulttuple)',
0.0,
0.0,
0,
0,
round
(
(
((Cconst feed filter[(.Id = "124")] sum[ElapsedTime]
- Cconst feed filter[(.Id = "123")] sum[ElapsedTime])
/ 1200000)
* 1000)
,6)]consume;
/*
1.1.11 Ksmallest
The costs contain a linear part which grows in the size of the relation.
The second part growths exponential with number of tuples that are used by the function.
First constant is evaluated through the relation Roads and the operator ksmallest limited to one Tuple, this ensures that the costs are near the basic cost of this function.
The second constant is then evaluated using a query that contains 800.000 and 1.200.000 tuples.
The runtime is then reduced by the feed...count part and the basic cost from ksmallest.
After that, the remaining time will be divided by the Term (Size x lognl(Size+1)) x 1000.
The mean of the 2 queries should then factor the growth per tuple.
----
query Roads feed ksmallest[1;Code] count
query Roads feed ksmallest[800000;Code] count
query Roads feed ksmallest[1200000;Code] count
----
*/
query Roads feed ksmallest[1;Code] count;
query Roads feed ksmallest[1;Code] count;
query Roads feed ksmallest[1;Code] count;
query Cconst inserttuple[
"125",
'query Roads feed ksmallest[1;Code] count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
Roads feedproject[Code] count,
1,
0.0] consume;
query Cconst inserttuple[
"25",
'query Roads feed ksmallest[k;Code] count (BasicCost)',
0.0,
0.0,
0,
0,
round
(
(
(((Cconst feed filter[(.Id = "125")] sum[ElapsedTime])
- Cconst feed filter[(.Id = "115")] sum[ElapsedTime])
/ Roads feedproject[Code] count)
* 1000)
,6)]consume;
query Roads feed ksmallest[800000;Code] count;
query Roads feed ksmallest[800000;Code] count;
query Roads feed ksmallest[800000;Code] count;
query Cconst inserttuple[
"126",
'query Roads feed ksmallest[800000;Code] count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
Roads feedproject[Code] count,
800000,
0.0] consume;
query Roads feed ksmallest[1200000;Code] count;
query Roads feed ksmallest[1200000;Code] count;
query Roads feed ksmallest[1200000;Code] count;
query Cconst inserttuple[
"127",
'query Roads feed ksmallest[1200000;Code] count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
Roads feedproject[Code] count,
1200000,
0.0] consume;
query Cconst inserttuple[
"128",
'query Roads feed ksmallest[800000;Code] count (Tuplecost Exp)',
0.0,
0.0,
0,
0,
round
(
(
((Cconst feed filter[(.Id = "126")] sum[ElapsedTime])
- (Cconst feed filter[(.Id = "125")] sum[ElapsedTime]))
/ (800000 * log(800000))
)* 1000,6)]consume;
query Cconst inserttuple[
"129",
'query Roads feed ksmallest[1200000;Code] count (Tuplecost Exp)',
0.0,
0.0,
0,
0,
round
(
(
((Cconst feed filter[(.Id = "127")] sum[ElapsedTime])
- (Cconst feed filter[(.Id = "125")] sum[ElapsedTime]))
/
(1200000 * log(1200000))
)* 1000,6)]consume;
query Cconst inserttuple[
"26",
'query Roads feed ksmallest[k;Code] count (Tuplecost Exp)',
0.0,
0.0,
0,
0,
((Cconst feed filter[(.Id = "128")] sum[ElapsedTime])
+ (Cconst feed filter[(.Id = "122")] sum[ElapsedTime]))
/ 2
]consume;
/*
1.2 Index Constants
1.2.1 CreateBTree
Special handling for distance queries which have to create a temporary index.
----
let Roads_Code = Roads createbtree[Code]
----
*/
let Roads_Code = Roads createbtree[Code];
let Roads_Code2 = Roads createbtree[Code];
let Roads_Code3 = Roads createbtree[Code];
query Cconst inserttuple[
"130",
'let Roads_Code = Roads createbtree[Code]',
SEC2COMMANDS feed tail[6]
filter[(.CmdStr contains "let ")] avg[ElapsedTime],
SEC2COMMANDS feed tail[6]
filter[(.CmdStr contains "let ")] avg[CpuTime],
Roads feedproject[Code] count,
Roads feedproject[Code] count,
0.0] consume;
query Cconst inserttuple[
"27",
'let Roads_Code = Roads createbtree[Code]',
0.0,
0.0,
0,
0,
round
(
(
(
(Cconst feed filter[(.Id = "130")] sum[ElapsedTime]
- Cconst feed filter[(.Id = "115")] sum[ElapsedTime])
/ (Roads feedproject[Code] count)
)
* 1000)
,6)]consume;
delete Roads_Code;
delete Roads_Code2;
delete Roads_Code3;
/*
1.2.2 Exactmatch
The constant is linear in the size of the result. It is evaluated through a rather simplistic query which uses the relation Roads again and filters all roads named Heideweg.
This query was originally designed to query all roads with name Hauptstra{\ss}e but execute the PDview correctly and for generating a PDF file of this script, it is needed to replace special German characters by latex symbols.
The queries would not be executed with these symbols. Consequently a compromise is made to query a name that contains none of such special characters.
As of 2019 the name that is most common in NRW, without special chars, is Heideweg.
----
query Roads_Name_btree Roads exactmatch['Heideweg'] count
----
*/
query Roads_Name_btree Roads exactmatch['Heideweg'] count;
query Roads_Name_btree Roads exactmatch['Heideweg'] count;
query Roads_Name_btree Roads exactmatch['Heideweg'] count;
query Cconst inserttuple[
"131",
'query Roads_Name_btree Roads exactmatch["Heideweg"] count',
SEC2COMMANDS feed tail[3] avg[ElapsedTime],
SEC2COMMANDS feed tail[3] avg[CpuTime],
Roads_Name_btree Roads exactmatch['Heideweg'] count,
Roads_Name_btree Roads exactmatch['Heideweg'] count,
0.0] consume;
query Cconst inserttuple[
"28",
'query Roads_Name_btree Roads exactmatch["xxx"] count',
0.0,
0.0,
0,
0,
round
(
(
(Cconst feed filter[(.Id = "131")] sum[ElapsedTime]
/ Roads feedproject[Code] count)
* 1000)
,6)]consume;
/*
1.2.3 Windowintersects
Same cost estimation as in exactmatch except that here R-Trees are used. The costs are linear to the arguments size.
An object eichlinghofen is created that represents a district of the city dortmund.
Then the relation Buildings (which has n R-tree index) is used to determine the constant.
----
let eichlinghofen = [const region value (
(
(
(7.419515247680575 51.47332155746125)
(7.394967670776298 51.47332155746125)
(7.394967670776298 51.48716614802665)
(7.419515247680575 51.48716614802665)))) ]
query Buildings_GeoData_rtree Buildings
windowintersects[bbox(eichlinghofen)]
filter[(.GeoData intersects eichlinghofen)] count
----
*/
let eichlinghofen = [const region value (
(
(
(7.419515247680575 51.47332155746125)
(7.394967670776298 51.47332155746125)
(7.394967670776298 51.48716614802665)
(7.419515247680575 51.48716614802665)))) ];
query Buildings_GeoData_rtree Buildings windowintersects[bbox(eichlinghofen)]
filter[(.GeoData intersects eichlinghofen)] count;
query Buildings_GeoData_rtree Buildings windowintersects[bbox(eichlinghofen)]
filter[(.GeoData intersects eichlinghofen)] count;
query Buildings_GeoData_rtree Buildings windowintersects[bbox(eichlinghofen)]
filter[(.GeoData intersects eichlinghofen)] count;
query Cconst inserttuple[
"132",
'query Buildings_GeoData_rtree Buildings windowintersects[bbox(eichlinghofen)]
filter[(.GeoData intersects eichlinghofen)] count',
SEC2COMMANDS feed tail[3] avg[ElapsedTime],
SEC2COMMANDS feed tail[3] avg[CpuTime],
Buildings_GeoData_rtree Buildings windowintersects[bbox(eichlinghofen)]
filter[(.GeoData intersects eichlinghofen)] count,
Buildings_GeoData_rtree Buildings windowintersects[bbox(eichlinghofen)]
filter[(.GeoData intersects eichlinghofen)] count,
0.0] consume;
query Cconst inserttuple[
"29",
'query Buildings_GeoData_rtree Buildings windowintersects[X]
filter[(.GeoData intersects X)] count',
0.0,
0.0,
0,
0,
round
(
(
(
(Cconst feed filter[(.Id = "132")] sum[ElapsedTime])
/
(Buildings feedproject[Code] count)
)
* 1000)
,6)]consume;
delete eichlinghofen;
/*
1.2.4 Leftrange
Searches in a given B-Tree all tuples that are left from the argument. The cost is linear so the constant will be determined using the B-Tree in relation Roads again.
Experiments showed that the range, left range and right range operators execute queries in almost identical runtime. So this constant is used for all three of them.
----
query Roads_Name_btree Roads leftrange['z'] count
----
*/
query Roads_Name_btree Roads leftrange['z'] count;
query Roads_Name_btree Roads leftrange['z'] count;
query Roads_Name_btree Roads leftrange['z'] count;
query Cconst inserttuple[
"133",
'query Roads_Name_btree Roads leftrange["z"] count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
Roads_Name_btree Roads leftrange['z'] count,
Roads_Name_btree Roads leftrange['z'] count,
0.0] consume;
query Cconst inserttuple[
"30",
'query Roads_Name_btree Roads leftrange["z"] count',
0.0,
0.0,
0,
0,
round
(
(
((Cconst feed filter[(.Id = "133")] sum[ElapsedTime])
/ (Roads feedproject[Code] count))
* 1000)
,6)]consume;
/*
1.3 Join Constants
1.3.1 Loopjoin
The costs are critically dependent on the buffer state of the system.
This means cold and warm states differ greatly in runtime. When an R-tree is used for the first time the buffer is in cold status, meaning it is empty and the R-tree must be loaded completely.
After the first load it resides in the buffer, which means a second query on that index will be much faster than the first one. This is the warm buffer state.
Examples have shown a factor by up to 10x in runtime.
For the cost evaluation it is very difficult to determine in which state the system is concerning the runtime of the query.
So a mix of warm and cold states is used. This will never estimate the true runtime but is more precisely then to assume an always cold or warm system.
This will be evaluated by a query that is running two times. The first one is cold and the second is in warm state.
The runtimes will be set in ratio and then divided by 2 to bring in the mix of warm and cold state.
This state-factor is a second constant that will be used in the query cost at the term (SizeX * CardY).
The base cost is left untouched and will be evaluated over an empty relation to isolate the basic cost constant.
The existing relation Leer is used again (from Section Product).
Here the ratio is ignored because of warm/cold buffer states.
----
query Roads feed head[5000] {r}
loopjoin[Buildings_GeoData_rtree Buildings windowintersects[.GeoData_r]
feed head[500000] {b}] filter[(.GeoData_r intersects .GeoData_b)]
count (cold)
query Roads feed head[5000] {r}
loopjoin[Buildings_GeoData_rtree Buildings windowintersects[.GeoData_r]
feed head[500000] {b}] filter[(.GeoData_r intersects .GeoData_b)]
count (warm)
----
*/
query Roads feed {r} loopjoin[Leer feed] count;
query Roads feed {r} loopjoin[Leer feed] count;
query Roads feed {r} loopjoin[Leer feed] count;
query Cconst inserttuple[
"134",
'query Roads feed {r} loopjoin[Leer feed] count',
SEC2COMMANDS feed tail[3] avg[ElapsedTime],
SEC2COMMANDS feed tail[3] avg[CpuTime],
Roads feedproject[Code] count,
Roads feedproject[Code] count,
0.0] consume;
query Cconst inserttuple[
"31",
'query Roads feed {r} loopjoin[...] count',
0.0,
0.0,
0,
0,
round
(
(
(
(Cconst feed filter[(.Id = "134")] sum[ElapsedTime]
- Cconst feed filter[(.Id = "115")] sum[ElapsedTime])
/ Roads feedproject[Code] count
)
* 1000)
,6)]consume;
query Roads feed head[5000] {r}
loopjoin[Buildings_GeoData_rtree Buildings windowintersects[.GeoData_r]
feed head[500000] {b}]
filter[(.GeoData_r intersects .GeoData_b)] count;
query Cconst inserttuple[
"135",
'query Roads feed head[5000] {r}
loopjoin[Buildings_GeoData_rtree Buildings windowintersects[.GeoData_r]
feed head[500000] {b}]
filter[(.GeoData_r intersects .GeoData_b)] count (Cold state)',
SEC2COMMANDS feed tail[1] avg[ElapsedTime],
SEC2COMMANDS feed tail[1] avg[CpuTime],
2500000000,
1,
0.0] consume;
query Roads feed head[5000] {r}
loopjoin[Buildings_GeoData_rtree Buildings windowintersects[.GeoData_r]
feed head[500000] {b}]
filter[(.GeoData_r intersects .GeoData_b)] count;
query Cconst inserttuple[
"136",
'query Roads feed head[5000] {r}
loopjoin[Buildings_GeoData_rtree Buildings windowintersects[.GeoData_r]
feed head[500000] {b}]
filter[(.GeoData_r intersects .GeoData_b)] count (Warm state)',
SEC2COMMANDS feed tail[1] avg[ElapsedTime],
SEC2COMMANDS feed tail[1] avg[CpuTime],
2500000000,
1,
0.0] consume;
query Cconst inserttuple[
"32",
'query Roads feed head[5000] {r}
loopjoin[Buildings_GeoData_rtree Buildings windowintersects[.GeoData_r]
feed head[500000] {b}]
filter[(.GeoData_r intersects .GeoData_b)] count (Warm state)',
0.0,
0.0,
0,
0,
round
(
(
(Cconst feed filter[(.Id = "135")] sum[ElapsedTime]
/ Cconst feed filter[(.Id = "136")] sum[ElapsedTime]) / 2
)
,6)]consume;
delete Leer;
/*
1.3.2 Symmjoin
This is mostly copied from the original version of costs2014.pl.
Like in the filter constant there is a threshold implemented to distinguish between a cheap and expensive predicate.
This threshold value is left untouched by the script and must be adjusted in the cost formula itself.
To determine the cost constant two new relations are defined.
Roads20 with all Roads that have a Maxspeed of 20 and Roads60 with Maxspeed 60.
These two are chosen because of the small amount of tuples and runtime. Both have around 3000 Tuples (as of 2019).
Experiments have shown that the cost is linear in size to the tuples.
The first query runs with 1000000 tuples, the second with 4000000.
The time difference between these two is the amount for 3000000 tuple in the symmjoin.
----
let Roads60 = Roads feed filter [(.Maxspeed = 60)] consume
let Roads20 = Roads feed filter [(.Maxspeed = 20)] consume
query Roads60 feed head [1000] {r60} Roads20 feed head[1000] {r20}
symmjoin[.Maxspeed_r60 = ..Maxspeed_r20] count
query Roads60 feed head [2000] {r60} Roads20 feed head[2000] {r20}
symmjoin[.Maxspeed_r60 = ..Maxspeed_r20] count
----
*/
let Roads60 = Roads feed filter [(.Maxspeed = 60)] consume;
let Roads20 = Roads feed filter [(.Maxspeed = 20)] consume;
query Roads60 feed head [1000] {r60} Roads20
feed head[1000] {r20} symmjoin[.Maxspeed_r60 = ..Maxspeed_r20] count;
query Roads60 feed head [1000] {r60} Roads20
feed head[1000] {r20} symmjoin[.Maxspeed_r60 = ..Maxspeed_r20] count;
query Roads60 feed head [1000] {r60} Roads20
feed head[1000] {r20} symmjoin[.Maxspeed_r60 = ..Maxspeed_r20] count;
query Cconst inserttuple[
"137",
'query Roads60 feed head [1000] {r60} Roads20
feed head[1000] {r20} symmjoin[.Maxspeed_r60 = ..Maxspeed_r20] count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
1000000,
0,
0.0] consume;
query Roads60 feed head [2000] {r60} Roads20
feed head[2000] {r20} symmjoin[.Maxspeed_r60 = ..Maxspeed_r20] count;
query Roads60 feed head [2000] {r60} Roads20
feed head[2000] {r20} symmjoin[.Maxspeed_r60 = ..Maxspeed_r20] count;
query Roads60 feed head [2000] {r60} Roads20
feed head[2000] {r20} symmjoin[.Maxspeed_r60 = ..Maxspeed_r20] count;
query Cconst inserttuple[
"138",
'query Roads60 feed head [2000] {r60} Roads20
feed head[2000] {r20} symmjoin[.Maxspeed_r60 = ..Maxspeed_r20] count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
4000000,
0,
0.0] consume;
query Cconst inserttuple[
"33",
'query Roads feed {r1} Roads feed {r2} symmjoin[...] count',
0.0,
0.0,
0,
0,
round
(
(
(
(Cconst feed filter[(.Id = "138")] sum[ElapsedTime]
- Cconst feed filter[(.Id = "137")] sum[ElapsedTime])
/ 3000000
)
* 1000)
,6)]consume;
delete Roads20;
delete Roads60;
/*
1.3.3 SortMergeJoin
The first query only sorts one stream by one argument. Its costs are linear to the size of the argument.
The second query is a join without producing a result. Time for this query contains the sorting of both relations as well as the probing phase.
The query is running 4 times to eliminate the cold buffer status effect.
----
query Roads feed sortby [Code] count
query Roads feed {r1} Roads feed {r2} sortmergejoin[Code_r1,Maxspeed_r2] count
----
*/
query Roads feed sortby [Code] count;
query Roads feed sortby [Code] count;
query Roads feed sortby [Code] count;
query Roads feed sortby [Code] count;
query Cconst inserttuple[
"139",
'query Roads feed sortby [Code] count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
Roads feedproject[Code] count,
Roads feedproject[Code] count,
0.0] consume;
query Cconst inserttuple[
"34",
'query Roads feed sortby [Code] count (Just sorting)',
0.0,
0.0,
0,
0,
round
(
(
(Cconst feed filter[(.Id = "139")] sum[ElapsedTime]
- Cconst feed filter[(.Id = "115")] sum[ElapsedTime])
/
(Roads feedproject[Code] count)
)
* 1000
,6)]consume;
query Roads feed {r1} Roads feed {r2} sortmergejoin[Code_r1,Maxspeed_r2] count;
query Roads feed {r1} Roads feed {r2} sortmergejoin[Code_r1,Maxspeed_r2] count;
query Roads feed {r1} Roads feed {r2} sortmergejoin[Code_r1,Maxspeed_r2] count;
query Cconst inserttuple[
"140",
'query Roads feed {r1} Roads feed {r2} sortmergejoin[Code_r1,Maxspeed_r2] count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
(Roads feedproject[Code] count)*2,
(Roads feedproject[Code] count)*2,
0.0] consume;
query Cconst inserttuple[
"35",
'query Roads feed {r1} Roads feed {r2}
sortmergejoin[Code_r1,Maxspeed_r2] count(Probing)',
0.0,
0.0,
0,
0,
round
(
(
(Cconst feed filter[(.Id = "140")] sum[ElapsedTime]
- (Cconst feed filter[(.Id = "139")] sum[ElapsedTime]*2)
)
/
(Roads feedproject[Code] count + Roads feedproject[Code] count)
)
* 1000,6)]consume;
/*
1.3.4 HashJoin
The hashjoin contains 3 phases.
The hash- and probephase as well as processing the result tuples.
The script evaluates the time for the hashphase and the probephase as the result is already determined through the attribute constant from chapter 1.1.4.
The first one is a query with joining but without producing results and thus without probing.
Time difference between the basic feedproject ... count and this query is the time for hashing the table.
The second query adds the probing phase but produces no results.
The costfunction only uses the probing constant as the hashing phase is ignored because experiments showed that this phase is always extremely shorter then probing.
----
query Roads feed {r1} Roads feed {r2}
hashjoin[Maxspeed_r1, Maxspeed_r2] head[1] count
query Roads feed {r1} Roads feed {r2}
hashjoin[Maxspeed_r1, Code_r2] count
----
*/
query Roads feedproject[Maxspeed] {r} count;
query Roads feedproject[Maxspeed] {r} count;
query Roads feedproject[Maxspeed] {r} count;
query Cconst inserttuple[
"141",
'query Roads feedproject[Maxspeed] {r} count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
Roads feedproject[Code] count,
Roads feedproject[Code] count,
0.0] consume;
query Roads feed {r1} Roads feed {r2}
hashjoin[Maxspeed_r1, Maxspeed_r2] head[1] count;
query Roads feed {r1} Roads feed {r2}
hashjoin[Maxspeed_r1, Maxspeed_r2] head[1] count;
query Roads feed {r1} Roads feed {r2}
hashjoin[Maxspeed_r1, Maxspeed_r2] head[1] count;
query Cconst inserttuple[
"142",
'query Roads20 feed {r1} Roads20
feed {r2} hashjoin[Maxspeed_r1, Maxspeed_r2] head[1] count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
Roads feedproject[Code] count,
1,
0.0] consume;
query Cconst inserttuple[
"36",
'query Roads feed {r1} Roads feed {r2} hashjoin[...] count (Hashphase)',
0.0,
0.0,
0,
0,
round
(
(
(
(Cconst feed filter[(.Id = "142")] sum[ElapsedTime]
- Cconst feed filter[(.Id = "141")] sum[ElapsedTime])
/ Roads feedproject[Code] count
)
* 1000)
,6)]consume
query Roads feed {r1} Roads feed {r2} hashjoin[Maxspeed_r1, Code_r2] count;
query Roads feed {r1} Roads feed {r2} hashjoin[Maxspeed_r1, Code_r2] count;
query Roads feed {r1} Roads feed {r2} hashjoin[Maxspeed_r1, Code_r2] count;
query Cconst inserttuple[
"143",
'query Roads feed {r1} Roads feed {r2} hashjoin[Maxspeed_r1, Code_r2] count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
((Roads feedproject[Code] count)*2),
0,
0.0] consume;
query Cconst inserttuple[
"37",
'query Roads feed {r1} Roads feed {r2} hashjoin[...] count (Probephase)',
0.0,
0.0,
0,
0,
round
(
(
(Cconst feed filter[(.Id = "143")] sum[ElapsedTime]
- Cconst feed filter[(.Id = "142")] sum[ElapsedTime]
- Cconst feed filter[(.Id = "141")] sum[ElapsedTime])
/
(
((Roads feedproject [Code] count)/99997)
* ((Roads feedproject [Code] count)/99997) * 99997
)
)
* 1000,6)]consume;
/*
1.3.5 SpatialJoin
The following query is used to determine the cost constant for the spatialjoin (without producing results).
As stated in costs2014.pl originalversion the costs are almost linear in size of the argument streams, therefore only one constant is used here.
The relations Roads and Waterways are joined through a spatialjoin to determine this constant.
----
query Roads feed {r} Waterways feed {w} spatialjoin[GeoData_r, GeoData_w]
filter[(.GeoData_r intersects .GeoData_w)] head[1] count
----
*/
query Roads feed {r} Waterways feed {w} spatialjoin[GeoData_r, GeoData_w]
filter[(.GeoData_r intersects .GeoData_w)] head[1] count;
query Roads feed {r} Waterways feed {w} spatialjoin[GeoData_r, GeoData_w]
filter[(.GeoData_r intersects .GeoData_w)] head[1] count;
query Roads feed {r} Waterways feed {w} spatialjoin[GeoData_r, GeoData_w]
filter[(.GeoData_r intersects .GeoData_w)] head[1] count;
query Cconst inserttuple[
"144",
'query Roads feed {r} Waterways feed {w}
spatialjoin[GeoData_r, GeoData_w]
filter[(.GeoData_r intersects .GeoData_w)] head[1] count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
Roads feedproject[Code] count + Waterways feedproject[Code] count,
1,
0.0] consume;
query Cconst inserttuple[
"38",
'query Roads feed {r} Waterways feed {w}
spatialjoin[GeoData_r, GeoData_w]
filter[(.GeoData_r intersects .GeoData_w)] head[1] count',
0.0,
0.0,
0,
0,
round
(
(
(
(Cconst feed filter[(.Id = "144")] sum[ElapsedTime])
/
(Roads feedproject[Code] count
+ Waterways feedproject[Code] count)
)
* 1000)
,6)]consume;
/*
1.3.6 itHashJoin
The cost function was first introduced in Costs2014.pl
It is modified to a version that only accounts the probing and not the hashing (like in hashjoin).
The documentation of the cost function can be found in costs2014.pl.
Cost constants are evaluated in a similar manner as for the hashjoin. The difference is that two relations with more tuples are used.
The itHashJoin has two additional constants for reading and writing of the tuple file for stream two.
The first two queries evaluate the constants without partitioning and thus without using the tuple file.
The second one determines the probing cost and uses the number of buckets 999997, that is the default value of the itHashJoin, for evaluating that constant.
For this a relation RoadsC2 is created to probe but not produce results.
For the read and write constants three queries are used:
A base query where the relation fits completely in the memory. After that a query with two partitions (and 300 MB Mem) and one with three partitions (and 150 MB Mem) are executed.
To determine the cost for reading the tuple file we subtract the time for the two-partitions-query from the three-partitions-query to determine the costs for one read process.
Then, this time is subtracted from the difference between two-partitions-query and the base cost to isolate the write process cost.
The memory values need to be adjusted if the queries are not executed in one, two and three partitions with 512MB, 300MB and 150MB memory each.
----
let RoadsC2 = Roads feed head[300000] extend[Code2: .Code*15] consume
query Roads feed head[300000] count
query Roads feed head[300000] {a} Roads feed head[300000] {b}
itHashJoin[Code_a, Code_b] head[1] count
query Roads feed head[300000] {a} RoadsC2 feed {b}
itHashJoin[Code_a, Code2_b] count
----
*/
let RoadsC2 = Roads feed head[300000] extend[Code2: .Code*15] project[Code2] consume;
query Roads feed head[300000] count;
query Roads feed head[300000] count;
query Roads feed head[300000] count;
query Cconst inserttuple[
"145",
'query Roads feed head[300000] count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
300000,
300000,
0.0] consume;
query Roads feed head[300000] {a} Roads feed head[300000] {b}
itHashJoin[Code_a, Code_b] head[1] count;
query Roads feed head[300000] {a} Roads feed head[300000] {b}
itHashJoin[Code_a, Code_b] head[1] count;
query Roads feed head[300000] {a} Roads feed head[300000] {b}
itHashJoin[Code_a, Code_b] head[1] count;
query Cconst inserttuple[
"146",
'query Roads feed head[300000] {a} Roads feed head[300000] {b}
itHashJoin[Code_a, Code_b] head[1] count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
300000,
1,
0.0] consume;
query Cconst inserttuple[
"39",
'query Roads feed {r1} Roads feed {r2} itHashJoin[...] count (Hashphase)',
0.0,
0.0,
0,
0,
round
(
(
(
((Cconst feed filter[(.Id = "146")] sum[ElapsedTime])
- (Cconst feed filter[(.Id = "145")] sum[ElapsedTime]))
/ 300000
)
* 1000)
,6)]consume;
query Roads feed head[300000] {a} RoadsC2 feed {b}
itHashJoin[Code_a, Code2_b] count;
query Roads feed head[300000] {a} RoadsC2 feed {b}
itHashJoin[Code_a, Code2_b] count;
query Roads feed head[300000] {a} RoadsC2 feed {b}
itHashJoin[Code_a, Code2_b] count;
query Cconst inserttuple[
"147",
'query Roads feed head[300000] {a} RoadsC2 feed {b}
itHashJoin[Code_a, Code2_b] count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
600000,
0,
0.0] consume;
query Cconst inserttuple[
"40",
'query Roads feed head[300000] {a} RoadsC2 feed {b}
itHashJoin[Code_a, Code2_b] count (Probephase)',
0.0,
0.0,
0,
0,
round
(
(
(Cconst feed filter[(.Id = "147")] sum[ElapsedTime]
- Cconst feed filter[(.Id = "146")] sum[ElapsedTime])
/
(((300000/999997) + 1) * ((300000/999997) + 1) * 999997)
)
* 1000,6)]consume;
/*
Here, the determination of the read and write constants begin.
First a few variables are defined for later checking if the number of partitions were really one, two and three respectively.
If not, the constants may be off, due to the fact that the calculation is explicitly build a to factor between two and three partitions (or one and two respectively).
The first query is executed four times to eliminate the cold buffer state.
This is written back to costs2014.pl as a commentary behind the constants.
CpuTimes are used because the elapsed times seemed to be off.
Tests showed that with two or three partitions the elapsed times are shorter than with only one partition.
*/
query meminit(512);
let Mem = memsize();
let CardX = Roads feedproject[Code] count;
let SizeX = Roads feed extend[Memory: .Code memattrsize] avg[Memory] + 136;
let P_Q1 = num2string(1 + floor((CardX * SizeX) / ((Mem * 1024 * 1024)*0.8)));
let P_Q2 = num2string(1 + floor((CardX * SizeX) / ((300 * 1024 * 1024)*0.8)));
let P_Q3 = num2string(1 + floor((CardX * SizeX) / ((150 * 1024 * 1024)*0.8)));
/*
Evaluation start
----
query Roads feedproject[Code] {a} RoadsC2 feedproject[Code2] {b}
itHashJoin[Code_a, Code2_b] head[1] count
query Roads feedproject[Code] {a} RoadsC2 feedproject[Code2] {b}
itHashJoin[Code_a, Code2_b] {memory 300} count
query Roads feedproject[Code] {a} RoadsC2 feedproject[Code2] {b}
itHashJoin[Code_a, Code2_b] {memory 150} count
----
*/
query Roads feedproject[Code] {a} RoadsC2 feedproject[Code2] {b}
itHashJoin[Code_a, Code2_b] head[1] count;
query Roads feedproject[Code] {a} RoadsC2 feedproject[Code2] {b}
itHashJoin[Code_a, Code2_b] head[1] count;
query Roads feedproject[Code] {a} RoadsC2 feedproject[Code2] {b}
itHashJoin[Code_a, Code2_b] head[1] count;
query Roads feedproject[Code] {a} RoadsC2 feedproject[Code2] {b}
itHashJoin[Code_a, Code2_b] head[1] count;
query Cconst inserttuple[
"148",
'query Roads feedproject[Code] {a} RoadsC2 feedproject[Code2] {b}
itHashJoin[Code_a, Code2_b] head [1] count (1 Partition)',
SEC2COMMANDS feed tail[3] avg[ElapsedTime],
SEC2COMMANDS feed tail[3] avg[CpuTime],
Roads feedproject[Code] count,
0,
0.0] consume;
query Roads feedproject[Code] {a} RoadsC2 feedproject[Code2] {b}
itHashJoin[Code_a, Code2_b] {memory 300} count;
query Roads feedproject[Code] {a} RoadsC2 feedproject[Code2] {b}
itHashJoin[Code_a, Code2_b] {memory 300} count;
query Roads feedproject[Code] {a} RoadsC2 feedproject[Code2] {b}
itHashJoin[Code_a, Code2_b] {memory 300} count;
query Roads feedproject[Code] {a} RoadsC2 feedproject[Code2] {b}
itHashJoin[Code_a, Code2_b] {memory 300} count;
query Cconst inserttuple[
"149",
'query Roads feedproject[Code] {a} RoadsC2 feedproject[Code2] {b}
itHashJoin[Code_a, Code2_b] {memory 300} count (2 Partitions)',
SEC2COMMANDS feed tail[3] avg[ElapsedTime],
SEC2COMMANDS feed tail[3] avg[CpuTime],
Roads feedproject[Code] count,
0,
0.0] consume;
query Roads feedproject[Code] {a} RoadsC2 feedproject[Code2] {b}
itHashJoin[Code_a, Code2_b] {memory 150} count;
query Roads feedproject[Code] {a} RoadsC2 feedproject[Code2] {b}
itHashJoin[Code_a, Code2_b] {memory 150} count;
query Roads feedproject[Code] {a} RoadsC2 feedproject[Code2] {b}
itHashJoin[Code_a, Code2_b] {memory 150} count;
query Roads feedproject[Code] {a} RoadsC2 feedproject[Code2] {b}
itHashJoin[Code_a, Code2_b] {memory 150} count;
query Cconst inserttuple[
"150",
'query Roads feedproject[Code] {a} RoadsC2 feedproject[Code2] {b}
itHashJoin[Code_a, Code2_b] {memory 150} count (3 Partitions)',
SEC2COMMANDS feed tail[3] avg[ElapsedTime],
SEC2COMMANDS feed tail[3] avg[CpuTime],
Roads feedproject[Code] count,
0,
0.0] consume;
query Cconst inserttuple[
"151",
'query Roads feedproject[Code] {a} RoadsC2 feedproject[Code2] {b}
itHashJoin[Code_a, Code2_b] {memory 300} count
(1x Read + 1x Write totaltime)',
((Cconst feed filter[(.Id = "149")] sum[CPUTime])
- (Cconst feed filter[(.Id = "148")] sum[CPUTime])),
((Cconst feed filter[(.Id = "149")] sum[CPUTime])
- (Cconst feed filter[(.Id = "148")] sum[CPUTime])),
0,
0,
0.0]consume;
query Cconst inserttuple[
"152",
'query Roads feedproject[Code] {a} RoadsC2 feedproject[Code2] {b}
itHashJoin[Code_a, Code2_b] count (1x Read totaltime)',
((Cconst feed filter[(.Id = "150")] sum[CPUTime])
- (Cconst feed filter[(.Id = "149")] sum[CPUTime])),
((Cconst feed filter[(.Id = "150")] sum[CPUTime])
- (Cconst feed filter[(.Id = "149")] sum[CPUTime])),
0,
0,
0.0]consume;
query Cconst inserttuple[
"41",
'query Roads feedproject[Code] {a} RoadsC2 feedproject[Code2] {b}
itHashJoin[Code_a, Code2_b] (1x Read)',
0.0,
0.0,
0,
0,
round
(
(Cconst feed filter[(.Id = "152")] sum[CPUTime]
/ (300000 * RoadsC2 feed tuplesize))
* 1000,6)]consume;
query Cconst inserttuple[
"42",
'query Roads feedproject[Code] {a} RoadsC2 feedproject[Code2] {b}
itHashJoin[Code_a, Code2_b] count (1x Write)',
0.0,
0.0,
0,
0,
round
(
(
(Cconst feed filter[(.Id = "151")] sum[CPUTime]
- Cconst feed filter[(.Id = "152")] sum[CPUTime])
/
(300000 * RoadsC2 feed tuplesize))
* 1000,6)]consume;
delete Mem;
delete CardX;
delete SizeX;
delete RoadsC2;
/*
1.3.7 itSpatialJoin
The cost function is the same as for itHashjoin only the constants differ.
Evaluation of the constants follows the same steps as for itHashJoin.
A relation based on Roads is created that contains tuples that have no overlapping BBoxes with relation Buildings. This is named RoadsNBuild.
It is needed for a query that probes but produces no results.
The evaluation of the read and write constants are done in a similar manner like for itHashJoin.
----
let RoadsNBuild = Roads feed{r} filter[(.Fclass_r = "bridleway")]
loopjoin[Buildings_GeoData_rtree Buildings windowintersects[.GeoData_r]
project[GeoData]{b}] filter[not((.GeoData_r intersects .GeoData_b))] consume
query Buildings feedproject[GeoData] head[500000] count
query Buildings feedproject[GeoData] head[500000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] head[1] count
query Buildings feedproject[GeoData] head[500000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] count
----
*/
let RoadsNBuild = Roads feed{r} filter[(.Fclass_r = "bridleway")]
loopjoin[Buildings_GeoData_rtree Buildings windowintersects[.GeoData_r]
project[GeoData]{b}] filter[not((.GeoData_r intersects .GeoData_b))]
project[GeoData_r] consume;
query Buildings feedproject[GeoData] head[500000] count;
query Buildings feedproject[GeoData] head[500000] count;
query Buildings feedproject[GeoData] head[500000] count;
query Buildings feedproject[GeoData] head[500000] count;
query Cconst inserttuple[
"153",
'query Buildings feedproject[GeoData] head[500000] count',
SEC2COMMANDS feed tail[3] avg[ElapsedTime],
SEC2COMMANDS feed tail[3] avg[CpuTime],
500000,
500000,
0.0] consume;
query Buildings feedproject[GeoData] head[500000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] head[1] count;
query Buildings feedproject[GeoData] head[500000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] head[1] count;
query Buildings feedproject[GeoData] head[500000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] head[1] count;
query Cconst inserttuple[
"154",
'query Buildings feedproject[GeoData] head[500000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] head[1] count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
1,
500000,
0.0] consume;
query Cconst inserttuple[
"43",
'query Buildings feedproject[GeoData] head[500000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] head[1] count (Hashing)',
0.0,
0.0,
0,
0,
round
(
(
(
((Cconst feed filter[(.Id = "154")] sum[ElapsedTime])
- (Cconst feed filter[(.Id = "153")] sum[ElapsedTime]))
/500000
)
* 1000)
,6)]consume;
query Buildings feedproject[GeoData] head[500000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] count;
query Buildings feedproject[GeoData] head[500000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] count;
query Buildings feedproject[GeoData] head[500000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] count;
query Cconst inserttuple[
"155",
'query Buildings feedproject[GeoData] head[500000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] count ',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
0,
500000 + (RoadsNBuild feed count),
0.0] consume;
query Cconst inserttuple[
"44",
'query Buildings feedproject[GeoData] head[500000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] count (Probing)',
0.0,
0.0,
0,
0,
round
(
(
(
(Cconst feed filter[(.Id = "155")] sum[ElapsedTime]
- Cconst feed filter[(.Id = "154")] sum[ElapsedTime])
/ (500000 + RoadsNBuild feed count)
)
* 1000)
,6)]consume;
/*
The evaluation of the read and write constants are done in a similiar manner like for the itHashJoin.
An overhead of 353 iss evaluated through a query with only the GeoData attribute and 512mb memory.
The join then finishes in 1 Partition and stores max 1062111 tuples. This would be (51210241024) / 1062111 = 505.48 bytes per tuple.
As GeoData in the relation Roads has a size of 152 the remaining size must be the overhead which is 353.
CpuTimes is used because the elapsed times seemed to be off.
Tests have shown that with two or three partitions the elapsed times are shorter than with only 1 partition.
----
query Buildings feedproject[GeoData] head[1000000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] head[1] count
query Buildings feedproject[GeoData] head[1000000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] {memory 400} count
query Buildings feedproject[GeoData] head[1000000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] {memory 150} count
----
*/
let Mem = memsize();
let CardX = 1000000;
let SizeX = Buildings feed extend[Memory: .GeoData memattrsize] avg[Memory] + 353;
let PS_Q1 = num2string(1 + floor((CardX * SizeX) / ((Mem * 1024 * 1024))));
let PS_Q2 = num2string(1 + floor((CardX * SizeX) / ((400 * 1024 * 1024))));
let PS_Q3 = num2string(1 + floor((CardX * SizeX) / ((230 * 1024 * 1024))));
/*
Evaluation start
*/
query Buildings feedproject[GeoData] head[1000000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] head[1] count;
query Buildings feedproject[GeoData] head[1000000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] head[1] count;
query Buildings feedproject[GeoData] head[1000000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] head[1] count;
query Buildings feedproject[GeoData] head[1000000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] head[1] count;
query Cconst inserttuple[
"156",
'query Buildings feedproject[GeoData] head[1000000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] count (1 Partition)',
SEC2COMMANDS feed tail[3] avg[ElapsedTime],
SEC2COMMANDS feed tail[3] avg[CpuTime],
1000000,
0,
0.0] consume;
query Buildings feedproject[GeoData] head[1000000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] {memory 400} count;
query Buildings feedproject[GeoData] head[1000000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] {memory 400} count;
query Buildings feedproject[GeoData] head[1000000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] {memory 400} count;
query Buildings feedproject[GeoData] head[1000000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] {memory 400} count;
query Cconst inserttuple[
"157",
'query Buildings feedproject[GeoData] head[1000000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] {memory 400} count (2 Partitions)',
SEC2COMMANDS feed tail[3] avg[ElapsedTime],
SEC2COMMANDS feed tail[3] avg[CpuTime],
1000000,
0,
0.0] consume;
query Buildings feedproject[GeoData] head[1000000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] {memory 230} count;
query Buildings feedproject[GeoData] head[1000000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] {memory 230} count;
query Buildings feedproject[GeoData] head[1000000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] {memory 230} count;
query Buildings feedproject[GeoData] head[1000000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] {memory 230} count;
query Cconst inserttuple[
"158",
'query Buildings feedproject[GeoData] head[1000000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] {memory 230} count (3 Partitions)',
SEC2COMMANDS feed tail[3] avg[ElapsedTime],
SEC2COMMANDS feed tail[3] avg[CpuTime],
1000000,
0,
0.0] consume;
query Cconst inserttuple[
"159",
'query Buildings feedproject[GeoData] head[1000000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] {memory 400} count
(1x Read + 1x Write totaltime)',
((Cconst feed filter[(.Id = "157")] sum[CPUTime])
- (Cconst feed filter[(.Id = "156")] sum[CPUTime])),
((Cconst feed filter[(.Id = "157")] sum[CPUTime])
- (Cconst feed filter[(.Id = "156")] sum[CPUTime])),
0,
0,
0.0]consume;
query Cconst inserttuple[
"160",
'query Buildings feedproject[GeoData] head[1000000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] count
(1x Read totaltime)',
((Cconst feed filter[(.Id = "158")] sum[CPUTime])
- (Cconst feed filter[(.Id = "157")] sum[CPUTime])),
((Cconst feed filter[(.Id = "158")] sum[CPUTime])
- (Cconst feed filter[(.Id = "157")] sum[CPUTime])),
0,
0,
0.0]consume;
query Cconst inserttuple[
"45",
'query Buildings feedproject[GeoData] head[1000000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] count (Read)',
0.0,
0.0,
0,
0,
round
(
(
(Cconst feed filter[(.Id = "158")] sum[CPUTime]
- Cconst feed filter[(.Id = "157")] sum[CPUTime])
/
(RoadsNBuild feed count * RoadsNBuild roottuplesize)
)* 1000
,6)]consume;
query Cconst inserttuple[
"46",
'query Buildings feedproject[GeoData] head[1000000] {b}
RoadsNBuild feed {r}
itSpatialJoin[GeoData_b, GeoData_r_r] count (Write)',
0.0,
0.0,
0,
0,
round
(
(
((Cconst feed filter[(.Id = "159")] sum[CPUTime])
- (Cconst feed filter[(.Id = "160")] sum[CPUTime]))
/
(RoadsNBuild feed count* RoadsNBuild roottuplesize)
)* 1000
,6)]consume;
delete RoadsNBuild;
delete Mem;
delete CardX;
delete SizeX;
/*
2 Export
The relation cConst is exported for further usage and analyzing.
If there is an old version of this file it will be overwritten.
*/
query Cconst feed csvexport['Cconst.csv', FALSE, TRUE, ";"] count;
/*
3 Other Constants
These constants are used in other algebras such as the symbolic trajectory algebra as well as the NearestNeighborAlgebra.
Indextmatches needs an mlbale attribut to work on. So the database symtrajsmall is restored, as this DB contains relations with the needed attributes.
To determine the constant a new relation Dotraj2 is created. This will be filled with round about 100.000 tuples. The mblabel attribute will also be indexed beforehand, as Indextmatches is an index operator.
After filling the relation and indexing it, the script is executing a query to determine the time for Indextmatches. Time is linear in size of the relation.
Distancescan on the other hand.
If these constants are determined in another database, it is not possible to write them back in the original relation Cconst in DB nrw. Instead another version of Cconst is implemented and these constants are directly appended into the csv-export.
Also the values are written directly back to costs2014.
All others are written at the end of the script.
*/
close database;
restore database symtrajsmall from '../bin/symtrajsmall';
let ratio = 1.5;
let nl = "
";
let Cconst = [const rel(tuple([
Id: string,
Query: text,
ElapsedTime: real,
CPUTime:real,
NoResTuple:int,
NoCompTuple:int,
Constant:real]))
value () ];
let Dotraj2 = Dotraj feed consume;
query Dotraj2 feed head[10000] Dotraj2 insert count;
query Dotraj2 feed head[10000] Dotraj2 insert count;
query Dotraj2 feed head[10000] Dotraj2 insert count;
query Dotraj2 feed head[10000] Dotraj2 insert count;
query Dotraj2 feed head[10000] Dotraj2 insert count;
query Dotraj2 feed head[10000] Dotraj2 insert count;
query Dotraj2 feed head[10000] Dotraj2 insert count;
query Dotraj2 feed head[10000] Dotraj2 insert count;
query Dotraj2 feed head[10000] Dotraj2 insert count;
query Dotraj2 feed head[10000] Dotraj2 insert count;
query Dotraj2 feed head[10000] Dotraj2 insert count;
let Dotraj2_traj = Dotraj2 bulkloadtupleindex[Trajectory];
query Dotraj2_traj Dotraj2
indextmatches[Trajectory, '(_ "Innenstadt-Ost") *'] count;
query Dotraj2_traj Dotraj2
indextmatches[Trajectory, '(_ "Innenstadt-Ost") *'] count;
query Dotraj2_traj Dotraj2
indextmatches[Trajectory, '(_ "Innenstadt-Ost") *'] count;
query Cconst inserttuple[
"161",
'query Dotraj2_traj Dotraj2
indextmatches[Trajectory, "(_ "Innenstadt-Ost") *"] count',
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[ElapsedTime],
SEC2COMMANDS feed tail[3]
filter[((.ElapsedTime / .CpuTime)< ratio)] avg[CpuTime],
Dotraj2 feed count,
Dotraj2_traj Dotraj2
indextmatches[Trajectory, '(_ "Innenstadt-Ost") *'] count,
0.0] consume;
query Cconst inserttuple[
"47",
'query Dotraj2_traj Dotraj2
indextmatches[Trajectory, "(_ "Innenstadt-Ost") *"] count',
0.0,
0.0,
0,
0,
round
(
(
(Cconst feed filter[(.Id = "161")] sum[ElapsedTime])
/
(Dotraj2 feed count)
)
* 1000
,6)]consume;
#Writing the reults back to costs2014
#Here the corresponding String will be build
let indextmatchesC = "indextmatchesC("+
num2string(Cconst feed filter[(.Id = "47")] sum[Constant])+")."+nl+nl;
query writeFile(indextmatchesC, 'costs2014.pl', TRUE);
delete indextmatches;
query Cconst feed csvexport['Cconst.csv', TRUE, FALSE, ";"] count;
close database;
delete database symtrajsmall;
open database nrw;
/*
4 Writing constants back
In this section all determined constants will be written back to the costs2014.pl file.
They will be added at the end of the file.
The Script builds a string that contains all relevant information. The constants are directly read from relation Cconst.
An example looks like the following:
----
let feedprC = "feedprojectC("+
num2string(Cconst feed filter[(.Id = "11")] sum[Constant])+","+
num2string(Cconst feed filter[(.Id = "10")] sum[Constant])+","+
num2string(Cconst feed filter[(.Id = "12")] sum[Constant])+")."+nl+nl;
query writeFile(feedprojectC, 'costs2014.pl', TRUE);
In Costs2014.pl:
feedprojectC(0.000636,0.000533,1e-06).
----
At the beginning of the script a timestamp is written in that file too, to distinguish between old and new determined constants.
All constants are written back in chronological order, except for indextmatches. This is always the first constant as it is written back directly after evaluation.
*/
let feedprC = "feedprojectC("+
num2string(Cconst feed filter[(.Id = "11")] sum[Constant])+","+
num2string(Cconst feed filter[(.Id = "10")] sum[Constant])+","+
num2string(Cconst feed filter[(.Id = "12")] sum[Constant])+")."+nl+nl;
query writeFile(feedprC, 'costs2014.pl', TRUE);
let feedC = "feedC("+
num2string(Cconst feed filter[(.Id = "14")] sum[Constant])+","+
num2string(Cconst feed filter[(.Id = "13")] sum[Constant])+")."+nl+nl;
query writeFile(feedC, 'costs2014.pl', TRUE);
let consumeC = "consumeC("+
num2string(Cconst feed filter[(.Id = "15")] sum[Constant])+")."+nl+nl;
query writeFile(consumeC, 'costs2014.pl', TRUE);
let attrC = "attrC("+
num2string(Cconst feed filter[(.Id = "16")] sum[Constant])+")."+nl+nl;
query writeFile(attrC, 'costs2014.pl', TRUE);
let filterC = "filterC("+
num2string(Cconst feed filter[(.Id = "17")] sum[Constant])+")."+nl+nl;
query writeFile(filterC, 'costs2014.pl', TRUE);
let renameC = "renameC("+
num2string(Cconst feed filter[(.Id = "18")] sum[Constant])+")."+nl+nl;
query writeFile(renameC, 'costs2014.pl', TRUE);
let extendC = "extendC("+
num2string(Cconst feed filter[(.Id = "19")] sum[Constant])+","+
num2string(Cconst feed filter[(.Id = "20")] sum[Constant])+")."+nl+nl;
query writeFile(extendC, 'costs2014.pl', TRUE);
let removeC = "removeC("+
num2string(Cconst feed filter[(.Id = "21")] sum[Constant])+")."+nl+nl;
query writeFile(removeC, 'costs2014.pl', TRUE);
let projectC = "projectC("+
num2string(Cconst feed filter[(.Id = "22")] sum[Constant])+")."+nl+nl;
query writeFile(projectC, 'costs2014.pl', TRUE);
let productC = "productC("+
num2string(Cconst feed filter[(.Id = "23")] sum[Constant])+","+
num2string(Cconst feed filter[(.Id = "24")] sum[Constant])+")."+nl+nl;
query writeFile(productC, 'costs2014.pl', TRUE);
let ksmallestC = "ksmallestC("+
num2string(Cconst feed filter[(.Id = "25")] sum[Constant])+","+
num2string(Cconst feed filter[(.Id = "26")] sum[Constant])+")."+nl+nl;
query writeFile(ksmallestC, 'costs2014.pl', TRUE);
let createbetreeC = "createbtreeC("+
num2string(Cconst feed filter[(.Id = "27")] sum[Constant])+")."+nl+nl;
query writeFile(createbetreeC, 'costs2014.pl', TRUE);
let exactmatchC = "exactmatchC("+
num2string(Cconst feed filter[(.Id = "28")] sum[Constant])+")."+nl+nl;
query writeFile(exactmatchC, 'costs2014.pl', TRUE);
let windowintersectsC = "windowintersectsC("+
num2string(Cconst feed filter[(.Id = "29")] sum[Constant])+")."+nl+nl;
query writeFile(windowintersectsC, 'costs2014.pl', TRUE);
let leftrangeC = "leftrangeC("+
num2string(Cconst feed filter[(.Id = "30")] sum[Constant])+")."+nl+nl;
query writeFile(leftrangeC, 'costs2014.pl', TRUE);
let loopjoinC = "loopjoinC("+
num2string(Cconst feed filter[(.Id = "31")] sum[Constant])+","+
num2string(Cconst feed filter[(.Id = "32")] sum[Constant])+"). "+nl;
let loopjoinInfo1 = "% the second constant should be greater than 3"+nl;
let loopjoinInfo2 = "% else Index was already buffered when evaluating it"+nl;
let loopjoinInfo3 = "% Constant then maybe off (reference value is 5)"+nl;
let loopjoinInfo4 = "% and should be changed"+nl+nl;
query writeFile(loopjoinC, 'costs2014.pl', TRUE);
query writeFile(loopjoinInfo1, 'costs2014.pl', TRUE);
query writeFile(loopjoinInfo2, 'costs2014.pl', TRUE);
query writeFile(loopjoinInfo3, 'costs2014.pl', TRUE);
query writeFile(loopjoinInfo4, 'costs2014.pl', TRUE);
let symmjoinC = "symmjoinC("+
num2string(Cconst feed filter[(.Id = "33")] sum[Constant])+")."+nl+nl;
query writeFile(symmjoinC, 'costs2014.pl', TRUE);
let sortmergejoinC = "sortmergejoinC("+
num2string(Cconst feed filter[(.Id = "34")] sum[Constant])+","+
num2string(Cconst feed filter[(.Id = "35")] sum[Constant])+")."+nl+nl;
query writeFile(sortmergejoinC, 'costs2014.pl', TRUE);
let hashjoinC = "hashjoinC("+
num2string(0)+","+
num2string(Cconst feed filter[(.Id = "37")] sum[Constant])+")."+nl+nl;
query writeFile(hashjoinC, 'costs2014.pl', TRUE);
let spatialjoinC = "spatialjoinC("+
num2string(Cconst feed filter[(.Id = "38")] sum[Constant])+")."+nl+nl;
query writeFile(spatialjoinC, 'costs2014.pl', TRUE);
let itHashJoinC1 = "itHashJoinC("+
num2string(Cconst feed filter[(.Id = "39")] sum[Constant])+","+
num2string(Cconst feed filter[(.Id = "40")] sum[Constant])+",";
let itHashJoinC2 =
num2string(Cconst feed filter[(.Id = "41")] sum[Constant])+","+
num2string(Cconst feed filter[(.Id = "42")] sum[Constant])+"). "+nl;
query writeFile(itHashJoinC1, 'costs2014.pl', TRUE);
query writeFile(itHashJoinC2, 'costs2014.pl', TRUE);
let p1 = "% "+"Partitions for first Query: "+P_Q1+", "+nl;
let p2 = "% "+"Partitions for second Query: "+P_Q2+", "+nl;
let p3 = "% "+"Partitions for third Query: "+P_Q3+", "+nl;
let info1 = "% these should be 1, 2 and 3, else "+nl;
let info2 = "% the read/write Constants may be off"+nl+nl;
query writeFile(p1, 'costs2014.pl', TRUE);
query writeFile(p2, 'costs2014.pl', TRUE);
query writeFile(p3, 'costs2014.pl', TRUE);
query writeFile(info1, 'costs2014.pl', TRUE);
query writeFile(info2, 'costs2014.pl', TRUE);
let itSpatialJoinC1 = "itSpatialJoinC("+
num2string(Cconst feed filter[(.Id = "43")] sum[Constant])+","+
num2string(Cconst feed filter[(.Id = "44")] sum[Constant])+",";
let itSpatialJoinC2 =
num2string(Cconst feed filter[(.Id = "45")] sum[Constant])+","+
num2string(Cconst feed filter[(.Id = "46")] sum[Constant])+"). "+nl;
query writeFile(itSpatialJoinC1, 'costs2014.pl', TRUE);
query writeFile(itSpatialJoinC2, 'costs2014.pl', TRUE);
let p_sq1 = "% "+"Partitions for first Query: "+PS_Q1+", "+nl;
let p_sq2 = "% "+"Partitions for second Query: "+PS_Q2+", "+nl;
let p_sq3 = "% "+"Partitions for third Query: "+PS_Q3+", "+nl;
let s_info1 = "% these should be 1, 2 and 3, else "+nl;
let s_info2 = "% the read/write Constants may be off"+nl+nl;
query writeFile(p_sq1, 'costs2014.pl', TRUE);
query writeFile(p_sq2, 'costs2014.pl', TRUE);
query writeFile(p_sq3, 'costs2014.pl', TRUE);
query writeFile(s_info1, 'costs2014.pl', TRUE);
query writeFile(s_info2, 'costs2014.pl', TRUE);
delete feedprC;
delete feedC;
delete consumeC;
delete attrC;
delete filterC;
delete renameC;
delete extendC;
delete removeC;
delete projectC;
delete productC;
delete ksmallestC;
delete createbetreeC;
delete exactmatchC;
delete windowintersectsC;
delete leftrangeC;
delete loopjoinC;
delete loopjoinInfo1;
delete loopjoinInfo2;
delete loopjoinInfo3;
delete loopjoinInfo4;
delete symmjoinC;
delete sortmergejoinC;
delete hashjoinC;
delete spatialjoinC;
delete itHashJoinC1;
delete itHashJoinC2;
delete itSpatialJoinC1;
delete itSpatialJoinC2;
delete P_Q1;
delete P_Q2;
delete P_Q3;
delete p1;
delete p2;
delete p3;
delete info1;
delete info2;
delete PS_Q1;
delete PS_Q2;
delete PS_Q3;
delete p_sq1;
delete p_sq2;
delete p_sq3;
delete s_info1;
delete s_info2;
delete ratio;
delete nl;
Reference in New Issue View Git Blame Copy Permalink