along/secondo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
dfd1e745480fabd88139d2804acb90d5b6dc055e
secondo/Algebras/DistributedClustering/DistributedClusteringAlgebra.cpp
along dfd1e74548 firs commit
2026-01-23 17:03:45 +08:00

1172 lines
38 KiB
C++
Raw Blame History

/*
----
This file is part of SECONDO.
Copyright (C) 2004, University in Hagen, Department of Computer Science,
Database Systems for New Applications.
SECONDO is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
SECONDO is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with SECONDO; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
----
//paragraph [1] Title: [{\Large \bf \begin{center}] [\end{center}}]
//paragraph [10] Footnote: [{\footnote{] [}}]
//[TOC] [\tableofcontents]
[1] Implementation of the distributedClustering Algebra
August-February 2015, Daniel Fuchs
[TOC]
1 Overview
This file contains the implementation of the distributedClustering Algebra
1.1 Includes
*/
#include "NestedList.h"
#include "QueryProcessor.h"
#include "ListUtils.h"
#include "Algebras/Relation-C++/RelationAlgebra.h"
#include "Stream.h"
#include "Attribute.h"
#include "StandardTypes.h"
#include "Algebras/Spatial/SpatialAlgebra.h"
#include "Operator.h"
#include "Symbols.h"
#include "Algebra.h"
#include "DbDacScanGen.h"
#include "Algebras/FText/FTextAlgebra.h"
#include "DistSampleSort.h"
#include "BinRelWriteRead.h"
#include "Member.h"
extern NestedList* nl;
extern QueryProcessor* qp;
using namespace std;
namespace distributedClustering{
const static string MEMBER_ID = "MemberId";
const static string IS_CLUSTER = "IsCluster";
const static string CLUSTER_TYPE = "ClusterType";
const static string NEIGHBORS_RELATION_NAME = "NeighborFName";
const static string PIC_X_REF = "PicXRef";
const static string PIC_Y_REF = "PicYRefVal";
/*
2 Operator dbDacScan
This operated realizes a inmemory db-scan for spatial objects
with a divide and conquer algorithm
2.1 Type Mapping ~dbDacScan~
*/
ListExpr
dbDacScanTM(ListExpr args){
//check Types in args
string errMsg = "";
if(!nl->HasLength(args,2)){
errMsg = "Two elements expexted. First must be a stream"
" of tuple and second must be an argument list";
ErrorReporter::ReportError(errMsg);
return nl->TypeError();
}
//check if first in-Parameter is a stream
ListExpr stream = nl->First(args);
if(!Stream<Tuple>::checkType(nl->First(args))) {
errMsg = "first argument is not stream(Tuple)";
ErrorReporter::ReportError(errMsg);
return nl->TypeError();
}
//check if second in-Parameter is a list of args
ListExpr arguments = nl->Second(args);
if(!nl->HasLength(arguments,5)){
errMsg = "Second argument is not a conform list of input"
" Parameters such, "
" Attributename x ClusterId x NeiborRelFileName x EPS x MinPts ";
ErrorReporter::ReportError(errMsg);
return nl->TypeError();
}
//check NeiborRelFileName
if(!FText::checkType(nl->Third(arguments))
) {
errMsg = "Third argument is NeiborRelFileName and must be a text";
ErrorReporter::ReportError(errMsg);
return nl->TypeError();
}
//check EPS
if(!CcReal::checkType(nl->Fourth(arguments))) {
errMsg = "Fourth argument is EPS and must be a real";
ErrorReporter::ReportError(errMsg);
return nl->TypeError();
}
//check MinPts
if(!CcInt::checkType(nl->Fifth(arguments))) {
errMsg = "Fifth argument is MinPts and must be a int";
ErrorReporter::ReportError(errMsg);
return nl->TypeError();
}
//check the cluster attribute name if it is existing in a tuple
ListExpr attrList = nl->Second(nl->Second(stream));
ListExpr attrType;
string attrName = nl->SymbolValue(nl->First(nl->Second(args)));
int found = FindAttribute(attrList, attrName, attrType);
if(found <= 0) {
errMsg= "Attribute "
+ attrName + " is no member of the tuple";
return listutils::typeError(errMsg);;
}
//Check attrType
if( !CcInt::checkType(attrType)
&& !CcReal::checkType(attrType)
&& !Point::checkType(attrType)
&& !Picture::checkType(attrType))
{
errMsg = "Attribute "+attrName+" is not of type "
+ CcInt::BasicType() + ", "
+ CcReal::BasicType() + ", "
+ Point::BasicType() + ", "
+ Picture::BasicType() + ", "
+"!";
return listutils::typeError(errMsg);
}
//check clusterID
ListExpr name = nl->Second(arguments);
ListExpr typeList;
string errormsg;
if(!listutils::isValidAttributeName(name, errormsg)){
return listutils::typeError(errormsg);
}//endif
string namestr = nl->SymbolValue(name);
int pos = FindAttribute(attrList,namestr,typeList);
if(pos!=0) {
ErrorReporter::ReportError("Attribute "+ namestr +
" already member of the tuple");
return nl->TypeError();
}//endif
//check if there exist picture coordinate references
bool pictureRefExist = false;
int foundXRef = 0;
if(Picture::checkType(attrType)){
ListExpr picXType,picYType;
foundXRef = FindAttribute(attrList, PIC_X_REF, picXType);
int foundYRef = FindAttribute(attrList, PIC_Y_REF, picYType);
if(foundXRef > 0 && foundYRef > 0
&& Picture::checkType(picXType)
&& CcReal::checkType(picYType))
{
pictureRefExist = true;
}
}
//Copy attrList to newAttrList
ListExpr oldAttrList = nl->Second(nl->Second(stream));
ListExpr newAttrList = nl->OneElemList(nl->First(oldAttrList));
ListExpr newAttrListEnd = newAttrList;
oldAttrList = nl->Rest(oldAttrList);
while(!(nl->IsEmpty(oldAttrList))){
newAttrListEnd = nl->Append(newAttrListEnd,nl->First(oldAttrList));
oldAttrList = nl->Rest(oldAttrList);
}
//if picture append x and y references
if(Picture::checkType(attrType) && !pictureRefExist){
newAttrListEnd = nl->Append(newAttrListEnd,
(nl->TwoElemList(
nl->SymbolAtom(PIC_X_REF),
nl->SymbolAtom(Picture::BasicType())
)));
newAttrListEnd = nl->Append(newAttrListEnd,
(nl->TwoElemList(
nl->SymbolAtom(PIC_Y_REF),
nl->SymbolAtom(CcReal::BasicType())
)));
}
// append LongInt for MemberId
newAttrListEnd = nl->Append(newAttrListEnd,
(nl->TwoElemList(
nl->SymbolAtom(MEMBER_ID),
nl->SymbolAtom(LongInt::BasicType()))));
//append int for ClusterID
newAttrListEnd = nl->Append(newAttrListEnd,
(nl->TwoElemList(
name,
nl->SymbolAtom(CcInt::BasicType()))));
//append bool for is Cluster
newAttrListEnd = nl->Append(newAttrListEnd,
(nl->TwoElemList(
nl->SymbolAtom(IS_CLUSTER),
nl->SymbolAtom(CcBool::BasicType())
)));
//append info for ClusterType (CLUSTER = 1 , LEFT = 2,
// RIGHT = 3, BOTH = 4,
//CLUSTERCAND = -1 ,NOISE = -2)
newAttrListEnd = nl->Append(newAttrListEnd,
(nl->TwoElemList(
nl->SymbolAtom(CLUSTER_TYPE),
nl->SymbolAtom(CcInt::BasicType())
)));
//append RelNames - NeighborsRelName
newAttrListEnd = nl->Append(newAttrListEnd,
(nl->TwoElemList(
nl->SymbolAtom(NEIGHBORS_RELATION_NAME),
nl->SymbolAtom(FText::BasicType())
)));
return nl->ThreeElemList(
nl->SymbolAtom(Symbol::APPEND())
,nl->ThreeElemList(nl->IntAtom(found-1),
nl->IntAtom(foundXRef-1),
nl->BoolAtom(
(Picture::checkType(attrType)
&& !pictureRefExist)))
,nl->TwoElemList(nl->SymbolAtom(Symbol::STREAM()),
nl->TwoElemList(
nl->SymbolAtom(Tuple::BasicType()),
newAttrList)));
}
/*
2.2 Value mapping mehtod ~dbDacScan~
*/
template <class TYPE, class MEMB_TYP_CLASS>
int dbDacScanVM1(Word* args, Word& result, int message,
Word& local, Supplier s) {
typedef DbDacScanGen<MEMB_TYP_CLASS,TYPE> dbdacscanClass;
dbdacscanClass* li = (dbdacscanClass*) local.addr;
switch (message)
{
case OPEN:
{
//get stream
Word inStream = args[0];
// get arguments
Word argument;
//get NewRelNamePrefix
Supplier sup = qp->GetSupplier(args[1].addr,2);
// Supplier sup = qp->GetSupplier(args[3].addr,2);
qp->Request(sup, argument);
string relName = static_cast<FText*>(argument.addr)->GetValue();
//get EPS
sup = qp->GetSupplier(args[1].addr,3);
// sup = qp->GetSupplier(args[3].addr,3);
qp->Request(sup, argument);
double eps = static_cast<CcReal*>(argument.addr)->GetValue();
//get MinPts
sup = qp->GetSupplier(args[1].addr, 4);
// sup = qp->GetSupplier(args[3].addr, 4);
qp->Request(sup, argument);
int minPts = static_cast<CcInt*>(argument.addr)->GetValue();
//set index of attribute position in tuple
int attrPos = static_cast<CcInt*>(args[2].addr)->GetIntval();
//get xRef Position for pictures
int xPicRefPos = static_cast<CcInt*>(args[3].addr)->GetIntval();
bool appendPictureRefs =
static_cast<CcBool*>(args[4].addr)->GetValue();
//set the result type of the tuple
ListExpr resultType = GetTupleResultType(s);
ListExpr tt = ( nl->Second( resultType ) );
if(li) {
delete li;
local.addr=0;
}
size_t maxMem = qp->GetMemorySize(s)*1024*1024;
// if(!eps->IsDefined() || eps->GetValue() < 0){
if(eps < 0 ){
return 0;
}
// if(!minPts->IsDefined() || minPts->GetValue() < 0){
if(minPts < 0){
return 0;
}
local.addr = new dbdacscanClass(inStream,tt,
relName,
eps,
minPts,
attrPos,
xPicRefPos,
appendPictureRefs,
maxMem);
return 0;
}
case REQUEST:
{
result.addr= li?li->next():0;
return result.addr?YIELD:CANCEL;
}
case CLOSE:
{
if(li){
delete li;
local.addr=0;
}
}
}
return 0;
}
/*
2.3 Value mapping array ~dbscanMVM~
*/
ValueMapping dbDacScanVM[] =
{
dbDacScanVM1<CcInt, IntMember>,
dbDacScanVM1<CcReal, RealMember>,
dbDacScanVM1<Point, PointMember>,
dbDacScanVM1<Picture, PictureMember>,
};
/*
2.4 Selection Function
selection function for one input stream
*/
int dbDacScanSel(ListExpr args) {
ListExpr attrList = nl->Second(nl->Second(nl->First(args)));
ListExpr attrType;
string attrName = nl->SymbolValue(nl->First(nl->Second(args)));
int found = FindAttribute(attrList, attrName, attrType);
if(found > 0 ){
if(CcInt::checkType(attrType)) {
return 0;
} else if(CcReal::checkType(attrType)) {
return 1;
} else if(Point::checkType(attrType)) {
return 2;
} else if(Picture::checkType(attrType)){
return 3;
}
}
else
{
cout << " no ValueMapping found" << endl;
}
return -1;
};
/*
2.5 Specification
*/
OperatorSpec dbDacScanSpec(
" stream(Tuple) x Attr x Attr x text x real x int -> stream(Tuple) ",
"_ dbdacscan[list]",
"Clusters an Inputstream according to the first Attribute Name"
" which is given in the Argument lsit. The second Attribute"
" Name is the name of ClusterId with which the relation is"
" expanded. The text is the filename in which the Neighbor"
" Relation would be stored.",
"query Kneipen feed dbdacscan [GeoData,"
" ClusterId,'testN',500.0, 5] sortby[ClusterId]"
" groupby[ClusterId ; C : group count] count;"
);
/*
2.6 Instance
*/
Operator dbDacScanOp("dbdacscan",
dbDacScanSpec.getStr(),
4,
dbDacScanVM,
dbDacScanSel,
dbDacScanTM
);
/*
3 Operator ~distclmerge~
3.1 Type Mapping of operator ~distclmerge~
*/
ListExpr
distClMergeTM(ListExpr args){
string errMsg = "";
if(!nl->HasLength(args,10))
{
errMsg = "10 elements expexted. First four must be a Filname"
" of Relations next two must be Attributenames then real and integer"
" value is expexted and at last two output Filenames";
return listutils::typeError(errMsg);
}
//check Files
ListExpr leftFileName = nl->First(args);
if(!nl->HasLength(leftFileName,2)){
return listutils::typeError("internal error");
}
ListExpr leftRelType; //equal to stream
if(!checkFile(leftFileName,leftRelType,errMsg)){
return listutils::typeError(errMsg);
}
ListExpr leftNFielName = nl->Second(args);
if(!nl->HasLength(leftNFielName,2)){
return listutils::typeError("internal error");
}
ListExpr leftNRelType; //equal to stream
if(!checkFile(leftNFielName,leftNRelType,errMsg)){
return listutils::typeError(errMsg);
}
ListExpr righFileName = nl->Third(args);
if(!nl->HasLength(righFileName,2)){
return listutils::typeError("internal error");
}
ListExpr rightRelType; //equal to stream
if(!checkFile(righFileName,rightRelType,errMsg)){
return listutils::typeError(errMsg);
}
ListExpr rightNFileName = nl->Fourth(args);
if(!nl->HasLength(rightNFileName,2)){
return listutils::typeError("internal error");
}
ListExpr rightNRelType; //equal to stream
if(!checkFile(rightNFileName,rightNRelType,errMsg)){
return listutils::typeError(errMsg);
}
//check EPS
if(!CcReal::checkType(nl->First(nl->Seventh(args))))
{
errMsg = "Seventh argument is EPS and must be a real";
ErrorReporter::ReportError(errMsg);
return nl->TypeError();
}
//check MinPts
if(!CcInt::checkType(nl->First(nl->Eigth(args))))
{
errMsg = "Eigth argument is MinPts and must be a int";
ErrorReporter::ReportError(errMsg);
return nl->TypeError();
}
//check OutRelFileName
if(!FText::checkType(nl->First(nl->Ninth(args))))
{
errMsg = "Ninth argument is NewRelNamePrefix and must be a string";
ErrorReporter::ReportError(errMsg);
return nl->TypeError();
}
//check OutNeighborFileName
if(!FText::checkType(nl->First(nl->Tenth(args))))
{
errMsg = "Tenth argument is NewRelNamePrefix and must be a string";
ErrorReporter::ReportError(errMsg);
return nl->TypeError();
}
//=====================================================
//NeighborRel
//=====================================================
//check neighborRelation if there are the same Attributes
ListExpr attrNListLeft = nl->Second(nl->Second(leftNRelType));
ListExpr attrNListRight = nl->Second(nl->Second(rightNRelType));
//check NeighborMembId
ListExpr membAttrL,membAttrR;
int foundMembIdNL = FindAttribute(attrNListLeft,
NEIGH_REL_MEMBER_ID, membAttrL);
int foundMembIdNR = FindAttribute(attrNListRight,
NEIGH_REL_MEMBER_ID, membAttrR);
if(foundMembIdNL <= 0 || foundMembIdNR <= 0
|| foundMembIdNL != foundMembIdNR || foundMembIdNL != 1)
{
errMsg= "Attribute "
+ NEIGH_REL_MEMBER_ID +
" is no member of both tuple on equal position in Neighbor Relation";
return listutils::typeError(errMsg);
}
if( !LongInt::checkType(membAttrL) || !LongInt::checkType(membAttrR))
{
errMsg = "Attribute "+NEIGH_REL_MEMBER_ID+"is not of type "
+ LongInt::BasicType() + "! ";
return listutils::typeError(errMsg);
}
//check Neighbor NeighborId
ListExpr neighborAttrL,neighborAttrR;
int foundNeighIdL = FindAttribute(attrNListLeft,
NEIGH_REL_NEIGHBOR_ID, neighborAttrL);
int foundNeighIdR = FindAttribute(attrNListRight,
NEIGH_REL_NEIGHBOR_ID, neighborAttrR);
if(foundNeighIdL <= 0 || foundNeighIdR <= 0
|| foundNeighIdL != foundNeighIdR || foundNeighIdL != 2) {
errMsg= "Attribute "
+ NEIGH_REL_NEIGHBOR_ID +
" is no member of both tuple on equal position in Neighbor Relation";
return listutils::typeError(errMsg);
}
if( !LongInt::checkType(neighborAttrL)
|| !LongInt::checkType(neighborAttrR))
{
errMsg = "Attribute "+NEIGH_REL_NEIGHBOR_ID+"is not of type "
+ LongInt::BasicType() + "! ";
return listutils::typeError(errMsg);
}
//========================================================================
//Member Relation
//========================================================================
//check the cluster attribute name if it is existing in a tuple
ListExpr attrListLeft = nl->Second(nl->Second(leftRelType));
ListExpr attrListRight = nl->Second(nl->Second(rightRelType));
//Check attribute Type of GeoData
ListExpr geDAttrL,geDAttrR;
string attrNGeoData = nl->SymbolValue(nl->First(nl->Fifth(args)));
int foundGeoL = FindAttribute(attrListLeft, attrNGeoData, geDAttrL);
int foundGeoR = FindAttribute(attrListRight, attrNGeoData, geDAttrR);
if(foundGeoL <= 0 || foundGeoR <= 0 || foundGeoL != foundGeoR ) {
errMsg= "Attribute "
+ attrNGeoData + " is no member of both tuple on equal position";
return listutils::typeError(errMsg);
}
if( !CcInt::checkType(geDAttrL)
&& !CcReal::checkType(geDAttrL)
&& !Point::checkType(geDAttrL)
&& !Picture::checkType(geDAttrL))
{
errMsg = "Attribute "+attrNGeoData+"is not of type "
+ CcInt::BasicType() + ", "
+ CcReal::BasicType() + ", "
+ Point::BasicType() + ", "
+ Picture::BasicType() + ", "
+"!";
return listutils::typeError(errMsg);
}
//Check attribute Type of ClusterId
ListExpr clNoAttrL,clNoAttrR;
string attrNClusterID = nl->SymbolValue(nl->First(nl->Sixth(args)));
int foundClIdL = FindAttribute(attrListLeft, attrNClusterID, clNoAttrL);
int foundClIdR = FindAttribute(attrListRight, attrNClusterID, clNoAttrR);
if(foundClIdL <= 0 || foundClIdR <= 0 || foundClIdL != foundClIdR) {
errMsg= "Attribute "
+ attrNClusterID + " is no member of the tuple";
return listutils::typeError(errMsg);
}
if( !CcInt::checkType(clNoAttrL) || !CcInt::checkType(clNoAttrR))
{
errMsg = "Attribute "+attrNClusterID+"is not of type "
+ CcInt::BasicType() + "! ";
return listutils::typeError(errMsg);
}
//Check ClsuterType
ListExpr clTypeL,clTypeR;
int foundClTyL = FindAttribute(attrListLeft, CLUSTER_TYPE, clTypeL);
int foundClTyR = FindAttribute(attrListRight, CLUSTER_TYPE, clTypeR);
if(foundClTyL <= 0 || foundClTyR <= 0 || foundClTyL != foundClTyR) {
errMsg= "Attribute "
+ CLUSTER_TYPE + " is no member of the tuple";
return listutils::typeError(errMsg);
}
if( !CcInt::checkType(clTypeL) || !CcInt::checkType(clTypeR))
{
errMsg = "Attribute "+CLUSTER_TYPE+"is not of type "
+ CcInt::BasicType() + "! ";
return listutils::typeError(errMsg);
}
//Check MemberId
ListExpr membL,membR;
int foundMembIdL = FindAttribute(attrListLeft, MEMBER_ID, membL);
int foundMembIdR = FindAttribute(attrListRight, MEMBER_ID, membR);
if(foundMembIdL <= 0 || foundMembIdR <= 0
|| foundMembIdL != foundMembIdR) {
errMsg= "Attribute "
+ MEMBER_ID + " is no member of the tuple";
return listutils::typeError(errMsg);
}
if( !LongInt::checkType(membL) || !LongInt::checkType(membR))
{
errMsg = "Attribute "+MEMBER_ID+" is not of type "
+ LongInt::BasicType() + "! ";
return listutils::typeError(errMsg);
}
//Check NeighborRelationName Attribute
ListExpr neighborRL,neighborRR;
int foundNeighRelL = FindAttribute(attrListLeft,
NEIGHBORS_RELATION_NAME, neighborRL);
int foundNeighRelR = FindAttribute(attrListRight,
NEIGHBORS_RELATION_NAME, neighborRR);
if(foundNeighRelL <= 0 || foundNeighRelR <= 0
|| foundNeighRelL != foundNeighRelR) {
errMsg= "Attribute "
+ NEIGHBORS_RELATION_NAME + " is no member of the tuple";
return listutils::typeError(errMsg);
}
if( !FText::checkType(neighborRL) || !FText::checkType(neighborRR))
{
errMsg = "Attribute "+NEIGHBORS_RELATION_NAME+" is not of type "
+ FText::BasicType() + "! ";
return listutils::typeError(errMsg);
}
//if it is Picture Type search and check coordinate references for x and y
int foundXRefL = 0;
if(Picture::checkType(geDAttrL)){
ListExpr picXTypeL,picYTypeL, picXTypeR, picYTypeR;
foundXRefL = FindAttribute(attrListLeft, PIC_X_REF, picXTypeL);
int foundYRefL = FindAttribute(attrListLeft, PIC_Y_REF, picYTypeL);
int foundXRefR = FindAttribute(attrListRight, PIC_X_REF, picXTypeR);
int foundYRefR = FindAttribute(attrListRight, PIC_Y_REF, picYTypeR);
if(foundXRefL <= 0 || foundYRefL <= 0 || foundYRefL != foundYRefR
|| foundXRefL != foundXRefR
|| !Picture::checkType(picXTypeL) || !Picture::checkType(picXTypeR)
|| !CcReal::checkType(picYTypeL) || !CcReal::checkType(picYTypeR) )
{
errMsg = "Attribute "+attrNGeoData+" is of type "
+ Picture::BasicType() + " but there are not correct"
" coordinate references for x and y! Reference "
"for coordinate x must be of type "+ Picture::BasicType() +
" and reference for y must be of type "+ CcReal::BasicType();
return listutils::typeError(errMsg);
}
}
return nl->ThreeElemList(
nl->SymbolAtom(Symbol::APPEND())
,nl->FourElemList(nl->IntAtom(foundGeoL-1) // position of GeoData
,nl->IntAtom(foundClIdL-1) // position of ClusterId
,nl->IntAtom(foundClTyL-1) // position of ClusterType
,nl->IntAtom(foundXRefL-1) // position of coordinate reference x
)
,nl->TwoElemList(nl->SymbolAtom(Symbol::STREAM()),
nl->Second(leftRelType)
));
}
/*
3.2 Value Mapping Template of operator ~distclmerge~
this the template method which is called from distClMergeVM
*/
template <class TYPE, class MEMB_TYP_CLASS>
int
distClMergeVMT(Word* args, Word& result, int message,
Word& local, Supplier s)
{
typedef DbDacScanGen<MEMB_TYP_CLASS,TYPE > distClMergeClass;
distClMergeClass* li = (distClMergeClass*) local.addr;
switch (message)
{
case OPEN:
{
ListExpr resultType = GetTupleResultType(s);
ListExpr tt = ( nl->Second( resultType ) ); //RelType
//TupleType for file
ListExpr relFileType = qp->GetType(s);
//get Filenames
FText* fnL = (FText*) args[0].addr;
if(!fnL->IsDefined()){
return 0;
}
FText* fnNL = (FText*) args[1].addr;
if(!fnNL->IsDefined()){
return 0;
}
FText* fnR = (FText*) args[2].addr;
if(!fnR->IsDefined()){
return 0;
}
FText* fnNR = (FText*) args[3].addr;
if(!fnNR->IsDefined()){
return 0;
}
//get EPS
double eps = static_cast<CcReal*>(args[6].addr)->GetValue();
//get MinPts
int minPts = static_cast<CcInt*>(args[7].addr)->GetValue();
//OutRelFileName
string outRelFileName = static_cast<FText*>(args[8].addr)->GetValue();
//OutNeighborFileName
string outNFileName = static_cast<FText*>(args[9].addr)->GetValue();
//set index of GeoData attribute position in tuple
int geoPos = static_cast<CcInt*>(args[10].addr)->GetIntval();
// set index of ClusterId attribute position in tuple
int clIdPos = static_cast<CcInt*>(args[11].addr)->GetIntval();
//set index of ClusterType attribute position in tuple
int clTypePos = static_cast<CcInt*>(args[12].addr)->GetIntval();
//get xRef Position for pictures
int xPicRefPos = static_cast<CcInt*>(args[13].addr)->GetIntval();
if(li) {
delete li;
local.addr=0;
}
size_t maxMem = qp->GetMemorySize(s)*1024*1024;
if(eps < 0){
return 0;
}
if(minPts < 0){
return 0;
}
local.addr =
new distClMergeClass(fnL->GetValue(), fnNL->GetValue(),
fnR->GetValue(), fnNR->GetValue(),
geoPos,clIdPos,clTypePos,xPicRefPos, maxMem
,tt, relFileType,
outRelFileName,outNFileName,
eps, minPts);
return 0;
}
case REQUEST:
{
result.addr= li?li->next():0;
return result.addr?YIELD:CANCEL;
}
case CLOSE:
{
if(li){
delete li;
local.addr=0;
}
}
}
return 0;
}
/*
3.3 Value Mapping of operator ~distclmerge~
value mapping acts as selection function.
Because the relation is passed as a file, there is no way to find
out the type of the passed Attributename in selection function.
*/
int
distClMergeVM(Word* args, Word& result, int message,
Word& local, Supplier s)
{
//set index of GeoData attribute position in tuple
int geoPos = static_cast<CcInt*>(args[10].addr)->GetIntval();
//determine AttrType
ListExpr relFileType = qp->GetType(s);
ListExpr attrType = nl->Second( nl->Second( relFileType ) );
for(int i= 0;i<geoPos; i++){
attrType = nl->Rest(attrType);
}
attrType = nl->Second(nl->First(attrType));
if(CcInt::checkType(attrType))
{
return
distClMergeVMT<CcInt ,IntMember>(args,result,message,local, s);
} else
if(CcReal::checkType(attrType))
{
return
distClMergeVMT<CcReal, RealMember > (args,result,message,local, s);
} else
if(Point::checkType(attrType))
{
return
distClMergeVMT<Point, PointMember >(args,result,message,local, s);
} else
if(Picture::checkType(attrType)){
return distClMergeVMT<Picture, PictureMember >
(args,result,message,local, s);
}else {
return 0;
}
}
/*
3.4 Specification
*/
OperatorSpec distClMergeSpec( " text x text x text x text "
"x Attr x Attr x real x int x text x text -> stream(Tuple)",
"distclmerge(_,_,_,_,_,_,_,_,_,_)",
"Performs a function to merge alredy clustered part relations.",
"query distclmerge('DbScan.bin' , 'NeighborFile.bin' "
",'DbScan.bin_1','NeighborFile.bin_1 ' , GeoData, ClusterId, "
"500.0 , 5 , 'DbScan.bin' ,'NeighborFile.bin' ) consume"
);
/*
3.5 Instance
*/
Operator distClMergeOp(
"distclmerge",
distClMergeSpec.getStr(),
distClMergeVM,
Operator::SimpleSelect,
distClMergeTM
);
/*
4 Operator ~distsamp~
4.1 Type Mapping of operator ~distsamp~
*/
ListExpr
distsampTM(ListExpr args){
string err = "two tuple streams and Attributename value expected";
if(!nl->HasLength(args,3))
{
return listutils::typeError(err);
}
//check streams
if(!Stream<Tuple>::checkType(nl->First(args)) ||
!Stream<Tuple>::checkType(nl->Second(args)))
{
return listutils::typeError(err);
}
ListExpr arguments = nl->Third(args);
//check CntWorkers
if(!CcInt::checkType(nl->Third(arguments))) {
err = "Fourth argument is Count Workers and must be a int";
ErrorReporter::ReportError(err);
return nl->TypeError();
}
//check AttrName
ListExpr attrList = nl->Second(nl->Second(nl->First(args)));
ListExpr attrListSamp = nl->Second(nl->Second(nl->Second(args)));
ListExpr attrType, attrTypeSamp;
string attrName = nl->SymbolValue(nl->First(arguments));
int found = FindAttribute(attrList, attrName, attrType);
int foundSamp = FindAttribute(attrListSamp, attrName, attrTypeSamp);
if(found <= 0 || foundSamp <= 0 || found != foundSamp) {
err= "Attribute " + attrName + " is no member of the tuple"
". Both streams must have the same Attribute on the same position.";
return listutils::typeError(err);
}
//Check attrType
if(( !CcInt::checkType(attrType)
&& !CcReal::checkType(attrType)
&& !Point::checkType(attrType)
&& !Picture::checkType(attrType)) ||
(!CcInt::checkType(attrTypeSamp)
&& !CcReal::checkType(attrTypeSamp)
&& !Point::checkType(attrTypeSamp)
&& !Picture::checkType(attrTypeSamp)
))
{
err = "Attribute "+attrName+" is not of type "
+ CcInt::BasicType() + ", "
+ CcReal::BasicType() + ", "
+ Point::BasicType() + ", "
+ Picture::BasicType() + ", "
+"! Both streams must have the same Attribute!";
return listutils::typeError(err);
}
//Check WorkerId
ListExpr name = nl->Second(arguments);
ListExpr typeList;
string errormsg;
if(!listutils::isValidAttributeName(name, errormsg)){
return listutils::typeError(errormsg);
}//endif
string namestr = nl->SymbolValue(name);
int pos = FindAttribute(attrList,namestr,typeList);
if(pos!=0) {
ErrorReporter::ReportError("Attribute "+ namestr +
" already member of the tuple");
return nl->TypeError();
}//endif
//check if there exist picture coordinate references
bool pictureRefExist = false;
int foundXRef = 0;
if(Picture::checkType(attrType)){
ListExpr picXType,picYType;
foundXRef = FindAttribute(attrList, PIC_X_REF, picXType);
int foundYRef = FindAttribute(attrList, PIC_Y_REF, picYType);
if(foundXRef > 0 && foundYRef > 0
&& Picture::checkType(picXType)
&& CcReal::checkType(picYType))
{
pictureRefExist = true;
}
}
//Copy attrList to newAttrList
ListExpr stream = nl->First(args);
ListExpr oldAttrList = nl->Second(nl->Second(stream));
ListExpr newAttrList = nl->OneElemList(nl->First(oldAttrList));
ListExpr newAttrListEnd = newAttrList;
oldAttrList = nl->Rest(oldAttrList);
while(!(nl->IsEmpty(oldAttrList))){
newAttrListEnd = nl->Append(newAttrListEnd,nl->First(oldAttrList));
oldAttrList = nl->Rest(oldAttrList);
}
//append int for WorkerID
newAttrListEnd = nl->Append(newAttrListEnd,
(nl->TwoElemList(
name,
nl->SymbolAtom(CcInt::BasicType()))));
//append after WorkerId -> so always the last
//two attributes are the references for picture
// at dbdacscan appen the attributes at first
if(Picture::checkType(attrType) && !pictureRefExist){
newAttrListEnd = nl->Append(newAttrListEnd,
(nl->TwoElemList(
nl->SymbolAtom(PIC_X_REF),
nl->SymbolAtom(Picture::BasicType())
)));
newAttrListEnd = nl->Append(newAttrListEnd,
(nl->TwoElemList(
nl->SymbolAtom(PIC_Y_REF),
nl->SymbolAtom(CcReal::BasicType())
)));
}
return nl->ThreeElemList(
nl->SymbolAtom(Symbol::APPEND())
// ,nl->OneElemList(nl->IntAtom(found-1))
,nl->ThreeElemList(nl->IntAtom(found-1),
nl->IntAtom(foundXRef-1),
nl->BoolAtom(
(Picture::checkType(attrType)
&& !pictureRefExist)))
,nl->TwoElemList(nl->SymbolAtom(Symbol::STREAM()),
nl->TwoElemList(
nl->SymbolAtom(Tuple::BasicType()),
newAttrList)));
}
/*
4.2 ValueMapping of operator ~distsamp~
*/
template <class TYPE, class MEMB_TYP_CLASS>
int
distsampVM1(Word* args, Word& result, int message,
Word& local, Supplier s)
{
typedef Distsamp<MEMB_TYP_CLASS,TYPE > distSample;
distSample* li = (distSample*) local.addr;
switch (message)
{
case OPEN:
{
Word inStream = args[0];
Word sampStream = args[1];
// Word sampStream;
//get CntWorkers
Word argument;
Supplier sup = qp->GetSupplier(args[2].addr, 2);
qp->Request(sup, argument);
int cntWorkers = (static_cast<CcInt*>(argument.addr))->GetIntval();
//get AttrPos:
int attrPos = static_cast<CcInt*>(args[3].addr)->GetIntval();
//get xRef Position for pictures
int xPicRefPos = static_cast<CcInt*>(args[4].addr)->GetIntval();
bool appendPictureRefs =
static_cast<CcBool*>(args[5].addr)->GetValue();
ListExpr resultType = GetTupleResultType(s);
ListExpr tt = ( nl->Second( resultType ) );
if(li) {
delete li;
local.addr=0;
}
size_t maxMem = qp->GetMemorySize(s)*1024*1024;
local.addr = new distSample(inStream, sampStream,
tt, attrPos,
xPicRefPos, appendPictureRefs,
cntWorkers, maxMem);
return 0;
}
case REQUEST:
{
result.addr= li?li->next():0;
return result.addr?YIELD:CANCEL;
}
case CLOSE:
{
if(li){
delete li;
local.addr=0;
}
}
}
return 0;
}
/*
4.3 struct of operator ~distsamp~
*/
struct distsampInfo : OperatorInfo
{
distsampInfo() : OperatorInfo()
{
name = "distsamp";
signature = "stream(T) x stream(T) x Attr x Attr x int -> stream(T)";
syntax = "_ _ distsamp[_,_,_]";
meaning = "As input a relation and a sample relation is excpected."
"According to the first attribute, the relation would be sorted."
" The second attributes which stores the WorkerID , would added."
" Based on the sample relation, boundary points are determined "
"once the tuple allocated to the relevant workers.";
example = "query Kneipen feed Kneipen feed head[50] "
"distsamp[GeoData, WId, 4] count ";
}
};
/*
4.4 Selection Function of operator ~distsamp~
selection funktion for 2 input streams
*/
int distsampSel(ListExpr args) {
ListExpr attrList = nl->Second(nl->Second(nl->First(args)));
ListExpr attrType;
string attrName = nl->SymbolValue(nl->First(nl->Third(args)));
int found = FindAttribute(attrList, attrName, attrType);
if(found > 0 ){
if(CcInt::checkType(attrType)) {
return 0;
} else if(CcReal::checkType(attrType)) {
return 1;
} else if(Point::checkType(attrType)) {
return 2;
} else if(Picture::checkType(attrType)){
return 3;
}
}
else
{
cout << " no ValueMapping found" << endl;
}
return -1;
}
/*
4.5 Value mapping array ~distsamp~
*/
ValueMapping distsampVM[] =
{
distsampVM1<CcInt, IntMember>,
distsampVM1<CcReal, RealMember>,
distsampVM1<Point, PointMember>,
distsampVM1<Picture, PictureMember>
};
/*
5 Algebra class ~ClusterDbDacScanAlgebra~
*/
class ClusterDbDacScanAlgebra : public Algebra
{
public:
ClusterDbDacScanAlgebra() : Algebra()
{
AddOperator(&dbDacScanOp);
// dbDacScanOp.SetUsesArgsInTypeMapping();
dbDacScanOp.SetUsesMemory();
AddOperator(&distClMergeOp);
distClMergeOp.SetUsesArgsInTypeMapping();
distClMergeOp.SetUsesMemory();
AddOperator(distsampInfo(), distsampVM,
distsampSel, distsampTM)->SetUsesMemory();
}
~ClusterDbDacScanAlgebra() {};
};
}
extern "C"
Algebra* InitializeDistributedClusteringAlgebra( NestedList* nlRef,
QueryProcessor* qpRef)
{
nl = nlRef;
qp = qpRef;
return (new distributedClustering::ClusterDbDacScanAlgebra());
}
Reference in New Issue View Git Blame Copy Permalink