Files
secondo/Algebras/KeyValueStore/OperatorsDistribution.cpp
2026-01-23 17:03:45 +08:00

639 lines
20 KiB
C++

/*
----
This file is part of SECONDO.
Copyright (C) 2015,
Faculty of Mathematics and Computer Science,
Database Systems for New Applications.
SECONDO is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
SECONDO is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with SECONDO; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
----
*/
#include "AlgebraOperators.h"
using namespace std;
namespace KVS {
extern KeyValueStoreIPC* kvsIPC;
// kvsServerId( {distribution, text}, {int, rect} [, requestOnly] )
ListExpr kvsServerIdTM(ListExpr args) {
if (!nl->HasLength(args, 2) && !nl->HasLength(args, 3)) {
return listutils::typeError(
"Expecting 2 or 3 arguments [ {distribution, text} x " +
Rectangle<2>::BasicType() + "," + CcInt::BasicType() +
"} x bool = ({rect, int}, {distribution, distributionName} "
"[,requestOnly])].");
}
ListExpr firstType = nl->First(nl->First(args));
if (!Distribution::checkType(firstType) && !FText::checkType(firstType)) {
return listutils::typeError(
"1st argument should be distribution OR text. [ {distribution, text} "
"x " +
Rectangle<2>::BasicType() + "," + CcInt::BasicType() +
"} x bool = ({rect, int}, {distribution, distributionName} "
"[,requestOnly])].");
}
ListExpr secondType = nl->First(nl->Second(args));
if (!Rectangle<2>::checkType(secondType) && !CcInt::checkType(secondType)) {
return listutils::typeError(
"2nd argument should be rectangle(bbox) OR int. [ {distribution, text} "
"x " +
Rectangle<2>::BasicType() + "," + CcInt::BasicType() +
"} x bool = ({rect, int}, {distribution, distributionName} "
"[,requestOnly])].");
}
if (nl->HasLength(args, 3) &&
!CcBool::checkType(nl->First(nl->Third(args)))) {
return listutils::typeError(
"3rd argument should be bool. [ {distribution, text} x " +
Rectangle<2>::BasicType() + "," + CcInt::BasicType() +
"} x bool = ({rect, int}, {distribution, distributionName} "
"[,requestOnly])].");
}
if (!kvsIPC->connected()) {
return listutils::typeError(
"IPC-Connection Error: Not connected to Key-Value Store application.");
}
ListExpr distname;
if (FText::checkType(firstType)) {
distname = nl->Second(nl->First(args)); // ((text 'distname')...)
} else {
distname = nl->TextAtom(nl->SymbolValue(
nl->Second(nl->First(args)))); // ( (..) (distribution distname))
}
return nl->ThreeElemList(nl->SymbolAtom(Symbol::APPEND()),
nl->OneElemList(distname),
NList(Symbol::STREAM(),
CcInt::BasicType())
.listExpr()); // return int stream
}
int kvsServerIdVM(Word* args, Word& result, int message, Word& local,
Supplier s) {
struct ServerIdResult {
set<int>::iterator iter;
set<int> outputServerIds;
};
ServerIdResult* res = static_cast<ServerIdResult*>(local.addr);
switch (message) {
case OPEN: {
int noSons = qp->GetNoSons(s);
// Request Only?
bool requestOnly = false;
if (noSons == 4) {
requestOnly = static_cast<CcBool*>(args[noSons - 2].addr)->GetValue();
}
// Distribution
string distributionName =
static_cast<FText*>(args[noSons - 1].addr)->GetValue();
int distRef = kvsIPC->getDistributionRef(distributionName);
if (distRef < 0) {
ListExpr distributionType = qp->GetSupplierTypeExpr(qp->GetSon(s, 0));
if (Distribution::checkType(distributionType)) {
Distribution* dist = static_cast<Distribution*>(args[0].addr);
distRef = kvsIPC->getDistributionRef(distributionName, dist->type,
dist->toBin());
} else {
cout << "Error: No Distribution specified";
return CANCEL;
}
}
if (distRef >= 0) {
// Prepare State
delete res;
res = new ServerIdResult;
set<int>* outputSet = &res->outputServerIds;
// Handle inputs
ListExpr inputType = qp->GetSupplierTypeExpr(qp->GetSon(s, 1));
if (Rectangle<2>::checkType(inputType)) {
Rectangle<2>* mbb = (Rectangle<2>*)args[1].addr;
double coords[4];
coords[0] = mbb->MinD(0) * 1000;
coords[1] = mbb->MinD(1) * 1000;
coords[2] = mbb->MaxD(0) * 1000;
coords[3] = mbb->MaxD(1) * 1000;
if (kvsIPC->distAddRect(distRef, 4, coords, outputSet, requestOnly)) {
if (outputSet->size() > 0) {
res->iter = res->outputServerIds.begin();
local.addr = res;
return 0;
} else {
cout << "Error: 0 results" << endl;
}
} else {
cout << "Error: distAddRect failed" << endl;
}
} else if (CcInt::checkType(inputType)) {
CcInt* val = static_cast<CcInt*>(args[1].addr);
if (kvsIPC->distAddInt(distRef, val->GetIntval(), outputSet,
requestOnly)) {
if (outputSet->size() > 0) {
res->iter = res->outputServerIds.begin();
local.addr = res;
return 0;
}
}
} else {
cout << "Error: Handling input type failed.\n";
}
} else {
cout << "Error: Distribution couldn't be loaded.\n";
}
// failed?
delete res;
return CANCEL;
}
case REQUEST: {
if (!res) {
return CANCEL;
} else {
if (res->iter != res->outputServerIds.end()) {
CcInt* serverId = new CcInt(true, *(res->iter));
result.addr = serverId;
res->iter++;
return YIELD;
} else {
result.addr = 0;
return CANCEL;
}
}
}
case CLOSE: {
delete res;
local.addr = 0;
}
return 0;
}
return 0;
}
ListExpr kvsSaveDistTM(ListExpr args) {
if (!nl->HasLength(args, 1)) {
return listutils::typeError(
"Expecting 1 argument [ distribution = (distribution)].");
}
ListExpr firstType = nl->First(nl->First(args));
if (!Distribution::checkType(firstType)) {
return listutils::typeError(
"1st argument should be distribution [ distribution = "
"(distribution)].");
}
ListExpr distname;
distname = nl->TextAtom(nl->SymbolValue(
nl->Second(nl->First(args)))); // ( (..) (distribution distname))
return nl->ThreeElemList(nl->SymbolAtom(Symbol::APPEND()),
nl->OneElemList(distname),
listutils::basicSymbol<CcBool>());
}
int kvsSaveDistVM(Word* args, Word& result, int message, Word& local,
Supplier s) {
result = qp->ResultStorage(s);
CcBool* res = static_cast<CcBool*>(result.addr);
if (!kvsIPC->connected()) {
// Distribution
Distribution* dist = static_cast<Distribution*>(args[0].addr);
int noSons = qp->GetNoSons(s);
string distributionName =
static_cast<FText*>(args[noSons - 1].addr)->GetValue();
int distRef = kvsIPC->getDistributionRef(distributionName);
if (distRef < 0) {
distRef = kvsIPC->getDistributionRef(distributionName, dist->type,
dist->toBin());
}
if (distRef >= 0) {
string data;
if (kvsIPC->getDistributionData(distRef, &data)) {
if (dist->fromBin(data)) {
qp->SetModified(qp->GetSon(s, 0));
res->Set(true, true);
return 0;
}
}
}
cout << "Error: Couldn't load distribution." << endl;
} else {
cout << "Error: No IPC-Connection" << endl;
}
res->Set(true, false);
return 0;
}
//_ kvsFilter( {distribution, distributionName} ,region ,globalId
//[,updateDistribution] )
ListExpr kvsFilterTM(ListExpr args) {
if (!nl->HasLength(args, 4) && !nl->HasLength(args, 5)) {
return listutils::typeError(
"4 OR 5 arguments expected. [stream (tuple (...)) x distribution x "
"region x int x bool = _ ({distribution, distributionName} ,region "
",globalId [,updateDistribution])].");
}
ListExpr firstType = nl->First(nl->First(args));
if (!Stream<Tuple>::checkType(firstType)) {
return listutils::typeError(
"1st argument should be stream (tuple (...)). [stream (tuple (...)) x "
"distribution x region x int x bool = _ ({distribution, "
"distributionName} ,region ,globalId [,updateDistribution])].");
}
ListExpr secondType = nl->First(nl->Second(args));
if (!FText::checkType(secondType) && !Distribution::checkType(secondType)) {
return listutils::typeError(
"2nd argument should be distribution OR text. [stream (tuple (...)) x "
"distribution x region x int x bool = _ ({distribution, "
"distributionName} ,region ,globalId [,updateDistribution])].");
}
ListExpr thirdType = nl->First(nl->Third(args));
if (nl->AtomType(thirdType) != SymbolType) {
return listutils::typeError(
"3rd argument should be symbol. [stream (tuple (...)) x distribution "
"x region x int x bool = _ ({distribution, distributionName} ,region "
",globalId [,updateDistribution])].");
}
ListExpr fouthType = nl->First(nl->Fourth(args));
if (nl->AtomType(fouthType) != SymbolType) {
return listutils::typeError(
"4th argument should be symbol. [stream (tuple (...)) x distribution "
"x region x int x bool = _ ({distribution, distributionName} ,region "
",globalId [,updateDistribution])].");
}
if (nl->HasLength(args, 5)) {
ListExpr fifthType = nl->First(nl->Fifth(args));
if (!CcBool::checkType(fifthType)) {
return listutils::typeError(
"5th argument should be bool. [stream (tuple (...)) x distribution "
"x region x int x bool = _ ({distribution, distributionName} ,region "
",globalId [,updateDistribution])].");
}
}
if (!kvsIPC->connected()) {
return listutils::typeError(
"IPC-Connection Error: Not connected to Key-Value Store application.");
}
ListExpr dist_name;
if (FText::checkType(secondType)) {
dist_name = nl->Second(nl->Second(args)); // ((text 'distname')...)
} else {
dist_name = nl->TextAtom(nl->SymbolValue(
nl->Second(nl->Second(args)))); // ( (..) (distribution distname))
}
// Debug:(((stream (tuple ((Osm_id string) (Name string) (Type string)
// (GeoData region) (ServerId int)))) (extendstream (head (feed Buildings) 1)
// ((ServerId (fun (tuple1 TUPLE) (kvsServerId testdist4 (bbox (attr tuple1
// GeoData)) TRUE)))))) (ServerId ServerId))
ListExpr tuple_desc = nl->Second(nl->First(nl->First(args)));
if (nl->IsEqual(nl->First(tuple_desc), Tuple::BasicType()) &&
nl->ListLength(tuple_desc) == 2) {
ListExpr attrL = nl->Second(tuple_desc);
if (IsTupleDescription(attrL)) {
string geo_attr_name = nl->SymbolValue(thirdType);
string id_attr_name = nl->SymbolValue(fouthType);
ListExpr attrType;
int geo_attr_index = FindAttribute(attrL, geo_attr_name, attrType);
int id_attr_index = FindAttribute(attrL, id_attr_name, attrType);
if (geo_attr_index > 0 && id_attr_index > 0) {
return nl->ThreeElemList(
nl->SymbolAtom(Symbol::APPEND()),
nl->ThreeElemList(nl->IntAtom(geo_attr_index),
nl->IntAtom(id_attr_index), dist_name),
nl->First(nl->First(args))); // return input stream
} else {
return listutils::typeError(
"input not as expected: Attributes not found in tuple...");
}
} else {
return listutils::typeError(
"input not as expected: Tuple description not recognized...");
}
} else {
return listutils::typeError(
"input not as expected: TupleType not recognized...");
}
}
int kvsFilterVM(Word* args, Word& result, int message, Word& local,
Supplier s) {
class FilterLocal {
public:
FilterLocal(Address streamAddr, int distRef)
: distRef(distRef),
inTupleStream(streamAddr),
filtered(0),
notfiltered(0) {}
int distRef;
Stream<Tuple> inTupleStream;
int filtered, notfiltered;
};
FilterLocal* filter = static_cast<FilterLocal*>(local.addr);
switch (message) {
case OPEN: {
delete filter;
int noSons = qp->GetNoSons(s);
// Distribution:
string distributionName =
static_cast<FText*>(args[noSons - 1].addr)->GetValue();
int distRef = kvsIPC->getDistributionRef(distributionName);
if (distRef < 0) {
ListExpr distributionType = qp->GetSupplierTypeExpr(qp->GetSon(s, 1));
if (Distribution::checkType(distributionType)) {
Distribution* dist = static_cast<Distribution*>(args[1].addr);
distRef = kvsIPC->getDistributionRef(distributionName, dist->type,
dist->toBin());
} else {
cout << "Error: No Distribution specified";
return CANCEL;
}
}
if (distRef >= 0) {
filter = new FilterLocal(args[0].addr, distRef);
filter->inTupleStream.open();
local.addr = filter;
return 0;
} else {
local.addr = 0;
return CANCEL;
}
}
case REQUEST: {
if (filter != 0) {
int noSons = qp->GetNoSons(s);
int geoAttrIndex = ((CcInt*)(args[noSons - 3].addr))->GetIntval() - 1;
int idAttrIndex = ((CcInt*)(args[noSons - 2].addr))->GetIntval() - 1;
bool updateDistribution = false;
if (noSons == 5 + 3) {
updateDistribution = ((CcBool*)(args[4].addr))->GetValue();
}
Tuple* tuple;
cout << "kvsFilter: Starting to process stream..." << endl;
while ((tuple = filter->inTupleStream.request()) != 0) {
const Rectangle<2> mbb =
static_cast<Region*>(tuple->GetAttribute(geoAttrIndex))
->BoundingBox();
unsigned int globalId =
static_cast<CcInt*>(tuple->GetAttribute(idAttrIndex))->GetValue();
double coords[4];
coords[0] = mbb.MinD(0) * 1000;
coords[1] = mbb.MinD(1) * 1000;
coords[2] = mbb.MaxD(0) * 1000;
coords[3] = mbb.MaxD(1) * 1000;
set<int> results;
int nrCoords = 4;
if (kvsIPC->distFilter(filter->distRef, nrCoords, coords, globalId,
updateDistribution)) {
filter->notfiltered++;
result.addr = tuple;
return YIELD;
} else {
filter->filtered++;
}
}
filter->inTupleStream.close();
cout << "Finished kvsFilter - filtered:" << filter->filtered
<< " not filtered:" << filter->notfiltered << endl;
delete filter;
local.addr = 0;
}
result.addr = 0;
return CANCEL;
}
case CLOSE: {
delete filter;
local.addr = 0;
}
return 0;
}
return 0;
}
ListExpr kvsDistStreamTM(ListExpr args) {
if (!nl->HasLength(args, 1)) {
return listutils::typeError(
"Expecting 1 argument "
"[{distribution, text} = ({distribution, distributionName})].");
}
if (!Distribution::checkType(nl->First(nl->First(args))) &&
!FText::checkType(nl->First(nl->First(args)))) {
return listutils::typeError(
"1st argument should be distribution OR text(distribution name). "
"[{distribution, text} = ({distribution, distributionName})].");
}
// append distribution name
ListExpr dist_name;
if (FText::checkType(nl->First(nl->First(args)))) {
dist_name = nl->Second(nl->First(args)); // ((text 'distname')...)
} else {
string dist_name_str = nl->SymbolValue(nl->Second(nl->First(args)));
dist_name = nl->TextAtom(dist_name_str); // ( (..) (distribution distname))
}
ListExpr resType = nl->TwoElemList(
listutils::basicSymbol<Stream<Tuple> >(),
nl->TwoElemList(
listutils::basicSymbol<Tuple>(),
nl->TwoElemList(
nl->TwoElemList(nl->SymbolAtom("Area"),
nl->SymbolAtom(Rectangle<2>::BasicType())),
nl->TwoElemList(nl->SymbolAtom("Object-Count"),
nl->SymbolAtom(CcInt::BasicType())))));
return nl->ThreeElemList(nl->SymbolAtom(Symbol::APPEND()),
nl->OneElemList(dist_name), resType);
}
int kvsDistStreamVM(Word* args, Word& result, int message, Word& local,
Supplier s) {
result = qp->ResultStorage(s);
std::list<Tuple*>* resultStream = static_cast<list<Tuple*>*>(local.addr);
switch (message) {
case OPEN: {
delete resultStream;
Distribution* dist = 0;
int noSons = qp->GetNoSons(s);
// Distribution:
string distributionName =
static_cast<FText*>(args[noSons - 1].addr)->GetValue();
int distRef = kvsIPC->getDistributionRef(distributionName);
if (distRef < 0) {
ListExpr distributionType = qp->GetSupplierTypeExpr(qp->GetSon(s, 0));
if (Distribution::checkType(distributionType)) {
dist = static_cast<Distribution*>(args[0].addr);
distRef = kvsIPC->getDistributionRef(distributionName, dist->type,
dist->toBin());
} else {
cout << "Error: No Distribution specified";
return CANCEL;
}
}
if (distRef >= 0) {
resultStream = new std::list<Tuple*>;
// create tupleType
ListExpr resultTupleType = nl->TwoElemList(
nl->SymbolAtom(Tuple::BasicType()),
nl->TwoElemList(
nl->TwoElemList(nl->SymbolAtom("Area"),
nl->SymbolAtom(Rectangle<2>::BasicType())),
nl->TwoElemList(nl->SymbolAtom("Object-Count"),
nl->SymbolAtom(CcInt::BasicType()))));
ListExpr numResultTupleType =
SecondoSystem::GetCatalog()->NumericType(resultTupleType);
TupleType* tt = new TupleType(numResultTupleType);
// retrieve distribution
if (dist == 0) {
// get distribution snapshot from application
string distData;
if (kvsIPC->getDistributionData(distRef, &distData)) {
dist = Distribution::getInstance(distData);
}
}
// retrieve list
std::list<std::pair<double*, int> > areaList;
dist->createAreaObjectCountList(&areaList);
// create result stream
while (!areaList.empty()) {
double* mbb = areaList.front().first;
int count = areaList.front().second;
Tuple* tempTuple = new Tuple(tt);
CcInt* tempCount = new CcInt(count);
Rectangle<2>* tempRect = new Rectangle<2>(true, &mbb[0], &mbb[2]);
tempTuple->PutAttribute(0, tempRect);
tempTuple->PutAttribute(1, tempCount);
resultStream->push_back(tempTuple);
delete[] mbb;
areaList.pop_front();
}
tt->DeleteIfAllowed();
local.addr = resultStream;
result.addr = 0;
return 0;
} else {
return CANCEL;
}
}
case REQUEST: {
if (resultStream != 0 && !resultStream->empty()) {
Tuple* tempTuple = resultStream->front();
resultStream->pop_front();
result = SetWord(tempTuple);
return YIELD;
} else {
result.addr = 0;
return CANCEL;
}
}
case CLOSE: {
delete resultStream;
local.addr = 0;
}
return CANCEL;
}
return CANCEL;
}
}