along/secondo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
dfd1e745480fabd88139d2804acb90d5b6dc055e
secondo/Algebras/Distributed2/DArray.cpp

1317 lines
33 KiB
C++
Raw Normal View History

firs commit
2026-01-23 17:03:45 +08:00
/*
----
This file is part of SECONDO.
Copyright (C) 2015,
Faculty of Mathematics and Computer Science,
Database Systems for New Applications.
SECONDO is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
SECONDO is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with SECONDO; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
----
//[$][\$]
*/
#include <set>
#include <string>
#include <utility>
#include "DArray.h"
#include "Dist2Helper.h"
#include <boost/thread/lock_guard.hpp>
#include <boost/thread/mutex.hpp>
#include <boost/thread/recursive_mutex.hpp>
using namespace std;
namespace distributed2{
/*
1.1 Implementation of Class ~DArrayElement~
This class represents information about a single worker of a DArray.
*/
DArrayElement::DArrayElement(const string& _server, const int _port,
const int _num, const string& _config):
server(_server), port(_port), num(_num),config(_config) {}
DArrayElement::DArrayElement(const DArrayElement& src):
server(src.server), port(src.port), num(src.num),
config(src.config) {}
DArrayElement& DArrayElement::operator=(const DArrayElement& src){
this->server = src.server;
this->port = src.port;
this->num = src.num;
this->config = src.config;
return *this;
}
DArrayElement::~DArrayElement() {}
void DArrayElement::set(const string& server, const int port,
const int num, const string& config){
this->server = server;
this->port = port;
this->num = num;
this->config = config;
}
bool DArrayElement::operator==(const DArrayElement& other) const{
return (this->port == other.port)
&& (this->num == other.num)
&& (this->server == other.server)
&& (this->config == other.config);
}
bool DArrayElement::operator<(const DArrayElement& other) const {
if(this->port < other.port) return true;
if(this->port > other.port) return false;
if(this->num < other.num) return true;
if(this->num > other.num) return false;
if(this->server < other.server) return true;
if(this->server > other.server) return false;
if(this->config < other.config) return true;
// equal or greater
return false;
}
bool DArrayElement::operator>(const DArrayElement& other) const {
if(this->port > other.port) return true;
if(this->port < other.port) return false;
if(this->num > other.num) return true;
if(this->num < other.num) return false;
if(this->server > other.server) return true;
if(this->server < other.server) return false;
if(this->config > other.config) return true;
// equal or greater
return false;
}
ListExpr DArrayElement::toListExpr() const{
return nl->ThreeElemList(
nl->TextAtom(server),
nl->IntAtom(port),
nl->TextAtom(config));
}
bool DArrayElement::readFrom(SmiRecord& valueRecord, size_t& offset){
string s;
if(!readVar(s,valueRecord,offset)){
return false;
}
uint32_t p;
if(!readVar(p,valueRecord,offset)){
return false;
}
string c;
if(!readVar(c,valueRecord,offset)){
return false;
}
server = s;
port = p;
num = -1;
config = c;
return true;
}
bool DArrayElement::saveTo(SmiRecord& valueRecord, size_t& offset) const{
if(!writeVar(server,valueRecord,offset)){
return false;
}
if(!writeVar(port,valueRecord,offset)){
return false;
}
if(!writeVar(config,valueRecord,offset)){
return false;
}
return true;
}
void DArrayElement::print(ostream& out)const{
out << "( S: " << server << ", P : " << port
<< "Num : " << num
<< ", C : " << config << ")";
}
bool InDArrayElement(ListExpr list, DArrayElement& result){
if(!nl->HasLength(list,3)){
return false;
}
ListExpr e1 = nl->First(list);
ListExpr e2 = nl->Second(list);
ListExpr e4 = nl->Third(list);
string server;
int port;
int num;
string config;
if(nl->AtomType(e1) == StringType){
server = nl->StringValue(e1);
} else if(nl->AtomType(e1) == TextType){
server = nl->Text2String(e1);
} else {
return false;
}
stringutils::trim(server);
if(server.empty()){
return false;
}
if(nl->AtomType(e2) != IntType){
return false;
}
port = nl->IntValue(e2);
if(port <=0){
return false;
}
if(nl->AtomType(e4) == StringType){
config = nl->StringValue(e4);
} else if(nl->AtomType(e4) == TextType){
config = nl->Text2String(e4);
} else {
return false;
}
stringutils::trim(config);
if(config.empty()){
return false;
}
num = -1;
result.set(server,port,num,config);
return true;
}
std::string getName(const arrayType a){
switch(a){
case DARRAY: return "darray";
case DFARRAY: return "dfarray";
case DFMATRIX: return "dfmatrix";
case SDARRAY: return "sdarray";
}
assert(false);
return "unknown";
}
std::string getPName(const arrayType a){
switch(a){
case DARRAY: return "pdarray";
case DFARRAY: return "pdfarray";
case SDARRAY: return "psdarray";
case DFMATRIX : return "unknown";//partially type of dmatrix does not exist
}
assert(false);
return "unknown";
}
ListExpr wrapType(const arrayType a, ListExpr subtype){
return nl->TwoElemList( nl->SymbolAtom(getName(a)),
subtype);
}
ListExpr wrapPType(const arrayType a, ListExpr subtype){
return nl->TwoElemList( nl->SymbolAtom(getPName(a)),
subtype);
}
/*
3.1 Implementation of DistTypeBase
3.1.1 Constructors
*/
DistTypeBase::DistTypeBase(const std::vector<DArrayElement>& _worker,
const string& _name):
worker(_worker), name(_name), defined(true),keepRemoteObjects(false){}
DistTypeBase::DistTypeBase( const string& _name):
worker(), name(_name), defined(true),keepRemoteObjects(false){}
DistTypeBase::DistTypeBase( const DistTypeBase& src): worker(src.worker),
name(src.name), defined(src.defined),
keepRemoteObjects(src.keepRemoteObjects) {}
DistTypeBase& DistTypeBase::operator=(const DistTypeBase& src){
worker = src.worker;
name = src.name;
defined = src.defined;
keepRemoteObjects = src.keepRemoteObjects;
return *this;
}
void DistTypeBase::set(const std::string& name,
const std::vector<DArrayElement>& worker){
if(!stringutils::isIdent(name)){ // invalid
this->name = "";
this->defined = false;
return;
}
defined = true;
this->name = name;
this->worker = worker;
}
bool DistTypeBase::equalWorkers(const DistTypeBase& other) const{
return equalWorkers(other.worker);
}
bool DistTypeBase::equalWorkers(const std::vector<DArrayElement>& w) const{
if(worker.size() != w.size()){
return false;
}
for(size_t i=0;i<worker.size();i++){
if(worker[i] != w[i]){
return false;
}
}
return true;
}
size_t DistTypeBase::numOfWorkers() const{
return worker.size();
}
const DArrayElement& DistTypeBase::getWorker(int i) const{
if(i< 0 || i >= (int) worker.size()){
assert(false);
// throw "Invalid worker number";
}
return worker[i];
}
std::string DistTypeBase::getName() const{
return name;
}
void DistTypeBase::makeUndefined(){
worker.clear();
name = "";
defined = false;
}
bool DistTypeBase::setName( const std::string& n){
if(!stringutils::isIdent(n)){
return false;
}
name = n;
return true;
}
/*
3.2 Implementation of DArrayT
*/
DArrayBase::DArrayBase(const std::vector<uint32_t>& _map,
const std::string& _name):
DistTypeBase(_name),
map(_map){
if(!stringutils::isIdent(name) || map.size() ==0 ){ // invalid
makeUndefined();
return;
}
defined = true;
}
DArrayBase::DArrayBase(const std::vector<uint32_t>& _map,
const std::string& _name,
const std::vector<DArrayElement>& _worker):
DistTypeBase(_worker, _name), map(_map){
if(!stringutils::isIdent(name) || map.size() ==0 || !checkMap()){
// invalid
name = "";
defined = false;
return;
}
defined = true;
}
DArrayBase::DArrayBase(const DArrayBase& src):
DistTypeBase(src), map(src.map) {}
DArrayBase& DArrayBase::operator=(const DArrayBase& src) {
boost::lock_guard<boost::recursive_mutex> guard(mapmtx);
DistTypeBase::operator=(src);
this->map = src.map;
return *this;
}
void DArrayBase::set(const std::string& name,
const std::vector<DArrayElement>& worker){
boost::lock_guard<boost::recursive_mutex> guard(mapmtx);
DistTypeBase::set(name, worker);
this->map = createStdMap(worker.size(),worker.size());
}
void DArrayBase::set(const size_t size, const std::string& name,
const std::vector<DArrayElement>& worker){
boost::lock_guard<boost::recursive_mutex> guard(mapmtx);
DistTypeBase::set(name, worker);
this->map = createStdMap(size,worker.size());
}
bool DArrayBase::equalMapping(DArrayBase& a, bool ignoreSize )const{
if(getType()==DFMATRIX){
return false;
}
boost::lock_guard<boost::recursive_mutex> guard(mapmtx);
const vector<uint32_t> m = a.getMap();
if(!ignoreSize && (map.size()!=m.size())){
return false;
}
size_t minV = std::min(map.size(), m.size());
for(size_t i=0;i<minV;i++){
if(map[i]!=m[i]){
return false;
}
}
return true;
}
void DArrayBase::set(const std::vector<uint32_t>& m,
const std::string& name,
const std::vector<DArrayElement>& worker){
if(!stringutils::isIdent(name) || m.size() ==0){ // invalid
makeUndefined();
return;
}
boost::lock_guard<boost::recursive_mutex> guard(mapmtx);
defined = true;
this->name = name;
this->map = m;
this->worker = worker;
if(!checkMap()){
makeUndefined();
}
}
void DArrayBase::setStdMap(size_t size){
boost::lock_guard<boost::recursive_mutex> guard(mapmtx);
map = createStdMap(size, worker.size());
}
const DArrayElement& DArrayBase::getWorkerForSlot(int i) const{
if(i<0 || i>= (int) map.size()){
cerr << "access for worker " << i << endl;
cerr << "number of workers is " << map.size() << endl;
assert(false);
}
boost::lock_guard<boost::recursive_mutex> guard(mapmtx);
return getWorker(map[i]);
}
size_t DArrayBase::getWorkerIndexForSlot(int i){
boost::lock_guard<boost::recursive_mutex> guard(mapmtx);
if(i<0 || i>= (int)map.size()){
assert(false);
}
return map[i];
}
void DArrayBase::setResponsible(size_t slot, size_t _worker){
boost::lock_guard<boost::recursive_mutex> guard(mapmtx);
assert(slot < map.size());
assert(_worker < worker.size());
map[slot] = _worker;
}
ListExpr DArrayBase::toListExpr() const{
if(!defined){
return listutils::getUndefined();
}
ListExpr wl;
if(worker.empty()){
wl = nl->TheEmptyList();
} else {
wl = nl->OneElemList(
worker[0].toListExpr());
ListExpr last = wl;
for(size_t i=1;i<worker.size();i++){
last = nl->Append(last, worker[i].toListExpr());
}
}
boost::lock_guard<boost::recursive_mutex> guard(mapmtx);
ListExpr usedSlots = nl->TheEmptyList();
ListExpr uslast = nl->TheEmptyList();
bool usfirst = true;
if(this->isPartially()){
for(size_t i=0;i<this->getSize(); i++){
if(this->isSlotUsed(i)){
if(usfirst){
usedSlots = nl->OneElemList( nl->IntAtom(i));
uslast = usedSlots;
usfirst = false;
} else {
uslast = nl->Append(uslast, nl->IntAtom(i));
}
}
}
}
if(isStdMap()){
if(!this->isPartially()){
return nl->ThreeElemList(nl->SymbolAtom(name),
nl->IntAtom(getSize()),
wl);
} else {
return nl->FourElemList(nl->SymbolAtom(name),
nl->IntAtom(getSize()),
wl, usedSlots);
}
} else if(map.empty()) {
if(!this->isPartially()){
return nl->ThreeElemList(nl->SymbolAtom(name),
nl->TheEmptyList(),
wl);
} else {
return nl->FourElemList(nl->SymbolAtom(name),
nl->TheEmptyList(),
wl, usedSlots);
}
} else {
ListExpr lmap = nl->OneElemList(nl->IntAtom(map[0]));
ListExpr last = lmap;
for(size_t i=1;i<map.size();i++){
last = nl->Append(last, nl->IntAtom(map[i]));
}
if(!this->isPartially()){
return nl->ThreeElemList(nl->SymbolAtom(name), lmap, wl);
} else {
return nl->FourElemList(nl->SymbolAtom(name), lmap, wl, usedSlots);
}
}
}
template<class R>
R* DArrayBase::readFrom(ListExpr list, bool isPartially){
if(listutils::isSymbolUndefined(list)){
std::vector<uint32_t> m;
return new R(m,"");
}
int requiredLength = isPartially?4:3;
if(!nl->HasLength(list,requiredLength)){
return 0;
}
ListExpr Name = nl->First(list);
ListExpr Workers = nl->Third(list);
if( (nl->AtomType(Name) != SymbolType)
||(nl->AtomType(Workers)!=NoAtom)){
return 0;
}
std::string name = nl->SymbolValue(Name);
if(!stringutils::isIdent(name)){
return 0;
}
std::vector<DArrayElement> v;
int wn = 0;
while(!nl->IsEmpty(Workers)){
DArrayElement elem("",0,0,"");
if(!InDArrayElement(nl->First(Workers), elem)){
return 0;
}
elem.setNum(wn);
wn++;
v.push_back(elem);
Workers = nl->Rest(Workers);
}
std::vector<uint32_t> m;
if(nl->AtomType(nl->Second(list))==IntType){
int size = nl->IntValue(nl->Second(list));
if(size <=0){
return 0;
}
m = createStdMap(size,v.size());
} else if(nl->AtomType(nl->Second(list))!=NoAtom){
return 0;
} else {
ListExpr lm = nl->Second(list);
while(!nl->IsEmpty(lm)){
ListExpr first = nl->First(lm);
lm = nl->Rest(lm);
if(nl->AtomType(first)!=IntType){
return 0;
}
int mv = nl->IntValue(first);
if(mv<0 || mv>=(int)v.size()){
return 0;
}
m.push_back(mv);
}
}
std::set<int> usedSlots;
if(isPartially){
ListExpr usedSlotsList = nl->Fourth(list);
if(nl->AtomType(usedSlotsList) != NoAtom){
return 0;
}
while(!nl->IsEmpty(usedSlotsList)){
ListExpr slot = nl->First(usedSlotsList);
usedSlotsList = nl->Rest(usedSlotsList);
if(nl->AtomType(slot) != IntType){
return 0;
}
int s = nl->IntValue(slot);
if(s<0 || (size_t) s >= m.size()){
return 0;
}
usedSlots.insert(s);
}
}
R* result = new R(m,name);
result->setUsedSlots(usedSlots);
swap(result->worker,v);
result->defined = true;
return result;
}
template<class R, bool partially>
bool DArrayBase::open(SmiRecord& valueRecord, size_t& offset,
const ListExpr __attribute__((unused)) typeInfo,
Word& result){
bool defined;
result.addr = 0;
if(!readVar<bool>(defined,valueRecord,offset)){
return false;
}
if(!defined){
std::vector<uint32_t> m;
result.addr = new R(m,"");
return true;
}
bool keepRemoteObjects;
if(!readVar<bool>(keepRemoteObjects,valueRecord,offset)){
return false;
}
// array in smirecord is defined, read size
size_t size;
if(!readVar<size_t>(size,valueRecord,offset)){
return false;
}
// read name
std::string name;
if(!readVar<std::string>(name,valueRecord, offset)){
return false;
}
// read map
uint32_t me;
std::vector<uint32_t> m;
DArrayBase* res = 0;
for(size_t i=0;i<size;i++){
if(!readVar<uint32_t>(me,valueRecord,offset)){
return false;
}
m.push_back(me);
}
res = new R(m,name);
res->setKeepRemoteObjects(keepRemoteObjects);
int wn = 0;
// append workers
size_t numWorkers;
if(!readVar<size_t>(numWorkers,valueRecord, offset)){
delete res;
return false;
}
for(size_t i=0; i< numWorkers; i++){
DArrayElement elem("",0,0,"");
if(!elem.readFrom(valueRecord, offset)){
delete res;
return false;
}
elem.setNum(wn);
wn++;
res->worker.push_back(elem);
}
if(partially){
size_t numUsedSlots;
if(!readVar<size_t>(numUsedSlots, valueRecord,offset)){
delete res;
return false;
}
std::set<int> used;
int us;
for(size_t i=0;i<numUsedSlots;i++){
if(!readVar<int>(us, valueRecord,offset)){
delete res;
return false;
}
if(us >=0 && (size_t)us < res->getSize()){
used.insert(us);
}
}
res->setUsedSlots(used);
}
result.addr = res;
return true;
}
template<bool partially>
bool DArrayBase::save(SmiRecord& valueRecord, size_t& offset,
const ListExpr __attribute__((unused)) typeInfo,
Word& value) {
DArrayBase* a = (DArrayBase*) value.addr;
// defined flag
if(!writeVar(a->defined,valueRecord,offset)){
return false;
}
if(!a->defined){
return true;
}
if(!writeVar(a->keepRemoteObjects,valueRecord,offset)){
return false;
}
// size
size_t s = a->getSize();
if(!writeVar(s,valueRecord,offset)){
return false;
}
// name
if(!writeVar(a->name, valueRecord, offset)){
return false;
}
// map
boost::lock_guard<boost::recursive_mutex> guard(a->mapmtx);
for(size_t i=0;i<a->map.size();i++){
if(!writeVar(a->map[i], valueRecord,offset)){
return false;
}
}
// workers
if(!writeVar(a->worker.size(), valueRecord, offset)){
return false;
}
for(size_t i=0;i<a->worker.size();i++){
if(!a->worker[i].saveTo(valueRecord,offset)){
return false;
}
}
if(partially){
std::set<int> used = a->getUsedSlots();
size_t s = used.size();
if(!writeVar(s,valueRecord,offset)){
return false;
}
int c = 0;
for(auto it = used.begin();it!=used.end();it++){
int slot = *it;
c++;
if(!writeVar(slot,valueRecord,offset)){
return false;
}
}
}
return true;
}
// instatiations
template
bool DArrayBase::save<true>(SmiRecord&, size_t& ,
const ListExpr __attribute__((unused)) ,
Word& );
template
bool DArrayBase::save<false>(SmiRecord&, size_t& ,
const ListExpr __attribute__((unused)) ,
Word& );
std::vector<uint32_t> DArrayBase::createStdMap(const uint32_t size,
const int numWorkers){
std::vector<uint32_t> map;
for(uint32_t i=0;i<size;i++){
map.push_back(i%numWorkers);
}
return map;
}
void DArrayBase::print(std::ostream& out)const{
if(!defined){
out << "undefined";
return;
}
boost::lock_guard<boost::recursive_mutex> guard(mapmtx);
out << "Name : " << name <<", size : " << map.size()
<< " workers : [" ;
for(size_t i =0;i<worker.size();i++){
if(i>0) out << ", ";
worker[i].print(out);
}
out << "]";
out << "map = [";
for(uint32_t i=0;i<map.size();i++){
if(i>0){
out << ", ";
}
out << i << " -> " << map[i];
}
out << "]";
}
void DArrayBase::makeUndefined(){
boost::lock_guard<boost::recursive_mutex> guard(mapmtx);
DistTypeBase::makeUndefined();
map.clear();
}
bool DArrayBase::checkMap(){
boost::lock_guard<boost::recursive_mutex> guard(mapmtx);
for(size_t i=0;i<map.size();i++){
if(map[i] >= worker.size()){
return false;
}
}
return true;
}
bool DArrayBase::isStdMap() const{
boost::lock_guard<boost::recursive_mutex> guard(mapmtx);
int s = worker.size();
for(size_t i=0;i<map.size();i++){
if(map[i]!= i%s){
return false;
}
}
return true;
}
ostream& operator<<(ostream& out, const DArrayElement& elem){
elem.print(out);
return out;
}
ostream& operator<<(ostream& out, const DArrayBase& a){
a.print(out);
return out;
}
template<arrayType T>
bool DArrayT<T>::checkType(const ListExpr list){
if(!nl->HasLength(list,2)){
return false;
}
if(!listutils::isSymbol(nl->First(list), BasicType())){
return false;
}
// for other than DARRAY, only relations are allowed as a
// subtype
if(T!=DARRAY){
return Relation::checkType(nl->Second(list));
}
// check that second arghument is an valid tyoe
SecondoCatalog* ctl = SecondoSystem::GetCatalog();
std::string name;
int algid, type;
if(!ctl->LookUpTypeExpr(nl->Second(list), name, algid, type)){
return false;
}
AlgebraManager* am = SecondoSystem::GetAlgebraManager();
ListExpr errorInfo = listutils::emptyErrorInfo();
if(!am->TypeCheck(algid,type,nl->Second(list),errorInfo)){
return false;
}
return true;
}
template<arrayType T>
bool PDArrayT<T>::checkType(const ListExpr list){
if(!nl->HasLength(list,2)){
return false;
}
if(!listutils::isSymbol(nl->First(list), BasicType())){
return false;
}
// for other than DARRAY, only relations are allowed as a
// subtype
if(T!=DARRAY){
return Relation::checkType(nl->Second(list));
}
// check that second arghument is an valid tyoe
SecondoCatalog* ctl = SecondoSystem::GetCatalog();
std::string name;
int algid, type;
if(!ctl->LookUpTypeExpr(nl->Second(list), name, algid, type)){
return false;
}
AlgebraManager* am = SecondoSystem::GetAlgebraManager();
ListExpr errorInfo = listutils::emptyErrorInfo();
if(!am->TypeCheck(algid,type,nl->Second(list),errorInfo)){
return false;
}
return true;
}
// instantiation
template
bool PDArrayT<DARRAY>::checkType(const ListExpr list);
template
bool PDArrayT<DFARRAY>::checkType(const ListExpr list);
/*
Implementation of ~DMatrix~
*/
const std::string DFMatrix::BasicType(){
return "dfmatrix";
}
DFMatrix::DFMatrix(const size_t _size , const std::string& _name):
DistTypeBase(_name), size(_size) {
if(!stringutils::isIdent(name) || size ==0 ){ // invalid
name = "";
defined = false;
size = 0;
return;
}
defined = true;
}
DFMatrix::DFMatrix(const size_t _size, const std::string& _name,
const std::vector<DArrayElement>& _worker):
DistTypeBase(_worker, _name), size(_size) {
if(!stringutils::isIdent(name) || size ==0 ){
// invalid
name = "";
defined = false;
size = 0;
return;
}
defined = true;
}
void DFMatrix::setSize(size_t newSize){
assert(newSize > 0);
this->size = newSize;
}
bool DFMatrix::open(SmiRecord& valueRecord, size_t& offset,
const ListExpr __attribute__((unused)) typeInfo,
Word& result){
bool defined;
result.addr = 0;
if(!readVar<bool>(defined,valueRecord,offset)){
return false;
}
if(!defined){
std::vector<uint32_t> m;
result.addr = new DFMatrix(0,"");
return true;
}
bool keepRemoteObjects;
if(!readVar<bool>(keepRemoteObjects,valueRecord,offset)){
return false;
}
// array in smirecord is defined, read size
size_t size;
if(!readVar<size_t>(size,valueRecord,offset)){
return false;
}
// read name
std::string name;
if(!readVar<std::string>(name,valueRecord, offset)){
return false;
}
DFMatrix* res = new DFMatrix(size,name);
res->setKeepRemoteObjects(keepRemoteObjects);
int wn = 0;
// append workers
size_t numWorkers;
if(!readVar<size_t>(numWorkers,valueRecord, offset)){
return false;
}
for(size_t i=0; i< numWorkers; i++){
DArrayElement elem("",0,0,"");
if(!elem.readFrom(valueRecord, offset)){
delete res;
return false;
}
elem.setNum(wn);
wn++;
res->worker.push_back(elem);
}
result.addr = res;
return true;
}
bool DFMatrix::save(SmiRecord& valueRecord, size_t& offset,
const ListExpr __attribute__((unused)) typeInfo,
Word& value) {
DFMatrix* a = (DFMatrix*) value.addr;
// defined flag
if(!writeVar(a->defined,valueRecord,offset)){
return false;
}
if(!a->defined){
return true;
}
if(!writeVar(a->keepRemoteObjects,valueRecord,offset)){
return false;
}
// size
size_t s = a->size;
if(!writeVar(s,valueRecord,offset)){
return false;
}
// name
if(!writeVar(a->name, valueRecord, offset)){
return false;
}
// workers
if(!writeVar(a->worker.size(), valueRecord, offset)){
return false;
}
for(size_t i=0;i<a->worker.size();i++){
if(!a->worker[i].saveTo(valueRecord,offset)){
return false;
}
}
return true;
}
bool DFMatrix::checkType(ListExpr e){
if(!nl->HasLength(e,2)) {
return false;
}
if(!listutils::isSymbol(nl->First(e), BasicType())){
return false;
}
return Relation::checkType(nl->Second(e));
}
ListExpr DFMatrix::toListExpr()const{
if(!defined){
return listutils::getUndefined();
}
ListExpr wl;
if(worker.empty()){
wl = nl->TheEmptyList();
} else {
wl = nl->OneElemList(
worker[0].toListExpr());
ListExpr last = wl;
for(size_t i=1;i<worker.size();i++){
last = nl->Append(last, worker[i].toListExpr());
}
}
return nl->ThreeElemList(nl->SymbolAtom(name),
nl->IntAtom(size),
wl);
}
DFMatrix* DFMatrix::readFrom(ListExpr list){
if(listutils::isSymbolUndefined(list)){
return new DFMatrix(0,"");
}
if(!nl->HasLength(list,3)){
return 0;
}
ListExpr Name = nl->First(list);
ListExpr Workers = nl->Third(list);
if( (nl->AtomType(Name) != SymbolType)
||(nl->AtomType(Workers)!=NoAtom)){
return 0;
}
std::string name = nl->SymbolValue(Name);
if(!stringutils::isIdent(name)){
return 0;
}
std::vector<DArrayElement> v;
int wn = 0;
while(!nl->IsEmpty(Workers)){
DArrayElement elem("",0,0,"");
if(!InDArrayElement(nl->First(Workers), elem)){
return 0;
}
elem.setNum(wn);
wn++;
v.push_back(elem);
Workers = nl->Rest(Workers);
}
std::vector<uint32_t> m;
size_t s;
if(nl->AtomType(nl->Second(list))==IntType){
int size = nl->IntValue(nl->Second(list));
if(size <=0){
return 0;
}
s = size;
} else {
return 0;
}
DFMatrix* result = new DFMatrix(s, name, v);
result->defined = true;
return result;
}
// template instantiaons
template bool DArrayBase::open<DArray, false>(SmiRecord&, size_t&,
const ListExpr, Word&);
template bool DArrayBase::open<DFArray, false>(SmiRecord&, size_t&,
const ListExpr, Word&);
template bool DArrayBase::open<PDArray, true>(SmiRecord&, size_t&,
const ListExpr, Word&);
template bool DArrayBase::open<PDFArray,true>(SmiRecord&, size_t&,
const ListExpr, Word&);
template DArray* DArrayBase::readFrom<DArray>(ListExpr, bool);
template DFArray* DArrayBase::readFrom<DFArray>(ListExpr, bool);
template PDArray* DArrayBase::readFrom<PDArray>(ListExpr, bool);
template PDFArray* DArrayBase::readFrom<PDFArray>(ListExpr, bool);
template class DArrayT<DARRAY>;
template class DArrayT<DFARRAY>;
ListExpr SDArray::Property(){
return ( nl->TwoElemList (
nl->FourElemList (
nl->StringAtom("Signature"),
nl->StringAtom("Example Type List"),
nl->StringAtom("List Rep"),
nl->StringAtom("Example List")),
nl->FourElemList (
nl->StringAtom("-> SIMPLE"),
nl->StringAtom(SDArray::BasicType()),
nl->StringAtom("(name (worker1 worker2 ...))"),
nl->StringAtom("(sda (('127.0.0.1', 1234, 'SecondoConfig.ini')))")
)));
}
Word SDArray::IN( const ListExpr typeInfo, const ListExpr instance,
const int errorPos, ListExpr& errorInfo, bool& correct ) {
correct = false;
if(listutils::isSymbolUndefined(instance)){
SDArray* sd = new SDArray("");
sd->makeUndefined();
Word res(sd);
correct = true;
return res;
}
if(!nl->HasLength(instance,2)){
cmsg.inFunError("two Elem list expected");
return SetWord((void*)0);
}
ListExpr nameList = nl->First(instance);
if(nl->AtomType(nameList)!=SymbolType){
cmsg.inFunError("invalid name (not a symbol)");
return SetWord((void*)0);
}
string name = nl->SymbolValue(nameList);
ListExpr workerList = nl->Second(instance);
if(nl->AtomType(workerList)!=NoAtom){
cmsg.inFunError("invalid worker list (not a list)");
return SetWord((void*)0);
}
vector<DArrayElement> workers;
std::set<std::pair<string,int> > usedServers;
while(!nl->IsEmpty(workerList)){
ListExpr singleWorker = nl->First(workerList);
workerList = nl->Rest(workerList);
DArrayElement worker("",0,0,"");
if(!InDArrayElement(singleWorker,worker)){
cmsg.inFunError("found invalid worker definition");
return SetWord((void*)0);
}
pair<string,int> con(worker.getHost(), worker.getPort());
if(usedServers.find(con)!=usedServers.end()){
cmsg.inFunError("found connection twice");
return SetWord((void*)0);
}
usedServers.insert(con);
workers.push_back(worker);
}
SDArray* res = new SDArray(workers,name);
correct = true;
return SetWord(res);
}
ListExpr SDArray::Out( ListExpr typeInfo, Word value ) {
SDArray* a = (SDArray*) value.addr;
if(!a->IsDefined()){
return listutils::getUndefined();
}
ListExpr nameList = nl->SymbolAtom(a->getName());
int s = a->getSize();
ListExpr workers = nl->TheEmptyList();
if(s>0){
workers = nl->OneElemList(a->getWorker(0).toListExpr());
ListExpr last = workers;
for(int i=1;i<s;i++){
last = nl->Append(last, a->getWorker(i).toListExpr());
}
}
return nl->TwoElemList(nameList, workers);
}
bool SDArray::Open( SmiRecord& valueRecord,
size_t& offset, const ListExpr typeInfo,
Word& value ) {
// check whether defined
bool def;
valueRecord.Read(&def, sizeof(bool), offset);
offset+=sizeof(bool);
if(!def){
SDArray* res = new SDArray("");
res->makeUndefined();
value.addr = res;
return true;
}
bool keepRemoteObjects;
if(!readVar<bool>(keepRemoteObjects,valueRecord,offset)){
return false;
}
// read number of workers and size of the name
size_t size;
valueRecord.Read(&size,sizeof(size_t), offset);
offset += sizeof(size_t);
size_t namesize;
valueRecord.Read(&namesize,sizeof(size_t),offset);
offset += sizeof(size_t);
// read the name
char* buffer = new char[namesize];
valueRecord.Read(buffer,namesize,offset);
offset += namesize;
string name(buffer,namesize);
delete[] buffer;
// read the workers
vector<DArrayElement> workers;
for(size_t i=0;i<size;i++){
DArrayElement elem("",0,0,"");
if(!elem.readFrom(valueRecord,offset)){
return false;
}
elem.setNum(i);
workers.push_back(elem);
}
// build result
SDArray* res = new SDArray(workers,name);
res->setKeepRemoteObjects(keepRemoteObjects);
value.addr = res;
return true;
}
bool SDArray::Save( SmiRecord& valueRecord, size_t& offset,
const ListExpr typeInfo, Word& value ) {
SDArray* a = (SDArray*) value.addr;
bool def = a->IsDefined();
valueRecord.Write(&def,sizeof(bool),offset);
offset+=sizeof(bool);
if(!def) return true;
if(!writeVar(a->keepRemoteObjects,valueRecord,offset)){
return false;
}
size_t size = a->getSize();
string name = a->getName();
size_t namesize = name.length();
const char* buffer = name.c_str();
valueRecord.Write(&size,sizeof(size_t),offset);
offset += sizeof(size_t);
valueRecord.Write(&namesize,sizeof(size_t), offset);
offset += sizeof(size_t);
valueRecord.Write(buffer,namesize,offset);
offset += namesize;
for(size_t i=0;i<size;i++){
a->getWorker(i).saveTo(valueRecord,offset);
}
return true;
}
} // end of namespace distributed2
Reference in New Issue Copy Permalink