along/secondo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
dfd1e745480fabd88139d2804acb90d5b6dc055e
secondo/Algebras/PartitionedStream/SortMerge.cpp
along dfd1e74548 firs commit
2026-01-23 17:03:45 +08:00

1466 lines
36 KiB
C++
Raw Blame History

/*
----
This file is part of SECONDO.
Copyright (C) 2004-2008, University in Hagen, Faculty of Mathematics and
Computer Science, Database Systems for New Applications.
SECONDO is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
SECONDO is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with SECONDO; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
----
//paragraph [1] Title: [{\Large \bf \begin{center}] [\end{center}}]
//paragraph [10] Footnote: [{\footnote{] [}}]
//[TOC] [\tableofcontents]
Dec 2008. M. Spiekermann: Integration of code implemented formerly in the
ExtRelation-Algebra.
[1] Implementation of special variants for sort merge joins
[TOC]
0 Overview
This file contains the implementation of algorithms for external sorting,
merging wich integrate sample techniques to guarantee that the output will
also start with a random prefix.
1 Includes and defines
*/
#include <vector>
#include <list>
#include <set>
#include <queue>
#include "LogMsg.h"
#include "StandardTypes.h"
#include "Algebras/Relation-C++/RelationAlgebra.h"
#include "CPUTimeMeasurer.h"
#include "QueryProcessor.h"
#include "SecondoInterface.h"
#include "StopWatch.h"
#include "Counter.h"
#include "Progress.h"
#include "RTuple.h"
#include "Algebras/ExtRelation-C++/Hashtable.h"
#include "StreamIterator.h"
#include "Algebras/ExtRelation-C++/Tupleorder.h"
extern NestedList* nl;
extern QueryProcessor* qp;
using namespace std;
/*
2 Operators
2.1 class ~SortByLocalInfo2~
This class contains some big changes compared to the basic implementation, thus for the
sake of simplicity and programming time efficiency we did not try to rearrange the code
in such a way that we can keep most of them in a base class.
As a result we accept to have many redundant code lines !!!
*/
class SortByLocalInfo2 : protected ProgressWrapper
{
public:
SortByLocalInfo2( Word stream, const bool lexicographic,
void *tupleCmp, ProgressLocalInfo* p ) :
ProgressWrapper(p),
stream( stream ),
currentIndex( 0 ),
lexiTupleCmp( lexicographic ?
(LexicographicalTupleSmaller*)tupleCmp :
0 ),
tupleCmpBy( lexicographic ? 0 : (TupleCompareBy*)tupleCmp ),
lexicographic( lexicographic),
pq( UniversalCompare<LexicographicalTupleSmaller>() )
{}
public:
void PrepareResultIteration(bool mkRndSubset = false)
{
InitRuns();
if (!mkRndSubset)
{
CreateRuns();
}
else
{
MakeRndSubset();
}
FinishRuns();
InitMerge();
}
void InitRuns()
{
currentRun = &queue[0];
nextRun = &queue[1];
c = 0, i = 0, a = 0, n = 0, m = 0, r = 0; // counter variables
newRelation = true;
MAX_MEMORY = qp->FixedMemory();
cmsg.info("ERA:ShowMemInfo")
<< "Sortby.MAX_MEMORY (" << MAX_MEMORY/1024 << " kb)" << endl;
cmsg.send();
lastTuple.setTuple(0);
minTuple.setTuple(0);
rel=0;
}
void CreateRuns()
{
StreamIterator<Tuple> is(stream);
while( is.valid() ) // consume the stream completely
{
Tuple* tuple = *is;
AppendTuple(tuple);
++is;
tuple->DeleteIfAllowed();
}
}
inline StreamIterator<Tuple> GetIterator()
{
return StreamIterator<Tuple>(stream);
}
void MakeRndSubset()
{
// Before a tuple will be passed to the sorting algorithm
// it may be chosen as member of a random subset.
StreamIterator<Tuple> is(stream);
while( is.valid() ) // consume the stream completely
{
// choose random tuples
size_t i = 0;
bool replaced = false;
Tuple* s = rtBuf.ReplacedByRandom(*is, i, replaced);
if ( replaced )
{
// s was replaced by *is
if (s != 0) {
MAX_MEMORY -= (*is)->GetSize();
MAX_MEMORY += s->GetSize();
AppendTuple(s);
s->DeleteIfAllowed();
}
}
else
{
assert(s == 0);
// s == 0, and *is was not stored in buffer
AppendTuple(*is);
(*is)->DeleteIfAllowed();
}
++is;
}
}
HashTable* CreateHashTable(int buckets, int i, int j)
{
// create a hash table for the tuples stored in
// rtBuf and return a pointer to it.
ht = new HashTable(buckets, CmpTuples(i,j));
RandomTBuf::iterator tuple = rtBuf.begin();
for( ; tuple != rtBuf.end(); tuple++) {
if (*tuple) {
ht->add( *tuple, ((*tuple)->HashValue(i) ) );
}
}
return ht;
}
HashTable* GetHashTable() const { return ht; }
void FinishRuns()
{
ShowPartitionInfo(c,a,n,m,r,rel);
Counter::getRef("Sortby:ExternPartitions") = relations.size();
// copy the lastRun and NextRun runs into tuple buffers
// which stay in memory.
CopyQueue2Vector(0);
CopyQueue2Vector(1);
}
inline void AppendTuple(Tuple* t)
{
progress->read++;
c++; // tuple counter;
TupleAndRelPos nextTuple(t, tupleCmpBy);
if( MAX_MEMORY > (size_t)t->GetSize() )
{
currentRun->push(nextTuple);
i++; // increment Tuples in memory counter
MAX_MEMORY -= t->GetSize();
}
else
{ // memory is completely used
progress->state = 1;
if ( newRelation )
{ // create new relation
r++;
rel = new TupleBuffer( 0 );
GenericRelationIterator *iter = 0;
relations.push_back( std::make_pair( rel, iter ) );
newRelation = false;
// get first tuple and store it in an relation
currentRun->push(nextTuple);
minTuple = currentRun->topTuple();
rel->AppendTuple( minTuple.tuple );
minTuple.tuple->DeleteIfAllowed();
lastTuple = minTuple;
currentRun->pop();
}
else
{ // check if nextTuple can be saved in current relation
if ( nextTuple < TupleAndRelPos(lastTuple.tuple, tupleCmpBy) )
{ // nextTuple is in order
// Push the next tuple int the heap and append the minimum to
// the current relation and push
currentRun->push(nextTuple);
minTuple = currentRun->topTuple();
rel->AppendTuple( minTuple.tuple );
minTuple.tuple->DeleteIfAllowed();
lastTuple = minTuple;
currentRun->pop();
m++;
}
else
{ // nextTuple is smaller, save it for the next relation
nextRun->push(nextTuple);
n++;
if ( !currentRun->empty() )
{
// Append the minimum to the current relation
minTuple.setTuple(currentRun->top().tuple());
rel->AppendTuple( minTuple.tuple );
minTuple.tuple->DeleteIfAllowed();
lastTuple = minTuple;
currentRun->pop();
}
else
{ //create a new run
newRelation = true;
// swap queues
Heap* helpRun = currentRun;
currentRun = nextRun;
nextRun = helpRun;
ShowPartitionInfo(c,a,n,m,r,rel);
i=n;
a=0;
n=0;
m=0;
} // end new run
} // end next tuple is smaller
} // end of check if nextTuple can be saved in current relation
}// end of memory is completely used
}
/*
It may happen, that the localinfo object will be destroyed
before all internal buffered tuples are delivered stream
upwards, e.g. queries which use a ~head~ operator.
In this case we need to delete also all tuples stored in memory.
*/
~SortByLocalInfo2()
{
while( !mergeTuples.empty() )
{
mergeTuples.topTuple()->DeleteIfAllowed();
mergeTuples.pop();
}
for( int i = 0; i < 2; i++ )
{
while( !queue[i].empty() )
{
queue[i].topTuple()->DeleteIfAllowed();
queue[i].pop();
}
}
// delete information about sorted runs
for( size_t i = 0; i < relations.size(); i++ )
{
delete relations[i].second;
relations[i].second = 0;
//relations[i].first->Clear();
delete relations[i].first;
relations[i].first = 0;
}
delete lexiTupleCmp;
lexiTupleCmp = 0;
delete tupleCmpBy;
tupleCmpBy = 0;
}
Tuple* NextResultTuple()
{
if( mergeTuples.empty() ) // stream finished
return 0;
else
{
// Take the first element out of the merge heap
TupleAndRelPos p = mergeTuples.top();
Tuple* result = p.tuple();
mergeTuples.pop();
// push next tuple into the merge heap
Tuple* t = relations[p.pos].second->GetNextTuple();
if( t != 0 )
{ // run not finished
mergeTuples.push( TupleAndRelPos(t, tupleCmpBy, p.pos) );
}
return result;
}
}
void InitMerge()
{
for( size_t i = 0; i < relations.size(); i++ )
{
if ( relations[i].second != 0 ) {
delete relations[i].second;
}
relations[i].second = relations[i].first->MakeScan();
// Get next tuple from each relation and push it into the heap.
Tuple *t = relations[i].second->GetNextTuple();
if( t != 0 )
{
mergeTuples.push( TupleAndRelPos(t, tupleCmpBy, i) );
}
}
}
protected:
void ShowPartitionInfo( int c, int a, int n,
int m, int r, GenericRelation* rel )
{
int rs = (rel != 0) ? rel->GetNoTuples() : 0;
if ( RTFlag::isActive("ERA:Sort:PartitionInfo") )
{
cmsg.info() << "Current run finished: "
<< " processed tuples=" << c
<< ", append minimum=" << m
<< ", append next=" << n << endl
<< " materialized runs=" << r
<< ", last partition's tuples=" << rs << endl
<< " Runs in memory: queue1= " << queue[0].size()
<< ", queue2= " << queue[1].size() << endl;
cmsg.send();
}
}
void CopyQueue2Vector(int i)
{
assert( i == 0 || i == 1 );
TupleBuffer* tbuf = new TupleBuffer();
GenericRelationIterator *iter = 0;
relations.push_back( std::make_pair( tbuf, iter ) );
while( !queue[i].empty() )
{
Tuple* t = queue[i].top().tuple();
tbuf->AppendTuple(t);
queue[i].pop();
}
}
Word stream;
size_t currentIndex;
// tuple information
LexicographicalTupleSmaller *lexiTupleCmp;
TupleCompareBy *tupleCmpBy;
bool lexicographic;
// sorted runs created by in memory heap filtering
size_t MAX_MEMORY;
typedef std::pair<TupleBuffer*, GenericRelationIterator*> SortedRun;
vector< SortedRun > relations;
typedef TupleQueue Heap;
Heap queue[2];
Heap mergeTuples;
// Alternate queue type which can be constructed with a user specific
// comparison function. Currently, this is only experimental code, the
// member pq is just instantiated not used.
priority_queue< TupleAndRelPos,
vector<TupleAndRelPos>,
UniversalCompare<LexicographicalTupleSmaller> > pq;
private:
Heap* currentRun;
Heap* nextRun;
TupleBuffer* rel;
size_t c, i, a, n, m, r; // counter variables
bool newRelation;
RTuple lastTuple;
RTuple minTuple;
RandomTBuf rtBuf;
HashTable* ht;
};
/*
2.1.1 Value mapping function of operator ~sortby~
The argument vector ~args~ contains in the first slot ~args[0]~ the stream and
in ~args[2]~ the number of sort attributes. ~args[3]~ contains the index of the
first sort attribute, ~args[4]~ a boolean indicating wether the stream should
be sorted in ascending order with regard to the sort first attribute. ~args[5]~
and ~args[6]~ contain these values for the second sort attribute and so on.
*/
template<bool lexicographically> int
SortBy(Word* args, Word& result, int message, Word& local, Supplier s)
{
// args[0] : stream
// args[1] : ignored
// args[2] : the number of sort attributes
// args[3] : the index of the first sort attribute
// args[4] : a boolean which indicates if sortorder should
// be asc or desc.
// args[5] : Same as 3 but for the second sort attribute
// args[6] : Same as 4
// ....
//
LocalInfo<SortByLocalInfo2>* li;
li = static_cast<LocalInfo<SortByLocalInfo2>*>( local.addr );
switch(message)
{
case OPEN:
{
if ( li ) delete li;
li = new LocalInfo<SortByLocalInfo2>();
local.addr = li;
// at this point the local value is well defined
// afterwards progress request calls are
// allowed.
li->ptr = 0;
qp->Open(args[0].addr);
return 0;
}
case REQUEST:
{
if ( li->ptr == 0 )
{
void *tupleCmp = CompareObject(lexicographically, args).getPtr();
//Sorting is done in the following constructor. It was moved from
//OPEN to REQUEST to avoid long delays in the OPEN method, which are
//a problem for progress estimation
li->ptr = new SortByLocalInfo2( args[0],
lexicographically,
tupleCmp, li );
li->ptr->PrepareResultIteration();
}
SortByLocalInfo2* sli = li->ptr;
result.setAddr( sli->NextResultTuple() );
li->returned++;
return result.addr != 0 ? YIELD : CANCEL;
}
case CLOSE:
qp->Close(args[0].addr);
return 0;
case CLOSEPROGRESS:
if ( li ) {
delete li;
local.addr = 0;
}
return 0;
case REQUESTPROGRESS:
ProgressInfo p1;
ProgressInfo *pRes;
const double uSortBy = 0.000396; //millisecs per byte input and sort
const double vSortBy = 0.000194; //millisecs per byte output
const double oSortBy = 0.00004; //offset due to writing to disk
//not yet measurable
pRes = (ProgressInfo*) result.addr;
if( !li ) return CANCEL;
else
{
if (qp->RequestProgress(args[0].addr, &p1))
{
pRes->Card = li->returned == 0 ? p1.Card : li->read;
pRes->CopySizes(p1);
pRes->Time = //li->state = 0 or 1
p1.Time
+ pRes->Card * p1.Size * (uSortBy + oSortBy * li->state)
+ pRes->Card * p1.Size * vSortBy;
pRes->Progress =
(p1.Progress * p1.Time
+ li->read * p1.Size * (uSortBy + oSortBy * li->state)
+ li->returned * p1.Size * vSortBy)
/ pRes->Time;
pRes->BTime = p1.Time + pRes->Card * p1.Size *
(uSortBy + oSortBy * li->state);
pRes->BProgress =
(p1.Progress * p1.Time
+ li->read * p1.Size * (uSortBy + oSortBy * li->state))
/ pRes->BTime;
return YIELD;
}
else return CANCEL;
}
}
return 0;
}
/*
2.2 Operator ~mergejoin~
This operator computes the equijoin of two streams. It uses a text book
algorithm as outlined in A. Silberschatz, H. F. Korth, S. Sudarshan,
McGraw-Hill, 3rd. Edition, 1997.
2.2.1 Auxiliary definitions for value mapping function of operator ~mergejoin~
*/
class MergeJoinLocalInfo2: protected ProgressWrapper
{
protected:
// buffer limits
size_t MAX_MEMORY;
size_t MAX_TUPLES_IN_MEMORY;
// buffer related members and iterators
TupleBuffer *grpB;
GenericRelationIterator *iter;
// members needed for sorting the input streams
typedef LocalInfo<SortByLocalInfo2> LocalSRT;
LocalSRT* liA;
SortByLocalInfo2* sliA;
LocalSRT* liB;
SortByLocalInfo2* sliB;
Word streamA;
Word streamB;
// the current pair of tuples
Word resultA;
Word resultB;
RTuple ptA;
RTuple ptB;
RTuple tmpB;
// the last comparison result
int cmp;
// the indexes of the attributes which will
// be merged and the result type
int attrIndexA;
int attrIndexB;
TupleType *resultTupleType;
// a flag which indicates if sorting is needed
bool expectSorted;
// Members needed for the random subset option
bool randomPrefix;
bool continueHashjoin;
bool continueProbe;
bool earlyExit;
StreamIterator<Tuple> iterB;
HashTable* ht;
// switch trace messages on/off
const bool traceFlag;
// a flag needed in function NextTuple which tells
// if the merge with grpB has been finished
bool continueMerge;
template<bool BOTH_B>
int CompareTuples(Tuple* t1, Tuple* t2)
{
Attribute* a = 0;
if (BOTH_B) {
a = static_cast<Attribute*>( t1->GetAttribute(attrIndexB) );
}
else {
a = static_cast<Attribute*>( t1->GetAttribute(attrIndexA) );
}
Attribute* b = static_cast<Attribute*>( t2->GetAttribute(attrIndexB) );
/* tuples with NULL-Values in the join attributes
are never matched with other tuples. */
if( !a->IsDefined() )
{
return -1;
}
if( !b->IsDefined() )
{
return 1;
}
int cmp = a->Compare(b);
if (traceFlag)
{
cmsg.info()
<< "CompareTuples:" << endl
<< " BOTH_B = " << BOTH_B << endl
<< " tuple_1 = " << *t1 << endl
<< " tuple_2 = " << *t2 << endl
<< " cmp(t1,t2) = " << cmp << endl;
cmsg.send();
}
return cmp;
}
inline int CompareTuplesB(Tuple* t1, Tuple* t2)
{
return CompareTuples<true>(t1, t2);
}
inline int CompareTuples(Tuple* t1, Tuple* t2)
{
return CompareTuples<false>(t1, t2);
}
inline Tuple* NextTuple(Word stream, SortByLocalInfo2* sli)
{
bool yield = false;
Word result( Address(0) );
if(!expectSorted) {
return sli->NextResultTuple();
}
qp->Request(stream.addr, result);
yield = qp->Received(stream.addr);
if(yield)
{
return static_cast<Tuple*>( result.addr );
}
else
{
result.addr = 0;
return static_cast<Tuple*>( result.addr );
}
}
inline Tuple* NextTupleA()
{
progress->readFirst++;
return NextTuple(streamA, sliA);
}
inline Tuple* NextTupleB()
{
progress->readSecond++;
return NextTuple(streamB, sliB);
}
SortByLocalInfo2* SortInput( const Word& stream, int attrIndex,
LocalSRT*& li)
{
// sort the input streams
SortOrderSpecification spec;
spec.push_back( std::pair<int, bool>(attrIndex + 1, true) );
void* tupleCmp = new TupleCompareBy( spec );
li = new LocalSRT();
return new SortByLocalInfo2( stream, false, tupleCmp, li);
}
void SortInputs()
{
// sort the input streams
SortOrderSpecification specA;
SortOrderSpecification specB;
specA.push_back( std::pair<int, bool>(attrIndexA + 1, true) );
specB.push_back( std::pair<int, bool>(attrIndexB + 1, true) );
void* tupleCmpA = new TupleCompareBy( specA );
void* tupleCmpB = new TupleCompareBy( specB );
liA = new LocalInfo<SortByLocalInfo2>();
progress->firstLocalInfo = liA;
sliA = new SortByLocalInfo2( streamA,
false,
tupleCmpA, liA );
liB = new LocalInfo<SortByLocalInfo2>();
progress->secondLocalInfo = liB;
sliB = new SortByLocalInfo2( streamB,
false,
tupleCmpB, liB );
}
public:
MergeJoinLocalInfo2( Word _streamA, Word wAttrIndexA,
Word _streamB, Word wAttrIndexB,
bool _expectSorted, Supplier s,
ProgressLocalInfo* p,
bool _randomPrefix = false,
bool _earlyExit = false )
: ProgressWrapper(p),
traceFlag( RTFlag::isActive("PSA:TraceMergeJoin") )
{
expectSorted = _expectSorted;
randomPrefix = _randomPrefix;
earlyExit = _earlyExit;
streamA = _streamA;
streamB = _streamB;
attrIndexA = StdTypes::GetInt( wAttrIndexA ) - 1;
attrIndexB = StdTypes::GetInt( wAttrIndexB ) - 1;
ListExpr resultType =
SecondoSystem::GetCatalog()->NumericType( qp->GetType( s ) );
resultTupleType = new TupleType( nl->Second( resultType ) );
MAX_MEMORY = qp->FixedMemory();
cmsg.info("ERA:ShowMemInfo")
<< "MergeJoin.MAX_MEMORY (" << MAX_MEMORY/1024 << " kb)" << endl;
cmsg.send();
liA = 0;
sliA = 0;
liB = 0;
grpB = 0;
sliB = 0;
ht = 0;
continueHashjoin = false;
continueProbe = false;
if ( randomPrefix ) {
sliA = SortInput(streamA, attrIndexA, liA);
sliA->PrepareResultIteration(true);
progress->firstLocalInfo = liA;
// Now a random subset S1 of 500 tuples is stored in a hash table.
// Next the tuples of streamB will be joined with S1 and passed to
// the Sorting-Algorithm for B. Finally, the sorted streams are merged.
sliB = SortInput(streamB, attrIndexB, liB);
if (traceFlag)
cerr << "Input B sorted" << endl;
progress->secondLocalInfo = liB;
iterB = sliB->GetIterator();
// prime numbers: 503, 701, 1009, 2003
ht = sliA->CreateHashTable(701, attrIndexA, attrIndexB);
if (traceFlag)
cerr << "HashTable created" << endl;
continueHashjoin = true;
sliB->InitRuns();
if (traceFlag)
cerr << "Input B initialized" << endl;
}
else
{
if( !expectSorted ) {
sliA = SortInput(streamA, attrIndexA, liA);
sliA->PrepareResultIteration();
progress->firstLocalInfo = liA;
sliB = SortInput(streamB, attrIndexB, liB);
sliB->PrepareResultIteration();
progress->secondLocalInfo = liB;
}
InitIteration();
}
}
~MergeJoinLocalInfo2()
{
//cerr << "calling ~MergeJoinLocalInfo2()" << endl;
if( !expectSorted )
{
// delete the objects instantiated for sorting
delete sliA;
delete sliB;
delete liA;
delete liB;
}
delete grpB;
resultTupleType->DeleteIfAllowed();
}
inline Tuple* NextResultTuple()
{
Tuple* resultTuple = 0;
while ( continueHashjoin ) { // probe hash buckets
if ( !continueProbe ) // initialize hash bucket iteration
{
if (traceFlag)
cerr << "Initialize hash bucket iteration" << endl;
if ( iterB.valid() )
{
(*iterB)->IncReference();
ht->initProbe( (*iterB)->HashValue(attrIndexB) );
continueProbe = true;
sliB->AppendTuple(*iterB);
}
else // end of stream B and end of hashjoin
{
if (traceFlag)
cerr << "End of stream B" << endl;
continueHashjoin = false;
continueProbe = false;
sliB->FinishRuns();
sliB->InitMerge();
InitIteration();
}
}
if ( continueProbe )
{
Tuple* b = *iterB;
Tuple* a = ht->probe(b);
if (a != 0) { // concat a and b
Tuple* result = new Tuple( resultTupleType );
Concat( a, b, result );
return result;
}
else // switch to next tuple of B
{
//cout << "b:refs =" << b->GetNumOfRefs() << endl;
b->DeleteIfAllowed();
++iterB;
continueProbe = false;
}
}
}
if (earlyExit) {
if (traceFlag)
cerr << "Early exit" << endl;
return 0;
}
if ( !continueMerge && ptB == 0)
return 0;
while( ptA != 0 ) {
if (!continueMerge && ptB != 0) {
//save ptB in tmpB
tmpB = ptB;
grpB->AppendTuple(tmpB.tuple);
// advance the tuple pointer
ptB.setTuple( NextTupleB() );
// collect a group of tuples from B which
// have the same attribute value
bool done = false;
while ( !done && ptB != 0 ) {
int cmp = CompareTuplesB( tmpB.tuple, ptB.tuple );
if ( cmp == 0)
{
// append equal tuples to group
grpB->AppendTuple(ptB.tuple);
// release tuple of input B
ptB.setTuple( NextTupleB() );
}
else
{
done = true;
}
} // end collect group
cmp = CompareTuples( ptA.tuple, tmpB.tuple );
while ( ptA != 0 && cmp < 0 )
{
// skip tuples from A while they are smaller than the
// value of the tuples in grpB
ptA.setTuple( NextTupleA() );
if (ptA != 0) {
cmp = CompareTuples( ptA.tuple, tmpB.tuple );
}
}
}
// continue or start a merge with grpB
while ( ptA != 0 && cmp == 0 )
{
// join ptA with grpB
if (!continueMerge)
{
iter = grpB->MakeScan();
continueMerge = true;
resultTuple = NextConcat();
if (resultTuple)
return resultTuple;
}
else
{
// continue merging, create the next result tuple
resultTuple = NextConcat();
if (resultTuple) {
return resultTuple;
}
else
{
// Iteration over the group finished.
// Continue with the next tuple of argument A
continueMerge = false;
delete iter;
iter = 0;
ptA.setTuple( NextTupleA() );
if (ptA != 0) {
cmp = CompareTuples( ptA.tuple, tmpB.tuple );
}
}
}
}
grpB->Clear();
// tpA > tmpB
if ( ptB == 0 ) {
// short exit
return 0;
}
} // end of main loop
return 0;
}
inline Tuple* NextConcat()
{
Tuple* t = iter->GetNextTuple();
if( t != 0 ) {
Tuple* result = new Tuple( resultTupleType );
Concat( ptA.tuple, t, result );
t->DeleteIfAllowed();
return result;
}
return 0;
}
void InitIteration()
{
// read in the first tuple of both input streams
Tuple* tA = NextTupleA();
ptA = RTuple( tA );
if(tA) tA->DeleteIfAllowed();
Tuple* tB = NextTupleB();
ptB = RTuple( tB );
if(tB) tB->DeleteIfAllowed();
// initialize the status for the result
// set iteration
tmpB = 0;
cmp = 0;
continueMerge = false;
if (grpB != 0)
delete grpB;
grpB = new TupleBuffer( MAX_MEMORY );
}
};
/*
2.2.2 MergeJoinLocalInfoSHF
A variant of a sortmergejoin which produces an output stream which starts with
a random sample of 500 tuples.
*/
class MergeJoinLocalInfoSHF : protected MergeJoinLocalInfo2
{
public:
MergeJoinLocalInfoSHF( Word _streamA,
Word wAttrIndexA,
Word _streamB,
Word wAttrIndexB,
bool _expectSorted,
Supplier s,
ProgressLocalInfo* p,
bool rnd = false, bool earlyexit = false )
: MergeJoinLocalInfo2( _streamA, wAttrIndexA,
_streamB, wAttrIndexB,
_expectSorted, s, p, rnd, earlyexit ),
streamPos(0),
positions(500,0),
memBufIter(0),
memBufFinished(false),
firstScanFinished(false),
trace(true)
{}
~MergeJoinLocalInfoSHF() {
cerr << "calling ~MergeJoinLocalInfoSHF()" << endl;
}
inline Tuple* NextResultTuple()
{
Tuple* res = 0;
if (!firstScanFinished)
{
res = MergeJoinLocalInfo2::NextResultTuple();
while (res != 0)
{
// decide if tuple replaces one of the buffer
streamPos++;
size_t i = 0;
bool replaced = false;
Tuple* v = rtBuf.ReplacedByRandom(res, i, replaced);
if ( replaced )
{
positions[i] = streamPos;
// v was replaced by res
if (v != 0) {
//persBuf.AppendTuple(v);
v->DeleteIfAllowed();
}
}
else
{
assert(v == 0);
// v == 0, and t was not stored in buffer
res->DeleteIfAllowed();
}
res = MergeJoinLocalInfo2::NextResultTuple();
}
if (trace)
cerr << "copy2TupleBuf" << endl;
rtBuf.copy2TupleBuf( memBuf );
// reset scan
firstScanFinished = true;
sliA->InitMerge();
sliB->InitMerge();
InitIteration();
sort(positions.begin(), positions.end());
posIter = positions.begin();
memBufIter = memBuf.MakeScan();
streamPos = 0;
if (trace)
cerr << "Start 2nd run" << endl;
}
if (firstScanFinished)
{
if (!memBufFinished)
{
res = memBufIter->GetNextTuple();
if (res == 0) {
if (trace) {
cerr << endl;
cerr << "streamPos: " << streamPos << endl;
cerr << "memBuf : " << memBuf.GetNoTuples() << endl;
}
memBufFinished = true;
delete memBufIter;
memBufIter = 0;
}
}
if ( memBufFinished == true)
{
res = MergeJoinLocalInfo2::NextResultTuple();
streamPos++;
while (streamPos == *posIter)
{
res->DeleteIfAllowed();
res = MergeJoinLocalInfo2::NextResultTuple();
streamPos++;
posIter++;
}
}
}
return res;
}
private:
size_t streamPos;
TupleBuffer memBuf;
vector<size_t> positions;
vector<size_t>::const_iterator posIter;
RandomTBuf rtBuf;
GenericRelationIterator* memBufIter;
bool memBufFinished;
bool firstScanFinished;
const bool trace;
};
/*
2.2.3 Value mapping function of operator ~mergejoin~
*/
//CPUTimeMeasurer mergeMeasurer;
template<class T, bool SRT, bool RND, bool R3> int
MergeJoin(Word* args, Word& result, int message, Word& local, Supplier s)
{
typedef LocalInfo<T> LocalType;
LocalType* li = static_cast<LocalType*>( local.addr );
switch(message)
{
case OPEN:
if ( li ) {
delete li->ptr;
delete li;
}
li = new LocalType();
local.addr = li;
qp->Open(args[0].addr);
qp->Open(args[1].addr);
li->ptr = 0;
return 0;
case REQUEST: {
//mergeMeasurer.Enter();
if ( li->ptr == 0 ) //first request;
//constructor put here to avoid delays in OPEN
//which are a problem for progress estimation
{
li->ptr = new T( args[0], args[4], args[1],
args[5], SRT, s, li, RND, R3 );
}
T* mli = li->ptr;
result.addr = mli->NextResultTuple();
li->returned++;
//mergeMeasurer.Exit();
return result.addr != 0 ? YIELD : CANCEL;
}
case CLOSE:
//mergeMeasurer.PrintCPUTimeAndReset("CPU Time for Merging Tuples : ");
qp->Close(args[0].addr);
qp->Close(args[1].addr);
//nothing is deleted on close because the substructures are still
//needed for progress estimation. Instea/*
//(repeated) OPEN and on CLOSEPROGRESS
return 0;
case CLOSEPROGRESS:
if ( li ) {
delete li;
local.addr = 0;
}
return 0;
case REQUESTPROGRESS:
{
ProgressInfo p1, p2;
ProgressInfo* pRes = static_cast<ProgressInfo*>( result.addr );
const double uMergeJoin = 0.041; //millisecs per tuple merge (merge)
const double vMergeJoin = 0.000076; //millisecs per byte merge (sortmerge)
const double uSortBy = 0.00043; //millisecs per byte sort
if( !li )
{
return CANCEL;
}
else
{
if (qp->RequestProgress(args[0].addr, &p1)
&& qp->RequestProgress(args[1].addr, &p2))
{
li->SetJoinSizes(p1, p2);
pRes->CopySizes(li);
if ( SRT ) // already sorted inputes
{
pRes->Time = p1.Time + p2.Time +
(p1.Card + p2.Card) * uMergeJoin;
pRes->Progress =
(p1.Progress * p1.Time + p2.Progress * p2.Time +
(((double) li->readFirst) + ((double) li->readSecond))
* uMergeJoin)
/ pRes->Time;
pRes->CopyBlocking(p1, p2); //non-blocking in this case
}
else // unsorted inputs
{
pRes->Time =
p1.Time +
p2.Time +
p1.Card * p1.Size * uSortBy +
p2.Card * p2.Size * uSortBy +
(p1.Card * p1.Size + p2.Card * p2.Size) * vMergeJoin;
typedef LocalInfo<SortByLocalInfo2> LocalSRT;
LocalSRT* liFirst = 0;
LocalSRT* liSecond = 0;
liFirst = static_cast<LocalSRT*>( li->firstLocalInfo );
liSecond = static_cast<LocalSRT*>( li->secondLocalInfo );
long readFirst = (liFirst ? liFirst->read : 0);
long readSecond = (liSecond ? liSecond->read : 0);
pRes->Progress =
(p1.Progress * p1.Time +
p2.Progress * p2.Time +
((double) readFirst) * p1.Size * uSortBy +
((double) readSecond) * p2.Size * uSortBy +
(((double) li->readFirst) * p1.Size +
((double) li->readSecond) * p2.Size)
* vMergeJoin)
/ pRes->Time;
pRes->BTime = p1.Time + p2.Time
+ p1.Card * p1.Size * uSortBy
+ p2.Card * p2.Size * uSortBy;
pRes->BProgress =
(p1.Progress * p1.Time + p2.Progress * p2.Time
+ ((double) readFirst) * p1.Size * uSortBy
+ ((double) readSecond) * p2.Size * uSortBy)
/ pRes->BTime;
}
if (li->returned > enoughSuccessesJoin ) // stable state
{
pRes->Card = ((double) li->returned * (p1.Card + p2.Card)
/ ((double) li->readFirst + (double) li->readSecond));
}
else
{
pRes->Card = p1.Card * p2.Card * qp->GetSelectivity(s);
}
return YIELD;
}
else return CANCEL;
}
}
}
return 0;
}
/*
3 Instantiation of Template Functions
*/
// sortmergejoin_r
template int
MergeJoin<MergeJoinLocalInfoSHF, false, false, false>( Word* args,
Word& result,
int message,
Word& local,
Supplier s );
int
sortmergejoinr_vm( Word* args, Word& result,
int message, Word& local, Supplier s )
{
return MergeJoin<MergeJoinLocalInfoSHF, false, false, false>(args, result,
message, local, s);
}
int
sortmergejoinr2_vm( Word* args, Word& result,
int message, Word& local, Supplier s )
{
return MergeJoin<MergeJoinLocalInfo2, false, true, false>(args, result,
message, local, s);
}
int
sortmergejoinr3_vm( Word* args, Word& result,
int message, Word& local, Supplier s )
{
return MergeJoin<MergeJoinLocalInfo2, false, true, true>(args, result,
message, local, s);
}
Reference in New Issue View Git Blame Copy Permalink