/*
----
This file is part of SECONDO.

Copyright (C) 2004-2008, University in Hagen, Faculty of Mathematics and
Computer Science, Database Systems for New Applications.

SECONDO is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.

SECONDO is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with SECONDO; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
----

//paragraph [1] Title: [{\Large \bf \begin{center}] [\end{center}}]
//paragraph [10] Footnote: [{\footnote{] [}}]
//[TOC] [\tableofcontents]
//[_] [\_]
//[x] [\ensuremath{\times}]
//[->] [\ensuremath{\rightarrow}]
//[>] [\ensuremath{>}]
//[<] [\ensuremath{<}]




[1] Implementation of Module Extended Relation Algebra

[1] Using Storage Manager Berkeley DB

June 1996 Claudia Freundorfer

May 2002 Frank Hoffmann port to C++

November 7, 2002 RHG Corrected the type mapping of ~tcount~.

November 30, 2002 RHG Introduced a function ~RelPersistValue~ instead of
~DefaultPersistValue~ which keeps relations that have been built in memory in a
small cache, so that they need not be rebuilt from then on.

April 2002 Victor Almeida Separated the relational algebra into two algebras
called Relational Algebra and Extended Relational Algebra.

December 2005, Victor Almeida deleted the deprecated algebra levels
(~executable~, ~descriptive~, and ~hibrid~). Only the executable
level remains. Models are also removed from type constructors.

January 2006 Victor Almeida replaced the ~free~ tuples concept to
reference counters. There are reference counters on tuples and also
on attributes. Some assertions were removed, since the code is
stable.

June 2006, Corrected a bug caused by improper reference counting of tuples
observed in operator ~mergesec~.

June 2006, Christian D[ue]ntgen added operators ~symmproduct~ and
~symmproductextend~.

August 2006, Christian D[ue]ntgen added signature ((stream T) int) -> (stream T)
to operator ~head~.

January 2007, M. Spiekermann. Reference counting in groupby corrected, since it
causes a segmentation fault, when the Tuplebuffer needs to be flushed on disk.

July 2007, C. Duentgen. Changed groupbyTypeMap. It will now accept an empty
list of grouping attributes (argument 1)). The result will then be constructed
from a single group containing ALL tuples from the input stream (argument 0).
Logically, the result tuples contain no original attributes from the input
stream, but only those created by aggregation functions from argument 2.

October 2007, M. Spiekermann, T. Behr. Reimplementation of the operators ~avg~
and ~sum~ in order to provide better example code since these should be used as
examples for the practical course.

January 2008, C. D[ue]ntgen adds aggregation operator ~var~, computing the
variance on a stream.

November 2011, F. Valdes. Operator ~toFields~ added, decomposing a stream of
tuples from a relation schema into its items.

December 2011, F. Valdes. Operator ~fromFields~ added, composing a stream of
tuples into a relation schema.

[TOC]

1 Includes and defines

*/

#include <vector>
#include <deque>
#include <sstream>
#include <stack>
#include <limits.h>
#include <set>

//#define TRACE_ON
#undef TRACE_ON
#include "LogMsg.h"
#define TRACE_OFF

#include "Algebras/Relation-C++/RelationAlgebra.h"
#include "QueryProcessor.h"
#include "AlgebraManager.h"
#include "CPUTimeMeasurer.h"
#include "StandardTypes.h"
#include "Counter.h"
#include "Algebras/TupleIdentifier/TupleIdentifier.h"
#include "Progress.h"
#include "RTuple.h"
#include "Symbols.h"
#include "ListUtils.h"
#include "Outerjoin.h"
#include "DateTime.h"
#include "Stream.h"
#include "Algebras/FText/FTextAlgebra.h"
#include "SecondoCatalog.h"
#include "Algebras/Standard-C++/LongInt.h"

#ifdef USE_PROGRESS
#include "../CostEstimation/ExtRelationAlgebraCostEstimation.h"
#endif

extern NestedList* nl;
extern QueryProcessor* qp;
extern AlgebraManager* am;
extern Operator extrelsmouterjoin;
extern Operator extrelsymmouterjoin;

using namespace listutils;
using namespace std;


namespace extrelationalg{

/*
2 Operators

2.2 Selection function for type operators

The selection function of a type operator always returns -1.

*/
int exttypeOperatorSelect(ListExpr args)
{
  return -1;
}

/*
2.3 Type Operator ~Group~

Type operators are used only for inferring argument types of parameter
functions. They have a type mapping but no evaluation function.

2.3.1 Type mapping function of operator ~group~

----  ((stream x))                -> (rel x)
----

*/
ListExpr GroupTypeMap(ListExpr args)
{
  if(nl->ListLength(args)<1){
     ErrorReporter::ReportError("one argument expected");
     return nl->TypeError();
  }

  ListExpr first = nl->First(args);

  if(!Stream<Tuple>::checkType(first)){
    return listutils::typeError("tuple stream expected");
  }

  return nl->TwoElemList(
          nl->SymbolAtom(Relation::BasicType()),
          nl->Second(first));
}
/*
2.3.2 Specification of operator ~Group~

*/
const string GroupSpec  = "( ( \"Signature\" \"Syntax\" \"Meaning\" "
                          "\"Remarks\" ) "
                          "( <text>((stream x)) -> (rel x)</text--->"
                          "<text>type operator</text--->"
                          "<text>Maps stream type to a rel.</text--->"
                          "<text>not for use with sos-syntax</text--->"
                          ") )";

/*
2.3.3 Definition of operator ~group~

*/
Operator extrelgroup (
         "GROUP",              // name
         GroupSpec,            // specification
         0,                    // no value mapping
         exttypeOperatorSelect,   // trivial selection function
         GroupTypeMap          // type mapping
);

/*
2.4 Operator ~sample~

Produces a stream representing a sample of a relation.

2.4.1 Function ~MakeRandomSubset~

Generates a random subset of the numbers 1 ... ~setSize~, the size
of which is ~subsetSize~. This function is needed for operator ~sample~.

The strategy for generating a random subset works as follows: The algorithm
maintains a set of already drawn numbers. The algorithm draws a new random
number (using the libc random number generator) and adds it to the set of
already drawn numbers, if it has not been
already drawn. This is repeated until the size of the drawn set equals
~subsetSize~. If ~subsetSize~ it not considerably smaller than ~setSize~, e.g.
~subsetSize~ = ~setSize~ - 1, this approach becomes very inefficient
or may even not terminate, because towards the end of the algorithm
it may take a very long (or infinitely long)
time until the random number generator hits one of the few numbers,
which have not been already drawn. Therefore, if ~subsetSize~ is more
than half of ~subSet~, we simple draw a subset of size
~setSize~ - ~subsetSize~ and take the complement of that set as result set.

If the optional parameter ~subsetSize~ is set to ~true~, the optional parameter
~randSeed~ (defaults to 0) is used as the starting sequence offset for the
random
number generator. Otherwise, the offset is calculated from the current time and
will differ for each call.

*/
void
MakeRandomSubset(vector<int>& result, int subsetSize, int setSize,
     bool useSeed = false, unsigned int randSeed = 1)
{
  set<int> drawnNumbers;
  set<int>::iterator iter;
  int drawSize = 0;
  int nDrawn = 0;
  int i = 0;
  int r = 0;
  bool doInvert = false;

  result.resize(0);

  // The variable below defines an offset into the
  // random number sequence. It will be incremented by
  // the number of rand() calls. Hence subsequent calls
  // will avoid to return the same sequence of numbers,
  // unless useSeed is true and both calls provide the
  // same randSeed.
  static unsigned int randCalls = (time(0) % 1000) * 1000;
  // For some experiments, one would like to get "known" samples.
  // To that end, useSeed can be set to true and an explicit random seed can be
  // passed.
  // To allow for "real" random sequences after such a seeding, we need to
  // remember the fact whether the current randCalls is due to an explicit
  // seed or not (otherwise the sample-result would be random)
  static bool lastWasSeeded = useSeed;
  if(useSeed){
    randCalls = randSeed;
    lastWasSeeded = true; // time-based seed overwritten!
  } else {
    if(lastWasSeeded){
      randCalls = (time(0) % 1000) * 1000; // re-init seed
    }
    lastWasSeeded = false;
  }
  srand(randCalls);
  if(((double)setSize) / ((double)subsetSize) <= 2)
  {
    doInvert = true;
    drawSize = setSize - subsetSize;
  }
  else
  {
    doInvert = false;
    drawSize = subsetSize;
  }

  // Using Windows RAND_MAX is very small (about 2^15) therefore we
  // need to limit the drawSize to 3/4 (to avoid long runtimes) of
  // this size.
  int drawMax = 3*(RAND_MAX/4);
  const int randMax = RAND_MAX;
  const int intMax = INT_MAX;
  const int newMax = max(randMax, intMax);
  static const int f = (intMax / randMax) - 1;
  static long& ctr = Counter::getRef("EXT::sample:randPos");
  static long& ctrMax = Counter::getRef("EXT::sample:maxDrawnRand");


  if ( drawSize > drawMax )
  {
    drawSize = drawMax;
    cerr << "Warning: Sample size reduced to 3/4*RAND_MAX." << endl;
  }

  TRACE("*** sample parameters ***")
  SHOW(f)
  SHOW(setSize)
  SHOW(subsetSize)
  SHOW(drawSize)

  while(nDrawn < drawSize)
  {
    // 28.04.04 M. Spiekermann.
    // The calculation of random numbers below is recommended in
    // the man page documentation of the rand() function.
    // Moreover, the factor f is used to retrieve values in the
    // range of LONG_MAX
    long nextRand = rand();
    randCalls++;
    if ( f > 1 )
    {
      nextRand += (f * rand());
      randCalls++;
    }
    if (nextRand > ctrMax)
      ctrMax = nextRand;

    r = (int) ((double)(setSize + 1) * nextRand/(newMax+1.0));

    if ( r != 0) {
      if(drawnNumbers.find(r) == drawnNumbers.end())
      {
        drawnNumbers.insert(r);
        ++nDrawn;
      }
    }
  }
  ctr = randCalls;
  SHOW(drawnNumbers.size())

  if(doInvert)
  {
    for(i = 1; i <= setSize; ++i)
    {
      if(drawnNumbers.find(i) == drawnNumbers.end())
      {
        result.push_back(i);
      }
    }
  }
  else
  {
    for(iter = drawnNumbers.begin();
        iter != drawnNumbers.end();
        ++iter)
      result.push_back(*iter);
  }
  SHOW(result.size())
}

/*
2.4.2 Type mapping function of operator ~sample~

A type mapping function takes a nested list as argument. Its contents are
type descriptions of an operator's input parameters. A nested list describing
the output type of the operator is returned.

Result type of feed operation.

----  ((rel x) int real [int] )    -> (stream x)
----

*/
ListExpr SampleTypeMap(ListExpr args)
{

  int len = nl->ListLength(args);
  if(len!=3 && len!=4){
    ErrorReporter::ReportError("three or four arguments expected");
    return nl->TypeError();
  }

  ListExpr rel = nl->First(args);
  ListExpr minSampleSize = nl->Second(args);
  ListExpr minSampleRate = nl->Third(args);

  if( !Relation::checkType(rel) ||
      !CcInt::checkType(minSampleSize)  ||
      !CcReal::checkType(minSampleRate)){
    ErrorReporter::ReportError("rel x int x real [ x int] expected");
    return nl->TypeError();
  }

  ListExpr streamDescription =
          nl->Cons(nl->SymbolAtom(Symbol::STREAM()), nl->Rest(rel));

  if(len==4){
    ListExpr randSeed = nl->Fourth(args);
    if(!CcInt::checkType(randSeed)){
      ErrorReporter::ReportError("rel x int x real [ x int] expected");
      return nl->TypeError();
    }
    return nl->ThreeElemList(nl->SymbolAtom(Symbol::APPEND()),
                             nl->OneElemList(nl->BoolAtom(true)),
                             streamDescription);
  }

  return nl->ThreeElemList(nl->SymbolAtom(Symbol::APPEND()),
                           nl->TwoElemList(nl->IntAtom(0),
                                           nl->BoolAtom(false)),
                           streamDescription);
}
/*
2.4.3 Value mapping function of operator ~sample~

*/
struct SampleLocalInfo
{
  vector<int> sampleIndices;
  vector<int>::iterator iter;
  int lastIndex;
  RandomRelationIterator* relIter;
  bool useSeed;
  unsigned int randSeed;
};

int Sample(Word* args, Word& result, int message, Word& local, Supplier s)
{
  SampleLocalInfo* localInfo = static_cast<SampleLocalInfo*>( local.addr );

  Relation* rel = 0;
  Tuple* tuple = 0;

  int sampleSize = 0;
  int relSize = 0;
  float sampleRate = 0;
  int i = 0;
  int currentIndex = 0;

  switch(message)
  {
    case OPEN :
      localInfo = new SampleLocalInfo();
      local.addr = localInfo;

      localInfo->randSeed = (unsigned int)((CcInt*)(args[3].addr))->GetIntval();
      localInfo->useSeed = ((CcBool*)(args[4].addr))->GetBoolval();

      rel = (Relation*)args[0].addr;
      relSize = rel->GetNoTuples();
      localInfo->relIter = rel->MakeRandomScan();
      sampleSize = StdTypes::GetInt(args[1]);
      sampleRate = StdTypes::GetReal(args[2]);
      if(sampleSize < 1)
      {
        sampleSize = 1;
      }
      if(sampleRate <= 0.0)
      {
        sampleRate = 0.0;
      }
      else if(sampleRate > 1.0)
      {
        sampleRate = 1.0;
      }
      if((int)(sampleRate * (float)relSize) > sampleSize)
      {
        sampleSize = (int)(sampleRate * (float)relSize);
      }

      if(relSize <= sampleSize)
      {
        for(i = 1; i <= relSize; ++i)
        {
          localInfo->sampleIndices.push_back(i);
        }
      }
      else
      {
        MakeRandomSubset(localInfo->sampleIndices, sampleSize, relSize,
                         localInfo->useSeed, localInfo->randSeed);
      }

      localInfo->iter = localInfo->sampleIndices.begin();
      localInfo->lastIndex = 0;
      return 0;

    case REQUEST:
      if(localInfo->iter == localInfo->sampleIndices.end())
      {
        return CANCEL;
      }
      else
      {
        currentIndex = *(localInfo->iter);
        int step = currentIndex - localInfo->lastIndex;
        if(!(tuple = localInfo->relIter->GetNextTuple(step)))
        {
          return CANCEL;
        }

        result.setAddr(tuple);
        localInfo->lastIndex = *(localInfo->iter);
        localInfo->iter++;
        return YIELD;
      }

    case CLOSE :
      if(local.addr){
         localInfo = (SampleLocalInfo*)local.addr;
         delete localInfo->relIter;
         delete localInfo;
         local.setAddr(0);
      }
      return 0;
  }
  return 0;
}
/*
2.4.4 Specification of operator ~sample~

*/
const string SampleSpec  = "( ( \"Signature\" \"Syntax\" \"Meaning\" "
                           "\"Example\" ) "
                           "( <text>rel(X) int x real [x int ] "
                           "-> (stream x)"
                           "</text--->"
                           "<text>_ sample [ Size , Fraction , Seed ]</text--->"
                           "<text>Produces a random sample of a relation."
                           " The sample size is min(relSize, "
                           "max(s, t * relSize)), where relSize is the size"
                           " of the argument relation, s is Size "
                           "argument, and t is Fraction. The optional fourth "
                           "parameter selects a seed for the used random "
                           "number generator."
                           "The sample has the same ordering as the original "
                           "relation. </text--->"
                          "<text>query cities sample[0, 0.45] count</text--->"
                           ") )";

/*
2.4.5 Definition of operator ~sample~

Non-overloaded operators are defined by constructing a new instance of
class ~Operator~, passing all operator functions as constructor arguments.

*/
Operator extrelsample (
          "sample",                // name
          SampleSpec,              // specification
          Sample,                  // value mapping
          Operator::SimpleSelect,  // trivial selection function
          SampleTypeMap            // type mapping
);



/*
2.6 Operator ~cancel~

Transmits tuple from its input stream to its output stream until a tuple
arrives fulfilling some condition.

2.6.1 Type mapping function of operator ~cancel~

Type mapping for ~cancel~ is the same, as type mapping for operator ~filter~.
Result type of cancel operation.

----    ((stream x) (map x bool)) -> (stream x)
----

*/
ListExpr CancelTypeMap(ListExpr args)
{
  if(nl->ListLength(args)!=2){
    return listutils::typeError("two arguments expected");
  }
  ListExpr stream = nl->First(args);
  ListExpr fun = nl->Second(args);

  string err = " stream(tuple(x)) x (tuple(x) -> bool) expected";
  if(!listutils::isTupleStream(stream) ||
     !listutils::isMap<1>(fun)){
    return listutils::typeError(err);
  }

  if(!listutils::isSymbol(nl->Third(fun),CcBool::BasicType())){
    return listutils::typeError(err);
  }
  if(!nl->Equal(nl->Second(stream),nl->Second(fun))){
    return listutils::typeError(err);
  }
  return stream;
}

/*
2.6.2 Value mapping function of operator ~cancel~

*/
int Cancel(Word* args, Word& result, int message, Word& local, Supplier s)
{
  Word t, value;
  Tuple* tuple;
  bool found;
  ArgVectorPointer vector;

  switch (message)
  {
    case OPEN :

      qp->Open(args[0].addr);
      return 0;

    case REQUEST :

      qp->Request(args[0].addr, t);
      found= false;
      if (qp->Received(args[0].addr))
      {
        tuple = (Tuple*)t.addr;
        vector = qp->Argument(args[1].addr);
        (*vector)[0] = t;
        qp->Request(args[1].addr, value);
        found =
          ((CcBool*)value.addr)->IsDefined()
          && ((CcBool*)value.addr)->GetBoolval();
        if (found)
        {
          tuple->DeleteIfAllowed();
          return CANCEL;
        }
        else
        {
          result.setAddr(tuple);
          return YIELD;
        }
      }
      else return CANCEL;

    case CLOSE :

      qp->Close(args[0].addr);
      return 0;
  }
  return 0;
}
/*
2.6.3 Specification of operator ~cancel~

*/
const string CancelSpec  = "( ( \"Signature\" \"Syntax\" \"Meaning\" "
                           "\"Example\" ) "
                           "( <text>((stream x) (map x bool)) -> "
                           "(stream x)</text--->"
                           "<text>_ cancel [ fun ]</text--->"
                           "<text>Transmits tuple from its input stream "
                           "to its output stream until a tuple arrives "
                           "fulfilling some condition.</text--->"
                           "<text>query cities feed cancel [.cityname = "
                           "\"Dortmund\"] consume</text--->"
                              ") )";

/*
2.6.4 Definition of operator ~cancel~

*/
Operator extrelcancel (
         "cancel",               // name
         CancelSpec,             // specification
         Cancel,                 // value mapping
         Operator::SimpleSelect, // trivial selection function
         CancelTypeMap           // type mapping
);

/*
2.7 Operator ~extract~

This operator has a stream of tuples and the name of an attribut as input and
returns the value of this attribute
from the first tuple of the input stream. If the input stream is empty a run
time error occurs. In this case value -1 will be returned.

2.7.1 Type mapping function of operator ~extract~

Type mapping for ~extract~ is

----  ((stream (tuple ((x1 t1)...(xn tn))) xi)  -> ti
              APPEND (i) ti)
----

*/
ListExpr ExtractTypeMap( ListExpr args )
{
   

  string err = "(stream( tuple[ a1 : t1, .., an : tn ])) x a_i"
               "   or stream(DATA)  expected";
  if(!nl->HasLength(args,2) && !nl->HasLength(args,1)){
   return listutils::typeError("one or two arguments expected");
  }

  if(nl->HasLength(args,1)){ // DATA stream version
     if(!Stream<Attribute>::checkType(nl->First(args))){
       return listutils::typeError("if called with one arg, "
                                   "this arg must be stream(DATA)");
     }
     return nl->Second(nl->First(args));
  }


  ListExpr stream = nl->First(args);
  ListExpr attrname = nl->Second(args);

  if(!listutils::isTupleStream(stream) ||
     nl->AtomType(attrname)!=SymbolType){
    return listutils::typeError(err);
  }

  string attr = nl->SymbolValue(attrname);
  ListExpr attrList = nl->Second(nl->Second(stream));
  ListExpr attrType;
  int j = listutils::findAttribute(attrList, attr, attrType);
  if (j) {
    if (nl->ListLength(attrType) == 2 && nl->SymbolValue
          (nl->First(attrType)) == "arel"){
      ErrorReporter::ReportError("Standard extract is not defined for arel");
      return nl->TypeError();
    }
    else{
      return nl->ThreeElemList(nl->SymbolAtom(Symbol::APPEND()),
           nl->OneElemList(nl->IntAtom(j)), attrType);
    }
  } else {
    return listutils::typeError("Attribute name " + attr +
                                " not known in the tuple");
  }
}
/*
2.7.2 Value mapping function of operator ~extract~

The argument vector ~args~ contains in the first slot ~args[0]~ the tuple
and in ~args[2]~ the position of the attribute as a number. Returns as
~result~ the value of an attribute at the given position ~args[2]~ in a
tuple object.

*/
int Extract(Word* args, Word& result, int message, Word& local, Supplier s)
{

  if(qp->GetNoSons(s)>1){
    int index;
    Attribute* res = (Attribute*)((qp->ResultStorage(s)).addr);
    result.setAddr(res);
    Stream<Tuple> stream(args[0]);
    stream.open();
    Tuple* tuple = stream.request(); 
  
    if(tuple) {
      index = ((CcInt*)args[2].addr)->GetIntval();
      res->CopyFrom(
        (const Attribute*)tuple->GetAttribute(index - 1));
      tuple->DeleteIfAllowed();
    }
    else
    {
      res->SetDefined(false);
    }
  
    stream.close();
    return 0;
  } else { // attribute stream version
    Stream<Attribute> stream(args[0]);
    result = qp->ResultStorage(s); 
    Attribute* res = (Attribute*)(result.addr);
    stream.open();
    Attribute* a = stream.request();
    if(a){
       res->CopyFrom(a);
       a->DeleteIfAllowed();
    } else {
       res->SetDefined(false);
    }
    stream.close();
    return 0;
  }
}
/*
2.7.3 Specification of operator ~extract~

*/
const string ExtractSpec  = 
     "( ( \"Signature\" \"Syntax\" \"Meaning\" "
     "\"Example\" ) "
     "( <text>((stream (tuple([a1:d1, ... ,an:dn]"
     "))) x ai) -> di || stream<X> -> X , X in DATA</text--->"
     "<text>_ extract [ _ ]</text--->"
     "<text>Returns the value of attribute ai of "
     "the first tuple in the input stream."
     "</text--->"
     "<text>query cities feed extract [population]"
     "</text--->"
     ") )";

/*
2.7.4 Definition of operator ~extract~

*/
Operator extrelextract (
         "extract",              // name
         ExtractSpec,            // specification
         Extract,                // value mapping
         Operator::SimpleSelect, // trivial selection function
         ExtractTypeMap          // type mapping
);


/*
2.7 Operator extractDef

This operator works similar to the extract operator but uses
a default value if the stream is empty instead of setting the
result to be undefined.

*/
ListExpr extractDefTM(ListExpr args){
  if(!nl->HasLength(args,3)){
    return listutils::typeError("three args expected");
  }
  if(!Stream<Tuple>::checkType(nl->First(args))){
    return listutils::typeError("first arg is not a tuple stream");
  }
  if(nl->AtomType(nl->Second(args))!=SymbolType){
    return listutils::typeError("Second arg is not a valid attribute name");
  }
  string attrName = nl->SymbolValue(nl->Second(args));
  ListExpr attrType;
  int index = listutils::findAttribute(nl->Second(nl->Second(nl->First(args))),
                                       attrName, attrType);
  if(!index){
    return listutils::typeError("Attribute name " + attrName 
                                + " not part of the tuple");
  }
  if(!nl->Equal(nl->Third(args), attrType)){
    return listutils::typeError("type of attribute and default value differ");
  }
  return nl->ThreeElemList( nl->SymbolAtom(Symbols::APPEND()),
                            nl->OneElemList(nl->IntAtom(index-1)),
                            attrType);
}


int extractDefVM(Word* args, Word& result, int message, Word& local, Supplier s)
{
   Stream<Tuple> stream(args[0]);
   stream.open();
   Tuple* t = stream.request();
   stream.close();
   result = qp->ResultStorage(s);
   Attribute* res = (Attribute*) result.addr;

   if(t){
     int index = ((CcInt*) args[3].addr)->GetValue();
     res->CopyFrom(t->GetAttribute(index));
     t->DeleteIfAllowed();
   } else {
     res->CopyFrom((Attribute*) args[2].addr);
   }
   return 0;
}

OperatorSpec extractDefSpec(
   "stream(tuple) x attrName x DATA -> DATA",
   " _ extractDel[_,_ ] ",
   "Extract the attribute with the specified name "
   "from the first tuple of the incoming stream. "
   "if the stream is empty, a default value (the third "
   "argument is used as the result.",
   " query ten feed extactDel[No,-1]"
);

Operator extractDefOp(
 "extractDef",
 extractDefSpec.getStr(),
 extractDefVM,
 Operator::SimpleSelect,
 extractDefTM
);

/*
2.8 Operator ~head~

This operator fetches the first n elements (e.g. tuples) from a stream.

2.8.1 Type mapping function of operator ~head~

Type mapping for ~head~ is

----  ((stream (tuple ((x1 t1)...(xn tn))) int)   ->
              ((stream (tuple ((x1 t1)...(xn tn))))

  or ((stream T) int) -> (stream T)  for T in kind DATA
----

*/
ListExpr HeadTypeMap( ListExpr args )
{

  if(nl->ListLength(args)!=2){
    return listutils::typeError("two arguments expected");
  }

  string err = " stream(tuple(...) x int or stream(DATA) x int expected";

  ListExpr stream = nl->First(args);
  ListExpr count = nl->Second(args);

  if(( !Stream<Tuple>::checkType(stream) &&
       !Stream<Attribute>::checkType(stream) ) ||
     !CcInt::checkType(count) ){
    return listutils::typeError(err);
  }
  return stream;
}
/*
2.8.3 Value mapping function of operator ~head~

*/

#ifndef USE_PROGRESS

struct HeadLocalInfo
{
  HeadLocalInfo( const int maxTuples = 0 ):
    numTuples( 0 ),
    maxTuples( maxTuples )
    {}

  int numTuples;
  int maxTuples;
};

int Head(Word* args, Word& result, int message, Word& local, Supplier s)
{
  HeadLocalInfo *localInfo;
  Word tupleWord;

  switch(message)
  {
    case OPEN:

      qp->Open(args[0].addr);
      localInfo =
        new HeadLocalInfo( ((CcInt*)args[1].addr)->GetIntval() );
      local.setAddr( localInfo );
      return 0;

    case REQUEST:

      localInfo = (HeadLocalInfo*)local.addr;
      if(localInfo->numTuples >= localInfo->maxTuples)
      {
        return CANCEL;
      }

      qp->Request(args[0].addr, tupleWord);
      if(qp->Received(args[0].addr))
      {
        result = tupleWord;
        localInfo->numTuples++;
        return YIELD;
      }
      else
      {
        return CANCEL;
      }
    case CLOSE:
      if(local.addr)
      {
        localInfo = (HeadLocalInfo*)local.addr;
        delete localInfo;
        local.setAddr(0);
      }
      qp->Close(args[0].addr);
      return 0;
  }
  return 0;
}

#else

struct HeadLocalInfo
{
  HeadLocalInfo( const int maxTuples = 0 ):
    numTuples( 0 ),
    maxTuples( maxTuples )
    {}

  int numTuples;
  int maxTuples;
};

int Head(Word* args, Word& result, int message, Word& local, Supplier s)
{
  HeadLocalInfo *hli;
  Word tupleWord;

  hli = (HeadLocalInfo*)local.addr;

  switch(message)
  {
    case OPEN:

      if ( hli ) delete hli;

      hli =
        new HeadLocalInfo( ((CcInt*)args[1].addr)->GetIntval() );
      local.setAddr( hli );

      qp->Open(args[0].addr);
      return 0;

    case REQUEST:


      if(hli->numTuples >= hli->maxTuples)
      {
        return CANCEL;
      }

      qp->Request(args[0].addr, tupleWord);
      if(qp->Received(args[0].addr))
      {
        result = tupleWord;
        hli->numTuples++;
        return YIELD;
      }
      else
      {
        return CANCEL;
      }

    case CLOSE:
      qp->Close(args[0].addr);
      return 0;

    case CLOSEPROGRESS:
      if ( hli )
      {
        delete hli;
        local.setAddr(0);
      }
      return 0;

    case REQUESTPROGRESS:

      ProgressInfo p1;
      ProgressInfo* pRes;
      const double uHead = 0.00056;

      pRes = (ProgressInfo*) result.addr;

      if ( !hli )  {
         return CANCEL;
      }

      if ( qp->RequestProgress(args[0].addr, &p1) )
      {
        pRes->Card = (p1.Card < hli->maxTuples ? p1.Card : hli->maxTuples);
        pRes->CopySizes(p1);

        double perTupleTime =
    (p1.Time - p1.BTime) / (p1.Card + 1);

        pRes->Time = p1.BTime + (double) pRes->Card * (perTupleTime + uHead);

        pRes->Progress =
    (p1.BProgress * p1.BTime
          + (double) hli->numTuples * (perTupleTime + uHead))
          / pRes->Time;

        pRes->CopyBlocking(p1);
        return YIELD;
      } else {
        return CANCEL;
      }

  }
  return 0;
}

#endif


/*
2.8.4 Specification of operator ~head~

*/
const string HeadSpec  = "( ( \"Signature\" \"Syntax\" \"Meaning\" "
                         "\"Example\" ) "
                         "( "
                         "<text>((stream (tuple([a1:d1, ... ,an:dn]"
                         "))) x int) -> (stream (tuple([a1:d1, ... ,"
                         "an:dn]))) or \n"
                         "((stream T) int) -> (stream T), "
                         "for T in kind DATA.</text--->"
                         "<text>_ head [ _ ]</text--->"
                         "<text>Returns the first n elements of the input "
                         "stream.</text--->"
                         "<text>query cities feed head[10] consume\n"
                         "query intstream(1,1000) head[10] printstream count"
                         "</text--->"
                              ") )";

/*
2.8.5 Definition of operator ~head~

*/
Operator extrelhead (
         "head",                 // name
         HeadSpec,               // specification
         Head,                   // value mapping
         Operator::SimpleSelect, // trivial selection function
         HeadTypeMap             // type mapping
);

/*
2.9 Operators ~max~ and ~min~

2.9.1 Type mapping function of Operators ~max~ and ~min~

Type mapping for ~max~ and ~min~ is

----  ((stream (tuple ((x1 t1)...(xn tn))) xi)  -> ti
              APPEND (i ti)
----

*/
template<bool isMax> ListExpr
MaxMinTypeMap( ListExpr args )
{
  if(nl->ListLength(args)!=2){
   return listutils::typeError("two arguments expected");
  }

  string err = "stream(tuple(...)) x attrname expected";
  ListExpr stream = nl->First(args);
  ListExpr attr = nl->Second(args);

  if(!listutils::isTupleStream(stream) ||
     nl->AtomType(attr)!=SymbolType){
   return listutils::typeError(err);
  }

  ListExpr attrtype;
  string attrname = nl->SymbolValue(attr);
  ListExpr attrlist = nl->Second(nl->Second(stream));
  int j = listutils::findAttribute(attrlist, attrname, attrtype);
  if ( j ) {
    if(!listutils::isSymbol(attrtype,CcReal::BasicType()) &&
       !listutils::isSymbol(attrtype,CcString::BasicType()) &&
       !listutils::isSymbol(attrtype,CcBool::BasicType()) &&
       !listutils::isSymbol(attrtype,CcInt::BasicType()) &&
       !listutils::isSymbol(attrtype,Instant::BasicType()) &&
       !listutils::isSymbol(attrtype,Duration::BasicType()) ){
      return listutils::typeError("result type not in {real, string, "
                                          "bool, int, instant, duration}");
    }
    return nl->ThreeElemList(nl->SymbolAtom(Symbol::APPEND()),
           nl->OneElemList(nl->IntAtom(j)), attrtype);
  } else {
    return listutils::typeError("attribute name " + attrname +
                                "not known in the tuple");
  }
}
/*
2.9.2 Value mapping function of operators ~max~ and ~min~

*/

template<bool isMax> int
MaxMinValueMapping(Word* args, Word& result, int message,
                   Word& local, Supplier s)
{
  bool definedValueFound = false;
  Word currentTupleWord;
  Attribute* extremum =
    (Attribute*)(qp->ResultStorage(s)).addr;
  extremum->SetDefined(false);
  result.setAddr(extremum);

  int attributeIndex = ((CcInt*)args[2].addr)->GetIntval() - 1;

  qp->Open(args[0].addr);
  qp->Request(args[0].addr, currentTupleWord);
  while(qp->Received(args[0].addr))
  {
    Tuple* currentTuple = (Tuple*)currentTupleWord.addr;
    const Attribute* currentAttr =
      (const Attribute*)currentTuple->GetAttribute(attributeIndex);
    if(currentAttr->IsDefined())
    {
      if(definedValueFound)
      {
        if(isMax)
        {
          if(currentAttr->Compare(extremum) > 0)
          {
            extremum->CopyFrom(currentAttr);
          }
        }
        else
        {
          if(currentAttr->Compare(extremum) < 0)
          {
            extremum->CopyFrom(currentAttr);
          }
        }
      }
      else
      {
        definedValueFound = true;
        extremum->CopyFrom(currentAttr);
      }
    }
    currentTuple->DeleteIfAllowed();
    qp->Request(args[0].addr, currentTupleWord);
  }
  qp->Close(args[0].addr);

  return 0;
}

/*
2.9.3 Specification of operator ~max~

*/
const string MaxOpSpec  = "( ( \"Signature\" \"Syntax\" \"Meaning\" "
                          "\"Example\" ) "
                          "( <text>((stream (tuple([a1:d1, ... ,an:dn]"
                          "))) x ai) -> di</text--->"
                          "<text>_ _ mergesec</text--->"
                          "<text>Returns the maximum value of attribute "
                          "ai over the input stream.</text--->"
                          "<text>query cities feed max [ cityname ]"
                          "</text--->"
                              ") )";

/*
2.9.4 Definition of operator ~max~

*/
Operator extrelmax (
         "max",             // name
         MaxOpSpec,           // specification
         MaxMinValueMapping<true>,               // value mapping
         Operator::SimpleSelect,          // trivial selection function
         MaxMinTypeMap<true>         // type mapping
);

/*
2.9.5 Specification of operator ~min~

*/
const string MinOpSpec  = "( ( \"Signature\" \"Syntax\" \"Meaning\" "
                          "\"Example\" ) "
                          "( <text>((stream (tuple([a1:d1, ... ,an:dn])))"
                          " x ai) -> di</text--->"
                          "<text>_ min [ _ ]</text--->"
                          "<text>Returns the minimum value of attribute ai "
                          "over the input stream.</text--->"
                          "<text>query cities feed min [ cityname ]"
                          "</text--->"
                              ") )";

/*
2.9.6 Definition of operator ~min~

*/
Operator extrelmin (
         "min",             // name
         MinOpSpec,           // specification
         MaxMinValueMapping<false>,               // value mapping
         Operator::SimpleSelect,          // trivial selection function
         MaxMinTypeMap<false>         // type mapping
);

/*
2.10 Operators ~avg~, ~sum~, and ~var~

2.10.1 Type mapping function of Operators ~avg~, ~sum~, and ~var~

Type mapping for ~avg~ is

----  ((stream (tuple ((x1 t1)...(xn tn))) xi)  -> real)
              APPEND (i ti)
----

Type mapping for ~sum~ is

----  ((stream (tuple ((x1 t1)...(xn tn))) xi)  -> ti)
              APPEND (i ti)
----

Type mapping for ~var~ is

----  ((stream (tuple ((x1 t1)...(xn tn))) xi)  -> real)
              APPEND (i ti)
----

*/

template<bool isAvg>
ListExpr
AvgSumTypeMap( ListExpr args )
{

  NList type(args);
  if ( !type.hasLength(2) )
  {
    return NList::typeError("Expecting two arguments.");
  }

  NList first = type.first();
  if (
    !first.hasLength(2)  ||
    !first.first().isSymbol(Symbol::STREAM()) ||
    !first.second().hasLength(2) ||
    !first.second().first().isSymbol(Tuple::BasicType()) ||
    !IsTupleDescription( first.second().second().listExpr() ) )
  {
    return NList::typeError("Error in first argument!");
  }

  NList second = type.second();
  if ( !second.isSymbol() ) {
    return NList::typeError( "Second argument must be an attribute name. "
                             "Perhaps the attribute's name may be the name "
           "of a Secondo object!" );
  }

  string attrname = type.second().str();
  ListExpr attrtype = nl->Empty();
  int j = FindAttribute(first.second().second().listExpr(), attrname, attrtype);

  if ( j != 0 )
  {
    if ( nl->SymbolValue(attrtype) != CcReal::BasicType() &&
   nl->SymbolValue(attrtype) != CcInt::BasicType() )
    {
      return NList::typeError("Attribute type is not of type real or int.");
    }
    NList resType = isAvg ? NList(CcReal::BasicType()) : NList(attrtype);
    return NList( NList(Symbol::APPEND()),
                  NList(j).enclose(), resType ).listExpr();
  }
  else
  {
    return NList::typeError( "Attribute name '" + attrname + "' is not known.");
  }
}

/*
Since the list structure has been already checked the selection function can
trust to have a list of the correct format. Hence we can ommit many checks and
access elements directly.

*/

int
AvgSumSelect( ListExpr args )
{
  NList type(args);
  NList first = type.first();
  string attrname = type.second().str();
  ListExpr attrtype = nl->Empty();
  FindAttribute(first.second().second().listExpr(), attrname, attrtype);

  if ( nl->SymbolValue(attrtype) == CcInt::BasicType() )
  {
    return 0;
  }
  return 1;
}
/*

2.10.2 Value mapping function of operators ~avg~, ~sum~, and ~var~

Here we use template functions which may be instantiated with the
following values:

----
    T = int, real
    R = CcInt, CcReal
----

*/
template<class T, class R> int
SumValueMapping(Word* args, Word& result, int message,
                   Word& local, Supplier s)
{
  T sum = 0;
  int number = 0;

  Word currentTupleWord(Address(0));
  int attributeIndex = static_cast<CcInt*>( args[2].addr)->GetIntval() - 1;

  qp->Open(args[0].addr);
  qp->Request(args[0].addr, currentTupleWord);
  while(qp->Received(args[0].addr))
  {
    Tuple* currentTuple = static_cast<Tuple*>( currentTupleWord.addr );
    R* currentAttr =
      static_cast<R*>( currentTuple->GetAttribute(attributeIndex) );

    if( currentAttr->IsDefined() ) // process only defined elements
    {
      number++;
      sum += currentAttr->GetValue();
    }
    currentTuple->DeleteIfAllowed();
    qp->Request(args[0].addr, currentTupleWord);
  }
  qp->Close(args[0].addr);

  result = qp->ResultStorage(s);
  static_cast<R*>( result.addr )->Set(true, sum);
  return 0;
}

template<class T, class R> int
AvgValueMapping(Word* args, Word& result, int message,
                   Word& local, Supplier s)
{
  T sum = 0;
  int number = 0;

  Word currentTupleWord(Address(0));
  int attributeIndex = static_cast<CcInt*>( args[2].addr )->GetIntval() - 1;

  qp->Open(args[0].addr);
  qp->Request(args[0].addr, currentTupleWord);
  while(qp->Received(args[0].addr))
  {
    Tuple* currentTuple = static_cast<Tuple*>( currentTupleWord.addr );
    R* currentAttr =
      static_cast<R*>( currentTuple->GetAttribute(attributeIndex) );

    if( currentAttr->IsDefined() ) // process only defined elements
    {
      number++;
      sum += currentAttr->GetValue();
    }
    currentTuple->DeleteIfAllowed();
    qp->Request(args[0].addr, currentTupleWord);
  }
  qp->Close(args[0].addr);

  result = qp->ResultStorage(s);
  if ( number == 0 ) {
    static_cast<Attribute*>( result.addr )->SetDefined(false);
  }
  else
  {
    SEC_STD_REAL sumreal = sum;
    static_cast<CcReal*>( result.addr )->Set(true, sumreal / number);
  }
  return 0;
}


template<class T, class R> int
VarValueMapping(Word* args, Word& result, int message,
                Word& local, Supplier s)
{
  TupleBuffer *tp = 0;
  GenericRelationIterator *relIter = 0;
  long MaxMem = qp->GetMemorySize(s);

  T sum = 0;
  int number = 0;
  int counter = 0;
  SEC_STD_REAL mean = 0.0;
  SEC_STD_REAL diffsum = 0.0;

  result = qp->ResultStorage(s);
  Word currentTupleWord(Address(0));
  int attributeIndex = static_cast<CcInt*>( args[2].addr )->GetIntval() - 1;

  tp = new TupleBuffer(MaxMem);

  // In a first scan, we compute the stream's MEAN:
  qp->Open(args[0].addr);
  qp->Request(args[0].addr, currentTupleWord);
  while(qp->Received(args[0].addr))
  {
    Tuple* currentTuple = static_cast<Tuple*>( currentTupleWord.addr );
    R* currentAttr =
        static_cast<R*>( currentTuple->GetAttribute(attributeIndex) );

    if( currentAttr->IsDefined() ) // process only defined elements
    {
      number++;
      sum += currentAttr->GetValue();
    }
    tp->AppendTuple(currentTuple);
    currentTuple->DeleteIfAllowed();
    qp->Request(args[0].addr, currentTupleWord);
  }
  qp->Close(args[0].addr);

  // if there was no defined value, we are finished. Otherwise we need a secomd
  // scan to compute the stream's VARIANCE
  if ( number < 2 ) {
    static_cast<Attribute*>( result.addr )->SetDefined(false);
  }
  else
  {
    mean = (sum * 1.0) / number;

    Tuple* currentTuple = 0;
    relIter = tp->MakeScan();
    currentTuple = relIter->GetNextTuple();

    while( currentTuple && (counter <= number) )
    {
      R* currentAttr =
          static_cast<R*>( currentTuple->GetAttribute(attributeIndex) );

      if( currentAttr->IsDefined() ) // process only defined elements
      {
        counter++;
        SEC_STD_REAL Diff = ( (currentAttr->GetValue() * 1.0) - mean );
        diffsum += Diff * Diff;
      }
      currentTuple->DeleteIfAllowed();
      currentTuple = relIter->GetNextTuple();
    }
    delete relIter;
    static_cast<CcReal*>( result.addr )->Set(true, diffsum / (counter - 1));
  }
  delete tp;
  return 0;
}


/*
2.10.3 Operator Descriptions for ~avg~, ~sum~, and ~var~

*/

struct avgInfo : OperatorInfo {

  avgInfo() : OperatorInfo() {

    name      = "avg";
    signature = "((stream (tuple([a1:d1, ...,"
                          " ai:int, ..., an:dn]))) x ai) -> real";
    appendSignature( "((stream (tuple([a1:d1, ...,"
                          " ai:real, ..., an:dn]))) x ai) -> real");
    syntax    = "_ avg [ _ ]";
    meaning   = "Returns the average value of attribute "
                "ai over the input stream. ";
  }

};

struct sumInfo : OperatorInfo {

  sumInfo() : OperatorInfo() {

    name      = "sum";
    signature = "((stream (tuple([a1:d1, ..., "
                            "ai:int, ..., an:dn]))) x ai) -> int";
    appendSignature( "((stream (tuple([a1:d1, ...,"
                         " ai:real, ..., an:dn]))) x ai) -> real" );
    syntax    = "_ sum [ _ ]";
    meaning   = "Returns the sum of the values of attribute "
                "ai over the input stream. ";
  }

};

struct varInfo : OperatorInfo {

  varInfo() : OperatorInfo() {

    name      = "var";
    signature = "((stream (tuple([a1:d1, ..., "
        "ai:int, ..., an:dn]))) x ai) -> real";
    appendSignature( "((stream (tuple([a1:d1, ...,"
        " ai:real, ..., an:dn]))) x ai) -> real" );
    syntax    = "_ var [ _ ]";
    meaning   = "Returns the variance of the values of attribute "
        "ai over the input stream. Needs to perform 2 scans!";
  }

};

/*
10.5 Operator ~stats~

This operator calculates several aggregation functions and statistics on a
stream of tuples containing two mumeric attributes. And returns a single tuple
with
all the results:

  * CountX - Number of defined instances for the first attribute X

  * MinX   - minimum instance of the first attribute

  * MaxX   - maximum instance of the first attribute

  * SumX   - sum for all defined instance of the first attribute

  * AvgX   - mean for all defined instance of the first attribute

  * VarX   - variance for all defined instance of the first attribute

  * CountY - Number of defined instances for the second attribute Y

  * MinY   - minimum instance of the second attribute

  * MaxY   - maximum instance of the second attribute

  * SumY   - sum for all defined instance of the second attribute

  * AvgY   - mean for all defined instance of the second attribute

  * VarY   - variance for all defined instance of the second attribute

  * Count  - Cardinality of the stream

  * CountXY - Number of tuples, where both arguments X and Y are defined

  * CovXY  - the covariance for X and Y

  * Corr   - the Pearson product-moment correlation coefficient for X and Y

*/

/*
10.5.1 Type mapping for ~stats~

Type mapping for ~stats~ is

----  ((stream (tuple ((x1 t1)...(xn tn))) xi xj)  -> stream(tuple(
       (CountX int) (MinX real) (MaxX real) (SumX real) (AvgX real) (VarX real)
       (CountY int) (MinY real) (MaxY real) (SumY real) (AvgY real) (VarY real)
             (Count int) (CountXY int) (CovXY real) (CorrXY real)))
              APPEND ((i ti) (j tj))

      where ti, ty in {int, real}
----

*/

ListExpr  StatsTypeMap( ListExpr args )
{

  NList type(args);
  if ( !type.hasLength(3) )
  {
    return NList::typeError("Expecting three arguments.");
  }

  NList first = type.first();
  if (
      !first.hasLength(2)  ||
      !first.first().isSymbol(Symbol::STREAM()) ||
      !first.second().hasLength(2) ||
      !first.second().first().isSymbol(Tuple::BasicType()) ||
      !IsTupleDescription( first.second().second().listExpr() ) )
  {
    return NList::typeError("First argument must be of type stream(tuple(X))");
  }

  NList second = type.second();
  if ( !second.isSymbol() ) {
    return NList::typeError( "Second argument must be an attribute name. "
        "Perhaps the attribute's name may be the name "
        "of a Secondo object!" );
  }

  NList third = type.third();
  if ( !second.isSymbol() ) {
    return NList::typeError( "Third argument must be an attribute name. "
        "Perhaps the attribute's name may be the name "
        "of a Secondo object!" );
  }

  string attrnameX = type.second().str();
  ListExpr attrtypeX = nl->Empty();
  int jX = FindAttribute(first.second().second().listExpr(),
                         attrnameX, attrtypeX);

  string attrnameY = type.third().str();
  ListExpr attrtypeY = nl->Empty();
  int jY = FindAttribute(first.second().second().listExpr(),
                         attrnameY, attrtypeY);

  if ( (jX != 0) && (jY != 0) )
  {
    if ( nl->SymbolValue(attrtypeX) != CcReal::BasicType() &&
         nl->SymbolValue(attrtypeX) != CcInt::BasicType() )
    {
      return NList::typeError(
          "Attribute type of 2nd argument is not of type real or int.");
    }
    if ( nl->SymbolValue(attrtypeY) != CcReal::BasicType() &&
         nl->SymbolValue(attrtypeY) != CcInt::BasicType() )
    {
      return NList::typeError(
          "Attribute type of 3rd argument is not of type real or int.");
    }
    NList resTupleType = NList(NList("CountX"),
                               NList(CcInt::BasicType())).enclose();
    resTupleType.append(NList(NList("MinX"),NList(CcReal::BasicType())));
    resTupleType.append(NList(NList("MaxX"),NList(CcReal::BasicType())));
    resTupleType.append(NList(NList("SumX"),NList(CcReal::BasicType())));
    resTupleType.append(NList(NList("AvgX"),NList(CcReal::BasicType())));
    resTupleType.append(NList(NList("VarX"),NList(CcReal::BasicType())));
    resTupleType.append(NList(NList("CountY"),NList(CcInt::BasicType())));
    resTupleType.append(NList(NList("MinY"),NList(CcReal::BasicType())));
    resTupleType.append(NList(NList("MaxY"),NList(CcReal::BasicType())));
    resTupleType.append(NList(NList("SumY"),NList(CcReal::BasicType())));
    resTupleType.append(NList(NList("AvgY"),NList(CcReal::BasicType())));
    resTupleType.append(NList(NList("VarY"),NList(CcReal::BasicType())));
    resTupleType.append(NList(NList("Count"),NList(CcInt::BasicType())));
    resTupleType.append(NList(NList("CountXY"),NList(CcInt::BasicType())));
    resTupleType.append(NList(NList("CovXY"),NList(CcReal::BasicType())));
    resTupleType.append(NList(NList("CorrXY"),NList(CcReal::BasicType())));
    NList resType = NList( NList(Symbol::APPEND()), NList(NList(jX), NList(jY)),
                           NList(NList(Symbol::STREAM()),
                                 NList(NList(Tuple::BasicType()),resTupleType))
                         );
    return resType.listExpr();
  }
  else
  {
    if (jX == 0)
      return NList::typeError( "Attribute '" + attrnameX + "' is not known.");
    else
      return NList::typeError( "Attribute '" + attrnameY + "' is not known.");
  }
}

/*
Since the list structure has been already checked the selection function can
trust to have a list of the correct format. Hence we can ommit many checks and
access elements directly.

*/

int StatsSelect( ListExpr args )
{
  NList type(args);
  NList first = type.first();
  string attrnameX = type.second().str();
  string attrnameY = type.third().str();
  ListExpr attrtypeX = nl->Empty();
  ListExpr attrtypeY = nl->Empty();
  FindAttribute(first.second().second().listExpr(), attrnameX, attrtypeX);
  FindAttribute(first.second().second().listExpr(), attrnameY, attrtypeY);

  if ((nl->SymbolValue(attrtypeX) == CcInt::BasicType())&&(
    nl->SymbolValue(attrtypeY) == CcInt::BasicType()))
    return 0;
  if ((nl->SymbolValue(attrtypeX) == CcInt::BasicType())&&
    (nl->SymbolValue(attrtypeY) == CcReal::BasicType()))
    return 1;
  if ((nl->SymbolValue(attrtypeX) == CcReal::BasicType())&&
    (nl->SymbolValue(attrtypeY) == CcInt::BasicType()))
    return 2;
  if ((nl->SymbolValue(attrtypeX) == CcReal::BasicType())&&
    (nl->SymbolValue(attrtypeY)==CcReal::BasicType()))
    return 3;
  assert( false );
  return -1;
}

/*
10.5.2 Value mapping for ~stats~

There are 4 template class parameters.
Tx and Ty are ~int~ or ~real~,
Rx and Ry are the corresponding types ~CcInt~ resp. ~CcReal~.

*/

template<class Tx, class Rx, class Ty, class Ry> int
    StatsValueMapping(Word* args, Word& result, int message,
                      Word& local, Supplier s)
{

  TupleBuffer *tp = 0;
  GenericRelationIterator *relIter = 0;
  long MaxMem = qp->GetMemorySize(s); 
  bool *finished = 0;
  TupleType *resultTupleType = 0;
  Tuple *newTuple = 0;
  CcInt *CCountX = 0;
  CcReal *CMinX = 0;
  CcReal *CMaxX = 0;
  CcReal *CSumX = 0;
  CcReal *CAvgX = 0;
  CcReal *CVarX = 0;
  CcInt *CCountY = 0;
  CcReal *CMinY = 0;
  CcReal *CMaxY = 0;
  CcReal *CSumY = 0;
  CcReal *CAvgY = 0;
  CcReal *CVarY = 0;
  CcInt *CCount = 0;
  CcInt *CCountXY = 0;
  CcReal *CCovXY = 0;
  CcReal *CCorrXY = 0;

  switch(message)
  {
    case OPEN:
    {
      finished = new bool(false);
      local.addr = finished;
      return 0;
    }

    case REQUEST:
    {
      if(local.addr == 0)
        return CANCEL;
      finished = (bool*) local.addr;
      if(*finished)
        return CANCEL;

      int countAll = 0, countX = 0, countY = 0, countXY = 0;
      SEC_STD_REAL minX = 0, maxX = 0, sumX = 0;
      SEC_STD_REAL minY = 0, maxY = 0, sumY = 0;
      SEC_STD_REAL meanX = 0, meanY  = 0;
      SEC_STD_REAL diffsumX = 0, diffsumY = 0, sumXY = 0;
      SEC_STD_REAL sumX2 = 0, sumY2 = 0, sumsqX = 0, sumsqY = 0;

      Tx currX = 0;
      Ty currY = 0;

      result = qp->ResultStorage(s);
      Word currentTupleWord(Address(0));
      int attributeIndexX = static_cast<CcInt*>(args[3].addr)->GetIntval() - 1;
      int attributeIndexY = static_cast<CcInt*>(args[4].addr)->GetIntval() - 1;

      tp = new TupleBuffer(MaxMem);

      // In a first scan, we compute the stream's MEANs for X,Y
      // and count defined values for X,Y:
      qp->Open(args[0].addr);
      qp->Request(args[0].addr, currentTupleWord);
      while(qp->Received(args[0].addr))
      {
        countAll++;
        Tuple* currentTuple = static_cast<Tuple*>( currentTupleWord.addr );
        Rx* currentAttrX =
            static_cast<Rx*>( currentTuple->GetAttribute(attributeIndexX) );
        Ry* currentAttrY =
            static_cast<Ry*>( currentTuple->GetAttribute(attributeIndexY) );

        if( currentAttrX->IsDefined() )
        {
          countX++;
          currX = currentAttrX->GetValue();
          if (countX == 1)
          {
            minX = currX;
            maxX = currX;
          }
          else
          {
            if (currX < minX)
              minX = currX;
            if (currX > maxX)
              maxX = currX;
          }
          sumX += currX;
        }
        if( currentAttrY->IsDefined() )
        {
          countY++;
          currY = currentAttrY->GetValue();
          if (countY == 1)
          {
            minY = currY;
            maxY = currY;
          }
          else
          {
            if (currY < minY)
              minY = currY;
            if (currY > maxY)
              maxY = currY;
          }
          sumY += currY;
        }
        tp->AppendTuple(currentTuple);
        if(currentTuple->DeleteIfAllowed()){
            cout << "stats :: deleting a tuple stored in a buffer !!!" << endl;
        }
        qp->Request(args[0].addr, currentTupleWord);
      }
      qp->Close(args[0].addr);

      if ( (countX + countY) > 1 )
      {
        meanX = (sumX * 1.0) / countX;
        meanY = (sumY * 1.0) / countY;

        // A second Scan to calculate the emprical covariance and
        // emprical correlation coefficient. Therefore, we recalculate the means
        // using only those tuples, where both attributes are defined...
        Tuple* currentTuple = 0;
        relIter = tp->MakeScan();
        currentTuple = relIter->GetNextTuple();

        while( currentTuple )
        {
          Rx* currentAttrX =
              static_cast<Rx*>( currentTuple->GetAttribute(attributeIndexX) );
          Ry* currentAttrY =
              static_cast<Ry*>( currentTuple->GetAttribute(attributeIndexY) );
          if( currentAttrX->IsDefined() )
          { // for var(X):
            currX = currentAttrX->GetValue();
            diffsumX += (currX - meanX) * (currX - meanX);
          }
          if( currentAttrY->IsDefined() )
          { // for var(Y):
            currY = currentAttrY->GetValue();
            diffsumY += (currY - meanY) * (currY - meanY);
          }
          if( currentAttrX->IsDefined() && currentAttrY->IsDefined() )
          { // for cov(X,Y) and corr(X,Y):
            countXY++;
            sumX2 += currX;
            sumY2 += currY;
            sumXY += currX * currY;
            sumsqX += currX * currX;
            sumsqY += currY * currY;
          }
          currentTuple->DeleteIfAllowed();
          currentTuple = relIter->GetNextTuple();
        }
        delete relIter;
      }
      delete tp;

      // create the resulttuple:
      resultTupleType = new TupleType(nl->Second(GetTupleResultType(s)));
      newTuple = new Tuple( resultTupleType );

      CCountX = new CcInt(true, countX);
      CMinX   = new CcReal((countX > 0), minX);
      CMaxX   = new CcReal((countX > 0), maxX);
      CSumX   = new CcReal(true, sumX);
      if (countX > 0){
        CAvgX = new CcReal(true, sumX / countX);
        CVarX = new CcReal(true, diffsumX / (countX - 1));
      }
      else{
        CAvgX = new CcReal(false, 0);
        CVarX = new CcReal(false, 0);
      }
      CCountY = new CcInt(true, countY);
      CMinY   = new CcReal((countY > 0), minY);
      CMaxY   = new CcReal((countY > 0), maxY);
      CSumY   = new CcReal(true, sumY);
      if (countY > 0){
        CAvgY = new CcReal(true, sumY / countY);
        CVarY = new CcReal(true, diffsumY / (countY - 1));
      }
      else {
        CAvgY = new CcReal(false, 0);
        CVarY = new CcReal(false, 0);
      }
      CCount  = new CcInt(true, countAll);
      CCountXY = new CcInt(true, countXY);
      if(countXY > 1){
        SEC_STD_REAL covXY =
            (sumXY - (sumX2 * sumY2 / countXY)) / (countXY - 1);
        CCovXY = new CcReal(true, covXY);
        SEC_STD_REAL meanX2 = sumX2/countXY;
        SEC_STD_REAL meanY2 = sumY2/countXY;
        SEC_STD_REAL denom =   sqrt(sumsqX - (countXY * (meanX2 * meanX2)))
                             * sqrt(sumsqY - (countXY * (meanY2 * meanY2)));
        if (denom == 0){
            CCorrXY = new CcReal(false, 0);
        }
        else{
          CCorrXY = new
              CcReal(true, (sumXY - (countXY * meanX2 * meanY2) ) / denom);
        }
      }
      else {
        CCovXY  = new CcReal(false, 0);
        CCorrXY = new CcReal(false, 0);
      }
      newTuple->PutAttribute(  0,(Attribute*)CCountX);
      newTuple->PutAttribute(  1,(Attribute*)CMinX);
      newTuple->PutAttribute(  2,(Attribute*)CMaxX);
      newTuple->PutAttribute(  3,(Attribute*)CSumX);
      newTuple->PutAttribute(  4,(Attribute*)CAvgX);
      newTuple->PutAttribute(  5,(Attribute*)CVarX);
      newTuple->PutAttribute(  6,(Attribute*)CCountY);
      newTuple->PutAttribute(  7,(Attribute*)CMinY);
      newTuple->PutAttribute(  8,(Attribute*)CMaxY);
      newTuple->PutAttribute(  9,(Attribute*)CSumY);
      newTuple->PutAttribute( 10,(Attribute*)CAvgY);
      newTuple->PutAttribute( 11,(Attribute*)CVarY);
      newTuple->PutAttribute( 12,(Attribute*)CCount);
      newTuple->PutAttribute( 13,(Attribute*)CCountXY);
      newTuple->PutAttribute( 14,(Attribute*)CCovXY);
      newTuple->PutAttribute( 15,(Attribute*)CCorrXY);

      result.setAddr(newTuple);
      resultTupleType->DeleteIfAllowed();

      *finished = true;
      return YIELD;
    }

    case CLOSE:
    {
      if(local.addr == 0)
        return 0;
      finished = (bool*) local.addr;
      delete(finished);
      local.setAddr(0);
      return 0;
    }
  } // end switch
  return -1;
}

/*
10.5.2 Operator Descriptions for ~stats~

*/

struct statsInfo : OperatorInfo {

  statsInfo() : OperatorInfo() {

    name      = "stats";
    signature = "((stream (tuple([a1:d1, ... ,an:dn]))) x ai x aj) -> "
    "stream(tuple( \n"
    "(CountX int) (MinX real) (MaxX real) (SumX real) (AvgX real) (VarX real)\n"
    "(CountY int) (MinY real) (MaxY real) (SumY real) (AvgY real) (VarY real)\n"
    "(Count int) (CountXY int) (CovXY real) (CorrXY real))\n, "
    "where ti, tj in {int,real}";
    syntax    = "_ stats [ attrnameX , attrnameY ]";
    meaning   = "Returns a tuple containing several statistics for 2 numerical "
        "attributes 'attrnameX', 'attrnameY' on the stream. The first 6 "
        "attributes contain the number of tuples where X is defined, the "
        "minimum and maximum value for X, the sum for X, the mean for X, and "
        "the variance of X. \n"
        "The 7th to 12th attribute hold the appropriate statistics for Y.\n"
        "the last 4 attributes hold the total count of tuples in the stream, "
        "the number of tuples, where both attributes X and Y contain defined "
        "values, the covariance of X and X, and finally Pearson's "
        "correlation coefficient";
  }

};


/*
2.10 Operator ~krdup~

This operator removes duplicates from a tuple stream. In contrast
to the usual rdup operator, the test of equality of tuples is
done using only the attributes from the given attribute list.

2.10.1 Type mapping of the operator ~krdup~

*/

ListExpr krdupTM(ListExpr args){
  if(nl->ListLength(args)<2){
    ErrorReporter::ReportError("minimum of 2 arguments required");
    return nl->TypeError();
  }

  ListExpr streamList = nl->First(args);
  ListExpr attrnameList = nl->Second(args);
  if(nl->ListLength(attrnameList)<1){
     ErrorReporter::ReportError("invalid number of attribute names");
     return nl->TypeError();
  }

  // extract the attribute names
  set<string> names;
  int pos = 0;
  while(!nl->IsEmpty(attrnameList)){
      ListExpr attr = nl->First(attrnameList);
      pos++;
      attrnameList = nl->Rest(attrnameList);
      if(nl->AtomType(attr)!=SymbolType){
        ErrorReporter::ReportError("argument is not "
                                   " an attribute name");
        return nl->TypeError();
      }
      string n = nl->SymbolValue(attr);
      // insert the name into the set of names
      names.insert(nl->SymbolValue(attr));
  }

  // check the stream argument
  if(nl->ListLength(streamList)!=2 ||
     !nl->IsEqual(nl->First(streamList),Symbol::STREAM())){
     ErrorReporter::ReportError("(stream (tuple(...)) "
                                " expected as first argument");
     return nl->TypeError();
  }

  ListExpr tupleList = nl->Second(streamList);
  if(nl->ListLength(tupleList)!=2 ||
     !nl->IsEqual(nl->First(tupleList),Tuple::BasicType())){
     ErrorReporter::ReportError("(stream (tuple(...)) "
                                " expected as first argument");
     return nl->TypeError();
  }
  ListExpr attrList = nl->Second(tupleList);

  if(names.empty()){ // should never occur
    ErrorReporter::ReportError("internal error, no attr names found.");
    return nl->TypeError();
  }

  pos = 0;

  ListExpr PosList = nl->TheEmptyList();
  ListExpr Last = nl->TheEmptyList();
  bool first = true;

  while(!nl->IsEmpty(attrList)){
     ListExpr pair = nl->First(attrList);
     if((nl->ListLength(pair)!= 2) ||
        (nl->AtomType(nl->First(pair))!=SymbolType)){
        ErrorReporter::ReportError("Error in tuple definition ");
        return nl->TypeError();
     }
     string name = nl->SymbolValue(nl->First(pair));
     if(names.find(name)!= names.end()){
         if(first){
           first = false;
           PosList = nl->OneElemList(nl->IntAtom(pos));
           Last = PosList;
           first = false;
         } else {
           Last = nl->Append(Last,nl->IntAtom(pos));
         }
         names.erase(name);
     }
     pos++;
     attrList = nl->Rest(attrList);
  }

  if(!names.empty()){
    ErrorReporter::ReportError("at least one given attribute name is "
                               "not part of the stream");
    return nl->TypeError();
  }


  ListExpr result =  nl->ThreeElemList(nl->SymbolAtom(Symbol::APPEND()),
                                      nl->OneElemList(PosList),
                                      streamList);
  return result;
}

/*
2.10.2 Value Mapping of the ~krdup~ operator

This value mapping is very similar the this one of the
rdup operator. The only difference is, only specified
attributes are used for the check of equality.

*/


class KrdupLocalInfo{
public:
/*
~Constructor~

Creates a localinfo for the krdup operator from the Supplier
containing the information about the indexes which should be used
for the comparison.

*/
  KrdupLocalInfo(vector<int>* _indexes, Word& _stream):
    stream(_stream),indexes(*_indexes),lastTuple(0){
    stream.open();
   }

   static vector<int>* getIndexes(Supplier s){
       vector<int>* indexes = new vector<int>();
       int count = qp->GetNoSons(s);
       Word elem;
       for(int i=0;i<count; i++){
          Supplier son = qp->GetSon(s,i);
          qp->Request(son,elem);
          int index = ((CcInt*)elem.addr)->GetIntval();
          indexes->push_back(index);
       }
       return indexes;
   }


/*
~Destructor~

Destroy this instance.

*/

   ~KrdupLocalInfo(){
       if(lastTuple!=0){
          lastTuple->DeleteIfAllowed();
       }
       lastTuple = 0;
       stream.close();
   }


   Tuple* next(){
      Tuple* tuple;
      while((tuple=stream.request())){
         if(!ReplaceIfNonEqual(tuple)){
            tuple->DeleteIfAllowed();
         } else {
            return tuple;
         }
      }
      return 0;
   }


private:
/*
~equals~

Checks the equality of the last tuple with the argument where only the
attributes stored in the vector are of interest.

*/
inline bool Equals(Tuple* tuple){
 int size = indexes.size();
 for(int i=0;i<size;i++){
   int pos = indexes[i];
   int cmp = ( ((Attribute*)lastTuple->GetAttribute(pos))->CompareAlmost(
               ((Attribute*)tuple->GetAttribute(pos))));
   if(cmp!=0){
       return false;
   }
 }
 return true;
}


/*
~ReplaceIfNonEqual~

If the argument differs from the last stored tuple, the old tuple
is replaced by the argument and the return value is true. Otherwise,
this instance is keep unchanged and the reuslt of the call is false.
The old tuple is removed automatically.

*/
inline bool  ReplaceIfNonEqual(Tuple* newTuple){
   if(lastTuple==0){ // first call
      newTuple->IncReference();
      lastTuple = newTuple;
      return true;
   }

   if(Equals(newTuple)){
     return false;
   } else{ // replace the tuple
     lastTuple->DeleteIfAllowed();
     newTuple->IncReference();
     lastTuple = newTuple;
     return true;
   }
}

   Stream<Tuple> stream;
   vector<int> indexes;
   Tuple* lastTuple;
};


int KrdupVM(Word* args, Word& result, int message,
                     Word& local, Supplier s)
{
 

  switch(message)
  {
    case INIT : {
      return 0;
    }
    case FINISH : {
       vector<int>* indexes = (vector<int>*) qp->GetLocal2(s).addr; 
       if(indexes){
         delete indexes;
         qp->GetLocal2(s).addr = 0;
       }
       return 0;
    }

    case OPEN:{
      vector<int>* indexes = (vector<int>*) qp->GetLocal2(s).addr;
      if(!indexes){
         indexes = KrdupLocalInfo::getIndexes(qp->GetSon(s,2));
         qp->GetLocal2(s).addr = indexes;
      } 
      local.setAddr(new KrdupLocalInfo(indexes,args[0]));
      return 0;
    }
    case REQUEST: {
      KrdupLocalInfo* localinfo = (KrdupLocalInfo*) local.addr;
      result.addr = localinfo?localinfo->next():0;
      return result.addr?YIELD:CANCEL;  
    }
    case CLOSE:
      if(local.addr) {
        KrdupLocalInfo* localinfo = (KrdupLocalInfo*) local.addr;
        delete localinfo;
        local.setAddr(0);
      }
      qp->Close(args[0].addr);
      return 0;
  }
  return 0;
}


/*
2.13.2 Specification of operator ~krdup~

*/
const string KrdupSpec  = "( ( \"Signature\" \"Syntax\" \"Meaning\" "
                         "\"Example\" ) "
                         "( <text>((stream (tuple([a1:d1, ... ,an:dn]"
                            " x ai1 .. ain))))"
                         " -> (stream (tuple([a1:d1, ... ,an:dn])))"
                         "</text--->"
                         "<text>_ krdup[ _ , _ , ...]</text--->"
                         "<text>Removes duplicates from a sorted "
                         "stream with respect to the attributes "
                         "given as arguments.</text--->"
                         "<text>query plz feed sortby [Ort] "
                         "krdup[Ort]  consume</text--->"
                              ") )";

/*
2.13.3 Definition of operator ~krdup~

*/
Operator krdup (
         "krdup",             // name
         KrdupSpec,           // specification
         KrdupVM,               // value mapping
         Operator::SimpleSelect,          // trivial selection function
         krdupTM         // type mapping
);


/*
2.10 Operator ~krduph~

This operator removes duplicates from a stream by comparing given key 
attributes.
In constrast to ~krdup~, this operator does not require a sorted input stream.

2.10.1 Type Mapping

  stream(tuple) x attrname[_]1 x attrname[_]2 ... [x int] -> stream(tuple)


*/
ListExpr krduphTM(ListExpr args){

   string err = "stream(tuple) x attrname_1 x attrname_2 ... [x int] expected";
   if(nl->ListLength(args) < 1){ 
     return listutils::typeError(err);
   }

   ListExpr stream = nl->First(args);
   if(!Stream<Tuple>::checkType(stream)){
     return listutils::typeError(err + "(first attr is not a tuple stream)");
   }

   args = nl->Rest(args);
   bool done = nl->IsEmpty(args);

   vector<int> positions;
   bool numGiven = false;
   ListExpr attrList = nl->Second(nl->Second(stream));
   ListExpr attrType;

   while(!done){
      ListExpr first = nl->First(args);
      if(CcInt::checkType(first)){
         done = true;
         numGiven = true;
         args = nl->Rest(args);
         if(!nl->IsEmpty(args)){
           return listutils::typeError(err + 
                " (int is allowed as the last argument only");
         }
     } else {
       if(!listutils::isSymbol(first)){
           return listutils::typeError(err + " (invalid attribute name)");
       }
       string attrName = nl->SymbolValue(first);
       int index = listutils::findAttribute(attrList, attrName, attrType);
       if(index == 0){
          return listutils::typeError("attribute " + attrName + 
                                      "not present");
       }
       positions.push_back(index -1); 
       args = nl->Rest(args);
       done = nl->IsEmpty(args);
     }
   }
  

   ListExpr appendList; 
   ListExpr last;
   bool appendInit = false;
   if(positions.empty()){ // use all attributes
      for(int i=0;i<nl->ListLength(attrList);i++){
         positions.push_back(i);
      }
      // append dummy parameters 
      appendList = nl->OneElemList(nl->IntAtom(0));
      last = appendList;
      for(unsigned int i=1;i<positions.size();i++) {
         last = nl->Append(last,nl->IntAtom(0));
      } 
      appendInit = true;
   }
   unsigned int start = 0;
   if(!numGiven){
      if(!appendInit){ 
        appendList = nl->OneElemList(nl->IntAtom(-1));
        last = appendList;
      } else {
        last = nl->Append(last,nl->IntAtom(-1));
      }
   } else {
      if(!appendInit){
        appendList = nl->OneElemList(nl->IntAtom(positions[0]));
        last = appendList;
        start = 1;
      }
   }

   for(unsigned int i=start;i<positions.size();i++){
      last = nl->Append(last, nl->IntAtom(positions[i]));
   }

 
   ListExpr resultList = nl->ThreeElemList( 
                             nl->SymbolAtom(Symbols::APPEND()),
                             appendList,
                             stream);

   cout << nl->ToString(resultList) << endl;

   return resultList;
}
// TODO

/*
2.10.2 LocalInfo

*/

class KrdupHInfo{

   public:
      KrdupHInfo(Word& _s, const int _buckNum, 
                 vector<int>& _positions, 
                 const size_t _maxMem,
                 TupleType* _tt) : 
                 stream(_s), buckNum(_buckNum), 
                 positions(_positions), maxMem(_maxMem), usedMem(0),
                 tupleType(_tt) {

          stream.open();
          if(buckNum<97){
             buckNum = 97;
          }
          if(buckNum > 9999997){
             buckNum = 9999997;
          }
          buckets = new vector<Tuple*>*[buckNum];
          memset(buckets,0, buckNum*sizeof(void*));
          usedMem = buckNum * sizeof(void*);
          if(usedMem>maxMem){
            maxMem = usedMem +128;
          }
          buffer1  = 0;
          buffer2 = 0;
          it = 0;
          tupleType->IncReference(); // reserved for local usage
          stored=0;
          scans = 1; // at least the stream must be scanned


      }

      ~KrdupHInfo(){
         stream.close();
         tupleType->DeleteIfAllowed(); 
         eraseBuckets();
         delete[] buckets;
         if(it){
            delete it;
         } 
         if(buffer1){
           buffer1->DeleteAndTruncate();
         }
         if(buffer2){
           buffer2->DeleteAndTruncate();
         }
         cout << "Krduph finished with " << scans << "scans" << endl;
       }



       Tuple* nextTuple(){
          Tuple* res = 0;
          while(true){ 
             res = nextInput();
             if(res==0){
               return 0;
             }
             bool ok = insert(res);
             if(ok){
                return res;
             } else {
               res->DeleteIfAllowed();
             }
          }
       }




   private:
      Stream<Tuple> stream;
      int buckNum;
      vector<int> positions;
      size_t maxMem; 
      vector<Tuple*>** buckets;
      Relation* buffer1; // buffer for reading tuples
      Relation* buffer2; // buffer fro writing tuples
      GenericRelationIterator* it;  
      size_t usedMem;
      TupleType* tupleType;
      int stored;
      int scans;

      void eraseBuckets(){
         for(int i=0;i<buckNum;i++){
            if(buckets[i]){
                vector<Tuple*>* bucket = buckets[i];
                for(unsigned int j=0;j<bucket->size();j++){
                   (*bucket)[j]->DeleteIfAllowed();
                } 
                delete bucket;
                buckets[i] = 0;
            }
         }
         stored = 0; 
         usedMem = sizeof(void*) * buckNum;
      }

      Tuple* nextInput(){
          if(!it){ // try to get data from stream
            Tuple* res = stream.request();
            if(res){ // tuple found
               return res;
            } 
            // no tuple found in original stream
            if(!buffer2){ // no tuples written out
               return 0;
            }
            eraseBuckets();
            buffer1 = buffer2; // change read buffer
            buffer2 = 0;
            it = buffer1->MakeScan();
          }

          Tuple* res = it->GetNextTuple();
          if(res!=0){
            return res;
          }
          // iterator finished
          delete it;
          it = 0;
          buffer1->DeleteAndTruncate();
          buffer1 = 0;
          if(!buffer2){ 
              return 0;
          }
          buffer1 = buffer2;
          buffer2 = 0;
          eraseBuckets();
          it = buffer1->MakeScan();
          return it->GetNextTuple();
      }

/*
Inserts a tuple into this structure.

*/
      bool insert(Tuple* tuple){
         if(usedMem>maxMem){
            if(!contains(tuple)){
               if(!buffer2){
                  buffer2 = new Relation(tupleType, true);
                  scans++;
               } 
               tuple->PinAttributes();
               buffer2->AppendTupleNoLOBs(tuple);
            }
            return false;
         }
         return insertMM(tuple);
      }


/*
Inserts a tuple into the memory part of this structure.

*/
      bool insertMM(Tuple* tuple){
         size_t hash = computeHash(tuple);
         vector<Tuple*>* bucket = buckets[hash];
         if(!bucket){
            bucket = new vector<Tuple*>();
            buckets[hash] = bucket;
            bucket->push_back(tuple);
            stored ++;
            tuple->IncReference();
            usedMem += tuple->GetMemSize() + sizeof(void*) + sizeof(*bucket);
            return true; 
         }
         for(size_t i=0;i<bucket->size();i++){
            if(equals((*bucket)[i],tuple)){ // tuple found
              return false;
            }
         }
         bucket->push_back(tuple);
         tuple->IncReference();
         usedMem += tuple->GetMemSize() + sizeof(void*);
         return true;
      }

/*
Checks whether the memory part of this structur contains a tuple;

*/
     bool contains(Tuple* tuple) const{
       size_t hash = computeHash(tuple);
       vector<Tuple*>* bucket = buckets[hash];
       if(!bucket){
         return false;
       }
       for(size_t i =0;i<bucket->size();i++){
         if(equals((*bucket)[i],tuple)){
            return true;
         }
       }
       return false;
     }
       

/*
Computes the hash value of a tuple based on the key attributes.

*/

   size_t computeHash(const Tuple* tuple) const{
     size_t hash = 0;
     for(size_t i=0;i<positions.size();i++){
        hash += tuple->GetAttribute(positions[i])->HashValue();
     }
     return hash % buckNum;
   }

/*
Returns true if the key attributes of the given tuples are euqal.

*/
    bool equals(const Tuple* t1, const Tuple* t2) const{
       for(size_t i=0;i<positions.size(); i++){
          int k = positions[i];
          if(t1->GetAttribute(k)->Compare(t2->GetAttribute(k)) != 0){
             return false;
          }
       }
       return true;
    }

};

/*
2.10.3

*/
int krduphVM(Word* args, Word& result, int message,
                     Word& local, Supplier s)
{
  KrdupHInfo* li = (KrdupHInfo*) local.addr;
  TupleType* tt = (TupleType*) qp->GetLocal2(s).addr;
  switch(message){

     case INIT : {
        assert(!tt);
        qp->GetLocal2(s).addr = new TupleType(nl->Second(
                                GetTupleResultType(s)));
        return 0;
     }

     case OPEN :{  if(li){
                     delete li;
                   }
                   int sc = qp->GetNoSons(s) / 2;
                   CcInt* b = (CcInt*) args[sc].addr;
                   int bm = 999997;
                   if(b->IsDefined() && b->GetValue()>0){
                       bm = b->GetValue();
                   }

                   vector<int> positions;

                       
                   for(int i=sc+1;i<qp->GetNoSons(s);i++){
                      positions.push_back(((CcInt*)args[i].addr)->GetValue());
                   }
                   
                   size_t mem = qp->GetMemorySize(s)*1024*1024; // in bytes
                   local.addr = new KrdupHInfo(args[0],bm, positions, mem,tt);
                }
    case REQUEST: { if(!li){
                      return CANCEL;
                    }
                    result.addr = li->nextTuple();
                    return result.addr?YIELD:CANCEL;
                  }
    case CLOSE : {
                     if(li){
                       delete li;
                       local.addr = 0;
                     }
                 }

    case FINISH : {
         if(tt){
            tt->DeleteIfAllowed();
            qp->GetLocal2(s).addr = 0;
         }
         return 0;
    }
               
  };
  return 0;
}

/*
2.13.2 Specification of operator ~krdup~

*/
const string krduphSpec  = "( ( \"Signature\" \"Syntax\" \"Meaning\" "
                         "\"Example\" ) "
                         "( <text>((stream (tuple([a1:d1, ... ,an:dn]"
                            " x ai1 .. ain [x int]))))"
                         " -> (stream (tuple([a1:d1, ... ,an:dn])))"
                         "</text--->"
                         "<text>_ krduph[ _ , _ , ... [, buckets]]</text--->"
                         "<text>Removes duplicates from a "
                         "stream with respect to the attributes "
                         "given as arguments.</text--->"
                         "<text>query plz feed  "
                         "krdup[Ort]  consume</text--->"
                              ") )";

/*
2.13.3 Definition of operator ~krdup~

*/
Operator krduph (
         "krduph",             // name
         krduphSpec,           // specification
         krduphVM,               // value mapping
         Operator::SimpleSelect,          // trivial selection function
         krduphTM         // type mapping
);


/*

2.10 Operator k-smallest

2.10.1 Type Mapping

   stream(tuple(...)) [x] int [x] attr[_]1 ... attr[_]n [->] tuple(...)


*/

ListExpr ksmallestTM(ListExpr args){

  string err = "stream(tuple(...)) x int x attr_1 x ... x attr_n   expected";
  if(nl->ListLength(args) < 3 ){
     ErrorReporter::ReportError(err);
     return nl->TypeError();
  }
  ListExpr Stream = nl->First(args);
  args = nl->Rest(args);
  ListExpr Int = nl->First(args);
  ListExpr AttrList = nl->Rest(args);

  if(nl->ListLength(AttrList)!=1 ){
     ErrorReporter::ReportError(err);
     return nl->TypeError();
  }
  AttrList = nl->First(AttrList);

  if(!IsStreamDescription(Stream)){
    ErrorReporter::ReportError(err);
    return nl->TypeError();
  }
  if(!nl->IsEqual(Int,CcInt::BasicType())){
    ErrorReporter::ReportError(err);
    return nl->TypeError();
  }

  ListExpr StreamList = nl->Second(nl->Second(Stream));

  int attrNo = 0;
  ListExpr NumberList=nl->TheEmptyList();
  ListExpr Last=nl->TheEmptyList();
  ListExpr attrType = nl->TheEmptyList();

  if(nl->AtomType(AttrList)!=NoAtom){
    return listutils::typeError(err);
  }


  while(!nl->IsEmpty(AttrList)){

     ListExpr Attr = nl->First(AttrList);
     if(nl->AtomType(Attr)!=SymbolType){
        ErrorReporter::ReportError(err);
        return nl->TypeError();
     }
     string attrName = nl->SymbolValue(Attr);
     int j = FindAttribute(StreamList,attrName, attrType);
     if(j==0){
        ErrorReporter::ReportError("Attribute name " + attrName + "not found");
        return nl->TypeError();
     }
     if(attrNo==0){
      NumberList = nl->OneElemList(nl->IntAtom(j-1));
      Last = NumberList;
     } else {
       Last = nl->Append(Last, nl->IntAtom(j-1));
     }
     attrNo++;
     AttrList = nl->Rest(AttrList);
  }

  return nl->ThreeElemList(
              nl->SymbolAtom(Symbol::APPEND()),
              nl->TwoElemList(nl->IntAtom(attrNo),
                              NumberList),
              Stream);
}


/*
2.10.2 LocalInfo


*/

#ifndef USE_PROGRESS
class KSmallestLocalInfo{
 public:

/*
~Constructor~

Constructs a localinfo tor given k and the attribute indexes by ~attrnumbers~.

*/
    KSmallestLocalInfo(int ak, vector<int>& attrnumbers,bool Smallest):
       elems(0),numbers(attrnumbers),pos(0),smallest(Smallest),
       firstRequest(true){
       if(ak<0){
          k = 0;
       } else {
          k = ak;
       }
    }


/*
~Destructor~

Destroy this instance and calls DeleteIfAllowed for all
non processed tuples.


*/
    ~KSmallestLocalInfo(){
        for(unsigned int i=0;i<elems.size();i++){
          if(elems[i]){
            elems[i]->DeleteIfAllowed();
            elems[i] = 0;
          }
        }
    }


/*
~nextTuple~

Returns the next tuple within the buffer, or 0 if no tuple is
available.

*/
   Tuple* nextTuple(Word& stream){
      if(firstRequest){
         Word elem;
         qp->Request(stream.addr,elem);
         Tuple* tuple;
         while(qp->Received(stream.addr)){
           tuple = static_cast<Tuple*>(elem.addr);
           insertTuple(tuple);
           qp->Request(stream.addr,elem);
           firstRequest = false;
         }
      }

      if(pos==0){
         // sort the elements
         if(elems.size()< k){
            initializeHeap();
         }
         for(unsigned int i=elems.size()-1; i>0; i--){
            Tuple* top = elems[0];
            Tuple* last = elems[i];
            elems[0] = last;
            elems[i] = top;
            sink(0,i);
         }
      }
      if(pos<elems.size()){
         Tuple* res = elems[pos];
         elems[pos] = 0;
         pos++;
         return res;
      } else {
        return 0;
      }
   }

 private:
    vector<Tuple*> elems;
    vector<int> numbers;
    unsigned int k;
    unsigned int pos;
    bool smallest;
    bool firstRequest;


/*
~insertTuple~

Inserts a tuple into the local buffer. If the buffer would be
overflow (size [>] k) , the maximum element is removed from the buffer.

*/
    void insertTuple(Tuple* tuple){
       if(elems.size() < k){
          elems.push_back(tuple);
          if(elems.size()==k){
            initializeHeap();
          }
       } else {
         Tuple* maxTuple = elems[0];

         int cmp = compareTuples(tuple,maxTuple);
         if(cmp>=0){ // tuple >= maxTuple
            tuple->DeleteIfAllowed();
         } else {
            maxTuple->DeleteIfAllowed();
            elems[0] = tuple;
            sink(0,elems.size());
         }
       }
    }

    inline int compareTuples(Tuple* t1, Tuple* t2){
      return smallest?compareTuplesSmaller(t1,t2): compareTuplesSmaller(t2,t1);
    }

    int compareTuplesSmaller(Tuple* t1, Tuple* t2){
       for(unsigned int i=0;i<numbers.size();i++){
          Attribute* a1 = t1->GetAttribute(numbers[i]);
          Attribute* a2 = t2->GetAttribute(numbers[i]);
          int cmp = a1->Compare(a2);
          if(cmp!=0){
            return cmp;
          }
       }
       return 0;
    }

    void initializeHeap(){
       int s = elems.size()/2;
       for(int i=s; i>=0; i--){
          sink(i,elems.size());
       }
    }

    void sink(unsigned int i, unsigned int max){
      unsigned int root = i;
      unsigned int son1 = ((i+1)*2) - 1;
      unsigned int son2 = ((i+1)*2+1) - 1;
      unsigned int swapWith;

      bool done = false;
      do{
        swapWith = root;
        son1 = ((root+1)*2) - 1;
        son2 = ((root+1)*2+1) - 1;
        if(son1<max){
          int cmp = compareTuples(elems[root],elems[son1]);
          if(cmp<0){
             swapWith = son1;
          }
        }
        if(son2 < max){
          int cmp = compareTuples(elems[swapWith],elems[son2]);
          if(cmp<0){
            swapWith = son2;
          }
        }
        if(swapWith!=root){
          Tuple* t1 = elems[root];
          Tuple* t2 = elems[swapWith];
          elems[swapWith] = t1;
          elems[root] = t2;
        }
        done = (swapWith == root);
        root = swapWith;

      }while(!done);
    }
};

template<bool smaller>
int ksmallestVM(Word* args, Word& result,
                           int message, Word& local, Supplier s)
{
  switch( message )
  {
    case OPEN:{
      qp->Open(args[0].addr);
      CcInt* cck = static_cast<CcInt*>(args[1].addr);
      int attrNum = (static_cast<CcInt*>(args[3].addr))->GetIntval();
      Supplier son;
      Word elem;
      vector<int> attrPos;
      for(int i=0;i<attrNum;i++){
         son = qp->GetSupplier(args[4].addr,i);
         qp->Request(son,elem);
         int anum = (static_cast<CcInt*>(elem.addr))->GetIntval();
         attrPos.push_back(anum);
      }
      int k = cck->IsDefined()?cck->GetIntval():0;
      KSmallestLocalInfo* linfo = new   KSmallestLocalInfo(k,attrPos,smaller);
      local.addr = linfo;
      return 0;
    }
    case REQUEST:{
       KSmallestLocalInfo* linfo;
       linfo = static_cast<KSmallestLocalInfo*>(local.addr);
       if(linfo){
         Tuple* tuple = linfo->nextTuple(args[0]);
         if(tuple){
           result.setAddr(tuple);
           return YIELD;
         }
       }
       return CANCEL;

    }
    case CLOSE:{
       qp->Close(args[0].addr);
       KSmallestLocalInfo* linfo;
       linfo = static_cast<KSmallestLocalInfo*>(local.addr);
       if(linfo){
          delete linfo;
          local.addr=0;
       }
       return 0;
    }
    default:{
       return  0;
    }
  }

}

#else

class KSmallestLocalInfo: public ProgressLocalInfo{
  public:

/*
~Constructor~

Constructs a localinfo tor given k and the attribute indexes by ~attrnumbers~.

*/

    KSmallestLocalInfo(int ak, vector<int>& attrnumbers,bool Smallest):
      elems(0),numbers(attrnumbers),pos(0),smallest(Smallest),
            firstRequest(true){
              if(ak<0){
                k = 0;
              } else {
                k = ak;
              }
            }


/*
            ~Destructor~

            Destroy this instance and calls DeleteIfAllowed for all
            non processed tuples.


*/
            ~KSmallestLocalInfo(){
              for(unsigned int i=0;i<elems.size();i++){
                if(elems[i]){
                  elems[i]->DeleteIfAllowed();
                  elems[i] = 0;
                }
              }
            }

            int getK()
            {
              return k;
            }

/*
            ~nextTuple~

            Returns the next tuple within the buffer, or 0 if no tuple is
            available.

*/
            Tuple* nextTuple(Word& stream){
              if(firstRequest){
                Word elem;
                qp->Request(stream.addr,elem);
                Tuple* tuple;
                while(qp->Received(stream.addr)){
                  tuple = static_cast<Tuple*>(elem.addr);
                  insertTuple(tuple);
                  qp->Request(stream.addr,elem);
                  firstRequest = false;
                }
              }

              if(pos==0){
         // sort the elements
                if(elems.size()< k){
                  initializeHeap();
                }
                if(elems.size()==0){
                   return 0;
                }
                for(unsigned int i=elems.size()-1; i>0; i--){
                  Tuple* top = elems[0];
                  Tuple* last = elems[i];
                  elems[0] = last;
                  elems[i] = top;
                  sink(0,i);
                }
              }
              if(pos<elems.size()){
                Tuple* res = elems[pos];
                elems[pos] = 0;
                pos++;
                return res;
              } else {
                return 0;
              }
            }

  private:
    vector<Tuple*> elems;
    vector<int> numbers;
    unsigned int k;
    unsigned int pos;
    bool smallest;
    bool firstRequest;


/*
~insertTuple~

Inserts a tuple into the local buffer. If the buffer would be
overflow (size [>] k) , the maximum element is removed from the buffer.

*/
    void insertTuple(Tuple* tuple){
      if(elems.size() < k){
        elems.push_back(tuple);
        if(elems.size()==k){
          initializeHeap();
        }
      } else {
        Tuple* maxTuple = elems[0];

        int cmp = compareTuples(tuple,maxTuple);
        if(cmp>=0){ // tuple >= maxTuple
          tuple->DeleteIfAllowed();
        } else {
          maxTuple->DeleteIfAllowed();
          elems[0] = tuple;
          sink(0,elems.size());
        }
      }
    }

    inline int compareTuples(Tuple* t1, Tuple* t2){
      return smallest?compareTuplesSmaller(t1,t2): compareTuplesSmaller(t2,t1);
    }

    int compareTuplesSmaller(Tuple* t1, Tuple* t2){
      for(unsigned int i=0;i<numbers.size();i++){
        Attribute* a1 = t1->GetAttribute(numbers[i]);
        Attribute* a2 = t2->GetAttribute(numbers[i]);
        int cmp = a1->Compare(a2);
        if(cmp!=0){
          return cmp;
        }
      }
      return 0;
    }

    void initializeHeap(){
      int s = elems.size()/2;
      for(int i=s; i>=0; i--){
        sink(i,elems.size());
      }
    }

    void sink(unsigned int i, unsigned int max){
      unsigned int root = i;
      unsigned int son1 = ((i+1)*2) - 1;
      unsigned int son2 = ((i+1)*2+1) - 1;
      unsigned int swapWith;

      bool done = false;
      do{
        swapWith = root;
        son1 = ((root+1)*2) - 1;
        son2 = ((root+1)*2+1) - 1;
        if(son1<max){
          int cmp = compareTuples(elems[root],elems[son1]);
          if(cmp<0){
            swapWith = son1;
          }
        }
        if(son2 < max){
          int cmp = compareTuples(elems[swapWith],elems[son2]);
          if(cmp<0){
            swapWith = son2;
          }
        }
        if(swapWith!=root){
          Tuple* t1 = elems[root];
          Tuple* t2 = elems[swapWith];
          elems[swapWith] = t1;
          elems[root] = t2;
        }
        done = (swapWith == root);
        root = swapWith;

      }while(!done);
    }
};

template<bool smaller>
    int ksmallestVM(Word* args, Word& result,
                    int message, Word& local, Supplier s)
{
  switch( message )
  {
    case OPEN:{
      qp->Open(args[0].addr);
      CcInt* cck = static_cast<CcInt*>(args[1].addr);
      int attrNum = (static_cast<CcInt*>(args[3].addr))->GetIntval();
      Supplier son;
      Word elem;
      vector<int> attrPos;
      for(int i=0;i<attrNum;i++){
        son = qp->GetSupplier(args[4].addr,i);
        qp->Request(son,elem);
        int anum = (static_cast<CcInt*>(elem.addr))->GetIntval();
        attrPos.push_back(anum);
      }
      int k = cck->IsDefined()?cck->GetIntval():0;
      KSmallestLocalInfo* linfo = new   KSmallestLocalInfo(k,attrPos,smaller);
      local.addr = linfo;
      return 0;
    }
    case REQUEST:{
      KSmallestLocalInfo* linfo;
      linfo = static_cast<KSmallestLocalInfo*>(local.addr);
      if(linfo){
        Tuple* tuple = linfo->nextTuple(args[0]);
        if(tuple){
          result.setAddr(tuple);
          return YIELD;
        }
      }
      return CANCEL;

    }
    case CLOSE:{
      qp->Close(args[0].addr);
#ifndef USE_PROGRESS
      KSmallestLocalInfo* linfo;
      linfo = static_cast<KSmallestLocalInfo*>(local.addr);
      if(linfo){
        delete linfo;
        local.addr=0;
      }
#endif
      return 0;
    }

    case CLOSEPROGRESS: {
      KSmallestLocalInfo* linfo;
      linfo = static_cast<KSmallestLocalInfo*>(local.addr);
      if(linfo){
        delete linfo;
        local.addr=0;
      }
      return 0;
    }
    case REQUESTPROGRESS: {

      ProgressInfo p1;
      ProgressInfo *pRes;
      KSmallestLocalInfo* linfo;

      const double uKsmallest = 0.0001287348; //millisecs per tuple
      const double vKsmallest = 0.0000021203; //millisecs per tuple
      const double wKsmallest = 0.0000000661; //millisecs per k

      pRes = (ProgressInfo*) result.addr;
      linfo = (KSmallestLocalInfo*) local.addr;
      if(!linfo){
         return CANCEL;
      }

      if (qp->RequestProgress(args[0].addr, &p1))
      {
        pRes->CopySizes(p1);
        int k = linfo->getK();
        if (k < p1.Card)
        {
          pRes->Card = k;
        }
        else
        {
          pRes->Card = p1.Card;
        }
        pRes->Time = p1.Time + p1.Card *
            (uKsmallest + p1.SizeExt * vKsmallest + k * wKsmallest);
        pRes->Progress = (p1.Progress*p1.Time + linfo->read *
            (uKsmallest + p1.SizeExt * vKsmallest + k * wKsmallest))/pRes->Time;
        pRes->BTime=pRes->Time;
        pRes->BProgress=pRes->Progress;

        return YIELD;
      } else {
         return CANCEL;
      }
    }

    default:{
      return  0;
    }
  }

}
#endif
const string ksmallestSpec  =
    "( ( \"Signature\" \"Syntax\" \"Meaning\" \"Example\" ) "
    "( <text>stream(tuple([a1:d1, ... ,an:dn])))"
    " x int x a_k x  ... a_m -> "
    "(stream (tuple(...)))</text--->"
    "<text>_ ksmallest [k ; list ]</text--->"
    "<text>returns the k smallest elements from the input stream"
    "</text--->"
    "<text>query employee feed ksmallest[10; Salary] consume "
    "</text--->"
    ") )";

const string kbiggestSpec  =
    "( ( \"Signature\" \"Syntax\" \"Meaning\" \"Example\" ) "
    "( <text>stream(tuple([a1:d1, ... ,an:dn])))"
    " x int x a_k x  ... a_m -> "
    "(stream (tuple(...)))</text--->"
    "<text>_ kbiggest [k ; list ]</text--->"
    "<text>returns the k biggest elements from the input stream"
    "</text--->"
    "<text>query employee feed kbiggest[10; Salary] consume "
    "</text--->"
    ") )";

Operator ksmallest (
         "ksmallest",
         ksmallestSpec,
         ksmallestVM<true>,
         Operator::SimpleSelect,
         ksmallestTM
);

Operator kbiggest (
         "kbiggest",
         kbiggestSpec,
         ksmallestVM<false>,
         Operator::SimpleSelect,
         ksmallestTM
);

/*
2.11 Operator ~sortBy~

This operator sorts a stream of tuples by a given list of attributes.
For each attribute it must be specified wether the list should be sorted
in ascending (asc) or descending (desc) order with regard to that attribute.

2.11.1 Type mapping function of operator ~sortBy~

Type mapping for ~sortBy~ is

----  ((stream (tuple ((x1 t1)...(xn tn))) ((xi1 asc/desc) ... (xij asc/desc)))
              -> (stream (tuple ((x1 t1)...(xn tn)))
                  APPEND (j i1 asc/desc i2 asc/desc ... ij asc/desc)
----

*/

static const char* sortAscending = "asc";
static const char* sortDescending = "desc";

ListExpr SortByTypeMap( ListExpr args )
{

  if(nl->ListLength(args)!=2){
    return listutils::typeError("two arguments expected");
  }

  ListExpr stream = nl->First(args);
  if(!listutils::isTupleStream(stream)){
    return listutils::typeError("first argument must be a tuple stream");
  }
  ListExpr attrtype;
  string  attrname, argstr;
  ListExpr sortSpecification  = nl->Second(args);
  int numberOfSortAttrs = nl->ListLength(sortSpecification);

  if(numberOfSortAttrs < 0){
    return listutils::typeError("Attribute list may not be enpty!");
  }

  ListExpr sortOrderDescription =
    nl->OneElemList(nl->IntAtom(numberOfSortAttrs));
  ListExpr sortOrderDescriptionLastElement = sortOrderDescription;

  ListExpr rest = sortSpecification;
  while(!nl->IsEmpty(rest))
  {
    ListExpr attributeSpecification = nl->First(rest);
    rest = nl->Rest(rest);

    nl->WriteToString(argstr, attributeSpecification);

    int length = nl->ListLength(attributeSpecification);
    string err = "second argument must be a list with"
                 " elements of form: (attrname [asc, desc])|attrname";
    if(!nl->IsAtom(attributeSpecification) &&  (length != 2)){
     return listutils::typeError(err);
    }
    bool isAscending=true;
    if(length==2) {
      if(nl->AtomType(nl->First(attributeSpecification))!=SymbolType){
         return listutils::typeError(err);
      }
      attrname = nl->SymbolValue(nl->First(attributeSpecification));
      ListExpr order = nl->Second(attributeSpecification);
      if(listutils::isSymbol(order, sortAscending)){
         isAscending = true;
      } else if(listutils::isSymbol(order, sortDescending)){
         isAscending = false;
      } else {
        return listutils::typeError("invalid sorting criterion");
      }
    } else  {
      if(nl->AtomType(attributeSpecification)!=SymbolType){
         return listutils::typeError(err);
      }
      attrname = nl->SymbolValue(attributeSpecification);
    }

    int j = FindAttribute(nl->Second(nl->Second(stream)),
                          attrname, attrtype);
    if (j > 0) {
      sortOrderDescriptionLastElement =
        nl->Append(sortOrderDescriptionLastElement, nl->IntAtom(j));
      sortOrderDescriptionLastElement =
        nl->Append(sortOrderDescriptionLastElement,
        nl->BoolAtom(isAscending));
    } else {
      return listutils::typeError("Unknown attribute name found");
    }
  }
  return nl->ThreeElemList(nl->SymbolAtom(Symbol::APPEND()),
    sortOrderDescription, stream);
}
/*
2.11.2 Value mapping function of operator ~sortBy~

The argument vector ~args~ contains in the first slot ~args[0]~ the
stream and in ~args[2]~ the number of sort attributes. ~args[3]~
contains the index of the first sort attribute, ~args[4]~ a boolean
indicating wether the stream is sorted in ascending order with regard
to the sort first attribute. ~args[5]~ and ~args[6]~ contain these
values for the second sort attribute  and so on.

*/
template<bool lexicographically> int
sortby_vm(Word* args, Word& result, int message, Word& local, Supplier s);
/*

2.11.3 Specification of operator ~sortBy~

*/
const string SortBySpec  = "( ( \"Signature\" \"Syntax\" \"Meaning\" "
                           "\"Example\" ) "
                           "( <text>((stream (tuple([a1:d1, ... ,an:dn])))"
                           " ((xi1 asc/desc) ... (xij [asc/desc]))) -> "
                           "(stream (tuple([a1:d1, ... ,an:dn])))</text--->"
                           "<text>_ sortby [list]</text--->"
                           "<text>Sorts input stream according to a list "
                           "of attributes ai1 ... aij. \n"
                           "If no order is specified, ascending is assumed."
                           "</text--->"
                           "<text>query employee feed sortby[DeptNo asc] "
                           "consume</text--->"
                              ") )";

/*
2.11.4 Definition of operator ~sortBy~

*/
Operator extrelsortby (
         "sortby_old",               // name
         SortBySpec,             // specification
         sortby_vm<false>,       // value mapping
         Operator::SimpleSelect, // trivial selection function
         SortByTypeMap           // type mapping
);

/*
2.12 Operator ~sort~

This operator sorts a stream of tuples lexicographically.

2.12.1 Type mapping function of operator ~sort~

Type mapping for ~sort~ is

----  ((stream (tuple ((x1 t1)...(xn tn))))
        -> (stream (tuple ((x1 t1)...(xn tn)))
----

*/
template<bool isSort> ListExpr
IdenticalTypeMap( ListExpr args )
{
  if(nl->ListLength(args)!=1){
    return listutils::typeError("one argument expected");
  }
  ListExpr arg = nl->First(args);
  if(!listutils::isTupleStream(arg)){
    return listutils::typeError("tuple stream expected");
  }
  return arg;

}
/*
2.12.2 Specification of operator ~sort\_old~

*/
const string SortSpec  = "( ( \"Signature\" \"Syntax\" \"Meaning\" "
                         "\"Example\" ) "
                         "( <text>((stream (tuple([a1:d1, ... ,an:dn]"
                         ")))) -> (stream (tuple([a1:d1, ... ,an:dn])))"
                         "</text--->"
                         "<text>_ sort_old</text--->"
                         "<text>Sorts input stream lexicographically."
                         "</text--->"
                         "<text>query cities feed sort_old consume</text--->"
                         ") )";

/*
2.12.3 Definition of operator ~sort\_old~

*/
Operator extrelsort (
         "sort_old",             // name
         SortSpec,           // specification
         sortby_vm<true>,               // value mapping
         Operator::SimpleSelect,          // trivial selection function
         IdenticalTypeMap<true>         // type mapping
);

/*
2.13 Operator ~rdup~

This operator removes duplicates from a sorted stream.

2.13.1 Value mapping function of operator ~rdup~

*/

#ifndef USE_PROGRESS

// standard version

int RdupValueMapping(Word* args, Word& result, int message,
                     Word& local, Supplier s)
{
  Word tuple(Address(0));
  LexicographicalTupleCmpAlmost cmp;
  Tuple* current = 0;
  RTuple* lastOutput = 0;

  switch(message)
  {
    case OPEN: {
      qp->Open(args[0].addr);
      local.addr = 0;
      return 0;
    }
    case REQUEST: {
      while(true)
      {
        qp->Request(args[0].addr, tuple);
        if(qp->Received(args[0].addr))
        {
          // stream delivered a new tuple
          if(local.addr != 0)
          {
            // there is a last tuple
            current = static_cast<Tuple*>(tuple.addr);
            lastOutput = static_cast<RTuple*>(local.addr);
            if(cmp(current, lastOutput->tuple)
              || cmp(lastOutput->tuple, current))
            {
              // tuples are not equal. Return the tuple
              // stored in local info and store the current one
              // there.

        *lastOutput = RTuple( current );
              result = tuple;
              return YIELD;
            }
            else
            {
              // tuples are equal
              current->DeleteIfAllowed();
            }
          }
          else
          {
            // no last tuple stored
      local.addr = new RTuple( static_cast<Tuple*>(tuple.addr) );
      result = tuple;
            return YIELD;
          }
        }
  else
        {
          result.addr = 0;
          return CANCEL;
        }
      }
    }
    case CLOSE: {
      if( local.addr != 0 ){ // check if local is present
         lastOutput = static_cast<RTuple*>(local.addr);
         delete lastOutput;
         local.setAddr(0);
      }
      qp->Close(args[0].addr);
      return 0;
    }
  }
  return 0;
}

# else

// progress version


struct RdupLocalInfo
{
  RdupLocalInfo(): localTuple(0),read(0),returned(0),stableValue(50),cmp1(){}
  ~RdupLocalInfo(){
    if(localTuple){
      localTuple->DeleteIfAllowed();
      localTuple=0;
    }
  }
  Tuple* localTuple;
  int read, returned, stableValue;
  LexicographicalTupleCmpAlmost cmp1;
};


int RdupValueMapping(Word* args, Word& result, int message,
                     Word& local, Supplier s)
{

  RdupLocalInfo* rli = (RdupLocalInfo*) local.addr;

  switch(message)
  {
    case OPEN: {

      if ( rli ) {
         delete rli;
      }
      rli = new RdupLocalInfo();
      local.setAddr(rli);
      qp->Open(args[0].addr);
      return 0;
    } case REQUEST: {
      if(!rli){
         return CANCEL;
      }
      Tuple* currentTuple;
      Tuple* lastOutputTuple;
      Word tuple;

      while(true)
      {
        qp->Request(args[0].addr, tuple);
        if(qp->Received(args[0].addr))
        {
          // stream deliverd a new tuple

          rli->read++;
          if(rli->localTuple != 0)
          {
            // there is a last tuple

            currentTuple = (Tuple*)tuple.addr;
            lastOutputTuple = rli->localTuple;
            if(rli->cmp1(currentTuple, lastOutputTuple)!=0)
            {
              // tuples are not equal. Return the tuple
              // stored in local info and store the current one
              // there.
              lastOutputTuple->DeleteIfAllowed();

              rli->returned++;
              rli->localTuple = currentTuple;

              currentTuple->IncReference();
              result.setAddr(currentTuple);
              return YIELD;
            }
            else
            {
              // tuples are equal
              currentTuple->DeleteIfAllowed();
            }
          }
          else
          {
            // no last tuple stored
            currentTuple = (Tuple*)tuple.addr;
            rli->returned++;
            rli->localTuple = currentTuple;
            currentTuple->IncReference();
            result.setAddr(currentTuple);
            return YIELD;
          }
        }
        else
        {
          // last tuple of the stream
          lastOutputTuple = rli->localTuple;
          if(lastOutputTuple != 0)
          {
            lastOutputTuple->DeleteIfAllowed();
            rli->localTuple = 0;
          }
          return CANCEL;
        }
      }
    } case CLOSE: {
      qp->Close(args[0].addr);
      return 0;

    } case CLOSEPROGRESS: {
      if ( rli )
      {
        delete rli;
        local.setAddr(0);
      }
      return 0;


    } case REQUESTPROGRESS: {
      ProgressInfo p1;
      ProgressInfo* pRes;
      const double uRdup = 0.01;  // time per tuple
      const double vRdup = 0.1;   // time per comparison
      const double wRdup = 0.9;   // default selectivity (= 10% duplicates)

      pRes = (ProgressInfo*) result.addr;

      if ( qp->RequestProgress(args[0].addr, &p1) )
      {
        pRes->CopySizes(p1);
        pRes->Time = p1.Time + p1.Card * uRdup * vRdup;

        pRes->CopyBlocking(p1);    //non-blocking operator

        if (rli) {
          if (rli->returned > rli->stableValue) {
            pRes->Card =  p1.Card *
              ((double) rli->returned / (double) (rli->read));

            if ( p1.BTime < 0.1 && pipelinedProgress ){  //non-blocking,
                                                        //use pipelining
              pRes->Progress = p1.Progress;
            } else {
              pRes->Progress = (p1.Progress * p1.Time
                + rli->read * uRdup * vRdup) / pRes->Time;
            }

            return YIELD;
          }
        }

        pRes->Card = p1.Card * wRdup;

        if ( p1.BTime < 0.1 && pipelinedProgress ){      //non-blocking,
                                                        //use pipelining
          pRes->Progress = p1.Progress;
        } else {
          pRes->Progress = (p1.Progress * p1.Time) / pRes->Time;
        }

        return YIELD;
      } else {
        return CANCEL;
      }
    }
  }
  return 0;
}


#endif



/*
2.13.2 Specification of operator ~rdup~

*/
const string RdupSpec  = "( ( \"Signature\" \"Syntax\" \"Meaning\" "
                         "\"Example\" ) "
                         "( <text>((stream (tuple([a1:d1, ... ,an:dn]))))"
                         " -> (stream (tuple([a1:d1, ... ,an:dn])))"
                         "</text--->"
                         "<text>_ rdup</text--->"
                         "<text>Removes duplicates from a sorted "
                         "stream.</text--->"
                         "<text>query twenty feed ten feed concat sort "
                         "rdup consume</text--->"
                              ") )";

/*
2.13.3 Definition of operator ~rdup~

*/
Operator extrelrdup (
         "rdup",             // name
         RdupSpec,           // specification
         RdupValueMapping,               // value mapping
         Operator::SimpleSelect,          // trivial selection function
         IdenticalTypeMap<false>         // type mapping
);

/*
2.14 Set Operators

These operators compute set operations on two sorted stream.

2.14.1 Generic Type Mapping for Set Operations

*/
template<int errorMessageIdx> ListExpr
SetOpTypeMap( ListExpr args )
{
  string err = " stream(tuple(X) x stream(tuple(X)) expected";
  if(nl->ListLength(args)!=2){
    return listutils::typeError(err);
  }
  if(!listutils::isTupleStream(nl->First(args)) ||
     !nl->Equal(nl->First(args), nl->Second(args))){
    return listutils::typeError(err);
  }
  return nl->First(args);
}
/*
2.14.2 Auxiliary Class for Set Operations

*/

class SetOperation
{
public:
  bool outputAWithoutB;
  bool outputBWithoutA;
  bool outputMatches;

private:
  LexicographicalTupleCmp tupleCmp;

  Word streamA;
  Word streamB;

  Tuple* currentATuple;
  Tuple* currentBTuple;

/*
~NextATuple~

Stores the next tuple from stream A into currentATuple which is
unequal to the currently store A tuple.

*/
  void NextATuple()
  {
    Word tuple;
    qp->Request(streamA.addr,tuple);
    if(!currentATuple){ // first tuple
       if(qp->Received(streamA.addr)){
          currentATuple = static_cast<Tuple*>(tuple.addr);
       }
    } else { // currentAtuple already exists
       while(qp->Received(streamA.addr) &&
             TuplesEqual(currentATuple, static_cast<Tuple*>(tuple.addr))){
          (static_cast<Tuple*>(tuple.addr))->DeleteIfAllowed();
          qp->Request(streamA.addr,tuple);
       }
       currentATuple->DeleteIfAllowed(); // remove from inputstream or buffer
       if(qp->Received(streamA.addr)){
         currentATuple = static_cast<Tuple*>(tuple.addr);
       } else {
         currentATuple=0;
       }
    }
  }

  void NextBTuple()
  {
    Word tuple;
    if(!currentBTuple){ // first tuple
       qp->Request(streamB.addr,tuple);
       if(qp->Received(streamB.addr)){
          currentBTuple = static_cast<Tuple*>( tuple.addr);
       }
    } else { // currentAtuple already exists
       qp->Request(streamB.addr,tuple);
       while(qp->Received(streamB.addr) &&
             TuplesEqual(currentBTuple, static_cast<Tuple*>(tuple.addr))){
          (static_cast<Tuple*>(tuple.addr))->DeleteIfAllowed();
          qp->Request(streamB.addr,tuple);
       }
       currentBTuple->DeleteIfAllowed();
       if(qp->Received(streamB.addr)){
         currentBTuple = static_cast<Tuple*>(tuple.addr);
       } else {
         currentBTuple=0;
       }
    }
  }


  bool TuplesEqual(Tuple* a, Tuple* b)
  {
    return (tupleCmp(a,b)==0);
  }

public:

  SetOperation(Word streamA, Word streamB)
  {
    this->streamA = streamA;
    this->streamB = streamB;

    currentATuple = 0;
    currentBTuple = 0;

    qp->Open(streamA.addr);
    qp->Open(streamB.addr);
    NextATuple();
    NextBTuple();
  }

  virtual ~SetOperation()
  {
    qp->Close(streamA.addr);
    qp->Close(streamB.addr);
    if(currentATuple){
      currentATuple->DeleteIfAllowed();
      currentATuple=0;
    }
    if(currentBTuple){
      currentBTuple->DeleteIfAllowed();
      currentBTuple=0;
    }
  }

  Tuple* NextResultTuple()
  {
    Tuple* result = 0;
    while(result == 0)
    {
      if(currentATuple == 0)
      {
        if(currentBTuple == 0)
        { // stream exhausted
          return 0;
        }
        else
        {
          if(outputBWithoutA)
          {
            result = currentBTuple;
            result->IncReference();
            NextBTuple();
          }
          else
          {
            currentBTuple->DeleteIfAllowed();
            currentBTuple = 0;
            return 0;
          }
        }
      }
      else
      {
        if(currentBTuple == 0)
        {
          if(outputAWithoutB)
          {
            result = currentATuple;
            result->IncReference();
            NextATuple();
          }
          else
          {
            currentATuple->DeleteIfAllowed();
            currentATuple=0;
            return 0;
          }
        }
        else
        {
          /* both current tuples != 0 */
          int cmp = tupleCmp(currentATuple, currentBTuple);
          if(cmp < 0)
          {
            if(outputAWithoutB)
            {
              result = currentATuple;
              result->IncReference();
            }
            NextATuple();
          }
          else if(cmp > 0)
          {
            if(outputBWithoutA)
            {
              result = currentBTuple;
              result->IncReference();
            }
            NextBTuple();
          }
          else
          {
            /* found match */
            Tuple* match = currentATuple;
            if(outputMatches)
            {
              result = match;
              result->IncReference();
            }
            NextATuple();
            NextBTuple();
          }
        }
      }
    }
    return result;
  }
};

/*
2.14.3 Generic Value Mapping Function for Set Operations

*/

template<bool outputAWithoutB,
         bool outputBWithoutA,
         bool outputMatches>
int
SetOpValueMapping(Word* args, Word& result, int message,
                  Word& local, Supplier s)
{
  SetOperation* localInfo;

  switch(message)
  {
    case OPEN:
      localInfo = new SetOperation(args[0], args[1]);
      localInfo->outputBWithoutA = outputBWithoutA;
      localInfo->outputAWithoutB = outputAWithoutB;
      localInfo->outputMatches = outputMatches;

      local.setAddr(localInfo);
      return 0;
    case REQUEST:
      localInfo = (SetOperation*)local.addr;
      result.setAddr(localInfo->NextResultTuple());
      return result.addr != 0 ? YIELD : CANCEL;
    case CLOSE:
      if(local.addr){
         localInfo = (SetOperation*)local.addr;
         delete localInfo;
         local.setAddr(0);
      }
      return 0;
  }
  return 0;
}

/*
2.14.4 Specification of Operator ~mergesec~

*/
const string MergeSecSpec  = "( ( \"Signature\" \"Syntax\" \"Meaning\" "
                         "\"Example\" ) "
                         "( <text>((stream (tuple ((x1 t1) ... "
                         "(xn tn)))) stream (tuple ((x1 t1) ... (xn tn)"
                         ")))) -> (stream (tuple ((x1 t1) ... (xn tn))))"
                         "</text--->"
                         "<text>_ _ mergesec</text--->"
                         "<text>Computes the intersection of two sorted "
                         "streams.</text--->"
                         "<text>query twenty feed oddtwenty feed mergesec"
                         " consume</text--->"
                         ") )";

/*
2.14.5 Definition of Operator ~mergesec~

*/
Operator extrelmergesec(
         "mergesec",        // name
         MergeSecSpec,     // specification
         SetOpValueMapping<false, false, true>,         // value mapping
         Operator::SimpleSelect,          // trivial selection function
         SetOpTypeMap<0>   // type mapping
);

/*
2.14.6 Specification of Operator ~mergediff~

*/
const string MergeDiffSpec  = "( ( \"Signature\" \"Syntax\" \"Meaning\" "
                              "\"Example\" ) "
                             "( <text>((stream (tuple ((x1 t1) ... (xn tn)"
                             "))) stream (tuple ((x1 t1) ... (xn tn))))) ->"
                             " (stream (tuple ((x1 t1) ... (xn tn))))"
                             "</text--->"
                             "<text>_ _ mergediff</text--->"
                             "<text>Computes the difference of two sorted "
                             "streams.</text--->"
                             "<text>query twenty feed oddtwenty feed"
                             " mergediff consume</text--->"
                              ") )";

/*
2.14.7 Definition of Operator ~mergediff~

*/
Operator extrelmergediff(
         "mergediff",        // name
         MergeDiffSpec,     // specification
         SetOpValueMapping<true, false, false>,         // value mapping
         Operator::SimpleSelect,          // trivial selection function
         SetOpTypeMap<1>   // type mapping
);

/*
2.14.8 Specification of Operator ~mergeunion~

*/
const string
MergeUnionSpec = "( ( \"Signature\" \"Syntax\" \"Meaning\" "
  "\"Example\" ) "
  "( <text>((stream (tuple ((x1 t1) ... (xn tn))))"
  "stream (tuple ((x1 t1) ... (xn tn))))) -> (stream"
  " (tuple ((x1 t1) ... (xn tn))))</text--->"
  "<text>_ _ mergeunion</text--->"
  "<text>Computes the union of two sorted streams."
  "</text--->"
  "<text>query twenty feed oddtwenty feed "
  "mergeunion consume</text--->"
  ") )";

/*
2.14.9 Definition of Operator ~mergeunion~

*/
Operator extrelmergeunion(
         "mergeunion",        // name
         MergeUnionSpec,     // specification
         SetOpValueMapping<true, true, true>,         // value mapping
         Operator::SimpleSelect,          // trivial selection function
         SetOpTypeMap<2>   // type mapping
);

/*
2.15 Operator ~mergejoin~

This operator computes the equijoin two streams.

2.15.1 Type mapping function of operators ~mergejoin~ and ~hashjoin~

Type mapping for ~mergejoin~ is

----  ((stream (tuple ((x1 t1) ... (xn tn))))
       (stream (tuple ((y1 d1) ... (ym dm)))) xi yj)
         -> (stream (tuple ((x1 t1) ... (xn tn) (y1 d1) ... (ym dm))))
            APPEND (i j)
----

Type mapping for ~hashjoin~ is

----  ((stream (tuple ((x1 t1) ... (xn tn))))
       (stream (tuple ((y1 d1) ... (ym dm)))) xi yj int)
         -> (stream (tuple ((x1 t1) ... (xn tn) (y1 d1) ... (ym dm))))
            APPEND (i j)
----


*/
bool arelTypeEqual (NList& t1, NList& t2)
{
  NList firstA, firstB;
  NList attrA = t1.second().second();
  NList attrB = t2.second().second();
  if (attrA.length() != attrB.length())
    return false;
  else
  {
    bool equal = true;
    while (!attrA.isEmpty() && equal)
    {
      firstA = attrA.first().second();
      firstB = attrB.first().second();
       if (firstA.hasLength(2) && firstB.hasLength(2) &&
           firstA.first().isSymbol() && firstB.first().isSymbol() &&
           firstA.first().str() == "arel" && firstB.first().str() == "arel")
        equal = arelTypeEqual(firstA, firstB);
      else
        equal = nl->Equal(firstA.listExpr(), firstB.listExpr());
      attrA.rest();
      attrB.rest();
    }
    return equal;
  }
}

template<bool OptionalIntAllowed, int defaultValue>
ListExpr JoinTypeMap (ListExpr args)
{
  string err = "stream(tuple[y1 : d1, ..., yn : dn]) x "
               "stream(tuple[z1 : e1, ..., zn : en]) x di x e1 ";



  if(OptionalIntAllowed){
    err += " [ x int]";
  }

  int len = nl->ListLength(args);

  // check for correct number of arguments
  if( (len!=4) && (len!=5)){
     return listutils::typeError(err);
  }
  if((len==5) && !OptionalIntAllowed){
     return listutils::typeError(err);
  }

  // check the first 4 arguments stream x stream x attrname x attrname
  ListExpr stream1 = nl->First(args);
  ListExpr stream2 = nl->Second(args);
  ListExpr attr1 = nl->Third(args);
  ListExpr attr2 = nl->Fourth(args);
  if(!listutils::isTupleStream(stream1) ||
     !listutils::isTupleStream(stream2) ||
     nl->AtomType(attr1)!=SymbolType ||
     nl->AtomType(attr2)!=SymbolType){
    return listutils::typeError(err);
  }

  // check the last element if present
  if(len==5){
     ListExpr size = nl->Fifth(args);
     if(!listutils::isSymbol(size,CcInt::BasicType())){
       return listutils::typeError(err + "(last arg is not an int");
     }
  }

  // check for correct naming of attributes
  ListExpr list1 = nl->Second(nl->Second(stream1));
  ListExpr list2 = nl->Second(nl->Second(stream2));
  if(!listutils::disjointAttrNames(list1,list2)){
    return listutils::typeError("Attribute lists are not disjoint");
  }


  ListExpr list = ConcatLists(list1, list2);
  ListExpr outlist = nl->TwoElemList(nl->SymbolAtom(Symbol::STREAM()),
      nl->TwoElemList(nl->SymbolAtom(Tuple::BasicType()), list));

  string attrAName = nl->SymbolValue(attr1);
  string attrBName = nl->SymbolValue(attr2);

  ListExpr attrTypeA, attrTypeB;

  int attrAIndex = listutils::findAttribute(list1, attrAName, attrTypeA);
  if(attrAIndex<1){
    return listutils::typeError("Attributename " + attrAName +
                                " not found in the first argument");
  }

  int attrBIndex = listutils::findAttribute(list2, attrBName, attrTypeB);
  if(attrBIndex<1){
    return listutils::typeError("Attributename " + attrBName +
                                " not found in the second argument");
  }

  NList attrA(attrTypeA);
  NList attrB(attrTypeB);
  if (attrA.hasLength(2) && attrB.hasLength(2) &&
      attrA.first().isSymbol() && attrB.first().isSymbol() &&
      attrA.first().str() == "arel" && attrB.first().str() == "arel")
  {
   if (!arelTypeEqual(attrA, attrB))
    return listutils::typeError("different types selected for operation");
  }
  else
  {
    if(!nl->Equal(attrTypeA, attrTypeB)){
      return listutils::typeError("different types selected for operation");
    }
  }

  ListExpr joinAttrDescription;

  if(!OptionalIntAllowed || len == 5){
      joinAttrDescription  = nl->TwoElemList(nl->IntAtom(attrAIndex),
                                             nl->IntAtom(attrBIndex));
  } else { // additionally add the default value
      joinAttrDescription = nl->ThreeElemList( nl->IntAtom(defaultValue),
                                               nl->IntAtom(attrAIndex),
                                               nl->IntAtom(attrBIndex));
  }
  return nl->ThreeElemList(nl->SymbolAtom(Symbol::APPEND()),
              joinAttrDescription, outlist);
}

/*
Some explicit instantiations in order to use them also
outside of the implementation file.

*/

template ListExpr
JoinTypeMap<false,1>(ListExpr args);


template ListExpr
JoinTypeMap<true,1>(ListExpr args);



/*
2.15.1 Create Cost Estimation 

*/
template<bool expectSorted> 
CostEstimation* MergeJoinCostEstimationFunc() {
   return new MergeJoinCostEstimation<expectSorted>();
}

/*
2.15.2 Value mapping functions of operator ~mergejoin~

*/

template<bool expectSorted> int
mergejoin_vm( Word* args, Word& result,
              int message, Word& local, Supplier s );

template<bool expectSorted> int
mergejoin_vm( Word* args, Word& result,
                  int message, Word& local, Supplier s );


/*

2.15.3 Specification of operator ~mergejoin~

*/
const string MergeJoinSpec  = "( ( \"Signature\" \"Syntax\" \"Meaning\" "
                              "\"Example\" ) "
                             "( <text>((stream (tuple ((x1 t1) ... "
                             "(xn tn)))) (stream (tuple ((y1 d1) ... "
                             "(ym dm)))) xi yj) -> (stream (tuple ((x1 t1)"
                             " ... (xn tn) (y1 d1) ... (ym dm))))"
                             "</text--->"
                             "<text>_ _ mergejoin [_, _]</text--->"
                             "<text>Computes the equijoin two streams. "
                             "Expects that input streams are sorted."
                             "</text--->"
                             "<text>query duplicates feed ten feed "
                             "rename[A] mergejoin[no, no_A] sort rdup "
                             "consume</text--->"
                              ") )";

/*
2.15.4 Definition of operator ~mergejoin~

*/
#ifndef USE_PROGRESS
Operator extrelmergejoin(
         "mergejoin",             // name
         MergeJoinSpec,           // specification
         mergejoin_vm<true>,      // value mapping
         Operator::SimpleSelect,  // trivial selection function
         JoinTypeMap<false,0>    // type mapping
);
#else
Operator extrelmergejoin(
         "mergejoin",             // name
         MergeJoinSpec,           // specification
         mergejoin_vm<true>,      // value mapping
         Operator::SimpleSelect,  // trivial selection function
         JoinTypeMap<false,0>,    // type mapping
         MergeJoinCostEstimationFunc<true> // cost estimation
);
#endif


/*
2.16 Operator ~sortmergejoin~

This operator sorts two input streams and computes their equijoin.

2.16.1 Specification of operator ~sortmergejoin~

*/
const string SortMergeJoinSpec  = "( ( \"Signature\" \"Syntax\" "
                                  "\"Meaning\" \"Example\" ) "
                             "( <text>((stream (tuple ((x1 t1) ... "
                             "(xn tn)))) (stream (tuple ((y1 d1) ..."
                             " (ym dm)))) xi yj) -> (stream (tuple "
                             "((x1 t1) ... (xn tn) (y1 d1) ... (ym dm)"
                             ")))</text--->"
                             "<text>_ _ sortmergejoin [ _ , _ ]"
                             "</text--->"
                             "<text>Computes the equijoin two streams."
                             "</text--->"
                             "<text>query duplicates feed ten feed "
                             "mergejoin[no, nr] consume</text--->"
                             ") )";

/*
2.16.2 Definition of operator ~sortmergejoin~

*/
#ifndef USE_PROGRESS
Operator extrelsortmergejoin(
         "sortmergejoin_old",            // name
         SortMergeJoinSpec,          // specification
         mergejoin_vm<false>,        // value mapping
         Operator::SimpleSelect,     // trivial selection function
         JoinTypeMap<false,0>       // type mapping
);

#else

Operator extrelsortmergejoin(
         "sortmergejoin_old",        // name
         SortMergeJoinSpec,          // specification
         mergejoin_vm<false>,        // value mapping
         Operator::SimpleSelect,     // trivial selection function
         JoinTypeMap<false,0>,       // type mapping
         MergeJoinCostEstimationFunc<false> // cost estimation
);
#endif

/*
2.17 Operator ~hashjoin~

This operator computes the equijoin two streams via a hash join.
The user can specify the number of hash buckets.

2.17.2 Specification of Operator ~hashjoin~

*/
const string HashJoinSpec  = "( ( \"Signature\" \"Syntax\" "
                             "\"Meaning\" \"Example\" \"Remark\" ) "
                          "( <text>((stream (tuple ((x1 t1) ... "
                          "(xn tn)))) (stream (tuple ((y1 d1) ... "
                          "(ym dm)))) xi yj [nbuckets]) -> (stream "
                          "(tuple ((x1 t1) ... (xn tn) (y1 d1) ..."
                          " (ym dm))))</text--->"
                          "<text> _ _ hashjoin [ _ , _ , _ ]"
                          "</text--->"
                          "<text>Computes the equijoin two streams "
                          "via a hash join. The number of hash buckets"
                          " is given by the parameter nBuckets. If the number"
                          " of buckets is omitted, a default value of 99997"
                          " is used"
                          "</text--->"
                          "<text>query Employee feed Dept feed "
                          "rename[A] hashjoin[DeptNr, DeptNr_A, 17] "
                          "sort consume</text--->"
                          "<text>The hash table is created from the 2nd "
                          "argument. If it does not fit into memory, "
                          "the 1st argument will also be materialized."
                          "</text--->"
                          ") )";

/*
2.17.1 Value Mapping Function of Operator ~hashjoin~

*/
int HashJoin(Word* args, Word& result, int message, Word& local, Supplier s);
/*

2.17.3 Definition of Operator ~hashjoin~

*/
Operator extrelhashjoin(
         "hashjoin",        // name
         HashJoinSpec,     // specification
         HashJoin,         // value mapping
         Operator::SimpleSelect,          // trivial selection function
         JoinTypeMap<true, 99997>   // type mapping
);


/*
2.18 Operator ~extend~

Extends each input tuple by new attributes as specified in the parameter list.

2.18.1 Type mapping function of operator ~extend~

Type mapping for ~extend~ is

----     ((stream X) ((b1 (map x y1)) ... (bm (map x ym))))

        -> (stream (tuple ((a1 x1) ... (an xn) (b1 y1) ... (bm ym)))))

        where X = (tuple ((a1 x1) ... (an xn)))
----

*/
ListExpr ExtendTypeMap( ListExpr args )
{
  ListExpr errorInfo = nl->OneElemList(nl->SymbolAtom("ERROR"));

  if(nl->ListLength(args)!=2){
    ErrorReporter::ReportError("two elements expected");
    return nl->TypeError();
  }

  ListExpr stream = nl->First(args);

  if(!IsStreamDescription(stream)){
    ErrorReporter::ReportError("first argument is not a tuple stream");
    return nl->TypeError();
  }

  ListExpr tuple = nl->Second(stream);

  ListExpr functions = nl->Second(args);
  if(nl->ListLength(functions)<1){
    ErrorReporter::ReportError("at least one function expected");
    return nl->TypeError();
  }

  // copy attrlist to newattrlist
  ListExpr attrList = nl->Second(nl->Second(stream));
  ListExpr newAttrList = nl->OneElemList(nl->First(attrList));
  ListExpr lastlistn = newAttrList;
  attrList = nl->Rest(attrList);
  while (!(nl->IsEmpty(attrList)))
  {
     lastlistn = nl->Append(lastlistn,nl->First(attrList));
     attrList = nl->Rest(attrList);
  }

  // reset attrList
  attrList = nl->Second(nl->Second(stream));
  ListExpr typeList;


  // check functions
  set<string> usedNames;

  while (!(nl->IsEmpty(functions)))
  {
    ListExpr function = nl->First(functions);
    functions = nl->Rest(functions);
    if(nl->ListLength(function)!=2){
      ErrorReporter::ReportError("invalid extension found");
      return nl->TypeError();
    }
    ListExpr name = nl->First(function);
    ListExpr map  = nl->Second(function);

    string errormsg;
    if(!listutils::isValidAttributeName(name, errormsg)){
      return listutils::typeError(errormsg);
    }

    string namestr = nl->SymbolValue(name);
    int pos = FindAttribute(attrList,namestr,typeList);
    if(pos!=0){
       ErrorReporter::ReportError("Attribute "+ namestr +
                                  " already member of the tuple");
       return nl->TypeError();
    }


    if(nl->ListLength(map)!=3){
      ErrorReporter::ReportError("invalid function");
      return nl->TypeError();
    }

    if(!nl->IsEqual(nl->First(map),Symbol::MAP())){
      ErrorReporter::ReportError("invalid function");
      return nl->TypeError();
    }

    ListExpr funResType = nl->Third(map);
    if(!am->CheckKind(Kind::DATA(),funResType, errorInfo)){
      ErrorReporter::ReportError("requested attribute " + namestr +
                                 "not in kind DATA");
      return nl->TypeError();
    }

    ListExpr funArgType = nl->Second(map);

    if(!nl->Equal(funArgType, tuple)){
      ErrorReporter::ReportError("function type different to the tuple type");
      return nl->TypeError();
    }

    if(usedNames.find(namestr)!=usedNames.end()){
      ErrorReporter::ReportError("Name "+ namestr + "occurs twice");
      return nl->TypeError();
    }
    usedNames.insert(namestr);
    // append attribute
    lastlistn = nl->Append(lastlistn, (nl->TwoElemList(name, funResType )));
  }

  return nl->TwoElemList(nl->SymbolAtom(Symbol::STREAM()),
            nl->TwoElemList(nl->SymbolAtom(Tuple::BasicType()),newAttrList));
}
/*
2.18.2 Value mapping function of operator ~extend~

*/

#ifndef USE_PROGRESS

// standard version

int Extend(Word* args, Word& result, int message, Word& local, Supplier s)
{
  Word t, value;
  Tuple* tup;
  Supplier supplier, supplier2, supplier3;
  int nooffun;
  ArgVectorPointer funargs;
  TupleType *resultTupleType = (TupleType*) qp->GetLocal2(s).addr;
   

  switch (message)
  {
    case INIT : {
       assert(!resultTupleType);
       ListExpr resultType = GetTupleResultType( s );
       resultTupleType = new TupleType( nl->Second( resultType ) );
       qp->GetLocal2(s).addr =  resultTupleType();
       return 0;
    }

    case FINISH : {
       resultTupleType->DeleteIfAllowed();
       qp->GetLocal2(s).addr = 0;
    }

    case OPEN :
      qp->Open(args[0].addr);
      return 0;

    case REQUEST :

      qp->Request(args[0].addr,t);
      if (qp->Received(args[0].addr))
      {
        tup = (Tuple*)t.addr;
        Tuple *newTuple = new Tuple( resultTupleType );
        for( int i = 0; i < tup->GetNoAttributes(); i++ ) {
          //cout << (void*) tup << endl;
          newTuple->CopyAttribute( i, tup, i );
        }
        supplier = args[1].addr;
        nooffun = qp->GetNoSons(supplier);
        for (int i=0; i < nooffun;i++)
        {
          supplier2 = qp->GetSupplier(supplier, i);
          supplier3 = qp->GetSupplier(supplier2, 1);
          funargs = qp->Argument(supplier3);
          ((*funargs)[0]).setAddr(tup);
          qp->Request(supplier3,value);
          newTuple->PutAttribute( tup->GetNoAttributes()+i,
                                  ((Attribute*)value.addr)->Clone() );
        }
        tup->DeleteIfAllowed();
        result.setAddr(newTuple);
        return YIELD;
      }
      else
        return CANCEL;

    case CLOSE :
      qp->Close(args[0].addr);
      return 0;
  }
  return 0;
}


# else

// progress version


class ExtendLocalInfo: public ProgressLocalInfo
{
public:

  ExtendLocalInfo():resultTupleType(0),stableValue(0),
                    sizesFinal(false), noOldAttrs(0),
                    noNewAttrs(0), attrSizeTmp(0),
                    attrSizeExtTmp(0) {};

  ~ExtendLocalInfo() {
    if(resultTupleType){
      resultTupleType->DeleteIfAllowed();
    }
    if(attrSizeTmp){
       delete [] attrSizeTmp;
    }
    if(attrSizeExtTmp){
       delete [] attrSizeExtTmp;
    }
  }

  TupleType *resultTupleType;
  int stableValue;
  bool sizesFinal;        //true after stableValue reached
  int noOldAttrs, noNewAttrs;
  double *attrSizeTmp;
  double *attrSizeExtTmp;
};


struct extendLI2{
    TupleType* tt;
    Word t;
    Word value;
};


int Extend(Word* args, Word& result, int message, Word& local, Supplier s)
{
  Tuple* tup;
  Supplier supplier, supplier2, supplier3;
  int nooffun;
  ArgVectorPointer funargs;
  ExtendLocalInfo *eli;

  eli = (ExtendLocalInfo*) local.addr;
  extendLI2* li2 = (extendLI2*) qp->GetLocal2(s).addr;


  switch (message)
  {
    case INIT : {
      assert(!li2); 
      li2 = new extendLI2;
      li2->tt =  new TupleType(nl->Second(
                                     GetTupleResultType(s))
                                  );
      qp->GetLocal2(s).addr = li2; 
      return 0;
  }

    case OPEN :

      if ( eli ) {
         delete eli;
      }

      eli = new ExtendLocalInfo;
      assert(li2->tt);
      li2->tt->IncReference();
      eli->resultTupleType = li2->tt;
      eli->read = 0;
      eli->stableValue = 50;
      eli->sizesFinal = false;
      eli->noNewAttrs = qp->GetNoSons(args[1].addr);
      eli->attrSizeTmp = new double[eli->noNewAttrs];
      eli->attrSizeExtTmp = new double[eli->noNewAttrs];
      for (int i = 0; i < eli->noNewAttrs; i++)
      {
        eli->attrSizeTmp[i] = 0.0;
        eli->attrSizeExtTmp[i] = 0.0;
      }

      local.setAddr(eli);

      qp->Open(args[0].addr);

      return 0;

    case REQUEST :
      if(!eli){
        return CANCEL;
      }
      qp->Request(args[0].addr,li2->t);
      if (qp->Received(args[0].addr))
      {
        tup = (Tuple*)li2->t.addr;
        Tuple *newTuple = new Tuple( eli->resultTupleType );
        eli->read++;
        for( int i = 0; i < tup->GetNoAttributes(); i++ ) {
          //cout << (void*) tup << endl;
          newTuple->CopyAttribute( i, tup, i );
        }
        supplier = args[1].addr;
        nooffun = qp->GetNoSons(supplier);
        for (int i=0; i < nooffun;i++)
        {
          supplier2 = qp->GetSupplier(supplier, i);
          supplier3 = qp->GetSupplier(supplier2, 1);
          funargs = qp->Argument(supplier3);
          ((*funargs)[0]).setAddr(tup);
          qp->Request(supplier3,li2->value);
          Attribute* newAttr = ((Attribute*)li2->value.addr)->Clone();
          newTuple->PutAttribute( tup->GetNoAttributes()+i,newAttr);

          if (eli->read <= eli->stableValue)
          {
            eli->attrSizeTmp[i] += newTuple->GetSize(tup->GetNoAttributes()+i);
            eli->attrSizeExtTmp[i] +=
              newTuple->GetExtSize(tup->GetNoAttributes()+i);
          }

        }
        tup->DeleteIfAllowed();
        result.setAddr(newTuple);
        return YIELD;
      }
      else
        return CANCEL;

    case CLOSE :
      qp->Close(args[0].addr);
      return 0;


    case FINISH:{
       if(li2){
         li2->tt->DeleteIfAllowed();
         delete li2;
         qp->GetLocal2(s).addr=0;
       } 
       return 0;
    }

    case CLOSEPROGRESS:
      if ( eli ){
         delete eli;
         local.setAddr(0);
      }
      return 0;


    case REQUESTPROGRESS:

      ProgressInfo p1;
      ProgressInfo *pRes;
      const double uExtend = 0.0034;   //millisecs per tuple
      const double vExtend = 0.0067;   //millisecs per tuple and attribute

      // see determination of constants in file ConstantsExtendStream

      pRes = (ProgressInfo*) result.addr;

      if ( !eli ) {
         return CANCEL;
      }

      if ( qp->RequestProgress(args[0].addr, &p1) )
      {
        eli->sizesChanged = false;

        if ( !eli->sizesInitialized )
        {
          eli->noOldAttrs = p1.noAttrs;
          eli->noAttrs = eli->noOldAttrs + eli->noNewAttrs;
          eli->attrSize = new double[eli->noAttrs];
          eli->attrSizeExt = new double[eli->noAttrs];
        }

        if ( !eli->sizesInitialized || p1.sizesChanged ||
             ( eli->read >= eli->stableValue && !eli->sizesFinal ) )
        {
          eli->Size = 0.0;
          eli->SizeExt = 0.0;

        //old attrs
          for (int i = 0; i < eli->noOldAttrs; i++)
          {
            eli->attrSize[i] = p1.attrSize[i];
            eli->attrSizeExt[i] = p1.attrSizeExt[i];
            eli->Size += eli->attrSize[i];
            eli->SizeExt += eli->attrSizeExt[i];
          }

            //new attrs
          if ( eli->read < eli->stableValue )
          {
            for (int j = 0; j < eli->noNewAttrs; j++)
            {
              eli->attrSize[j + eli->noOldAttrs] = 12;
              eli->attrSizeExt[j + eli->noOldAttrs] = 12;
              eli->Size += eli->attrSize[j + eli->noOldAttrs];
              eli->SizeExt += eli->attrSizeExt[j + eli->noOldAttrs];
            }
          }
          else
          {
            for (int j = 0; j < eli->noNewAttrs; j++)
            {
              eli->attrSize[j + eli->noOldAttrs] = eli->attrSizeTmp[j] /
                eli->stableValue;
              eli->attrSizeExt[j + eli->noOldAttrs] = eli->attrSizeExtTmp[j] /
                eli->stableValue;
              eli->Size += eli->attrSize[j + eli->noOldAttrs];
              eli->SizeExt += eli->attrSizeExt[j + eli->noOldAttrs];
            }
          }
          eli->sizesInitialized = true;
          eli->sizesChanged = true;
        }
        //ensure sizes are updated only once for passing the threshold
        if ( eli->read >= eli->stableValue ) eli->sizesFinal = true;


        pRes->Card = p1.Card;

        pRes->CopySizes(eli);

        pRes->Time = p1.Time + p1.Card * (uExtend + eli->noNewAttrs * vExtend);


        if ( p1.BTime < 0.1 && pipelinedProgress )      //non-blocking,
                                                        //use pipelining
          pRes->Progress = p1.Progress;
        else
          pRes->Progress =
            (p1.Progress * p1.Time +
              eli->read * (uExtend + eli->noNewAttrs * vExtend))
            / pRes->Time;

        pRes->CopyBlocking(p1);    //non-blocking operator

        return YIELD;
      } else {
        return CANCEL;
      }
  }
  return 0;
}

#endif


/*
2.18.3 Specification of operator ~extend~

*/
const string ExtendSpec  = "( ( \"Signature\" \"Syntax\" \"Meaning\" "
                           "\"Example\" ) "
                           "( <text>(stream(tuple(x)) x [(a1, (tuple(x)"
                           " -> d1)) ... (an, (tuple(x) -> dn))] -> "
                           "stream(tuple(x@[a1:d1, ... , an:dn])))"
                           "</text--->"
                           "<text>_ extend [funlist]</text--->"
                           "<text>Extends each input tuple by new "
                           "attributes as specified in the parameter"
                           " list.</text--->"
                           "<text>query ten feed extend [Mult5 : "
                           ".nr * 5, Mod2 : .nr mod 2] consume"
                           "</text--->"
                           ") )";

/*
2.18.4 Definition of operator ~extend~

*/
Operator extrelextend (
         "extend",              // name
         ExtendSpec,            // specification
         Extend,                // value mapping
         Operator::SimpleSelect,          // trivial selection function
         ExtendTypeMap          // type mapping
);

/*
2.19 Operator ~loopjoin~

This operator will fulfill a join of two relations. Tuples in the
cartesian product which satisfy certain conditions are passed on
to the output stream.

For instance,

----    query Staedte feed {s1} loopjoin[ fun(t1: TUPLE) plz feed
          filter [ attr(t1, SName_s1) = .Ort]] count;

        (query (count (loopjoin (rename (feed Staedte) s1)
          (fun (t1 TUPLE) (filter (feed plz)
          (fun (tuple1 TUPLE) (= (attr t1 SName_s1) (attr tuple1 Ort))))))))
----

The renaming is necessary whenever the underlying relations have at
least one common attribute name in order to assure that the output
tuple stream consists of different named attributes.

The type mapping function of the loopjoin operation is as follows:

----    ((stream (tuple x)) (map (tuple x) (stream (tuple y))))
          -> (stream (tuple x * y))
  where x = ((x1 t1) ... (xn tn)) and y = ((y1 d1) ... (ym dm))
----

*/
ListExpr LoopjoinTypeMap(ListExpr args)
{

 if(nl->ListLength(args)!=2){
   return listutils::typeError("two arguments expected");
 }
 ListExpr stream = nl->First(args);
 ListExpr map = nl->Second(args);
 string err = "stream(tuple (a)) x ( tuple(a) -> stream(tuple(b))) expected";
 if(!listutils::isTupleStream(stream) ||
    !listutils::isMap<1>(map)){
   return listutils::typeError(err);
 }

 ListExpr maparg = nl->Second(map);
 ListExpr mapres = nl->Third(map);

 if(!nl->Equal(nl->Second(stream), maparg)){
   return listutils::typeError( err +
                               " (function argument differs from tuple)");
 }

 if(!listutils::isTupleStream(mapres)){
   return listutils::typeError(err +
                               " (function result is not a tuple stream");
 }

 ListExpr alist1 = nl->Second(nl->Second(stream));
 ListExpr alist2 = nl->Second(nl->Second(mapres));

 if(!listutils::disjointAttrNames(alist1, alist2)){
   return listutils::typeError(err +
                               " ( name conflict in  tuples");
 }
 ListExpr list = ConcatLists(alist1,alist2);

 return nl->TwoElemList(nl->SymbolAtom(Symbol::STREAM()),
                        nl->TwoElemList(nl->SymbolAtom(Tuple::BasicType()),
                                        list));
}

/*
2.19.2 Value mapping function of operator ~loopjoin~

SPM: There is a problem when this operator is requested after
it has returned CANCEL it will chrash since it tryes to do a
request on a NULL pointer.

*/

#ifndef USE_PROGRESS

// standard version

struct LoopjoinLocalInfo
{
  Word tuplex;
  Word streamy;
  TupleType *resultTupleType;
};



int Loopjoin(Word* args, Word& result, int message,
             Word& local, Supplier s)
{
  ArgVectorPointer funargs = 0;

  Word tuplex(Address(0));
  Word tupley(Address(0));
  Word tuplexy(Address(0));
  Word streamy(Address(0));

  Tuple* ctuplex = 0;
  Tuple* ctupley = 0;
  Tuple* ctuplexy = 0;

  LoopjoinLocalInfo *localinfo = 0;

  TupleType* tt = (TupleType*) qp->GetLocal2(s).addr;

  switch ( message )
  {
    case INIT: {
       assert(!tt);
       ListExpr resultType = GetTupleResultType( s );
       qp->GetLocal2(s).addr = new TupleType(nl->Second(resultType));
       return 0;
    }

    case FINISH: {
        tt->DeleteIfAllowed();
        qp->GetLocal2(s).addr=0;
        return 0;
    }


    case OPEN:
      qp->Open (args[0].addr);
      qp->Request(args[0].addr, tuplex);
      if (qp->Received(args[0].addr))
      {
        funargs = qp->Argument(args[1].addr);
        (*funargs)[0] = tuplex;
        streamy=args[1];
        qp->Open (streamy.addr);

        localinfo = new LoopjoinLocalInfo;
        tt->IncReference();
        localinfo->resultTupleType = tt;
        localinfo->tuplex = tuplex;
        localinfo->streamy = streamy;
        local.setAddr(localinfo);
      }
      else
      {
        local.setAddr(0);
      }
      return 0;

    case REQUEST:
      if (local.addr ==0) return CANCEL;
      localinfo=(LoopjoinLocalInfo *) local.addr;
      tuplex=localinfo->tuplex;
      ctuplex=(Tuple*)tuplex.addr;
      streamy=localinfo->streamy;
      tupley.setAddr(0);
      while (tupley.addr==0)
      {
        qp->Request(streamy.addr, tupley);
        if (!(qp->Received(streamy.addr)))
        {
          qp->Close(streamy.addr);
          ((Tuple*)tuplex.addr)->DeleteIfAllowed();
          qp->Request(args[0].addr, tuplex);
          if (qp->Received(args[0].addr))
          {
            funargs = qp->Argument(args[1].addr);
            ctuplex=(Tuple*)tuplex.addr;
            (*funargs)[0] = tuplex;
            streamy=args[1];
            qp->Open (streamy.addr);
            tupley.setAddr(0);

            localinfo->tuplex=tuplex;
            localinfo->streamy=streamy;
            local.setAddr(localinfo);
          }
          else
          {
            localinfo->streamy.setAddr(0);
            localinfo->tuplex.setAddr(0);
            return CANCEL;
          }
        }
        else
        {
          ctupley=(Tuple*)tupley.addr;
        }
      }
      ctuplexy = new Tuple( localinfo->resultTupleType );
      tuplexy.setAddr(ctuplexy);
      Concat(ctuplex, ctupley, ctuplexy);
      ctupley->DeleteIfAllowed();
      result = tuplexy;
      return YIELD;

    case CLOSE:
      if( local.addr != 0 )
      {
        localinfo=(LoopjoinLocalInfo *) local.addr;
        if( localinfo->streamy.addr != 0 )
          qp->Close( localinfo->streamy.addr );

        if( localinfo->tuplex.addr != 0 )
          ((Tuple*)localinfo->tuplex.addr)->DeleteIfAllowed();

        if( localinfo->resultTupleType != 0 )
          localinfo->resultTupleType->DeleteIfAllowed();
        delete localinfo;
        local.setAddr(0);
      }
      qp->Close(args[0].addr);
      return 0;
  }

  return 0;
}


#else

// with support for progress queries



class LoopjoinLocalInfo: public ProgressLocalInfo
{
public:
  LoopjoinLocalInfo():tuplex(Address(0)),streamy(Address(0)),
                      resultTupleType(0), costEstimation(0)
                      {}

  ~LoopjoinLocalInfo() {
     if(resultTupleType){
       resultTupleType->DeleteIfAllowed();
       resultTupleType=0;
     }
  }

  Word tuplex;
  Word streamy;
  TupleType *resultTupleType;
  LoopJoinCostEstimation<true> *costEstimation;
};

CostEstimation* LoopJoinCostEstimationFunc() {
  return new LoopJoinCostEstimation<true>();
}



struct loopJoinLI2{
  TupleType* tt;
  Word tuplex;
  Word tupley;
  Word tuplexy;
  Word streamy;
};


int Loopjoin(Word* args, Word& result, int message,
             Word& local, Supplier s)
{
  ArgVectorPointer funargs = 0;
  Tuple* ctuplex = 0;
  Tuple* ctupley = 0;
  Tuple* ctuplexy = 0;


  LoopjoinLocalInfo* lli;

  lli = (LoopjoinLocalInfo *) local.addr;

  loopJoinLI2* li2 = (loopJoinLI2*) qp->GetLocal2(s).addr;


  switch ( message )
  {
    case INIT : {
       ListExpr resultType = GetTupleResultType(s);
       li2 = new loopJoinLI2();
       qp->GetLocal2(s).addr = li2;
       li2->tt = new TupleType(nl->Second(resultType));
       return 0; 
    }

    case FINISH: {
       if(li2){
           li2->tt->DeleteIfAllowed();
           delete li2;
       }
       qp->GetLocal2(s).addr = 0;
       return 0;
    }

    case OPEN:

      if ( lli ) {
         delete lli;
      }

      lli = new LoopjoinLocalInfo();
      li2->tt->IncReference();
      lli->resultTupleType = li2->tt;

      local.setAddr(lli);

      // cost estimation
      lli->costEstimation = 
         (LoopJoinCostEstimation<true>*) qp->getCostEstimation(s); 
      lli->costEstimation->init(NULL, NULL);

      qp->Open (args[0].addr);
      qp->Request(args[0].addr, li2->tuplex);
      if (qp->Received(args[0].addr))
      {
        lli->readFirst++;
        lli->costEstimation->processedTupleInStream1();

        funargs = qp->Argument(args[1].addr);
        (*funargs)[0] = li2->tuplex;
        li2->streamy=args[1];
        qp->Open (li2->streamy.addr);

        lli->tuplex = li2->tuplex;
        lli->streamy = li2->streamy;

      }
      else
      {
        lli->tuplex.setAddr(0);
        lli->streamy.setAddr(0);
      }
      return 0;

    case REQUEST:
      if(!lli){
        return CANCEL;
      }
      if ( lli->tuplex.addr == 0) {
         return CANCEL;
      }

      li2->tuplex=lli->tuplex;
      ctuplex=(Tuple*)li2->tuplex.addr;
      li2->streamy=lli->streamy;
      li2->tupley.setAddr(0);
      while (li2->tupley.addr==0)
      {
        qp->Request(li2->streamy.addr, li2->tupley);
        if (!(qp->Received(li2->streamy.addr)))
        {
          qp->Close(li2->streamy.addr);
          ((Tuple*)li2->tuplex.addr)->DeleteIfAllowed();
          qp->Request(args[0].addr, li2->tuplex);
          if (qp->Received(args[0].addr))
          {
            lli->readFirst++;
            lli->costEstimation->processedTupleInStream1();
            funargs = qp->Argument(args[1].addr);
            ctuplex=(Tuple*)li2->tuplex.addr;
            (*funargs)[0] = li2->tuplex;
            li2->streamy=args[1];
            qp->Open (li2->streamy.addr);
            li2->tupley.setAddr(0);

            lli->tuplex=li2->tuplex;
            lli->streamy=li2->streamy;
          }
          else
          {
            lli->costEstimation->setStream1Exhausted(true);
            lli->streamy.setAddr(0);
            lli->tuplex.setAddr(0);
            return CANCEL;
          }
        }
        else
        {
          ctupley=(Tuple*)li2->tupley.addr;
        }
      }
      ctuplexy = new Tuple( lli->resultTupleType );
      li2->tuplexy.setAddr(ctuplexy);
      Concat(ctuplex, ctupley, ctuplexy);
      ctupley->DeleteIfAllowed();

      lli->returned++;
      result = li2->tuplexy;
      return YIELD;

    case CLOSE:
      qp->Close(args[0].addr);

      if ( lli )
      {
         if ( lli->tuplex.addr != 0 )
         {
           if( lli->streamy.addr != 0 )
            qp->Close( lli->streamy.addr );

           if( lli->tuplex.addr != 0 )
             ((Tuple*)lli->tuplex.addr)->DeleteIfAllowed();

         }
         delete lli;
         local.setAddr(0);
      }
      return 0;
  }
  return 0;
}


#endif



/*
2.19.3 Specification of operator ~loopjoin~

*/
const string LoopjoinSpec  = "( ( \"Signature\" \"Syntax\" \"Meaning\" "
  "\"Example\" ) "
  "( <text>((stream tuple1) (fun (tuple1 -> "
  "stream(tuple2)))) -> (stream tuple1*tuple2)"
  "</text--->"
  "<text>_ loopjoin [ fun ]</text--->"
  "<text>Creates the product of all tuples from the first argument "
  "with all tuples from the stream created by the second (function) argument. "
  "Note: The input tuples must "
  "have different attribute names, hence renaming may be applied "
  "to one of the input streams.</text--->"
  "<text>query Staedte feed {s1} loopjoin[ fun(t1: TUPLE) plz feed"
  " filter [ attr(t1, SName_s1) = .Ort]] count</text--->"
  ") )";

/*
2.19.4 Definition of operator ~loopjoin~

*/
#ifndef USE_PROGRESS
Operator extrelloopjoin (
         "loopjoin",                // name
         LoopjoinSpec,              // specification
         Loopjoin,                  // value mapping
         Operator::SimpleSelect,    // trivial selection function
         LoopjoinTypeMap            // type mapping
);
#else
Operator extrelloopjoin (
         "loopjoin",                // name
         LoopjoinSpec,              // specification
         Loopjoin,                  // value mapping
         Operator::SimpleSelect,    // trivial selection function
         LoopjoinTypeMap,           // type mapping
         LoopJoinCostEstimationFunc // costestimation
);

#endif

/*

2.20 Operator ~loopselect~

This operator is similar to the ~loopjoin~ operator except that it
only returns the inner tuple (instead of the concatination of two
tuples). Tuples in the cartesian product which satisfy certain conditions
are passed on to the output stream.

For instance,

----    query Staedte feed loopsel [ fun(t1: TUPLE) plz feed
          filter [ attr(t1, SName) = .Ort ]] count;

        (query (count (loopsel (feed Staedte)
          (fun (t1 TUPLE) (filter (feed plz)
          (fun (tuple1 TUPLE) (= (attr t1 SName) (attr tuple1 Ort))))))))

----

7.4.1 Type mapping function of operator ~loopsel~

The type mapping function of the loopsel operation is as follows:

----    ((stream (tuple x)) (map (tuple x) (stream (tuple y))))
          -> (stream (tuple y))
        where x = ((x1 t1) ... (xn tn)) and y = ((y1 d1) ... (ym dm))
----

*/
ListExpr
LoopselectTypeMap(ListExpr args)
{
 if(nl->ListLength(args)!=2){
   return listutils::typeError("two arguments expected");
 }
 ListExpr stream = nl->First(args);
 ListExpr map = nl->Second(args);
 string err = "stream(tuple (a)) x ( tuple(a) -> stream(tuple(b))) expected";
 if(!listutils::isTupleStream(stream) ||
    !listutils::isMap<1>(map)){
   return listutils::typeError(err);
 }

 ListExpr maparg = nl->Second(map);
 ListExpr mapres = nl->Third(map);

 if(!nl->Equal(nl->Second(stream), maparg)){
   return listutils::typeError( err +
                               " (function argument differs from tuple)");
 }

 if(!listutils::isTupleStream(mapres)){
   return listutils::typeError(err +
                               " (function result is not a tuple stream");
 }

 return mapres;
}

/*

4.1.2 Value mapping function of operator ~loopsel~

*/

#ifndef USE_PROGRESS

struct LoopselectLocalInfo
{
  Word tuplex;
  Word streamy;
  TupleType *resultTupleType;
};

int
Loopselect(Word* args, Word& result, int message,
           Word& local, Supplier s)
{
  ArgVectorPointer funargs;
  Word tuplex, tupley, streamy;
  Tuple* ctuplex;
  Tuple* ctupley;
  LoopselectLocalInfo *localinfo;

  TupleType* tt = (TupleType*) qp->GetLocal2(s).addr;

  switch ( message )
  {
    case INIT : {
       ListExpr resultType = GetTupleResultType( s );
       tt = new TupleType (nl->Second(resultType));
       qp->GetLocal2(s).addr = tt;
       return 0;
    }

    case FINISH : {
        if(tt){
           tt->DeleteIfAllowed();
           qp->GetLocal2(s).addr = 0;
        }
        return 0;
    }

    case OPEN:
      // open the stream and initiate the variables
      qp->Open (args[0].addr);
      qp->Request(args[0].addr, tuplex);
      if (qp->Received(args[0].addr))
      {
        // compute the rely which corresponding to tuplex
        funargs = qp->Argument(args[1].addr);
        (*funargs)[0] = tuplex;
        streamy = args[1];
        qp->Open(streamy.addr);

        // put the information of tuplex and rely into local
        localinfo = new LoopselectLocalInfo;
        tt->IncReference();
        localinfo->resultTupleType = tt;
        localinfo->tuplex=tuplex;
        localinfo->streamy=streamy;
        local.setAddr(localinfo);
      }
      else
      {
        local.setAddr(0);
      }
      return 0;

    case REQUEST:
      if (local.addr ==0) return CANCEL;

      // restore localinformation from the local variable.
      localinfo = (LoopselectLocalInfo *) local.addr;
      tuplex = localinfo->tuplex;
      ctuplex = (Tuple*)tuplex.addr;
      streamy = localinfo->streamy;
      // prepare tuplex and tupley for processing.
      // if rely is exausted: fetch next tuplex.
      tupley.setAddr(0);
      while (tupley.addr==0)
      {
        qp->Request(streamy.addr, tupley);
        if (!(qp->Received(streamy.addr)))
        {
          qp->Close(streamy.addr);
          ((Tuple*)tuplex.addr)->DeleteIfAllowed();
          qp->Request(args[0].addr, tuplex);
          if (qp->Received(args[0].addr))
          {
            funargs = qp->Argument(args[1].addr);
            ctuplex = (Tuple*)tuplex.addr;
            (*funargs)[0] = tuplex;
            streamy = args[1];
            qp->Open(streamy.addr);
            tupley.setAddr(0);

            localinfo->tuplex=tuplex;
            localinfo->streamy=streamy;
            local.setAddr(localinfo);
          }
          else
          {
            localinfo->streamy.setAddr(0);
            localinfo->tuplex.setAddr(0);
            return CANCEL;
          }
        }
        else
        {
          ctupley = (Tuple*)tupley.addr;
        }
      }

      result = tupley;
      return YIELD;

    case CLOSE:
      if( local.addr != 0 )
      {
        localinfo=(LoopselectLocalInfo *) local.addr;

        if( localinfo->streamy.addr != 0 )
          qp->Close( localinfo->streamy.addr );

        if( localinfo->tuplex.addr != 0 )
          ((Tuple*)localinfo->tuplex.addr)->DeleteIfAllowed();

        if( localinfo->resultTupleType != 0 )
          localinfo->resultTupleType->DeleteIfAllowed();
        delete localinfo;
        local.setAddr(0);
      }
      qp->Close(args[0].addr);
      return 0;
  }

  return 0;
}
#else

struct LoopselectLocalInfo: public ProgressLocalInfo
{
  LoopselectLocalInfo(): ::ProgressLocalInfo(),tuplex(Address(0)),
                         streamy(Address(0)), resultTupleType(0),
                         costEstimation(0) {}

  ~LoopselectLocalInfo(){
     if(resultTupleType){
       resultTupleType->DeleteIfAllowed();
       resultTupleType = 0;
     }
  }

  Word tuplex;
  Word streamy;
  TupleType *resultTupleType;
  LoopJoinCostEstimation<false> *costEstimation;
};

CostEstimation* LoopSelCostEstimationFunc() {
  return new LoopJoinCostEstimation<false>();
}



int Loopselect(Word* args, Word& result, int message,
               Word& local, Supplier s)
{
  ArgVectorPointer funargs;
  Word tuplex, tupley, streamy;
  LoopselectLocalInfo *localinfo;

  TupleType* tt = (TupleType*) qp->GetLocal2(s).addr;

  switch ( message )
  {
     case INIT : {
        ListExpr resultType = GetTupleResultType( s );
        tt = new TupleType(nl->Second(resultType));
        qp->GetLocal2(s).addr = tt;
        return 0;
     }

     case FINISH : {
         if(tt){
            tt->DeleteIfAllowed();
            qp->GetLocal2(s).addr=0;
         }
         return 0;
     }


    case OPEN:
      // open the stream and initiate the variables
      qp->Open (args[0].addr);
      qp->Request(args[0].addr, tuplex);
      localinfo = static_cast<LoopselectLocalInfo*>(local.addr);
      if(localinfo){
        delete localinfo;
        localinfo=0;
      }
      if (qp->Received(args[0].addr))
      {
        // compute the rely which corresponding to tuplex
        funargs = qp->Argument(args[1].addr);
        (*funargs)[0] = tuplex;
        streamy = args[1];
        qp->Open(streamy.addr);

        // put the information of tuplex and rely into local
        localinfo = new LoopselectLocalInfo;
        tt->IncReference();
        localinfo->resultTupleType = tt;
        localinfo->tuplex=tuplex;
        localinfo->streamy=streamy;
        localinfo->readFirst=1;   // first tuple read
        localinfo->returned=0;
        local.setAddr(localinfo);
      
        // CostEstimation
        localinfo -> costEstimation = 
         (LoopJoinCostEstimation<false>*) qp->getCostEstimation(s);
        localinfo->costEstimation->init(NULL, NULL);
        localinfo->costEstimation->processedTupleInStream1();
      }
      else
      {
        local.setAddr(0);
      }
      return 0;

    case REQUEST:
      if (local.addr ==0){
          return CANCEL;
      }

      // restore localinformation from the local variable.
      localinfo = (LoopselectLocalInfo *) local.addr;
      tuplex = localinfo->tuplex;
    //  ctuplex = (Tuple*)tuplex.addr;
      streamy = localinfo->streamy;
      // prepare tuplex and tupley for processing.
      // if rely is exausted: fetch next tuplex.
      tupley.setAddr(0);
      while (tupley.addr==0)
      {
        qp->Request(streamy.addr, tupley);
        if (!(qp->Received(streamy.addr)))
        {
          qp->Close(streamy.addr);
          ((Tuple*)tuplex.addr)->DeleteIfAllowed();
          qp->Request(args[0].addr, tuplex);
          if (qp->Received(args[0].addr))
          {
            localinfo->readFirst++;         // another tuple read
            localinfo->costEstimation->processedTupleInStream1();
            funargs = qp->Argument(args[1].addr);
            //ctuplex = (Tuple*)tuplex.addr;
            (*funargs)[0] = tuplex;
            streamy = args[1];
            qp->Open(streamy.addr);
            tupley.setAddr(0);

            localinfo->tuplex=tuplex;
            localinfo->streamy=streamy;
            local.setAddr(localinfo);
          }
          else
          {
            localinfo->streamy.setAddr(0);
            localinfo->tuplex.setAddr(0);
            localinfo->costEstimation->setStream1Exhausted(true);
            return CANCEL;
          }
        }
        else
        {
          //ctupley = (Tuple*)tupley.addr;
        }
      }
      localinfo->returned++;
      result = tupley;
      return YIELD;

    case CLOSE:
      if( local.addr != 0 )
      {
        localinfo=(LoopselectLocalInfo *) local.addr;

        if( localinfo->streamy.addr != 0 )
          qp->Close( localinfo->streamy.addr );

        if( localinfo->tuplex.addr != 0 )
          ((Tuple*)localinfo->tuplex.addr)->DeleteIfAllowed();

        delete localinfo;
        local.setAddr(0);
      }
      qp->Close(args[0].addr);
      return 0;
   }

  return -1;
}
#endif
/*

4.1.3 Specification of operator ~loopsel~

*/
const string LoopselectSpec  = "( ( \"Signature\" \"Syntax\" \"Meaning\" "
  "\"Example\" ) "
  "( <text>((stream tuple1) (map tuple1 "
  "rel(tuple2))) -> (stream tuple2)"
  "</text--->"
  "<text>_ loopselect [ fun ]</text--->"
  "<text>Only tuples in the cartesian product "
  "which satisfy certain conditions are passed on"
  " to the output stream.</text--->"
  "<text>query Staedte feed loopsel [ fun(t1: TUPLE) plz feed "
  "filter [ attr(t1, SName) = .Ort ]] count</text--->"
  ") )";

/*

4.1.3 Definition of operator ~loopsel~

*/
#ifndef USE_PROGRESS
Operator extrelloopsel (
         "loopsel",                // name
         LoopselectSpec,           // specification
         Loopselect,               // value mapping
         Operator::SimpleSelect,   // trivial selection function
         LoopselectTypeMap         // type mapping
);
#else
Operator extrelloopsel (
         "loopsel",                // name
         LoopselectSpec,           // specification
         Loopselect,               // value mapping
         Operator::SimpleSelect,   // trivial selection function
         LoopselectTypeMap,        // type mapping
         LoopSelCostEstimationFunc // CostEstimation
);
#endif

/*
2.21 Operator ~extendstream~

This operator is implemented by the idea of LOOPJOIN and EXTEND

The type mapping function of the extendstream operation is as follows:

----    ((stream (tuple x)) (map (tuple x) (stream (y))))
          -> (stream (tuple x * y))
  where x = ((x1 t1) ... (xn tn)) and
        y is a simple data type such as string, integer, or real ...
----

----    ((stream (tuple x)) ( (b1 (map (tuple x) (stream (y)))))
          -> (stream (tuple x * y))
  where x = ((x1 t1) ... (xn tn)) and
        y is a simple data type such as string, integer, or real ...
----

For instance,

----  query people feed extendstream [parts : components (.name) ] consume;
----

*/



ListExpr ExtendStreamTypeMap(ListExpr args)
{
  NList Args(args);
  if(!Args.hasLength(2)){
     return NList::typeError("expecting two arguments");
  }
  NList Stream = Args.first();


  // check the tuple stream
  if(!Stream.hasLength(2) ||
     !Stream.first().isSymbol(Symbol::STREAM()) ||
     !Stream.second().hasLength(2) ||
     !Stream.second().first().isSymbol(Tuple::BasicType()) ||
     !IsTupleDescription(Stream.second().second().listExpr())
     ){
     return NList::typeError("first argument is not a tuple stream");
  }

  // second argument must be a list of length one
  NList Second = Args.second();
  if(!Second.isNoAtom() ||
     !Second.hasLength(1)){
     return NList::typeError("error in second argument");
  }

  // check for ( <attrname> <map> )
  NList  NameMap = Second.first();
  if(!NameMap.hasLength(2)){
    return NList::typeError("expecting (attrname map) as second argument");
  }

  // attrname must be an identifier
  if(!NameMap.first().isSymbol()){
    return NList::typeError("invalid representation of <attrname>");
  }

  // check map
  NList Map = NameMap.second();

  // Map must have format (map <tuple> <stream>)
  if(!Map.hasLength(3) ||
     !Map.first().isSymbol(Symbol::MAP()) ||
     !Map.second().hasLength(2) ||
     !Map.second().first().isSymbol(Tuple::BasicType()) ||
     !Map.third().hasLength(2) ||
     !Map.third().first().isSymbol(Symbol::STREAM())){
     return NList::typeError("expecting (map ( (tuple(...) (stream DATA))))"
                             " as second argument");
  }

  // the argumenttype for map must be equal to the tuple type from the stream
  if(Stream.second() != Map.second()){
     return NList::typeError("different tuple descriptions detected");
  }

  NList  typelist  = Map.third().second();

  ListExpr errorInfo = nl->OneElemList(nl->SymbolAtom("ERROR"));
  // check for Kind Data
  if(!am->CheckKind(Kind::DATA(),typelist.listExpr(),errorInfo)){
    return NList::typeError("stream elements not in kind DATA");
  }

  // check if argname is already used in the original tuple
  ListExpr attrNameList = NameMap.first().listExpr();
  string errormsg;
  if(!listutils::isValidAttributeName(attrNameList,errormsg)){
     return listutils::typeError(errormsg);
  }
  string attrname = NameMap.first().str();
  ListExpr attrtype=nl->TheEmptyList();
  int index = FindAttribute(Stream.second().second().listExpr(),
                            attrname,attrtype);
  if(index){ // attrname already used
    return NList::typeError("attrname already used in the original stream");
  }

  ListExpr attrlist = ConcatLists(Stream.second().second().listExpr(),
                                  nl->OneElemList(
                                    nl->TwoElemList(
                                      nl->SymbolAtom(attrname),
                                      Map.third().second().listExpr())));

  return  nl->TwoElemList(nl->SymbolAtom(Symbol::STREAM()),
                nl->TwoElemList( nl->SymbolAtom(Tuple::BasicType()),attrlist));
}

/*
2.19.2 Value mapping function of operator ~extendstream~

*/

#ifndef USE_PROGRESS

// standard version

struct ExtendStreamLocalInfo
{
  Tuple *tupleX;
  Word streamY;
  TupleType *resultTupleType;
};

int ExtendStream(Word* args, Word& result, int message,
                 Word& local, Supplier s)
{
  ArgVectorPointer funargs;
  Word wTupleX, wValueY;
  Tuple* tupleXY;
  ExtendStreamLocalInfo *localinfo;

  TupleType *resultTupleType;

  Supplier supplier, supplier2, supplier3;

  TupleType* tt = (TupleType*) qp->GetLocal2(s).addr;

  switch( message )
  {
    case INIT: {
        ListExpr resultType = GetTupleResultType( s );
        tt = new TupleType(nl->Second(resulType);
        qp->GetLocal2(s).addr = tt;
        return 0;
    }

    case FINISH: {
        if(tt){
           tt->DeleteIfAllowed();
           qp->GetLocal2(s).addr = 0;
        }
        return 0;
    }


    case OPEN:
    {
      //1. open the input stream and initiate the arguments
      qp->Open( args[0].addr );
      qp->Request( args[0].addr, wTupleX );
      if( qp->Received( args[0].addr ) )
      {
        //2. compute the result "stream y" from tuple x
        //funargs = qp->Argument(args[1].addr);
        //here should be changed to the following...
        supplier = args[1].addr;
        supplier2 = qp->GetSupplier( supplier, 0 );
        supplier3 = qp->GetSupplier( supplier2, 1 );
        funargs = qp->Argument( supplier3 );

        (*funargs)[0] = wTupleX;
        qp->Open( supplier3 );

        //3. save the local information
        localinfo = new ExtendStreamLocalInfo;
        tt->IncReference();
        localinfo->resultTupleType = tt;
        localinfo->tupleX = (Tuple*)wTupleX.addr;
        localinfo->streamY.setAddr( supplier3 );
        local.setAddr(localinfo);
      }
      else
      {
        local.setAddr(0);
      }
      return 0;
    }
    case REQUEST:
    {
      if( local.addr == 0 )
        return CANCEL;

      //1. recover local information
      localinfo=(ExtendStreamLocalInfo *) local.addr;
      resultTupleType = (TupleType *)localinfo->resultTupleType;

      //2. prepare tupleX and wValueY.
      //If wValueY is empty, then get next tupleX
      wValueY.setAddr(0);
      while( wValueY.addr == 0 )
      {
        qp->Request( localinfo->streamY.addr, wValueY );
        if( !(qp->Received( localinfo->streamY.addr )) )
        {
          qp->Close( localinfo->streamY.addr );
          localinfo->tupleX->DeleteIfAllowed();
          qp->Request( args[0].addr, wTupleX );
          if( qp->Received(args[0].addr) )
          {
            supplier = args[1].addr;
            supplier2 = qp->GetSupplier(supplier, 0);
            supplier3 = qp->GetSupplier(supplier2, 1);
            funargs = qp->Argument(supplier3);

            localinfo->tupleX = (Tuple*)wTupleX.addr;
            (*funargs)[0] = wTupleX;
            qp->Open( supplier3 );

            localinfo->tupleX = (Tuple*)wTupleX.addr;
            localinfo->streamY.setAddr(supplier3);

            wValueY.setAddr(0);
          }
          else  //streamx is exausted
          {
            localinfo->streamY.setAddr(0);
            localinfo->tupleX = 0;
            return CANCEL;
          }
        }
      }

      //3. compute tupleXY from tupleX and wValueY
      tupleXY = new Tuple( localinfo->resultTupleType );

      for( int i = 0; i < localinfo->tupleX->GetNoAttributes(); i++ )
        tupleXY->CopyAttribute( i, localinfo->tupleX, i );

      tupleXY->PutAttribute( localinfo->tupleX->GetNoAttributes(),
                             (Attribute*)wValueY.addr );

      // setting the result
      result.setAddr( tupleXY );
      return YIELD;
    }
    case CLOSE:
    {
      if( local.addr != 0 )
      {
        localinfo = (ExtendStreamLocalInfo *)local.addr;

        if( localinfo->streamY.addr != 0 )
          qp->Close( localinfo->streamY.addr );

        if( localinfo->tupleX != 0 )
          localinfo->tupleX->DeleteIfAllowed();

        if( localinfo->resultTupleType != 0 )
          localinfo->resultTupleType->DeleteIfAllowed();
        delete localinfo;
        local.setAddr(0);
      }
      qp->Close( args[0].addr );
      return 0;
    }
  }
  return 0;
}

# else

// progress version

struct ExtendStreamLocalInfo: public ProgressLocalInfo
{
public:

  ExtendStreamLocalInfo():
    tupleX(0), resultTupleType(0) {};

  ~ExtendStreamLocalInfo() {
    if( streamY.addr != 0 )
      qp->Close( streamY.addr );

    if( tupleX != 0 )
      tupleX->DeleteIfAllowed();

    if( resultTupleType != 0 )
      resultTupleType->DeleteIfAllowed();
  }

  Tuple *tupleX;
  Word streamY;
  TupleType *resultTupleType;

  double newAttrSize, newAttrSizeExt;
  int stableValue;
  bool sizesFinal;
};

int ExtendStream(Word* args, Word& result, int message,
                 Word& local, Supplier s)
{
  ArgVectorPointer funargs;
  Word wTupleX, wValueY;
  Tuple* tupleXY;

  ExtendStreamLocalInfo *eli;
  eli = (ExtendStreamLocalInfo*) local.addr;

  Supplier supplier, supplier2, supplier3;
  TupleType* tt = (TupleType*) qp->GetLocal2(s).addr;


  switch( message )
  {

    case INIT : {
      tt = new TupleType(nl->Second(GetTupleResultType(s)));
      qp->GetLocal2(s).addr = tt;
      return 0;  
    }

    case FINISH : {
       if(tt){
          tt->DeleteIfAllowed();
           qp->GetLocal2(s).addr=0;
       }
       return 0;
    }

    case OPEN:
    {
      if ( eli ) {
        delete eli;
      }

      eli = new ExtendStreamLocalInfo;
      tt->IncReference();
      eli->resultTupleType = tt;
      eli->newAttrSize = 0.0;
      eli->newAttrSizeExt = 0.0;
      eli->stableValue = 50;
      eli->sizesFinal = false;

      // 1. open the input stream and initiate the arguments
      qp->Open( args[0].addr );
      qp->Request( args[0].addr, wTupleX );
      if( qp->Received( args[0].addr ) )
      {
        eli->read = 1;

        // 2. compute the result "stream y" from tuple x
        // funargs = qp->Argument(args[1].addr);
        // here should be changed to the following...
        supplier = args[1].addr;
        supplier2 = qp->GetSupplier( supplier, 0 );
        supplier3 = qp->GetSupplier( supplier2, 1 );
        funargs = qp->Argument( supplier3 );

        (*funargs)[0] = wTupleX;
        qp->Open( supplier3 );

        // 3. save the local information
 
        eli->tupleX = (Tuple*)wTupleX.addr;
        eli->streamY.setAddr( supplier3 );
        local.setAddr(eli);
      }
      else
      {
        local.setAddr(0);
      }
      return 0;
    }
    case REQUEST:
    {
      if( local.addr == 0 )
        return CANCEL;

      // prepare tupleX and wValueY.
      // if wValueY is empty, then get next tupleX
      wValueY.setAddr(0);
      while( wValueY.addr == 0 )
      {
        qp->Request( eli->streamY.addr, wValueY );
        if( !(qp->Received( eli->streamY.addr )) )
        {
          qp->Close( eli->streamY.addr );
          eli->tupleX->DeleteIfAllowed();
          qp->Request( args[0].addr, wTupleX );
          if( qp->Received(args[0].addr) )
          {
            eli->read++;

            supplier = args[1].addr;
            supplier2 = qp->GetSupplier(supplier, 0);
            supplier3 = qp->GetSupplier(supplier2, 1);
            funargs = qp->Argument(supplier3);

            eli->tupleX = (Tuple*)wTupleX.addr;
            (*funargs)[0] = wTupleX;
            qp->Open( supplier3 );

            eli->tupleX = (Tuple*)wTupleX.addr;
            eli->streamY.setAddr(supplier3);

            wValueY.setAddr(0);
          }
          else  //streamx is exausted
          {
            eli->streamY.setAddr(0);
            eli->tupleX = 0;
            return CANCEL;
          }
        }
      }

      // 3. compute tupleXY from tupleX and wValueY
      tupleXY = new Tuple( eli->resultTupleType );

      for( int i = 0; i < eli->tupleX->GetNoAttributes(); i++ )
        tupleXY->CopyAttribute( i, eli->tupleX, i );

      tupleXY->PutAttribute( eli->tupleX->GetNoAttributes(),
                             (Attribute*)wValueY.addr );

      // for the first 50 tuples returned, observe attribute sizes of the 
      // new attribute.
      if ( eli->returned <= eli->stableValue )
      {
        eli->newAttrSize += 
          tupleXY->GetSize( eli->tupleX->GetNoAttributes() );
        eli->newAttrSizeExt += 
          tupleXY->GetExtSize( eli->tupleX->GetNoAttributes() );
      }

      // setting the result
      result.setAddr( tupleXY );
      eli->returned++;
      return YIELD;
    }
    case CLOSE:
    {
      qp->Close( args[0].addr );
      return 0;
    }

    case CLOSEPROGRESS:
    {
      if ( eli ) {
         delete eli;
         local.setAddr(0);
      }
      return 0;
    }


    case REQUESTPROGRESS:

      ProgressInfo p1;
      ProgressInfo *pRes;

      // for the determination of constants see file ConstantsExtendStream
      // and file bin/UpdateProgressConstants

      static const double wExtendStream = 
        ProgressConstants::getValue("ExtRelation-C++", "extendstream",
          "wExtendStream"); // 0.005963; millisecs per tuple read
      static const double vExtendStream = 
        ProgressConstants::getValue("ExtRelation-C++", "extendstream",
          "vExtendStream"); //  0.00014405; millisecs per attr returned
      static const double uExtendStream = 
        ProgressConstants::getValue("ExtRelation-C++", "extendstream",
          "uExtendStream"); // 0.0067; millisecs per tuple returned

      pRes = (ProgressInfo*) result.addr;

      if ( !eli ) {
        return CANCEL;
      }

      if ( qp->RequestProgress(args[0].addr, &p1) )
      {

    // 1. attribute sizes
        eli->sizesChanged = false;

        if ( !eli->sizesInitialized )
        {
          eli->noAttrs = p1.noAttrs + 1;
          eli->attrSize = new double[eli->noAttrs];
          eli->attrSizeExt = new double[eli->noAttrs];
        }

        if ( !eli->sizesInitialized || p1.sizesChanged ||
           ( eli->returned >= eli->stableValue && !eli->sizesFinal ) )
        {
      eli->Size = 0.0;
      eli->SizeExt = 0.0;

          for( int i = 0; i < eli->noAttrs - 1; i++)    //old attrs
          {
            eli->attrSize[i] = p1.attrSize[i];
            eli->attrSizeExt[i] = p1.attrSizeExt[i];
            eli->Size += eli->attrSize[i];
            eli->SizeExt += eli->attrSizeExt[i];
          }

          if ( eli->returned < eli->stableValue )       //new attrs
          {
            eli->attrSize[eli->noAttrs - 1] = 144;
            eli->attrSizeExt[eli->noAttrs - 1] = 144;
                        // size yet unknown. The default 144 is taken from
                        // the size of a upoint
          }
          else
          {
            eli->attrSize[eli->noAttrs - 1] = 
                eli->newAttrSize / ( eli->stableValue + 1 );
            eli->attrSizeExt[eli->noAttrs - 1] = 
        eli->newAttrSizeExt / ( eli->stableValue + 1 );
          }
          eli->Size += eli->attrSize[eli->noAttrs - 1];
          eli->SizeExt += eli->attrSizeExt[eli->noAttrs - 1];
          
          eli->sizesInitialized = true;
          eli->sizesChanged = true;
        }
        if ( eli->returned >= eli->stableValue ) eli->sizesFinal = true;

        pRes->CopySizes(eli);


        // 2. result cardinality

        if ( eli->returned > enoughSuccessesSelection )
          pRes->Card = 
            p1.Card * ( (double) (eli->returned + 1) / (eli->read + 1) );
        else
        {
          if ( qp->GetSelectivity(s) != 0.1 )
            pRes->Card = p1.Card * qp->GetSelectivity(s);
          else
            pRes->Card = p1.Card;
        }


    // 3. total time

        pRes->Time = p1.Time +
          p1.Card * wExtendStream + // time per input tuple wo results
          pRes->Card * (uExtendStream + eli->noAttrs * vExtendStream);
                    // time per output tuple created

    // 4. progress
        if ( p1.BTime < 0.1 && pipelinedProgress )      //non-blocking,
                                                        //use pipelining
          pRes->Progress = p1.Progress;
        else
          pRes->Progress =
            (p1.Progress * p1.Time +
            eli->read * wExtendStream +
            eli->returned * (uExtendStream + eli->noAttrs * vExtendStream) )    
            / pRes->Time;


    // 5. blocking time and progress

        pRes->CopyBlocking(p1);    //non-blocking operator

        return YIELD;
      } else {
        return CANCEL;
      }

  }
  return 0;
}

#endif

/*
2.19.3 Specification of operator ~extendstream~

*/
const string ExtendStreamSpec  =
  "( ( \"Signature\" \"Syntax\" \"Meaning\" "
  "\"Example\" ) "
  "( <text>((stream tuple1) (map tuple1 "
  "stream(type))) -> (stream tuple1*tuple2)"
  "</text--->"
  "<text>_ extendstream [ fun ]</text--->"
  "<text>This operator do the loopjoin between"
  "a stream of tuples and a stream of objects of a certain type."
  " the result is a stream of tuples.</text--->"
  "<text>query UBahn feed extendstream"
  "[ newattr:  units(.Trajectory)] consume</text--->"
  ") )";

/*
2.19.4 Definition of operator ~extendstream~

*/
Operator extrelextendstream (
         "extendstream",                // name
         ExtendStreamSpec,              // specification
         ExtendStream,                  // value mapping
         Operator::SimpleSelect,            // trivial selection function
         ExtendStreamTypeMap            // type mapping
);

/*
2.20 Operator ~projectextend~

This operator does the same as a combination of ~extend~ with subsequent
~project~, but in a single operation. This will saves a lot of time, because
attributes are only copied once and attributes to be removed will not be
copied at all!

The signature of the ~projectextend~ operation is as follows:

----

      stream(tuple((a1 t1) ... (an tn)))
    x (ai1, ... aij)
    x ( (bk1 (tuple((a1 t1) ... (an tn)) --> tk1))
        ...
        (bkm (tuple((a1 t1) ... (an tn)) --> tkm))
      )
 --> stream(tuple((ai1 ti1) ... (aij tij) (bk1 tk1) ... (bkm tkm)))

    where ai, ..., aij in {a1, ..., an}.

----

For instance,

----
  query people feed
  projectext[id;
             wood_perimeter: perimeter(.wood),
             wood_size: area(.wood)
            ]
  consume;

----

*/

/*
2.20.1 Type Mapping for operator ~projectextend~

*/
ListExpr ExtProjectExtendTypeMap(ListExpr args)
{

 string err = "stream(tuple(...)) x (a1,..., an) x "
              "( (b1  map1) ... (bm : mapm)) expected";

 if(nl->ListLength(args)!=3){
   return listutils::typeError("wrong number of arguments, expected: 3");
 }

 ListExpr stream = nl->First(args);
 ListExpr attrList = nl->Second(args);
 ListExpr extList = nl->Third(args);

 if(!listutils::isTupleStream(stream)){
   return listutils::typeError(err + " first arg is not a tuple stream");
 }

 // check whether all elements of attrlist are attrnames in the
 // first argument
 if(nl->AtomType(attrList)!=NoAtom){
   return listutils::typeError(err +
                          " second arg is not a list of attribute names");
 }

 ListExpr streamattr = nl->Second(nl->Second(stream));
 set<string> names;
 int noAttrs=0;
 ListExpr numberList=nl->TheEmptyList();

 ListExpr newAttrList=nl->TheEmptyList();
 ListExpr lastNewAttrList = nl->TheEmptyList();
 ListExpr lastNumberList = nl->TheEmptyList();
 bool firstCall = true;

 if(nl->IsEmpty(attrList)){
   noAttrs = 0;
   numberList = nl->TheEmptyList();
   newAttrList = nl->TheEmptyList();
 } else {
   noAttrs = nl->ListLength(attrList);
   ListExpr attrType;
   while(!nl->IsEmpty(attrList)){
     ListExpr attr = nl->First(attrList);
     attrList = nl->Rest(attrList);
     if(nl->AtomType(attr)!=SymbolType){
       return listutils::typeError(err + "(invalid attr name found)");
     }
     string attrName = nl->SymbolValue(attr);
     if(names.find(attrName)!=names.end()){
       return listutils::typeError(err + "(attr name found twice)");
     }
     names.insert(attrName);
     int j = listutils::findAttribute(streamattr, attrName, attrType);
     if(j==0){
       return listutils::typeError(err + "(attr name "+
                                   attrName + " not found)");
     }
     if (firstCall) {
       firstCall = false;
       newAttrList = nl->OneElemList(nl->TwoElemList(attr, attrType));
       lastNewAttrList = newAttrList;
       numberList = nl->OneElemList(nl->IntAtom(j));
       lastNumberList = numberList;
     } else {
       lastNewAttrList = nl->Append(lastNewAttrList,
                                    nl->TwoElemList(attr, attrType));
       lastNumberList = nl->Append(lastNumberList, nl->IntAtom(j));
     }
   }
 }

 // check the third argument (must be a list of pairs (attrname map))
 // the argument of map must be the tuple type in the stream and the
 // result must be in kind DATA
 if(nl->IsAtom(extList)){
   return listutils::typeError(err + " (wrong extension list)");
 }

 ListExpr tupletype = nl->Second(stream);
 while(!nl->IsEmpty(extList)){
   ListExpr ext = nl->First(extList);
   extList = nl->Rest(extList);
   if(nl->ListLength(ext)!=2){
     return listutils::typeError(err + " (problem in extension list");
   }
   ListExpr nameL = nl->First(ext);
   ListExpr map = nl->Second(ext);
   string err;
   if(!listutils::isValidAttributeName(nameL,err) ||
      !listutils::isMap<1>(map)){
     return listutils::typeError(err + " (problem in extension list");
   }
   string name = nl->SymbolValue(nameL);
   if(names.find(name)!=names.end()){
     return listutils::typeError(err +
                              "( conflicting names found " + name+")");
   }
   names.insert(name);
   if(!nl->Equal(tupletype,nl->Second(map))){
    return listutils::typeError(err + "( invalid argument type for map)");
   }
   ListExpr mapres = nl->Third(map);
   if(!listutils::isDATA(mapres)){
     return listutils::typeError(err + " (map result for " + name
                                 + " not in kind DATA)");
   }
   if(firstCall) {
     firstCall = false;
     newAttrList = nl->OneElemList(nl->TwoElemList(nameL, mapres));
     lastNewAttrList = newAttrList;
   } else {
     lastNewAttrList =
     nl->Append(lastNewAttrList, nl->TwoElemList(nameL,mapres));
   }
 }

 if(nl->IsEmpty(newAttrList)){
   return listutils::typeError(err +  "(resulting tuple would be empty");
 }


  return nl->ThreeElemList(
               nl->SymbolAtom(Symbol::APPEND()),
               nl->TwoElemList( nl->IntAtom(noAttrs),
                                numberList),
               nl->TwoElemList( nl->SymbolAtom(Symbol::STREAM()),
                            nl->TwoElemList( nl->SymbolAtom(Tuple::BasicType()),
                                                 newAttrList)));

}

/*
2.20.2 Value Mapping for operator ~projectextend~

*/

#ifndef USE_PROGRESS

// standard version

int
ExtProjectExtendValueMap(Word* args, Word& result, int message,
                         Word& local, Supplier s)
{
  //  cout << "ExtProjectExtendValueMap() called." << endl;

  TupleType* tt = (TupleType*)  qp->GetLocal2(s).addr;

  switch (message)
  {
    case INIT: {
      tt = new TupleType(nl->Second(GetTupleResultType(s)));
      qp->GetLocal2(s).addr = tt;
      return 0;
    }

    case FINISH: {
       if(tt){
         tt->DeleteIfAllowed();
         qp->GetLocal2(s).addr=0;
       }
       return 0;
    }

    case OPEN :
    {
      tt->IncReference();
      TupleType *tupleType = tt;
      local.addr = tupleType;
      qp->Open(args[0].addr);
      return 0;
    }
    case REQUEST :
    {
      Word elem1, elem2, value;
      int i=0, noOfAttrs=0, index=0, nooffun=0;
      Supplier son, supplier, supplier2, supplier3;
      ArgVectorPointer extFunArgs;

      qp->Request(args[0].addr, elem1);
      if (qp->Received(args[0].addr))
      {
        TupleType *tupleType = (TupleType *)local.addr;
        Tuple *currTuple     = (Tuple*) elem1.addr;
        Tuple *resultTuple   = new Tuple( tupleType );

        // copy attrs from projection list
        noOfAttrs = ((CcInt*)args[3].addr)->GetIntval();
        for(i = 0; i < noOfAttrs; i++)
        {
          son = qp->GetSupplier(args[4].addr, i);
          qp->Request(son, elem2);
          index = ((CcInt*)elem2.addr)->GetIntval();
          resultTuple->CopyAttribute(index-1, currTuple, i);
        }

        // evaluate and add attrs from extension list
        supplier = args[2].addr;           // get list of ext-functions
        nooffun = qp->GetNoSons(supplier); // get num of ext-functions
        for(i = 0; i< nooffun; i++)
        {
          supplier2 = qp->GetSupplier(supplier, i); // get an ext-function
          supplier3 = qp->GetSupplier(supplier2, 1);
          extFunArgs = qp->Argument(supplier3);
          ((*extFunArgs)[0]).setAddr(currTuple);     // pass argument
          qp->Request(supplier3,value);              // call extattr mapping
//  The original implementation tried to avoid copying the function result,
//  but somehow, this results in a strongly growing tuplebuffer on disk:
//           resultTuple->PutAttribute(
//             noOfAttrs + i, (Attribute*)value.addr );
//           qp->ReInitResultStorage( supplier3 );
          resultTuple->PutAttribute( noOfAttrs + i,
                ((Attribute*)value.addr)->Clone() );
        }
        currTuple->DeleteIfAllowed();
        result.setAddr(resultTuple);
        return YIELD;
      }
      else return CANCEL;
    }
    case CLOSE :
    {
      if(local.addr)
      {
        ((TupleType *)local.addr)->DeleteIfAllowed();
        qp->Close(args[0].addr);
        local.setAddr(0);
      }
      return 0;
    }
  }
  return 0;
}

# else

// progress version

class ProjectExtendLocalInfo: public ProgressLocalInfo
{
public:

  ProjectExtendLocalInfo():
        resultTupleType(0),stableValue(0), sizesFinal(false),
        noOldAttrs(0),noNewAttrs(0),attrSizeTmp(0), attrSizeExtTmp(0) {};

  ~ProjectExtendLocalInfo() {
    if(resultTupleType){
       resultTupleType->DeleteIfAllowed();
       resultTupleType=0;
    }
    if(attrSizeTmp){
       delete [] attrSizeTmp;
       attrSizeTmp =0;
    }
    if(attrSizeExtTmp){
       delete [] attrSizeExtTmp;
       attrSizeExtTmp =0;
    }
  }

  TupleType *resultTupleType;
  int stableValue;
  bool sizesFinal;
  int noOldAttrs, noNewAttrs;
  double *attrSizeTmp;
  double *attrSizeExtTmp;
};


struct projectextendLI2{
  TupleType* tt;
  Word elem1;
  Word elem2;
  Word value;
};


int
ExtProjectExtendValueMap(Word* args, Word& result, int message,
                         Word& local, Supplier s)
{
  //  cout << "ExtProjectExtendValueMap() called." << endl;
  int index=0;
  Supplier son, supplier, supplier2, supplier3;
  ArgVectorPointer extFunArgs;

  ProjectExtendLocalInfo *eli;
  eli = (ProjectExtendLocalInfo*) local.addr;

  projectextendLI2* li2 = (projectextendLI2*) qp->GetLocal2(s).addr;

  switch (message)
  {

    case INIT: {
       li2 = new projectextendLI2();
       li2->tt = new TupleType(nl->Second(GetTupleResultType(s)));
       qp->GetLocal2(s).addr = li2;
       return 0;
    }

    case FINISH: {
       if(li2){
         li2->tt->DeleteIfAllowed();
         delete li2;
         qp->GetLocal2(s).addr=0;
       }
       return 0;
    }

    case OPEN :
    {
      if ( eli ) {
        delete eli;
      }

      eli = new ProjectExtendLocalInfo;
      li2->tt->IncReference();
      eli->resultTupleType = li2->tt;
      eli->read = 0;
      eli->stableValue = 50;
      eli->sizesFinal = false;
      eli->noOldAttrs = ((CcInt*)args[3].addr)->GetIntval();
      eli->noNewAttrs = qp->GetNoSons(args[2].addr);
      eli->attrSizeTmp = new double[eli->noNewAttrs];
      eli->attrSizeExtTmp = new double[eli->noNewAttrs];
      for (int i = 0; i < eli->noNewAttrs; i++)
      {
        eli->attrSizeTmp[i] = 0.0;
        eli->attrSizeExtTmp[i] = 0.0;
      }

      local.setAddr(eli);

      qp->Open(args[0].addr);

      return 0;
    }
    case REQUEST :
    {
      if(!eli){
        return CANCEL;
      }
      qp->Request(args[0].addr, li2->elem1);
      if (qp->Received(args[0].addr))
      {
        eli->read++;

        Tuple *currTuple     = (Tuple*) li2->elem1.addr;
        Tuple *resultTuple   = new Tuple( eli->resultTupleType );

        // copy attrs from projection list

        for(int i = 0; i < eli->noOldAttrs; i++)
        {
          son = qp->GetSupplier(args[4].addr, i);
          qp->Request(son, li2->elem2);
          index = ((CcInt*)li2->elem2.addr)->GetIntval();
          resultTuple->CopyAttribute(index-1, currTuple, i);
        }

        // evaluate and add attrs from extension list
        supplier = args[2].addr;           // get list of ext-functions
        for(int i = 0; i < eli->noNewAttrs; i++)
        {
          supplier2 = qp->GetSupplier(supplier, i); // get an ext-function
          supplier3 = qp->GetSupplier(supplier2, 1);
          extFunArgs = qp->Argument(supplier3);
          ((*extFunArgs)[0]).setAddr(currTuple);     // pass argument
          qp->Request(supplier3,li2->value);         // call extattr mapping

          resultTuple->PutAttribute( eli->noOldAttrs + i,
                ((Attribute*)li2->value.addr)->Clone() );

          if (eli->read <= eli->stableValue)
          {
            eli->attrSizeTmp[i] += resultTuple->GetSize( eli->noOldAttrs + i );
            eli->attrSizeExtTmp[i] +=
              resultTuple->GetExtSize( eli->noOldAttrs + i );
          }

        }
        currTuple->DeleteIfAllowed();
        result.setAddr(resultTuple);
        return YIELD;
      } else {
        return CANCEL;
      }
    }
    case CLOSE :
    {
      qp->Close(args[0].addr);
      return 0;
    }

    case CLOSEPROGRESS:
    {
      if ( eli ) {
         delete eli;
         local.setAddr(0);
      }
      return 0;
    }

    case REQUESTPROGRESS:

      ProgressInfo p1;
      ProgressInfo *pRes;
      const double uProjectExtend = 0.0012;    //millisecs per tuple
      const double vProjectExtend = 0.00085;   //millisecs per tuple
                                               //and attribute

      pRes = (ProgressInfo*) result.addr;

      if ( !eli ) {
        return CANCEL;
      }

      if ( qp->RequestProgress(args[0].addr, &p1) )
      {
        eli->sizesChanged = false;

        if ( !eli->sizesInitialized )
        {
          eli->noAttrs = eli->noOldAttrs + eli->noNewAttrs;
          eli->attrSize = new double[eli->noAttrs];
          eli->attrSizeExt = new double[eli->noAttrs];
        }

        if ( !eli->sizesInitialized || p1.sizesChanged ||
           ( eli->read >= eli->stableValue && !eli->sizesFinal ) )
        {
      eli->Size = 0.0;
      eli->SizeExt = 0.0;

          for( int i = 0; i < eli->noOldAttrs; i++)    //old attrs
          {
            son = qp->GetSupplier(args[4].addr, i);
            qp->Request(son, li2->elem2);
            index = ((CcInt*)li2->elem2.addr)->GetIntval();
            eli->attrSize[i] = p1.attrSize[index-1];
            eli->attrSizeExt[i] = p1.attrSizeExt[index-1];
            eli->Size += eli->attrSize[i];
            eli->SizeExt += eli->attrSizeExt[i];
          }

          if ( eli->read < eli->stableValue )        //new attrs
          {
            for (int j = 0; j < eli->noNewAttrs; j++)
            {
              eli->attrSize[j + eli->noOldAttrs] = 12;   //size yet unknown
              eli->attrSizeExt[j + eli->noOldAttrs] = 12;
              eli->Size += eli->attrSize[j + eli->noOldAttrs];
              eli->SizeExt += eli->attrSizeExt[j + eli->noOldAttrs];
            }
          }
          else
          {
            for (int j = 0; j < eli->noNewAttrs; j++)
            {
              eli->attrSize[j + eli->noOldAttrs] = eli->attrSizeTmp[j] /
                eli->stableValue;
              eli->attrSizeExt[j + eli->noOldAttrs] = eli->attrSizeExtTmp[j] /
                eli->stableValue;
              eli->Size += eli->attrSize[j + eli->noOldAttrs];
              eli->SizeExt += eli->attrSizeExt[j + eli->noOldAttrs];
            }
          }
          eli->sizesInitialized = true;
          eli->sizesChanged = true;
        }
        if ( eli->read >= eli->stableValue ) eli->sizesFinal = true;

        pRes->Card = p1.Card;

        pRes->CopySizes(eli);

        pRes->Time = p1.Time +
          p1.Card * (uProjectExtend + eli->noAttrs * vProjectExtend);


        if ( p1.BTime < 0.1 && pipelinedProgress )      //non-blocking,
                                                        //use pipelining
          pRes->Progress = p1.Progress;
        else
          pRes->Progress =
            (p1.Progress * p1.Time +
              eli->read * (uProjectExtend + eli->noAttrs * vProjectExtend))
            / pRes->Time;

        pRes->CopyBlocking(p1);    //non-blocking operator

        return YIELD;
      } else {
        return CANCEL;
      }
  }
  return 0;
}

#endif

/*
2.20.3 Specification of operator ~projectextend~

*/
const string ExtProjectExtendSpec =
  "( ( \"Signature\" \"Syntax\" \"Meaning\" "
  "\"Example\" ) "
  "( <text>"
  "stream(tuple((a1 t1) ... (an tn))) \n"
  "  x (ai1, ... aij)\n"
  "  x ( (bk1 (tuple((a1 t1) ... (an tn)) --> tk1))\n"
  "      ...\n"
  "      (bkm (tuple((a1 t1) ... (an tn)) --> tkm))\n"
  "    )\n"
  " --> stream(tuple((ai1 ti1) ... (aij tij) (bk1 tk1) ... (bkm tkm)))\n"
  " where ai, ..., aij in {a1, ..., an}"
  "</text--->"
  "<text>_ projectextend[ list; funlist ]</text--->"
  "<text>Project tuple to list of attributes and extend with "
  "new ones from funlist. The projection list may be empty, "
  "but the extension list must contain at least one entry."
  "You may even 'replace' an attribute (but be careful with that)."
  "</text--->"
  "<text>query Orte feed projectextend"
  "[Ort, Vorwahl; BevT2: .BevT*2, BevT: (.BevT + 30)] consume</text--->"
  ") )";

/*
2.20.4 Definition of operator ~projectextend~

*/
Operator extrelprojectextend (
         "projectextend",           // name
         ExtProjectExtendSpec,      // specification
         ExtProjectExtendValueMap,  // value mapping
         Operator::SimpleSelect,    // trivial selection function
         ExtProjectExtendTypeMap    // type mapping
);

/*
2.21 Operator ~projectextendstream~

This operator does the same as ~extendstream~ with a projection on some
attributes. It is very often that a big attribute is converted into a
stream of a small pieces, for example, a text into keywords. When
applying the ~extendstream~ operator, the big attribute belongs to the
result type, and is copied for every occurrence of its smaller pieces.
A projection is a normal operation after such operation. To avoid this
copying, the projection operation is now built in the operation.

The type mapping function of the ~projectextendstream~ operation is
as follows:

----  ((stream (tuple ((x1 T1) ... (xn Tn)))) (ai1 ... aik)
        (map (tuple ((x1 T1) ... (xn Tn))) (b stream(Tb))))  ->
      (APPEND
        (k (i1 ... ik))
        (stream (tuple ((ai1 Ti1) ... (aik Tik)(b Tb)))))
----

For instance,

----  query people feed
        projectextendstream [id, age; parts : .name keywords]
        consume;
----

*/
ListExpr ProjectExtendStreamTypeMap(ListExpr args)
{

 if(nl->ListLength(args)!=3){
   return listutils::typeError("three arguments expected");
 }

 ListExpr stream = nl->First(args);
 ListExpr attrList = nl->Second(args);
 ListExpr namedMapL = nl->Third(args);
 string err ="stream(tuple(K) x (a1..an) x "
             "(b (tuple(K) -> stream(L))) expected";

 if(!listutils::isTupleStream(stream) ||
    nl->IsAtom(attrList) ||
    nl->ListLength(namedMapL)!=1){
   return listutils::typeError(err);
 }

 ListExpr namedMap = nl->First(namedMapL);
 if(nl->ListLength(namedMap)!=2){
   return listutils::typeError(err +
                        "(third argument must be a pair of name , map)");
 }

 // process the projection list
 ListExpr streamattr = nl->Second(nl->Second(stream));
 set<string> names;
 // int noAttrs;
 ListExpr numberList = nl->TheEmptyList();

 ListExpr newAttrList = nl->TheEmptyList();
 ListExpr lastNewAttrList = nl->TheEmptyList();
 ListExpr lastNumberList = nl->TheEmptyList();
 bool firstCall = true;

 int attrno = nl->ListLength(attrList);
 if(nl->IsEmpty(attrList)){
   //noAttrs = 0;
   numberList = nl->TheEmptyList();
   newAttrList = nl->TheEmptyList();
 } else {
   //noAttrs = nl->ListLength(attrList);
   ListExpr attrType;
   while(!nl->IsEmpty(attrList)){
     ListExpr attr = nl->First(attrList);
     attrList = nl->Rest(attrList);
     if(nl->AtomType(attr)!=SymbolType){
       return listutils::typeError(err + "(invalid attr name found)");
     }
     string attrName = nl->SymbolValue(attr);
     if(names.find(attrName)!=names.end()){
       return listutils::typeError(err + "(attr name found twice)");
     }
     names.insert(attrName);
     int j = listutils::findAttribute(streamattr, attrName, attrType);
     if(j==0){
       return listutils::typeError(err + "(attr name "+
                                   attrName + " not found)");
     }
     if (firstCall) {
       firstCall = false;
       newAttrList = nl->OneElemList(nl->TwoElemList(attr, attrType));
       lastNewAttrList = newAttrList;
       numberList = nl->OneElemList(nl->IntAtom(j));
       lastNumberList = numberList;
     } else {
       lastNewAttrList = nl->Append(lastNewAttrList,
                                    nl->TwoElemList(attr, attrType));
       lastNumberList = nl->Append(lastNumberList, nl->IntAtom(j));
     }
   }
 }

 ListExpr tupletype = nl->Second(stream);
 ListExpr mapname = nl->First(namedMap);
 ListExpr map = nl->Second(namedMap);
 if(nl->AtomType(mapname)!=SymbolType ||
    !listutils::isMap<1>(map)){
   return listutils::typeError(err +
          "(third argument must be a pair of name and map)");
 }
 string errmsg;
 if(!listutils::isValidAttributeName(mapname, errmsg)){
   return listutils::typeError(errmsg);
 }
 if(!nl->Equal(tupletype,nl->Second(map))){
   return listutils::typeError(err + "(map argument differs from tuple type");
 }
 if(!listutils::isDATAStream(nl->Third(map))){
   return listutils::typeError(err + " (result of mpa is not a DATA stream");
 }

 ListExpr mapData = nl->Second(nl->Third(map));
 if(firstCall){
   newAttrList = nl->OneElemList(nl->TwoElemList(mapname,mapData));
 } else {
   lastNewAttrList = nl->Append(lastNewAttrList,
                                nl->TwoElemList(mapname,mapData));
 }


 return nl->ThreeElemList( nl->SymbolAtom(Symbol::APPEND()),
                           nl->TwoElemList( nl->IntAtom( attrno ),
                                             numberList),
                           nl->TwoElemList( nl->SymbolAtom(Symbol::STREAM()),
                                            nl->TwoElemList(
                                            nl->SymbolAtom(Tuple::BasicType()),
                                            newAttrList)));

}

/*
2.19.2 Value mapping function of operator ~projectextendstream~

*/

#ifndef USE_PROGRESS

// standard version

struct ProjectExtendStreamLocalInfo
{
  Tuple *tupleX;
  Word streamY;
  TupleType *resultTupleType;
  vector<long> attrs;
};

int ProjectExtendStream(Word* args, Word& result, int message,
                        Word& local, Supplier s)
{
  ArgVectorPointer funargs;
  Word wTupleX, wValueY;
  Tuple* tupleXY;
  ProjectExtendStreamLocalInfo *localinfo;

  TupleType *resultTupleType;

  Supplier supplier, supplier2, supplier3;

  TupleType* tt = (TupleType*) qp->GetLocal2(s).addr;

  switch( message )
  {

    case INIT: {
      tt = new TupleType(nl->Second(GetTupleResultType(s)));
      qp->GetLocal2(s).addr = tt;
      return 0;
    }
    case FINISH: {
       if(tt){
         tt->DeleteIfAllowed();
         qp->GetLocal2(s).addr=0;
       }
       return 0;
    }


    case OPEN:
    {
      //1. open the input stream and initiate the arguments
      qp->Open( args[0].addr );
      qp->Request( args[0].addr, wTupleX );
      if( qp->Received( args[0].addr ) )
      {
        //2. compute the result "stream y" from tuple x
        supplier = args[2].addr;
        supplier2 = qp->GetSupplier( supplier, 0 );
        supplier3 = qp->GetSupplier( supplier2, 1 );
        funargs = qp->Argument( supplier3 );

        (*funargs)[0] = wTupleX;
        qp->Open( supplier3 );

        //3. save the local information
        localinfo = new ProjectExtendStreamLocalInfo;
        tt->IncReference();
        localinfo->resultTupleType = tt;
        localinfo->tupleX = (Tuple*)wTupleX.addr;
        localinfo->streamY.setAddr( supplier3 );

        //4. get the attribute numbers
        int noOfAttrs = ((CcInt*)args[3].addr)->GetIntval();
        for( int i = 0; i < noOfAttrs; i++)
        {
          Supplier son = qp->GetSupplier(args[4].addr, i);
          Word elem2;
          qp->Request(son, elem2);
          localinfo->attrs.push_back( ((CcInt*)elem2.addr)->GetIntval()-1 );
        }

        local.setAddr(localinfo);
      }
      else
      {
        local.setAddr(0);
      }
      return 0;
    }
    case REQUEST:
    {
      if( local.addr == 0 )
        return CANCEL;

      //1. recover local information
      localinfo=(ProjectExtendStreamLocalInfo *) local.addr;
      resultTupleType = (TupleType *)localinfo->resultTupleType;

      //2. prepare tupleX and wValueY.
      //If wValueY is empty, then get next tupleX
      wValueY.setAddr(0);
      while( wValueY.addr == 0 )
      {
        qp->Request( localinfo->streamY.addr, wValueY );
        if( !(qp->Received( localinfo->streamY.addr )) )
        {
          qp->Close( localinfo->streamY.addr );
          localinfo->tupleX->DeleteIfAllowed();
          qp->Request( args[0].addr, wTupleX );
          if( qp->Received(args[0].addr) )
          {
            supplier = args[2].addr;
            supplier2 = qp->GetSupplier(supplier, 0);
            supplier3 = qp->GetSupplier(supplier2, 1);
            funargs = qp->Argument(supplier3);

            localinfo->tupleX = (Tuple*)wTupleX.addr;
            (*funargs)[0] = wTupleX;
            qp->Open( supplier3 );

            localinfo->tupleX = (Tuple*)wTupleX.addr;
            localinfo->streamY.setAddr(supplier3);

            wValueY.setAddr(0);
          }
          else  //streamx is exausted
          {
            localinfo->streamY.setAddr(0);
            localinfo->tupleX = 0;
            return CANCEL;
          }
        }
      }

      //3. compute tupleXY from tupleX and wValueY
      tupleXY = new Tuple( localinfo->resultTupleType );

      size_t i;
      for( i = 0; i < localinfo->attrs.size(); i++ )
        tupleXY->CopyAttribute( localinfo->attrs[i], localinfo->tupleX, i );

      tupleXY->PutAttribute( i, (Attribute*)wValueY.addr );

      // setting the result
      result.setAddr( tupleXY );
      return YIELD;
    }
    case CLOSE:
    {
      if( local.addr != 0 )
      {
        localinfo = (ProjectExtendStreamLocalInfo *)local.addr;

        if( localinfo->streamY.addr != 0 )
          qp->Close( localinfo->streamY.addr );

        if( localinfo->tupleX != 0 )
          localinfo->tupleX->DeleteIfAllowed();

        if( localinfo->resultTupleType != 0 )
          localinfo->resultTupleType->DeleteIfAllowed();
        delete localinfo;
        local.setAddr(0);
      }
      qp->Close( args[0].addr );
      return 0;
    }
  }
  return 0;
}

# else

// progress version


struct ProjectExtendStreamLocalInfo: public ProgressLocalInfo
{
public:

  ProjectExtendStreamLocalInfo():
    tupleX(0), resultTupleType(0), noOldAttrs(0) { streamY.setAddr(0); };

  ~ProjectExtendStreamLocalInfo() {
    if( streamY.addr != 0 )
          qp->Close( streamY.addr );

    if( tupleX != 0 )
          tupleX->DeleteIfAllowed();

    if( resultTupleType != 0 )
          resultTupleType->DeleteIfAllowed();
  }

  Tuple *tupleX;
  Word streamY;
  TupleType *resultTupleType;
  vector<long> attrs;

  int noOldAttrs;
  double newAttrSize, newAttrSizeExt;
  int stableValue;
  bool sizesFinal;
};


int ProjectExtendStream(Word* args, Word& result, int message,
                        Word& local, Supplier s)
{
  ArgVectorPointer funargs;
  Word wTupleX, wValueY, elem2;
  int index = 0;
  Tuple* tupleXY;

  ProjectExtendStreamLocalInfo *eli;
  eli = (ProjectExtendStreamLocalInfo*) local.addr;

  Supplier son, supplier, supplier2, supplier3;

  TupleType* tt = (TupleType*) qp->GetLocal2(s).addr;

  switch( message )
  {
    case INIT: {
      tt = new TupleType(nl->Second(GetTupleResultType(s)));
      qp->GetLocal2(s).addr = tt;
      return 0;
    }

    case FINISH: {
      if(tt){
        tt->DeleteIfAllowed();
        qp->GetLocal2(s).addr=0;
      }
      return 0;
    }

    case OPEN:
    {
      if ( eli ) {
        delete eli;
      }

      eli = new ProjectExtendStreamLocalInfo;
      tt->IncReference();
      eli->resultTupleType = tt;
      eli->newAttrSize = 0.0;
      eli->newAttrSizeExt = 0.0;
      eli->stableValue = 50;
      eli->sizesFinal = false;
      eli->noOldAttrs = ((CcInt*)args[3].addr)->GetIntval();

      // 1. open the input stream and initiate the arguments
      qp->Open( args[0].addr );
      qp->Request( args[0].addr, wTupleX );
      if( qp->Received( args[0].addr ) )
      {
        eli->read = 1;

        // 2. compute the result "stream y" from tuple x
        supplier = args[2].addr;
        supplier2 = qp->GetSupplier( supplier, 0 );
        supplier3 = qp->GetSupplier( supplier2, 1 );
        funargs = qp->Argument( supplier3 );

        (*funargs)[0] = wTupleX;
        qp->Open( supplier3 );

        // 3. save the local information

        eli->tupleX = (Tuple*)wTupleX.addr;
        eli->streamY.setAddr( supplier3 );

        // 4. get the attribute numbers
        int noOfAttrs = ((CcInt*)args[3].addr)->GetIntval();
        eli->noOldAttrs = noOfAttrs;
        for( int i = 0; i < noOfAttrs; i++)
        {
          Supplier son = qp->GetSupplier(args[4].addr, i);
          Word elem2;
          qp->Request(son, elem2);
          eli->attrs.push_back( ((CcInt*)elem2.addr)->GetIntval()-1 );
        }

        local.setAddr(eli);
      }
      else
      {
        local.setAddr(0);
      }
      return 0;
    }
    case REQUEST:
    {
      if( local.addr == 0 )
        return CANCEL;

      // 1. prepare tupleX and wValueY.
      // if wValueY is empty, then get next tupleX
      wValueY.setAddr(0);
      while( wValueY.addr == 0 )
      {
        qp->Request( eli->streamY.addr, wValueY );
        if( !(qp->Received( eli->streamY.addr )) )
        {
          qp->Close( eli->streamY.addr );
          eli->tupleX->DeleteIfAllowed();
          qp->Request( args[0].addr, wTupleX );
          if( qp->Received(args[0].addr) )
          {
            eli->read++;

            supplier = args[2].addr;
            supplier2 = qp->GetSupplier(supplier, 0);
            supplier3 = qp->GetSupplier(supplier2, 1);
            funargs = qp->Argument(supplier3);

            eli->tupleX = (Tuple*)wTupleX.addr;
            (*funargs)[0] = wTupleX;
            qp->Open( supplier3 );

            eli->tupleX = (Tuple*)wTupleX.addr;
            eli->streamY.setAddr(supplier3);

            wValueY.setAddr(0);
          }
          else  //streamx is exausted
          {
            eli->streamY.setAddr(0);
            eli->tupleX = 0;
            return CANCEL;
          }
        }
      }

      // 2. compute tupleXY from tupleX and wValueY
      tupleXY = new Tuple( eli->resultTupleType );

      size_t i;
      for(i = 0; i < eli->attrs.size(); i++ )
        tupleXY->CopyAttribute( eli->attrs[i], eli->tupleX, i );

      tupleXY->PutAttribute( eli->attrs.size(), 
                    (Attribute*)wValueY.addr );

      // for the first 50 tuples returned, observe attribute sizes of the 
      // new attribute.
      if ( eli->returned <= eli->stableValue )
      {
        eli->newAttrSize += 
          tupleXY->GetSize( eli->attrs.size() );
        eli->newAttrSizeExt += 
          tupleXY->GetExtSize( eli->attrs.size() );
      }

      // setting the result
      result.setAddr( tupleXY );
      eli->returned++;
      return YIELD;
    }

    case CLOSE:
    {
      qp->Close( args[0].addr );
      return 0;
    }

    case CLOSEPROGRESS:
    {
      if ( eli ) {
         delete eli;
         local.setAddr(0);
      }
      return 0;
    }

    case REQUESTPROGRESS:

      ProgressInfo p1;
      ProgressInfo *pRes;

      // For the determination of constants see file ConstantsExtendStream.
      // We assume the same constants are valid here.

      const double wExtendStream = 0.005963;    //millisecs per tuple read
      const double uExtendStream = 0.0067;   //millisecs per tuple returned
      const double vExtendStream = 0.00014405; //millisec per attr returned

      pRes = (ProgressInfo*) result.addr;

      if ( !eli ) {
        return CANCEL;
      }

      if ( qp->RequestProgress(args[0].addr, &p1) )
      {

    // 1. attribute sizes
        eli->sizesChanged = false;

        if ( !eli->sizesInitialized )
        {
          eli->noAttrs = eli->noOldAttrs + 1;
          eli->attrSize = new double[eli->noAttrs];
          eli->attrSizeExt = new double[eli->noAttrs];
        }

        if ( !eli->sizesInitialized || p1.sizesChanged ||
           ( eli->returned >= eli->stableValue && !eli->sizesFinal ) )
        {
      eli->Size = 0.0;
      eli->SizeExt = 0.0;

          for( int i = 0; i < eli->noOldAttrs; i++) //old attrs
          {
            son = qp->GetSupplier(args[4].addr, i);
            qp->Request(son, elem2);
            index = ((CcInt*)elem2.addr)->GetIntval();
            eli->attrSize[i] = p1.attrSize[index-1];
            eli->attrSizeExt[i] = p1.attrSizeExt[index-1];
            eli->Size += eli->attrSize[i];
            eli->SizeExt += eli->attrSizeExt[i];
          }

          if ( eli->returned < eli->stableValue )   //new attr
          {
            eli->attrSize[eli->noAttrs - 1] = 144;
            eli->attrSizeExt[eli->noAttrs - 1] = 144;
                        // size yet unknown. The default 144 is taken from
                        // the size of a upoint
          }
          else
          {
            eli->attrSize[eli->noAttrs - 1] = 
                eli->newAttrSize / ( eli->stableValue + 1 );
            eli->attrSizeExt[eli->noAttrs - 1] = 
        eli->newAttrSizeExt / ( eli->stableValue + 1 );
          }
          eli->Size += eli->attrSize[eli->noAttrs - 1];
          eli->SizeExt += eli->attrSizeExt[eli->noAttrs - 1];
          
          eli->sizesInitialized = true;
          eli->sizesChanged = true;
        }
        if ( eli->returned >= eli->stableValue ) eli->sizesFinal = true;

        pRes->CopySizes(eli);


        // 2. result cardinality

        pRes->Card = 
          p1.Card * ( (double) (eli->returned + 1) / (eli->read + 1) );


    // 3. total time

        pRes->Time = p1.Time +
          p1.Card * wExtendStream + // time per input tuple wo results
          pRes->Card * (uExtendStream + eli->noAttrs * vExtendStream);
                    // time per output tuple created

    // 4. progress
        if ( p1.BTime < 0.1 && pipelinedProgress )      //non-blocking,
                                                        //use pipelining
          pRes->Progress = p1.Progress;
        else
          pRes->Progress =
            (p1.Progress * p1.Time +
            eli->read * wExtendStream +
            eli->returned * (uExtendStream + eli->noAttrs * vExtendStream) )    
            / pRes->Time;


    // 5. blocking time and progress

        pRes->CopyBlocking(p1);    //non-blocking operator

        return YIELD;
      } else {
        return CANCEL;
      }

  }
  return 0;
}


#endif

/*
2.19.3 Specification of operator ~projectextendstream~

*/
const string ProjectExtendStreamSpec =
  "( ( \"Signature\" \"Syntax\" \"Meaning\" "
  "\"Example\" ) "
  "( <text>((stream tuple1) (ai1 ... aik) (map tuple1 "
  "stream(type))) -> (stream tuple1[ai1 ... aik]*type)"
  "</text--->"
  "<text>_ projectextendstream [ list; funlist ]</text--->"
  "<text>This operator does the same as the extendstream does,"
  " projecting the result stream of tuples to some specified"
  " list of attribute names.</text--->"
  "<text>query UBahn feed projectextendstream"
  "[ Id, Up; newattr:  units(.Trajectory)] consume</text--->"
  ") )";

/*
2.19.4 Definition of operator ~projectextendstream~

*/
Operator extrelprojectextendstream (
         "projectextendstream",                // name
         ProjectExtendStreamSpec,              // specification
         ProjectExtendStream,                  // value mapping
         Operator::SimpleSelect,               // trivial selection function
         ProjectExtendStreamTypeMap            // type mapping
);

/*
2.20 Operator ~concat~

2.20.1 Type mapping function of operator ~concat~

Type mapping for ~concat~ is

----    ((stream (tuple (a1:d1 ... an:dn)))
         (stream (tuple (b1:d1 ... bn:dn))))
            -> (stream (tuple (a1:d1 ... an:dn)))
----

*/
ListExpr GetAttrTypeList (ListExpr l)
{
  ListExpr first = nl->TheEmptyList();
  ListExpr olist = first,
           lastolist = first;

  ListExpr attrlist = l;
  while (!nl->IsEmpty(attrlist))
  {
    first = nl->First(attrlist);
    attrlist = nl->Rest(attrlist);
    if (olist == nl->TheEmptyList())
    {
      olist = nl->Cons(nl->Second(first), nl->TheEmptyList());
      lastolist = olist;
    }
    else
    {
      lastolist = nl->Append(lastolist, nl->Second(first));
    }
  }
  return olist;
}



ListExpr ConcatTypeMap( ListExpr args )
{
  if(nl->ListLength(args)!=2){
    return listutils::typeError("two streams expected");
  }
  if(!nl->Equal(nl->First(args),nl->Second(args))){
      return listutils::typeError("both arguments must be of the same type");
  }
  if(!listutils::isStream(nl->First(args))){
    return listutils::typeError("arguments are no streams");
  }
  return nl->First(args);
}
/*
2.20.2 Value mapping function of operator ~concat~

*/
int Concat(Word* args, Word& result, int message, Word& local, Supplier s)
{
  Word t;

  switch (message)
  {
    case OPEN :

      qp->Open(args[0].addr);
      qp->Open(args[1].addr);
      // start using 1st stream
      if(local.addr){
        *(static_cast<bool*>(local.addr)) = false;
      } else {
        local.setAddr(new bool(false));
      }
      return 0;

    case REQUEST :
      if(!local.addr) {
        result.setAddr(0);
        return CANCEL;
      }
      if ( !(*static_cast<bool*>(local.addr)) ){
        qp->Request(args[0].addr, t);
        if (qp->Received(args[0].addr)){
          result.setAddr(t.addr);
          return YIELD;
        } else {
          *(static_cast<bool*>(local.addr)) = true; // start using 2nd stream
        }
      }
      qp->Request(args[1].addr, t);
      if (qp->Received(args[1].addr)){
        result.setAddr(t.addr);
        return YIELD;
      } else {
        return CANCEL;
      }

    case CLOSE :

      qp->Close(args[0].addr);
      qp->Close(args[1].addr);
      if( local.addr ) {
        delete (static_cast<bool*>(local.addr));
        local.setAddr(0);
      }
      return 0;
  }
  return 0;
}

/*
2.20.3 Specification of operator ~concat~

*/
const string ConcatSpec  = "( ( \"Signature\" \"Syntax\" \"Meaning\" "
                           "\"Example\" ) "
                           "(<text>stream(X) x stream(X) -> stream(X)</text--->"
                           "<text>_ _ concat</text--->"
                           "<text>Returns all elements of the first argument "
                           "followed by all elements from the second argument."
                           "</text--->"
                           "<text>query ten feed five feed concat consume"
                           "</text--->"
                           ") )";

/*
2.20.4 Definition of operator ~concat~

*/
Operator extrelconcat (
         "concat",               // name
         ConcatSpec,             // specification
         Concat,                 // value mapping
         Operator::SimpleSelect, // trivial selection function
         ConcatTypeMap           // type mapping
);

/*
2.21 Operator ~groupby~

2.21.1 Type mapping function of operator ~groupby~

Result type of ~groupby~ operation.

----   Let X = tuple ((x1 t1) ... (xn tn)), R = rel(X):

       ( (stream X) (xi1 ... xik) ( (y1 (map R T1)) ... (ym (map R Tm)) ) )

        -> ( APPEND (m p1 ... pm)
               (stream (tuple (xj1 tj1)... (xjl tjl) (y1 T1) ... (ym Tm))))

       with tj,Ti in kind DATA, xi <> xj and k+l=n, pi <> pj and 1 <= pi <= m.
       This means attributes xi ... xik are removed from the stream and
       attributes y1 ... ym are appended. These new attributes represent
       aggregated values computed by maps of R -> Ti which must have a
       result type of kind DATA.
----

*/
ListExpr GroupByTypeMap(ListExpr args)
{

 string err = "stream(tuple(X)) x attrlist x funlist expected";
 if(!nl->HasLength(args,3)){
   return listutils::typeError(err + " ( wrong number of args");
 } 

 ListExpr stream = nl->First(args);
 ListExpr groupList = nl->Second(args);
 ListExpr funList = nl->Third(args);

 if(!Stream<Tuple>::checkType(stream) ||
    (nl->AtomType(groupList) != NoAtom) ||
    (nl->AtomType(funList) != NoAtom)){
    return listutils::typeError(err);
 }

 ListExpr attrList = nl->Second(nl->Second(stream));

 // 
 set<string> usedNames;

 ListExpr indexList = nl->TheEmptyList();
 ListExpr indexLast = nl->TheEmptyList();
 ListExpr resAttrList = nl->TheEmptyList();
 ListExpr resAttrLast = nl->TheEmptyList();
 bool first = true;

 // process group list

 while(!nl->IsEmpty(groupList)){
    ListExpr attrNameList = nl->First(groupList);
    groupList = nl->Rest(groupList);
    if(!listutils::isSymbol(attrNameList)){
       return listutils::typeError(" invalid attribute name: " 
                                   + nl->ToString(attrNameList));
    }
    string attrName = nl->SymbolValue(attrNameList);
    if(usedNames.find(attrName)!=usedNames.end()){
       return listutils::typeError("Attribute " + attrName + " used twice");
    }
    usedNames.insert(attrName);
    ListExpr type;
    int index = listutils::findAttribute(attrList, attrName, type);
    if(!index){
      return listutils::typeError("attribute " + attrName 
                                  + " not part of tuple");
    }
    ListExpr newAttr = nl->TwoElemList(attrNameList, type);
    if(first){
       indexList = nl->OneElemList(nl->IntAtom(index));
       indexLast = indexList;
       resAttrList = nl->OneElemList(newAttr);
       resAttrLast = resAttrList;
       first = false;
    }  else {
       indexLast = nl->Append(indexLast, nl->IntAtom(index));
       resAttrLast = nl->Append(resAttrLast, newAttr);
    }
 }

 // process funlist
 ListExpr tupleType = nl->Second(stream);
 ListExpr relType = nl->TwoElemList( 
              listutils::basicSymbol<Relation>(),
              tupleType);
 
 while(!nl->IsEmpty(funList)) {
   ListExpr fun = nl->First(funList);
   funList = nl->Rest(funList);
   if(!nl->HasLength(fun,2)){ // (name map)
      return listutils::typeError("invalid function definition "
                                  "found (missing name or map)");
   } 
   // check attribute name
   ListExpr attrNameList = nl->First(fun);
   if(!listutils::isValidAttributeName(attrNameList,err)){
     return listutils::typeError("Attribute name " 
                                 + nl->ToString(attrNameList) 
                                 + " is not valid:" + err);  
   }
   string attrname = nl->SymbolValue(attrNameList);
   if(usedNames.find(attrname)!=usedNames.end()){
     return listutils::typeError("Attribute " + attrname + " used twice"); 
   }
   usedNames.insert(attrname);
   // check map
   ListExpr fundef = nl->Second(fun);
   if(!listutils::isMap<1>(fundef)){
     return listutils::typeError(nl->ToString(fundef) 
                                   + " is not a valid function definition");
   }
   // check argument
   if(!nl->Equal(relType, nl->Second(fundef))){
     return listutils::typeError("invalid argument type for attribute " 
                          + attrname + " expected " + nl->ToString(relType) 
                          + " but got " + nl->ToString(nl->Second(fundef) ));
   }
   // check result
   if(!Attribute::checkType(nl->Third(fundef))){
     return listutils::typeError("result of function for " + attrname
                                 + " is not an attribute");
   }
   ListExpr newAttr = nl->TwoElemList(attrNameList, nl->Third(fundef));
   if(first){
      indexList = nl->TheEmptyList();
      resAttrList = nl->OneElemList(newAttr);
      resAttrLast = resAttrList;
      first = false;
   } else {
      resAttrLast = nl->Append(resAttrLast, newAttr);
   }
 }
 // create result list
 ListExpr resList = nl->TwoElemList(
                      listutils::basicSymbol<Stream<Tuple> >(),
                      nl->TwoElemList(
                        listutils::basicSymbol<Tuple>(),
                        resAttrList));
 return nl->ThreeElemList(
            nl->SymbolAtom(Symbol::APPEND()),
            nl->Cons(nl->IntAtom(nl->ListLength(indexList)), indexList),
            resList);
}


/*
2.22 Operator ~slidingwindow~

2.21.1 Type mapping function of operator ~slidingwindow~

Result type of ~slidingwindow~ operation.

----   Let X = tuple ((x1 t1) ... (xn tn)), R = rel(X):

       ( (stream X) (xi1 ... xik) ( (y1 (map R T1)) ... (ym (map R Tm)) ) )

        -> ( APPEND (m p1 ... pm)
               (stream (tuple (xj1 tj1)... (xjl tjl) (y1 T1) ... (ym Tm))))

       with tj,Ti in kind DATA, xi <> xj and k+l=n, pi <> pj and 1 <= pi <= m.
       This means attributes xi ... xik are removed from the stream and
       attributes y1 ... ym are appended. These new attributes represent
       aggregated values computed by maps of R -> Ti which must have a
       result type of kind DATA.
----

*/
ListExpr SlidingWindowTypeMap(ListExpr args)
{
  bool debugme= true;
  ListExpr first, second, third, fourth;     // list used for analysing input
  ListExpr listn, lastlistn, listp;  // list used for constructing output

  first = second = third = nl->TheEmptyList();
  listn = lastlistn = listp = nl->TheEmptyList();

  string tupleSymbolStr = Tuple::BasicType();

  bool listOk = true;
  listOk = listOk && ( nl->ListLength(args) == 4 );

  if ( listOk ) {
    first  = nl->First(args);
    second = nl->Second(args);
    third  = nl->Third(args);
    fourth= nl->Fourth(args);

    // check input list structure
    listOk = listOk && (nl->ListLength(first) == 2);
    listOk = listOk && !nl->IsEmpty( fourth );
  }

  if( !listOk )
  {
    stringstream errMsg;
    errMsg << "slidingwindow: Invalid input list structure. "
        << "The structure should be a four elem list "
        << "like (stream (" << tupleSymbolStr
        << "((x1 t1) ... (xn tn)) int int "
        << "( (y1 (map R T1)) ... (ym (map R Tm))!";

    ErrorReporter::ReportError(errMsg.str());
    return nl->SymbolAtom(Symbol::TYPEERROR());
  }


  // check for tuple stream
  listOk = listOk && Stream<Tuple>::checkType(first);

  if ( !listOk ) {

    ErrorReporter::ReportError( "slidingwindow: Input is not of type (stream "
        + tupleSymbolStr + "(...))." );
    return nl->SymbolAtom(Symbol::TYPEERROR());
  }

  // list seems to be ok. Check the second and the third arguments
  if(! nl->IsEqual(second, CcInt::BasicType()) )
  {
    ErrorReporter::ReportError(
        "slidingwindow: expected int as a second argument");
    return nl->TypeError();
  }

  if(! nl->IsEqual(third, CcInt::BasicType()) )
  {
    ErrorReporter::ReportError(
        "slidingwindow: expected int as a third argument");
    return nl->TypeError();
  }

  // Check the last argument
  ListExpr rest = fourth;

  if(nl->IsAtom(rest)){
    return listutils::typeError("fourth arg must be a list");
  }

  while (!(nl->IsEmpty(rest))) // check functions y1 .. ym
  {
      // iterate over elements of the 3rd input list
    ListExpr firstr = nl->First(rest);
    rest = nl->Rest(rest);

    ListExpr newAttr = nl->First(firstr);
    ListExpr mapDef = nl->Second(firstr);
    ListExpr mapOut = nl->Third(mapDef);

      // check list structure
    bool listOk = true;
    listOk = listOk && ( nl->IsAtom(newAttr) );
    listOk = listOk && ( nl->ListLength(mapDef) == 3 );
    listOk = listOk && ( nl->AtomType(newAttr) == SymbolType );
    listOk = listOk && ( listutils::isAnyMap(mapDef)); 
//    listOk = listOk && ( nl->Equal(groupType, nl->Second(mapDef)) );

    if( !listOk )
        // Todo: there could be more fine grained error messages
    {
      ErrorReporter::ReportError(
          "slidingwindow: Function definition is not correct!");
      return nl->SymbolAtom(Symbol::TYPEERROR());
    }

    // check if the Type Constructor belongs to KIND DATA
    // If the functions result type is typeerror this check will also fail
    ListExpr errorInfo = nl->OneElemList(nl->SymbolAtom("ErrorInfo"));
    if ( !am->CheckKind(Kind::DATA(), mapOut, errorInfo) ) {

      stringstream errMsg;
      errMsg << "slidingwindow: The aggregate function for attribute \""
          << nl->SymbolValue(newAttr) << "\""
          << " returns a type which is not usable in tuples."
          << " The type constructor \""
          <<  nl->ToString(mapOut) << "\""
          << " belongs not to kind DATA!"
          << ends;

      ErrorReporter::ReportError(errMsg.str());
      return nl->SymbolAtom(Symbol::TYPEERROR());

    }

    if (    (nl->EndOfList( lastlistn ) == true)
         && (nl->IsEmpty( lastlistn ) == false)
         && (nl->IsAtom( lastlistn ) == false)
       )
    { // list already contains group-attributes (not empty)
      lastlistn = nl->Append(lastlistn,(nl->TwoElemList(newAttr,mapOut)));
    }
    else
    { // no group attribute (list is still empty)
      listn = nl->OneElemList(nl->TwoElemList(newAttr,mapOut));
      lastlistn = listn;
    }
  } // end of while check functions

  if ( !CompareNames(listn) )
  { // check if attribute names are unique
    ErrorReporter::ReportError("slidingwindow: Attribute names are not unique");
    return nl->SymbolAtom(Symbol::TYPEERROR());
  }

  // Type mapping is correct, return result type.
  ListExpr groupType = nl->TwoElemList( nl->SymbolAtom(Symbol::STREAM()),
      nl->TwoElemList(nl->SymbolAtom(Tuple::BasicType()), listn) );

  if(debugme)
  {
    string resstring;
    nl->WriteToString(resstring, groupType);
    cerr << "groupbyTypeMap: result = " << resstring << endl;
  }
  return groupType;
}


/*
2.21.2 Value mapping function of operator ~groupby~

*/


#ifndef USE_PROGRESS


struct GroupByLocalInfo
{
  Tuple *t;
  TupleType *resultTupleType;
  long MAX_MEMORY;
  GroupByLocalInfo() : t(0), resultTupleType(0), MAX_MEMORY(0) {}
};

int GroupByValueMapping
(Word* args, Word& result, int message, Word& local, Supplier supplier)
{
  Tuple *s = 0;
  Word sWord(Address(0));
  TupleBuffer* tp = 0;
  GenericRelationIterator* relIter = 0;
  int i = 0, j = 0, k = 0;
  int numberatt = 0;
  bool ifequal = false;
  Word value(Address(0));
  Supplier  value2;
  Supplier supplier1;
  Supplier supplier2;
  int noOffun = 0;
  ArgVectorPointer vector;
  const int indexOfCountArgument = 3;
  const int startIndexOfExtraArguments = indexOfCountArgument +1;
  int attribIdx = 0;
  GroupByLocalInfo *gbli = 0;

  TupleType* tt = (TupleType*) qp->GetLocal2(s).addr;


  // The argument vector contains the following values:
  // args[0] = stream of tuples
  // args[1] = list of identifiers
  // args[2] = list of functions
  // args[3] = Number of extra arguments
  // args[4 ...] = args added by APPEND

  switch(message)
  {
    case INIT : {
      tt = new TupleType(nl->Second(GetTupleResultType(s)));
      qp->GetLocal2(s).addr=tt;
      return 0;
    }

    case FINISH: {
       if(tt){
         tt->DeleteIfAllowed();
         qp->GetLocal2(s).addr=0;
       }
       return 0;
    }


    case OPEN:
    {
      // Get the first tuple pointer and store it in the
      // GroupBylocalInfo structure
      qp->Open (args[0].addr);
      qp->Request(args[0].addr, sWord);
      if (qp->Received(args[0].addr))
      {
        gbli = new GroupByLocalInfo();
        gbli->t = (Tuple*)sWord.addr;
        gbli->t->IncReference();
        tt->IncReference();
        gbli->resultTupleType = tt;
        gbli->MAX_MEMORY = (qp->GetMemorySize(s) * 1024 * 1024);
        local.setAddr(gbli);

        cmsg.info("ERA:ShowMemInfo")
          << "GroupBy.MAX_MEMORY ("
          << gbli->MAX_MEMORY / 1024 << " MB): = "
          << gbli->MAX_MEMORY / gbli->t->GetExtSize()
          << " Tuples" << endl;
        cmsg.send();
      }
      else
      {
        local.setAddr(0);
      }
      return 0;
    }
    case REQUEST:
    {
      Counter::getRef("GroupBy:Request")++;
      if(local.addr == 0)
      {
        return CANCEL;
      }
      gbli = (GroupByLocalInfo *)local.addr;
      if( gbli->t == 0 ) { // Stream ends
          return CANCEL;
      }
      tp = new TupleBuffer(gbli->MAX_MEMORY);
      tp->AppendTuple(gbli->t);


      // get number of attributes
      numberatt = ((CcInt*)args[indexOfCountArgument].addr)->GetIntval();

      ifequal = true;
      // Get next tuple
      qp->Request(args[0].addr, sWord);
      while ((qp->Received(args[0].addr)) && ifequal)
      {
        s = (Tuple*)sWord.addr;
        for (k = 0; k < numberatt; k++) // check if  tuples t = s
        {
          // loop over all grouping attributes
          attribIdx =
            ((CcInt*)args[startIndexOfExtraArguments+k].addr)->GetIntval();
          j = attribIdx - 1;
          if (((Attribute*)gbli->t->GetAttribute(j))->
               Compare((Attribute *)s->GetAttribute(j)))
          {
            ifequal = false;
            break;
          }
        }

        if (ifequal) // store in tuple buffer
        {
          tp->AppendTuple(s);
          s->DeleteIfAllowed();
          qp->Request(args[0].addr, sWord);
          // get next tuple
        }
        else
        {
          // store tuple pointer in local info
          gbli->t->DecReference();
          gbli->t->DeleteIfAllowed();
          gbli->t = s;
          gbli->t->IncReference();
        }
      }
      if (ifequal)
      // last group finished, stream ends
      {
        gbli->t->DecReference();
        gbli->t->DeleteIfAllowed();
        gbli->t = 0;
      }

      // create result tuple
      Tuple *t = new Tuple( gbli->resultTupleType );
      relIter = tp->MakeScan();
      s = relIter->GetNextTuple();

      // copy in grouping attributes
      for(i = 0; i < numberatt; i++)
      {
        attribIdx =
          ((CcInt*)args[startIndexOfExtraArguments+i].addr)->GetIntval();
        t->CopyAttribute(attribIdx - 1, s, i);
      }
      s->DeleteIfAllowed();
      value2 = (Supplier)args[2].addr; // list of functions
      noOffun  =  qp->GetNoSons(value2);
      delete relIter;

      for(i = 0; i < noOffun; i++)
      {
        // prepare arguments for function i
        supplier1 = qp->GetSupplier(value2, i);
        supplier2 = qp->GetSupplier(supplier1, 1);
        vector = qp->Argument(supplier2);
        // The group was stored in a relation identified by symbol group
        // which is a typemap operator. Here it is stored in the
        // argument vector
        ((*vector)[0]).setAddr(tp);

        // compute value of function i and put it into the result tuple
        qp->Request(supplier2, value);
        t->PutAttribute(numberatt + i, (Attribute*)value.addr);
        qp->ReInitResultStorage(supplier2);
      }
      result.setAddr(t);
      delete tp;
      return YIELD;
    }
    case CLOSE:
    {
      if( local.addr != 0 )
      {
        gbli = (GroupByLocalInfo *)local.addr;
        if( gbli->resultTupleType != 0 )
          gbli->resultTupleType->DeleteIfAllowed();
        if( gbli->t != 0 )
        {
          gbli->t->DecReference();
          gbli->t->DeleteIfAllowed();
        }
        delete gbli;
        local.setAddr(0);
      }
      qp->Close(args[0].addr);
      return 0;
    }
  }
  return 0;
}

#else

//progress version


class GroupByLocalInfo: public ProgressLocalInfo
{
public:
  Tuple *t;
  TupleType *resultTupleType;
  long MAX_MEMORY;

  //new for progress

  int stableValue;
  bool sizesFinal;

  int noGroupAttrs;
  int noAggrAttrs;
  double *attrSizeTmp;
  double *attrSizeExtTmp;

  // initialization
  GroupByLocalInfo() : ProgressLocalInfo(),
    t(0), resultTupleType(0), MAX_MEMORY(0),
    stableValue(0),sizesFinal(false),
    noGroupAttrs(0), noAggrAttrs(0),
    attrSizeTmp(0), attrSizeExtTmp(0)
  {}

  ~GroupByLocalInfo(){
     if(t){
       t->DeleteIfAllowed();
       t = 0;
     }
     if(resultTupleType){
       resultTupleType->DeleteIfAllowed();
       resultTupleType = 0;
     }
     if(attrSizeTmp){
       delete[] attrSizeTmp;
       attrSizeTmp=0;
     }
     if(attrSizeExtTmp){
       delete[] attrSizeExtTmp;
       attrSizeExtTmp = 0;
     }
  }

};

int GroupByValueMapping
(Word* args, Word& result, int message, Word& local, Supplier s)
{
  Tuple *ss = 0;
  Word sWord(Address(0));
  TupleBuffer* tp = 0;
  GenericRelationIterator* relIter = 0;
  int i = 0, j = 0, k = 0;
  int numberatt = 0;
  bool ifequal = false;
  Word value(Address(0));
  Supplier  value2;
  Supplier supplier1;
  Supplier supplier2;
  int noOffun = 0;
  ArgVectorPointer vector;
  const int indexOfCountArgument = 3;
  const int startIndexOfExtraArguments = indexOfCountArgument +1;
  int attribIdx = 0;
  GroupByLocalInfo *gbli = (GroupByLocalInfo *)local.addr;
  TupleType* tt = (TupleType*) qp->GetLocal2(s).addr;


  // The argument vector contains the following values:
  // args[0] = stream of tuples
  // args[1] = list of identifiers
  // args[2] = list of functions
  // args[3] = Number of extra arguments
  // args[4 ...] = args added by APPEND

  switch(message)
  {

    case INIT: {
      tt = new TupleType(nl->Second(GetTupleResultType(s)));
      qp->GetLocal2(s).addr = tt;
      return 0;
    }
    case FINISH: {
       if(tt){
         tt->DeleteIfAllowed();
         qp->GetLocal2(s).addr=0;
       }
       return 0;
    }


    case OPEN:
    {

      if (gbli) {
        delete gbli;
      }
      gbli = new GroupByLocalInfo();
      gbli->stableValue = 5;
      gbli->sizesFinal = false;

      numberatt = ((CcInt*)args[indexOfCountArgument].addr)->GetIntval();
      value2 = (Supplier)args[2].addr; // list of functions
      noOffun  =  qp->GetNoSons(value2);

      gbli->noAttrs = numberatt + noOffun;
      gbli->noGroupAttrs = numberatt;
      gbli->noAggrAttrs = noOffun;
      gbli->attrSizeTmp = new double[gbli->noAttrs];
      gbli->attrSizeExtTmp = new double[gbli->noAttrs];

      for (int i = 0; i < gbli->noAttrs; i++)
      {
        gbli->attrSizeTmp[i] = 0.0;
        gbli->attrSizeExtTmp[i] = 0.0;
      }

      local.setAddr(gbli);  //from now, progress queries possible

      // Get the first tuple pointer and store it in the
      // GroupBylocalInfo structure
      qp->Open (args[0].addr);
      qp->Request(args[0].addr, sWord);
      if (qp->Received(args[0].addr))
      {
        gbli->read++;
        gbli->t = (Tuple*)sWord.addr;
        //gbli->t->IncReference();
        tt->IncReference();
        gbli->resultTupleType = tt;
        gbli->MAX_MEMORY = (qp->GetMemorySize(s) * 1024 * 1024);


        cmsg.info("ERA:ShowMemInfo")
          << "GroupBy.MAX_MEMORY ("
          << gbli->MAX_MEMORY / 1024 << " MB): = "
          << gbli->MAX_MEMORY / gbli->t->GetExtSize()
          << " Tuples" << endl;
        cmsg.send();
      }

      return 0;
    }
    case REQUEST:
    {
      Counter::getRef("GroupBy:Request")++;
      if(!gbli){
        return CANCEL;
      }

      if(gbli->t == 0){
        return CANCEL;
      } else {
        tp = new TupleBuffer(gbli->MAX_MEMORY);
        tp->AppendTuple(gbli->t);
      }

      // get number of attributes
      numberatt = ((CcInt*)args[indexOfCountArgument].addr)->GetIntval();

      ifequal = true;
      // Get next tuple
      qp->Request(args[0].addr, sWord);
      while ((qp->Received(args[0].addr)) && ifequal)
      {
        gbli->read++;
        ss = (Tuple*)sWord.addr;
        for (k = 0; k < numberatt; k++) // check if  tuples t = s
        {
          // loop over all grouping attributes
          attribIdx =
            ((CcInt*)args[startIndexOfExtraArguments+k].addr)->GetIntval();
          j = attribIdx - 1;
          if (((Attribute*)gbli->t->GetAttribute(j))->
               Compare((Attribute *)ss->GetAttribute(j)))
          {
            ifequal = false;
            break;
          }
        }

        if (ifequal) // store in tuple buffer
        {
          tp->AppendTuple(ss);
          ss->DeleteIfAllowed();
          qp->Request(args[0].addr, sWord);
          // get next tuple
        }
        else
        {
          // store tuple pointer in local info
          gbli->t->DeleteIfAllowed();
          gbli->t = ss;
          //gbli->t->IncReference();
        }
      }
      if (ifequal)
      // last group finished, stream ends
      {
        gbli->t->DeleteIfAllowed();
        gbli->t = 0;
      }

      // create result tuple
      Tuple *t = new Tuple( gbli->resultTupleType );
      relIter = tp->MakeScan();
      ss = relIter->GetNextTuple();

      // copy in grouping attributes
      for(i = 0; i < numberatt; i++)
      {
        attribIdx =
          ((CcInt*)args[startIndexOfExtraArguments+i].addr)->GetIntval();
        t->CopyAttribute(attribIdx - 1, ss, i);
      }
      ss->DeleteIfAllowed();
      value2 = (Supplier)args[2].addr; // list of functions
      noOffun  =  qp->GetNoSons(value2);
      delete relIter;

      for(i = 0; i < noOffun; i++)
      {
        // prepare arguments for function i
        supplier1 = qp->GetSupplier(value2, i);
        supplier2 = qp->GetSupplier(supplier1, 1);
        vector = qp->Argument(supplier2);
        // The group was stored in a relation identified by symbol group
        // which is a typemap operator. Here it is stored in the
        // argument vector
        ((*vector)[0]) = SetWord(tp);

        // compute value of function i and put it into the result tuple
        qp->Request(supplier2, value);
        t->PutAttribute(numberatt + i, (Attribute*)value.addr);
        qp->ReInitResultStorage(supplier2);
      }

      if ( gbli->returned < gbli->stableValue )
      {
        for (int i = 0; i < gbli->noAttrs; i++)
        {
          gbli->attrSizeTmp[i] += t->GetSize(i);
          gbli->attrSizeExtTmp[i] += t->GetExtSize(i);
        }
      }

      gbli->returned++;
      result.setAddr(t);
      if(tp){
         delete tp;
      }
      tp = 0;
      return YIELD;
    }
    case CLOSE:
    {
      qp->Close(args[0].addr);
      return 0;
    }

    case CLOSEPROGRESS:
      if (gbli)
      {
        delete gbli;
        local.setAddr(0);
      }
      return 0;


    case REQUESTPROGRESS:

      ProgressInfo p1;
      ProgressInfo *pRes;
      const double uGroup = 0.013;  //millisecs per tuple and new attribute

      pRes = (ProgressInfo*) result.addr;

      if ( !gbli ){
         return CANCEL;
      }

      if ( qp->RequestProgress(args[0].addr, &p1) )
      {
        gbli->sizesChanged = false;

        if ( !gbli->sizesInitialized )
        {
          gbli->attrSize = new double[gbli->noAttrs];
          gbli->attrSizeExt = new double[gbli->noAttrs];
        }

        if ( !gbli->sizesInitialized || p1.sizesChanged ||
           ( gbli->returned >= gbli->stableValue && !gbli->sizesFinal ) )
        {

          if ( gbli->returned < gbli->stableValue )
          {
            for (int i = 0; i < gbli->noGroupAttrs; i++)
            {
              gbli->attrSize[i] = 56;  //guessing string attributes
              gbli->attrSizeExt[i] = 56;
            }
            for (int j = 0; j < gbli->noAggrAttrs; j++)
            {        //guessing int attributes
              gbli->attrSize[j + gbli->noGroupAttrs] = 12;
              gbli->attrSizeExt[j + gbli->noGroupAttrs] = 12;
            }
          }
          else
          {
            for (int i = 0; i < gbli->noAttrs; i++)
            {
              gbli->attrSize[i] = gbli->attrSizeTmp[i] / gbli->stableValue;
              gbli->attrSizeExt[i] = gbli->attrSizeExtTmp[i] /
                gbli->stableValue;
            }
          }
          gbli->Size = 0.0;
          gbli->SizeExt = 0.0;
          for (int i = 0; i < gbli->noAttrs; i++)
          {
            gbli->Size += gbli->attrSize[i];
            gbli->SizeExt += gbli->attrSizeExt[i];
          }
          gbli->sizesInitialized = true;
          gbli->sizesChanged = true;
        }
        if ( gbli->returned >= gbli->stableValue ) gbli->sizesFinal = true;



        //As long as we have not seen 5 result tuples (groups), we guess groups
        //of size 10

        pRes->Card = (gbli->returned < 5 ? p1.Card/10.0 :
          p1.Card * (double) gbli->returned / (double) gbli->read);

        pRes->CopySizes(gbli);

        pRes->Time = p1.Time + p1.Card * gbli->noAggrAttrs * uGroup;

        pRes->Progress = (p1.Progress * p1.Time
          + gbli->read * gbli->noAggrAttrs * uGroup)
          / pRes->Time;

        pRes->CopyBlocking(p1);    //non-blocking operator

        return YIELD;
      } else {
        return CANCEL;
      }

  }
  return 0;
}

#endif



struct SlidingWindowLocalInfo
{
  TupleType *resultTupleType;
  bool firstWindow;
  CircularTupleBuffer* tb;

  SlidingWindowLocalInfo() : resultTupleType(0), firstWindow(true), tb(0) {}
};

int SlidingWindowValueMapping
(Word* args, Word& result, int message, Word& local, Supplier supplier)
{
  bool debugme= false;
  if(debugme)
    qp->ListOfTree(supplier, cerr);
  Tuple *s = 0;
  Word sWord(Address(0));
  Word value(Address(0));
  Supplier  value2;
  Supplier supplier1;
  Supplier supplier2;
  int noOffun = 0;
  ArgVectorPointer vector;
  SlidingWindowLocalInfo *gbli = 0;

  TupleType* tt = (TupleType*) qp->GetLocal2(supplier).addr;

  // The argument vector contains the following values:
  // args[0] = stream of tuples
  // args[1] = list of identifiers
  // args[2] = list of functions
  // args[3] = Number of extra arguments
  // args[4 ...] = args added by APPEND

  switch(message)
  {
    case INIT : {
      tt = new TupleType(nl->Second(GetTupleResultType(supplier)));
      qp->GetLocal2(supplier).addr = tt;
      return 0;
    }

    case FINISH : {
       if(tt){
          tt->DeleteIfAllowed();
          qp->GetLocal2(supplier).addr=0;
       }
       return 0;
    }


    case OPEN:
    {
      // Get the first tuple pointer and store it in the
      // GroupBylocalInfo structure
      gbli = new SlidingWindowLocalInfo();
      tt->IncReference();
      gbli->resultTupleType = tt;
      gbli->firstWindow= true;
      local.setAddr(gbli);
      qp->Open (args[0].addr);

      return 0;
    }
    case REQUEST:
    {
      int windowSize, stepSize;
      windowSize = static_cast<CcInt*>(args[1].addr)->GetIntval();
      stepSize = static_cast<CcInt*>(args[2].addr)->GetIntval();

      if(local.addr == 0)
      {
        return CANCEL;
      }
      gbli = (SlidingWindowLocalInfo *)local.addr;

      int i= 0;
      if(gbli->firstWindow)
      {
        gbli->firstWindow= false;
        gbli->tb= new CircularTupleBuffer(
            true, gbli->resultTupleType, windowSize);
        qp->Request(args[0].addr, sWord);
        while (i++ < windowSize && qp->Received(args[0].addr))
        {
          s= static_cast<Tuple*>(sWord.addr);
          gbli->tb->AppendTuple(s);
          if(i < windowSize){
             qp->Request(args[0].addr, sWord);
          }
          s->DeleteIfAllowed();
        }
      }
      else
      {
        qp->Request(args[0].addr, sWord);
        while (i++ < stepSize && qp->Received(args[0].addr))
        {
          s= static_cast<Tuple*>(sWord.addr);
          gbli->tb->AppendTuple(s);
          if(i < stepSize){
            qp->Request(args[0].addr, sWord);
          }
          s->DeleteIfAllowed();
        }
      }
      if(i == 1) //Steam end
        return CANCEL;

      // create result tuple
      Tuple *t = new Tuple( gbli->resultTupleType );
      value2 = (Supplier)args[3].addr; // list of functions
      noOffun  =  qp->GetNoSons(value2);

      for(i = 0; i < noOffun; i++)
      {
        // prepare arguments for function i
        supplier1 = qp->GetSupplier(value2, i);
        supplier2 = qp->GetSupplier(supplier1, 1);
        vector = qp->Argument(supplier2);
        // The group was stored in a relation identified by symbol group
        // which is a typemap operator. Here it is stored in the
        // argument vector
        ((*vector)[0]).setAddr(gbli->tb);

        // compute value of function i and put it into the result tuple
        qp->Request(supplier2, value);
        t->PutAttribute(i, (Attribute*)value.addr);
        qp->ReInitResultStorage(supplier2);
      }
      result.setAddr(t);
      return YIELD;
    }
    case CLOSE:
    {
      if( local.addr != 0 )
      {
        gbli = (SlidingWindowLocalInfo *)local.addr;
        if( gbli->resultTupleType != 0 )
          gbli->resultTupleType->DeleteIfAllowed();
        if( gbli->tb != 0 )
        {
          gbli->tb->Clear();
          delete gbli->tb;
        }
        delete gbli;
        local.setAddr(0);
      }
      qp->Close(args[0].addr);
      return 0;
    }
  }
  return 0;
}


/*
2.21.3 Specification of operator ~groupby~

*/
const string GroupBySpec  = "( ( \"Signature\" \"Syntax\" \"Meaning\" "
                            "\"Example\" ) "
                            "( <text>((stream (tuple (a1:d1 ... an:dn))) "
                            "(ai1 ... aik) ((bj1 (fun (rel (tuple (a1:d1"
                            " ... an:dn))) (_))) ... (bjl (fun (rel (tuple"
                            " (a1:d1 ... an:dn))) (_))))) -> (stream (tuple"
                            " (ai1:di1 ... aik:dik bj1 ... bjl)))</text--->"
                            "<text>_ groupby [list; funlist]</text--->"
                            "<text>Groups a relation according to attributes "
                            "ai1, ..., aik and feeds the groups to other "
                            "functions. The results of those functions are "
                            "appended to the grouping attributes. The empty "
                            "list is allowed for the grouping attributes (this "
                            "results in a single group with all input tuples)."
                            "</text--->"
                            "<text>query Employee feed sortby[DeptNr asc] "
                            "groupby[DeptNr; anz : group feed count] consume"
                            "</text--->"
                            ") )";

/*
2.21.6 Specification of operator ~slidingwindow~

*/
const string SlidingWindowSpec  = "( ( \"Signature\" \"Syntax\" \"Meaning\" "
                            "\"Example\" ) "
                            "( <text>((stream (tuple (a1:d1 ... an:dn))) "
                            "int int ((bj1 (fun (rel (tuple (a1:d1"
                            " ... an:dn))) (_))) ... (bjl (fun (rel (tuple"
                            " (a1:d1 ... an:dn))) (_))))) -> (stream (tuple"
                            " (bj1 ... bjl)))</text--->"
                            "<text>_ slidingwindow [int, int; funlist]"
                            "</text--->"
                            "<text>Apply a sliding window on a stream of tuples"
                            " grouping together the tuples within the window."
                            " The window size and the step size are arguments."
                            " The groups are fed to the functions. The result"
                            " tuples are constructed from the results of those"
                            " functions. If the stream has tuples less than the"
                            " window size, these tuples are considered one "
                            " group.</text--->"
                            "<text>query Employee feed sortby[DeptNr asc] "
                            "groupby[DeptNr; anz : group feed count] consume"
                            "</text--->"
                            ") )";


/*
2.21.4 Definition of operator ~groupby~

*/
Operator extrelgroupby (
         "groupby",             // name
         GroupBySpec,           // specification
         GroupByValueMapping,   // value mapping
         Operator::SimpleSelect,          // trivial selection function
         GroupByTypeMap         // type mapping
);

/*
2.21.5 Definition of operator ~slidingwindow~

*/
Operator extrelslidingwindow (
         "slidingwindow",             // name
         SlidingWindowSpec,           // specification
         SlidingWindowValueMapping,   // value mapping
         Operator::SimpleSelect,          // trivial selection function
         SlidingWindowTypeMap         // type mapping
);


/*
2.22 Operator ~aggregate~

2.22.1 Type mapping function of operator ~aggregate~

Type mapping for ~aggregate~ is

----    ((stream (tuple ((x1 T1) ... (xn Tn))))
          xi (map Ti Ti Tx) Tx                   ->

        (APPEND i Tx)
----

*/
ListExpr AggregateTypeMap( ListExpr args )
{
  if(nl->ListLength(args)!=4){
    return listutils::typeError("4 arguments expected");
  }

  ListExpr stream = nl->First(args);
  ListExpr attrNameList = nl->Second(args);
  ListExpr map = nl->Third(args);
  ListExpr defaultValue = nl->Fourth(args);

  if(!listutils::isTupleStream(stream)){
    return listutils::typeError("first argument must be a tuple stream");
  }
  if(nl->AtomType(attrNameList)!=SymbolType){
    return listutils::typeError("second argument is not a valid attr name");
  }
  if(!listutils::isMap<2>(map)){
    return listutils::typeError("third argument is not a map");
  }

  ListExpr attrType;
  string attrName = nl->SymbolValue(attrNameList);
  int j = listutils::findAttribute(nl->Second(nl->Second(stream)),
                                   attrName, attrType);
  if(j==0){
    return listutils::typeError("attrname " + attrName +
                                " not known in the tuple");
  }
  if(!nl->Equal(defaultValue,attrType)){
    return listutils::typeError("default value has another"
                                " type than the selected attribute");
  }

  if(nl->ListLength(map)!=4){
   return listutils::typeError("map has the wrong number of arguments");
  }

  ListExpr maparg1 = nl->Second(map);
  ListExpr maparg2 = nl->Third(map);
  ListExpr mapres  = nl->Fourth(map);
  string err = "stream(tuple([a1 : t1, ..., an : tn])) x "
               "ai x ( ti x ti -> ti) x ti expected";

  if(!nl->Equal(maparg1, maparg2) ||
     !nl->Equal(maparg1, mapres) ||
     !nl->Equal(attrType,maparg1)){
     return listutils::typeError(err);
  }

  return nl->ThreeElemList(
           nl->SymbolAtom( Symbol::APPEND() ),
           nl->OneElemList(nl->IntAtom(j)),
           defaultValue );
}

/*
2.18.2 Value mapping function of operator ~aggregate~

*/
#ifndef USE_PROGRESS


int Aggregate(Word* args, Word& result, int message, Word& local, Supplier s)
{
  // The argument vector contains the following values:
  // args[0] = stream of tuples
  // args[1] = attribute name
  // args[2] = mapping function
  // args[3] = zero value
  // args[4] = attribute index added by APPEND

  Word t( Address(0) );
  ArgVectorPointer vector;

  qp->Open(args[0].addr);
  int index = ((CcInt*)args[4].addr)->GetIntval();
  result = qp->ResultStorage(s);
  // Get the initial value
  Attribute* tmpres = ((Attribute*)args[3].addr)->Clone();
  qp->Request( args[0].addr, t );
  Word fctres;
  while( qp->Received( args[0].addr ) )
  {
    vector = qp->Argument(args[2].addr);
    ((*vector)[0]).setAddr(tmpres);
    ((*vector)[1]).setAddr(((Tuple*)t.addr)->GetAttribute( index-1 ) );
    qp->Request(args[2].addr, fctres);

    delete tmpres; //delete intermediate result
    tmpres = (Attribute*) fctres.addr;

    qp->ReInitResultStorage( args[2].addr );

    ((Tuple*)t.addr)->DeleteIfAllowed();
    qp->Request( args[0].addr, t );
  }

  ((Attribute*)result.addr)->
    CopyFrom( (const Attribute*)tmpres );

  delete tmpres;
  qp->Close(args[0].addr);

  return 0;
}
#else
int Aggregate(Word* args, Word& result, int message, Word& local, Supplier s)
{
  // The argument vector contains the following values:
  // args[0] = stream of tuples
  // args[1] = attribute name
  // args[2] = mapping function
  // args[3] = zero value
  // args[4] = attribute index added by APPEND

  Word t( Address(0) );
  ArgVectorPointer vector;
  ProgressLocalInfo* ali=0;
  ali = (ProgressLocalInfo*) local.addr;
  switch ( message )
  {

    case OPEN :
    case REQUEST:
    case CLOSE: {
      if(ali){
         delete ali;
      }
      ali = new ProgressLocalInfo();
      local.setAddr(ali);
      qp->Open(args[0].addr);
      int index = ((CcInt*)args[4].addr)->GetIntval();
      result = qp->ResultStorage(s);
      // Get the initial value
      Attribute* tmpres = ((Attribute*)args[3].addr)->Clone();
      qp->Request( args[0].addr, t );
      Word fctres;
      while( qp->Received( args[0].addr ) )
      {
        vector = qp->Argument(args[2].addr);
        ((*vector)[0]).setAddr(tmpres);
        ((*vector)[1]).setAddr(((Tuple*)t.addr)->GetAttribute( index-1 ) );
        qp->Request(args[2].addr, fctres);

        delete tmpres; //delete intermediate result
        tmpres = (Attribute*) fctres.addr;

        qp->ReInitResultStorage( args[2].addr );

        ((Tuple*)t.addr)->DeleteIfAllowed();
        qp->Request( args[0].addr, t );
        if (ali) ali->read++;
      }
      ((Attribute*)result.addr)->CopyFrom( (const Attribute*)tmpres );
      delete tmpres;
      qp->Close(args[0].addr);
      return 0;
    }
    case CLOSEPROGRESS:{
      if ( ali )        // if local info structure exists
      {
        delete ali;     // remove it
        local.setAddr(0); // and set adress to 0
      }
      return 0;
    }

    case REQUESTPROGRESS :
    {
      ProgressInfo p1;
      ProgressInfo *pRes;
      pRes = (ProgressInfo*) result.addr;
      double uAggregate=0.0121937626;
      if (qp->RequestProgress(args[0].addr, &p1))
      {
        pRes->Card=1;
        pRes->Time = p1.Time + p1.Card * uAggregate;
        pRes->Progress = (p1.Progress * p1.Time + ali->read * uAggregate)/
            pRes->Time;
        return YIELD;
      } else{
        return CANCEL;
      }
    }
  }
  return -1;
}

#endif


/* 
2.19 ~aggregateC~

This operator works similar to the ~aggregate~ operator. In constrast, the
aggregateC operator uses a function tuple x t -> t, t in 
DATA for aggregation.

2.19.1 Type Mapping

 
*/
ListExpr aggregateCTM(ListExpr args){



   string err = "stream(tuple(X)) x  (tuple(X) x t ->t ) x t"
                "t in DATA expected";
   if(!nl->HasLength(args,3)){
      return listutils::typeError(err);
   }       

   ListExpr stream = nl->First(args); 
   if(!Stream<Tuple>::checkType(stream)){
      return listutils::typeError(err);
   }       


   ListExpr start = nl->Third(args);
   if(!listutils::isDATA(start)){
      return listutils::typeError(err);
   }

   ListExpr map = nl->Second(args);
   if(!listutils::isMap<2>(map)){
       return listutils::typeError(err);
   }

   ListExpr t = nl->Second(stream);
   if(!nl->Equal(t, nl->Second(map))){
      return listutils::typeError("map argument 1 unequals to tuple in stream");
   }
   if(!nl->Equal(start,nl->Third(map))){
     return listutils::typeError("second map argument unequal to start value");
   } 

   if(!nl->Equal(start, nl->Fourth(map))){
      return listutils::typeError("result of function differs from "
                                  "start value");
   }


   return start;
}

/*
2.19.2 Value Mapping

*/

int aggregateCVM(Word* args, Word& result, int message,
               Word& local, Supplier s)
{


   Stream<Tuple> stream(args[0]);

   ArgVectorPointer funargs;

   result = qp->ResultStorage(s);
   Attribute* res = (Attribute*) result.addr;

   Attribute* tmp = ((Attribute*) args[2].addr)->Clone();

   stream.open();
   Tuple* nextTuple = stream.request();
   funargs= qp->Argument(args[1].addr);
   while(nextTuple){
     (*funargs)[0].setAddr(nextTuple);
     (*funargs)[1].setAddr(tmp);
     Word funres;
     qp->Request(args[1].addr, funres);
     tmp->DeleteIfAllowed();
     tmp = ((Attribute*) funres.addr)->Clone();
     nextTuple->DeleteIfAllowed();
     nextTuple = stream.request();
   }

   res->CopyFrom(tmp);
   tmp->DeleteIfAllowed();
   stream.close();
   return 0;
}


const string aggregateCSpec  = 
       "( ( \"Signature\" \"Syntax\" \"Meaning\" \"Example\" ) "
     "('stream(tuple(X)) x (tuple(X) x t -> t ) x t, t in DATA -> t'"
    "' _ aggregateC [ fun, t ] '"
    "' Aggregates values of a tuple stream'"
    "' query ten feed aggregateC[ fun( t : TUPLE, "
    "i : int) attr(t,no) + i ; 5] '"
    ") )";


Operator aggregateC (
         "aggregateC",              // name
         aggregateCSpec,            // specification
         aggregateCVM,                // value mapping
         Operator::SimpleSelect,          // trivial selection function
         aggregateCTM          // type mapping
);



/*
2.18.2 Value mapping function of operator ~aggregateB~

The ~aggregateB~ operator uses a stack to compute the aggregation
balanced. This will have advantages in geomatric aggregation.
It may also help to reduce numeric errors in aggregation using
double values.


2.18.2.1 ~StackEntry~

A stack entry consist of the level within the (simulated)
balanced tree and the corresponding value.
Note:
  The attributes at level 0 come directly from the input stream.
  We have to free them using the deleteIfAllowed function.
  On all other levels, the attribute is computes using the
  parameter function. Because this is outside of stream and
  tuples, no reference counting is available and we have to delete
  them using the usual delete function.

*/
struct AggrStackEntry
{
  inline AggrStackEntry(): level(-1),value(0)
  { }

  inline AggrStackEntry( long level, Attribute* value):
  level( level )
  { this->value = value;}

  inline AggrStackEntry( const AggrStackEntry& a ):
  level( a.level )
  { this->value = a.value;}

  inline AggrStackEntry& operator=( const AggrStackEntry& a )
  { level = a.level; value = a.value; return *this; }

  inline ~AggrStackEntry(){ } // use destroy !!


  inline void destroy(){
     if(level<0){
        return;
     }
     if(level==0){ // original from tuple
        value->DeleteIfAllowed();
     } else {
        delete value;
     }
     value = 0;
     level = -1;
  }

  long level;
  Attribute* value;
};

int AggregateB(Word* args, Word& result, int message,
               Word& local, Supplier s)
{
  // The argument vector contains the following values:
  // args[0] = stream of tuples
  // args[1] = attribute name
  // args[2] = mapping function
  // args[3] = zero value
  // args[4] = attribute index added by APPEND

  Word t1,resultWord;
  ArgVectorPointer vector = qp->Argument(args[2].addr);

  qp->Open(args[0].addr);
  result = qp->ResultStorage(s);
  int index = ((CcInt*)args[4].addr)->GetIntval();

  // read the first tuple
  qp->Request( args[0].addr, t1 );


  if( !qp->Received( args[0].addr ) ){
    // special case: stream is empty
    // use the third argument as result
    ((Attribute*)result.addr)->
      CopyFrom( (const Attribute*)args[3].addr );

  } else {
    // nonempty stream, consume it
    stack<AggrStackEntry> theStack;
    while( qp->Received(args[0].addr)){
       // get the attribute from the current tuple
       Attribute* attr = ((Tuple*)t1.addr)->GetAttribute(index-1)->Copy();
       // the tuple is not longer needed here
       ((Tuple*)t1.addr)->DeleteIfAllowed();
       // put the attribute on the stack merging with existing entries
       // while possible
       int level = 0;
       while(!theStack.empty() && level==theStack.top().level){
         // merging is possible
         AggrStackEntry top = theStack.top();
         theStack.pop();
         // call the parameter function
         ((*vector)[0]).setAddr(top.value);
         ((*vector)[1]).setAddr(attr);
         qp->Request(args[2].addr, resultWord);
         qp->ReInitResultStorage(args[2].addr);
         top.destroy();  // remove stack content
         if(level==0){ // delete attr;
            attr->DeleteIfAllowed();
         } else {
            delete attr;
         }
         attr = (Attribute*) resultWord.addr;
         level++;
       }
       AggrStackEntry entry(level,attr);
       theStack.push(entry);
       qp->Request(args[0].addr,t1);
   }
   // stream ends, merge stack elements regardless of their level
   assert(!theStack.empty()); // at least one element must be exist
   AggrStackEntry tmpResult = theStack.top();
   theStack.pop();
   while(!theStack.empty()){
     AggrStackEntry top = theStack.top();
     theStack.pop();
     ((*vector)[0]).setAddr(top.value);
     ((*vector)[1]).setAddr(tmpResult.value);
     qp->Request(args[2].addr, resultWord);
     qp->ReInitResultStorage(args[2].addr);
     tmpResult.destroy(); // destroy temporarly result
     tmpResult.level = 1; // mark as computed
     tmpResult.value = (Attribute*) resultWord.addr;
     top.destroy();
   }
   ((Attribute*)result.addr)->
                          CopyFrom((Attribute*)tmpResult.value);
   tmpResult.destroy();
  }
  // close input stream
  qp->Close(args[0].addr);
  return 0;

}



/*
2.18.3 Specification of operator ~aggregate~

*/
const string AggregateSpec  =
  "( ( \"Signature\" \"Syntax\" \"Meaning\" "
  "\"Example\" ) "
  "( <text>(stream(tuple((a1 t1) ... (an tn))) ai"
  "((ti ti) -> ti) ti -> ti</text--->"
  "<text>_ aggregate [ AttrName; fun; InitialValue]</text--->"
  "<text>Aggregates the values from the attribute 'AttrName'"
  "in the tuplestream using function 'fun' and starting "
  "with 'InitialValue' as first left argument for 'fun'. "
  "Returns 'InitialValue', if the stream is empty.</text--->"
  "<text>query ten feed aggregate[no; "
  "fun(i1: int, i2: int) i1+i2; 0]</text--->"
  ") )";

/*
2.18.3 Specification of operator ~aggregateB~

*/
const string AggregateBSpec  =
  "( ( \"Signature\" \"Syntax\" \"Meaning\" "
  "\"Example\" ) "
  "( <text>(stream(tuple((a1 t1) ... (an tn))) ai"
  "((ti ti) -> ti) ti -> ti</text--->"
  "<text>_ aggregateB [ AttrName ; fun; ZeroValue ]</text--->"
  "<text>Aggregates the values of attribute 'AttrName' "
  "from the tuple stream using the function 'fun' in a balanced "
  "fashion. Returns 'ZeroValue', if the stream is empty.</text--->"
  "<text>query ten feed aggregateB[no; "
  "fun(i1: int, i2: int) i1+i2; 0]</text--->"
  ") )";

/*
2.18.4 Definition of operator ~aggregate~

*/
Operator extrelaggregate (
         "aggregate",              // name
         AggregateSpec,            // specification
         Aggregate,                // value mapping
         Operator::SimpleSelect,          // trivial selection function
         AggregateTypeMap          // type mapping
);

/*
2.18.4 Definition of operator ~aggregate\_new~

*/
Operator extrelaggregateB (
         "aggregateB",              // name
         AggregateBSpec,            // specification
         AggregateB,                // value mapping
         Operator::SimpleSelect,          // trivial selection function
         AggregateTypeMap          // type mapping
);

/*

5.10 Operator ~symmjoin~

5.10.1 Type mapping function of operator ~symmjoin~

Result type of symmjoin operation.

----    ((stream (tuple (x1 ... xn))) (stream (tuple (y1 ... ym)))
        (map tuple tuple bool)

        -> (stream (tuple (x1 ... xn y1 ... ym)))
----

*/
ListExpr SymmJoinTypeMap(ListExpr args)
{
  if(nl->ListLength(args)!=3){
    return listutils::typeError("three arguments expected");
  }
  ListExpr stream1 = nl->First(args);
  ListExpr stream2 = nl->Second(args);
  ListExpr map = nl->Third(args);

  string err = "stream(tuple1) x stream(tuple2) x "
               "( tuple1 x tuple2 -> bool) expected";
  if(!listutils::isTupleStream(stream1) ||
     !listutils::isTupleStream(stream2) ||
     !listutils::isMap<2>(map)){
    return listutils::typeError(err);
  }

  if(!nl->Equal(nl->Second(stream1), nl->Second(map)) ||
     !nl->Equal(nl->Second(stream2), nl->Third(map)) ||
     !listutils::isSymbol(nl->Fourth(map),CcBool::BasicType())){
    return listutils::typeError(err +"(wrong mapping)");
  }

  ListExpr a1List = nl->Second(nl->Second(stream1));
  ListExpr a2List = nl->Second(nl->Second(stream2));

  if(!listutils::disjointAttrNames(a1List,a2List)){
    return listutils::typeError(err + "(name conflict in tuples");
  }
  ListExpr list = ConcatLists(a1List, a2List);

  return nl->TwoElemList(nl->SymbolAtom(Symbol::STREAM()),
           nl->TwoElemList(nl->SymbolAtom(Tuple::BasicType()),
             list));
}
/*

5.10.2 Value mapping function of operator ~symmjoin~

*/

#ifndef USE_PROGRESS

// standard version


struct SymmJoinLocalInfo
{
  TupleType *resultTupleType;

  TupleBuffer *rightRel;
  GenericRelationIterator *rightIter;
  TupleBuffer *leftRel;
  GenericRelationIterator *leftIter;
  bool right;
  Tuple *currTuple;
  bool rightFinished;
  bool leftFinished;
};

int
SymmJoin(Word* args, Word& result, int message, Word& local, Supplier s)
{
  Word r, l;
  SymmJoinLocalInfo* pli;

  TupleType* tt = (TupleType*) qp->GetLocal2(s).addr;

  switch (message)
  {
    case INIT: {
       tt = new TupleType(nl->Second(GetResultTupleType(s)));
       qp->GetLocal2(s).addr=tt;
       return 0;
    }

    case FINISH: {
       if(tt){
         tt->DeleteIfAllowed();
         qp->GetLocal2(s).addr=0;
       }
       return 0;
    }


    case OPEN :
    {
      long MAX_MEMORY = (qp->GetMemorySize(s) * 1024 * 1024);
      cmsg.info("ERA:ShowMemInfo") << "SymmJoin.MAX_MEMORY ("
                                   << MAX_MEMORY/1024 << " kB): " << endl;
      cmsg.send();
      pli = new SymmJoinLocalInfo;
      pli->rightRel = new TupleBuffer( MAX_MEMORY / 2 );
      pli->rightIter = 0;
      pli->leftRel = new TupleBuffer( MAX_MEMORY / 2 );
      pli->leftIter = 0;
      pli->right = true;
      pli->currTuple = 0;
      pli->rightFinished = false;
      pli->leftFinished = false;

      tt->IncReference();
      pli->resultTupleType = tt;

      qp->Open(args[0].addr);
      qp->Open(args[1].addr);

      local.setAddr(pli);
      return 0;
    }
    case REQUEST :
    {
      pli = (SymmJoinLocalInfo*)local.addr;

      while( 1 )
        // This loop will end in some of the returns.
      {
        if( pli->right )
          // Get the tuple from the right stream and match it with the
          // left stored buffer
        {
          if( pli->currTuple == 0 )
          {
            qp->Request(args[0].addr, r);
            if( qp->Received( args[0].addr ) )
            {
              pli->currTuple = (Tuple*)r.addr;
              pli->leftIter = pli->leftRel->MakeScan();
            }
            else
            {
              pli->rightFinished = true;
              if( pli->leftFinished )
                return CANCEL;
              else
              {
                pli->right = false;
                continue; // Go back to the loop
              }
            }
          }

          // Now we have a tuple from the right stream in currTuple
          // and an open iterator on the left stored buffer.
          Tuple *leftTuple = pli->leftIter->GetNextTuple();

          if( leftTuple == 0 )
            // There are no more tuples in the left iterator. We then
            // store the current tuple in the right buffer and close the
            // left iterator.
          {
            if( !pli->leftFinished )
              // We only need to keep track of the right tuples if the
              // left stream is not finished.
            {
              pli->rightRel->AppendTuple( pli->currTuple );
              pli->right = false;
            }

            pli->currTuple->DeleteIfAllowed();
            pli->currTuple = 0;

            delete pli->leftIter;
            pli->leftIter = 0;

            continue; // Go back to the loop
          }
          else
            // We match the tuples.
          {
            ArgVectorPointer funArgs = qp->Argument(args[2].addr);
            ((*funArgs)[0]).setAddr( pli->currTuple );
            ((*funArgs)[1]).setAddr( leftTuple );
            Word funResult;
            qp->Request(args[2].addr, funResult);
            CcBool *boolFunResult = (CcBool*)funResult.addr;

            if( boolFunResult->IsDefined() &&
                boolFunResult->GetBoolval() )
            {
              Tuple *resultTuple = new Tuple( pli->resultTupleType );
              Concat( pli->currTuple, leftTuple, resultTuple );
              leftTuple->DeleteIfAllowed();
              leftTuple = 0;
              result.setAddr( resultTuple );
              return YIELD;
            }
            else
            {
              leftTuple->DeleteIfAllowed();
              leftTuple = 0;
              continue; // Go back to the loop
            }
          }
        }
        else
          // Get the tuple from the left stream and match it with the
          // right stored buffer
        {
          if( pli->currTuple == 0 )
          {
            qp->Request(args[1].addr, l);
            if( qp->Received( args[1].addr ) )
            {
              pli->currTuple = (Tuple*)l.addr;
              pli->rightIter = pli->rightRel->MakeScan();
            }
            else
            {
              pli->leftFinished = true;
              if( pli->rightFinished )
                return CANCEL;
              else
              {
                pli->right = true;
                continue; // Go back to the loop
              }
            }
          }

          // Now we have a tuple from the left stream in currTuple and
          // an open iterator on the right stored buffer.
          Tuple *rightTuple = pli->rightIter->GetNextTuple();

          if( rightTuple == 0 )
            // There are no more tuples in the right iterator. We then
            // store the current tuple in the left buffer and close
            // the right iterator.
          {
            if( !pli->rightFinished )
              // We only need to keep track of the left tuples if the
              // right stream is not finished.
            {
              pli->leftRel->AppendTuple( pli->currTuple );
              pli->right = true;
            }

            pli->currTuple->DeleteIfAllowed();
            pli->currTuple = 0;

            delete pli->rightIter;
            pli->rightIter = 0;

            continue; // Go back to the loop
          }
          else
            // We match the tuples.
          {
            ArgVectorPointer funArgs = qp->Argument(args[2].addr);
            ((*funArgs)[0]).setAddr( rightTuple );
            ((*funArgs)[1]).setAddr( pli->currTuple );
            Word funResult;
            qp->Request(args[2].addr, funResult);
            CcBool *boolFunResult = (CcBool*)funResult.addr;

            if( boolFunResult->IsDefined() &&
                boolFunResult->GetBoolval() )
            {
              Tuple *resultTuple = new Tuple( pli->resultTupleType );
              Concat( rightTuple, pli->currTuple, resultTuple );
              rightTuple->DeleteIfAllowed();
              rightTuple = 0;
              result.setAddr( resultTuple );
              return YIELD;
            }
            else
            {

              rightTuple->DeleteIfAllowed();
              rightTuple = 0;
              continue; // Go back to the loop
            }
          }
        }
      }
    }
    case CLOSE :
    {
      pli = (SymmJoinLocalInfo*)local.addr;
      if(pli)
      {
         if( pli->currTuple != 0 )
           pli->currTuple->DeleteIfAllowed();

         delete pli->leftIter;
         delete pli->rightIter;
         if( pli->resultTupleType != 0 )
           pli->resultTupleType->DeleteIfAllowed();

         if( pli->rightRel != 0 )
         {
           pli->rightRel->Clear();
           delete pli->rightRel;
         }

         if( pli->leftRel != 0 )
         {
           pli->leftRel->Clear();
           delete pli->leftRel;
         }

         delete pli;
         local.setAddr(0);
      }

      qp->Close(args[0].addr);
      qp->Close(args[1].addr);

      return 0;
    }
  }
  return 0;
}


#else

// with support for progress queries

class SymmJoinLocalInfo: public ProgressLocalInfo
{
public:

   SymmJoinLocalInfo(): resultTupleType(0), rightRel(0),
       rightIter(0),leftRel(0),leftIter(0),right(false),
       currTuple(0),rightFinished(false),leftFinished(false),
       costEstimation(0) {}

   ~SymmJoinLocalInfo() {
      if(resultTupleType){
        resultTupleType->DeleteIfAllowed();
        resultTupleType =0;
      }
      if(currTuple){
        currTuple->DeleteIfAllowed();
        currTuple =0;
      }
      if(rightRel){
        delete rightRel;
        rightRel=0;
      }
      if(rightIter){
        delete rightIter;
        rightIter=0;
      }
      if(leftRel){
        delete leftRel;
        leftRel=0;
      }
      if(leftIter){
        delete leftIter;
        leftIter=0;
      }
   }

  TupleType *resultTupleType;

  TupleBuffer *rightRel;
  GenericRelationIterator *rightIter;
  TupleBuffer *leftRel;
  GenericRelationIterator *leftIter;
  bool right;
  Tuple *currTuple;
  bool rightFinished;
  bool leftFinished;
  SymmjoinCostEstimation *costEstimation;
};

CostEstimation* SymmjoinCostEstimationFunc() {
  return new SymmjoinCostEstimation();
}

int
SymmJoin(Word* args, Word& result, int message, Word& local, Supplier s)
{
  Word r, l;
  SymmJoinLocalInfo* pli;

  pli = (SymmJoinLocalInfo*) local.addr;
  TupleType* tt = (TupleType*) qp->GetLocal2(s).addr;

  switch (message)
  {
    case INIT: {
      tt = new TupleType(nl->Second(GetTupleResultType(s)));
      qp->GetLocal2(s).addr=tt;
      return 0;
    }
    case FINISH: {
      if(tt){
        tt->DeleteIfAllowed();
        qp->GetLocal2(s).addr=0;
      }
      return 0;
    }

    case OPEN :
    {

      long MAX_MEMORY = (qp->GetMemorySize(s) * 1024 * 1024);
      cmsg.info("ERA:ShowMemInfo") << "SymmJoin.MAX_MEMORY ("
                                   << MAX_MEMORY/1024 << " MB): " << endl;
      cmsg.send();


      if ( pli ){
         delete pli;
      }
      pli = new SymmJoinLocalInfo;
      pli->rightRel = new TupleBuffer( MAX_MEMORY / 2 );
      pli->rightIter = 0;
      pli->leftRel = new TupleBuffer( MAX_MEMORY / 2 );
      pli->leftIter = 0;
      pli->right = true;
      pli->currTuple = 0;
      pli->rightFinished = false;
      pli->leftFinished = false;

      pli -> costEstimation =
         (SymmjoinCostEstimation*) qp->getCostEstimation(s);
      pli -> costEstimation -> init(NULL, NULL);
      tt->IncReference();
      pli->resultTupleType =tt;
      pli->readFirst = 0;
      pli->readSecond = 0;
      pli->returned = 0;
      local.setAddr(pli);

      qp->Open(args[0].addr);
      qp->Open(args[1].addr);

      return 0;
    }

    case REQUEST :
    {
      if(!pli){
        return CANCEL;
      }
      while( true )
        // This loop will end in some of the returns.
      {
        if( pli->right )
          // Get the tuple from the right stream and match it with the
          // left stored buffer
        {
          if( pli->currTuple == 0 )
          {
            qp->Request(args[0].addr, r);
            if( qp->Received( args[0].addr ) )
            {
              pli->currTuple = (Tuple*)r.addr;
              pli->leftIter = pli->leftRel->MakeScan();
              pli->readFirst++;
              pli->costEstimation->processedTupleInStream2();
              pli->costEstimation->setTupleBuffer2OnDisk(
                                           ! pli->rightRel->InMemory());
            }
            else
            {
              pli->rightFinished = true;
              pli->costEstimation-> setStream2Exhausted(true);
              if( pli->leftFinished ) {
                return CANCEL;
              } else {
                pli->right = false;
                continue; // Go back to the loop
              }
            }
          }

          // Now we have a tuple from the right stream in currTuple
          // and an open iterator on the left stored buffer.
          Tuple *leftTuple = pli->leftIter->GetNextTuple();

          if( leftTuple == 0 )
            // There are no more tuples in the left iterator. We then
            // store the current tuple in the right buffer and close the
            // left iterator.
          {
            if( !pli->leftFinished )
              // We only need to keep track of the right tuples if the
              // left stream is not finished.
            {
              pli->rightRel->AppendTuple( pli->currTuple );
              pli->right = false;
            }

            pli->currTuple->DeleteIfAllowed();
            pli->currTuple = 0;

            delete pli->leftIter;
            pli->leftIter = 0;

            continue; // Go back to the loop
          }
          else
            // We match the tuples.
          {
            ArgVectorPointer funArgs = qp->Argument(args[2].addr);
            ((*funArgs)[0]).setAddr( pli->currTuple );
            ((*funArgs)[1]).setAddr( leftTuple );
            Word funResult;
            qp->Request(args[2].addr, funResult);
            CcBool *boolFunResult = (CcBool*)funResult.addr;

            if( boolFunResult->IsDefined() &&
                boolFunResult->GetBoolval() )
            {
              Tuple *resultTuple = new Tuple( pli->resultTupleType );
              Concat( pli->currTuple, leftTuple, resultTuple );
              leftTuple->DeleteIfAllowed();
              leftTuple = 0;
              result.setAddr( resultTuple );
              pli->returned++;
              return YIELD;
            }
            else
            {
              leftTuple->DeleteIfAllowed();
              leftTuple = 0;
              continue; // Go back to the loop
            }
          }
        }
        else
          // Get the tuple from the left stream and match it with the
          // right stored buffer
        {
          if( pli->currTuple == 0 )
          {
            qp->Request(args[1].addr, l);
            if( qp->Received( args[1].addr ) )
            {
              pli->currTuple = (Tuple*)l.addr;
              pli->rightIter = pli->rightRel->MakeScan();
              pli->readSecond++;
              pli->costEstimation->processedTupleInStream1();
              pli->costEstimation->setTupleBuffer1OnDisk(
                                         ! pli->leftRel->InMemory());
            }
            else
            {
              pli->leftFinished = true;
              pli->costEstimation->setStream1Exhausted(true);
              if( pli->rightFinished ) {
                return CANCEL;
             } else {
                pli->right = true;
                continue; // Go back to the loop
              }
            }
          }

          // Now we have a tuple from the left stream in currTuple and
          // an open iterator on the right stored buffer.
          Tuple *rightTuple = pli->rightIter->GetNextTuple();

          if( rightTuple == 0 )
            // There are no more tuples in the right iterator. We then
            // store the current tuple in the left buffer and close
            // the right iterator.
          {
            if( !pli->rightFinished )
              // We only need to keep track of the left tuples if the
              // right stream is not finished.
            {
              pli->leftRel->AppendTuple( pli->currTuple );
              pli->right = true;
            }

            pli->currTuple->DeleteIfAllowed();
            pli->currTuple = 0;

            delete pli->rightIter;
            pli->rightIter = 0;

            continue; // Go back to the loop
          }
          else
            // We match the tuples.
          {
            ArgVectorPointer funArgs = qp->Argument(args[2].addr);
            ((*funArgs)[0]).setAddr( rightTuple );
            ((*funArgs)[1]).setAddr( pli->currTuple );
            Word funResult;
            qp->Request(args[2].addr, funResult);
            CcBool *boolFunResult = (CcBool*)funResult.addr;

            if( boolFunResult->IsDefined() &&
                boolFunResult->GetBoolval() )
            {
              Tuple *resultTuple = new Tuple( pli->resultTupleType );
              Concat( rightTuple, pli->currTuple, resultTuple );
              rightTuple->DeleteIfAllowed();
              rightTuple = 0;
              result.setAddr( resultTuple );
              pli->returned++;
              return YIELD;
            }
            else
            {
              rightTuple->DeleteIfAllowed();
              rightTuple = 0;
              continue; // Go back to the loop
            }
          }
        }
      }
    }
    case CLOSE :
    {
      if(pli)
      {
        if( pli->currTuple != 0 ){
          pli->currTuple->DeleteIfAllowed();
          pli->currTuple=0;
        }

        if(pli->leftIter){
          delete pli->leftIter;
          pli->leftIter = 0;
        }
        if(pli->rightIter){
           delete pli->rightIter;
           pli->rightIter=0;
        }
        if( pli->resultTupleType != 0 ){
          pli->resultTupleType->DeleteIfAllowed();
          pli->resultTupleType=0;
        }

        if( pli->rightRel != 0 )
        {
          pli->rightRel->Clear();
          delete pli->rightRel;
          pli->rightRel=0;
        }

        if( pli->leftRel != 0 )
        {
          pli->leftRel->Clear();
          delete pli->leftRel;
          pli->leftRel=0;
        }
         
       delete pli;
       local.setAddr(0);
      }

      qp->Close(args[0].addr);
      qp->Close(args[1].addr);

      return 0;
    }
  }
  return 0;
}

#endif



/*

5.10.3 Specification of operator ~symmjoin~

*/
const string SymmJoinSpec  =
  "( ( \"Signature\" \"Syntax\" \"Meaning\" "
  "\"Example\" ) "
  "( <text>((stream (tuple (x1 ... xn))) (stream "
  "(tuple (y1 ... ym)))) (map (tuple (x1 ... xn)) "
  "(tuple (y1 ... ym)) -> bool) -> (stream (tuple (x1 "
  "... xn y1 ... ym)))</text--->"
  "<text>_ _ symmjoin[ fun ]</text--->"
  "<text>Computes a Cartesian product stream from "
  "its two argument streams filtering by the third "
  "argument.</text--->"
  "<text>query ten feed {a} twenty feed {b} "
  "symmjoin[.no_a = .no_b] count</text--->"
  " ) )";

/*

5.10.4 Definition of operator ~symmjoin~

*/

#ifndef USE_PROGRESS
Operator extrelsymmjoin (
         "symmjoin",                // name
         SymmJoinSpec,              // specification
         SymmJoin,                  // value mapping
         Operator::SimpleSelect,    // trivial selection function
         SymmJoinTypeMap            // type mapping
//         true                     // needs large amounts of memory
);
#else
Operator extrelsymmjoin (
         "symmjoin",                // name
         SymmJoinSpec,              // specification
         SymmJoin,                  // value mapping
         Operator::SimpleSelect,    // trivial selection function
         SymmJoinTypeMap,           // type mapping
         SymmjoinCostEstimationFunc // CostEstimation
//         true                     // needs large amounts of memory
);
#endif



/*

5.10.5 Operator ~symmproductextend~

02.06.2006 Christian Duentgen

In queries, it is advantageous to avoid the multiple evaluation of common
subexpressions (CSE). One way is to evaluate a CSE when it is needed for
the first time, and append a new attribute with the CSE's value to the tuple,
using operator ~extend~. When a CSE first appears within a join condition, and
the CSE depends on attributes from both input streams to join, this method
fails. To cope with such situations and optimize the benefit of CSE
substitution, we implement the ~symmproductextend~ operator.

The operator has three arguments, the first and second being the input tuple
streams. The third argument is a function list (i.e. a list of pairs
(attributename function), where ~attributename~ is a new identifier and
~function~ is a function calculating the new attribute's value from a pair
of tuples, one taken from each input tuple stream.

Subsequent to a ~symmproductextend~, a ~filter~ can be applied, e.g. to express
a ``join condition'' using the attributes extended (representing CSEs).

The operator works similar to

---- stream(tuple(X)) stream(tuple(Y)) symmjoin[TRUE] extend[ funlist ].
----

5.10.5.1 Typemapping for operator ~symmproductextend~

----
    (stream (tuple (A)))
  x (stream (tuple (B)))
  x ( ( c1 (map tuple(A) tuple(B) tc1))
          ...

      ( ck (map tuple(A) tuple(B) tck)))
 -> (stream (tuple(A*B*C)))

where A = (ta1 a1) ... (tan an)
      B = (tb1 b1) ... (tbm bm)
      C = (tc1 c1) ... (tck ck)
----

*/

/*
Typemapping operator for operator ~symmproductextend~

*/

ListExpr SymmProductExtendTypeMap(ListExpr args)
{
  if(nl->ListLength(args)!=3){
    return listutils::typeError("three arguments expected");
  }

  ListExpr stream1 = nl->First(args);
  ListExpr stream2 = nl->Second(args);
  ListExpr mapList = nl->Third(args);

  string err = "stream(tupleA) x stream(tupleB) x "
                "((name (tupleA x tupleB -> DATA))+) expected";

  if(!listutils::isTupleStream(stream1) ||
     !listutils::isTupleStream(stream2) ||
     nl->IsAtom(mapList)){
    return listutils::typeError(err);
  }


  ListExpr TType1 = nl->Second(stream1);
  ListExpr TType2 = nl->Second(stream2);

  ListExpr attrL1 = nl->Second(TType1);
  // copy all attributes from the first list
  ListExpr newAttrList = nl->OneElemList(nl->First(attrL1));
  ListExpr lastAttrList = newAttrList;
  attrL1 = nl->Rest(attrL1);
  while(!nl->IsEmpty(attrL1)){
    lastAttrList = nl->Append(lastAttrList,nl->First(attrL1));
    attrL1 = nl->Rest(attrL1);
  }
  // copy the attributes of the second tuple stream
  ListExpr attrL2 = nl->Second(TType2);
  while(!nl->IsEmpty(attrL2)){
   lastAttrList = nl->Append(lastAttrList, nl->First(attrL2));
   attrL2 = nl->Rest(attrL2);
  }

  // for each mapping

  while(!nl->IsEmpty(mapList)){
    ListExpr namedMap = nl->First(mapList);
    mapList = nl->Rest(mapList);

    if(nl->ListLength(namedMap)!=2){
      return listutils::typeError(err +"( invalid named map found)");
    }

    if(nl->AtomType(nl->First(namedMap))!=SymbolType){
      return listutils::typeError(err+ "(invalid attribute name found");
    }
    ListExpr map = nl->Second(namedMap);

    if(!listutils::isMap<2>(map)){
     return listutils::typeError(err+" ( found a incorrect map in list)");
    }
    if(!nl->Equal(TType1, nl->Second(map)) ||
       !nl->Equal(TType2, nl->Third(map))){
      return listutils::typeError(err +"( ivalid argument in mapping)");
    }
    lastAttrList = nl->Append(lastAttrList,
                              nl->TwoElemList(nl->First(namedMap),
                                              nl->Fourth(map)));
  }

  if(!listutils::isAttrList(newAttrList)){
    return listutils::typeError(err + " ( error in result stream: "
                               "conflicting names or non-DATA attributes)");
  }

  return nl->TwoElemList(nl->SymbolAtom(Symbol::STREAM()),
                         nl->TwoElemList(nl->SymbolAtom(Tuple::BasicType()),
                                         newAttrList));

}


/*
5.10.5.2 Value mapping function of operator ~symmproductextend~

*/
struct SymmProductExtendLocalInfo
{
  TupleType *resultTupleType;
  TupleBuffer *rightRel;
  GenericRelationIterator *rightIter;
  TupleBuffer *leftRel;
  GenericRelationIterator *leftIter;
  bool right;
  Tuple *currTuple;
  bool rightFinished;
  bool leftFinished;
};

int
SymmProductExtend(Word* args, Word& result,
                              int message, Word& local, Supplier s)
{
  Word r, l, value;
  SymmProductExtendLocalInfo* pli;
  int i, nooffun;
  Supplier supplier, supplier2, supplier3;
  ArgVectorPointer extFunArgs;
  Tuple *resultTuple;

  TupleType* tt = (TupleType*) qp->GetLocal2(s).addr;

  switch (message)
  {

    case INIT: {
      tt = new TupleType(nl->Second(GetTupleResultType(s)));
      qp->GetLocal2(s).addr=tt;
      return 0;
    }
    case FINISH : {
      if(tt){
        tt->DeleteIfAllowed();
        qp->GetLocal2(s).addr=0;
      }
      return 0;
    }


    case OPEN :
    {
      long MAX_MEMORY = (qp->GetMemorySize(s) * 1024 * 1024);
      cmsg.info("ERA:ShowMemInfo") << "SymmProductExtend.MAX_MEMORY ("
                                   << MAX_MEMORY/1024 << " MB): " << endl;
      cmsg.send();
      pli = new SymmProductExtendLocalInfo;
      pli->rightRel = new TupleBuffer( MAX_MEMORY / 2 );
      pli->rightIter = 0;
      pli->leftRel = new TupleBuffer( MAX_MEMORY / 2 );
      pli->leftIter = 0;
      pli->right = true;
      pli->currTuple = 0;
      pli->rightFinished = false;
      pli->leftFinished = false;

      tt->IncReference();
      pli->resultTupleType = tt;
      qp->Open(args[0].addr);
      qp->Open(args[1].addr);

      local.setAddr(pli);
      return 0;
    }
    case REQUEST :
    {
      pli = (SymmProductExtendLocalInfo*)local.addr;

      while( 1 )
        // This loop will end in some of the returns.
      {
        if( pli->right )
          // Get the tuple from the right stream and match it with the
          // left stored buffer
        {
          if( pli->currTuple == 0 )
          {
            qp->Request(args[1].addr, r);
            if( qp->Received( args[1].addr ) )
            {
              pli->currTuple = (Tuple*)r.addr;
              pli->leftIter = pli->leftRel->MakeScan();
            }
            else
            {
              pli->rightFinished = true;
              if( pli->leftFinished )
                return CANCEL;
              else
              {
                pli->right = false;
                continue; // Go back to the loop
              }
            }
          }

          // Now we have a tuple from the right stream in currTuple
          // and an open iterator on the left stored buffer.
          Tuple *leftTuple = pli->leftIter->GetNextTuple();

          if( leftTuple == 0 )
            // There are no more tuples in the left iterator. We then
            // store the current tuple in the right buffer and close the
            // left iterator.
          {
            if( !pli->leftFinished )
              // We only need to keep track of the right tuples if the
              // left stream is not finished.
            {
              pli->rightRel->AppendTuple( pli->currTuple );
              pli->right = false;
            }

            pli->currTuple->DeleteIfAllowed();
            pli->currTuple = 0;

            delete pli->leftIter;
            pli->leftIter = 0;

            continue; // Go back to the loop
          }
          else
            // We match the tuples.
          {
            // XRIS: We must calculate the new attributes' values here
            // and extend currTuple with them

            resultTuple = new Tuple( pli->resultTupleType );
            Concat( leftTuple, pli->currTuple, resultTuple );

            supplier = args[2].addr;           // get list of ext-functions
            nooffun = qp->GetNoSons(supplier); // get num of ext-functions
            for(i = 0; i< nooffun; i++)
            {
              supplier2 = qp->GetSupplier(supplier, i); // get an ext-function
              //noofsons = qp->GetNoSons(supplier2);
              supplier3 = qp->GetSupplier(supplier2, 1);
              extFunArgs = qp->Argument(supplier3);
              ((*extFunArgs)[0]).setAddr(leftTuple);     // pass first argument
              ((*extFunArgs)[1]).setAddr(pli->currTuple);// pass second argument
              qp->Request(supplier3,value);              // call extattr mapping
//  The original implementation tried to avoid copying the function result,
//  but somehow, this results in a strongly growing tuplebuffer on disk:
//               resultTuple->PutAttribute(
//                 leftTuple->GetNoAttributes()
//                 + pli->currTuple->GetNoAttributes()
//                 + i, (Attribute*)value.addr);
//               qp->ReInitResultStorage( supplier3 );
              resultTuple->PutAttribute(
                leftTuple->GetNoAttributes()
                + pli->currTuple->GetNoAttributes()
                + i, ((Attribute*)value.addr)->Clone() );
            }
            leftTuple->DeleteIfAllowed();
            leftTuple = 0;
            result.setAddr( resultTuple );
            return YIELD;
          }
        }
        else
          // Get the tuple from the left stream and match it with the
          // right stored buffer
        {
          if( pli->currTuple == 0 )
          {
            qp->Request(args[0].addr, l);
            if( qp->Received( args[0].addr ) )
            {
              pli->currTuple = (Tuple*)l.addr;
              pli->rightIter = pli->rightRel->MakeScan();
            }
            else
            {
              pli->leftFinished = true;
              if( pli->rightFinished )
                return CANCEL;
              else
              {
                pli->right = true;
                continue; // Go back to the loop
              }
            }
          }

          // Now we have a tuple from the left stream in currTuple and
          // an open iterator on the right stored buffer.
          Tuple *rightTuple = pli->rightIter->GetNextTuple();

          if( rightTuple == 0 )
            // There are no more tuples in the right iterator. We then
            // store the current tuple in the left buffer and close
            // the right iterator.
          {
            if( !pli->rightFinished )
              // We only need to keep track of the left tuples if the
              // right stream is not finished.
            {
              pli->leftRel->AppendTuple( pli->currTuple );
              pli->right = true;
            }

            pli->currTuple->DeleteIfAllowed();
            pli->currTuple = 0;

            delete pli->rightIter;
            pli->rightIter = 0;

            continue; // Go back to the loop
          }
          else
            // We match the tuples.
          {
            // XRIS: We must calculate the new attribute's values here
            // and extend rightTuple
            resultTuple = new Tuple( pli->resultTupleType );
            Concat( pli->currTuple, rightTuple, resultTuple );
            supplier = args[2].addr;           // get list of ext-functions
            nooffun = qp->GetNoSons(supplier); // get num of ext-functions
            for(i = 0; i< nooffun; i++)
            {
              supplier2 = qp->GetSupplier(supplier, i); // get an ext-function
              //noofsons = qp->GetNoSons(supplier2);
              supplier3 = qp->GetSupplier(supplier2, 1);
              extFunArgs = qp->Argument(supplier3);
              ((*extFunArgs)[0]).setAddr(pli->currTuple);// pass 1st argument
              ((*extFunArgs)[1]).setAddr(rightTuple);    // pass 2nd argument
              qp->Request(supplier3,value);              // call extattr mapping
//  The original implementation tried to avoid copying the function result,
//  but somehow, this results in a strongly growing tuplebuffer on disk:
//               resultTuple->PutAttribute(
//                 pli->currTuple->GetNoAttributes()
//                 + rightTuple->GetNoAttributes()
//                 + i, (Attribute*)value.addr);
//               // extend effective left tuple
//               qp->ReInitResultStorage( supplier3 );

              resultTuple->PutAttribute(
                pli->currTuple->GetNoAttributes()
                + rightTuple->GetNoAttributes()
                + i, ((Attribute*)value.addr)->Clone() );
              // extend effective left tuple
            }
            rightTuple->DeleteIfAllowed();
            rightTuple = 0;
            result.setAddr( resultTuple );
            return YIELD;
          }
        }
      }
    }
    case CLOSE :
    {
      pli = (SymmProductExtendLocalInfo*)local.addr;
      if(pli)
      {
        if( pli->currTuple != 0 )
          pli->currTuple->DeleteIfAllowed();

        delete pli->leftIter;
        delete pli->rightIter;
        if( pli->resultTupleType != 0 )
          pli->resultTupleType->DeleteIfAllowed();

        if( pli->rightRel != 0 )
        {
          pli->rightRel->Clear();
          delete pli->rightRel;
        }

        if( pli->leftRel != 0 )
        {
          pli->leftRel->Clear();
          delete pli->leftRel;
        }

        delete pli;
        local.setAddr(0);
      }

      qp->Close(args[0].addr);
      qp->Close(args[1].addr);

      return 0;
    }
  }
  return 0;
}




/*

5.10.5.3 Specification of operator ~symmproductextend~

*/
const string SymmProductExtendSpec  =
  "( ( \"Signature\" \"Syntax\" \"Meaning\" "
  "\"Example\" ) "
  "( <text>(stream (tuple(X))) (stream (tuple(Y))) "
  " [(z1, (tuple(X) tuple(Y) -> t1)) "
  "... (zj, (tuple(X) tuple(Y) -> tj))]  -> (stream (tuple(X*Y*Z))) "
  "))</text--->"
  "<text>_ _ symmproductextend[ funlist ]</text--->"
  "<text>Computes a Cartesian product stream from "
  "its two argument streams, extending it with "
  "new attributes respecting mapping rules passed as "
  "third argument.</text--->"
  "<text>query ten feed {a} ten feed {b} "
  "symmproductextend[ [prod: (.no_a * ..no_b)] ] "
  "count</text--->"
  " ) )";

/*

5.10.5.4 Definition of operator ~symmproductextend~

*/
Operator extrelsymmproductextend (
         "symmproductextend",     // name
         SymmProductExtendSpec,   // specification
         SymmProductExtend,       // value mapping
         Operator::SimpleSelect,  // trivial selection function
         SymmProductExtendTypeMap // type mapping
//         true                   // needs large amounts of memory
);


/*

5.10.6 Operator ~symmproduct~

This operator calculates the cartesian product of two tuplestreams in 
a symmetrical and non-blocking manner. It behaves like

---- _ _ symmjoin [ TRUE ]
----
but does not evaluate a predictate.

5.10.6.1 Typemapping for operator ~symmproduct~

----
    (stream (tuple (A)))
  x (stream (tuple (B)))
 -> (stream (tuple(A*B)))

where A = (ta1 a1) ... (tan an)
      B = (tb1 b1) ... (tbm bm)
----

*/

/*
Typemapping operator for operator ~symmproduct~

*/

ListExpr SymmProductTypeMap(ListExpr args)
{
  if(nl->ListLength(args)!=2){
    return listutils::typeError("two arguments expected");
  }
  ListExpr stream1 = nl->First(args);
  ListExpr stream2 = nl->Second(args);
  string err = "stream(tuple1) x stream(tuple2) expected";
  if(!listutils::isTupleStream(stream1) ||
     !listutils::isTupleStream(stream2)){
    return listutils::typeError(err +
                 "(one of the arguments is not a tuple stream)");
  }
  ListExpr newAttrList = ConcatLists(nl->Second(nl->Second(stream1)),
                                     nl->Second(nl->Second(stream2)));
  if(!listutils::isAttrList(newAttrList)){
    return listutils::typeError(err + "( name conflict )");
  }
  return nl->TwoElemList(nl->SymbolAtom(Symbol::STREAM()),
                         nl->TwoElemList(nl->SymbolAtom(Tuple::BasicType()),
                                         newAttrList));

}

/*
5.10.5.2 Value mapping function of operator ~symmproduct~

*/

struct SymmProductLocalInfo
{
  TupleType *resultTupleType;

  TupleBuffer *rightRel;
  GenericRelationIterator *rightIter;
  TupleBuffer *leftRel;
  GenericRelationIterator *leftIter;
  bool right;
  Tuple *currTuple;
  bool rightFinished;
  bool leftFinished;
};

int
SymmProduct(Word* args, Word& result, int message, Word& local, Supplier s)
{
  Word r, l;
  SymmProductLocalInfo* pli;
  TupleType* tt = (TupleType*) qp->GetLocal2(s).addr;

  switch (message)
  {
    case INIT: {
       tt = new TupleType(nl->Second(GetTupleResultType(s)));
       qp->GetLocal2(s).addr=tt;
       return 0;
    }
    case FINISH: {
       if(tt){
         tt->DeleteIfAllowed();
         qp->GetLocal2(s).addr=0;
       }
       return 0;
    }

    case OPEN :
    {
      long MAX_MEMORY = (qp->GetMemorySize(s) * 1024 * 1024);
      cmsg.info("ERA:ShowMemInfo") << "SymmProduct.MAX_MEMORY ("
                                   << MAX_MEMORY/1024 << " kB): " << endl;
      cmsg.send();
      pli = new SymmProductLocalInfo;
      pli->rightRel = new TupleBuffer( MAX_MEMORY / 2 );
      pli->rightIter = 0;
      pli->leftRel = new TupleBuffer( MAX_MEMORY / 2 );
      pli->leftIter = 0;
      pli->right = true;
      pli->currTuple = 0;
      pli->rightFinished = false;
      pli->leftFinished = false;

      tt->IncReference();
      pli->resultTupleType = tt;

      qp->Open(args[0].addr);
      qp->Open(args[1].addr);

      local.setAddr(pli);
      return 0;
    }
    case REQUEST :
    {
      pli = (SymmProductLocalInfo*)local.addr;

      while( 1 )
        // This loop will end in some of the returns.
      {
        if( pli->right )
          // Get the tuple from the right stream and match it with the
          // left stored buffer
        {
          if( pli->currTuple == 0 )
          {
            qp->Request(args[1].addr, r);
            if( qp->Received( args[1].addr ) )
            {
              pli->currTuple = (Tuple*)r.addr;
              pli->leftIter = pli->leftRel->MakeScan();
            }
            else
            {
              pli->rightFinished = true;
              if( pli->leftFinished )
                return CANCEL;
              else
              {
                pli->right = false;
                continue; // Go back to the loop
              }
            }
          }

          // Now we have a tuple from the right stream in currTuple
          // and an open iterator on the left stored buffer.
          Tuple *leftTuple = pli->leftIter->GetNextTuple();

          if( leftTuple == 0 )
            // There are no more tuples in the left iterator. We then
            // store the current tuple in the right buffer and close the
            // left iterator.
          {
            if( !pli->leftFinished )
              // We only need to keep track of the right tuples if the
              // left stream is not finished.
            {
              pli->rightRel->AppendTuple( pli->currTuple );
              pli->right = false;
            }

            pli->currTuple->DeleteIfAllowed();
            pli->currTuple = 0;

            delete pli->leftIter;
            pli->leftIter = 0;

            continue; // Go back to the loop
          }
          else
            // We match the tuples.
          {
            Tuple *resultTuple = new Tuple( pli->resultTupleType );
            Concat( leftTuple, pli->currTuple, resultTuple );
            leftTuple->DeleteIfAllowed();
            leftTuple = 0;
            result.setAddr( resultTuple );
            return YIELD;
          }
        }
        else
          // Get the tuple from the left stream and match it with the
          // right stored buffer
        {
          if( pli->currTuple == 0 )
          {
            qp->Request(args[0].addr, l);
            if( qp->Received( args[0].addr ) )
            {
              pli->currTuple = (Tuple*)l.addr;
              pli->rightIter = pli->rightRel->MakeScan();
            }
            else
            {
              pli->leftFinished = true;
              if( pli->rightFinished )
                return CANCEL;
              else
              {
                pli->right = true;
                continue; // Go back to the loop
              }
            }
          }

          // Now we have a tuple from the left stream in currTuple and
          // an open iterator on the right stored buffer.
          Tuple *rightTuple = pli->rightIter->GetNextTuple();

          if( rightTuple == 0 )
            // There are no more tuples in the right iterator. We then
            // store the current tuple in the left buffer and close
            // the right iterator.
          {
            if( !pli->rightFinished )
              // We only need to keep track of the left tuples if the
              // right stream is not finished.
            {
              pli->leftRel->AppendTuple( pli->currTuple );
              pli->right = true;
            }

            pli->currTuple->DeleteIfAllowed();
            pli->currTuple = 0;

            delete pli->rightIter;
            pli->rightIter = 0;

            continue; // Go back to the loop
          }
          else
            // We match the tuples.
          {
            Tuple *resultTuple = new Tuple( pli->resultTupleType );
            Concat( pli->currTuple, rightTuple, resultTuple );
            rightTuple->DeleteIfAllowed();
            rightTuple = 0;
            result.setAddr( resultTuple );
            return YIELD;
          }
        }
      }
    }
    case CLOSE :
    {
      pli = (SymmProductLocalInfo*)local.addr;
      if(pli)
      {
        if( pli->currTuple != 0 )
          pli->currTuple->DeleteIfAllowed();

        delete pli->leftIter;
        delete pli->rightIter;
        if( pli->resultTupleType != 0 )
          pli->resultTupleType->DeleteIfAllowed();

        if( pli->rightRel != 0 )
        {
          pli->rightRel->Clear();
          delete pli->rightRel;
        }

        if( pli->leftRel != 0 )
        {
          pli->leftRel->Clear();
          delete pli->leftRel;
        }

        delete pli;
        local.setAddr(0);
      }

      qp->Close(args[0].addr);
      qp->Close(args[1].addr);

      return 0;
    }
  }
  return 0;
}


/*

5.10.6.3 Specification of operator ~symmproduct~

*/
const string SymmProductSpec  =
  "( ( \"Signature\" \"Syntax\" \"Meaning\" "
  "\"Example\" ) "
  "( <text>(stream (tuple(X))) (stream (tuple(Y))) "
  " -> (stream (tuple(X*Y))) "
  "))</text--->"
  "<text>_ _ symmproduct</text--->"
  "<text>Computes a Cartesian product stream from "
  "its two argument streams, in a symmetrical and "
  "non-blocking way.</text--->"
  "<text>query ten feed {a} ten feed {b} "
  "symmproduct count</text--->"
  " ) )";

/*

5.10.6.4 Definition of operator ~symmproduct~

*/
Operator extrelsymmproduct (
         "symmproduct",           // name
         SymmProductSpec,         // specification
         SymmProduct,             // value mapping
         Operator::SimpleSelect,  // trivial selection function
         SymmProductTypeMap       // type mapping
//         true                   // needs large amounts of memory
);



/*
5.10.7 Operator ~addCounter~

5.10.7.1 Type Mapping function

This operator receives a tuple stream, an attribute name
as well as an initial value. It extends each tuple by
a counter initialized with the given value and the given name.

*/
ListExpr AddCounterTypeMap(ListExpr args){

if(nl->ListLength(args)!=3){
   ErrorReporter::ReportError("three arguments required.");
   return nl->TypeError();
}
ListExpr stream = nl->First(args);
ListExpr nameL   = nl->Second(args);
ListExpr init   = nl->Third(args);

// check init
if(!CcInt::checkType(init) && !LongInt::checkType(init) ){
  return listutils::typeError("third argument must be of "
                              "type int or longint");
}

string errmsg;
if(!listutils::isValidAttributeName(nameL,errmsg)){
  return listutils::typeError(errmsg);
}
string name = nl->SymbolValue(nameL);

// check streamlist
if(nl->ListLength(stream)!=2 ||
   !nl->IsEqual(nl->First(stream),Symbol::STREAM())){
 ErrorReporter::ReportError("first argument is not a stream");
 return nl->TypeError();
}
ListExpr tuple = nl->Second(stream);
if(nl->ListLength(tuple)!=2 ||
   !nl->IsEqual(nl->First(tuple),Tuple::BasicType())){
 ErrorReporter::ReportError("first argument is not a tuple stream");
 return nl->TypeError();
}
set<string> usednames;
ListExpr attributes = nl->Second(tuple);
if(nl->AtomType(attributes)!=NoAtom){
   ErrorReporter::ReportError("invalid representation in tuple stream"
                              "(not a list)");
   return nl->TypeError();
}
ListExpr last = nl->TheEmptyList();
while(!nl->IsEmpty(attributes)){
   last = nl->First(attributes);
   if(nl->ListLength(last)!=2){
      ErrorReporter::ReportError("invalid representation in tuple stream"
                                 " wrong listlength");
      return nl->TypeError();
   }
   if(nl->AtomType(nl->First(last))!=SymbolType){
      ErrorReporter::ReportError("invalid representation in tuple stream"
                                 "(syntax of attribute name)");
      return nl->TypeError();
   }
   usednames.insert(nl->SymbolValue(nl->First(last)));
   attributes = nl->Rest(attributes);
}

if(usednames.find(name)!=usednames.end()){
  ErrorReporter::ReportError("Name" + name +" is already used.");
  return  nl->TypeError();
}

// all fine, construct the result

if(nl->IsEmpty(last)){ // stream without attributes
  return nl->TwoElemList( nl->SymbolAtom(Symbol::STREAM()),
                          nl->TwoElemList(
                              nl->SymbolAtom(Tuple::BasicType()),
                              nl->OneElemList(
                                 nl->TwoElemList(
                                     nl->SymbolAtom(name),
                                     init))));
}

// make a copy of the attributes
  attributes = nl->Second(tuple);
  ListExpr reslist = nl->OneElemList(nl->First(attributes));
  ListExpr lastlist = reslist;
  attributes = nl->Rest(attributes);
  while (!(nl->IsEmpty(attributes))) {
    lastlist = nl->Append(lastlist,nl->First(attributes));
    attributes = nl->Rest(attributes);
  }
  lastlist = nl->Append(lastlist,nl->TwoElemList(
                        nl->SymbolAtom(name),
                        init));

  return nl->TwoElemList( nl->SymbolAtom(Symbol::STREAM()),
                nl->TwoElemList( nl->SymbolAtom(Tuple::BasicType()),
                                 reslist));

}

/*
5.10.7.2 Value mapping of the ~addcounter~ operator

*/
template< class T>
class AddCounterLocalInfo{
public:
  AddCounterLocalInfo(T* init, TupleType*  tt){
     defined = init->IsDefined();
     value = *init;
     tt->IncReference();
     tupleType = tt;
  }
  ~AddCounterLocalInfo(){
     tupleType->DeleteIfAllowed();
     tupleType = 0;
  }
  Tuple* createTuple(Tuple* orig){
    if(!defined){
       return 0;
    }
    Tuple* result = new Tuple(tupleType);
    int size = orig->GetNoAttributes();
    for(int i=0;i<size;i++){
       result->CopyAttribute(i,orig,i);
    }
    result->PutAttribute(size, value.Clone());
    value.Inc();
    return result;
  }


private:
  bool  defined;
  T value;
  TupleType* tupleType;
};

template<class T>
int AddCounterValueMap(Word* args, Word& result, int message,
               Word& local, Supplier s){

   Word orig;
   TupleType* tt = (TupleType*) qp->GetLocal2(s).addr;

   switch(message){
    case INIT: {
       tt = new TupleType(nl->Second(GetTupleResultType(s)));
       qp->GetLocal2(s).addr = tt;
       return 0;
    }

    case FINISH : {
       if(tt){
          tt->DeleteIfAllowed();
          qp->GetLocal2(s).addr=0;
       }
       return 0;
    }

    case OPEN: {
       T* Init = ((T*)args[2].addr);
       qp->Open(args[0].addr);
       local.addr = new AddCounterLocalInfo<T>(Init,tt);
       break;
    }
    case REQUEST: {
       qp->Request(args[0].addr,orig);
       if(!qp->Received(args[0].addr)){
          return CANCEL;
       }  else {
          Tuple* tmp = ((AddCounterLocalInfo<T>*)local.addr)->
                          createTuple((Tuple*)orig.addr);
          ((Tuple*)orig.addr)->DeleteIfAllowed();
          result.setAddr(tmp);
          return YIELD;
       }

    }
    case CLOSE: {
       qp->Close(args[0].addr);
       if(local.addr)
       {
         AddCounterLocalInfo<T>* acli = (AddCounterLocalInfo<T>*)local.addr;
         delete acli;
         local.addr=0;
       }
       return 0;
    }
    default: assert(false); // unknown message
   }
   return 0;
}


/*

5.10.7.3 Specification of operator ~addcounter~

*/
const string AddCounterSpec  =
  "( ( \"Signature\" \"Syntax\" \"Meaning\" "
  "\"Example\" ) "
  "( <text>(stream (tuple(X))) x name x {int,longint} "
  " -> (stream (tuple(X (name int)))) "
  "))</text--->"
  "<text> stream addcounter[AttrName, Initial]  </text--->"
  "<text>Adds a counter with the given name to the stream "
  "starting at the initial value </text--->"
  "<text>query ten feed addCounter[ Cnt , 1] consume</text--->"
  " ) )";

/*
ValueMapping Array and SelectionFunction

*/
ValueMapping AddCounterVM[] = {
     AddCounterValueMap<CcInt>,
     AddCounterValueMap<LongInt>
  };

int AddCounterSelect(ListExpr args){
  return CcInt::checkType(nl->Third(args))?0:1;
}


/*

5.10.7.4 Definition of operator ~addcounter~

*/
Operator extreladdcounter (
         "addcounter",           // name
         AddCounterSpec,         // specification
         2,
         AddCounterVM,      // value mapping
         AddCounterSelect,  // trivial selection function
         AddCounterTypeMap       // type mapping
);


struct printrefsInfo : OperatorInfo {

  printrefsInfo()
  {
    name      = "printrefs";

    signature = "stream(tuple(...)) -> stream(tuple(...))";
    syntax    = "_ printrefs _";
    meaning   = "Prints out the values of the tuple's "
                "and attribute's reference counter";
  }
};

static ListExpr printrefs_tm(ListExpr args)
{
  NList l(args);

  static const string err1 = "printrefs expects stream(tuple(...))!";

  if ( !l.hasLength(1) )
    return l.typeError(err1);

  NList attrs;
  if ( l.checkStreamTuple( attrs ) )
    return NList::typeError(err1);

  return l.first().listExpr();
}


int printrefs_vm( Word* args, Word& result, int message,
                 Word& local, Supplier s)
{
  Word w(Address(0));

  switch (message)
  {
    case OPEN :

      qp->Open(args[0].addr);
      return 0;

    case REQUEST :

      qp->Request(args[0].addr, w);
      if (qp->Received(args[0].addr))
      {
        Tuple* t = static_cast<Tuple*>( w.addr );
        int tRefs = t->GetNumOfRefs();
        cout << (void*)t << ": " << tRefs << "(";
        for(int i = 0; i < t->GetNoAttributes(); i++)
        {
          cout << " " << t->GetAttribute(i)->NoRefs();
        }
        cout << " )" << endl;

        result.addr = t;
        return YIELD;
      }
      result.addr = 0;
      return CANCEL;

    case CLOSE :

      qp->Close(args[0].addr);
      return 0;
  }
  return 0;
}



/*
2.113 Operator extend[_]aggr

This operator computes the aggregation of an attribute of a tuple stream.
Each provisonal result is appended to the original tuple.


2.113.1 Type Mapping

The Signature is:

  tuplestream x attrName x newAttrName x fun

*/

ListExpr extend_aggrTM(ListExpr args){

  string err = "stream(tuple(...)) x attr x newAttr x fun expected";
  if(!nl->HasLength(args,3)){
    return listutils::typeError(err);
  }
  ListExpr stream = nl->First(args);
  ListExpr AttrNameList = nl->Second(args);
  ListExpr FunList1 = nl->Third(args);

  if(!nl->HasLength(FunList1,1)){ // only one function is allowed
     return listutils::typeError(err + " (only one fun allowed)");
  }
  ListExpr FunList2 = nl->First(FunList1);
  if(!nl->HasLength(FunList2,2)){
     return listutils::typeError(err + " invalid function");
  }


  ListExpr NewAttrName = nl->First(FunList2);
  ListExpr Fun = nl->Second(FunList2);

  if(!Stream<Tuple>::checkType(stream)){
    return listutils::typeError(err + 
                                " (first arg is not a tuple stream) ");
  }

  int AttrNameListLength = nl->ListLength(AttrNameList);
  if(AttrNameListLength!=1 && AttrNameListLength!=2){
     return listutils::typeError(err);
  }
  ListExpr AttrName1 = nl->First(AttrNameList);
  
  if(!listutils::isSymbol(AttrName1)){
    return listutils::typeError(err + 
                        " (second arg is not a valid attribute name ");
  }
  string errmsg;
  if(!listutils::isValidAttributeName(NewAttrName,errmsg)){
    return listutils::typeError(errmsg);
  }
  
  if(!listutils::isMap<2>(Fun)){
    return listutils::typeError(err + 
                        " (third arg is not a function");
  }
  ListExpr FunArgType1 = nl->Second(Fun);
  ListExpr FunArgType2 = nl->Third(Fun);
  ListExpr FunResType = nl->Fourth(Fun);
  ListExpr AttrType;
  string attrName1 = nl->SymbolValue(AttrName1);

  ListExpr AttrList = nl->Second(nl->Second(stream));

  int pos = listutils::findAttribute(AttrList, attrName1, AttrType);
  if(pos==0){
    return listutils::typeError("Attribute name " + attrName1 + 
                                " not present in stream");
  }

  if(!nl->Equal(FunArgType1, AttrType)){
    return listutils::typeError("Attribute type doesn't match "
                                "function argument type");
  }
  if(!nl->Equal(FunArgType2, AttrType)){
    return listutils::typeError("Attribute type doesn't match "
                                "function argument type");
  }
  if(!nl->Equal(FunResType, AttrType)){
    return listutils::typeError("Attribute type doesn't match "
                                "function result type");
  }

ListExpr NewAttr = nl->OneElemList(nl->TwoElemList(NewAttrName, AttrType));
ListExpr ResAttrList = listutils::concat(AttrList, NewAttr);

if(!listutils::isAttrList(ResAttrList)){
    return listutils::typeError("Attribute " + nl->ToString(NewAttr) +
                                " already present in stream");
}

ListExpr Res = nl->TwoElemList(
                             nl->SymbolAtom(Stream<Tuple>::BasicType()),
                             nl->TwoElemList(
                                    nl->SymbolAtom(Tuple::BasicType()),
                                    ResAttrList)) ;

int resetPos = -1;
if(AttrNameListLength==2){
  ListExpr AttrName2 = nl->Second(AttrNameList);
  if(!listutils::isSymbol(AttrName2)){
    return listutils::typeError(err);
  }
  string attrName2=nl->SymbolValue(AttrName2);
  ListExpr at;
  int index = listutils::findAttribute(AttrList,attrName2, at);
  if(index==0){
     return listutils::typeError("Attributre " + attrName2 + " not found");
  }
  if(!CcBool::checkType(at)){
    return listutils::typeError("Attrbute " + attrName2 + " not of type bool");
  } 
  resetPos = index -1;
}

return nl->ThreeElemList(nl->SymbolAtom(Symbol::APPEND()),
                         nl->TwoElemList(nl->IntAtom(pos-1),
                         nl->IntAtom(resetPos)
                        ),
                        Res);
}



class ExtendAggrInfo{

public: 
     ExtendAggrInfo(Word& _stream, Supplier _fun, int _attrPos, 
                  int _resetPos, TupleType* type):
     stream(_stream),first(true), fun(_fun), attrPos(_attrPos),
     resetPos(_resetPos), value(0) {
         tupleType = type;
         tupleType->IncReference();
         stream.open();
     }
    
     ~ExtendAggrInfo(){
            stream.close();
            tupleType->DeleteIfAllowed();
            if(value){
                 value->DeleteIfAllowed();
            }
     }

     Tuple* getNextTuple(){

            Tuple* srcTuple = stream.request();
            if(srcTuple==0){
                return 0;
            }

            Tuple* resTuple = new Tuple(tupleType); 
            // copy old attributes
            for(int i=0;i<srcTuple->GetNoAttributes(); i++){
                resTuple->CopyAttribute(i,srcTuple,i);
            }
      bool reset=false;
      if(resetPos>=0){
        CcBool* b = (CcBool*) srcTuple->GetAttribute(resetPos);
        reset = b->IsDefined() && b->GetValue();
      }
        if(first){ 
           value = srcTuple->GetAttribute(attrPos)->Clone();
           first = false;
        } else if(reset){
          value->DeleteIfAllowed();
          value = srcTuple->GetAttribute(attrPos)->Clone();
        } else {
                // compute value from current value and next attribute
                Attribute* attr = srcTuple->GetAttribute(attrPos);
                ArgVectorPointer funargs = qp->Argument(fun);
                (*funargs)[0] = value;
                (*funargs)[1] = attr;
                Word funres;
                qp->Request(fun,funres);
                Attribute* victim = value; 
                value = ((Attribute*)funres.addr)->Copy(); 
                victim->DeleteIfAllowed();
            }
            resTuple->PutAttribute(srcTuple->GetNoAttributes(), value->Clone());
            srcTuple->DeleteIfAllowed();
            return resTuple;
     }

private:
    Stream<Tuple> stream;
    bool first;
    Supplier fun;
    int attrPos;
    int resetPos;
    Attribute* value; 
    TupleType* tupleType;   
};


int
extend_aggrVM(Word* args, Word& result, int message,
                 Word& local, Supplier s1){

   ExtendAggrInfo* li = (ExtendAggrInfo*) local.addr;
   TupleType* tt = (TupleType*) qp->GetLocal2(s1).addr;
    
   switch(message){
    case INIT : {
      tt = new TupleType(nl->Second(GetTupleResultType(s1)));
      qp->GetLocal2(s1).addr=tt;
      return 0;
    }
    case FINISH: {
      if(tt){
        tt->DeleteIfAllowed();
        qp->GetLocal2(s1).addr=0;
      }
      return 0;
    }
 
    case OPEN : {
                  if(li){
                     delete li;
                  }
                  int attrPos = ((CcInt*)args[3].addr)->GetValue();
                  int resetPos = ((CcInt*)args[4].addr)->GetValue();
                  Supplier s = args[2].addr;
                  Supplier s2 = qp->GetSupplier(s,0);
                  Supplier s3 = qp->GetSupplier(s2,1);
                  local.addr = new ExtendAggrInfo(args[0], s3, attrPos,
                                      resetPos, 
                                      tt);
                  return 0;
                  }
    case REQUEST: {
                     if(!li){
                        return CANCEL;
                     }
                     result.addr = li->getNextTuple();
                     return result.addr?YIELD:CANCEL;
                   }
    case CLOSE: {
                   if(li){
                      delete li;
                      local.addr=0;
                    }
                    return 0;
                }

   }
   return -1;
}

const string extend_aggrSpec  =
"( ( \"Signature\" \"Syntax\" \"Meaning\" \"Example\" ) "
"( <text>stream(tuple(x)) x attrName x newAttrName x fun -> "
  "stream(tuple(x @ newAttrName))</text--->"
  "<text>stream extend_aggr [ attrName ; newAttrName : fun ] )</text--->"
  "<text>Compute an aggrgation over an attribute of a tuple and stored all "
  "provisional results into the" 
  "  result tuples"
  "</text--->"
  "<text>query ten feed extend_aggr[no; aNo : "
  "fun(i1: int, i2: int) i1 + i2 ] </text--->"
  ") )";



/*
2.113.4 Operator instance

*/

Operator extend_aggr(
          "extend_aggr",
          extend_aggrSpec,
          extend_aggrVM,
          Operator::SimpleSelect,
          extend_aggrTM);



/*
2.114 Operator extend[_]last

This operator extends a tuple using values from that tuple and from
values of the last tuple from the stream. For the first tuple, the 
values are given in a separate list.


2.114.1 Type Mapping

signature is  stream(tuple) x funlist x list


*/
ListExpr extend_lastTM(ListExpr args){
  string err = "stream(tuple) x funlist with defaults expected";
  if(!nl->HasLength(args, 2)){
    return listutils::typeError(err + " (wrong number of arguments)");
  }
  ListExpr stream = nl->First(args);
  ListExpr fundeflist = nl->Second(args);

  if(!Stream<Tuple>::checkType(stream)){
    return listutils::typeError(err + " (first arg is not a tuple stream");
  }
  if(nl->AtomType(fundeflist)!=NoAtom){
    return listutils::typeError("secondo argument is not a function list");
  }
  if(nl->IsEmpty(fundeflist)){
    return listutils::typeError(err + " (funlist is empty)");
  }

  ListExpr TupleList  = nl->Second(stream);
  ListExpr attrList   = nl->Second(TupleList);
  ListExpr ExtList = nl->TheEmptyList();
  ListExpr last = nl->TheEmptyList();
  bool first = true;


  while(!nl->IsEmpty(fundeflist)){
    ListExpr firstFun = nl->First(fundeflist);
    fundeflist = nl->Rest(fundeflist);
    if(!nl->HasLength(firstFun,3)){ // (name map default)
      return listutils::typeError(err + " (Invalid named function)");
    }

    ListExpr newName = nl->First(firstFun);

    string error;
    if(!listutils::isValidAttributeName(newName, error)){
       return listutils::typeError(error);
    }

    ListExpr mapList = nl->Second(firstFun);
    if(!listutils::isMap<2>(mapList)){
       return listutils::typeError(err + " (invalid map found)"); 
    } 
    if(!nl->Equal(nl->Second(mapList),TupleList) ||
       !nl->Equal(nl->Third(mapList),TupleList)){
       return listutils::typeError(err + " ( invalid function arguments )");
    } 
    ListExpr resType = nl->Fourth(mapList);
    ListExpr defType = nl->Third(firstFun);
    if(!nl->Equal(resType,defType)){
       return listutils::typeError(err + " (type mismatch between fun "
                                  "result and default)");
    }
    ListExpr dummy;
    string name = nl->SymbolValue(newName);
    if(listutils::findAttribute(attrList,name,dummy)>0){
      return listutils::typeError(err + " ( attribute " + name +
                                   " already exists"); 
    }
    ListExpr ext = nl->TwoElemList( newName, resType);
    if(first){
        ExtList = nl->OneElemList(ext);
        last = ExtList;
        first = false;
    } else {
        last = nl->Append(last, ext);
    }
  }

  ListExpr resAttrList = listutils::concat(attrList, ExtList);
  return nl->TwoElemList(
              nl->SymbolAtom(Stream<Tuple>::BasicType()),
              nl->TwoElemList(
                  nl->SymbolAtom(Tuple::BasicType()),
                  resAttrList)); 
}


/*
2.114.2 Value Mapping

*/
class ExtendLastInfo{
  public:
    ExtendLastInfo(Word& _stream, Supplier _s1, 
                   TupleType* resType):
     stream(_stream), s1(_s1), first(true), lastTuple(0){
      tupleType = resType;
      tupleType->IncReference();
      stream.open();
    }
    ~ExtendLastInfo(){
      stream.close();
      tupleType->DeleteIfAllowed();
      if(lastTuple){
         lastTuple->DeleteIfAllowed();
      }
    }

    Tuple* getNextTuple(){

      Tuple* srcTuple = stream.request();
      if(srcTuple==0){
         return 0;
      }
      Tuple* resTuple = new Tuple(tupleType);
      // copy the original attributesA
      int no_attr = srcTuple->GetNoAttributes();
      for(int i=0;i<no_attr; i++){
        resTuple->CopyAttribute(i,srcTuple,i);
      }

      if(first){
         // copy default values 
         int noDefaults = qp->GetNoSons(s1);
         for(int i=0;i<noDefaults;i++){
            Supplier s = qp->GetSon(qp->GetSon(s1,i),2);
            Word res;
            qp->Request(s,res);
            Attribute* resAttr = (Attribute*) res.addr;
            resTuple->PutAttribute(i + no_attr, resAttr->Clone());
         }
         first = false;
         lastTuple = srcTuple;
      } else { // lastTuple is present, use functions
         int noFuns = qp->GetNoSons(s1);
         for(int i=0; i < noFuns; i++){
            Supplier fun1 = qp->GetSon(s1,i); // (Name Function default)

            Supplier fun = qp->GetSon(fun1,1); // get function
            ArgVectorPointer funargs = qp->Argument(fun);
            (*funargs)[0] = srcTuple;
            (*funargs)[1] = lastTuple;
            Word funres;
            qp->Request(fun, funres);
            resTuple->PutAttribute(i+no_attr,
                              ((Attribute*) funres.addr)->Clone());
         }
         lastTuple->DeleteIfAllowed();
         lastTuple = srcTuple;
      }
      return resTuple; 
    }

  private:
    Stream<Tuple> stream;
    Supplier s1;
    bool first;
    TupleType* tupleType;
    Tuple* lastTuple;
};



template<class T>
int extend_lastVMT(Word* args, Word& result, int message,
                 Word& local, Supplier s1){

   T* li = (T*) local.addr;
   TupleType* tt = (TupleType*) qp->GetLocal2(s1).addr;

   switch(message){
    case INIT: {
        tt = new TupleType(nl->Second(GetTupleResultType(s1)));
        qp->GetLocal2(s1).addr = tt;
        return 0;
    }
    case FINISH: {
       if(tt){
          tt->DeleteIfAllowed();
          qp->GetLocal2(s1).addr=0;
       }
       return 0;
   }

    case OPEN : {
                  if(li){
                     delete li;
                  }
                  local.addr = new T(args[0], args[1].addr, tt);
                  return 0;
                  }
    case REQUEST: {
                     if(!li){
                        return CANCEL;
                     }
                     result.addr = li->getNextTuple();
                     return result.addr?YIELD:CANCEL;
                   }
    case CLOSE: {
                   if(li){
                      delete li;
                      local.addr=0;
                    }
                    return 0;
                }

   }
   return -1;
}




const string extend_lastSpec  =
"( ( \"Signature\" \"Syntax\" \"Meaning\" \"Example\" ) "
"( <text>stream(tuple(x)) x funlist x list  -> "
  "stream(tuple(x @ newAttributes))</text--->"
  "<text>stream extend_last [ funlist ; list ] )</text--->"
  "<text>Computes new attributes from the current tuple and the"
  " previous one in the stream.  The current tuple can be accessed using"
  " the dot(.). Attribute values from the previous  tuple can be accessed "
  " by ..\n"
  " For the first tuple, no previous tuple is available. For this reason,"
  " the default value from the list is usied instead of the funtion."
  "</text--->"
  "<text>query ten feed extend_last[ ANum : .no - ..no ; 3] tconsume "
  " </text--->"
  ") )";



/*
2.114.4 Operator instance

*/

Operator extend_last(
          "extend_last",
          extend_lastSpec,
          extend_lastVMT<ExtendLastInfo>,
          Operator::SimpleSelect,
          extend_lastTM);


/*
2.116 Operator extend\_next


2.116.1 Type Mapping

The Type mapping is the same as for the extend[_]last operator.

*/


class ExtendNextInfo{
  public:
    ExtendNextInfo(Word& _stream, Supplier _s1, 
                   TupleType* resType):
      stream(_stream), s1(_s1), lastTuple(0){
      tupleType = resType;
      tupleType->IncReference();
      stream.open();
    }
    ~ExtendNextInfo(){
      stream.close();
      tupleType->DeleteIfAllowed();
      if(lastTuple){
         lastTuple->DeleteIfAllowed();
      }
    }

    Tuple* getNextTuple(){

      Tuple* srcTuple = stream.request();
      if(srcTuple==0){
         if(lastTuple){ // copy default values
            int no_attr = lastTuple->GetNoAttributes();
            Tuple* resTuple = getResTuple(lastTuple);
            lastTuple->DeleteIfAllowed();
            lastTuple = 0;
            int noDefaults = qp->GetNoSons(s1);
            for(int i=0;i<noDefaults;i++){
               Supplier s = qp->GetSon(qp->GetSon(s1,i),2);
               Word res;
               qp->Request(s,res);
               Attribute* resAttr = (Attribute*) res.addr;
               resTuple->PutAttribute(i + no_attr, resAttr->Clone());
            }
            return resTuple;
         }
         return 0;
      }

      if(!lastTuple){ // it's the first tuple in stream
         lastTuple = srcTuple;
         return getNextTuple();     
      }
     

      int no_attr = lastTuple->GetNoAttributes();
      Tuple* resTuple= getResTuple(lastTuple);   
      int noFuns = qp->GetNoSons(s1);
      for(int i=0; i < noFuns; i++){
          Supplier fun1 = qp->GetSon(s1,i); // (Name Function default)
          Supplier fun = qp->GetSon(fun1,1); // get function
          ArgVectorPointer funargs = qp->Argument(fun);
          (*funargs)[0] = lastTuple;
          (*funargs)[1] = srcTuple;
          Word funres;
          qp->Request(fun, funres);
          resTuple->PutAttribute(i+no_attr,
                            ((Attribute*) funres.addr)->Clone());
      }
      lastTuple->DeleteIfAllowed();
      lastTuple = srcTuple;
      return resTuple; 
    }


  private:
    Stream<Tuple> stream;
    Supplier s1;
    TupleType* tupleType;
    Tuple* lastTuple;

    // creates a new tuple and copies the content of srctuple
    // to the first attributes of the result
    Tuple* getResTuple(Tuple* srcTuple){
         Tuple* resTuple = new Tuple(tupleType);
         // copy the original attributesA
         int no_attr = srcTuple->GetNoAttributes();
         for(int i=0;i<no_attr; i++){
             resTuple->CopyAttribute(i,srcTuple,i);
         }
         return resTuple;
    }


};




const string extend_nextSpec  =
"( ( \"Signature\" \"Syntax\" \"Meaning\" \"Example\" ) "
"( <text>stream(tuple(x)) x funlist x list  -> "
  "stream(tuple(x @ newAttributes))</text--->"
  "<text>stream extend_next [ funlist ; list ] )</text--->"
  "<text>Computes new attributes from the current tuple and the"
  " nexxt one in the stream.  The current tuple can be accessed using"
  " the dot(.). Attribute values from the next  tuple can be accessed "
  " by ..\n"
  " For the last tuple, no next tuple is available. For this reason,"
  " the default value from the list is used instead of the funtion."
  "</text--->"
  "<text>query ten feed extend_next[ NNum : .no - ..no ; 3] tconsume "
  " </text--->"
  ") )";



/*
2.114.4 Operator instance

*/

Operator extend_next(
          "extend_next",
          extend_nextSpec,
          extend_lastVMT<ExtendNextInfo>,
          Operator::SimpleSelect,
          extend_lastTM);



/*


2.115 Operator ~toFields~


*/

/*

2.115.1 Type mapping for ~toFields~ is
(stream(tuple(a1:d1, ..., an:dn))) x (aj) 
-> (stream(tuple(aj:dj, Field: string, Type: text, Value:text)))
 
*/
ListExpr toFieldsType( ListExpr args ) {
  if ( !nl->HasLength( args, 2 ) ) {
    return listutils::typeError( "two arguments expected" );
  }
  if ( !Stream<Tuple>::checkType( nl->First( args ) ) ) {
    return listutils::typeError( "first argument must be a stream of tuples" );
  }
  if ( !listutils::isSymbol( nl->Second( args )) ) {
    return listutils::typeError( "second argument must be an attribute name" );
  }
  ListExpr attrlist = nl->Second(nl->Second(nl->First( args )));
  string attrname = nl->SymbolValue(nl->Second( args ));
  ListExpr type;
  int attrpos = listutils::findAttribute(attrlist, attrname, type);
  if ( !attrpos ) {
    return listutils::typeError( "attribute " + attrname + " does not exist" );
  }
  ListExpr resultattrlist = nl->FourElemList(
  nl->TwoElemList( nl->SymbolAtom( attrname ),
    type ),
    nl->TwoElemList( nl->SymbolAtom( "Field" ),
             nl->SymbolAtom( CcString::BasicType() ) ),
    nl->TwoElemList( nl->SymbolAtom( "Type" ),
             nl->SymbolAtom( FText::BasicType() ) ),
    nl->TwoElemList( nl->SymbolAtom( "Value" ),
             nl->SymbolAtom( FText::BasicType() ) ) );
  ListExpr resultlist = nl->TwoElemList(
    nl->SymbolAtom( Stream<Tuple>::BasicType() ), 
    nl->TwoElemList( nl->SymbolAtom( Tuple::BasicType() ),
             resultattrlist ) );
  
  ListExpr attrs = nl->OneElemList( nl->StringAtom( nl->SymbolValue
                                  ( nl->First( nl->First( attrlist ) ) ) ) );
  ListExpr lastAttr = attrs;
  ListExpr types = nl->OneElemList( nl->TextAtom( nl->SymbolValue
                                  ( nl->Second( nl->First( attrlist ) ) ) ) );
  ListExpr lastType = types;
  attrlist = nl->Rest( attrlist );
  while ( !nl->IsEmpty( attrlist ) ) {
    ListExpr first = nl->First( attrlist );
    ListExpr firstfirst = nl->First( first );
    ListExpr firstsecond = nl->Second( first );
    lastAttr = nl->Append( lastAttr, nl->StringAtom
    ( nl->SymbolValue( firstfirst ) ) );
    lastType = nl->Append( lastType, nl->TextAtom
                                   ( nl->ToString( firstsecond ) ) );
    attrlist = nl->Rest( attrlist ); // Iteration
  }
  
  ListExpr infolist1 = nl->OneElemList( nl->IntAtom( attrpos ) );
  ListExpr infolist2 = listutils::concat(infolist1,attrs);
  ListExpr infolist = listutils::concat( infolist2, types );
  
  ListExpr result = nl->ThreeElemList( nl->SymbolAtom( "APPEND" ),
                       infolist,
                       resultlist );
  return result;
}

/*

2.155.2 Class ~ToFieldsInfo~

*/

class ToFieldsInfo {
  public:
    ToFieldsInfo( Word& is, vector<string> &fields1, vector<string> &types1,
          int position, TupleType* _tt );
    ~ToFieldsInfo();
    Tuple* nextTuple();
  private:
    Stream<Tuple> stream;
    Tuple* tuple;
    unsigned int pos;
    int keypos;
    vector<string> fields, types;
    TupleType* tt;
    int algid, typid;
    bool istype;
    vector<OutObject> outfuns;
    vector<ListExpr> typelists;
    ListExpr typelist;
};

ToFieldsInfo::ToFieldsInfo( Word& is, vector<string> &fields1,
                            vector<string> &types1, int position, 
                            TupleType* _tt): 
    stream( is ), tuple( 0 ), pos(0), keypos( position ), fields( fields1 ),
    types(types1) {
  stream.open();
  tuple = stream.request();
  tt = _tt;
  tt->IncReference();
  SecondoCatalog* sc = SecondoSystem::GetCatalog();
  AlgebraManager* am = SecondoSystem::GetAlgebraManager();
  for ( int i = 0; i < tuple->GetNoAttributes(); i++ ) {
    // check whether types[ i ] is an atomic type. Otherwise cut the nested list
    bool listok = nl->ReadFromString( types[ i ], typelist );
    while ( !nl->IsAtom( typelist ) ) {
      typelist = nl->First( typelist );
    }   
    istype = sc->GetTypeId( nl->ToString( typelist ), algid, typid );
    // get and save the Out functions
    // depending on algebra id and type id of the attributes
    outfuns.push_back( am->OutObj( algid, typid ) );
    listok = nl->ReadFromString( types[ i ], typelist );
    if ( !listok ) {
      cout << "unknown type" << endl;
    }
    typelists.push_back( typelist );
  }
}

ToFieldsInfo::~ToFieldsInfo() {
  if ( tuple ) {
    tuple->DeleteIfAllowed();
  }
  stream.close();
  tt->DeleteIfAllowed();
}

Tuple* ToFieldsInfo::nextTuple() {
  if ( !tuple ) {
    return 0;
  }
  if ( pos == fields.size() ) {   // tuple finished
    tuple->DeleteIfAllowed();
    tuple = stream.request();
    pos = 0;
    if ( !tuple ) {   // last tuple finished
      return 0;
    }
  }
  Tuple* result = new Tuple( tt );
  // set 1st entry to key attribute
  result->PutAttribute( 0, tuple->GetAttribute(keypos-1)->Copy() );
  // set 2nd entry to field name
  result->PutAttribute( 1, new CcString( true, fields[ pos ] ) ); 
  // set 3rd entry to attribute type
  result->PutAttribute( 2, new FText( true, types[ pos ] ) ); 
  string value;
  Attribute* attrvalue = tuple->GetAttribute( pos );
  if ( attrvalue->hasTextRepresentation() ) {
    value = attrvalue->toText(); 
  }
  else {
     // call the Out function if necessary
    ListExpr output = outfuns[ pos ]( typelists[ pos ], attrvalue ); 
    value = nl->ToString( output );
  }
  // set 4th entry to attribute value  
  result->PutAttribute( 3, new FText( true, value ) );
  pos++; // proceed through input tuple
  return result;
}

/*

2.155.3 Value mapping for ~toFields~

*/

int toFieldsFun ( Word* args, Word& result, int message,
          Word& local, Supplier s ) {
  ToFieldsInfo* tfi = (ToFieldsInfo*) local.addr;
  TupleType* tt = (TupleType*) qp->GetLocal2(s).addr;

  switch( message ) {
    case INIT : {
       tt = new TupleType(nl->Second(GetTupleResultType(s)));
       qp->GetLocal2(s).addr=tt;
       return 0;
    }
    case FINISH: {
       if(tt){
         tt->DeleteIfAllowed();
         qp->GetLocal2(s).addr=0;
       }
       return 0;
    }

    case OPEN: {
      if ( tfi ) {
         delete tfi;
      }
      int pos = ( (CcInt*)args[ 2 ].addr )->GetIntval();
      vector<string> fields, types;
      int n = qp->GetNoSons( s );
      // there are 2*n+3 arguments
      int noattrs = ( n-3 ) / 2;
      for ( int i = 0; i < noattrs; i++ ) {
    fields.push_back( ( (CcString*)args[ i + 3 ].addr )->GetValue() );
    types.push_back( ( (FText*)args[ i + noattrs + 3 ].addr )->GetValue() );
      }      
      local.addr = new ToFieldsInfo( args[ 0 ], fields, types, pos, tt);
      return 0;
    }
    case REQUEST: {
      if ( !tfi ) {
    return CANCEL;
      }
      result.addr = tfi->nextTuple();
      return result.addr?YIELD:CANCEL;
    }
    case CLOSE: {
      if ( tfi ) {
    delete tfi;
    local.addr = 0;
      }
      return 0;
    }
    default: {
      return -1;
    }
  }
}

/*
 
2.155.4 Description for operator ~toFields~

*/

struct toFieldsInfo : OperatorInfo {

  toFieldsInfo() : OperatorInfo() {

    name      = "toFields";
    signature = "((stream (tuple([a1:d1, ...,an:dn]))) x ai) -> "
        "stream(tuple([ai, attrname, dj, aj]))";
    syntax    = "_ feed toFields [ _ ]";
    meaning   = "Decomposes a stream of tuples into its items.";
  }

};

/*


2.116 Operator ~fromFields~
inverts the operator toFields

*/

/*

2.116.1 Type mapping for ~fromFields~ is
(stream(tuple(keyvalue: keytype, Field: string, Type: text, Value: text))) 
x (relation(X)) -> (stream(X))

*/

ListExpr fromFieldsType( ListExpr args ) {
  if ( !nl->HasLength( args, 2 ) ) {
    return listutils::typeError( "two arguments expected" );
  }
  if ( !Stream<Tuple>::checkType( nl->First( args ) ) ) {
    return listutils::typeError( "first argument must be a stream of tuples" );
  }
  if ( !listutils::isRelDescription( nl->Second( args ) ) ) {
    return listutils::typeError( "second argument must be a relation" );
  }
  ListExpr attrlist = nl->Second( nl->Second( nl->First( args ) ) );
  string error = "stream must contain tuples of type"
                 "((x y) (Field string) (Value text)) [in arbitrary order]";
  // retrieve the input tuple positions of key, field and value, ignore type
  int posV = -1;
  int posK = -1;
  int posF = -1;
  int pos = 0;
  while ( !nl->IsEmpty( attrlist ) ) {
    ListExpr attr = nl->First( attrlist );
    attrlist = nl->Rest( attrlist );
    if ( CcString::checkType( nl->Second( attr ) )
        && ( listutils::isSymbol( nl->First( attr ), "Field" ) ) ) {
      posF = pos;
    }
    else if ( FText::checkType( nl->Second( attr ) )
        && ( listutils::isSymbol( nl->First( attr ), "Value" ) ) ) {
      posV = pos;
    }
    else if ( !listutils::isSymbol( nl->First( attr ), "Type" )
        && ( posK < 0 ) ) {
      posK = pos;
    }
    pos++;
  }
  // value, field or key were not found
  if ( posV == -1 || posK == -1 || posF == -1 ) {
      return listutils::typeError( error );
  }
  ListExpr tuple = nl->Second( nl->Second( args ) );
  ListExpr result = nl->TwoElemList( nl->SymbolAtom
                                   ( Stream<Tuple>::BasicType() ) , tuple );
  return nl->ThreeElemList( nl->SymbolAtom( "APPEND" ),
                            nl->ThreeElemList( nl->IntAtom( posF ),
                                               nl->IntAtom( posK ),
                                               nl->IntAtom( posV ) ),
                            result );
}
/*

2.116.2 Class ~fromFieldsInfo~

*/

class FromFieldsInfo {
  private:
    Stream<Tuple> stream;
    Tuple* tuple;
    Tuple* inTuple;
    TupleType* tt;
    vector<InObject> infuns;
    vector<ObjectCreation> createfuns;
    map<string,int> field2pos;
    map<string,int>::iterator attrPos;
    int attrMax; // number of attributes in every complete output tuple
    Attribute* key;
    vector<ListExpr> typelists;
    int posF, posK, posV; // input tuple positions of Field, Key and Value
    
    pair<int, int> getMainType( ListExpr type ){
      while( !( nl->IsAtom( nl->First( type ) ) &&
              ( nl->AtomType( nl->First( type ) == IntType ) ) ) ) {
        type = nl->First( type );
      }
      pair<int, int> result( nl->IntValue( nl->First( type ) ),
                             nl->IntValue( nl->Second( type ) ) );
      return result;
    }
    
  public:
    FromFieldsInfo( Word& is, TupleType* _tt , int pF, int pK, int pV,
                    Supplier s ): 
              stream( is ), tuple( 0 ), posF( pF ), posK( pK ), posV( pV ) {
      inTuple = 0;                      
      key = 0;                        
      stream.open();
      tt = _tt; 
      tt->IncReference();
      AlgebraManager* am = SecondoSystem::GetAlgebraManager();
      ListExpr tl = nl->Second(GetTupleResultType(s));
      ListExpr attrList = nl->Second( tl );
      int pos = -1;
      while( !nl->IsEmpty( attrList ) ) {
        pos++;
        ListExpr attr = nl->First( attrList );
        attrList = nl->Rest( attrList );
        string field = nl->SymbolValue( nl->First( attr ) );
        field2pos[field] = pos; // assign each attribute name to its position
        pair<int, int> t = getMainType( nl->Second( attr ) );
        // collect In functions, typelists and Create functions
        infuns.push_back( am->InObj( t.first, t.second ) );
        typelists.push_back( nl->Second( attr ) );
        createfuns.push_back( am->CreateObj( t.first, t.second ) );
      }
      pos++;
      attrMax = pos;
    }
    
    ~FromFieldsInfo(){
      if ( tuple ) {
        tuple->DeleteIfAllowed();
      }
      stream.close();
      tt->DeleteIfAllowed(); 
      if ( key ) {
        key->DeleteIfAllowed();
      }
    }
    
    // this function converts an inTuple into a tuple attribute
    bool processTuple( Tuple* inTuple, Tuple* tuple ){
      CcString* fieldCc = ( CcString* ) inTuple->GetAttribute( posF );
      if ( fieldCc->IsDefined() ) {
        attrPos = field2pos.find( fieldCc->GetValue() );
        if( attrPos != field2pos.end() ) {       
          FText* attrvalue = (FText*) inTuple->GetAttribute( posV );
          string valuestr;
          if ( attrvalue->IsDefined() ) {
            valuestr = attrvalue->GetValue();
          }
          // call Create function depending on attribute
          Word creation = createfuns[ attrPos->second ]
                        ( typelists[ attrPos->second ] );
          Attribute* newAttr = (Attribute*)creation.addr;                 
          if ( newAttr->hasTextRepresentation() ) {
            bool fromtextok = newAttr->fromText( valuestr );
            if ( !fromtextok ) {
              cout << "string could not be read" << endl;
              newAttr->SetDefined(false);
            }
          }
          else { // In function necessary
            ListExpr instance; // needed to call the In function
            bool instanceok = nl->ReadFromString( valuestr, instance );
            if( !instanceok ) {
              cout << "string could not be read" << endl;
              newAttr->SetDefined(false);
            }
            else {
              int errorPos = 0;
              ListExpr errorInfo;
              bool correct;
              Word in = infuns[ attrPos->second ] // call In function
                      ( typelists[ attrPos->second ],
                        instance, errorPos, errorInfo, correct );
              if ( correct ) {
                newAttr->DeleteIfAllowed();
                newAttr = ( Attribute* )in.addr;             
              }
              else {
                  cout << "In function failed" << endl;
                  newAttr->SetDefined(false);
              }
            }           
          }
          // attribute already set -> overwrite
          if ( tuple->GetAttribute( attrPos->second ) != 0 ) {
            cout << "attribute " ;
            inTuple->GetAttribute( posF )->Print(cout) <<
                 " was already defined for key " << key <<
                 " and is now overwritten" << endl;
            // delete old attribute value
            tuple->GetAttribute( attrPos->second )->DeleteIfAllowed();
          }            
          tuple->PutAttribute( attrPos->second, newAttr );
        } 
        else { // attribute does not exist -> input tuple is ignored
          cout << "attribute ";
          inTuple->GetAttribute( posF )->Print(cout) <<
               " does not exist" << endl;
          return false;           
        }                                              
      }
      else { // undefined attribute -> input tuple is ignored
        cout << "undefined attribute " << endl;
        return false;
      }     
      return true;   
    }
    
    // fills the output tuple with undefined values if necessary
    bool fillTuple( Tuple* tuple ){
      for ( int pos = 0; pos < attrMax; pos++ ) {
        if ( !tuple->GetAttribute( pos ) ) {
          Word creation = createfuns[ pos ]( typelists[ pos ] );
          Attribute* emptyAttr = ( Attribute* )creation.addr;
          emptyAttr->SetDefined( false );
          tuple->PutAttribute( pos, emptyAttr );
        }
      }
      return true;
    }
    
    Tuple* nextTuple() {
      Tuple* tuple = new Tuple( tt );
      int attrsStored = 0; // #attributes currently stored in the output tuple
      while ( attrsStored < attrMax ) {
        if ( !inTuple ) {
          inTuple = stream.request(); 
        }
        // not enough attributes for output tuple -> fill
        if ( ( !inTuple && attrsStored < attrMax - 1 && attrsStored > 0 ) ) {
          bool filled = fillTuple( tuple );
          if ( filled ) {
            cout << "not enough attributes for key " << key->toText()
                 << ", tuple filled with undefined values" << endl;
            return tuple;
          }
          else {
            tuple->DeleteIfAllowed();
            return 0;
          }
        }
        if ( !inTuple ) {
          tuple->DeleteIfAllowed();
          return 0;
        }
        // new key, output tuple not completed -> fill
        if ( key && key->Compare( inTuple->GetAttribute( posK ) )
                 && attrsStored < attrMax - 1 && attrsStored > 0 ) {
          bool filled = fillTuple( tuple );
          cout << "unexpected new key " << key
               << ", tuple filled with undefined values" << endl;
          if ( filled ) {
            return tuple;
          }
          else {
            tuple->DeleteIfAllowed();
            return 0;
          }
        }
        if ( key ) {
          key->DeleteIfAllowed();
        }
        key = inTuple->GetAttribute( posK )->Copy();
        bool ok = processTuple( inTuple, tuple ); // the operator's main task
        if ( ok ) {
          attrsStored++;
        }
        inTuple->DeleteIfAllowed();
        inTuple = 0;
      }
      return tuple;
    }   
};



/*

2.116.3 Value mapping for ~fromFields~

*/

int fromFieldsFun( Word* args, Word& result, int message,
           Word& local, Supplier s ) {
  FromFieldsInfo* ffi = (FromFieldsInfo*) local.addr;
  TupleType* tt = (TupleType*) qp->GetLocal2(s).addr;

  switch( message ) {
    case INIT: {
      tt = new TupleType(nl->Second(GetTupleResultType(s)));
      qp->GetLocal2(s).addr=tt;
      return 0;
    }
    case FINISH: {
       if(tt){
          tt->DeleteIfAllowed();
          qp->GetLocal2(s).addr=0;
       }
       return 0;
    }

    case OPEN: {
      if ( ffi ) {
           delete ffi;
      }
      int posF = ( ( CcInt* )args[ 2 ].addr )->GetValue();
      int posK = ( ( CcInt* )args[ 3 ].addr )->GetValue();
      int posV = ( ( CcInt* )args[ 4 ].addr )->GetValue();
      local.addr = new FromFieldsInfo( args[ 0 ], tt, posF, posK, posV,s );
      return 0;
    }
    case REQUEST: {
      if ( !ffi ) {
           return CANCEL;
      }
      result.addr = ffi->nextTuple();
      return result.addr?YIELD:CANCEL;
    }
    case CLOSE: {
      if ( ffi ) {
           delete ffi;
           local.addr = 0;
      }
      return 0;
    }
    default: {
      return -1;
    }
  }
}
       
/*

2.116.4 Description for Operator ~fromFields~

*/

struct fromFieldsInfo : OperatorInfo {
  fromFieldsInfo() : OperatorInfo() {
    name      = "fromFields";
    signature = "(stream(tuple(keyvalue: keytype, Field: string, "
                  "Type: text, Value: text))) x (rel(tuple(X))) "
          "-> (stream(tuple(X)))";
    syntax    = "_ feed fromFields [ _ ]";
    meaning   = "Composes a stream of tuple items into a certain relation.";
  }
};



/*
2.117 Operator ~applyToAll~

This operator receives a tuple stream as well as a function  t -> x, x,t in DATA
and applies this function to all attributes of type t within each tuple. The 
attribute is replaced by the new value.

2.117.1 Type Mapping

*/
ListExpr applyToAllTM(ListExpr args){
   string err = "stream(Tuple) x (DATA -> DATA) expected";
   if(!nl->HasLength(args,2)){
     return listutils::typeError(err + " (wrong number of arguments)");
   }
   if(!Stream<Tuple>::checkType(nl->First(args))){
     return listutils::typeError(err + " (first arg is not a tuple stream)");
   }
   ListExpr fun = nl->Second(args);
   if(!listutils::isMap<1>(fun)){
     return listutils::typeError(err +" (second arg is not a function)");
   }
   ListExpr funArg = nl->Second(fun);
   ListExpr funRes = nl->Third(fun);
   if(!listutils::isDATA(funArg) || !listutils::isDATA(funRes)){
     return listutils::typeError(err + " ( function not of type DATA -> DATA");
   }

   // tests are ok, build result lists
   ListExpr resList = nl->TheEmptyList();
   ListExpr resLast = nl->TheEmptyList();
   ListExpr replaceList = nl->TheEmptyList();
   ListExpr replaceLast = nl->TheEmptyList();

   ListExpr attrList = nl->Second(nl->Second(nl->First(args)));
   bool firstRes = true;
   bool firstReplace = true;
   int attrNo = 0;

   while(!nl->IsEmpty(attrList)){
     ListExpr attr = nl->First(attrList);
     if(nl->Equal(funArg, nl->Second(attr))){
        if(firstReplace) {
          replaceList = nl->OneElemList(nl->IntAtom(attrNo));
          replaceLast = replaceList;
          firstReplace = false;
        } else {
          replaceLast = nl->Append(replaceLast, nl->IntAtom(attrNo));
        }
        attr = nl->TwoElemList( nl->First(attr), funRes);
     }
     if(firstRes){
        resList = nl->OneElemList(attr);
        resLast = resList;
        firstRes = false;
     } else {
        resLast = nl->Append(resLast, attr);
    }
    attrNo++;
    attrList = nl->Rest(attrList);
   }

   return nl->ThreeElemList(
                nl->SymbolAtom(Symbols::APPEND()),
                replaceList,
                 nl->TwoElemList(nl->SymbolAtom(Stream<Tuple>::BasicType()),
                    nl->TwoElemList( nl->SymbolAtom(Tuple::BasicType()),
                                     resList)));
}

/*
2.117.2 LocalInfo for applyToAll operator

*/

class applyToAllLocalInfo{

 public:
   
   applyToAllLocalInfo(Word& s, Word& f, vector<int>& pos, TupleType* _tt):
       stream(s), fun(f), positions(pos) {
       stream.open();
       tt = _tt;
       tt->IncReference();
       positions.push_back(-1);
   }
   ~applyToAllLocalInfo(){
       tt->DeleteIfAllowed();
       stream.close();
    }

   Tuple* nextTuple(){
     Tuple* inTuple = stream.request();
     if(!inTuple){
       return 0;
     }
     if(positions.empty()){
        return inTuple;
     }
     int p_index=0;
     int p = positions[p_index];
     Tuple* resTuple = new Tuple(tt);
     for(int i=0;i<inTuple->GetNoAttributes();i++){
       if(i!=p){  // do not apply the function
          resTuple->CopyAttribute(i,inTuple, i);
       } else {
         Attribute* inAttr = inTuple->GetAttribute(i); 
         ArgVectorPointer funArgs = qp->Argument(fun.addr);
         (*funArgs)[0] = inAttr;
          Word funRes;
          qp->Request(fun.addr, funRes);
          resTuple->PutAttribute(i, ((Attribute*)funRes.addr)->Clone());
          p_index++;
          p = positions[p_index];
       }
     }
     inTuple->DeleteIfAllowed();
     return resTuple;
   }


 private:
    Stream<Tuple> stream;
    Word fun;
    vector<int> positions;
    TupleType* tt;
};


/*
2.117.3 ValueMapping for applyToAll

*/

int applyToAllVM( Word* args, Word& result, int message,
           Word& local, Supplier s ) {


  applyToAllLocalInfo* li = (applyToAllLocalInfo*) local.addr;
  TupleType* tt = (TupleType*) qp->GetLocal2(s).addr;
 
  switch(message){
    case INIT : {
      tt = new TupleType(nl->Second(GetTupleResultType(s)));
      qp->GetLocal2(s).addr=tt;
      return 0;
    }

    case FINISH: {
      if(tt){
        tt->DeleteIfAllowed();
        qp->GetLocal2(s).addr=0;
      }
      return 0;
    }

    case OPEN: {
                  if(li){
                    delete li;
                  }
                  vector<int> attrPos;
                  for(int i=2;i<qp->GetNoSons(s); i++){
                     attrPos.push_back(((CcInt*)args[i].addr)->GetValue());
                  }  
                  local.addr = new applyToAllLocalInfo(args[0], args[1], 
                                                      attrPos, tt);
                  return 0;
               }
     case REQUEST: {
                     if(!li){
                       return CANCEL;
                     }
                     result.addr = li->nextTuple();
                     return result.addr?YIELD:CANCEL;
                   }
    case CLOSE: {
                   if(li){
                     delete li;
                     local.addr=0;
                   }
                   return  0;
                }
                 
  }
  return -1; 
}


const string applyToAllSpec  =
"( ( \"Signature\" \"Syntax\" \"Meaning\" \"Example\" ) "
"( <text>stream(tuple(x)) x fun  -> stream(tuple(y))"
  "</text--->"
  "<text>stream applyToAll [ fun  ] </text--->"
  "<text>Replaces all attributes of argument type of fun by the "
  " result of the function."
  "</text--->"
  "<text>query ten feed applyToAll[ fun(int i) i * i] tconsume "
  " </text--->"
  ") )";


Operator applyToAll(
          "applyToAll",
          applyToAllSpec,
          applyToAllVM,
          Operator::SimpleSelect,
          applyToAllTM);



/*
2.117 Operator = 

This operator compares two stream of the same type for equality.
The stream type can be either a tuple stream or an attribute stream.

2.117.1 Type Mapping

*/

ListExpr equalStreamsTM(ListExpr args){
  string err = "stream(X) x stream(X) expected";
  if(!nl->HasLength(args,2)){
    return listutils::typeError(err + " (wrong number of arguments");
  }
  ListExpr arg1 = nl->First(args);
  ListExpr arg2 = nl->Second(args);
  if(!nl->Equal(arg1,arg2)){
    return listutils::typeError(err + " (arguments have different types)");
  }  
  if(!Stream<Attribute>::checkType(arg1) &&
     !Stream<Tuple>::checkType(arg1)){
    return listutils::typeError(err + " ( arguments are not streams)");
  }
  return listutils::basicSymbol<CcBool>();
}

/*
2.117.2 ValueMapping

*/
template<class StreamType, class Cmp>
int equalStreamsVM1( Word* args, Word& result, int message,
           Word& local, Supplier s ) {

   result = qp->ResultStorage(s);
   CcBool* res = (CcBool*) result.addr;

   Stream<StreamType> s1(args[0]);
   Stream<StreamType> s2(args[1]);

   s1.open();
   s2.open();

   StreamType* e1 = s1.request();
   StreamType* e2 = s2.request();
   Cmp compare;   

   while( (e1!=0) && (e2!=0)){
       int cmp = compare(e1,e2);
       e1->DeleteIfAllowed();
       e2->DeleteIfAllowed();
       if(cmp!=0){ // different elements found
           s1.close();
           s2.close();
           res->Set(true,false);
           return 0; 
       }
       e1 = s1.request();
       e2 = s2.request(); 
   }
   s1.close();
   s2.close();

   if(e1){ // s1 has more elements
     e1->DeleteIfAllowed();
     res->Set(true,false);
     return 0;
   }
   if(e2) {
     e2->DeleteIfAllowed();
     res->Set(true,false);
     return 0;
   }
   res->Set(true,true);
   return 0; 
}


/*
2.117.4 Auxiliary classes for Value Mapping

*/
class AttrCompareAlmost{
  public:
  int operator()(Attribute* a1, Attribute* a2){
     return a1->CompareAlmost(a2);
  }  
};

/*
2.117.4 ValueMapping Array and SelectionFunction

*/

ValueMapping equalStreamsVM[] = {
     equalStreamsVM1<Tuple,LexicographicalTupleCmpAlmost>,
     equalStreamsVM1<Attribute,AttrCompareAlmost>  
   };

int equalStreamsSelect(ListExpr args){

  ListExpr arg1 = nl->First(args);
  if(Stream<Attribute>::checkType(arg1)){ // attribute stream
     return 1;
  } else { // tuple stream
     return 0;
  }
}

/*
2.117.5 Specification

*/
OperatorSpec equalStreamsSpec(
    "stream(X) x stream(X) -> bool; with X in {attribute, tuple(Y)}",
    "  _ = _ ",
    " Check streams for equality ",
    " query (plz feed) = (plz feed)");

/*
2.117.6 Operator instance

*/
Operator equalStreams(
     "=",
     equalStreamsSpec.getStr(),
     2,
     equalStreamsVM,
     equalStreamsSelect,
     equalStreamsTM
  );



/*
2.118 Operator ~replaceAttr~

This operator replaces attributes by function values

*/
ListExpr replaceAttrTM(ListExpr args){
  string err = "stream(tuple(x)) x funlist expected";
  if(!nl->HasLength(args,2)) {
    return listutils::typeError(err + " (wrong number of arguments)");
  }
  ListExpr stream = nl->First(args);
  if(!Stream<Tuple>::checkType(stream)){
     return listutils::typeError(err + " ( first arg is not a tuple stream)");
  }
  ListExpr funList = nl->Second(args);
  if(nl->AtomType(funList) != NoAtom){
     return listutils::typeError(err + " ( second argument is atomar)");
  }
  if(nl->IsEmpty(funList)){
      return listutils::typeError(err + " (funlist is empty)");
  }

  // first, we store all functions within a map<string, ListExpr>
  // the listexpr is the return type of the map
  ListExpr tupleT = nl->Second(stream);
  ListExpr attrList = nl->Second(tupleT);

  string error;
  ListExpr attrType;
  map<string,ListExpr> newTypes;
  ListExpr indexList = nl->TheEmptyList();
  bool first = true;
  ListExpr last = nl->TheEmptyList();

  while(!nl->IsEmpty(funList)){
    ListExpr fun = nl->First(funList);
    funList = nl->Rest(funList);
    if(!nl->HasLength(fun,2)){ // (name, map)
       return listutils::typeError(err + " ( invalid function found)");
    }
    ListExpr name = nl->First(fun);
    if(!listutils::isValidAttributeName(name, error)){
      return listutils::typeError(nl->ToString(name) + 
                                  " is not a valid attribute name (" 
                                  + error +")");
    }
    string nameS = nl->SymbolValue(name);
    int index = listutils::findAttribute(attrList,nameS,attrType);
    if(index==0){
       return listutils::typeError("AttrName + " + nameS + 
                                   " not present in Tuple");
    }
    if(newTypes.find(nameS)!=newTypes.end()){
      return listutils::typeError("Attribute " + nameS 
                                   + " is mentioned twice");
    }

    ListExpr maplist = nl->Second(fun);
    if(!listutils::isMap<1>(maplist)){
       return listutils::typeError(nl->ToString(maplist) 
                                   + " is not a map with 1 argument");
    }
    if(!nl->Equal(tupleT, nl->Second(maplist))){
      return listutils::typeError("function argument for " + nameS
                       + "does not correspond to the tuple in stream");
    }
    ListExpr resType = nl->Third(maplist);
    if(!Attribute::checkType(resType)){
      return listutils::typeError("function result for " + nameS
                         + " is not an attribute");
    }
    // store index for appending
    if(first){
       indexList = nl->OneElemList(nl->IntAtom(index-1));
       last = indexList;
       first = false;
    } else {
       last = nl->Append(last, nl->IntAtom(index-1));  
    }
    // store result type in map
    newTypes[nameS] = resType;
  }

  // now, we create the result attr type
  ListExpr resAttrList=nl->TheEmptyList();
  first = true;
  while(!nl->IsEmpty(attrList)){
    ListExpr attr = nl->First(attrList);
    attrList = nl->Rest(attrList);
    string name = nl->SymbolValue(nl->First(attr));
    ListExpr newAttr;
    if(newTypes.find(name)==newTypes.end()){ // no change
       newAttr = attr;
    } else {
       newAttr = nl->TwoElemList(nl->First(attr),
                                 newTypes[name]);
    }
    if(first){
       resAttrList = nl->OneElemList(newAttr);
       last = resAttrList;
       first = false;
    } else {
       last = nl->Append(last, newAttr);
    }
  }
  ListExpr resList = nl->TwoElemList(listutils::basicSymbol<Stream<Tuple> >(),
                            nl->TwoElemList(listutils::basicSymbol<Tuple>(),
                            resAttrList));


  return nl->ThreeElemList( nl->SymbolAtom(Symbol::APPEND()),
                            indexList,
                            resList);
}

/*
2.118.2 LocalInfo 

*/

class replaceAttrLocalInfo{

 public:
    replaceAttrLocalInfo(Word& _stream,
                         Supplier _funs,
                         map<int,int> _mapping,
                         TupleType* _tupleType): 
                stream(_stream), funs(_funs),mapping(_mapping){
       stream.open();
       tt = _tupleType;
       tt->IncReference();
    }

   ~replaceAttrLocalInfo(){
       tt->DeleteIfAllowed();
       stream.close();
    }

    Tuple* next(){
       Tuple* orig = stream.request();
       if(!orig){
          return 0;
       } 
       Tuple* res = new Tuple(tt);
       for(int i=0;i<res->GetNoAttributes();i++){
          if(mapping.find(i)==mapping.end()){
              // no function known
              res->CopyAttribute(i,orig,i);
          } else {
             // evaluate function
             Supplier sf = qp->GetSon(funs,mapping[i]);
             Supplier sfm = qp->GetSon(sf,1); // access mapping
             ((*qp->Argument(sfm))[0]).setAddr(orig);
             qp->Request(sfm,value);
             Attribute* a = (Attribute*) value.addr;
             res->PutAttribute(i,a->Clone());
          }
       }
       orig->DeleteIfAllowed();
       return res;

    }

 private:
    Stream<Tuple> stream;
    Supplier funs;
    map<int,int> mapping;
    TupleType* tt;
    Word value; // class member to avoid constructor calls of word
};



/*
2.118.3 ValueMapping

*/

int replaceAttrVM( Word* args, Word& result, int message,
           Word& local, Supplier s ) {

  replaceAttrLocalInfo* li = (replaceAttrLocalInfo*) local.addr;
  TupleType* tt = (TupleType*) qp->GetLocal2(s).addr;
  switch(message){
    case INIT: {
      tt = new TupleType(nl->Second(GetTupleResultType(s)));
      qp->GetLocal2(s).addr=tt;
      return 0;
    }
    case FINISH: {
       if(tt){
         tt->DeleteIfAllowed();
         qp->GetLocal2(s).addr=0;
       }
       return 0;
    }

    case OPEN: {
          if(li){
            delete li;
            local.addr = 0;
          }
          map<int,int> pos;
          for(int i=2;i<qp->GetNoSons(s);i++){
             int index = ( ((CcInt*) args[i].addr)->GetValue());
             pos[index] = i-2;
          }
          local.addr = new replaceAttrLocalInfo(args[0], qp->GetSon(s,1), pos,
                            tt);
          return 0;
    }
    case REQUEST : {
         result.addr = li?li->next():0;
         return result.addr?YIELD:CANCEL;  
    }
    case CLOSE : {
         if(li){
           delete li;
           local.addr=0;
         }
         return 0;
    }
    default: return -1;
  }
  return -1;
}

/*
1.118.4 Specification

*/

OperatorSpec replaceAttrSpec(
    "stream(X) x funlist -> stream(Y)",
    " _ replaceAttr [ funlist ]  ",
    " Replaces attributes within a tuple stream ",
    " query plz feed replaceAttr[ Ort : .PLZ , PLZ : .PLZ + 23] consume" );

/*
2.117.6 Operator instance

*/
Operator replaceAttr(
     "replaceAttr",
     replaceAttrSpec.getStr(),
     replaceAttrVM,
     Operator::SimpleSelect,
     replaceAttrTM
  );


/*
2.118 Operator ~pfilter~


2.118.1 Type Mapping


*/

ListExpr pfilterTM(ListExpr args){

  string err = "stream(tuple(X))  x ( (tuple(X) , tuple(X) -> bool) expected";

  if(!nl->HasLength(args,2)){
     return listutils::typeError(err);
  }
  ListExpr stream = nl->First(args);
  ListExpr map = nl->Second(args);
  if(!Stream<Tuple>::checkType(stream) ||
     !listutils::isMap<2>(map)){
    return listutils::typeError(err);
  }
  ListExpr m1 = nl->Second(map);
  ListExpr m2 = nl->Third(map);
  ListExpr mr = nl->Fourth(map);
  ListExpr tuple = nl->Second(stream);
  if(!nl->Equal(tuple,m1) || !nl->Equal(tuple,m2)){
    return listutils::typeError("function arguments does not fi"
                                 "t the tuple type");
  }
  if(!CcBool::checkType(mr)){
    return listutils::typeError("function result not bool");
  }
  return stream;

}

int pfilterVM( Word* args, Word& result, int message,
           Word& local, Supplier s ) {

  Tuple* lastTuple = (Tuple*) local.addr;
  switch(message){
    case OPEN: {
        if(lastTuple){
          lastTuple->DeleteIfAllowed();
           local.addr = 0;  
        }
        qp->Open(args[0].addr);
        return 0;
    }
    case REQUEST: {
      if(!lastTuple){ // first tuple
         Word tupleW;
         qp->Request(args[0].addr, tupleW);
         if(! qp->Received(args[0].addr)){
            return CANCEL;
         }
         local.addr = tupleW.addr;
         lastTuple = (Tuple*) local.addr;
         lastTuple->IncReference();
         result.addr = lastTuple;
         return YIELD;   
      }
      ArgVectorPointer funargs = qp->Argument(args[1].addr);
      while(1){ 
         // get next tuple
         Word tupleW;
         qp->Request(args[0].addr, tupleW);
         if(! qp->Received(args[0].addr)){
            return CANCEL;
         }
         // evaluate function
         (*funargs)[0] = tupleW;
         (*funargs)[1].addr = lastTuple;
         Word funres;   
         qp->Request(args[1].addr, funres);
         CcBool * res = (CcBool*) funres.addr;
         if(res->IsDefined() && res->GetBoolval()){
             lastTuple->DeleteIfAllowed();
             lastTuple = (Tuple*) tupleW.addr;
             lastTuple->IncReference();
             result.addr = lastTuple;
             local.addr = lastTuple;
             return  YIELD;
         } else {
            (  (Tuple*) tupleW.addr) -> DeleteIfAllowed();
         }
      }
    }

    case CLOSE :{
        if(lastTuple){
           lastTuple->DeleteIfAllowed();
            local.addr = 0;
        }
        qp->Close(args[0].addr);
        return 0;
    }

  }

  return -1;  


}


OperatorSpec pfilterSpec(
    "stream(tuple(X)) x (tuple(X) x tuple(X) -> bool) -> stream(tuple(X))",
    " _ pfilter [ fun ]  ",
    " Returns the first tuple in the stream and all tuples fulfilling the"
    " filter consition which uses the current and the last tuple",
    " query ten feed pfilter[ (. - ..) > 3]  consume" );

/*
2.117.6 Operator instance

*/
Operator pfilter(
     "pfilter",
     pfilterSpec.getStr(),
     pfilterVM,
     Operator::SimpleSelect,
     pfilterTM
  );


/*
2.118 Operator extendX

*/

ListExpr extendXTM(ListExpr args){

  int len = nl->ListLength(args);

  if(len!=4){
    return listutils::typeError("3 arguments expected");
  }

  string err = "stream(tuple) x (AttrName+)  x "
               "(tuple -> stream(DATA)) x DATA expected";

  // check for tuple stream
  ListExpr ts = nl->First(args);
  if(!Stream<Tuple>::checkType(ts)){
    return listutils::typeError(err);
  }

  // check for list of attribute names
  ListExpr anames = nl->Second(args);
  if(nl->AtomType(anames)!=NoAtom || nl->IsEmpty(anames)){
     return listutils::typeError(err);
  }

  while(!nl->IsEmpty(anames)){
     ListExpr fa = nl->First(anames);
     string msg;
     if(!listutils::isValidAttributeName(fa,msg)){
       return listutils::typeError("invalid Attribute name " + msg);
     }
     anames = nl->Rest(anames);
  }
  anames = nl->Second(args);

  ListExpr fun = nl->Third(args);

  if(!listutils::isMap<1>(fun)){
     return listutils::typeError(err);
  }
  if(!nl->Equal(nl->Second(ts), nl->Second(fun))){
     return listutils::typeError("Fun argument does not fit the tuple type");
  }

  ListExpr res = nl->Third(fun);

  if(!Stream<Attribute>::checkType(res)){
    return listutils::typeError("result of function is not a stream(DATA)");
  }

  res = nl->Second(res); // remove stream

  // build result attributes
  ListExpr extAttrList = nl->OneElemList( nl->TwoElemList(
                                             nl->First(anames),
                                             res));
  ListExpr last = extAttrList;
  anames = nl->Rest(anames);
  while(!nl->IsEmpty(anames)){
      last = nl->Append(last, nl->TwoElemList(nl->First(anames), res));
      anames = nl->Rest(anames);
  }

  ListExpr resAttrList = listutils::concat( nl->Second(nl->Second(ts)), 
                                            extAttrList);

  ListExpr resultList = nl->TwoElemList(
                            listutils::basicSymbol<Stream<Tuple> >(),
                            nl->TwoElemList(listutils::basicSymbol<Tuple>(),
                                            resAttrList));



  if(!Stream<Tuple>::checkType(resultList)){
    return listutils::typeError("Name conflicts in result tuple");
  }
  ListExpr defaultValue = nl->Fourth(args);
  if(!nl->Equal(res,defaultValue)){
    return listutils::typeError("Default value" + nl->ToString(defaultValue) +
                          " != stream element from fun " + nl->ToString(res) );
  }
  return resultList;
}


class extendXLocal{
   public:

      extendXLocal(Word _istream, 
                   Supplier _fun, int _no,
                   Attribute* _defaultValue, 
                   TupleType* _tupleType):
      istream(_istream), fun(_fun), no(_no), defaultValue(_defaultValue){
      tt = _tupleType;
      tt->IncReference();
      istream.open();
   }

   ~extendXLocal(){
       tt->DeleteIfAllowed();
       istream.close();
    }

   Tuple* nextTuple(){
     Tuple* ituple = istream.request();
     if(ituple==0){
       return 0;
     }
     Tuple* rtuple = new Tuple(tt);
     for(int i=0;i<ituple->GetNoAttributes();i++){
       rtuple->CopyAttribute(i,ituple,i);
     }
     ArgVectorPointer funargs = qp->Argument(fun);
     ((*funargs)[0]).setAddr(ituple);


     int pos = 0;
     bool ok = true;
     Word result;
     qp->Open(fun);
     while(pos < no && ok) {
        qp->Request(fun,result);
        ok = qp->Received(fun);
        if(ok){
           rtuple->PutAttribute(ituple->GetNoAttributes()+pos,
                               (Attribute*) result.addr);
        } else {
           rtuple->PutAttribute(ituple->GetNoAttributes()+pos,
                            defaultValue->Copy());
        }
        pos++;
     }
     qp->Close(fun);
     while(pos < no){
      rtuple->PutAttribute(ituple->GetNoAttributes()+pos,
                       defaultValue->Copy());
      pos++;
     }
     ituple->DeleteIfAllowed();
     return rtuple;
   }

  private:
    Stream<Tuple> istream;
    Supplier fun;
    int no;
    Attribute* defaultValue;
    TupleType* tt;
};

int extendXVM( Word* args, Word& result, int message,
           Word& local, Supplier s ) {

 extendXLocal* li = (extendXLocal*)local.addr;
 TupleType* tt = (TupleType*) qp->GetLocal2(s).addr;

  switch(message){
    case INIT : {
       tt = new TupleType(nl->Second(GetTupleResultType(s)));
       qp->GetLocal2(s).addr = tt;
       return 0;
    } 

    case FINISH: {
       if(tt){
          tt->DeleteIfAllowed();
          qp->GetLocal2(s).addr = 0;
       }
       return 0;
    }

    case OPEN : {
                  if(li) delete li;
                  // stream attrlist fun default
                  //   0      1       2     3
                  int ns = qp->GetNoSons(qp->GetSon(s,1));
                  Attribute* dv = (Attribute*) args[3].addr;
                  local.addr = new extendXLocal( args[0], args[2].addr,
                                                 ns, dv, tt);
                }
                return 0;
    case REQUEST: result.addr = li?li->nextTuple():0;
                  return result.addr?YIELD:CANCEL;
    case CLOSE :{
        if(li){
           delete li;
           local.addr = 0;
        }
        return 0;
    }
  } 
  return -1; 

}

OperatorSpec extendXSpec(
     "stream(tuple(X)) x AttrName + x fun(tuple) -> stream(DATA) "
     "x DATA  -> stream(tuple())",
     " _ extendX [ attrnames ; fun ; default ]  ",
     " Appends so much attributes from the stream created by the function"
     " to the original tuple as new attributes names are given. "
     " If less attributes comes out from the function stream, the default"
     " value is used to fill up the missing attributes",
     " query Kinos feed extendX[ S1, S2, S3 ; intstream(4,5), 0); \"\"" );

 Operator extendXOP(
      "extendX",
      extendXSpec.getStr(),
      extendXVM,
      Operator::SimpleSelect,
      extendXTM
 );


/*
2.118 Operator countMt

2.118.1 Type Mapping

*/

ListExpr countMtTM(ListExpr args){
   string err = "stream(Tuple) x int expected";
   if(!nl->HasLength(args,2)){
     return listutils::typeError(err);
   }
   if(!Stream<Tuple>::checkType(nl->First(args)) ||
      !CcInt::checkType(nl->Second(args))){
     return listutils::typeError(err);
   }
   return listutils::basicSymbol<CcInt>();
}
/*
2.118.2 Value Mapping

*/
int countMtVM( Word* args, Word& result, int message,
           Word& local, Supplier s ) {

    result =  qp->ResultStorage(s);
    CcInt* res = (CcInt*) result.addr;
    int count = 0;
    CcInt* interval = (CcInt*) args[1].addr;
    int iv = interval->IsDefined()?interval->GetValue():0;
    if(iv<0){
       iv = 0;
    }  
    Stream<Tuple> stream(args[0]);    
    stream.open();
    Tuple* tuple=0;
    clock_t time = clock();
    iv = (iv * CLOCKS_PER_SEC ) / 1000;
    while( (tuple = stream.request())!=0){
       count++;
       tuple->DeleteIfAllowed();
       clock_t ct = clock();
       if((ct-time)>=iv){
          cout << "sw ta" << endl;
          qp->switchTransaction(s);
          time = ct;
       }
    }
    res->Set(true,count);
    return 0;
}


OperatorSpec countMtSpec(
     " stream(tuple) x int -> int",
     " _ contMt[_]",
     "Count the number of tuples in the result."
     "If used as the root of the operator tree,"
     " after a certain amount of time (milliseconds "
     "given in  the 2th parameter), the current state"
     " of this query is committed to the database.",
     " query strassen feed countMt[100]\"\"" );

 Operator countMtOP(
      "countMt",
      countMtSpec.getStr(),
      countMtVM,
      Operator::SimpleSelect,
      countMtTM
 );



/*
Operator bringToMemory

*/

ListExpr bringToMemoryTM(ListExpr args){

  string err = "tuple expected";
  if(!nl->HasLength(args,1)){
    return listutils::typeError(err);
  }
  if(!Tuple::checkType(nl->First(args))){
    return listutils::typeError(err);
  }
  return listutils::basicSymbol<CcBool>();
}

int bringToMemoryVM( Word* args, Word& result, int message,
           Word& local, Supplier s ) {
  Tuple* tuple = (Tuple*) args[0].addr;
  tuple->bringToMemory();
  result=qp->ResultStorage(s);
  ((CcBool*) result.addr)->Set(true,true);
  return 0; 
}


OperatorSpec bringToMemorySpec(
     " tuple->tuple ",
     " _ bringToMemory",
     "Brings all parts of a tuple into main memory.",
     " query strassen feed extend[Dummy : . bringToMemory] consume ");


 Operator bringToMemoryOP(
      "bringToMemory",
      bringToMemorySpec.getStr(),
      bringToMemoryVM,
      Operator::SimpleSelect,
      bringToMemoryTM
 );



ListExpr feedSTM(ListExpr args){
    string err = "stream({text,string}) x rel(X) expected";
    if(!nl->HasLength(args,2)){
      return listutils::typeError(err + " (wrong number of args)");
    }
    if(   !Stream<CcString>::checkType(nl->First(args))
       && !Stream<FText>::checkType(nl->First(args))){
      return listutils::typeError(err + " (invalid type for arg 1)");
   }
   if(!Relation::checkType(nl->Second(args))){
      return listutils::typeError(err + " (second arg is not a relation)");
   }
   return nl->TwoElemList( listutils::basicSymbol<Stream<Tuple> >(),
                           nl->Second(nl->Second(args)));
}

template<class T>
class feedSLI{

   public:
      feedSLI(Word _stream, ListExpr _relType):stream(_stream), 
               relType(_relType), relation(0),iterator(0){
         stream.open();
         ctlg = SecondoSystem::GetCatalog();
      }

      ~feedSLI(){
          destroyRelAndIt();
          stream.close();
      }

      Tuple* next(){

        while(true){
           if(iterator){
               Tuple* t = iterator->GetNextTuple();
               if(t){
                 return t;
               }  else {
                  destroyRelAndIt();
               }        
           }
           retrieveNextIterator();
           if(iterator==0){
             return 0;
           }
        }

      }

   private:
      Stream<T> stream;
      ListExpr relType;
      Relation* relation;
      GenericRelationIterator* iterator;
      SecondoCatalog* ctlg;


      void destroyRelAndIt(){
         if(iterator){delete iterator;}
         if(relation){delete relation;}
         iterator=0;
         relation=0;
      }

      void retrieveNextIterator(){
        T* objName;
        while( (objName = stream.request())) {
           if(!objName->IsDefined()){
              objName->DeleteIfAllowed();
           } else {
              string name = objName->GetValue();
              objName->DeleteIfAllowed();
              if(retrieveNextIterator(name)){
                 return;
              }               
           }
        }
      }

      bool retrieveNextIterator(const string& name){
         assert(iterator==0);
         assert(relation==0);
         if(!ctlg->IsObjectName(name)){
            return false;
         } 
         ListExpr ot = ctlg->GetObjectTypeExpr(name);
         if(!nl->Equal(ot,relType)){
            return false;
         }
         Word r;
         bool def;
         if(!ctlg->GetObject(name,r,def)){
            return false;
         }
         if(!def){
            return false;
         }
         relation = (Relation*) r.addr;
         iterator = relation->MakeScan();
         return true;
      }
};

template<class T>
int feedSVMT( Word* args, Word& result, int message,
               Word& local, Supplier s ) {
    feedSLI<T>* li = (feedSLI<T>*) local.addr;
    switch(message){
       case OPEN:
               if(li){
                  delete li;
               }
               local.addr = new feedSLI<T>(args[0], 
                                     qp->GetType(qp->GetSon(s,1)));
               return 0;
       case REQUEST:
               result.addr=li?li->next():0;
               return result.addr?YIELD:CANCEL;
       case CLOSE:
                if(li){
                  delete li;
                  local.addr=0;
                }
    }
    return -51765; // should never happen
}

ValueMapping feedSVM[] = {
   feedSVMT<CcString>,
   feedSVMT<FText>
};

int feedSSelect(ListExpr args){
   return Stream<CcString>::checkType(nl->First(args))?0:1;
}
OperatorSpec feedSSpec(
   "stream({text,string}) x rel(tuple(X)) -> stream(tuple(X))",
   "_ feedS[_]",
   "Feeds the tuples of several relation objects having "
   "the type given in the second argument into a single stream of tuples."
   "the object names come from the input stream. Names not representing "
   "a relation with the expected type, are ignored. The value of the "
   "relation argument is also ignored." ,
   "query [const rel(tupl([N : string])) value ( (\"ten\")(\"ten\") )] "
   " feed namedtransformstream[N] feedS[ten] count"
);

Operator feedSOp(
  "feedS",
  feedSSpec.getStr(),
  2,
  feedSVM,
  feedSSelect,
  feedSTM
);



/*
2.119 Operator nth


 

2.118.1 Type Mapping

*/

ListExpr nthType( ListExpr args)
{
  if(!nl->HasLength(args,3))
   {
    return listutils::typeError("three arguments expected");
   }
   
ListExpr arg1 = nl->First(args);
  
    
if((!listutils::isTupleStream(arg1)))
   {
    return listutils::typeError("first argument must be a stream of tuples");
   }
 
 
ListExpr arg2 = nl->Second(args);

if(!CcInt::checkType(arg2))
  {
    return listutils::typeError("second argument musst be a int value");
  } 

  

ListExpr arg3 = nl->Third(args);
  
if(!CcBool::checkType(arg3))
  {
    return listutils::typeError("third argument musst be a bool value");
  } 
  
  
   
  return arg1;
  
}

 
/*
2.119.2 Value Mapping

*/

class nthInfo{

public:
 nthInfo(int iv, bool bv):
    intvalue(iv), boolvalue(bv)  {}  

 int intvalue;
 bool boolvalue; 
};


int nthValueMapping(Word* args, Word& result, int message,
                     Word& local, Supplier s) {

  nthInfo* li = (nthInfo*) local.addr;


 switch(message) {
   
    case OPEN: {     
     int intvalue = 0;
     bool boolvalue = false;
     CcInt* currentval = static_cast<CcInt*>(args[1].addr);
     CcBool* currentbool = static_cast<CcBool*>(args[2].addr);
  
     if(currentval->IsDefined() ) {
       intvalue = currentval->GetIntval(); 
     }
        
     if (intvalue <= 0) {
        intvalue = 1;
     }
  
  
     if(currentbool->IsDefined() ) {
        boolvalue = currentbool->GetBoolval(); 
     }
    
     srand(time(0)); 
     qp->Open(args[0].addr);
     if(li){
       delete li;
     }
     local.addr = new nthInfo(intvalue,boolvalue);
     return 0;
   }
    
    
   case REQUEST:  {
      if(!li) { 
         return CANCEL;
      }
      Word tuple(Address(0));
      Tuple* current = 0;
      int randvalue;       
      randvalue = rand()%li->intvalue + 1;
      if(li->boolvalue) { // exact case
        for (int i=1; i< li->intvalue; i++){ 
          qp->Request(args[0].addr, tuple);
          if(!qp->Received(args[0].addr)) {
            result.addr = 0;
            return CANCEL;
          }  else {
            current = static_cast<Tuple*>(tuple.addr);
            current -> DeleteIfAllowed();                
          }
        }   
        qp->Request(args[0].addr, tuple);
        if(qp->Received(args[0].addr)) {
           result= tuple;
           return YIELD;
        } else {
          result.addr = 0;
          return CANCEL;
        } 
      }  else { // random case     
        for (int i=1; i< randvalue; i++) {
          qp->Request(args[0].addr, tuple);
          if(!qp->Received(args[0].addr)) {
            result.addr = 0;
            return CANCEL;
          }  else {
            current = static_cast<Tuple*>(tuple.addr);
            current -> DeleteIfAllowed();                
          }
        }   
        qp->Request(args[0].addr, tuple);
        if(qp->Received(args[0].addr)) { 
           result= tuple;
           for (int i=randvalue; i<li->intvalue; i++) {
             qp->Request(args[0].addr, tuple);
             if(!qp->Received(args[0].addr)) { 
               current = static_cast<Tuple*>(result.addr);
               current -> DeleteIfAllowed();      
               result.addr = 0;
               return CANCEL;
             } else {
               current = static_cast<Tuple*>(tuple.addr);
               current -> DeleteIfAllowed();                
             }
           }    
           return YIELD;
        } else {
           result.addr = 0;
           return CANCEL;
        } 
      } //random case
    // end request case 
   } 
  
  case CLOSE: {
      qp->Close(args[0].addr);
      return 0;
  }
  
  case CLOSEPROGRESS:
    if(li){
      delete li;
      local.addr = 0;
    }
    return 0;
  
  
  case REQUESTPROGRESS: {
    ProgressInfo p1;
    ProgressInfo* pRes;
      
    pRes = (ProgressInfo*) result.addr;
    if (qp-> RequestProgress(args[0].addr, &p1) ) {  
       pRes->Copy(p1);       
       pRes->Card = p1.Card/li->intvalue;
       return YIELD;
    } else {
       return CANCEL;
    }
  }
 } // end of switch

  return 0;
 
 }
  











const string nthSpec  = "( ( \"Signature\" \"Syntax\" \"Meaning\" "
                         "\"Example\" ) "
                         "( <text>((stream (tuple([a1:d1, ... ,an:dn]))))" 
                         "x int x bool"
                         " -> (stream (tuple([ai:di,...... ,am:dm])))"
                         "</text--->"
                         "<text>_ nth[_,_]</text--->"
                         "<text>normal (TRUE): get every nth tuple"
                         " specified by the intvalue."
                         " random (FALSE):  get" 
                         " one random tuple"
                         " in the intervals specified by intvalue "
                         "stream.</text--->"
                         "<text>query Orte feed nth[2, TRUE] count"
                         "</text--->"
                              ") )";











 
 
 Operator extrelnth (
         "nth",             // name
         nthSpec,           // specification
         nthValueMapping,               // value mapping
         Operator::SimpleSelect,          // trivial selection function
         nthType                      // type mapping
);

 /*
 2.120 Operator ~exist~

 The operator checks whether at least one element is present in a tuple stream.

 2.120.1 Type mapping function of operator ~exist~

 Type mapping for ~exist~ is

 ----  stream(tuple) -> bool

 */
 ListExpr ExistTypeMap( ListExpr args )
 {

   if(nl->ListLength(args)!=1){
     return listutils::typeError("one argument expected");
   }

   string err = "stream(tuple) or stream(DATA) expected";

   ListExpr stream = nl->First(args);

   if( !Stream<Tuple>::checkType(stream)
      && !Stream<Attribute>::checkType(stream) ) {
     return listutils::typeError(err);
   }

   return listutils::basicSymbol<CcBool>();
 }
 /*
 2.120.3 Value mapping function of operator ~exist~

 */

 template<class Elem>
 int ExistVMT(Word* args, Word& result, int message, Word& local, Supplier s) {
	 Word sWord;

  result = qp->ResultStorage(s);
  CcBool* b = static_cast<CcBool*>( result.addr );

  qp->Open(args[0].addr);
  qp->Request(args[0].addr, sWord);
  if(qp->Received(args[0].addr)) {
     b ->Set(true,true);
     ((Elem*)sWord.addr)->DeleteIfAllowed();
  } else {
    b ->Set(true,false);
  }

  qp->Close(args[0].addr);

  return 0;
}

 ValueMapping ExistVM[] = {
    ExistVMT<Tuple>, ExistVMT<Attribute>
 };

 int ExistSelect(ListExpr args){
   return Stream<Attribute>::checkType(nl->First(args))?1:0;
 }

 /*
 2.120.4 Specification of operator ~exist~

 */
 const string ExistSpec  = "( ( \"Signature\" \"Syntax\" \"Meaning\" "
                          "\"Example\" ) "
                          "( "
                          "<text>stream(tuple) -> bool || stream(DATA) -> bool"
                          "</text--->"
                          "<text>_ exist</text--->"
                          "<text>Returns true, if the stream "
                          "contains at least one tuple"
		 	 	 	 	  "</text--->"
                          "<text>query Kinos feed exist"
                          "</text--->"
                           ") )";

 

 /*
 2.120.5 Definition of operator ~exist~

 */
 Operator extrelexist (
          "exist",                 	// name
          ExistSpec,               	// specification
          2,
          ExistVM,                  	// value mapping
          ExistSelect, 	// trivial selection function
          ExistTypeMap             	// type mapping
 );

/*
Operator ~obojoin~

Connects two tuple streams one by one

*/
ListExpr obojoinTM(ListExpr args){

 if(!nl->HasLength(args,2)){
   return listutils::typeError("two tuple streams expected");
 }
 if(!Stream<Tuple>::checkType(nl->First(args))){
   return listutils::typeError("first arg is not a tuple stream");
 }
 if(!Stream<Tuple>::checkType(nl->Second(args))){
   return listutils::typeError("second arg is not a tuple stream");
 }

 ListExpr al1 = nl->Second(nl->Second(nl->First(args)));
 ListExpr al2 = nl->Second(nl->Second(nl->Second(args)));
 if(!listutils::disjointAttrNames(al1,al2)){
    return listutils::typeError("name conflict in attribute names found");
 }
 return nl->TwoElemList(
            listutils::basicSymbol<Stream<Tuple> >(),
            nl->TwoElemList(listutils::basicSymbol<Tuple>(),
            listutils::concat(al1,al2)));
}

template<class E1, class E2>
class obojoinInfo{

  public:
    obojoinInfo(Word& s1, Word& s2, TupleType* _tt): stream1(s1), stream2(s2){
      stream1.open();
      stream2.open();
      tt = _tt;
      tt->IncReference();
    }
    ~obojoinInfo(){
        stream1.close();
        stream2.close();
        tt->DeleteIfAllowed();
     }
   
    Tuple* next(){
      E1* e1 = stream1.request();
      E2* e2 = stream2.request();
      if(!e1 && !e2){
         return 0;
      }
      if(!e1){
         e2->DeleteIfAllowed();
         return 0;
      }
      if(!e2){
         e1->DeleteIfAllowed();
         return 0;
      }
      Tuple* res = new Tuple(tt);
      concat(e1,e2,res);
      return res;
    }
 
  private:
    Stream<E1> stream1;
    Stream<E2> stream2;
    TupleType* tt;

    void concat(Attribute* e1, Attribute* e2, Tuple* res){
       res->PutAttribute(0,e1);
       res->PutAttribute(1,e2);
    }

    void concat(Attribute* e1, Tuple* e2, Tuple* res){
      res->PutAttribute(0,e1);
      for(int i=0;i<e2->GetNoAttributes(); i++){
         res->CopyAttribute(i, e2, i+1);
      }
      e2->DeleteIfAllowed();
    }

    void concat(Tuple* e1, Attribute* e2, Tuple* res){
      for(int i=0;i<e1->GetNoAttributes(); i++){
         res->CopyAttribute(i, e1, i);
      }
      res->PutAttribute(e1->GetNoAttributes(),e2);
      e1->DeleteIfAllowed(); 
    }

    void concat(Tuple* e1, Tuple* e2, Tuple* res){
       int e1num = e1->GetNoAttributes();
       for(int i=0;i<e1num; i++){
         res->CopyAttribute(i, e1, i);
       }
       for(int i=0; i<e2->GetNoAttributes();i++){
          res->CopyAttribute(i, e2, i + e1num);
       }
       e1->DeleteIfAllowed();
       e2->DeleteIfAllowed();
    }
};

template<class e1, class e2> 
int obojoinVM(Word* args, Word& result, int message,
                     Word& local, Supplier s) {

  obojoinInfo<e1,e2>* li = (obojoinInfo<e1,e2>*) local.addr;
  TupleType* tt = (TupleType*) qp->GetLocal2(s).addr;
  switch(message){
     case INIT : {
        tt = new TupleType(nl->Second(GetTupleResultType(s)));
        qp->GetLocal2(s).addr=tt;
        return 0;
     }
     case FINISH: {
        if(tt){
           tt->DeleteIfAllowed();
           qp->GetLocal2(s).addr=0;
        }
        return 0;
     }
     case OPEN: {
            if(li){
               delete li;
            }
            local.addr = new obojoinInfo<e1,e2>(args[0],args[1],tt);
            return 0;
      }
      case REQUEST :{
            result.addr = li?li->next():0;
            return result.addr?YIELD:CANCEL;
      }
      case CLOSE :{
            if(li){
              delete li;
              local.addr = 0;
            }
            return 0;
      }
  }
  return -1;
}

OperatorSpec obojoinSpec(
  "stream(tuple(X)) x stream(tuple(Y)) -> stream(tuple(XY))",
  "_ _ obojoin",
  "Joins two tuple streams one by one, meaning the first tuple"
  " of the first stream is joined with the first tuple of the "
  " second stream, both second tuples are joined and so on.",
  " query ten feed ten feed {a} obojoind consume"
);

Operator obojoinOP(
  "obojoin",
  obojoinSpec.getStr(),
  obojoinVM<Tuple,Tuple>,
  Operator::SimpleSelect,
  obojoinTM
);


ListExpr obojoinDTM(ListExpr args){
  if(!nl->HasMinLength(args,3)){
    return listutils::typeError("three args expected");
  }
  int noDATA = 0;  // number of data streams
  bool fd = false; // first stream is a data stream

  if(Stream<Attribute>::checkType(nl->First(args))){
    noDATA++;
    fd = true;
  } else if(!Stream<Tuple>::checkType(nl->First(args))){
    return listutils::typeError("first argument must be a stream "
                                 "of DATA or a stream of tuples");
  }
  if(Stream<Attribute>::checkType(nl->Second(args))){
    noDATA++;
  } else if(!Stream<Tuple>::checkType(nl->Second(args))){
    return listutils::typeError("second argument must be a stream "
                                 "of DATA or a stream of tuples");
  }

  if(nl->AtomType(nl->Third(args))!=NoAtom){
     return listutils::typeError("third arg is not a list of attr names");
  }

  if(!nl->HasLength(nl->Third(args),noDATA)){
    return listutils::typeError("invalid number of attribute names");
  }


  if(noDATA == 0){ // two tuple streams
     ListExpr al1 = nl->Second(nl->Second(nl->First(args)));
     ListExpr al2 = nl->Second(nl->Second(nl->Second(args)));
     if(!listutils::disjointAttrNames(al1,al2)){
       return listutils::typeError("name conflict in attribute names found");
     }
     return nl->TwoElemList(
            listutils::basicSymbol<Stream<Tuple> >(),
            nl->TwoElemList(listutils::basicSymbol<Tuple>(),
            listutils::concat(al1,al2)));
  }
  // noDATA > 0
  ListExpr third = nl->Third(args);

  if(nl->AtomType(nl->First(third))!=SymbolType){
    return listutils::typeError("third arg is not an attribute name");
  }
  string n1 = nl->SymbolValue(nl->First(third));
  if(!stringutils::isIdent(n1)){
     return listutils::typeError(n1 + " is not a valid attribute name");
  }
  if(noDATA==1){
     ListExpr al1;
     ListExpr al2;
     if(fd){
       al1 = nl->OneElemList(nl->TwoElemList(nl->First(third), 
                             nl->Second(nl->First(args))));
       al2 = nl->Second(nl->Second(nl->Second(args)));
     } else {
       al1 = nl->Second(nl->Second(nl->First(args)));
       al2 = nl->OneElemList(nl->TwoElemList(nl->First(third), 
                               nl->Second(nl->Second(args))));
     }
     if(!listutils::disjointAttrNames(al1,al2)){
       return listutils::typeError("name conflict in attribute names found");
     }
     return nl->TwoElemList(
            listutils::basicSymbol<Stream<Tuple> >(),
            nl->TwoElemList(listutils::basicSymbol<Tuple>(),
            listutils::concat(al1,al2)));
  }
  // noDATA is 2
  if(nl->AtomType(nl->Second(third))!=SymbolType){
    return listutils::typeError("fourth arg is not an attribute name");
  }
  string n2 = nl->SymbolValue(nl->Second(third));
  if(!stringutils::isIdent(n2)){
     return listutils::typeError(n1 + " is not a valid attribute name");
  }

  if(n1==n2){
    return listutils::typeError("cannot use the same attribute name twice");
  }

  ListExpr attrList = nl->TwoElemList(
              nl->TwoElemList(
                    nl->First(third),
                    nl->Second(nl->First(args))),
              nl->TwoElemList(
                    nl->Second(third),
                    nl->Second(nl->Second(args))));
   return nl->TwoElemList(
            listutils::basicSymbol<Stream<Tuple> >(),
            nl->TwoElemList(listutils::basicSymbol<Tuple>(),
            attrList));
}

ValueMapping obojoinDVM[] = {
   obojoinVM<Attribute,Attribute>,
   obojoinVM<Attribute,Tuple>,
   obojoinVM<Tuple,Attribute>,
   obojoinVM<Tuple,Tuple>
};

int obojoinDSelect(ListExpr args){
  int n1 = Stream<Attribute>::checkType(nl->First(args))?0:2;
  int n2 = Stream<Attribute>::checkType(nl->Second(args))?0:1;
  return n1 + n2;
}



OperatorSpec obojoinDSpec(
  "stream({tuple(X),DATA}) x stream({tuple(Y),DATA}) "
  "[x ID [ x ID]]-> stream(tuple(XY))",
  "_ _ obojoinD[_,_]",
  "Joins two streams one by one, meaning the first element"
  " of the first stream is joined with the first element of the "
  " second stream, both second elements are joined and so on."
  "For each data stream, an attribute name must be given within the "
  "parameter list.",
  "query ten feed intstream(0,100) obojoinD[K]  consume"
);

Operator obojoinDOP(
  "obojoinD",
  obojoinDSpec.getStr(),
  4,
  obojoinDVM,
  obojoinDSelect,
  obojoinDTM
);


/*
2.29 ~gettuples~

This operator retrieves tuples from arelation given by a stream of 
tuple ids.

*/
ListExpr gettuplesTM(ListExpr args){

  string err = " stream(tid) x rel expected";
  if(!nl->HasLength(args,2)){
    return listutils::typeError(err + " (wrong number of args)");
  }
  if(!Stream<TupleIdentifier>::checkType(nl->First(args))){
    return listutils::typeError(err + " (first arg is not a stream of tids)");
  }
  if(!Relation::checkType(nl->Second(args))){
    return listutils::typeError(err + "(second arg is not a relation)");
  }
  return nl->TwoElemList(
               listutils::basicSymbol<Stream<Tuple> >(),
               nl->Second(nl->Second(args)));
}


class gettuplesInfo{

  public:
     gettuplesInfo(Word w, GenericRelation* _rel): stream(w), rel(_rel){
         stream.open();
     }
   
     ~gettuplesInfo(){
        stream.close();
      }

     Tuple* next(){
       TupleIdentifier* res;
       while( (res = stream.request())){
          if(!res->IsDefined()){
            res->DeleteIfAllowed();
          } else {
            TupleId id = res->GetTid();
            res->DeleteIfAllowed();
            if(id!=0){
              Tuple* tuple = rel->GetTuple(id,true);
              if(tuple){
                 return tuple;
              }
            }
          }
       }
       return 0;
     }

  private:
     Stream<TupleIdentifier> stream;
     GenericRelation* rel;

};


int gettuplesVM(Word* args, Word& result, int message,
                     Word& local, Supplier s) {
  
     gettuplesInfo* li = (gettuplesInfo*) local.addr;
     switch(message){
         case OPEN : if(li){
                       delete li;
                       local.addr = 0;
                     }
                     local.addr = new gettuplesInfo(args[0], 
                                          (GenericRelation*) args[1].addr);
                     return 0;
         case REQUEST:
                     result.addr = li?li->next():0;
                     return result.addr?YIELD:CANCEL;
         case CLOSE:
                  if(li){
                      delete li;
                      local.addr = 0;
                  }
                  return 0;    
     } 
     return -1;
}

OperatorSpec gettuplesSpec(
  "stream(tid) x rel",
  " _ _ gettuples",
  "Retrieves tuples form a relation based on a tuple id stream",
  "query intstream(1,0) use( int2tid(.) ) Kinos gettuples consume"
);

Operator gettuplesOP(
  "gettuples",
  gettuplesSpec.getStr(),
  gettuplesVM,
  Operator::SimpleSelect,
  gettuplesTM
);



/*
2.30 operator isOrdered

checks whether a tuple stream is ascending ordered

2.30.1 Type Mapping

*/
ListExpr isOrderedTM(ListExpr args){
  string err = "stream(tuple) expected";
  if(!nl->HasLength(args,1)){
    return listutils::typeError(err + "wrong number of args");
  }
  if(!Stream<Tuple>::checkType(nl->First(args))){
    return listutils::typeError(err);
  }
  return listutils::basicSymbol<CcBool>();
}

/*
2.30.2 Value Mapping

*/

int isOrderedVM(Word* args, Word& result, int message,
                     Word& local, Supplier s) {

   result = qp->ResultStorage(s);
   CcBool* res = (CcBool*) result.addr;

   Stream<Tuple> stream(args[0]);
   stream.open();
   Tuple* last = stream.request();
   res->Set(true,true);
   LexicographicalTupleCmp cmp;

   if(last){
      Tuple* current;
      while( (current = stream.request()) ){
         int c = cmp(last,current);
         if(c>0){ // invalid order found
            last->DeleteIfAllowed();
            current->DeleteIfAllowed();
            stream.close();
            res->Set(true,false); 
            return 0;
         } 
         last->DeleteIfAllowed();
         last = current;
      }
      last->DeleteIfAllowed();
   }
   stream.close();
   return 0;
}

/*
2.30.3 Specification

*/

OperatorSpec isOrderedSpec(
  " stream(tuple) -> bool ",
  " _ isOrdered",
  "checks whether a tuple stream is in ascending order",
  "query Kinos feed sort isOrdered"
);

/*
2.30.4 Instance

*/

Operator isOrderedOP(
  "isOrdered",
  isOrderedSpec.getStr(),
  isOrderedVM,
  Operator::SimpleSelect,
  isOrderedTM
);


/*
2.34 ~isOrderedBy~

2.34.1 Type Mapping

*/
ListExpr isOrderedByTM(ListExpr args){
  string err="stream(tuple) x attrlist expected";
  if(!nl->HasLength(args,2)){
    return listutils::typeError(err + " (wrong number of args)");
  }
  if(!Stream<Tuple>::checkType(nl->First(args))){
    return listutils::typeError(err + " first arg is not a tuple stream)");
  }
  ListExpr alist = nl->Second(args);
  if(nl->AtomType(alist) != NoAtom){
    return listutils::typeError("Expected list as second arg");
  }
  vector<pair<int,bool> > orderspec;
  ListExpr attrList = nl->Second(nl->Second(nl->First(args)));


  while(!nl->IsEmpty(alist)){
    ListExpr first = nl->First(alist);
    alist = nl->Rest(alist);
    string aname;
    bool asc;
    if(nl->AtomType(first)==SymbolType) {
        aname = nl->SymbolValue(first);
        asc = true;
    } else if(!nl->HasLength(args,2)){
         return listutils::typeError("invalid by part");
    } else if( (nl->AtomType(nl->First(first)) != SymbolType) 
            ||(nl->AtomType(nl->Second(first)) != SymbolType)) {
         return listutils::typeError("invalid by part");
    } else {
      aname = nl->SymbolValue(nl->First(first));
      ListExpr o = nl->Second(first);
      if(!listutils::isSymbol(o,"asc") && !listutils::isSymbol(o,"desc")){
          return listutils::typeError("sorting criterion must be asc or desc");
      }
      asc = listutils::isSymbol(o,"asc");
    }
    ListExpr type;
    int index = listutils::findAttribute(attrList, aname, type);
    if(!index){
        return listutils::typeError("Attribute " + aname 
                                    + " not present in tuple");
    }
    orderspec.push_back(make_pair(index,asc));
  }
  if(orderspec.empty()){
    return listutils::typeError("missing order specification");
  }
  ListExpr appendList = nl->OneElemList( 
                             nl->IntAtom(orderspec[0].first));
  ListExpr last = appendList;
  last = nl->Append(last, nl->BoolAtom(orderspec[0].second));
  for(size_t i=1;i<orderspec.size();i++){
    last = nl->Append(last, nl->IntAtom(orderspec[i].first));
    last = nl->Append(last, nl->BoolAtom(orderspec[i].second));
  }


  return nl->ThreeElemList(
                  nl->SymbolAtom(Symbols::APPEND()),
                  appendList,
                  listutils::basicSymbol<CcBool>());
}

/*
2.34.2 Value Mapping

*/

int isOrderedByVM(Word* args, Word& result, int message,
                     Word& local, Supplier s) {

   result= qp->ResultStorage(s);
   CcBool* res = (CcBool*) result.addr;
   res->Set(true,true);
   Stream<Tuple> stream(args[0]);
   stream.open();

   SortOrderSpecification orderspec;
   int arg = 2;
   while(arg < qp->GetNoSons(s)){
      int pos = ((CcInt*) args[arg].addr)->GetValue();
      arg++;
      bool asc = ((CcBool*) args[arg].addr)->GetValue();
      arg++;
      orderspec.push_back(make_pair(pos,asc));
   } 
 
   TupleCmpBy cmp(orderspec);
   Tuple* last = stream.request();
   if(last){
      Tuple* current;
      while( (current = stream.request()) ){
         int c = cmp(last,current);
         if(c>0){ // invalid order found
            last->DeleteIfAllowed();
            current->DeleteIfAllowed();
            res->Set(true,false);
            stream.close(); 
            return 0;
         } 
         last->DeleteIfAllowed();
         last = current;
      }
      last->DeleteIfAllowed();
   }
   stream.close();
   return 0;
}

/*
2.34.3 Specification

*/

OperatorSpec isOrderedBySpec(
  " stream(tuple) x orderspec -> bool ",
  " _ isOrderedBy[<orderspec> ",
  "Checks whether a tuple stream is ordered in the way that "
  "given by orderspec. where orderspec is a list of pairs constisting "
  "of an attribute name and {asc,desc}. The latter can be omitted.",
  "query Kinos feed sortby[Name desc, GeoData] isOrderedBy[Name desc, GeoData]"
);

/*
2.35.4 Instance

*/

Operator isOrderedByOP(
  "isOrderedBy",
  isOrderedBySpec.getStr(),
  isOrderedByVM,
  Operator::SimpleSelect,
  isOrderedByTM
);


/*
2.36 Operator ~tids~

Converts a tuple stream into a stream of tids.

2.26.1 Type Mapping

*/

ListExpr tidsTM(ListExpr args){
  string err = "stream(tuple) expected";
  if(!nl->HasLength(args,1)){
    return listutils::typeError(err+ " ( wrong number of args)");
  } 
  if(!Stream<Tuple>::checkType(nl->First(args))){
    return listutils::typeError(err);
  }
  return nl->TwoElemList( listutils::basicSymbol<Stream<TupleIdentifier> >(),
                          listutils::basicSymbol<TupleIdentifier>());
                          
}

/*
2.36.2 Value Mapping

*/
int tidsVM(Word* args, Word& result, int message,
                     Word& local, Supplier s) {
  Stream<Tuple>* li = (Stream<Tuple>*) local.addr;
  switch(message){
    case OPEN:
            if(li){
              li->close();
              delete li;
              local.addr = 0;
            }
            li = new Stream<Tuple>(args[0]);
            li->open();
            local.addr = li;
            return 0;
   case REQUEST:{
           if(!li) return CANCEL;
           Tuple* t = li->request();
           if(t){
              result.addr = new TupleIdentifier(true, t->GetTupleId());
              t->DeleteIfAllowed();
           } else {
              result.addr = 0;
           }
           return result.addr?YIELD:CANCEL;
        }
   case CLOSE: 
          if(li){
            li->close();
            delete li;
            local.addr = 0;
          }
          return 0;
  }
  return -1;
}

/*
2.36.3 Specification

*/

OperatorSpec tidsSpec(
  "stream(tuple) -> stream(tid) ",
  " _ tids ",
  "Replaces tuples in a stream by their tids.",
  "query ten feed tids count"
);

/*
2.36.3 Operator

*/

Operator tidsOp(
  "tids",
  tidsSpec.getStr(),
  tidsVM,
  Operator::SimpleSelect,
  tidsTM
);


/*
2.37 Operator ~noRefs~

Retrieves the number of references of a tuple.

*/
ListExpr noRefsTM(ListExpr args){
  if(!nl->HasLength(args,1)){
    return listutils::typeError("one argument expected");
  }
  ListExpr a = nl->First(args);
  if(!Tuple::checkType(a) && !Attribute::checkType(a)){
    return listutils::typeError("tuple expected");
  }
  return listutils::basicSymbol<CcInt>();  
}

template<class T>
int noRefsVMT(Word* args, Word& result, int message,
                     Word& local, Supplier s) {

  T* a = (T*) args[0].addr;
  int refs = a->GetNumOfRefs();
  result=qp->ResultStorage(s);
  CcInt* res = (CcInt*) result.addr;
  res->Set(true,refs);
  return 0;
}


ValueMapping noRefsVM[] = {
   noRefsVMT<Attribute>,
   noRefsVMT<Tuple>
};

int noRefsSelect(ListExpr args){
  return Attribute::checkType(nl->First(args))?0:1;
}

OperatorSpec noRefsSpec(
  "{tuple, DATA} -> int",
  "noRefs(_)",
  "Retrieves the number of refeences for the attribute. "
  "This operator is useful for debugging purposes.",
  "query ten feed extend[RefT : noRefs(.), RefNo : noRefs(.No)] consume"
);

Operator noRefsOp(
  "noRefs",
  noRefsSpec.getStr(),
  2,
  noRefsVM,
  noRefsSelect,
  noRefsTM

);

/*
Operator ~addModCounter~

*/
ListExpr addModCounterTM(ListExpr args){
 // stream(t) x IDENT x int x (t x int -> bool) x (t x int -> int)
 if(!nl->HasLength(args,5)){
   return listutils::typeError("5 args expected");
 }
 if(!Stream<Tuple>::checkType(nl->First(args))){
   return listutils::typeError("first arg is not a tuple stream");
 }
 string err;
 if(!listutils::isValidAttributeName(nl->Second(args),err)){
   return listutils::typeError("second arg is not a valid attribute name");
 }
 if(!CcInt::checkType(nl->Third(args))){
   return listutils::typeError("third arg is not an int");
 }
 if(!listutils::isMap<2>(nl->Fourth(args))){
   return listutils::typeError("Fourth arg is not a binary funtion");
 }
 if(!listutils::isMap<2>(nl->Fifth(args))){
   return listutils::typeError("Fifth arg is not a binary funtion");
 }
  
 string n = nl->SymbolValue(nl->Second(args));
 ListExpr attrList = nl->Second(nl->Second(nl->First(args)));
 ListExpr type;
 if(listutils::findAttribute(attrList,n,type)){
   return listutils::typeError("Attribute " + n 
                               + " already part of the tuple");
 }
 ListExpr tt = nl->Second(nl->First(args));
 ListExpr fun1 = nl->Fourth(args);
 if(!nl->Equal(tt,nl->Second(fun1))){
   return listutils::typeError("first arg of fun1 and tuple type "
                               "in stream differ");
 }
 if(!CcInt::checkType(nl->Third(fun1))){
   return listutils::typeError("second arg of fun 1 is not an int");
 } 
 if(!CcBool::checkType(nl->Fourth(fun1))){
   return listutils::typeError("fun 1 does not create a bool");
 }
 ListExpr fun2 = nl->Fifth(args);
 if(!nl->Equal(tt,nl->Second(fun2))){
   return listutils::typeError("first arg of fun2 and tuple type "
                               "in stream differ");
 }
 if(!CcInt::checkType(nl->Third(fun2))){
   return listutils::typeError("second arg of fun 2 is not an int");
 } 
 if(!CcInt::checkType(nl->Fourth(fun2))){
   return listutils::typeError("fun 2 does not create an int");
 }

 ListExpr al = nl->OneElemList(nl->TwoElemList( nl->SymbolAtom(n),
                                             listutils::basicSymbol<CcInt>()));
 attrList = listutils::concat(attrList,al);
 if(!listutils::isAttrList(attrList)){
    return listutils::typeError("result is not a valid attrlist");
 }
 tt = nl->TwoElemList(listutils::basicSymbol<Tuple>(), attrList);
 return nl->TwoElemList(listutils::basicSymbol<Stream<Tuple> >(), tt);
}

class addModCounterInfo{
  public:
     addModCounterInfo( Word& _stream, CcInt* initial,
                        Word& _fun1, Word& _fun2, TupleType* _tt):
     stream(_stream), fun1(_fun1.addr), fun2(_fun2.addr), tt(_tt)
     {
       tt->IncReference();
       f1args = qp->Argument(fun1);
       f2args = qp->Argument(fun2);
       currentValue = 0;
       if(initial->IsDefined()){
         currentValue = initial->GetValue();
       }
       initValue = currentValue;
       stream.open();
     }

     ~addModCounterInfo(){
       stream.close();
       tt->DeleteIfAllowed();  
     }

     Tuple* next(){
        Tuple* in = stream.request();
        if(!in){
          return 0;  
        }
        CcInt* c = new CcInt(true,currentValue);
        (*f1args)[0] = in;
        (*f1args)[1] = c;
        qp->Request(fun1,funres);
        CcBool* b = (CcBool*) funres.addr;
        if(b->IsDefined() && b->GetValue()){
          // modify counter
          (*f2args)[0] = in;
          (*f2args)[1] = c;
          qp->Request(fun2,funres);
          CcInt* nv = (CcInt*) funres.addr;
          currentValue = nv->IsDefined()?nv->GetValue():initValue;
          c->Set(true,currentValue);
        }
        Tuple* res = new Tuple(tt);
        for(int i=0;i<in->GetNoAttributes();i++){
          res->CopyAttribute(i,in,i);
        }
        res->PutAttribute(in->GetNoAttributes(),c);
        in->DeleteIfAllowed();
        currentValue++;
        return res;
     }

  private:
     Stream<Tuple> stream;
     Supplier fun1;
     Supplier fun2;
     TupleType* tt;
     ArgVectorPointer f1args;
     ArgVectorPointer f2args;
     int currentValue;
     int initValue;
     Word funres;
};


int addModCounterVM(Word* args, Word& result, int message,
                     Word& local, Supplier s) {

   addModCounterInfo* li = (addModCounterInfo*)local.addr;
   switch(message){
     case INIT: {
        qp->GetLocal2(s).addr = new TupleType(
                                     nl->Second(GetTupleResultType(s)));
        return 0;
     }
     case FINISH: {
         TupleType* tt = (TupleType*) qp->GetLocal2(s).addr;
         if(tt){
           tt->DeleteIfAllowed();
           qp->GetLocal2(s).addr = 0;
         }
         return 0;
     }
     case OPEN: {
         TupleType* tt = (TupleType*) qp->GetLocal2(s).addr;
         if(li){
          delete li;
         }
         local.addr = new addModCounterInfo(args[0], (CcInt*) args[2].addr, 
                                            args[3],args[4], tt);
         return 0;
     }
     case REQUEST: {
          result.addr = li?li->next():0;
          return result.addr?YIELD:CANCEL;
     }
     case CLOSE: {
         if(li){
            delete li;
            local.addr = 0;
         }
         return 0;
     }

   }
   return -1;
}

OperatorSpec addModCounterSpec(
  "stream(t) x IDENT x int x ( t x int -> bool) x (t x int -> int) "
  "-> stream(T@(I int)",
  " _ addModCounter[Name, initial, fun1, fun2] ",
  "Adds a counter to a tuple stream. The counter starts with the value "
  "initial and is inremeneted for each tuple. " 
  "The first function takes the tuple and the current value of the "
  "counter as arguments. If this function evaluates to true, "
  "the current counter value is replaced by the result of the second "
  "function. If this result is undefined, the counter uses it's initial "
  " value.",
  "query plz feed addModCounter[C ,1, .PLZ * .. > 80000, 0] consume"
);

Operator addModCounterOp(
  "addModCounter",
  addModCounterSpec.getStr(),
  addModCounterVM,
  Operator::SimpleSelect,
  addModCounterTM
);

/*
Operator ~rduph~

*/
ListExpr Rdup2TypeMap( ListExpr args ) {

  NList type(args);
  if ( ! (type.hasLength(1) || type.hasLength(2)) )
  {
    return NList::typeError(
        "Expecting one stream argument and a optional int (buckets).");
  }

  NList first = type.first();
  if (  
    !first.hasLength(2)  ||
    !first.first().isSymbol(Symbol::STREAM()) ||
    !first.second().hasLength(2) ||
    !first.second().first().isSymbol(Tuple::BasicType()) ||
    !IsTupleDescription( first.second().second().listExpr() ) )
  {
    return NList::typeError("Error in first argument!");
  }

  bool bucketsGiven = false;

  // Check bucket parameter
  if(type.hasLength(2)) {
     if(!CcInt::checkType(nl->Second(args))) {
       return NList::typeError("Error in second (buckets) argument!");
     }

     bucketsGiven = true;
  }

   ListExpr stream = first.listExpr();
   ListExpr attrList = nl->Second(nl->Second(stream));
   ListExpr appendList;
   ListExpr last;

   // TODO: 

   // Append all attribute positions for krdup tm
   for(int i = 0; i < nl->ListLength(attrList); i++) {
      if(i == 0){
        appendList = nl->OneElemList(nl->IntAtom(i));
        last = appendList;
      } else {
        last = nl->Append(last,nl->IntAtom(i));
      }
   }

   if(bucketsGiven) {
        last = nl->Append(last,nl->IntAtom(0));
   } else {
        last = nl->Append(last,nl->IntAtom(-1));
   }

   // Append all attribute positions for krdup tm
   for(int i = 0; i < nl->ListLength(attrList); i++) {
        last = nl->Append(last,nl->IntAtom(i));
   }

   ListExpr resultList = nl->ThreeElemList( 
                             nl->SymbolAtom(Symbols::APPEND()),
                             appendList,
                             stream);

   cout << nl->ToString(resultList) << endl;

   return resultList;
}

const string rduphSpec  = "( ( \"Signature\" \"Syntax\" \"Meaning\" "
                         "\"Example\" ) "
                         "( <text>((stream (tuple(...))) x int)"
                         " -> (stream (tuple(...)))"
                         "</text--->"
                         "<text>_ rduph[buckets]</text--->"
                         "<text>This operator removes duplicate tuples "
                         " from a stream using a hashmap.</text--->"
                         "<text>query Orte feed project[BevT] "
                         "rduph[999997] count</text--->"
                         ") )";

/*
2.9.4 Definition of operator ~rduph~

*/
Operator extrelrduph (
         "rduph",             // name
         rduphSpec,           // specification
         krduphVM,            // value mapping (reuse krduph operator)
         Operator::SimpleSelect,          // trivial selection function
         Rdup2TypeMap         // type mapping
);



/*
Operator ~swap~

*/
ListExpr swapTM(ListExpr args){
  if(!nl->HasLength(args,3)){
    return listutils::typeError("three arguments expected");
  }
  if(!Stream<Tuple>::checkType(nl->First(args))){
    return listutils::typeError("first arg is not an tuple stream");
  }
  if(nl->AtomType(nl->Second(args))!=SymbolType){
    return listutils::typeError("second arg is not a valid attribute name");
  }
  if(nl->AtomType(nl->Third(args))!=SymbolType){
    return listutils::typeError("third arg is not a valid attribute name");
  }
  ListExpr attrList = nl->Second(nl->Second(nl->First(args)));
  ListExpr t1;
  string n1 = nl->SymbolValue(nl->Second(args));
  int index1 = listutils::findAttribute(attrList,n1, t1);
  if(!index1){
    return listutils::typeError("attribute " + n1 + " not part of the tuple");
  }
  ListExpr t2;
  string n2 = nl->SymbolValue(nl->Third(args));
  int index2 = listutils::findAttribute(attrList,n2, t2);
  if(!index2){
    return listutils::typeError("attribute " + n2 + " not part of the tuple");
  }
  if(!nl->Equal(t1,t2)){
    return listutils::typeError("attributes " + n1 + " and " + n1 
                                + " have different types");
  }
  return nl->ThreeElemList(
             nl->SymbolAtom(Symbols::APPEND()),
             nl->TwoElemList(
               nl->IntAtom(index1-1),
               nl->IntAtom(index2-1)
             ),
             nl->First(args));

        
}

class swapInfo{

  public:
    swapInfo(Word& _stream, int _index1, int _index2):
        stream(_stream), index1(_index1), index2(_index2){
        stream.open();
    }
    ~swapInfo(){
      stream.close();
    }

    Tuple* next(){
      Tuple* tuple = stream.request();
      if(tuple){
        Attribute* a1 = tuple->GetAttribute(index1)->Copy();
        Attribute* a2 = tuple->GetAttribute(index2)->Copy();
        tuple->PutAttribute(index1,a2);
        tuple->PutAttribute(index2,a1);
      }
      return tuple;
    }
   private:
     Stream<Tuple> stream;
     int index1;
     int index2;
};


int swapVM(Word* args, Word& result, int message,
                     Word& local, Supplier s) {

  swapInfo* li = (swapInfo*) local.addr;
  switch(message){
     case OPEN: 
         if(li) delete li;
         local.addr = new swapInfo(args[0],
                           ((CcInt*) args[3].addr)->GetValue(),
                           ((CcInt*) args[4].addr)->GetValue());
         return 0;
     case REQUEST:
          result.addr = li?li->next():0;
          return result.addr?YIELD:CANCEL;
     case CLOSE:
         if(li){
            delete li;
            local.addr = 0;
         }    
         return 0;

  }
  return -1;
}

OperatorSpec swapSpec(
   "stream(tuple) x IDENT x INDENT -> stream(tuple)",
   "_ swap [_,_]",
   "Swaps two attribute values within each tuple of "
   "a stream. The attributes must be of the same type.",
   "query plz feed addcounter[c,3] swap[PLZ,C] count"
);

Operator swapOp(
  "swap",
  swapSpec.getStr(),
  swapVM,
  Operator::SimpleSelect,
  swapTM
);


ListExpr hashvalueTM(ListExpr args){
  if(!nl->HasLength(args,1) && !nl->HasLength(args,2)){
    return listutils::typeError("one argument expected");
  }
  if(!Tuple::checkType(nl->First(args))){
    return listutils::typeError("tuple [x int] expected");
  }
  if(nl->HasLength(args,2)){
    if(!CcInt::checkType(nl->Second(args))){
      return listutils::typeError("second argument must be an int");
    }
    return listutils::basicSymbol<CcInt>();
  }
  return nl->ThreeElemList( nl->SymbolAtom(Symbols::APPEND()),
                nl->OneElemList(nl->IntAtom(0)),
                listutils::basicSymbol<CcInt>());

}

int hashvalueVM(Word* args, Word& result, int message,
                     Word& local, Supplier s) {

    Tuple* arg = (Tuple*) args[0].addr;
    size_t hash = arg->HashValue();

    int mv = -1;
    CcInt* Mv = (CcInt*) args[1].addr;
    if(Mv->IsDefined()){
      mv = Mv->GetValue();
    }
    if(mv!=0){
      hash = hash % mv;
    }

    result = qp->ResultStorage(s);
    ((CcInt*) result.addr)->Set(true,hash);
    return 0;
}

OperatorSpec hashvalueSpec(
   "tuple -> int",
   "hashvalue(_)",
   "Returns a hashvalue for a complete tuple. ",
   "query plz feed extend[H : hashvalue(.)] consume"
);

Operator hashvalueOp(
  "hashvalue",
  hashvalueSpec.getStr(),
  hashvalueVM,
  Operator::SimpleSelect,
  hashvalueTM
);


/*
Type Map Operator ~AGGRTYPE~

This opertaor takes a tuple stream, an attribute name, and 
may be further elements. It extracts the type of the attribute
with given name from the tuple stream. Other attributes are
ignored.

*/
ListExpr AGGRTYPETM(ListExpr args){
  if(!nl->HasMinLength(args,2)){
    return listutils::typeError("at least 2 arguments expected");
  }
  if(!Stream<Tuple>::checkType(nl->First(args))){
    return listutils::typeError("First argument is not a tuple stream");
  }
  if(nl->AtomType(nl->Second(args)) != SymbolType){
    return listutils::typeError("Second argument ist not "
                                "a valid attribute name");
  }
  string aname = nl->SymbolValue(nl->Second(args));
  ListExpr attrList = nl->Second(nl->Second(nl->First(args)));
  ListExpr attrType = nl->TheEmptyList();
  int index = listutils::findAttribute(attrList, aname, attrType);
  if(index == 0){
    return listutils::typeError("Attribute " + aname + 
                                " not part of the tuple");
  }
  return attrType;
}

OperatorSpec AGGRTYPESpec(
   "stream(tuple) x IDENT [ x ... ] -> DATA",
   "AGGRTYPE(_,_,...)",
   "Extracts the type of an attribute from a tuple "
   "stream description. Type Map Operator. ",
   "query ten feed aggregteB[No; . + ..; 0] "
);

Operator AGGRTYPEOp(
  "AGGRTYPE",
  AGGRTYPESpec.getStr(),
  0,
  Operator::SimpleSelect,
  AGGRTYPETM
);





/*
2.10 Operator ~buffer~
 
Decuples two streams by buffering up to n 
elements. After the buffer is filled, the buffered 
is emitted and re-filled.

2.10.1 Type mapping function of operator ~buffer~

Type mapping for ~buffer~ is

----  ((stream (tuple ((x1 t1)...(xn tn))) int)   ->
              ((stream (tuple ((x1 t1)...(xn tn))))

  or ((stream T) int) -> (stream T)  for T in kind DATA
----

*/
ListExpr BufferTypeMap( ListExpr args )
{

  if(nl->ListLength(args)!=2){
    return listutils::typeError("two arguments expected");
  }

  string err = " stream(tuple(...) x int or stream(DATA) x int expected";

  ListExpr stream = nl->First(args);
  ListExpr count = nl->Second(args);

  if(( !Stream<Tuple>::checkType(stream) &&
       !Stream<Attribute>::checkType(stream) ) ||
     !CcInt::checkType(count) ){
    return listutils::typeError(err);
  }
  return stream;
}

/*
2.10.2 Value mapping function of operator ~buffer~

*/


struct BufferLocalInfo
{
  BufferLocalInfo( const size_t maxTuples = 0 ):
    maxTuples( maxTuples ), streamExhausted(false)
    {}

  std::queue<Word> buffer;
  size_t maxTuples;
  bool streamExhausted;
};

int Buffer(Word* args, Word& result, int message, Word& local, Supplier s)
{
  BufferLocalInfo *bli;
  Word tupleWord;

  bli = (BufferLocalInfo*) local.addr;

  CcInt* maxTupleCCInt = ((CcInt*) args[1].addr);

  switch(message)
  {
    case OPEN:

      if ( bli ) delete bli;


      if(! maxTupleCCInt->IsDefined()) {
        cerr << "Error: Max buffer value is not defined" << endl;
        return CANCEL;
      }

      bli = new BufferLocalInfo( maxTupleCCInt->GetIntval() );
      local.setAddr( bli );

      qp->Open(args[0].addr);
      return 0;

    case REQUEST:

      // Case 1: We have buffered tuples
      if(! bli -> buffer.empty()) {
        //cout << "Debug: Serving request from queue" << endl;
        result = bli -> buffer.front();
        bli -> buffer.pop();
        return YIELD;
      }

      // Case 2: Buffer is empty but stream is ready
      if(! bli->streamExhausted) {
        //cout << "Debug: Re-fill queue" << endl;

        while(bli -> buffer.size() < bli -> maxTuples) {
          qp->Request(args[0].addr, tupleWord);

          if(! qp->Received(args[0].addr)) {
            bli->streamExhausted = true;
            break;
          }

          bli -> buffer.push(tupleWord);
        }
      }
    
      if(! bli -> buffer.empty()) {
        result = bli -> buffer.front();
        bli -> buffer.pop();
        return YIELD;
      }

      //  Case 3: Buffer is empty and stream is exhauted
      return CANCEL;

    case CLOSE:
      qp->Close(args[0].addr);
      return 0;

    case CLOSEPROGRESS:
      if ( bli )
      {
        delete bli;
        local.setAddr(0);
      }
      return 0;

    case REQUESTPROGRESS:

      ProgressInfo p1;
      ProgressInfo* pRes;

      pRes = (ProgressInfo*) result.addr;

      if ( !bli )  {
         return CANCEL;
      }

      if ( qp->RequestProgress(args[0].addr, &p1) )
      {
        pRes->Card = p1.Card;
        pRes->CopySizes(p1);
        pRes->Time = p1.Time;
        pRes->Progress = p1.Progress;
        pRes->BTime = p1.BTime;
        pRes->BProgress = p1.BProgress;    
        return YIELD;
      } else {
        return CANCEL;
      }

  }
  return 0;
}



/*
2.10.3 Specification of operator ~buffer~

*/
const string BufferSpec  = "( ( \"Signature\" \"Syntax\" \"Meaning\" "
                         "\"Example\" ) "
                         "( "
                         "<text>((stream (tuple([a1:d1, ... ,an:dn]"
                         "))) x int) -> (stream (tuple([a1:d1, ... ,"
                         "an:dn]))) or \n"
                         "((stream T) int) -> (stream T), "
                         "for T in kind DATA.</text--->"
                         "<text>_ buffer [ _ ]</text--->"
                         "<text>Decuples two streams by buffering up to n "
                         "elements. After the buffer is filled, the buffered "
                         "is emitted and re-filled.</text--->"
                         "<text>query cities feed buffer[10] consume\n"
                         "query intstream(1,1000) buffer[10] printstream count"
                         "</text--->"
                              ") )";

/*
2.10.4 Definition of operator ~buffer~

*/
Operator extrelbuffer (
         "buffer",                 // name
         BufferSpec,               // specification
         Buffer,                   // value mapping
         Operator::SimpleSelect, // trivial selection function
         BufferTypeMap             // type mapping
);




/*

3 Class ~ExtRelationAlgebra~

A new subclass ~ExtRelationAlgebra~ of class ~Algebra~ is declared. The only
specialization with respect to class ~Algebra~ takes place within the
constructor: all type constructors and operators are registered at the
actual algebra.

After declaring the new class, its only instance ~extendedRelationAlgebra~
is defined.

*/


class ExtRelationAlgebra : public Algebra
{
 public:
  ExtRelationAlgebra() : Algebra()
  {
    AddOperator(&extrelsample);
    AddOperator(&extrelgroup); // type map operator
    AddOperator(&extrelcancel);
    AddOperator(&extrelextract);
    AddOperator(&extrelextend);
    extrelextend.enableInitFinishSupport();
    AddOperator(&extrelconcat);
    AddOperator(&extrelmin);
    AddOperator(&extrelmax);

    ValueMapping avgFuns[] = { AvgValueMapping<int,CcInt>,
                         AvgValueMapping<SEC_STD_REAL,CcReal>, 0 };
    ValueMapping sumFuns[] = { SumValueMapping<int,CcInt>,
                         SumValueMapping<SEC_STD_REAL,CcReal>, 0 };
    ValueMapping varFuns[] = { VarValueMapping<int,CcInt>,
                         VarValueMapping<SEC_STD_REAL,CcReal>, 0 };

    // Operator* extrelavg =
    AddOperator(avgInfo(), avgFuns, AvgSumSelect, AvgSumTypeMap<true>);
      // extrelavg->SetUsesMemory();


    AddOperator(sumInfo(), sumFuns, AvgSumSelect, AvgSumTypeMap<false>);

    AddOperator(printrefsInfo(), printrefs_vm, printrefs_tm);

    Operator* vfo = AddOperator(varInfo(), varFuns, 
                                AvgSumSelect, AvgSumTypeMap<true>);
    vfo->SetUsesMemory();

    ValueMapping statsFuns[] =
    {
      StatsValueMapping<int,CcInt,int,CcInt>,
      StatsValueMapping<int,CcInt,SEC_STD_REAL,CcReal>,
      StatsValueMapping<SEC_STD_REAL,CcReal,int,CcInt>,
      StatsValueMapping<SEC_STD_REAL,CcReal,SEC_STD_REAL,CcReal>, 0
    };
    Operator* sio = AddOperator(statsInfo(), statsFuns, 
                                StatsSelect, StatsTypeMap);
    sio->SetUsesMemory();

    AddOperator(&extrelhead);
    AddOperator(&extrelbuffer);
    AddOperator(&extrelsortby);
      extrelsortby.SetUsesMemory();
    AddOperator(&extrelsort);
      extrelsort.SetUsesMemory();

    AddOperator(&extrelrdup);
    AddOperator(&extrelmergesec);
    AddOperator(&extrelmergediff);
    AddOperator(&extrelmergeunion);
    AddOperator(&extrelmergejoin);
      extrelmergejoin.SetUsesMemory();

    AddOperator(&extrelsortmergejoin);
      extrelsortmergejoin.SetUsesMemory();
    AddOperator(&extrelsmouterjoin);
      extrelsmouterjoin.SetUsesMemory();
    AddOperator(&extrelhashjoin);
      extrelhashjoin.SetUsesMemory();
    AddOperator(&extrelloopjoin);
    extrelloopjoin.enableInitFinishSupport();
    AddOperator(&extrelextendstream);
    extrelextendstream.enableInitFinishSupport();
    AddOperator(&extrelprojectextendstream);
    extrelprojectextendstream.enableInitFinishSupport();
    AddOperator(&extrelloopsel);
    extrelloopsel.enableInitFinishSupport();
    AddOperator(&extrelgroupby);
      extrelgroupby.SetUsesMemory();
    extrelgroupby.enableInitFinishSupport();
    AddOperator(&extrelaggregate);
    AddOperator(&extrelaggregateB);
    AddOperator(&extrelsymmjoin);
     extrelsymmjoin.SetUsesMemory();
     extrelsymmjoin.enableInitFinishSupport();
    AddOperator(&extrelsymmouterjoin);
      extrelsymmouterjoin.SetUsesMemory();
    AddOperator(&extrelsymmproductextend);
      extrelsymmproductextend.SetUsesMemory();
      extrelsymmproductextend.enableInitFinishSupport();
    AddOperator(&extrelsymmproduct);
    extrelsymmproduct.enableInitFinishSupport();
      extrelsymmproduct.SetUsesMemory();
    AddOperator(&extrelprojectextend);
    extrelprojectextend.enableInitFinishSupport(); 
    AddOperator(&krdup);
    krdup.enableInitFinishSupport();
    AddOperator(&krduph);
    krduph.enableInitFinishSupport();
    krduph.SetUsesMemory();
    AddOperator(&extrelrduph);
    extrelrduph.SetUsesMemory();
    extrelrduph.enableInitFinishSupport();
    AddOperator(&extreladdcounter);
    extreladdcounter.enableInitFinishSupport();
    AddOperator(&ksmallest);
    AddOperator(&kbiggest);
    AddOperator(&extrelslidingwindow);
    extrelslidingwindow.enableInitFinishSupport();
    AddOperator(&extend_aggr);
    extend_aggr.enableInitFinishSupport();
    AddOperator(&extend_last);
    extend_last.enableInitFinishSupport();
    AddOperator(&extend_next);
    extend_next.enableInitFinishSupport();
    AddOperator(&aggregateC);
    Operator* tfo = AddOperator(toFieldsInfo(), toFieldsFun, toFieldsType );
    tfo->enableInitFinishSupport();
    Operator* ffo = AddOperator(fromFieldsInfo(), fromFieldsFun, 
                            fromFieldsType );
    ffo->enableInitFinishSupport();
    AddOperator(&applyToAll);
    applyToAll.enableInitFinishSupport();
    AddOperator(&equalStreams);
    AddOperator(&replaceAttr);
    replaceAttr.enableInitFinishSupport();
    AddOperator(&pfilter);
    AddOperator(&extendXOP);
    extendXOP.enableInitFinishSupport();
    AddOperator(&countMtOP);
    AddOperator(&bringToMemoryOP);
    AddOperator(&feedSOp);
    AddOperator(&extrelnth);
    AddOperator(&obojoinOP);
    obojoinOP.enableInitFinishSupport();
    AddOperator(&obojoinDOP);
    obojoinDOP.enableInitFinishSupport();
    AddOperator(&gettuplesOP);
    AddOperator(&isOrderedOP);
    AddOperator(&isOrderedByOP);
    AddOperator(&tidsOp);
    AddOperator(&noRefsOp);
    AddOperator(&extractDefOp);
    AddOperator(&addModCounterOp);
    addModCounterOp.enableInitFinishSupport();
    AddOperator(&swapOp);
    AddOperator(&hashvalueOp);
    AddOperator(&AGGRTYPEOp);
    AddOperator(&extrelexist);
     
#ifdef USE_PROGRESS
// support for progress queries
   extrelextend.EnableProgress();
   extrelhead.EnableProgress();
   extrelbuffer.EnableProgress();
   extrelsortby.EnableProgress();
   extrelsort.EnableProgress();
   extrelrdup.EnableProgress();
   extrelmergejoin.EnableProgress();
   extrelsortmergejoin.EnableProgress();
   extrelsmouterjoin.EnableProgress();
   extrelsymmouterjoin.EnableProgress();
   extrelhashjoin.EnableProgress();
   extrelloopjoin.EnableProgress();
   extrelextendstream.EnableProgress();
   extrelprojectextendstream.EnableProgress();
   extrelgroupby.EnableProgress();
   extrelsymmjoin.EnableProgress();
   extrelprojectextend.EnableProgress();
   ksmallest.EnableProgress();
   kbiggest.EnableProgress();
   extrelloopsel.EnableProgress();
   extrelaggregate.EnableProgress();
   extrelnth.EnableProgress();
#endif
  }

  ~ExtRelationAlgebra() {};
};

} // end of namespace extrelationalg

/*

4 Initialization

Each algebra module needs an initialization function. The algebra manager
has a reference to this function if this algebra is included in the list
of required algebras, thus forcing the linker to include this module.

The algebra manager invokes this function to get a reference to the instance
of the algebra class and to provide references to the global nested list
container (used to store constructor, type, operator and object information)
and to the query processor.

The function has a C interface to make it possible to load the algebra
dynamically at runtime.

*/

extern "C"
Algebra*
InitializeExtRelationAlgebra( NestedList* nlRef,
                              QueryProcessor* qpRef,
                              AlgebraManager* amRef )
{
  nl = nlRef;
  qp = qpRef;
  am = amRef;
  return (new extrelationalg::ExtRelationAlgebra());
}