along/secondo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
dfd1e745480fabd88139d2804acb90d5b6dc055e
secondo/Tools/Flob/NativeFlobCache.h
along dfd1e74548 firs commit
2026-01-23 17:03:45 +08:00

1544 lines
37 KiB
C++
Raw Blame History

/*
----
This file is part of SECONDO.
Copyright (C) 2009, University in Hagen, Department of Computer Science,
Database Systems for New Applications.
SECONDO is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
SECONDO is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with SECONDO; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
----
//paragraph [10] Title: [{\Large \bf \begin {center}] [\end {center}}]
//[ae] [\"a]
//[ue] [\"u]
//[oe] [\"o]
[10] Header File of Module NativeFlobCache
June 2010, C. D[ue]ntgen: Added comments
1 Overview
The ~NativeFlobCache~ is responsible for accelerating the access to data
referenced by native Flobs. Native Flobs are temporary Flobs, i.e. Flobs that
are created e.g. within local variables of a Flob-containing data type. The
data referenced by these native Flobs is maintained within a dedicated file, the
native Flob file, which is managed by the FlobManager.
Since temporary Flobs are often accessed repeatedly by many algorithms, e.g. to
append data, not every single access should rely on disk access. Thus, a main
memory buffer is used for cached read/ write access to the according data.
If a native Flob is destroyed or deleted by the FlobManager, its content is
never modified on disk, regardless whether is is marked modified or not.
The according
cache entries are just removed from the cache. This is due to the temporary
nature of native Flobs.
1.1 Slot Based Approach
The FlobCache's view of a Flob is a sequence of fixed size memory slices.
Each slice can be kept separately within the cache, which consists of a limited
number of fixed size memory buffers (slices), that are organized in an open
hashtable containing instances of class ~NativeCacheEntry~.
As a secondary organisation, a LRU-priority list linking the cache entries
is used, since LRU-policy is used to replace data if necessary.
If cached Flob data is modified, it is marked as ~changed~. Whenever changed
Flob data is removed from the Cache, it is written to disk. Unchanged Flob data
does not need to be written back to disk.
1.2 Whole-Flob-Based Approach
In this variant, the Flob data is organized using an AVLTree. A Flob is
copied to the cache either whole or not at all (e.g. if it does not fit into the
cache).
*/
#include "Flob.h"
#include "FlobManager.h"
#include <assert.h>
#include <iostream>
#include <cstdlib>
#undef __TRACE_ENTER__
#undef __TRACE_LEAVE__
#define __TRACE_ENTER__
#define __TRACE_LEAVE__
/*#define __TRACE_ENTER__ std::cerr << "Enter : " << \
__PRETTY_FUNCTION__ << std::endl;
#define __TRACE_LEAVE__ std::cerr << "Leave : " << \
__PRETTY_FUNCTION__ << "@" << __LINE__ << std::endl;
*/
//#undef SLOT_BASED_APPROACH
#define SLOT_BASED_APPROACH
#ifdef SLOT_BASED_APPROACH
/*
1 Class NativeCacheEntry
*/
class NativeCacheEntry{
public:
/*
1.1 Constructor
Creates a new entry for Flob __flob__ and slot number __slotNo__
with given slot size.
*/
NativeCacheEntry(const Flob& _flob, const size_t _slotNo, size_t _slotSize):
flobId(_flob.id), slotNo(_slotNo), tableNext(0), tablePrev(0),
lruPrev(0), lruNext(0), changed(false) {
assert(_flob.size>0);
size_t offset = _slotNo * _slotSize;
size = std::min(_slotSize, (_flob.getSize() - offset));
assert(offset <= _flob.getSize());
mem = (char*) malloc(size);
}
void resize(const size_t _newFlobSize, const size_t _slotSize){
size_t offset = slotNo * _slotSize;
assert(offset <= _newFlobSize);
size_t mySize = std::min(_slotSize, (_newFlobSize - offset));
if(mySize==size){
return;
}
if(mySize>0){
mem = (char*) realloc(mem, mySize);
} else {
free(mem);
mem = 0;
}
size = mySize;
}
/*
1.2 Destructor
Detroys an entry of a Flob.
*/
virtual ~NativeCacheEntry(){
if(mem){
free(mem);
}
mem = 0;
}
/*
1.3 hashValue
Returns a hash value for this entry. The hash value is used to quickly test
whether a slice of Flob data is within the cache.
*/
size_t hashValue(size_t tableSize){
return (flobId.hashValue() + slotNo) % tableSize;
}
/*
1.4 check for equality/ inequality
A pair (Flob-id, slot number) uniquely identifies a cache entry.
*/
bool operator==(const NativeCacheEntry& e){
return flobId == e.flobId &&
slotNo == e.slotNo;
}
bool operator!=(const NativeCacheEntry& e){
return !(*this == e);
}
bool matches(const Flob& f, const size_t slotNo) const{
return (f.id == flobId) && (this->slotNo == slotNo);
}
bool matches(const FlobId& fid, const size_t slotNo) const{
return (fid == flobId) && (this->slotNo == slotNo);
}
std::ostream& print(std::ostream& o) const{
o << "FlobId = " << flobId << ", slot = " << slotNo << ", size = " << size;
return o;
}
/*
1.5 Members
All members are private because only the ~FlobCache~ class knows about and uses
the ~NativeCacheEntry~ class.
*/
FlobId flobId;
size_t slotNo;
NativeCacheEntry* tableNext;
NativeCacheEntry* tablePrev;
NativeCacheEntry* lruPrev;
NativeCacheEntry* lruNext;
bool changed;
char* mem;
size_t size;
};
/*
1.6 Output operator
For simple output of an entry to an ostream.
*/
std::ostream& operator<<(std::ostream& o, const NativeCacheEntry& e){
o << "[" << e.flobId << ", " << e.slotNo << "]" ;
return o;
}
class NativeFlobCache{
public:
/*
2.1 Constructor
Creates a new cache with a given maximum size and a given slotsize.
*/
NativeFlobCache(size_t _maxSize, size_t _slotSize, size_t _avgSize):
maxSize(_maxSize), slotSize(_slotSize), usedSize(0), first(0), last(0) {
assert(maxSize > slotSize);
assert(_avgSize <= slotSize);
assert(slotSize);
tableSize = ((maxSize / _avgSize) * 2);
if(tableSize < 1u){
tableSize = 1u;
}
// initialize hashTable
hashtable = new NativeCacheEntry*[tableSize];
for(unsigned int i=0;i<tableSize; i++){
hashtable[i] = 0;
}
//assert(check());
}
/*
2.2 Destructor
destroys the cache.
*/
~NativeFlobCache(){
//assert(check());
clear();
delete[] hashtable;
hashtable = 0;
}
/*
2.3 clear
removes all entries from the cache.
*/
void clear(){
// first, kill entries from hashtable without deleting them from lru
//assert(check());
for(unsigned int i=0;i<tableSize;i++){
hashtable[i] =0;
}
NativeCacheEntry* entry = first;
size_t killed = 0;
while(entry){
NativeCacheEntry* victim = entry;
entry = entry->lruNext;
killed += victim->size;
delete victim;
}
first = 0;
last = 0;
usedSize = 0;
//assert(check());
}
/*
2.4 getData
Retrieves Flob data. First, it tries to get it from the cache.
If the data are not cached, the FlobManager is used to access the data and bring
it into the cache. Empty Flobs are not cached.
*/
bool getData(
const Flob& flob,
char* buffer,
const SmiSize offset,
const SmiSize size ){
if(size==0){ // nothing to do
return true;
}
assert(offset+size <= flob.size);
//assert(check());
size_t slotNo = offset / slotSize;
size_t slotOffset = offset % slotSize;
size_t bufferOffset(0);
while(bufferOffset < size){
if(!getDataFromSlot(flob, slotNo, slotOffset,
bufferOffset, size, buffer)){
std::cerr << "Warning getData failed" << std::endl;
//assert(check());
return false;
}
slotNo++;
slotOffset = 0; // slotoffset only neeeded for the first slot
//if(bufferOffset < size){
// std::cout << "Required second slot for Flob" << std::endl;
// std::cout << "bufferoffset: " << bufferOffset << std::endl;
// std::cout << "required size: " << size << std::endl;
// std::cout << "flob. size " << flob.size << std::endl;
// std::cout << "initial offset : " << offset << std::endl;
// }
}
//assert(check());
return true;
}
/*
2.5 putData
Updates the data of a Flob. If the data is not resident within the FlobCache,
it is brought to memory. All touched slices are marked ~modified~.
*/
bool putData(
const Flob& flob,
const char* buffer,
const SmiSize offset,
const SmiSize size) {
//assert(check());
size_t slotNo = offset / slotSize;
size_t slotOffset = offset % slotSize;
size_t bufferOffset(0);
while(bufferOffset < size){
putDataToFlobSlot(flob, slotNo, slotOffset, bufferOffset, size, buffer);
slotNo++;
slotOffset = 0;
}
//assert(check());
return true;
}
bool saveToDisk(Flob& flob, NativeCacheEntry* e){
if(e==0 || e->size==0){ // nothing to do
return true;
}
return FlobManager::getInstance().putData(flob, e->mem,
e->slotNo * slotSize,
e->size, true);
}
bool saveToDisk(const FlobId& flobId, NativeCacheEntry* e){
if(e==0 || e->size==0){ // nothing to do
return true;
}
return FlobManager::getInstance().putData(flobId, e->mem,
e->slotNo * slotSize, e->size);
}
/*
2.7 eraseSlot
Removes a slot from the cache. If saveChanges is set to true,
the slot is stored to disk if the ~modified~ flag is set.
*/
bool eraseSlot(Flob& flob, const size_t slotNo, const bool saveChanges){
//assert(check());
size_t index = (flob.id.hashValue() + slotNo) % tableSize;
NativeCacheEntry* e = hashtable[index];
while(e && !e->matches(flob, slotNo)){
e = e->tableNext;
}
if(e==0){ // slot not chached
return true;
}
if(saveChanges && e->changed){
if(!this->saveToDisk(flob,e)){
return false;
}
}
// remove slot from hashtable
if(hashtable[index] == e){ // e is the first entry in the table
hashtable[index] = e->tableNext;
if(hashtable[index]){
hashtable[index]->tablePrev = 0;
}
} else { // not the top entry
e->tablePrev->tableNext = e->tableNext;
if(e->tableNext){
e->tableNext->tablePrev = e->tablePrev;
}
}
e->tablePrev = 0;
e->tableNext = 0;
// remove from lru
if(e==first){
if(e==last){
first = 0;
last = 0;
} else {
first = e->lruNext;
first->lruPrev = 0;
e->lruNext = 0;
}
} else { // e is not the top lru element
if(e==last){
last = e->lruPrev;
last->lruNext = 0;
e->lruPrev = 0;
} else {
e->lruPrev->lruNext = e->lruNext;
e->lruNext->lruPrev = e->lruPrev;
e->lruPrev = 0;
e->lruNext = 0;
}
}
assert(usedSize >= e->size);
usedSize -= e->size;
delete e;
//assert(check());
return true;
}
/*
removes a slot from cache
*/
bool erase(Flob& flob, const bool saveChanges=false){
if(flob.size > 536870912){ // 512 MB
std::cerr << "Warning try to erase very big flob , size = "
<< flob.size <<std::endl;
}
//assert(check());
if(flob.size/slotSize < tableSize*4 ){
size_t numFlobs = flob.getSize() / slotSize;
for(unsigned int i = 0 ; i<= numFlobs; i++){
if(!eraseSlot(flob, i, saveChanges)){
//assert(check());
return false;
}
}
} else {
// scan the whole cache, because its faster in most cases
for(unsigned int i=0;i<tableSize;i++){
NativeCacheEntry* e = hashtable[i];
while(e){
if(e->flobId==flob.id){
if(!eraseSlot(flob,e->slotNo, saveChanges)){
return false;
} else {
e = hashtable[i];
}
} else {
e = e->tableNext;
}
}
}
}
return true;
}
/*
*/
bool create(Flob& flob){
//assert(check());
if(flob.size >0){
size_t num = flob.size/slotSize;
for(size_t i =0; i<num; i++){
NativeCacheEntry* e = new NativeCacheEntry(flob,i, slotSize );
if(e->size>0){
usedSize += e->size;
putAtFront(e);
size_t index = e->hashValue(tableSize);
if(hashtable[index]){
e->tableNext = hashtable[index];
hashtable[index]->tablePrev = e;
}
hashtable[index] = e;
reduce();
} else {
delete e;
}
}
}
//assert(check());
return true;
}
/*
2.7 getDataFromSlot
retrieves flob data for a specified flob from the cache and copies it to the
indicated position of a provided buffer.
*/
bool getDataFromSlot(const Flob& flob,
const size_t slotNo,
const size_t slotOffset,
size_t& bufferOffset,
const size_t size,
char* buffer) {
unsigned int index = (flob.hashValue() + slotNo) % tableSize;
if(hashtable[index]==0){
NativeCacheEntry* newEntry = createEntry(flob, slotNo,true);
if(newEntry){
hashtable[index] = newEntry;
usedSize += newEntry->size;
putAtFront(newEntry);
reduce(); // remove entries if too much memory is used
getData(newEntry, slotOffset, bufferOffset, size, buffer);
}
return true;
}
// hashtable[index] is already used
NativeCacheEntry* entry = hashtable[index];
while(entry->tableNext && !(entry->matches(flob.id, slotNo))){
entry = entry->tableNext;
}
if(!entry->matches(flob.id, slotNo)){ // no hit
NativeCacheEntry* newEntry = createEntry(flob, slotNo, true);
if(newEntry){
newEntry->tablePrev = entry;
entry->tableNext = newEntry;
assert(first);
putAtFront(newEntry);
usedSize += newEntry->size;
reduce();
getData(newEntry, slotOffset, bufferOffset, size, buffer);
}
return true;
} else { // hit, does not access disk data
getData(entry, slotOffset, bufferOffset, size, buffer);
bringToFront(entry);
return true;
}
}
/*
2.8 getData
Copies data from a given NativeCacheEntry to a buffer.
*/
void getData(NativeCacheEntry* entry,
size_t slotOffset,
size_t& bufferOffset,
const size_t size,
char* buffer){
size_t mb = size-bufferOffset; // missing bytes
size_t ab = entry->size - slotOffset; // bytes available in slot
size_t pb = std::min(mb,ab); // provided bytes
memcpy(buffer+bufferOffset, entry->mem + slotOffset, pb);
bufferOffset += pb;
}
/*
2.9 putData
puts data from a buffer to a given NativeCacheEntry and marks it modified.
*/
void putData(NativeCacheEntry* entry,
const size_t slotOffset,
size_t& bufferOffset,
const size_t size,
const char* buffer){
size_t mb = size-bufferOffset; // missing bytes
size_t ab = entry->size - slotOffset; // bytes available in slot
size_t pb = std::min(mb,ab); // provided bytes
if(pb>0){
memcpy(entry->mem + slotOffset, buffer+bufferOffset, pb);
bufferOffset += pb;
entry->changed=true;
}
}
/*
2.10 createEntry
produces a new NativeCacheEntry. If readData are set to be __true__ (default),
the slot data are loaded from disk using the ~FlobManager~.
*/
NativeCacheEntry* createEntry(const Flob& flob,
const size_t slotNo,
const bool readData=true) const{
NativeCacheEntry* res = new NativeCacheEntry(flob, slotNo, slotSize);
if(res->size==0){
delete res;
return 0;
}
if(readData){
FlobManager::getInstance().getData(flob,res->mem,
slotNo*slotSize, res->size, true);
res->changed = false;
}
return res;
}
/*
2.11 reduce
removes slots from the cache until the size is smaller than the cache's maximum
size.
*/
void reduce(){
while(last && usedSize > maxSize){
NativeCacheEntry* victim = last;
// remove from lru
last = victim->lruPrev;
last->lruNext = 0;
victim->lruPrev = 0;
assert(victim->lruNext == 0);
// remove from hashtable
if(victim->tablePrev){ // not the first entry in table
if(victim->tableNext){
victim->tablePrev->tableNext = victim->tableNext;
victim->tableNext->tablePrev = victim->tablePrev;
victim->tablePrev = 0;
victim->tableNext = 0;
} else { // last entry in table
victim->tablePrev->tableNext = 0;
victim->tablePrev = 0;
}
} else { // first entry in table
size_t index = victim->hashValue(tableSize);
assert(hashtable[index] == victim);
if(victim->tableNext==0){ // the only entry in hashtable
hashtable[index] = 0;
} else {
hashtable[index] = victim->tableNext;
victim->tableNext->tablePrev = 0;
victim->tableNext = 0;
}
}
assert(usedSize >= victim->size);
usedSize -= victim->size;
if(victim->changed){
saveToDisk(victim->flobId, victim);
}
delete victim;
}
if(!last){
first = 0;
}
}
/*
2.12 putDataToFlobSlot
puts data to a given slot. If the slot is not present in the
cache, it is created.
*/
bool putDataToFlobSlot(
const Flob& flob,
const size_t slotNo,
const size_t slotOffset,
size_t& bufferOffset,
const size_t size,
const char* buffer){
//assert(check());
size_t index = (flob.hashValue() + slotNo) % tableSize;
NativeCacheEntry* entry = hashtable[index];
while(entry && !entry->matches(flob.id, slotNo)){
entry = entry->tableNext;
}
if(!entry){ // slot not cached
entry = createEntry(flob, slotNo);
if(entry){
putAtFront(entry);
entry->tableNext = hashtable[index];
if(hashtable[index]){
hashtable[index]->tablePrev = entry;
}
hashtable[index] = entry;
usedSize += entry->size;
reduce();
} else {
return true;
}
}
putData(entry, slotOffset, bufferOffset, size,buffer);
return true;
}
bool resize(Flob& flob, const size_t newSize){
if(flob.size==0){
flob.size= newSize;
create(flob);
return true;
}
if(newSize==0){
erase(flob,false);
flob.size = 0;
return true;
}
if(newSize < flob.size){
size_t start = newSize/slotSize + 1;
size_t end = flob.size/slotSize + 1;
for(size_t i = start; i< end; i++){
eraseSlot(flob,i,false);
}
resizeSlot(flob,start-1,newSize);
flob.size=newSize;
return true;
}
// flob is enlarged
size_t start = flob.size/slotSize + 1;
resizeSlot(flob, start-1, newSize);
size_t end = newSize/slotSize +1;
flob.size = newSize;
for(size_t i=start;i<end;i++){
NativeCacheEntry* e = new NativeCacheEntry(flob,i,slotSize);
if(e->size>0){
usedSize += e->size;
putAtFront(e);
size_t index = e->hashValue(tableSize);
if(hashtable[index]){
e->tableNext = hashtable[index];
hashtable[index]->tablePrev = e;
}
hashtable[index] = e;
} else {
delete e;
}
}
reduce();
return true;
}
bool resizeSlot(Flob& flob, size_t slot, size_t newFlobSize){
size_t index = (flob.id.hashValue()+slot)%tableSize;
NativeCacheEntry* e = hashtable[index];
while( e && !e->matches(flob.id, slot)){
e = e->tableNext;
}
if(e){
size_t oldSize = e->size;
e->resize(newFlobSize, slotSize);
if(e->size==0){ // delete entry because size is 0
usedSize -= oldSize;
// delete from lru
if(first==e){
first = first->lruNext;
if(first){
first->lruPrev = 0;
} else {
last = 0;
}
} else if(e==last){
last = last->lruPrev;
last->lruNext = 0;
} else {
e->lruPrev->lruNext = e->lruNext;
e->lruNext->lruPrev = e->lruPrev;
}
e->lruNext=0;
e->lruPrev=0;
// delete from hashtbale
if(e==hashtable[index]){
hashtable[index] = e->tableNext;
if(hashtable[index]){
hashtable[index]->tablePrev = 0;
}
} else {
e->tablePrev->tableNext = e->tableNext;
if(e->tableNext){
e->tableNext->tablePrev = e->tablePrev;
}
}
e->tableNext = 0;
e->tablePrev = 0;
delete e;
} else {
usedSize = (usedSize + e->size) - oldSize;
}
}
return true;
}
/*
2.13 putAtFront
puts an unused NativeCacheEntry to the front of the LRU list.
*/
void putAtFront(NativeCacheEntry* newEntry){
assert(newEntry->lruPrev==0);
assert(newEntry->lruNext==0);
if(!first){
assert(!last);
first = newEntry;
last = newEntry;
} else {
newEntry->lruNext = first;
first->lruPrev = newEntry;
first = newEntry;
}
}
/*
2.14 bringToFront
moves an entry already present in the lru-list to the top of that list.
*/
void bringToFront(NativeCacheEntry* entry){
if(first==entry){ // already on front
return;
}
if(last==entry){ // the last element
assert(last->lruNext==0);
last= last->lruPrev;
last->lruNext=0;
entry->lruPrev=0;
entry->lruNext = first;
first->lruPrev = entry;
first = entry;
return;
}
// entry in in the middle of the list
assert(entry->lruPrev);
assert(entry->lruNext);
entry->lruPrev->lruNext = entry->lruNext;
entry->lruNext->lruPrev = entry->lruPrev;
entry->lruPrev = 0;
entry->lruNext = first;
first->lruPrev = entry;
first = entry;
}
/*
2.14 check
Debugging function.
*/
bool check(){
if(first==0 || last==0){
if(first!=last){
std::cerr << "inconsistence in lru list, first = " << (void*) first
<< " , last = " << (void*) last << std::endl;
return false;
}
for(unsigned int i=0;i< tableSize;i++){
if(hashtable[i]){
std::cerr << "lru is empty, but hashtable[" << i
<< "] contains an element" << std::endl;
return false;
}
}
// empty cache
return true;
}
// lru is not empty, check first and last element
if(first->lruPrev){
std::cerr << "lru: first has a predecessor" << std::endl;
return false;
}
if(last->lruNext){
std::cerr << "lru: last has a successor" << std::endl;
return false;
}
// check whether ech element in lru is also an element in hashtable
NativeCacheEntry* e = first;
int lrucount = 0;
size_t compSize(0);
while(e){
if(e->size > slotSize){
std::cerr << "entry found having a size > slotSize" << std::endl;
std::cerr << " slotSize = " << slotSize << " entry = " << (*e)
<< std::endl;
return false;
}
compSize += e->size ;
lrucount++;
size_t index = e->hashValue(tableSize);
if(!hashtable[index]){
std::cerr << "element " << (*e) << " stored in lru but hashtable["
<< index << " is null" << std::endl;
return false;
}
NativeCacheEntry* e2 = hashtable[index];
while(e2 && (*e)!=(*e2)){
e2 = e2->tableNext;
}
if(!e2){
std::cerr << "element " << (*e) << " stored in lru but not in hashtable["
<< index << "]" << std::endl;
return false;
}
e = e->lruNext;
}
if(compSize!=usedSize){
std::cerr << "difference in usedSize and stored size " << std::endl;
std::cerr << "usedSiez = " << usedSize << ", stored Size = "
<< compSize << std::endl;
return false;
}
// check hashtable
int tablecount = 0;
for(unsigned int i=0; i<tableSize;i++){
if(hashtable[i]){
if(hashtable[i]->tablePrev){
std::cerr << " hashtable[" << i << " has a predecessor" << std::endl;
}
e = hashtable[i];
while(e){
tablecount++;
if(e->hashValue(tableSize)!=i){
std::cerr << "element << " << (*e) << " has hashvalue "
<< e->hashValue(tableSize) << " but is stored at position "
<< i << std::endl;
return false;
}
if(e->tableNext){
if(e->tableNext->tablePrev!=e){
std::cerr << "error in tablelist found" << std::endl;
return false;
}
}
e = e->tableNext;
}
}
}
if(lrucount!=tablecount){
std::cerr << "lrucount = " << lrucount << " # tablecount = "
<< tablecount << std::endl;
return false;
}
return true;
}
private:
size_t maxSize; // maximum allocated memory
size_t slotSize; // size of a single slot
size_t usedSize; // currently used memory
NativeCacheEntry** hashtable; // open hash table
size_t tableSize; // size of the hashtable
NativeCacheEntry* first; // pointer to the first entry for lru
NativeCacheEntry* last; // pointer to the last entry for lru
};
#else // whole flob based approach
#include "AvlTree.h"
#include <set>
class NativeCacheEntry{
public:
FlobId flobId;
SmiSize size;
char* mem;
NativeCacheEntry* prev;
NativeCacheEntry* next;
bool changed;
NativeCacheEntry(): flobId(0,0,0,true), size(0),mem(0),prev(0),
next(0),changed(true){}
static void putAtFront(NativeCacheEntry* e,
NativeCacheEntry*& first,
NativeCacheEntry*& last){
if(first==0){ // first element in the list
assert(last==0);
first = e;
last = e;
e->next = 0;
e->prev = 0;
return;
}
assert(last!=0);
e->prev = 0;
e->next = first;
first->prev = e;
first = e;
}
static void connect(NativeCacheEntry* e1, NativeCacheEntry* e2,
NativeCacheEntry*& first, NativeCacheEntry*& last){
__TRACE_ENTER__;
if(e1==0 && e2==0){ // connect nothing
first = 0;
last = 0;
return;
}
if(e1==0){
e2->prev = 0;
first = e2;
return;
}
if(e2==0){
e1->next=0;
last = e1;
return;
}
e1->next = e2;
e2->prev = e1;
}
NativeCacheEntry(const Flob& flob, const bool alloc = false):
flobId(flob.id), size(flob.size),
mem(0), prev(0), next(0), changed(false) {
if(alloc){
mem = (char*) malloc(size);
}
}
bool operator==(const NativeCacheEntry& e) const{
__TRACE_ENTER__;
return flobId == e.flobId;
}
bool operator<(const NativeCacheEntry& e) const {
__TRACE_ENTER__;
return flobId < e.flobId;
}
bool operator>(const NativeCacheEntry& e) const{
__TRACE_ENTER__;
return flobId > e.flobId;
}
std::ostream& print(std::ostream& o) const{
o << " [ FlobId = " << flobId << ", "
<< " size = " << size << ", " ;
if(mem){
o << " mem : allocated ,";
} else {
o << " mem : NULL ,";
}
if(prev){
o << "prev = " << prev->flobId <<", ";
} else {
o << "prev = NULL, ";
}
if(next){
o << "next = " << next->flobId <<", ";
} else {
o << "next = NULL, ";
}
o << "changed = " << changed << "]";
return o;
}
};
std::ostream& operator<<(std::ostream& o, const NativeCacheEntry entry);
class NativeFlobCache{
public:
/*
~Constructor~
Creates an empy cache with a given capacity.
*/
NativeFlobCache(const size_t _maxSize): cache(),
maxSize(_maxSize),
size(0),
first(0),
last(0){
__TRACE_ENTER__;
// assert(check());
}
/*
~getData~
Returns data stored in the cahce. If the flob in not in cache and
even the flob cannot be put into the cache, the result is zero.
*/
bool getData(const Flob& flob,
char* dest,
const SmiSize offset,
const SmiSize size) {
__TRACE_ENTER__;
NativeCacheEntry* entry = useFlob(flob);
if(entry==0){ // entry not in cache and not cachable
return FlobManager::getInstance().getData(flob, dest,
offset, size, true);
}
assert(offset + size <= entry->size);
memcpy(dest, entry->mem + offset, size);
return true;
}
~NativeFlobCache(){
__TRACE_ENTER__;
// assert(check());
clear();
// assert(check());
}
/*
~clear~
Removes all entries from the cache.
*/
void clear(const bool saveChanges = false ){
__TRACE_ENTER__;
// assert(check());
if(size > maxSize){
std::cerr << "NativeFlobCache::clear(), size > maxSize detected"
<< std::endl;
std::cerr << "size = " << size << "maxSize = " << maxSize << std::endl;
}
NativeCacheEntry* e = first;
while(e){
if(saveChanges){
this->saveChanges(e);
}
free(e->mem);
size -= e->size;
e = e->next;
}
cache.Empty();
first = 0;
last = 0;
if(size!=0){
std::cerr << "FlobCache::clear() : size computation failed,"
<< " remaining size:"
<< size << std::endl;
size = 0;
}
}
/*
~erase~
Removes a given flob from the cache without
flushing changes to disk. If the flob was not cached,
the result will be false.
*/
bool erase(const Flob& victim){
__TRACE_ENTER__;
NativeCacheEntry finder(victim);
NativeCacheEntry* entry = (NativeCacheEntry*)cache.getMember(finder);
if(entry==0){
return false;
}
// delete from double linked list
NativeCacheEntry::connect(entry->prev, entry->next, first, last);
// give up memory
free(entry->mem);
entry->mem = 0;
size -= entry->size;
// delete from tree
cache.remove(finder);
return true;
}
/*
~create~
Allocates memory for the (uncached) flob.
*/
bool create(const Flob& flob){
__TRACE_ENTER__;
if((flob.size==0 ) || (flob.size>maxSize)){ // non-cachable sizes
return false;
}
NativeCacheEntry* entry = (NativeCacheEntry*)
cache.insert2(NativeCacheEntry(flob,true));
NativeCacheEntry::putAtFront(entry,first,last);
size += entry->size;
if(size>maxSize){
reduce();
}
return true;
}
/*
~eraseFromCache~
Stores chached made only in cache to disk and erases the flob from cache.
*/
bool eraseFromCache(const Flob& victim){
__TRACE_ENTER__;
return saveChanges(victim) && erase(victim);
}
/*
~putData~
Stores new data of a flob.
*/
bool putData(const Flob& dest,
const char* buffer,
const SmiSize& targetoffset,
const SmiSize& length){
__TRACE_ENTER__;
assert(targetoffset + length <= dest.size);
NativeCacheEntry* entry = useFlob(dest);
if(entry==0){
return FlobManager::getInstance().putData(dest,buffer,
targetoffset, length, true);
}
// put data into memory representation
memcpy(entry->mem+targetoffset, buffer, length);
entry->changed = true;
return true;
}
/*
~resize~
resizes the given flob using the size stored in the argument.
*/
bool resize(Flob& flob, SmiSize newSize){
__TRACE_ENTER__;
if(newSize > maxSize ){
eraseFromCache(flob); // saveChanges and kill cache
return FlobManager::getInstance().resize(flob, newSize,true);
}
if(newSize==0){
erase(flob); // kill cache without saving changes
flob.size = 0;
return true;
}
NativeCacheEntry finder(flob);
NativeCacheEntry* entry = (NativeCacheEntry*) cache.getMember(finder);
if(entry==0){
if(flob.size==0){
flob.size = newSize;
return create(flob);
}
flob.size = newSize;
bool ok = useFlob(flob)!=0; // chache flob
assert(ok);
return true;
}
NativeCacheEntry::connect(entry->prev, entry->next, first, last);
NativeCacheEntry::putAtFront(entry, first, last);
if(entry->size == newSize){ // no change
return true;
}
entry->mem = (char*)realloc(entry->mem, newSize);
size = size + - entry->size + newSize;
entry->size = newSize; // store size
flob.size = newSize;
return true;
}
private:
avltree::AVLTree<NativeCacheEntry> cache;
size_t maxSize;
size_t size;
NativeCacheEntry* first;
NativeCacheEntry* last;
/*
~check~
This is debugging function checking the cache for invalid state.
In the double linked list, each flobid must be unique.
The content of the avl tree and the list muts be the same.
If the chache is not empty, last and first cannot be null.
The size is the sum of all stored sizes.
*/
bool check(){
if(cache.IsEmpty()){
if(first!=0){
cout << "cache is empty, but first is not null !" << std::endl;
return false;
}
if(last!=0){
cout << "cache is empty, but last is not null !" << std::endl;
return false;
}
return true;
}
if(first==0){
cout << "cache is not empty, but first is null" << std::endl;
return false;
}
if(last==0){
cout << "cache is not empty, but last is null" << std::endl;
return false;
}
NativeCacheEntry* e = first;
int length = 0;
int csize = 0;
set<NativeCacheEntry> testset;
while(e){
length++;
csize += e->size;
if(testset.find(*e) != testset.end()){
cout << "Element found twice in the list " << (*e) << std::endl;
return false;
}
testset.insert(*e);
if(e->next==0){
if(last!=e){
cout << "last does not point to the last list element"
<< std::endl;
return false;
}
}
e = e->next;
}
// list structure ok, check content with the avltree
if(length!=cache.Size()){
cout << "different number of elements in cache and list " << std::endl;
cout << "listLength = " << length << std::endl;
cout << "treeSize = " << cache.Size() << std::endl;
return false;
}
if(csize!=(int)size){
cout << "different sizes), computed : " << csize
<< " , stored : " << size << std::endl;
return false;
}
set<NativeCacheEntry>::iterator it = testset.begin();
while(it!=testset.end()){
if(cache.getMember(*it) == 0){
cout << "Element " << (*it)
<< "stored in list but not in tree " << std::endl;
return false;
}
it++;
}
return true;
}
NativeCacheEntry* useFlob(const Flob& flob){
__TRACE_ENTER__;
NativeCacheEntry finder(flob);
NativeCacheEntry* entry1 = (NativeCacheEntry*) cache.getMember(finder);
if(entry1!=0){
// entry cached
NativeCacheEntry::connect(entry1->prev, entry1->next, first, last);
NativeCacheEntry::putAtFront(entry1, first, last);
return entry1;
}
// flob not already stored
if(flob.size>maxSize || flob.size==0){ // non-cachable sizes
return 0;
}
// bring entry to chache
NativeCacheEntry entry(flob);
entry.changed = false;
entry.mem = (char*) malloc(flob.size);
FlobManager::getInstance().getData(flob, entry.mem,0, flob.size, true);
NativeCacheEntry* stored = (NativeCacheEntry*) cache.insert2(entry);
NativeCacheEntry::putAtFront(stored, first, last);
size += flob.size;
if(size > maxSize){
reduce();
}
return stored;
}
bool saveChanges(const Flob& flob){
__TRACE_ENTER__;
NativeCacheEntry finder(flob);
NativeCacheEntry* entry = (NativeCacheEntry*) cache.getMember(finder);
if(entry==0){
return true;
}
return saveChanges(entry);
}
bool saveChanges(NativeCacheEntry* entry){
if(!entry->changed){
return true;
}
Flob flob(entry->flobId, entry->size);
bool res1 = FlobManager::getInstance().resize(flob, entry->size, true);
bool res = FlobManager::getInstance().putData(flob,
entry->mem,
0,
entry->size,
true);
if(res){
entry->changed=false;
}
return res&&res1;
}
bool reduce(){
__TRACE_ENTER__;
while(!cache.IsEmpty() && size > maxSize){
NativeCacheEntry* e = last;
saveChanges(last);
assert(e!=0);
// remove from list
last = e->prev;
if(last==0){
first = 0;
}else {
last->next = 0;
}
e->prev = 0;
e->next = 0;
free(e->mem);
e->mem=0;
size -= e->size;
bool rm = cache.remove(*e);
assert(rm);
}
return true;
}
};
#endif
Reference in New Issue View Git Blame Copy Permalink