I'm developing a LRU-cache in C++, using boost mutexes and locks, in a multi-threaded environment. The architecture is based on a boost::unordered_map + a lock-free-queue Insertions work in non-blocking mode (try_lock), but removals should lock the map and proceed. The problem is that very rarely, the cache-access deadlocks in the removal.
.h
typedef boost::function<std::string ( const std::string &key )> LoaderFunction;
class ListNode;
struct CacheEntry {
CacheEntry(std::string key="", std::string value="");
ListNode * createLruListNode() const;
ListNode * getLruListNode() const;
virtual ~CacheEntry();
const std::string key;
const std::string value;
private:
ListNode ** listNodePP;
};
struct ListNode {
ListNode(const CacheEntry* entry = NULL);
~ListNode();
void setCacheEntry(const CacheEntry* entry);
const CacheEntry * getCacheEntry();
void setDirty();
private:
const CacheEntry * cacheEntry;
bool dirty;
};
typedef LockFreeQueue<ListNode*> List;
typedef boost::unordered_map
< const string , const CacheEntry * >
Cache;
typedef Cache::iterator CacheIter;
.cpp
#include "LockFreeQueue.h"
#include <unistd.h>
using namespace std;
/* ... */
ListNode::ListNode(const CacheEntry* e2) : cacheEntry(e2) {
dirty=false;
}
void ListNode::setCacheEntry(const CacheEntry* entry) {
cacheEntry=entry;
}
const CacheEntry* ListNode::getCacheEntry() {
if(dirty) {
return NULL;
}
return cacheEntry;
}
void ListNode::setDirty() {
dirty=true;
}
std::string PeachCachePartition::get(const string key) {
CacheIter iter=cache->find(key);
string value;
if(iter!=cache->end()) {
__sync_fetch_and_add(_hits,1);
const CacheEntry* entry=iter->second;
value=(entry->value);
lruList->enqueue(entry->getLruListNode());
if(size() > max) { // removes some
int howMany = (int) ceil((*_misses)/(*_hits))+1;
int k=0;
ListNode removedListNode=ListNode();
ListNode * p=&removedListNode;
ListNode ** pp=&p;
while(size() > max && k<howMany) {
if(lruList->dequeue(pp)) {
const CacheEntry * toBeRemoved=p->getCacheEntry();
if(toBeRemoved) {
remove(toBeRemoved->key);
k++;
}
}
}
}
} else {
__sync_fetch_and_add(_misses,1);
value=loader(key);
if(value.size()>0) {
put(key,value);
}
}
return value;
}
void PeachCachePartition::remove(const std::string &key) {
try {
boost::lock_guard<boost::mutex> mapLockGuard(mapMutex);
CacheIter iter = cache->find(key);
if(iter!=cache->end()) {
const CacheEntry * toBeRemoved=iter->second;
if(toBeRemoved->getLruListNode()) {
toBeRemoved->getLruListNode()->setDirty();
}
delete(toBeRemoved);
cache->erase(iter);
__sync_sub_and_fetch(_size,1);
}
} catch (std::exception &e) {
Logger::err(e.what());
}
}
void PeachCachePartition::put(const std::string &key, std::string &value) {
try {
boost::unique_lock<boost::mutex> mapLockGuard(mapMutex,boost::try_to_lock);
if(mapLockGuard.owns_lock()) {
CacheIter iter=cache->find(key);
const CacheEntry * entry;
if(iter!=cache->end()) {
entry=iter->second;
entry->getLruListNode()->setDirty();
} else {
entry = new CacheEntry(key,value);
__sync_add_and_fetch(_size,1);
(*cache)[key] = entry;
}
entry->createLruListNode()->setCacheEntry(entry);
lruList->enqueue(entry->getLruListNode());
}
} catch (std::exception &e) {
Logger::err(e.what());
}
}
Can you explain me what's wrong? I'm almost sure it deadlocks in the removal as it is the only lock it must acquire.
Thanks everybody
edit: I'm using this cache in an apache module which runs mpm_prefork_module: could this be the problem? Should I use boost::interprocess instead of boost::thread?