mongodb/mongo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update, delete

Browse Source
This commit is contained in:
Dwight 2007-10-30 11:35:17 -04:00
parent 1e7d144631
commit fc316ba2be
7 changed files with 238 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
db/db.cpp
View File

@ -22,7 +22,7 @@ struct MyStartupTests {
*/
void quicktest() {
cout << "quicktest\n";
cout << "quicktest()\n";
MemoryMappedFile mmf;
char *m = (char *) mmf.map("/tmp/abc", 16384);
@ -106,23 +106,6 @@ Record* findByOID(const char *ns, OID *oid) {
return 0;
}
void updateByOID(const char *ns, char *objdata, int objsize, OID *oid) {
Record *r = findByOID(ns, oid);
if( r == 0 ) {
cout << "updateByOID: no such record " << ns << endl;
return;
}
if( objsize > r->netLength() ) {
cout << "ERROR: updateByOID: growing records not implemented yet." << endl;
return;
}
/* note: need to be smarter if it gets a lot smaller? */
/* this really dumb for now as it gets smaller but doesn't allow regrowth
to the original size! */
memcpy(r->data, objdata, objsize);
r->setNewLength(objsize);
}
#pragma pack(push)
#pragma pack(1)
struct EmptyObject {

3
db/makefile
View File

@ -18,3 +18,6 @@ db: $(OBJS) db.o
clean:
-rm -f $(OBJS) db.o
-rm -f db
cleandb:
rm /data/namespace.idx /data/temp.dat

217
db/pdfile.cpp
View File

@ -5,6 +5,7 @@ todo:
_ manage deleted records. bucket?
_ use deleted on inserts!
_ quantize allocations
_ table scans must be sequential, not next/prev pointers
*/
#include "stdafx.h"
@ -13,9 +14,6 @@ _ quantize allocations
#include "../util/mmap.h"
#include "../util/hashtab.h"
#include <map>
#include <string>
DataFileMgr theDataFileMgr;
/* just temporary */
@ -29,6 +27,7 @@ int bucketSizes[] = {
0x400000, 0x800000
};
const int Buckets = 19;
const int MaxBucket = 18;
class NamespaceDetails {
public:
@ -44,15 +43,101 @@ public:
return Buckets-1;
}
void addDeletedRec(Record *d, DiskLoc dloc) {
void addDeletedRec(DeletedRecord *d, DiskLoc dloc) {
int b = bucket(d->lengthWithHeaders);
DiskLoc& list = deletedList[b];
DiskLoc oldHead = list;
list = dloc;
d->nextDeleted() = oldHead;
d->nextDeleted = oldHead;
}
DiskLoc alloc(int lenToAlloc, DiskLoc& extentLoc);
private:
DiskLoc _alloc(int len);
};
DiskLoc NamespaceDetails::alloc(int lenToAlloc, DiskLoc& extentLoc) {
lenToAlloc = (lenToAlloc + 3) & 0xfffffffc;
DiskLoc loc = _alloc(lenToAlloc);
if( loc.isNull() )
return loc;
DeletedRecord *r = loc.drec();
/* note we want to grab from the front so our next pointers on disk tend
to go in a forward direction which is important for performance. */
int regionlen = r->lengthWithHeaders;
extentLoc.set(loc.a(), r->extentOfs);
int left = regionlen - lenToAlloc;
if( left < 24 ) {
// you get the whole thing.
return loc;
}
/* split off some for further use. */
r->lengthWithHeaders = lenToAlloc;
DiskLoc newDelLoc = loc;
newDelLoc.inc(lenToAlloc);
DeletedRecord *newDel = newDelLoc.drec();
newDel->extentOfs = r->extentOfs;
newDel->lengthWithHeaders = left;
newDel->nextDeleted.Null();
addDeletedRec(newDel, newDelLoc);
return loc;
}
/* returned item is out of the deleted list upon return */
DiskLoc NamespaceDetails::_alloc(int len) {
DiskLoc *prev;
DiskLoc *bestprev = 0;
DiskLoc bestmatch;
int bestmatchlen = 0x7fffffff;
int b = bucket(len);
DiskLoc cur = deletedList[b]; prev = &deletedList[b];
int extra = 5; // look for a better fit, a little.
int chain = 0;
while( 1 ) {
if( cur.isNull() ) {
// move to next bucket. if we were doing "extra", just break
if( bestmatchlen < 0x7fffffff )
break;
b++;
if( b > MaxBucket ) {
// out of space. alloc a new extent.
return DiskLoc();
}
cur = deletedList[b]; prev = &deletedList[b];
continue;
}
DeletedRecord *r = cur.drec();
if( r->lengthWithHeaders >= len &&
r->lengthWithHeaders < bestmatchlen ) {
bestmatchlen = r->lengthWithHeaders;
bestmatch = cur;
bestprev = prev;
}
if( bestmatchlen < 0x7fffffff && --extra <= 0 )
break;
if( ++chain > 30 && b < MaxBucket ) {
// too slow, force move to next bucket to grab a big chunk
b++;
chain = 0;
cur.Null();
}
else {
cur = r->nextDeleted; prev = &r->nextDeleted;
}
}
/* unlink ourself from the deleted list */
*bestprev = bestmatch.drec()->nextDeleted;
return bestmatch;
}
class NamespaceIndex {
public:
NamespaceIndex() { }
@ -98,7 +183,8 @@ void PhysicalDataFile::open(const char *filename, int length) {
}
/* prev - previous extent for this namespace. null=this is the first one. */
Extent* PhysicalDataFile::newExtent(const char *ns, DiskLoc& loc, Extent *prev) {
Extent* PhysicalDataFile::newExtent(const char *ns) {
DiskLoc loc;
int left = header->unusedLength - ExtentSize;
if( left < 0 ) {
cout << "ERROR: newExtent: no more room for extents. write more code" << endl;
@ -110,22 +196,29 @@ Extent* PhysicalDataFile::newExtent(const char *ns, DiskLoc& loc, Extent *prev)
header->unusedLength -= ExtentSize;
loc.setOfs(offset);
Extent *e = _getExtent(loc);
e->init(ns, ExtentSize, offset);
if( prev ) {
assert( prev->xnext.isNull() );
prev->xnext = e->myLoc;
e->xprev = prev->myLoc;
} else {
e->xprev.Null();
DiskLoc emptyLoc = e->init(ns, ExtentSize, offset);
DiskLoc oldExtentLoc;
if( namespaceIndex.find(ns, oldExtentLoc) ) {
Extent *old = oldExtentLoc.ext();
assert( old->xprev.isNull() );
old->xprev = loc;
e->xnext = oldExtentLoc;
namespaceIndex.details(ns)->firstExtent = loc;
}
e->xnext.Null();
else {
namespaceIndex.add(ns, loc);
}
namespaceIndex.details(ns)->addDeletedRec(emptyLoc.drec(), emptyLoc);
return e;
}
/*---------------------------------------------------------------------*/
/* assumes already zeroed -- insufficient for block 'reuse' perhaps */
void Extent::init(const char *nsname, int _length, int _offset) {
DiskLoc Extent::init(const char *nsname, int _length, int _offset) {
magic = 0x41424344;
myLoc.setOfs(_offset);
xnext.Null(); xprev.Null();
@ -133,16 +226,18 @@ void Extent::init(const char *nsname, int _length, int _offset) {
length = _length;
firstRecord.Null(); lastRecord.Null();
firstEmptyRegion = myLoc;
firstEmptyRegion.inc( (extentData-(char*)this) );
DiskLoc emptyLoc = myLoc;
emptyLoc.inc( (extentData-(char*)this) );
Record *empty1 = (Record *) extentData;
Record *empty = getRecord(firstEmptyRegion);
DeletedRecord *empty1 = (DeletedRecord *) extentData;
DeletedRecord *empty = (DeletedRecord *) getRecord(emptyLoc);
assert( empty == empty1 );
empty->lengthWithHeaders = _length - (extentData - (char *) this);
empty->next.Null();
empty->extentOfs = myLoc.getOfs();
return emptyLoc;
}
/*
Record* Extent::newRecord(int len) {
if( firstEmptyRegion.isNull() )
return 0;
@ -153,7 +248,7 @@ Record* Extent::newRecord(int len) {
Record *r = getRecord(newRecordLoc);
int left = r->netLength() - len;
if( left < 0 ) {
/* this might be wasteful if huge variance in record sizes in a namespace */
//
firstEmptyRegion.Null();
return 0;
}
@ -186,6 +281,7 @@ Record* Extent::newRecord(int len) {
return r;
}
*/
/*---------------------------------------------------------------------*/
@ -200,32 +296,29 @@ Cursor DataFileMgr::findAll(const char *ns) {
return Cursor( e->firstRecord );
}
void DataFileMgr::deleteRecord(const char *ns, Record *todelete, const DiskLoc& dl) {
void DataFileMgr::deleteRecord(const char *ns, Record *todelete, const DiskLoc& dl)
{
/* remove ourself from the record next/prev chain */
DiskLoc prev = todelete->prev.getPrev(dl);
if( !prev.isNull() )
getRecord(prev)->next.set( todelete->next.getNext(dl) );
DiskLoc next = todelete->next.getNext(dl);
if( !next.isNull() )
getRecord(next)->prev.set( todelete->prev.getPrev(dl) );
{
if( todelete->prevOfs != DiskLoc::NullOfs )
todelete->getPrev(dl).rec()->nextOfs = todelete->nextOfs;
if( todelete->nextOfs != DiskLoc::NullOfs )
todelete->getNext(dl).rec()->prevOfs = todelete->prevOfs;
}
/* remove ourself from extent pointers */
DiskLoc ext = todelete->prev.myExtent(dl);
if( !ext.isNull() ) {
// we are first.
Extent *e = DataFileMgr::getExtent(ext);
assert( e->firstRecord == dl );
e->firstRecord = next;
}
ext = todelete->next.myExtent(dl);
if( !ext.isNull() ) {
Extent *e = DataFileMgr::getExtent(ext);
assert( e->lastRecord == dl );
e->lastRecord = next;
{
Extent *e = todelete->myExtent(dl);
if( e->firstRecord == dl )
e->firstRecord.setOfs(todelete->nextOfs);
if( e->lastRecord == dl )
e->lastRecord.setOfs(todelete->prevOfs);
}
NamespaceDetails* d = namespaceIndex.details(ns);
d->addDeletedRec(todelete, dl);
{
NamespaceDetails* d = namespaceIndex.details(ns);
d->addDeletedRec((DeletedRecord*)todelete, dl);
}
}
/** Note: as written so far, if the object shrinks a lot, we don't free up space. */
@ -246,16 +339,42 @@ void DataFileMgr::update(
}
void DataFileMgr::insert(const char *ns, const void *buf, int len) {
DiskLoc loc;
bool found = namespaceIndex.find(ns, loc);
if( !found ) {
NamespaceDetails *d = namespaceIndex.details(ns);
if( d == 0 ) {
cout << "New namespace: " << ns << endl;
temp.newExtent(ns, loc, 0);
namespaceIndex.add(ns, loc);
temp.newExtent(ns);
d = namespaceIndex.details(ns);
}
Extent *e = temp.getExtent(loc);
Record *r = e->newRecord(len); /*todo: if zero returned, need new extent */
DiskLoc extentLoc;
int lenWHdr = len + Record::HeaderSize;
DiskLoc loc = d->alloc(lenWHdr, extentLoc);
if( loc.isNull() ) {
// out of space
cout << "allocating new extent for " << ns << endl;
temp.newExtent(ns);
loc = d->alloc(lenWHdr, extentLoc);
if( loc.isNull() ) {
cout << "ERROR: out of space in datafile. write more code." << endl;
assert(false);
return;
}
}
Record *r = loc.rec();
assert( r->lengthWithHeaders >= lenWHdr );
memcpy(r->data, buf, len);
Extent *e = r->myExtent(loc);
if( e->lastRecord.isNull() ) {
e->firstRecord = e->lastRecord = loc;
r->prevOfs = r->nextOfs = DiskLoc::NullOfs;
}
else {
Record *oldlast = e->lastRecord.rec();
r->prevOfs = e->lastRecord.getOfs();
r->nextOfs = DiskLoc::NullOfs;
e->lastRecord = loc;
}
}
void DataFileMgr::init() {

117
db/pdfile.h
View File

@ -48,7 +48,7 @@ public:
void open(const char *filename, int length = 64 * 1024 * 1024);
private:
Extent* newExtent(const char *ns, DiskLoc& loc, Extent *prev);
Extent* newExtent(const char *ns);
Extent* getExtent(DiskLoc loc);
Extent* _getExtent(DiskLoc loc);
Record* recordAt(DiskLoc dl);
@ -82,53 +82,31 @@ extern DataFileMgr theDataFileMgr;
#pragma pack(push)
#pragma pack(1)
/* lots of code to make our next/prev pointers 4 bytes instead of 8! */
class SmartLoc {
public:
SmartLoc() { x = 0; }
DiskLoc getNextEmpty(const DiskLoc& myLoc) {
assert( x >= 0 );
return DiskLoc(myLoc.a(), x);
}
DiskLoc getNext(const DiskLoc& myLoc);
DiskLoc getPrev(const DiskLoc& myLoc);
/* if a next pointer, marks as last. if a prev pointer, marks as first */
void markAsFirstOrLastInExtent(Extent *e);
bool firstInExtent() { return x < 0; }
bool lastInExtent() { return x < 0; }
void set(const DiskLoc& nextprevRecordLoc) { x = nextprevRecordLoc.getOfs(); }
void Null() { x = 0; } /* this is for empty records only. nonempties point to the extent. */
DiskLoc myExtent(const DiskLoc& myLoc);
private:
int x;
};
class DeletedRecord {
public:
DiskLoc nextDeleted;
int lengthWithHeaders;
int myOfs;
DiskLoc myExtent;
void init(const DiskLoc& extent, const DiskLoc& myLoc) {
myExtent = extent;
myOfs = myLoc.getOfs();
}
int extentOfs;
DiskLoc nextDeleted;
};
const int MinRecordSize = sizeof(DeletedRecord);
class Record {
public:
enum { HeaderSize = 12 };
SmartLoc next, prev;
enum { HeaderSize = 16 };
int lengthWithHeaders;
int extentOfs, nextOfs, prevOfs;
char data[4];
// bool haveNext() { return !next.isNull(); }
int netLength() { return lengthWithHeaders - HeaderSize; }
void setNewLength(int netlen) { lengthWithHeaders = netlen + HeaderSize; }
//void setNewLength(int netlen) { lengthWithHeaders = netlen + HeaderSize; }
/* use this when a record is deleted. basically a union with next/prev fields */
DiskLoc& asDeleted() { return *((DeletedRecord*) this); }
DeletedRecord& asDeleted() { return *((DeletedRecord*) this); }
Extent* myExtent(const DiskLoc& myLoc) {
return DataFileMgr::getExtent(DiskLoc(myLoc.a(), extentOfs));
}
/* get the next record in the namespace, traversing extents as necessary */
DiskLoc getNext(const DiskLoc& myLoc);
DiskLoc getPrev(const DiskLoc& myLoc);
};
/* extents are regions where all the records within the region
@ -145,8 +123,11 @@ public:
DiskLoc firstRecord, lastRecord;
char extentData[4];
/* assumes already zeroed -- insufficient for block 'reuse' perhaps */
void init(const char *nsname, int _length, int _offset);
/* assumes already zeroed -- insufficient for block 'reuse' perhaps
Returns a DeletedRecord location which is the data in the extent ready for us.
Caller will need to add that to the freelist structure in namespacedetail.
*/
DiskLoc init(const char *nsname, int _length, int _offset);
void assertOk() { assert(magic == 0x41424344); }
@ -164,37 +145,6 @@ public:
Extent* getPrevExtent() { return xprev.isNull() ? 0 : DataFileMgr::getExtent(xprev); }
};
inline DiskLoc SmartLoc::getNext(const DiskLoc& myLoc) {
assert( x != 0 );
if( x > 0 )
return DiskLoc(myLoc.a(), x);
// we are the last one in this extent.
DiskLoc extLoc(myLoc.a(), -x);
Extent *e = DataFileMgr::getExtent(extLoc);
assert( e->lastRecord == myLoc );
Extent *nxt = e->getNextExtent();
return nxt ? nxt->firstRecord : DiskLoc();
}
inline DiskLoc SmartLoc::getPrev(const DiskLoc& myLoc) {
assert( x != 0 );
if( x > 0 )
return DiskLoc(myLoc.a(), x);
// we are the first one in this extent.
DiskLoc extLoc(myLoc.a(), -x);
Extent *e = DataFileMgr::getExtent(extLoc);
assert( e->firstRecord == myLoc );
Extent *prv = e->getPrevExtent();
return prv ? prv->lastRecord : DiskLoc();
}
/* only works if first (or last for 'next') record in the extent. */
inline DiskLoc SmartLoc::myExtent(const DiskLoc& myLoc) {
return x < 0 ? DiskLoc(myLoc.a(), -x) : DiskLoc();
}
/* if a next pointer, marks as last. if a prev pointer, marks as first */
inline void SmartLoc::markAsFirstOrLastInExtent(Extent *e) {
x = -e->myLoc.getOfs();
}
/*
----------------------
Header
@ -270,7 +220,7 @@ public:
if( eof() )
return false;
Record *r = current();
curr = r->next.getNext(curr);
curr = r->getNext(curr);
return ok();
}
@ -288,3 +238,30 @@ inline Extent* DataFileMgr::getExtent(const DiskLoc& dl) {
inline Record* DataFileMgr::getRecord(const DiskLoc& dl) {
return theDataFileMgr.temp.recordAt(dl);
}
inline DiskLoc Record::getNext(const DiskLoc& myLoc) {
if( nextOfs )
return DiskLoc(myLoc.a(), nextOfs);
Extent *e = myLoc.ext();
if( e->xnext.isNull() )
return DiskLoc(); // end of table.
return e->xnext.ext()->firstRecord;
}
inline DiskLoc Record::getPrev(const DiskLoc& myLoc) {
if( prevOfs )
return DiskLoc(myLoc.a(), prevOfs);
Extent *e = myLoc.ext();
if( e->xprev.isNull() )
return DiskLoc();
return e->xprev.ext()->firstRecord;
}
inline Record* DiskLoc::rec() const {
return DataFileMgr::getRecord(*this);
}
inline DeletedRecord* DiskLoc::drec() const {
return (DeletedRecord*) rec();
}
inline Extent* DiskLoc::ext() const {
return DataFileMgr::getExtent(*this);
}

18
db/storage.h
View File

@ -9,21 +9,27 @@
#pragma pack(push)
#pragma pack(1)
class Record;
class DeletedRecord;
class Extent;
class DiskLoc {
int reserved; /* this will be volume, file #, etc. */
int ofs;
public:
enum { NullOfs = -1 };
int a() const { return reserved; }
DiskLoc(int a, int b) : reserved(a), ofs(b) { }
DiskLoc() { reserved = -1; ofs = -1; }
DiskLoc() { reserved = -1; ofs = NullOfs; }
bool isNull() { return ofs == -1; }
void Null() { reserved = -1; ofs = -1; }
bool isNull() { return ofs == NullOfs; }
void Null() { reserved = -1; ofs = NullOfs; }
void assertOk() { assert(!isNull()); }
int getOfs() const { return ofs; }
void set(int a, int b) { reserved=a; ofs=b; }
void setOfs(int _ofs) {
reserved = -2;
reserved = -2; /*temp: fix for multiple datafiles */
ofs = _ofs;
}
@ -35,6 +41,10 @@ public:
bool sameFile(DiskLoc b) { return reserved == b.reserved; /* not really done...*/ }
bool operator==(const DiskLoc& b) { return reserved==b.reserved && ofs == b.ofs; }
Record* rec() const;
DeletedRecord* drec() const;
Extent* ext() const;
};
#pragma pack(pop)

4
stdafx.h
View File

@ -21,6 +21,10 @@ typedef char _TCHAR;
#include <fstream>
using namespace std;
#include <map>
#include <string>
#include <vector>
#if !defined(_WIN32)
typedef int HANDLE;
inline void strcpy_s(char *dst, unsigned len, const char *src) { strcpy(dst, src); }

2
util/sock.h
View File

@ -61,7 +61,7 @@ inline bool UDPConnection::init(const SockAddr& myAddr) {
cout << "invalid socket? " << errno << endl;
return false;
}
cout << sizeof(sockaddr_in) << ' ' << myAddr.addressSize << endl;
//cout << sizeof(sockaddr_in) << ' ' << myAddr.addressSize << endl;
if( bind(sock, (sockaddr *) &myAddr.sa, myAddr.addressSize) != 0 ) {
cout << "udp init failed" << endl;
closesocket(sock);