cap connection pool size

This reverts commit 17338d81b0.

SConstruct

@@ -1243,7 +1243,7 @@ elif not onlyServer:
         shellEnv["CPPPATH"].remove( "/usr/64/include" )
         shellEnv["LIBPATH"].remove( "/usr/64/lib" )
         shellEnv.Append( CPPPATH=filterExists(["/sw/include" , "/opt/local/include"]) )
-        shellEnv.Append( LIBPATH=filterExists(["/sw/lib/", "/opt/local/lib" , "/usr/lib"]) )
+        shellEnv.Append( LIBPATH=filterExists(["/sw/lib/", "/opt/local/lib" , "/usr/lib", "/usr/local/lib" ]) )
 
     l = shellEnv["LIBS"]
 
@@ -1361,6 +1361,9 @@ smokeEnv.AlwaysBuild( "addMongodReqNoJsTargets" )
 smokeEnv.Alias( "smokeAllNoJs", [ "smoke", "mongosTest", "addMongodReqNoJsTargets" ] )
 smokeEnv.AlwaysBuild( "smokeAllNoJs" )
 
+# dummy smokeDur for 1.6
+smokeEnv.Alias( "smokeDur" , "util/concurrency/" )
+
 def recordPerformance( env, target, source ):
     from buildscripts import benchmark_tools
     global perftest

bson/bsonelement.h

@@ -511,7 +511,7 @@ private:
         }
     }
 
-    /** Retrieve int value for the element safely.  Zero returned if not a number. */
+    /** Retrieve int value for the element safely.  Zero returned if not a number. Converted to int if another numeric type. */
     inline int BSONElement::numberInt() const { 
         switch( type() ) {
         case NumberDouble:

bson/bsoninlines.h

@@ -387,16 +387,7 @@ namespace mongo {
             }
             break;
         case NumberDouble:
-            {
-                stringstream tmp;
-                tmp.precision( 16 );
-                tmp << number();
-                string n = tmp.str();
-                s << n;
-                // indicate this is a double:
-                if( strchr(n.c_str(), '.') == 0 && strchr(n.c_str(), 'E') == 0 && strchr(n.c_str(), 'N') == 0 )
-                    s << ".0";
-            }
+            s.appendDoubleNice( number() );
             break;
         case NumberLong:
             s << _numberLong();

bson/bsonobj.h

@@ -40,7 +40,7 @@ namespace mongo {
 
      BSON object format:
      
-     \code
+     code
      <unsigned totalSize> {<byte BSONType><cstring FieldName><Data>}* EOO
      
      totalSize includes itself.

bson/util/builder.h

@@ -164,6 +164,7 @@ namespace mongo {
 #define SBNUM(val,maxSize,macro) \
             int prev = _buf.l; \
             int z = sprintf( _buf.grow(maxSize) , macro , (val) );  \
+            assert( z >= 0 ); \
             _buf.l = prev + z; \
             return *this; 
 
@@ -197,6 +198,17 @@ namespace mongo {
         }
 #undef SBNUM
 
+        void appendDoubleNice( double x ){
+            int prev = _buf.l;
+            char * start = _buf.grow( 32 );
+            int z = sprintf( start , "%.16g" , x );
+            assert( z >= 0 );
+            _buf.l = prev + z;
+            if( strchr(start, '.') == 0 && strchr(start, 'E') == 0 && strchr(start, 'N') == 0 ){
+                write( ".0" , 2 );
+            }
+        }
+
         void write( const char* buf, int len){
             memcpy( _buf.grow( len ) , buf , len );
         }

client/connpool.h

@@ -87,8 +87,17 @@ namespace mongo {
                 delete c;
                 return;
             }
-            scoped_lock L(_mutex);
-            _pools[host].pool.push(c);
+
+            {
+                scoped_lock L(_mutex);
+                if ( _pools[host].pool.size() < 50 ) {
+                    _pools[host].pool.push(c);
+                    return;
+                }
+            }
+
+            // if we get here it means we didn't add for some reason
+            delete c;
         }
         void addHook( DBConnectionHook * hook );
         void appendInfo( BSONObjBuilder& b );

client/dbclient.cpp

@@ -502,7 +502,7 @@ namespace mongo {
 
         if ( !p->connect(*server) ) {
             stringstream ss;
-            ss << "couldn't connect to server " << _serverString << '}';
+            ss << "couldn't connect to server " << _serverString;
             errmsg = ss.str();
             failed = true;
             return false;
@@ -1063,6 +1063,18 @@ namespace mongo {
         return checkMaster()->findOne(a,b,c,d);
     }
 
+    bool DBClientReplicaSet::isMember( const DBConnector * conn ) const {
+        if ( conn == this )
+            return true;
+        
+        for ( unsigned i=0; i<_conns.size(); i++ )
+            if ( _conns[i]->isMember( conn ) )
+                return true;
+        
+        return false;
+    }
+    
+
     bool serverAlive( const string &uri ) {
         DBClientConnection c( false, 0, 20 ); // potentially the connection to server could fail while we're checking if it's alive - so use timeouts
         string err;

client/dbclient.h

@@ -106,10 +106,11 @@ namespace mongo {
             _finishInit();
         }
 
-        ConnectionString( ConnectionType type , const vector<HostAndPort>& servers )
-            : _type( type ) , _servers( servers ){
-            _finishInit();
-        }
+        // TODO Delete if nobody is using
+        //ConnectionString( ConnectionType type , const vector<HostAndPort>& servers )
+        //    : _type( type ) , _servers( servers ){
+        //    _finishInit();
+        //}
         
         ConnectionString( ConnectionType type , const string& s , const string& setName = "" ){
             _type = type;
@@ -156,6 +157,14 @@ namespace mongo {
 
         static ConnectionString parse( const string& url , string& errmsg );
         
+        string getSetName() const{
+            return _setName;
+        }
+
+        vector<HostAndPort> getServers() const {
+            return _servers;
+        }
+        
     private:
 
         ConnectionString(){
@@ -314,7 +323,7 @@ namespace mongo {
 /** Typically one uses the QUERY(...) macro to construct a Query object.
     Example: QUERY( "age" << 33 << "school" << "UCLA" )
 */
-#define QUERY(x) Query( BSON(x) )
+#define QUERY(x) mongo::Query( BSON(x) )
 
     /**
        interface that handles communication with the db
@@ -329,6 +338,8 @@ namespace mongo {
 
         /* used by QueryOption_Exhaust.  To use that your subclass must implement this. */
         virtual void recv( Message& m ) { assert(false); }
+
+        virtual string getServerAddress() const = 0;
     };
 
     /**
@@ -739,8 +750,6 @@ namespace mongo {
          */
         virtual void update( const string &ns , Query query , BSONObj obj , bool upsert = false , bool multi = false );
         
-        virtual string getServerAddress() const = 0;
-        
         virtual bool isFailed() const = 0;
         
         virtual void killCursor( long long cursorID ) = 0;
@@ -758,6 +767,9 @@ namespace mongo {
         virtual void say( Message& toSend  ) = 0;
 
         virtual ConnectionString::ConnectionType type() const = 0;
+
+        /** @return true if conn is either equal to or contained in this connection */
+        virtual bool isMember( const DBConnector * conn ) const = 0;
     }; // DBClientBase
     
     class DBClientReplicaSet;
@@ -784,7 +796,7 @@ namespace mongo {
         void _checkConnection();
         void checkConnection() { if( failed ) _checkConnection(); }
 		map< string, pair<string,string> > authCache;
-        int _timeout;
+        double _timeout;
         
         bool _connect( string& errmsg );
     public:
@@ -795,7 +807,7 @@ namespace mongo {
            @param timeout tcp timeout in seconds - this is for read/write, not connect.  
            Connect timeout is fixed, but short, at 5 seconds.
          */
-        DBClientConnection(bool _autoReconnect=false, DBClientReplicaSet* cp=0, int timeout=0) :
+        DBClientConnection(bool _autoReconnect=false, DBClientReplicaSet* cp=0, double timeout=0) :
                 clientSet(cp), failed(false), autoReconnect(_autoReconnect), lastReconnectTry(0), _timeout(timeout) { }
 
         /** Connect to a Mongo database server.
@@ -892,11 +904,16 @@ namespace mongo {
         virtual bool call( Message &toSend, Message &response, bool assertOk = true );
         
         virtual ConnectionString::ConnectionType type() const { return ConnectionString::MASTER; }  
+
+        virtual bool isMember( const DBConnector * conn ) const { return this == conn; };
+
+        virtual void checkResponse( const char *data, int nReturned );
+
     protected:
         friend class SyncClusterConnection;
         virtual void recv( Message& m );
         virtual void sayPiggyBack( Message &toSend );
-        virtual void checkResponse( const char *data, int nReturned );
+
     };
     
     /** Use this class to connect to a replica set of servers.  The class will manage
@@ -987,9 +1004,12 @@ namespace mongo {
 
         virtual ConnectionString::ConnectionType type() const { return ConnectionString::SET; }  
 
+        virtual bool isMember( const DBConnector * conn ) const;
+
+        virtual void checkResponse( const char *data, int nReturned ) { checkMaster()->checkResponse( data , nReturned ); }
+
     protected:                
-        virtual void sayPiggyBack( Message &toSend ) { assert(false); }
-        virtual void checkResponse( const char *data, int nReturned ) { assert(false); }
+        virtual void sayPiggyBack( Message &toSend ) { checkMaster()->say( toSend ); }
         
         bool isFailed() const {
             return _currentMaster == 0 || _currentMaster->isFailed();

client/dbclientcursor.cpp

@@ -193,7 +193,7 @@ namespace mongo {
 
     void DBClientCursor::attach( AScopedConnection * conn ){
         assert( _scopedHost.size() == 0 );
-        assert( connector == conn->get() );
+        assert( conn->get()->isMember( connector ) );
         _scopedHost = conn->getHost();
         conn->done();
         connector = 0;

client/distlock.cpp

@@ -47,6 +47,8 @@ namespace mongo {
     }
     
     void distLockPingThread( ConnectionString addr ){
+        setThreadName( "LockPinger" );
+
         static int loops = 0;
         while( ! inShutdown() ){
             try {
@@ -109,9 +111,6 @@ namespace mongo {
 
        
     bool DistributedLock::lock_try( string why , BSONObj * other ){
-        // check for recrusive
-        assert( getState() == 0 );
-
         ScopedDbConnection conn( _conn );
             
         BSONObjBuilder queryBuilder;
@@ -136,7 +135,17 @@ namespace mongo {
                     return false;
                 }
 
-                Date_t elapsed = jsTime() - lastPing["ping"].Date(); // in ms
+                unsigned long long now = jsTime();
+                unsigned long long pingTime = lastPing["ping"].Date();
+                
+                if ( now < pingTime ) {
+                    // clock skew
+                    warning() << "dist_lock has detected clock skew of " << ( pingTime - now ) << "ms" << endl;
+                    conn.done();
+                    return false;
+                }
+
+                unsigned long long elapsed = now - pingTime;
                 elapsed = elapsed / ( 1000 * 60 ); // convert to minutes
 
                 if ( elapsed <= _takeoverMinutes ){
@@ -208,18 +217,33 @@ namespace mongo {
         if ( ! gotLock )
             return false;
             
-        _state.set( 1 );
         return true;
     }
 
     void DistributedLock::unlock(){
-        ScopedDbConnection conn( _conn );
-        conn->update( _ns , _id, BSON( "$set" << BSON( "state" << 0 ) ) );
-        log(1) << "dist_lock unlock: " << conn->findOne( _ns , _id ) << endl;
-        conn.done();
+        const int maxAttempts = 3;
+        int attempted = 0;
+        while ( ++attempted <= maxAttempts ) {
+
+            try {
+                ScopedDbConnection conn( _conn );
+                conn->update( _ns , _id, BSON( "$set" << BSON( "state" << 0 ) ) );
+                log(1) << "dist_lock unlock: " << conn->findOne( _ns , _id ) << endl;
+                conn.done();
+
+                return;
+
             
-        _state.set( 0 );
+            } catch ( std::exception& e) {
+                log( LL_WARNING ) << "dist_lock  " << _name << " failed to contact config server in unlock attempt " 
+                                  << attempted << ": " << e.what() <<  endl;
+
+                sleepsecs(1 << attempted);
+            }
+        }
+
+        log( LL_WARNING ) << "dist_lock couldn't consumate unlock request. " << "Lock " << _name 
+                              << " will be taken over after " <<  _takeoverMinutes << " minutes timeout" << endl;
     }
 
-
 }

client/distlock.h

@@ -36,14 +36,6 @@ namespace mongo {
          */
         DistributedLock( const ConnectionString& conn , const string& name , unsigned takeoverMinutes = 10 );
 
-        int getState(){
-            return _state.get();
-        }
-
-        bool isLocked(){
-            return _state.get() != 0;
-        }
-        
         bool lock_try( string why , BSONObj * other = 0 );
         void unlock();
 
@@ -54,8 +46,6 @@ namespace mongo {
         
         string _ns;
         BSONObj _id;
-        
-        ThreadLocalValue<int> _state;
     };
     
     class dist_lock_try {

client/gridfs.cpp

@@ -66,7 +66,6 @@ namespace mongo {
     }
 
     BSONObj GridFS::storeFile( const char* data , size_t length , const string& remoteName , const string& contentType){
-        massert( 10279 , "large files not yet implemented", length <= 0xffffffff);
         char const * const end = data + length;
 
         OID id;
@@ -127,8 +126,6 @@ namespace mongo {
         if (fd != stdin)
             fclose( fd );
         
-        massert( 10280 , "large files not yet implemented", length <= 0xffffffff);
-
         return insertFile((remoteName.empty() ? fileName : remoteName), id, length, contentType);
     }
 

client/parallel.cpp

@@ -328,6 +328,38 @@ namespace mongo {
     void ParallelSortClusteredCursor::_finishCons(){
         _numServers = _servers.size();
         _cursors = 0;
+
+        if ( ! _sortKey.isEmpty() && ! _fields.isEmpty() ){
+            // we need to make sure the sort key is in the project
+            bool isNegative = false;
+            BSONObjBuilder b;
+            {
+                BSONObjIterator i( _fields );
+                while ( i.more() ){
+                    BSONElement e = i.next();
+                    b.append( e );
+                    if ( ! e.trueValue() )
+                        isNegative = true;
+                }
+            }                    
+            
+            {
+                BSONObjIterator i( _sortKey );
+                while ( i.more() ){
+                    BSONElement e = i.next();
+                    BSONElement f = _fields.getField( e.fieldName() );
+                    if ( isNegative ){
+                        uassert( 13431 , "have to have sort key in projection and removing it" , f.eoo() );
+                    }
+                    else if ( f.eoo() ){
+                        // add to projection
+                        b.append( e );
+                    }
+                }
+            }
+            
+            _fields = b.obj();
+        }
     }
     
     void ParallelSortClusteredCursor::_init(){
@@ -420,37 +452,31 @@ namespace mongo {
     }
 
     bool Future::CommandResult::join(){
-        while ( ! _done )
-            sleepmicros( 50 );
+        _thr->join();
+        assert( _done );
         return _ok;
     }
 
-    void Future::commandThread(){
-        assert( _grab );
-        shared_ptr<CommandResult> res = *_grab;
-        _grab = 0;
-        
-        ScopedDbConnection conn( res->_server );
-        res->_ok = conn->runCommand( res->_db , res->_cmd , res->_res );
+    void Future::commandThread( shared_ptr<CommandResult> res ){
+        setThreadName( "future" );
+
+        try {
+            ScopedDbConnection conn( res->_server );
+            res->_ok = conn->runCommand( res->_db , res->_cmd , res->_res );
+            conn.done();
+        }
+        catch ( std::exception& e ){
+            error() << "Future::commandThread exception: " << e.what() << endl;
+            res->_ok = false;
+        }
         res->_done = true;
-        conn.done();
     }
 
     shared_ptr<Future::CommandResult> Future::spawnCommand( const string& server , const string& db , const BSONObj& cmd ){
-        shared_ptr<Future::CommandResult> res;
-        res.reset( new Future::CommandResult( server , db , cmd ) );
-        
-        _grab = &res;
-        
-        boost::thread thr( Future::commandThread );
-
-        while ( _grab )
-            sleepmicros(2);
-
+        shared_ptr<Future::CommandResult> res( new Future::CommandResult( server , db , cmd ) );
+        res->_thr.reset( new boost::thread( boost::bind( Future::commandThread , res ) ) );
         return res;
     }
-
-    shared_ptr<Future::CommandResult> * Future::_grab;
     
     
 }

client/parallel.h

@@ -274,7 +274,7 @@ namespace mongo {
             string _db;
             BSONObj _cmd;
 
-            boost::thread _thr;
+            scoped_ptr<boost::thread> _thr;
             
             BSONObj _res;
             bool _done;
@@ -283,12 +283,9 @@ namespace mongo {
             friend class Future;
         };
         
-        static void commandThread();
+        static void commandThread( shared_ptr<CommandResult> res );
         
         static shared_ptr<CommandResult> spawnCommand( const string& server , const string& db , const BSONObj& cmd );
-
-    private:
-        static shared_ptr<CommandResult> * _grab;
     };
 
     

client/syncclusterconnection.cpp

@@ -369,4 +369,16 @@ namespace mongo {
         // should never need to do this
         assert(0);
     }
+
+    bool SyncClusterConnection::isMember( const DBConnector * conn ) const {
+        if ( conn == this )
+            return true;
+        
+        for ( unsigned i=0; i<_conns.size(); i++ )
+            if ( _conns[i]->isMember( conn ) )
+                return true;
+        
+        return false;
+    }
+
 }

client/syncclusterconnection.h

@@ -90,6 +90,8 @@ namespace mongo {
 
         virtual ConnectionString::ConnectionType type() const { return ConnectionString::SYNC; }  
 
+        virtual bool isMember( const DBConnector * conn ) const;
+
     private:
         SyncClusterConnection( SyncClusterConnection& prev );
         string _toString() const;        

db/client.cpp

@@ -35,16 +35,18 @@
 
 namespace mongo {
 
+    Client* Client::syncThread;
     mongo::mutex Client::clientsMutex("clientsMutex");
     set<Client*> Client::clients; // always be in clientsMutex when manipulating this
     boost::thread_specific_ptr<Client> currentClient;
 
-    Client::Client(const char *desc) : 
+    Client::Client(const char *desc, MessagingPort *p) : 
       _context(0),
       _shutdown(false),
       _desc(desc),
       _god(0),
-      _lastOp(0)
+      _lastOp(0), 
+      _mp(p)
     {
         _curOp = new CurOp( this );
         scoped_lock bl(clientsMutex);
@@ -52,15 +54,21 @@ namespace mongo {
     }
 
     Client::~Client() { 
-        delete _curOp;
         _god = 0;
 
         if ( _context )
-            cout << "ERROR: Client::~Client _context should be NULL: " << _desc << endl;
-        if ( !_shutdown ) 
-            cout << "ERROR: Client::shutdown not called: " << _desc << endl;
-    }
+            error() << "Client::~Client _context should be null but is not; client:" << _desc << endl;
 
+        if ( ! _shutdown ) {
+            error() << "Client::shutdown not called: " << _desc << endl;
+        }
+        
+        scoped_lock bl(clientsMutex);
+        if ( ! _shutdown )
+            clients.erase(this);
+        delete _curOp;
+    }
+    
     void Client::_dropns( const string& ns ){
         Top::global.collectionDropped( ns );
                     
@@ -75,7 +83,7 @@ namespace mongo {
             dropCollection( ns , err , b );
         }
         catch ( ... ){
-            log() << "error dropping temp collection: " << ns << endl;
+            warning() << "error dropping temp collection: " << ns << endl;
         }
 
     }
@@ -196,12 +204,18 @@ namespace mongo {
         if ( doauth )
             _auth( lockState );
 
-        if ( _client->_curOp->getOp() != dbGetMore ){ // getMore's are special and should be handled else where
+        switch ( _client->_curOp->getOp() ){
+        case dbGetMore: // getMore's are special and should be handled else where
+        case dbUpdate: // update & delete check shard version in instance.cpp, so don't check here as well
+        case dbDelete: 
+            break;
+        default: {
             string errmsg;
-            if ( ! shardVersionOk( _ns , errmsg ) ){
+            if ( ! shardVersionOk( _ns , lockState > 0 , errmsg ) ){
                 msgasserted( StaleConfigInContextCode , (string)"[" + _ns + "] shard version not ok in Client::Context: " + errmsg );
             }
         }
+        }
     }
     
     void Client::Context::_auth( int lockState ){
@@ -237,7 +251,7 @@ namespace mongo {
 
     string sayClientState(){
         Client* c = currentClient.get();
-        if ( ! c )
+        if ( !c )
             return "no client";
         return c->toString();
     }
@@ -259,6 +273,38 @@ namespace mongo {
         }
     }
 
+    CurOp::~CurOp(){
+        if ( _wrapped ){
+            scoped_lock bl(Client::clientsMutex);
+            _client->_curOp = _wrapped;
+        }
+        
+        _client = 0;
+    }
+
+    BSONObj CurOp::query( bool threadSafe ) {
+        if( querySize() == 1 ) { 
+            return _tooBig;
+        }
+        
+        if ( ! threadSafe ){
+            BSONObj o(_queryBuf);
+            return o;
+        }
+
+        int size = querySize();        
+        int before = checksum( _queryBuf , size );
+        BSONObj a(_queryBuf);
+        BSONObj b = a.copy();
+        int after = checksum( _queryBuf , size );
+        
+        if ( before == after )
+            return b;
+        
+        return BSON( "msg" << "query changed while capturing" );
+    }
+
+
     BSONObj CurOp::infoNoauth( int attempt ) {
         BSONObjBuilder b;
         b.append("opid", _opNum);
@@ -402,7 +448,7 @@ namespace mongo {
                     tablecell( ss , co.getOp() );
                     tablecell( ss , co.getNS() );
                     if ( co.haveQuery() )
-                        tablecell( ss , co.query() );
+                        tablecell( ss , co.query( true ) );
                     else
                         tablecell( ss , "" );
                     tablecell( ss , co.getRemoteString() );

db/client.h

@@ -29,27 +29,33 @@
 #include "namespace.h"
 #include "lasterror.h"
 #include "stats/top.h"
-//#include "repl/rs.h"
 
 namespace mongo { 
 
     extern class ReplSet *theReplSet;
-
     class AuthenticationInfo;
     class Database;
     class CurOp;
     class Command;
     class Client;
+    class MessagingPort;
 
     extern boost::thread_specific_ptr<Client> currentClient;
 
     class Client : boost::noncopyable { 
     public:
+        static Client *syncThread;
+        void iAmSyncThread() { 
+            wassert( syncThread == 0 );
+            syncThread = this; 
+        }
+        bool isSyncThread() const { return this == syncThread; } // true if this client is the replication secondary pull thread
+
         static mongo::mutex clientsMutex;
         static set<Client*> clients; // always be in clientsMutex when manipulating this
-
         static int recommendedYieldMicros( int * writers = 0 , int * readers = 0 );
 
+        /* set _god=true temporarily, safely */
         class GodScope {
             bool _prev;
         public:
@@ -148,9 +154,11 @@ namespace mongo {
             }
 
             friend class CurOp;
-        };
+        }; // class Client::Context
         
     private:
+        void _dropns( const string& ns );
+
         CurOp * _curOp;
         Context * _context;
         bool _shutdown;
@@ -162,9 +170,9 @@ namespace mongo {
         BSONObj _handshake;
         BSONObj _remoteId;
 
-        void _dropns( const string& ns );
-
     public:
+        MessagingPort * const _mp;
+
         string clientAddress() const;
         AuthenticationInfo * getAuthenticationInfo(){ return &_ai; }
         bool isAdmin() { return _ai.isAuthorized( "admin" ); }
@@ -174,24 +182,19 @@ namespace mongo {
         const char *ns() const { return _context->ns(); }
         const char *desc() const { return _desc; }
         
-        Client(const char *desc);
+        Client(const char *desc, MessagingPort *p = 0);
         ~Client();
 
         void addTempCollection( const string& ns );
         
         void _invalidateDB(const string& db);
         static void invalidateDB(const string& db);
-
         static void invalidateNS( const string& ns );
 
-        void setLastOp( ReplTime op ) {
-            _lastOp = op;
-        }
-
-        ReplTime getLastOp() const {
-            return _lastOp;
-        }
+        void setLastOp( ReplTime op ) { _lastOp = op; }
+        ReplTime getLastOp() const { return _lastOp; }
 
+        /* report what the last operation was.  used by getlasterror */
         void appendLastOp( BSONObjBuilder& b ) {
             if( theReplSet ) { 
                 b.append("lastOp" , (long long) _lastOp);
@@ -206,14 +209,13 @@ namespace mongo {
         /* each thread which does db operations has a Client object in TLS.  
            call this when your thread starts. 
         */
-        static void initThread(const char *desc);
+        static Client& initThread(const char *desc, MessagingPort *mp = 0);
 
         /* 
            this has to be called as the client goes away, but before thread termination
            @return true if anything was done
          */
         bool shutdown();
-
         
         /* this is for map/reduce writes */
         bool isGod() const { return _god; }
@@ -221,13 +223,12 @@ namespace mongo {
         friend class CurOp;
 
         string toString() const;
-
         void gotHandshake( const BSONObj& o );
-
         BSONObj getRemoteID() const { return _remoteId; }
         BSONObj getHandshake() const { return _handshake; }
     };
     
+    /** get the Client object for this thread. */
     inline Client& cc() { 
         Client * c = currentClient.get();
         assert( c );
@@ -237,11 +238,13 @@ namespace mongo {
     /* each thread which does db operations has a Client object in TLS.  
        call this when your thread starts. 
     */
-    inline void Client::initThread(const char *desc) {
+    inline Client& Client::initThread(const char *desc, MessagingPort *mp) {
         setThreadName(desc);
         assert( currentClient.get() == 0 );
-        currentClient.reset( new Client(desc) );
+        Client *c = new Client(desc, mp);
+        currentClient.reset(c);
         mongo::lastError.initThread();
+        return *c;
     }
 
     inline Client::GodScope::GodScope(){
@@ -276,8 +279,5 @@ namespace mongo {
 
     string sayClientState();
   
-    inline bool haveClient(){ 
-        return currentClient.get() > 0;
-    }
+    inline bool haveClient() { return currentClient.get() > 0; }
 };
-

db/clientcursor.cpp

@@ -32,27 +32,37 @@
 
 namespace mongo {
 
-    CCById ClientCursor::clientCursorsById;
-    CCByLoc ClientCursor::byLoc;
-    boost::recursive_mutex ClientCursor::ccmutex;
+    typedef multimap<DiskLoc, ClientCursor*> CCByLoc;
 
-    unsigned ClientCursor::byLocSize() { 
+    CCById ClientCursor::clientCursorsById;
+    boost::recursive_mutex ClientCursor::ccmutex;
+    long long ClientCursor::numberTimedOut = 0;
+
+    /*static*/ void ClientCursor::assertNoCursors() { 
         recursive_scoped_lock lock(ccmutex);
-        return byLoc.size();
+        if( clientCursorsById.size() ) { 
+            log() << "ERROR clientcursors exist but should not at this point" << endl;
+            ClientCursor *cc = clientCursorsById.begin()->second;
+            log() << "first one: " << cc->cursorid << ' ' << cc->ns << endl;
+            clientCursorsById.clear();
+            assert(false);
+        }
     }
 
+
     void ClientCursor::setLastLoc_inlock(DiskLoc L) {
         if ( L == _lastLoc )
             return;
 
+        CCByLoc& bl = byLoc();
         if ( !_lastLoc.isNull() ) {
-            CCByLoc::iterator i = kv_find(byLoc, _lastLoc, this);
-            if ( i != byLoc.end() )
-                byLoc.erase(i);
+            CCByLoc::iterator i = kv_find(bl, _lastLoc, this);
+            if ( i != bl.end() )
+                bl.erase(i);
         }
 
         if ( !L.isNull() )
-            byLoc.insert( make_pair(L, this) );
+            bl.insert( make_pair(L, this) );
         _lastLoc = L;
     }
 
@@ -76,24 +86,52 @@ namespace mongo {
         {
             recursive_scoped_lock lock(ccmutex);
 
-            for ( CCByLoc::iterator i = byLoc.begin(); i != byLoc.end(); ++i ) {
+            Database *db = cc().database();
+            assert(db);
+            assert( str::startsWith(nsPrefix, db->name) );
+
+            for( CCById::iterator i = clientCursorsById.begin(); i != clientCursorsById.end(); ++i ) {
                 ClientCursor *cc = i->second;
-                if ( strncmp(nsPrefix, cc->ns.c_str(), len) == 0 )
+                if( cc->_db != db ) 
+                    continue;
+                if ( strncmp(nsPrefix, cc->ns.c_str(), len) == 0 ) {
                     toDelete.push_back(i->second);
+                }
             }
 
+            /*
+            note : we can't iterate byloc because clientcursors may exist with a loc of null in which case
+                   they are not in the map.  perhaps they should not exist though in the future?  something to 
+                   change???
+                   
+            CCByLoc& bl = db->ccByLoc;
+            for ( CCByLoc::iterator i = bl.begin(); i != bl.end(); ++i ) {
+                ClientCursor *cc = i->second;
+                if ( strncmp(nsPrefix, cc->ns.c_str(), len) == 0 ) {
+                    assert( cc->_db == db );
+                    toDelete.push_back(i->second);
+                }
+            }*/
+
             for ( vector<ClientCursor*>::iterator i = toDelete.begin(); i != toDelete.end(); ++i )
                 delete (*i);
         }
     }
 
+    bool ClientCursor::shouldTimeout( unsigned millis ){
+        _idleAgeMillis += millis;
+        return _idleAgeMillis > 600000 && _pinValue == 0;
+    }
+
     /* called every 4 seconds.  millis is amount of idle time passed since the last call -- could be zero */
     void ClientCursor::idleTimeReport(unsigned millis) {
+        readlock lk("");
         recursive_scoped_lock lock(ccmutex);
-        for ( CCByLoc::iterator i = byLoc.begin(); i != byLoc.end();  ) {
-            CCByLoc::iterator j = i;
+        for ( CCById::iterator i = clientCursorsById.begin(); i != clientCursorsById.end();  ) {
+            CCById::iterator j = i;
             i++;
             if( j->second->shouldTimeout( millis ) ){
+                numberTimedOut++;
                 log(1) << "killing old cursor " << j->second->cursorid << ' ' << j->second->ns 
                        << " idle:" << j->second->idleTime() << "ms\n";
                 delete j->second;
@@ -106,10 +144,12 @@ namespace mongo {
     */
     void ClientCursor::informAboutToDeleteBucket(const DiskLoc& b) {
         recursive_scoped_lock lock(ccmutex);
-        RARELY if ( byLoc.size() > 70 ) {
-            log() << "perf warning: byLoc.size=" << byLoc.size() << " in aboutToDeleteBucket\n";
+        Database *db = cc().database();
+        CCByLoc& bl = db->ccByLoc;
+        RARELY if ( bl.size() > 70 ) {
+            log() << "perf warning: byLoc.size=" << bl.size() << " in aboutToDeleteBucket\n";
         }
-        for ( CCByLoc::iterator i = byLoc.begin(); i != byLoc.end(); i++ )
+        for ( CCByLoc::iterator i = bl.begin(); i != bl.end(); i++ )
             i->second->c->aboutToDeleteBucket(b);
     }
     void aboutToDeleteBucket(const DiskLoc& b) {
@@ -120,8 +160,11 @@ namespace mongo {
     void ClientCursor::aboutToDelete(const DiskLoc& dl) {
         recursive_scoped_lock lock(ccmutex);
 
-        CCByLoc::iterator j = byLoc.lower_bound(dl);
-        CCByLoc::iterator stop = byLoc.upper_bound(dl);
+        Database *db = cc().database();
+        assert(db);
+        CCByLoc& bl = db->ccByLoc;
+        CCByLoc::iterator j = bl.lower_bound(dl);
+        CCByLoc::iterator stop = bl.upper_bound(dl);
         if ( j == stop )
             return;
 
@@ -139,6 +182,7 @@ namespace mongo {
         
         for ( vector<ClientCursor*>::iterator i = toAdvance.begin(); i != toAdvance.end(); ++i ){
             ClientCursor* cc = *i;
+            wassert(cc->_db == db);
             
             if ( cc->_doingDeletes ) continue;
 
@@ -157,7 +201,9 @@ namespace mongo {
             c->advance();
             if ( c->eof() ) {
                 // advanced to end
-                // leave ClieneCursor in place so next getMore doesn't fail
+                // leave ClientCursor in place so next getMore doesn't fail
+                // still need to mark new location though
+                cc->updateLocation();
             }
             else {
                 wassert( c->refLoc() != dl );
@@ -296,6 +342,13 @@ namespace mongo {
 
     int ctmLast = 0; // so we don't have to do find() which is a little slow very often.
     long long ClientCursor::allocCursorId_inlock() {
+        if( 0 ) { 
+            static long long z;
+            ++z;
+            cout << "TEMP alloccursorid " << z << endl;
+            return z;
+        }
+
         long long x;
         int ctm = (int) curTimeMillis();
         while ( 1 ) {
@@ -328,7 +381,13 @@ namespace mongo {
     }
 
 
-
+    void ClientCursor::appendStats( BSONObjBuilder& result ){
+        recursive_scoped_lock lock(ccmutex);
+        result.appendNumber("totalOpen", (int)clientCursorsById.size() );
+        result.appendNumber("clientCursors_size", (int) numCursors());
+        result.appendNumber("timedOut" , (int)numberTimedOut);
+    }
+    
     // QUESTION: Restrict to the namespace from which this command was issued?
     // Alternatively, make this command admin-only?
     class CmdCursorInfo : public Command {
@@ -339,11 +398,8 @@ namespace mongo {
             help << " example: { cursorInfo : 1 }";
         }
         virtual LockType locktype() const { return NONE; }
-        bool run(const string&, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
-            recursive_scoped_lock lock(ClientCursor::ccmutex);
-            result.append("totalOpen", unsigned( ClientCursor::clientCursorsById.size() ) );
-            result.append("byLocation_size", unsigned( ClientCursor::byLoc.size() ) );
-            result.append("clientCursors_size", unsigned( ClientCursor::clientCursorsById.size() ) );
+        bool run(const string& dbname, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
+            ClientCursor::appendStats( result );
             return true;
         }
     } cmdCursorInfo;

db/clientcursor.h

@@ -45,8 +45,6 @@ namespace mongo {
     */
     typedef map<CursorId, ClientCursor*> CCById;
 
-    typedef multimap<DiskLoc, ClientCursor*> CCByLoc;
-
     extern BSONObj id_obj;
 
     class ClientCursor {
@@ -64,14 +62,13 @@ namespace mongo {
         ElapsedTracker _yieldSometimesTracker;
 
         static CCById clientCursorsById;
-        static CCByLoc byLoc;
-        static boost::recursive_mutex ccmutex;   // must use this for all statics above!
-        
-        static CursorId allocCursorId_inlock();
-
-
+        static long long numberTimedOut;
+        static boost::recursive_mutex ccmutex;   // must use this for all statics above!        
+        static CursorId allocCursorId_inlock();        
         
     public:
+        static void assertNoCursors();
+
         /* use this to assure we don't in the background time out cursor while it is under use.
            if you are using noTimeout() already, there is no risk anyway.
            Further, this mechanism guards against two getMore requests on the same cursor executing
@@ -139,19 +136,24 @@ namespace mongo {
         };
 
         /*const*/ CursorId cursorid;
-        string ns;
-        shared_ptr<Cursor> c;
-        int pos;                  // # objects into the cursor so far 
-        BSONObj query;
-        int _queryOptions;        // see enum QueryOptions dbclient.h
+        const string ns;
+        const shared_ptr<Cursor> c;
+        int pos;                        // # objects into the cursor so far 
+        const BSONObj query;            // used for logging diags only; optional in constructor
+        const int _queryOptions;        // see enum QueryOptions dbclient.h
         OpTime _slaveReadTill;
+        Database * const _db;
 
-        ClientCursor(int queryOptions, shared_ptr<Cursor>& _c, const string& _ns) :
+        ClientCursor(int queryOptions, shared_ptr<Cursor>& _c, const string& _ns, BSONObj _query = BSONObj()) :
             _idleAgeMillis(0), _pinValue(0), 
             _doingDeletes(false), _yieldSometimesTracker(128,10),
             ns(_ns), c(_c), 
-            pos(0), _queryOptions(queryOptions)
+            pos(0), query(_query), 
+            _queryOptions(queryOptions), 
+            _db( cc().database() )
         {
+            assert( _db );
+            assert( str::startsWith(_ns, _db->name) );
             if( queryOptions & QueryOption_NoCursorTimeout )
                 noTimeout();
             recursive_scoped_lock lock(ccmutex);
@@ -308,10 +310,7 @@ namespace mongo {
         /**
          * @param millis amount of idle passed time since last call
          */
-        bool shouldTimeout( unsigned millis ){
-            _idleAgeMillis += millis;
-            return _idleAgeMillis > 600000 && _pinValue == 0;
-        }
+        bool shouldTimeout( unsigned millis );
 
         void storeOpForSlave( DiskLoc last );
         void updateSlaveLocation( CurOp& curop );
@@ -327,12 +326,18 @@ private:
         void noTimeout() { 
             _pinValue++;
         }
+
+        multimap<DiskLoc, ClientCursor*>& byLoc() { 
+            return _db->ccByLoc;
+        }
 public:
         void setDoingDeletes( bool doingDeletes ){
             _doingDeletes = doingDeletes;
         }
+        
+        static void appendStats( BSONObjBuilder& result );
 
-        static unsigned byLocSize();        // just for diagnostics
+        static unsigned numCursors() { return clientCursorsById.size(); }
 
         static void informAboutToDeleteBucket(const DiskLoc& b);
         static void aboutToDelete(const DiskLoc& dl);

db/cloner.cpp

@@ -49,7 +49,7 @@ namespace mongo {
         void setConnection( DBClientWithCommands *c ) { conn.reset( c ); }
         bool go(const char *masterHost, string& errmsg, const string& fromdb, bool logForRepl, bool slaveOk, bool useReplAuth, bool snapshot);
 
-        bool copyCollection( const string& from , const string& ns , const BSONObj& query , string& errmsg , bool copyIndexes = true );
+        bool copyCollection( const string& from , const string& ns , const BSONObj& query , string& errmsg , bool copyIndexes = true, bool logForRepl = true );
     };
 
     /* for index info object:
@@ -198,12 +198,12 @@ namespace mongo {
         }
     }
 
-    bool copyCollectionFromRemote(const string& host, const string& ns, const BSONObj& query, string errmsg) {
+    bool copyCollectionFromRemote(const string& host, const string& ns, const BSONObj& query, string& errmsg, bool logForRepl) {
         Cloner c;
-        return c.copyCollection(host, ns, query, errmsg , /*copyIndexes*/ true);
+        return c.copyCollection(host, ns, query, errmsg , /*copyIndexes*/ true, logForRepl);
     }
 
-    bool Cloner::copyCollection( const string& from , const string& ns , const BSONObj& query , string& errmsg , bool copyIndexes ){
+    bool Cloner::copyCollection( const string& from , const string& ns , const BSONObj& query , string& errmsg , bool copyIndexes, bool logForRepl ) {
         auto_ptr<DBClientConnection> myconn;
         myconn.reset( new DBClientConnection() );
         if ( ! myconn->connect( from , errmsg ) )
@@ -223,12 +223,17 @@ namespace mongo {
         }
         
         { // main data
-            copy( ns.c_str() , ns.c_str() , false , true , false , true , Query(query).snapshot() );
+            copy( ns.c_str() , ns.c_str() , /*isindex*/false , logForRepl , false , true , Query(query).snapshot() );
         }
         
+        /* TODO : copyIndexes bool does not seem to be implemented! */
+        if( !copyIndexes ) { 
+            log() << "ERROR copy collection copyIndexes not implemented? " << ns << endl;
+        }
+
         { // indexes
             string temp = ctx.db()->name + ".system.indexes";
-            copy( temp.c_str() , temp.c_str() , true , true , false , true , BSON( "ns" << ns ) );
+            copy( temp.c_str() , temp.c_str() , /*isindex*/true , logForRepl , false , true , BSON( "ns" << ns ) );
         }
         return true;
     }

db/cmdline.cpp

@@ -45,7 +45,7 @@ namespace mongo {
             ("bind_ip", po::value<string>(&cmdLine.bind_ip), "comma separated list of ip addresses to listen on - all local ips by default")
             ("logpath", po::value<string>() , "file to send all output to instead of stdout" )
             ("logappend" , "append to logpath instead of over-writing" )
-            ("pidfilepath", po::value<string>(), "directory for pidfile (if not set, no pidfile is created)")
+            ("pidfilepath", po::value<string>(), "full path to pidfile (if not set, no pidfile is created)")
 #ifndef _WIN32
             ("fork" , "fork server process" )
 #endif

db/commands.h

@@ -85,9 +85,7 @@ namespace mongo {
 
            Note if run() returns false, we do NOT log.
         */
-        virtual bool logTheOp() {
-            return false;
-        }
+        virtual bool logTheOp() { return false; }
 
         virtual void help( stringstream& help ) const;
 

db/curop.h

@@ -85,13 +85,7 @@ namespace mongo {
         int querySize() const { return *((int *) _queryBuf); }
         bool haveQuery() const { return querySize() != 0; }
 
-        BSONObj query() {
-            if( querySize() == 1 ) { 
-                return _tooBig;
-            }
-            BSONObj o(_queryBuf);
-            return o;
-        }
+        BSONObj query( bool threadSafe = false);
 
         void ensureStarted(){
             if ( _start == 0 )
@@ -228,10 +222,7 @@ namespace mongo {
             memset(_queryBuf, 0, sizeof(_queryBuf));
         }
         
-        ~CurOp(){
-            if ( _wrapped )
-                _client->_curOp = _wrapped;
-        }
+        ~CurOp();
 
         BSONObj info() { 
             if( ! cc().getAuthenticationInfo()->isAuthorized("admin") ) { 

db/database.h

@@ -22,6 +22,8 @@
 
 namespace mongo {
 
+    class ClientCursor;
+
     /**
      * Database represents a database database
      * Each database database has its own set of files -- dbname.ns, dbname.0, dbname.1, ...
@@ -196,6 +198,9 @@ namespace mongo {
         NamespaceIndex namespaceIndex;
         int profile; // 0=off.
         string profileName; // "alleyinsider.system.profile"
+
+        multimap<DiskLoc, ClientCursor*> ccByLoc;
+
         int magic; // used for making sure the object is still loaded in memory 
     };
 

db/db.cpp

@@ -67,7 +67,6 @@ namespace mongo {
 #endif
 
     void setupSignals();
-    void closeAllSockets();
     void startReplSets(ReplSetCmdline*);
     void startReplication();
     void pairWith(const char *remoteEnd, const char *arb);
@@ -104,7 +103,7 @@ namespace mongo {
         virtual void accepted(MessagingPort *mp) {
 
             if ( ! connTicketHolder.tryAcquire() ){
-                log() << "connection refused because too many open connections: " << connTicketHolder.used() << endl;
+                log() << "connection refused because too many open connections: " << connTicketHolder.used() << " of " << connTicketHolder.outof() << endl;
                 // TODO: would be nice if we notified them...
                 mp->shutdown();
                 delete mp;
@@ -207,16 +206,14 @@ namespace mongo {
     void connThread( MessagingPort * inPort )
     {
         TicketHolderReleaser connTicketReleaser( &connTicketHolder );
-        Client::initThread("conn");
 
         /* todo: move to Client object */
         LastError *le = new LastError();
         lastError.reset(le);
 
+        inPort->_logLevel = 1;
         auto_ptr<MessagingPort> dbMsgPort( inPort );
-
-        dbMsgPort->_logLevel = 1;
-        Client& c = cc();
+        Client& c = Client::initThread("conn", inPort);
 
         try {
 
@@ -522,7 +519,7 @@ sendmore:
             l << ( is32bit ? " 32" : " 64" ) << "-bit " << endl;
         }
         DEV log() << "_DEBUG build (which is slower)" << endl;
-        show_32_warning();
+        show_warnings();
         log() << mongodVersion() << endl;
         printGitVersion();
         printSysInfo();
@@ -629,7 +626,7 @@ using namespace mongo;
 namespace po = boost::program_options;
 
 void show_help_text(po::options_description options) {
-    show_32_warning();
+    show_warnings();
     cout << options << endl;
 };
 
@@ -1111,9 +1108,6 @@ int main(int argc, char* argv[], char *envp[] )
 
 namespace mongo {
 
-    /* we do not use log() below as it uses a mutex and that could cause deadlocks.
-    */
-
     string getDbContext();
 
 #undef out

db/db.vcxproj

@@ -574,14 +574,26 @@
     <ClCompile Include="repl\rs_config.cpp" />
   </ItemGroup>
   <ItemGroup>
-    <None Include="..\jstests\rs\rs_basic.js" />
-    <None Include="..\jstests\rs\test_framework.js" />
+    <None Include="..\jstests\replsets\replset1.js" />
+    <None Include="..\jstests\replsets\replset2.js" />
+    <None Include="..\jstests\replsets\replset3.js" />
+    <None Include="..\jstests\replsets\replset4.js" />
+    <None Include="..\jstests\replsets\replset5.js" />
+    <None Include="..\jstests\replsets\replsetadd.js" />
+    <None Include="..\jstests\replsets\replsetarb1.js" />
+    <None Include="..\jstests\replsets\replsetarb2.js" />
+    <None Include="..\jstests\replsets\replsetprio1.js" />
+    <None Include="..\jstests\replsets\replsetrestart1.js" />
+    <None Include="..\jstests\replsets\replsetrestart2.js" />
+    <None Include="..\jstests\replsets\replset_remove_node.js" />
+    <None Include="..\jstests\replsets\rollback.js" />
+    <None Include="..\jstests\replsets\rollback2.js" />
+    <None Include="..\jstests\replsets\sync1.js" />
+    <None Include="..\jstests\replsets\twosets.js" />
     <None Include="..\SConstruct" />
     <None Include="..\util\mongoutils\README" />
     <None Include="mongo.ico" />
     <None Include="repl\notes.txt" />
-    <None Include="repl\test.html" />
-    <None Include="repl\testing.js" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\client\dbclientcursor.h" />

db/db.vcxproj.filters

@@ -840,7 +840,7 @@
     <Filter Include="replSets">
       <UniqueIdentifier>{3b73f786-d352-446f-a5f5-df49384baf7a}</UniqueIdentifier>
     </Filter>
-    <Filter Include="replSets\test stuff">
+    <Filter Include="replSets\testing">
       <UniqueIdentifier>{4a1ea357-1077-4ad1-85b4-db48a6e1eb46}</UniqueIdentifier>
     </Filter>
   </ItemGroup>
@@ -851,24 +851,60 @@
     <None Include="..\util\mongoutils\README">
       <Filter>util\mongoutils</Filter>
     </None>
-    <None Include="repl\testing.js">
-      <Filter>replSets\test stuff</Filter>
-    </None>
-    <None Include="repl\test.html">
-      <Filter>replSets\test stuff</Filter>
-    </None>
     <None Include="..\SConstruct">
       <Filter>db</Filter>
     </None>
-    <None Include="..\jstests\rs\rs_basic.js">
-      <Filter>replSets\test stuff</Filter>
-    </None>
-    <None Include="..\jstests\rs\test_framework.js">
-      <Filter>replSets\test stuff</Filter>
-    </None>
     <None Include="mongo.ico">
       <Filter>Resource Files</Filter>
     </None>
+    <None Include="..\jstests\replsets\replset_remove_node.js">
+      <Filter>replSets\testing</Filter>
+    </None>
+    <None Include="..\jstests\replsets\replset1.js">
+      <Filter>replSets\testing</Filter>
+    </None>
+    <None Include="..\jstests\replsets\replset2.js">
+      <Filter>replSets\testing</Filter>
+    </None>
+    <None Include="..\jstests\replsets\replset3.js">
+      <Filter>replSets\testing</Filter>
+    </None>
+    <None Include="..\jstests\replsets\replset4.js">
+      <Filter>replSets\testing</Filter>
+    </None>
+    <None Include="..\jstests\replsets\replset5.js">
+      <Filter>replSets\testing</Filter>
+    </None>
+    <None Include="..\jstests\replsets\replsetadd.js">
+      <Filter>replSets\testing</Filter>
+    </None>
+    <None Include="..\jstests\replsets\replsetarb1.js">
+      <Filter>replSets\testing</Filter>
+    </None>
+    <None Include="..\jstests\replsets\replsetarb2.js">
+      <Filter>replSets\testing</Filter>
+    </None>
+    <None Include="..\jstests\replsets\replsetprio1.js">
+      <Filter>replSets\testing</Filter>
+    </None>
+    <None Include="..\jstests\replsets\replsetrestart1.js">
+      <Filter>replSets\testing</Filter>
+    </None>
+    <None Include="..\jstests\replsets\replsetrestart2.js">
+      <Filter>replSets\testing</Filter>
+    </None>
+    <None Include="..\jstests\replsets\rollback.js">
+      <Filter>replSets\testing</Filter>
+    </None>
+    <None Include="..\jstests\replsets\rollback2.js">
+      <Filter>replSets\testing</Filter>
+    </None>
+    <None Include="..\jstests\replsets\sync1.js">
+      <Filter>replSets\testing</Filter>
+    </None>
+    <None Include="..\jstests\replsets\twosets.js">
+      <Filter>replSets\testing</Filter>
+    </None>
   </ItemGroup>
   <ItemGroup>
     <Library Include="..\..\js\js64r.lib">

db/db_10.sln

@@ -12,7 +12,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{2B262D59
 		..\tools\bsondump.cpp = ..\tools\bsondump.cpp
 		..\tools\dump.cpp = ..\tools\dump.cpp
 		..\tools\export.cpp = ..\tools\export.cpp
-		..\tools\files.cpp = ..\tools\files.cpp
 		..\tools\import.cpp = ..\tools\import.cpp
 		..\tools\restore.cpp = ..\tools\restore.cpp
 		..\tools\sniffer.cpp = ..\tools\sniffer.cpp
@@ -29,9 +28,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "unix files", "unix files",
 	EndProjectSection
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "shell", "shell", "{407B4B88-3451-433C-B74F-31B31FEB5791}"
-	ProjectSection(SolutionItems) = preProject
-		..\shell\utils.h = ..\shell\utils.h
-	EndProjectSection
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "other", "other", "{12B11474-2D74-48C3-BB3D-F03249BEA88F}"
 EndProject

db/dbcommands.cpp

@@ -74,6 +74,14 @@ namespace mongo {
         }
     } cmdResetError;
 
+    /* set by replica sets if specified in the configuration. 
+       a pointer is used to avoid any possible locking issues with lockless reading (see below locktype() is NONE 
+       and would like to keep that)
+       (for now, it simply orphans any old copy as config changes should be extremely rare).
+       note: once non-null, never goes to null again.
+    */
+    BSONObj *getLastErrorDefault = 0;
+
     class CmdGetLastError : public Command {
     public:
         virtual LockType locktype() const { return NONE; } 
@@ -88,7 +96,7 @@ namespace mongo {
             help << "return error status of the last operation on this connection";
         }
         CmdGetLastError() : Command("getLastError", false, "getlasterror") {}
-        bool run(const string& dbnamne, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+        bool run(const string& dbnamne, BSONObj& _cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
             LastError *le = lastError.disableForCommand();
             if ( le->nPrev != 1 )
                 LastError::noError.appendSelf( result );
@@ -98,6 +106,18 @@ namespace mongo {
             Client& c = cc();
             c.appendLastOp( result );
 
+            BSONObj cmdObj = _cmdObj;
+            { 
+                BSONObj::iterator i(_cmdObj);
+                i.next();
+                if( !i.more() ) { 
+                    /* empty, use default */
+                    BSONObj *def = getLastErrorDefault;
+                    if( def )
+                        cmdObj = *def;
+                }
+            }
+
             if ( cmdObj["fsync"].trueValue() ){
                 log() << "fsync from getlasterror" << endl;
                 result.append( "fsyncFiles" , MemoryMappedFile::flushAll( true ) );
@@ -126,6 +146,7 @@ namespace mongo {
                     assert( sprintf( buf , "w block pass: %lld" , ++passes ) < 30 );
                     c.curop()->setMessage( buf );
                     sleepmillis(1);
+                    killCurrentOp.checkForInterrupt();
                 }
                 result.appendNumber( "wtime" , t.millis() );
             }
@@ -159,32 +180,6 @@ namespace mongo {
         }
     } cmdGetPrevError;
 
-    class CmdSwitchToClientErrors : public Command {
-    public:
-        virtual bool requiresAuth() { return false; }
-        virtual bool logTheOp() {
-            return false;
-        }
-        virtual void help( stringstream& help ) const {
-            help << "convert to id based errors rather than connection based";
-        }
-        virtual bool slaveOk() const {
-            return true;
-        }
-        virtual LockType locktype() const { return NONE; } 
-        CmdSwitchToClientErrors() : Command("switchToClientErrors", false, "switchtoclienterrors") {}
-        bool run(const string& dbnamne , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
-            if ( lastError.getID() ){
-                errmsg = "already in client id mode";
-                return false;
-            }
-            LastError *le = lastError.disableForCommand();
-            le->overridenById = true;
-            result << "ok" << 1;
-            return true;
-        }
-    } cmdSwitchToClientErrors;
-
     class CmdDropDatabase : public Command {
     public:
         virtual bool logTheOp() {
@@ -292,7 +287,6 @@ namespace mongo {
         }
 
         bool run(const string& dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
-            
             long long start = Listener::getElapsedTimeMillis();
             BSONObjBuilder timeBuilder(128);
 
@@ -345,6 +339,8 @@ namespace mongo {
                     t.appendBool( "supported" , false );
                 }
                     
+                timeBuilder.appendNumber( "middle of mem" , Listener::getElapsedTimeMillis() - start );
+
                 t.appendNumber( "mapped" , MemoryMappedFile::totalMappedLength() / ( 1024 * 1024 ) );
 
                 t.done();
@@ -381,7 +377,13 @@ namespace mongo {
                 globalFlushCounters.append( bb );
                 bb.done();
             }
-
+            
+            {
+                BSONObjBuilder bb( result.subobjStart( "cursors" ) );
+                ClientCursor::appendStats( bb );
+                bb.done();
+            }
+            
             timeBuilder.appendNumber( "after counters" , Listener::getElapsedTimeMillis() - start );            
 
             if ( anyReplEnabled() ){
@@ -409,8 +411,11 @@ namespace mongo {
             if ( ! authed )
                 result.append( "note" , "run against admin for more info" );
             
-            if ( Listener::getElapsedTimeMillis() - start > 1000 )
-                result.append( "timing" , timeBuilder.obj() );
+            if ( Listener::getElapsedTimeMillis() - start > 1000 ){
+                BSONObj t = timeBuilder.obj();
+                log() << "serverStatus was very slow: " << t << endl;
+                result.append( "timing" , t );
+            }
 
             return true;
         }
@@ -648,10 +653,10 @@ namespace mongo {
         virtual void help( stringstream& help ) const {
             help << "create a collection";
         }
-        virtual bool run(const string& dbname , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool) {
+        virtual bool run(const string& dbname , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl ) {
             string ns = dbname + '.' + cmdObj.firstElement().valuestr();
             string err;
-            bool ok = userCreateNS(ns.c_str(), cmdObj, err, true);
+            bool ok = userCreateNS(ns.c_str(), cmdObj, err, ! fromRepl );
             if ( !ok && !err.empty() )
                 errmsg = err;
             return ok;
@@ -710,12 +715,8 @@ namespace mongo {
 
     class CmdReIndex : public Command {
     public:
-        virtual bool logTheOp() {
-            return true;
-        }
-        virtual bool slaveOk() const {
-            return false;
-        }
+        virtual bool logTheOp() { return false; } // only reindexes on the one node
+        virtual bool slaveOk() const { return true; }    // can reindex on a secondary
         virtual LockType locktype() const { return WRITE; } 
         virtual void help( stringstream& help ) const {
             help << "re-index a collection";
@@ -765,9 +766,6 @@ namespace mongo {
 
     class CmdListDatabases : public Command {
     public:
-        virtual bool logTheOp() {
-            return false;
-        }
         virtual bool slaveOk() const {
             return true;
         }
@@ -945,15 +943,34 @@ namespace mongo {
                 "\nnote: This command may take a while to run";
         }
         bool run(const string& dbname, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
+            Timer timer;
+
             string ns = jsobj.firstElement().String();
             BSONObj min = jsobj.getObjectField( "min" );
             BSONObj max = jsobj.getObjectField( "max" );
             BSONObj keyPattern = jsobj.getObjectField( "keyPattern" );
+            bool estimate = jsobj["estimate"].trueValue();
 
             Client::Context ctx( ns );
+            NamespaceDetails *d = nsdetails(ns.c_str());
             
+            if ( ! d || d->nrecords == 0 ){
+                result.appendNumber( "size" , 0 );
+                result.appendNumber( "numObjects" , 0 );
+                result.append( "millis" , timer.millis() );
+                return true;
+            }
+            
+            result.appendBool( "estimate" , estimate );
+
             shared_ptr<Cursor> c;
             if ( min.isEmpty() && max.isEmpty() ) {
+                if ( estimate ){
+                    result.appendNumber( "size" , d->datasize );
+                    result.appendNumber( "numObjects" , d->nrecords );
+                    result.append( "millis" , timer.millis() );
+                    return 1;
+                }
                 c = theDataFileMgr.findAll( ns.c_str() );
             } 
             else if ( min.isEmpty() || max.isEmpty() ) {
@@ -964,18 +981,24 @@ namespace mongo {
                 IndexDetails *idx = cmdIndexDetailsForRange( ns.c_str(), errmsg, min, max, keyPattern );
                 if ( idx == 0 )
                     return false;
-                NamespaceDetails *d = nsdetails(ns.c_str());
+                
                 c.reset( new BtreeCursor( d, d->idxNo(*idx), *idx, min, max, false, 1 ) );
             }
             
+            long long avgObjSize = d->datasize / d->nrecords;
+
             long long maxSize = jsobj["maxSize"].numberLong();
             long long maxObjects = jsobj["maxObjects"].numberLong();
 
-            Timer timer;
             long long size = 0;
             long long numObjects = 0;
             while( c->ok() ) {
-                size += c->currLoc().rec()->netLength();
+
+                if ( estimate )
+                    size += avgObjSize;
+                else
+                    size += c->currLoc().rec()->netLength();
+                
                 numObjects++;
                 
                 if ( ( maxSize && size > maxSize ) || 
@@ -994,8 +1017,8 @@ namespace mongo {
             }
             logIfSlow( timer , os.str() );
 
-            result.append( "size", (double)size );
-            result.append( "numObjects" , (double)numObjects );
+            result.appendNumber( "size", size );
+            result.appendNumber( "numObjects" , numObjects );
             result.append( "millis" , timer.millis() );
             return true;
         }
@@ -1450,17 +1473,18 @@ namespace mongo {
             
             BSONElementSet values;
             shared_ptr<Cursor> cursor = bestGuessCursor(ns.c_str() , query , BSONObj() );
+            scoped_ptr<ClientCursor> cc (new ClientCursor(QueryOption_NoCursorTimeout, cursor, ns));
 
             while ( cursor->ok() ){
-                if ( cursor->matcher() && ! cursor->matcher()->matchesCurrent( cursor.get() ) ){
-                    cursor->advance();
-                    continue;
+                if ( !cursor->matcher() || cursor->matcher()->matchesCurrent( cursor.get() ) ){
+                    BSONObj o = cursor->current();
+                    o.getFieldsDotted( key, values );
                 }
 
-                BSONObj o = cursor->current();
                 cursor->advance();
-                
-                o.getFieldsDotted( key.c_str(), values );
+
+                if (!cc->yieldSometimes())
+                    break;
             }
 
             BSONArrayBuilder b( result.subarrayStart( "values" ) );
@@ -1574,9 +1598,6 @@ namespace mongo {
     class CmdWhatsMyUri : public Command {
     public:
         CmdWhatsMyUri() : Command("whatsmyuri") { }
-        virtual bool logTheOp() {
-            return false; // the modification will be logged directly
-        }
         virtual bool slaveOk() const {
             return true;
         }
@@ -1712,12 +1733,8 @@ namespace mongo {
     public:
         virtual LockType locktype() const { return NONE; } 
         virtual bool adminOnly() const { return true; }
-        virtual bool logTheOp() {
-            return false;
-        }
-        virtual bool slaveOk() const {
-            return true;
-        }
+        virtual bool logTheOp() { return false; }
+        virtual bool slaveOk() const { return true; }
         virtual void help( stringstream& help ) const {
             help << "internal testing command.  Makes db block (in a read lock) for 100 seconds\n";
             help << "w:true write lock";
@@ -1817,7 +1834,7 @@ namespace mongo {
         
 
         if ( c->adminOnly() && ! fromRepl && dbname != "admin" ) {
-            result.append( "errmsg" ,  "access denied- use admin db" );
+            result.append( "errmsg" ,  "access denied; use admin db" );
             log() << "command denied: " << cmdObj.toString() << endl;
             return false;
         }        

db/dbcommands_generic.cpp

@@ -60,6 +60,13 @@ namespace mongo {
         bool run(const string& dbname, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
             result << "version" << versionString << "gitVersion" << gitVersion() << "sysInfo" << sysInfo();
             result << "bits" << ( sizeof( int* ) == 4 ? 32 : 64 );
+            result.appendBool( "debug" , 
+#ifdef _DEBUG
+                               true
+#else
+                               false
+#endif
+                               );
             return true;
         }
     } cmdBuildInfo;
@@ -189,8 +196,9 @@ namespace mongo {
         CmdShutdown() : Command("shutdown") {}
         bool run(const string& dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
             Client * c = currentClient.get();
-            if ( c )
+            if ( c ) {
                 c->shutdown();
+            }
             log() << "terminating, shutdown command received" << endl;
             dbexit( EXIT_CLEAN ); // this never returns
             return true;

db/dbhelpers.cpp

@@ -301,6 +301,11 @@ namespace mongo {
 
             c->checkLocation();
 
+            if ( yield && ! cc->yieldSometimes() ){
+                // cursor got finished by someone else, so we're done
+                cc.release(); // if the collection/db is dropped, cc may be deleted
+                break;
+            }
         }
 
         return num;

db/dbwebserver.cpp

@@ -432,7 +432,7 @@ namespace mongo {
             ss << p( a("/", "back", "Home") );
             ss << p( "<b>MongoDB List of <a href=\"http://www.mongodb.org/display/DOCS/Commands\">Commands</a></b>\n" );
             const map<string, Command*> *m = Command::commandsByBestName();
-            ss << "S:slave-only  N:no-lock  R:read-lock  W:write-lock  A:admin-only<br>\n";
+            ss << "S:slave-ok  R:read-lock  W:write-lock  A:admin-only<br>\n";
             ss << table();
             ss << "<tr><th>Command</th><th>Attributes</th><th>Help</th></tr>\n";
             for( map<string, Command*>::const_iterator i = m->begin(); i != m->end(); i++ ) 

db/geo/2d.cpp

@@ -1257,7 +1257,7 @@ namespace mongo {
 
             _want._min = Point( _g , _bl );
             _want._max = Point( _g , _tr );
-            
+
             uassert( 13064 , "need an area > 0 " , _want.area() > 0 );
 
             _state = START;
@@ -1268,12 +1268,14 @@ namespace mongo {
             
             GEODEBUG( "center : " << center.toString() << "\t" << _prefix );
 
-	    {
-	      GeoHash a(0LL,32);
-	      GeoHash b(0LL,32);
-	      b.move(1,1);
-	      _fudge = _g->distance(a,b);
-	    }
+            {
+                GeoHash a(0LL,32);
+                GeoHash b(0LL,32);
+                b.move(1,1);
+                _fudge = _g->distance(a,b);
+            }
+
+            _wantLen = _fudge + std::max((_want._max._x - _want._min._x), (_want._max._y - _want._min._y));
 
             ok();
         }
@@ -1308,32 +1310,47 @@ namespace mongo {
                         _state = DONE;
                         return;
                     }
-                    
-                    Box cur( _g , _prefix );
-                    if ( cur._min._x + _fudge < _want._min._x &&
-                         cur._min._y + _fudge < _want._min._y &&
-                         cur._max._x - _fudge > _want._max._x &&
-                         cur._max._y - _fudge > _want._max._y ){
-                        
-                        _state = DONE;
-                        GeoHash temp = _prefix.commonPrefix( cur._max.hash( _g ) );
 
-                        GEODEBUG( "box done : " << cur.toString() << " prefix:" << _prefix << " common:" << temp );
-                        
-                        if ( temp == _prefix )
-                            return;
-                        _prefix = temp;
-                        GEODEBUG( "\t one more loop" );
-                        continue;
-                    }
-                    else {
+                    if (_g->sizeEdge(_prefix) < _wantLen){
                         _prefix = _prefix.up();
+                    } else {
+                        for (int i=-1; i<=1; i++){
+                            for (int j=-1; j<=1; j++){
+
+                                if (i == 0 && j == 0)
+                                    continue; // main box
+
+                                GeoHash newBox = _prefix;
+                                newBox.move(i, j);
+
+                                PREFIXDEBUG(newBox, _g);
+
+                                Box cur( _g , newBox );
+                                if (_want.intersects(cur)){
+                                    // TODO consider splitting into quadrants
+                                    getPointsForPrefix(newBox);
+                                } else  {
+                                    GEODEBUG("skipping box");
+                                }
+                            }
+                        }
+                        _state = DONE;
                     }
+                    
                 }
                 return;
             }
 
         }
+
+        void getPointsForPrefix(const GeoHash& prefix){
+            if ( ! BtreeLocation::initial( *_id , _spec , _min , _max , prefix , _found , this ) ){
+                return;
+            }
+
+            while ( _min.hasPrefix( prefix ) && _min.advance( -1 , _found , this ) );
+            while ( _max.hasPrefix( prefix ) && _max.advance( 1 , _found , this ) );
+        }
         
         virtual bool checkDistance( const GeoHash& h , double& d ){
             bool res = _want.inside( Point( _g , h ) , _fudge );
@@ -1346,6 +1363,7 @@ namespace mongo {
         GeoHash _bl;
         GeoHash _tr;
         Box _want;
+        double _wantLen;
 
         int _found;
         

db/instance.cpp

@@ -62,7 +62,6 @@ namespace mongo {
     bool useCursors = true;
     bool useHints = true;
     
-    void closeAllSockets();
     void flushOpLog( stringstream &ss ) {
         if( _diaglog.f && _diaglog.f->is_open() ) {
             ss << "flushing op log and files\n";
@@ -332,9 +331,10 @@ namespace mongo {
                     }
                 }
                 catch ( AssertionException& e ) {
-                    tlog() << " Caught Assertion in " << opToString(op) << " , continuing" << endl;
+                    static int n;
+                    tlog(3) << " Caught Assertion in " << opToString(op) << ", continuing" << endl;
                     ss << " exception " + e.toString();
-                    log = true;
+                    log = ++n < 10;
                 }
             }
         }
@@ -446,6 +446,7 @@ namespace mongo {
 
         mongolock lk(1);
 
+        // if this ever moves to outside of lock, need to adjust check Client::Context::_finishInit
         if ( ! broadcast && handlePossibleShardedMessage( m , 0 ) )
             return;
 
@@ -472,8 +473,10 @@ namespace mongo {
         }        
 
         writelock lk(ns);
+        // if this ever moves to outside of lock, need to adjust check Client::Context::_finishInit
         if ( ! broadcast & handlePossibleShardedMessage( m , 0 ) )
             return;
+        
         Client::Context ctx(ns);
         
         long long n = deleteObjects(ns, pattern, justOne, true);
@@ -496,6 +499,7 @@ namespace mongo {
         if( ntoreturn ) 
             ss << " ntoreturn:" << ntoreturn;
 
+		time_t start = 0;
         int pass = 0;        
         bool exhaust = false;
         QueryResult* msgdata;
@@ -508,6 +512,17 @@ namespace mongo {
             catch ( GetMoreWaitException& ) { 
                 exhaust = false;
                 massert(13073, "shutting down", !inShutdown() );
+				if( pass == 0 ) { 
+  				    start = time(0);
+				}
+				else { 
+				  if( time(0) - start >= 4 ) {
+					// after about 4 seconds, return.  this is a sanity check.  pass stops at 1000 normally 
+					// for DEV this helps and also if sleep is highly inaccurate on a platform.  we want to 
+					// return occasionally so slave can checkpoint.
+					pass = 10000;
+				  }
+				}
                 pass++;
                 DEV 
                     sleepmillis(20);
@@ -709,32 +724,32 @@ namespace mongo {
         }
         catch (...) { }
         
-        tryToOutputFatal( "dbexit: really exiting now\n" );
+        tryToOutputFatal( "dbexit: really exiting now" );
         if ( c ) c->shutdown();
         ::exit(rc);
     }
     
     void shutdown() {
 
-        log() << "\t shutdown: going to close listening sockets..." << endl;        
+        log() << "shutdown: going to close listening sockets..." << endl;        
         ListeningSockets::get()->closeAll();
 
-        log() << "\t shutdown: going to flush oplog..." << endl;
+        log() << "shutdown: going to flush oplog..." << endl;
         stringstream ss2;
         flushOpLog( ss2 );
         rawOut( ss2.str() );
 
         /* must do this before unmapping mem or you may get a seg fault */
-        log() << "\t shutdown: going to close sockets..." << endl;
-        boost::thread close_socket_thread(closeAllSockets);
+        log() << "shutdown: going to close sockets..." << endl;
+        boost::thread close_socket_thread( boost::bind(MessagingPort::closeAllSockets, 0) );
 
         // wait until file preallocation finishes
         // we would only hang here if the file_allocator code generates a
         // synchronous signal, which we don't expect
-        log() << "\t shutdown: waiting for fs preallocator..." << endl;
+        log() << "shutdown: waiting for fs preallocator..." << endl;
         theFileAllocator().waitUntilFinished();
         
-        log() << "\t shutdown: closing all files..." << endl;
+        log() << "shutdown: closing all files..." << endl;
         stringstream ss3;
         MemoryMappedFile::closeAllFiles( ss3 );
         rawOut( ss3.str() );
@@ -744,9 +759,9 @@ namespace mongo {
         
 #if !defined(_WIN32) && !defined(__sunos__)
         if ( lockFile ){
-            log() << "\t shutdown: removing fs lock..." << endl;
+            log() << "shutdown: removing fs lock..." << endl;
             if( ftruncate( lockFile , 0 ) ) 
-                log() << "\t couldn't remove fs lock " << errnoWithDescription() << endl;
+                log() << "couldn't remove fs lock " << errnoWithDescription() << endl;
             flock( lockFile, LOCK_UN );
         }
 #endif
@@ -766,12 +781,14 @@ namespace mongo {
 
         bool oldFile = false;
 
-        if ( boost::filesystem::exists( name ) && boost::filesystem::file_size( name ) > 0 ){
+        if ( boost::filesystem::exists( name ) && boost::filesystem::file_size( name ) > 0 ) {
             oldFile = true;
         }
 
         lockFile = open( name.c_str(), O_RDWR | O_CREAT , S_IRWXU | S_IRWXG | S_IRWXO );
-        uassert( 10309 ,  "Unable to create / open lock file for lockfilepath: " + name, lockFile > 0 );
+		if( lockFile <= 0 ) {
+		    uasserted( 10309 , str::stream() << "Unable to create / open lock file for lockfilepath: " << name << ' ' << errnoWithDescription());
+        }
         if (flock( lockFile, LOCK_EX | LOCK_NB ) != 0) {
             close ( lockFile );
             lockFile = 0;

db/instance.h

@@ -143,6 +143,7 @@ namespace mongo {
         }
 
         virtual ConnectionString::ConnectionType type() const { return ConnectionString::MASTER; }  
+        virtual bool isMember( const DBConnector * conn ) const { return this == conn; };
     };
 
     extern int lockFile;

db/lasterror.cpp

@@ -34,8 +34,11 @@ namespace mongo {
     void raiseError(int code , const char *msg) {
         LastError *le = lastError.get();
         if ( le == 0 ) {
-            /* might be intentional (non-user thread) */            
-            OCCASIONALLY DEV if( !isShell ) log() << "warning dev: lastError==0 won't report:" << msg << endl;
+            /* might be intentional (non-user thread) */    
+            DEV {
+                static unsigned n;
+                if( ++n < 4 && !isShell ) log() << "dev: lastError==0 won't report:" << msg << endl;
+            }
         } else if ( le->disabled ) {
             log() << "lastError disabled, can't report: " << code << ":" << msg << endl;
         } else {
@@ -168,9 +171,7 @@ namespace mongo {
     }
     
     LastError * LastErrorHolder::startRequest( Message& m , int clientId ) {
-
-        if ( clientId == 0 )
-            clientId = m.header()->id & 0xFFFF0000;
+        assert( clientId );
         setID( clientId );
 
         LastError * le = _get( true );
@@ -179,11 +180,7 @@ namespace mongo {
     }
 
     void LastErrorHolder::startRequest( Message& m , LastError * connectionOwned ) {
-        if ( !connectionOwned->overridenById ) {
-            prepareErrForNewRequest( m, connectionOwned );
-            return;
-        }
-        startRequest(m);
+        prepareErrForNewRequest( m, connectionOwned );
     }
 
     void LastErrorHolder::disconnect( int clientId ){

db/lasterror.h

@@ -32,7 +32,6 @@ namespace mongo {
         long long nObjects;
         int nPrev;
         bool valid;
-        bool overridenById;
         bool disabled;
         void writeback( OID& oid ){
             reset( true );
@@ -56,7 +55,6 @@ namespace mongo {
             nObjects = nDeleted;
         }
         LastError() {
-            overridenById = false;
             reset();
         }
         void reset( bool _valid = false ) {
@@ -127,7 +125,7 @@ namespace mongo {
         
         /** when db receives a message/request, call this */
         void startRequest( Message& m , LastError * connectionOwned );
-        LastError * startRequest( Message& m , int clientId = 0 );
+        LastError * startRequest( Message& m , int clientId );
         
         void disconnect( int clientId );
 

db/matcher.cpp

@@ -259,7 +259,7 @@ namespace mongo {
     }
     
     void Matcher::parseOr( const BSONElement &e, bool subMatcher, list< shared_ptr< Matcher > > &matchers ) {
-        uassert( 13090, "recursive $or/$nor not allowed", !subMatcher );
+        uassert( 13090, "nested $or/$nor not allowed", !subMatcher );
         uassert( 13086, "$or/$nor must be a nonempty array", e.type() == Array && e.embeddedObject().nFields() > 0 );
         BSONObjIterator j( e.embeddedObject() );
         while( j.more() ) {

db/mr.cpp

@@ -176,11 +176,11 @@ namespace mongo {
                 { // query options
                     if ( cmdObj["query"].type() == Object ){
                         filter = cmdObj["query"].embeddedObjectUserCheck();
-                        q = filter;
                     }
                     
-                    if ( cmdObj["sort"].type() == Object )
-                        q.sort( cmdObj["sort"].embeddedObjectUserCheck() );
+                    if ( cmdObj["sort"].type() == Object ){
+                        sort = cmdObj["sort"].embeddedObjectUserCheck();
+                    }
 
                     if ( cmdObj["limit"].isNumber() )
                         limit = cmdObj["limit"].numberLong();
@@ -203,10 +203,8 @@ namespace mongo {
             long long renameIfNeeded( DBDirectClient& db ){
                 if ( finalLong != tempLong ){
                     db.dropCollection( finalLong );
-                    if ( db.count( tempLong ) ){
-                        BSONObj info;
-                        uassert( 10076 ,  "rename failed" , db.runCommand( "admin" , BSON( "renameCollection" << tempLong << "to" << finalLong ) , info ) );
-                    }
+                    BSONObj info;
+                    uassert( 10076 ,  "rename failed" , db.runCommand( "admin" , BSON( "renameCollection" << tempLong << "to" << finalLong ) , info ) );
                 }
                 return db.count( finalLong );
             }
@@ -222,7 +220,7 @@ namespace mongo {
             // query options
             
             BSONObj filter;
-            Query q;
+            BSONObj sort;
             long long limit;
 
             // functions
@@ -444,7 +442,7 @@ namespace mongo {
                         readlock lock( mr.ns );
                         Client::Context ctx( mr.ns );
                         
-                        shared_ptr<Cursor> temp = bestGuessCursor( mr.ns.c_str(), mr.filter, BSONObj() );
+                        shared_ptr<Cursor> temp = bestGuessCursor( mr.ns.c_str(), mr.filter, mr.sort );
                         auto_ptr<ClientCursor> cursor( new ClientCursor( QueryOption_NoCursorTimeout , temp , mr.ns.c_str() ) );
 
                         Timer mt;

db/namespace.cpp

@@ -564,8 +564,10 @@ namespace mongo {
     }
 
     void renameNamespace( const char *from, const char *to ) {
-		NamespaceIndex *ni = nsindex( from );
-		assert( ni && ni->details( from ) && !ni->details( to ) );
+        NamespaceIndex *ni = nsindex( from );
+		assert( ni );
+        assert( ni->details( from ) );
+        assert( ! ni->details( to ) );
 		
 		// Our namespace and index details will move to a different 
 		// memory location.  The only references to namespace and 
@@ -637,6 +639,8 @@ namespace mongo {
 	}
 
     bool legalClientSystemNS( const string& ns , bool write ){
+        if( ns == "local.system.replset" ) return true;
+
         if ( ns.find( ".system.users" ) != string::npos )
             return true;
 

db/oplog.cpp

@@ -142,8 +142,10 @@ namespace mongo {
                      this code (or code in now() maybe) should be improved.
                      */
             if( theReplSet ) {
-                if( !(theReplSet->lastOpTimeWritten<ts) ) 
-                    log() << "replSet ERROR possible failover clock skew issue? " << theReplSet->lastOpTimeWritten << ' ' << ts << endl;
+                if( !(theReplSet->lastOpTimeWritten<ts) ) {
+                    log() << "replSet ERROR possible failover clock skew issue? " << theReplSet->lastOpTimeWritten << ' ' << ts << rsLog;
+                    log() << "replSet " << theReplSet->isPrimary() << rsLog;
+                }
                 theReplSet->lastOpTimeWritten = ts;
                 theReplSet->lastH = hNew;
                 ctx.getClient()->setLastOp( ts.asDate() );

db/oplog.h

@@ -195,7 +195,7 @@ namespace mongo {
             // Use a ClientCursor here so we can release db mutex while scanning
             // oplog (can take quite a while with large oplogs).
             shared_ptr<Cursor> c = _qp.newReverseCursor();
-            _findingStartCursor = new ClientCursor(QueryOption_NoCursorTimeout, c, _qp.ns());
+            _findingStartCursor = new ClientCursor(QueryOption_NoCursorTimeout, c, _qp.ns(), BSONObj());
             _findingStartTimer.reset();
             _findingStartMode = Initial;
             BSONElement tsElt = _qp.originalQuery()[ "ts" ];

db/pdfile.cpp

@@ -56,8 +56,6 @@ namespace mongo {
         }
     };
 
-    const int MaxExtentSize = 0x7ff00000;
-
     map<string, unsigned> BackgroundOperation::dbsInProg;
     set<string> BackgroundOperation::nsInProg;
 
@@ -157,7 +155,7 @@ namespace mongo {
             sz = 1000000000;
         int z = ((int)sz) & 0xffffff00;
         assert( z > len );
-        DEV tlog() << "initialExtentSize(" << len << ") returns " << z << endl;
+        //DEV tlog() << "initialExtentSize(" << len << ") returns " << z << endl;
         return z;
     }
 
@@ -272,10 +270,19 @@ namespace mongo {
     /*---------------------------------------------------------------------*/
 
     int MongoDataFile::maxSize() {
-        if ( sizeof( int* ) == 4 )
+        if ( sizeof( int* ) == 4 ) {
             return 512 * 1024 * 1024;
-        else
+        } else if ( cmdLine.smallfiles ) {
+            return 0x7ff00000 >> 2;
+        } else {
             return 0x7ff00000;
+        }
+    }
+
+    void MongoDataFile::badOfs(int ofs) const { 
+        stringstream ss;
+        ss << "bad offset:" << ofs << " accessing file: " << mmf.filename() << " - consider repairing database";
+        uasserted(13440, ss.str());
     }
 
     int MongoDataFile::defaultSize( const char *filename ) const {
@@ -380,7 +387,7 @@ namespace mongo {
 
     Extent* MongoDataFile::createExtent(const char *ns, int approxSize, bool newCapped, int loops) {
         massert( 10357 ,  "shutdown in progress", !goingAway );
-        massert( 10358 ,  "bad new extent size", approxSize >= 0 && approxSize <= MaxExtentSize );
+        massert( 10358 ,  "bad new extent size", approxSize >= 0 && approxSize <= Extent::maxSize() );
         massert( 10359 ,  "header==0 on new extent: 32 bit mmap space exceeded?", header ); // null if file open failed
         int ExtentSize = approxSize <= header->unusedLength ? approxSize : header->unusedLength;
         DiskLoc loc;
@@ -403,8 +410,8 @@ namespace mongo {
 
         addNewExtentToNamespace(ns, e, loc, emptyLoc, newCapped);
 
-        DEV tlog() << "new extent " << ns << " size: 0x" << hex << ExtentSize << " loc: 0x" << hex << offset
-                   << " emptyLoc:" << hex << emptyLoc.getOfs() << dec << endl;
+        DEV tlog(1) << "new extent " << ns << " size: 0x" << hex << ExtentSize << " loc: 0x" << hex << offset
+                    << " emptyLoc:" << hex << emptyLoc.getOfs() << dec << endl;
         return e;
     }
 
@@ -568,6 +575,14 @@ namespace mongo {
       }
     */
 
+    int Extent::maxSize() {
+        int maxExtentSize = 0x7ff00000;
+        if ( cmdLine.smallfiles ) {
+            maxExtentSize >>= 2;
+        }
+        return maxExtentSize;
+    }
+    
     /*---------------------------------------------------------------------*/
 
     shared_ptr<Cursor> DataFileMgr::findAll(const char *ns, const DiskLoc &startLoc) {
@@ -728,8 +743,11 @@ namespace mongo {
             try { 
                 assert( dropIndexes(d, name.c_str(), "*", errmsg, result, true) );
             }
-            catch( DBException& ) {
-                uasserted(12503,"drop: dropIndexes for collection failed - consider trying repair");
+            catch( DBException& e ) {
+                stringstream ss;
+                ss << "drop: dropIndexes for collection failed - consider trying repair ";
+                ss << " cause: " << e.what();
+                uasserted(12503,ss.str());
             }
             assert( d->nIndexes == 0 );
         }
@@ -894,7 +912,7 @@ namespace mongo {
 
         if ( toupdate->netLength() < objNew.objsize() ) {
             // doesn't fit.  reallocate -----------------------------------------------------
-            uassert( 10003 , "E10003 failing update: objects in a capped ns cannot grow", !(d && d->capped));
+            uassert( 10003 , "failing update: objects in a capped ns cannot grow", !(d && d->capped));
             d->paddingTooSmall();
             if ( cc().database()->profile )
                 ss << " moved ";
@@ -947,15 +965,19 @@ namespace mongo {
     }
 
     int followupExtentSize(int len, int lastExtentLen) {
-        assert( len < MaxExtentSize );
+        assert( len < Extent::maxSize() );
         int x = initialExtentSize(len);
         int y = (int) (lastExtentLen < 4000000 ? lastExtentLen * 4.0 : lastExtentLen * 1.2);
         int sz = y > x ? y : x;
 
-        if ( sz < lastExtentLen )
-            sz = lastExtentLen;
-        else if ( sz > MaxExtentSize )
-            sz = MaxExtentSize;
+        if ( sz < lastExtentLen ){
+            // this means there was an int overflow
+            // so we should turn it into maxSize
+            sz = Extent::maxSize();
+        }
+        else if ( sz > Extent::maxSize() ){
+            sz = Extent::maxSize();
+        }
         
         sz = ((int)sz) & 0xffffff00;
         assert( sz > len );
@@ -1026,7 +1048,7 @@ namespace mongo {
 
         Timer t;
 
-        tlog() << "Buildindex " << ns << " idxNo:" << idxNo << ' ' << idx.info.obj().toString() << endl;
+        tlog(1) << "fastBuildIndex " << ns << " idxNo:" << idxNo << ' ' << idx.info.obj().toString() << endl;
 
         bool dupsAllowed = !idx.unique();
         bool dropDups = idx.dropDups() || inDBRepair;
@@ -1511,6 +1533,13 @@ namespace mongo {
 
             BSONObj info = loc.obj();
             bool background = info["background"].trueValue();
+            if( background && cc().isSyncThread() ) { 
+                /* don't do background indexing on slaves.  there are nuances.  this could be added later 
+                   but requires more code.
+                   */
+                log() << "info: indexing in foreground on this replica; was a background index build on the primary" << endl;
+                background = false;
+            }
 
             int idxNo = tableToIndex->nIndexes;
             IndexDetails& idx = tableToIndex->addIndex(tabletoidxns.c_str(), !background); // clear transient info caches so they refresh; increments nIndexes
@@ -1892,7 +1921,10 @@ namespace mongo {
         bb.done();
         if( nNotClosed )
             result.append("nNotClosed", nNotClosed);
-        
+        else {
+            ClientCursor::assertNoCursors();
+        }
+
         return true;
     }
     

db/pdfile.h

@@ -79,6 +79,8 @@ namespace mongo {
         void flush( bool sync );
         
     private:
+        void badOfs(int) const;
+
         int defaultSize( const char *filename ) const;
 
         Extent* getExtent(DiskLoc loc);
@@ -255,6 +257,8 @@ namespace mongo {
         Extent* getPrevExtent() {
             return xprev.isNull() ? 0 : DataFileMgr::getExtent(xprev);
         }
+        
+        static int maxSize();
     };
 
     /*
@@ -339,7 +343,7 @@ namespace mongo {
 
     inline Record* MongoDataFile::recordAt(DiskLoc dl) {
         int ofs = dl.getOfs();
-        assert( ofs >= DataFileHeader::HeaderSize );
+        if( ofs < DataFileHeader::HeaderSize ) badOfs(ofs); // will uassert - external call to keep out of the normal code path
         return (Record*) _p.at(ofs, -1);
     }
 

db/query.cpp

@@ -320,7 +320,6 @@ namespace mongo {
 
             while ( 1 ) {
                 if ( !c->ok() ) {
-//                    log() << "TEMP Tailable : " << c->tailable() << ' ' << (queryOptions & QueryOption_AwaitData) << endl;
                     if ( c->tailable() ) {
                         /* when a tailable cursor hits "EOF", ok() goes false, and current() is null.  however 
                            advance() can still be retries as a reactivation attempt.  when there is new data, it will 
@@ -654,11 +653,12 @@ namespace mongo {
                 if ( !ClientCursor::recoverFromYield( _yieldData ) ) {
                     _c.reset();
                     _cc.reset();
+                    _so.reset();
                     massert( 13338, "cursor dropped during query", false );
                     // TODO maybe we want to prevent recording the winning plan as well?
                 } 
             }
-        }        
+        }
         
         virtual void next() {
             if ( _findingStartCursor.get() ) {
@@ -772,7 +772,8 @@ namespace mongo {
                 _n = _inMemSort ? _so->size() : _n;
             } 
             else if ( _inMemSort ) {
-                _so->fill( _buf, _pq.getFields() , _n );
+                if( _so.get() )
+                    _so->fill( _buf, _pq.getFields() , _n );
             }
             
             if ( _pq.hasOption( QueryOption_CursorTailable ) && _pq.getNumToReturn() != 1 )
@@ -785,8 +786,10 @@ namespace mongo {
             if ( _pq.isExplain()) {
                 _eb.noteScan( _c.get(), _nscanned, _nscannedObjects, _n, scanAndOrderRequired(), _curop.elapsedMillis(), useHints && !_pq.getHint().eoo() );
             } else {
-                _response.appendData( _buf.buf(), _buf.len() );
-                _buf.decouple();
+                if (_buf.len()) {
+                    _response.appendData( _buf.buf(), _buf.len() );
+                    _buf.decouple();
+                }
             }
             if ( stop ) {
                 setStop();
@@ -1042,14 +1045,13 @@ namespace mongo {
             if ( moreClauses ) {
                 // this MultiCursor will use a dumb NoOp to advance(), so no need to specify mayYield
                 shared_ptr< Cursor > multi( new MultiCursor( mps, cursor, dqo.matcher(), dqo ) );
-                cc = new ClientCursor(queryOptions, multi, ns);
+                cc = new ClientCursor(queryOptions, multi, ns, jsobj.getOwned());
             } else {
                 cursor->setMatcher( dqo.matcher() );
-                cc = new ClientCursor( queryOptions, cursor, ns );
+                cc = new ClientCursor( queryOptions, cursor, ns, jsobj.getOwned() );
             }
             cursorid = cc->cursorid;
-            cc->query = jsobj.getOwned();
-            DEV tlog() << "query has more, cursorid: " << cursorid << endl;
+            DEV tlog(2) << "query has more, cursorid: " << cursorid << endl;
             cc->pos = n;
             cc->pq = pq_shared;
             cc->fields = pq.getFieldPtr();

db/queryoptimizer.cpp

@@ -361,7 +361,7 @@ namespace mongo {
                 const IndexSpec& spec = ii.getSpec();
                 if ( spec.getTypeName() == _special && spec.suitability( _originalQuery , order_ ) ){
                     usingPrerecordedPlan_ = true;
-                    mayRecordPlan_ = true;
+                    mayRecordPlan_ = false;
                     plans_.push_back( PlanPtr( new QueryPlan( d , j , *fbs_ , _originalQuery, order_ , 
                                                               BSONObj() , BSONObj() , _special ) ) );
                     return;

db/queryutil.cpp

@@ -944,6 +944,8 @@ namespace mongo {
         return ( l % 2 == 0 ); // if we're inside an interval        
     }
     
+    // binary search for interval containing the specified element
+    // an even return value indicates that the element is contained within a valid interval
     int FieldRangeVector::matchingLowElement( const BSONElement &e, int i, bool forward ) const {
         int l = -1;
         int h = _ranges[ i ].intervals().size() * 2;
@@ -980,23 +982,16 @@ namespace mongo {
             BSONElement kk = k.next();
             int number = (int) kk.number();
             bool forward = ( number >= 0 ? 1 : -1 ) * ( _direction >= 0 ? 1 : -1 ) > 0;
-            BSONElement e = obj.getField( kk.fieldName() );
-            if ( e.eoo() ) {
-                e = staticNull.firstElement();
+            BSONElementSet keys;
+            obj.getFieldsDotted( kk.fieldName(), keys );
+            bool match = false;
+            for( BSONElementSet::const_iterator j = keys.begin(); j != keys.end(); ++j ) {
+                if ( matchesElement( *j, i, forward ) ) {
+                    match = true;
+                    break;
+                }
             }
-            if ( e.type() == Array ) {
-                BSONObjIterator j( e.embeddedObject() );
-                bool match = false;
-                while( j.more() ) {
-                    if ( matchesElement( j.next(), i, forward ) ) {
-                        match = true;
-                        break;
-                    }
-                }
-                if ( !match ) {
-                    return false;
-                }
-            } else if ( !matchesElement( e, i, forward ) ) {
+            if ( !match ) {
                 return false;
             }
         }
@@ -1007,9 +1002,13 @@ namespace mongo {
     int FieldRangeVector::Iterator::advance( const BSONObj &curr ) {
         BSONObjIterator j( curr );
         BSONObjIterator o( _v._keyPattern );
+        // track first field for which we are not at the end of the valid values,
+        // since we may need to advance from the key prefix ending with this field
         int latestNonEndpoint = -1;
+        // iterate over fields to determine appropriate advance method
         for( int i = 0; i < (int)_i.size(); ++i ) {
             if ( i > 0 && !_v._ranges[ i - 1 ].intervals()[ _i[ i - 1 ] ].equality() ) {
+                // if last bound was inequality, we don't know anything about where we are for this field
                 // TODO if possible avoid this certain cases when field in prev key is the same
                 setMinus( i );
             }
@@ -1017,9 +1016,9 @@ namespace mongo {
             BSONElement oo = o.next();
             bool reverse = ( ( oo.number() < 0 ) ^ ( _v._direction < 0 ) );
             BSONElement jj = j.next();
-            if ( _i[ i ] == -1 ) {
+            if ( _i[ i ] == -1 ) { // unknown position for this field, do binary search
                 int l = _v.matchingLowElement( jj, i, !reverse );
-                if ( l % 2 == 0 ) {
+                if ( l % 2 == 0 ) { // we are in a valid range for this field
                     _i[ i ] = l / 2;
                     int diff = (int)_v._ranges[ i ].intervals().size() - _i[ i ];
                     if ( diff > 1 ) {
@@ -1031,7 +1030,8 @@ namespace mongo {
                         }
                     }
                     continue;
-                } else {
+                } else { // not in a valid range for this field - determine if and how to advance
+                    // check if we're after the last interval for this field
                     if ( l == (int)_v._ranges[ i ].intervals().size() * 2 - 1 ) {
                         if ( latestNonEndpoint == -1 ) {
                             return -2;
@@ -1053,7 +1053,11 @@ namespace mongo {
                 }
             }
             bool first = true;
+            // _i[ i ] != -1, so we have a starting interval for this field
+            // which serves as a lower/equal bound on the first iteration -
+            // we advance from this interval to find a matching interval
             while( _i[ i ] < (int)_v._ranges[ i ].intervals().size() ) {
+                // compare to current interval's upper bound
                 int x = _v._ranges[ i ].intervals()[ _i[ i ] ]._upper._bound.woCompare( jj, false );
                 if ( reverse ) {
                     x = -x;
@@ -1062,16 +1066,22 @@ namespace mongo {
                     eq = true;
                     break;
                 }
+                // see if we're less than the upper bound
                 if ( x > 0 ) {
                     if ( i == 0 && first ) {
-                        break; // the value of 1st field won't go backward
+                        // the value of 1st field won't go backward, so don't check lower bound
+                        // TODO maybe we can check first only?
+                        break;
                     }
+                    // if it's an equality interval, don't need to compare separately to lower bound
                     if ( !_v._ranges[ i ].intervals()[ _i[ i ] ].equality() ) {
+                        // compare to current interval's lower bound
                         x = _v._ranges[ i ].intervals()[ _i[ i ] ]._lower._bound.woCompare( jj, false );
                         if ( reverse ) {
                             x = -x;
                         }
                     }
+                    // if we're less than the lower bound, advance
                     if ( x > 0 ) {
                         setZero( i + 1 );
                         // skip to curr / i / nextbounds
@@ -1084,17 +1094,20 @@ namespace mongo {
                         break;
                     }
                 }
+                // we're above the upper bound, so try next interval and reset remaining fields
                 ++_i[ i ];
                 setZero( i + 1 );
                 first = false;
             }
             int diff = (int)_v._ranges[ i ].intervals().size() - _i[ i ];
             if ( diff > 1 || ( !eq && diff == 1 ) ) {
+                 // check if we're not at the end of valid values for this field
                 latestNonEndpoint = i;
-            } else if ( diff == 0 ) {
+            } else if ( diff == 0 ) { // check if we're past the last interval for this field
                 if ( latestNonEndpoint == -1 ) {
                     return -2;
                 }
+                // more values possible, skip...
                 setZero( latestNonEndpoint + 1 );
                 // skip to curr / latestNonEndpoint + 1 / superlative
                 for( int j = latestNonEndpoint + 1; j < (int)_i.size(); ++j ) {

db/repl.cpp

@@ -137,9 +137,6 @@ namespace mongo {
         virtual bool adminOnly() const {
             return true;
         }
-        virtual bool logTheOp() {
-            return false;
-        }
         virtual LockType locktype() const { return WRITE; }
         void help(stringstream&h) const { h << "replace a node in a replica pair"; }
         CmdReplacePeer() : Command("replacePeer", false, "replacepeer") { }
@@ -199,9 +196,6 @@ namespace mongo {
         virtual bool adminOnly() const {
             return true;
         }
-        virtual bool logTheOp() {
-            return false;   
-        }
         virtual void help(stringstream& h) const { h << "internal"; }
         virtual LockType locktype() const { return WRITE; }
         CmdForceDead() : Command("forcedead") { }
@@ -222,9 +216,7 @@ namespace mongo {
         virtual bool adminOnly() const {
             return true;
         }
-        virtual bool logTheOp() {
-            return false;
-        }
+        virtual bool logTheOp() { return false; }
         virtual LockType locktype() const { return WRITE; }
         void help(stringstream&h) const { h << "resync (from scratch) an out of date replica slave.\nhttp://www.mongodb.org/display/DOCS/Master+Slave"; }
         CmdResync() : Command("resync") { }
@@ -290,11 +282,13 @@ namespace mongo {
             }
         }
         else {
-            result.append("ismaster", replSettings.master || replSettings.slave == 0 ? 1 : 0);
-            //result.append("msg", "not paired");
+            result.appendBool("ismaster", _isMaster() );
         }
-        
-        if ( level ){
+
+        if ( level && replSet ){
+            result.append( "info" , "is replica set" );
+        }
+        else if ( level ){
             BSONObjBuilder sources( result.subarrayStart( "sources" ) );
             
             readlock lk( "local.sources" );
@@ -664,6 +658,8 @@ namespace mongo {
                 ReplSource tmp(c->current());
                 if ( tmp.hostName != cmdLine.source ) {
                     log() << "repl: --source " << cmdLine.source << " != " << tmp.hostName << " from local.sources collection" << endl;
+                    log() << "repl: for instructions on changing this slave's source, see:" << endl;
+                    log() << "http://dochub.mongodb.org/core/masterslave" << endl;
                     log() << "repl: terminating mongod after 30 seconds" << endl;
                     sleepsecs(30);
                     dbexit( EXIT_REPLICATION_ERROR );
@@ -856,11 +852,12 @@ namespace mongo {
        see logOp() comments.
     */
     void ReplSource::sync_pullOpLog_applyOperation(BSONObj& op, OpTime *localLogTail) {
-        log( 6 ) << "processing op: " << op << endl;
-        // skip no-op
-        if ( op.getStringField( "op" )[ 0 ] == 'n' )
+        if( logLevel >= 6 ) // op.tostring is expensive so doing this check explicitly
+            log(6) << "processing op: " << op << endl;
+
+        if( op.getStringField("op")[0] == 'n' )
             return;
-        
+
         char clientName[MaxDatabaseLen];
         const char *ns = op.getStringField("ns");
         nsToDatabase(ns, clientName);
@@ -1256,8 +1253,14 @@ namespace mongo {
             if ( tailing || initial ) {
                 if ( initial )
                     log(1) << "repl:   initial run\n";
-                else
-                    assert( syncedTo < nextOpTime );
+                else {
+                    if( !( syncedTo <= nextOpTime ) ) { 
+                        log() << "repl ASSERTION failed : syncedTo <= nextOpTime" << endl;
+                        log() << "repl syncTo:     " << syncedTo.toStringLong() << endl;
+                        log() << "repl nextOpTime: " << nextOpTime.toStringLong() << endl;
+                        assert(false);
+                    }
+                }
                 oplogReader.putBack( op ); // op will be processed in the loop below
                 nextOpTime = OpTime(); // will reread the op below
             }
@@ -1301,12 +1304,15 @@ namespace mongo {
                    1) find most recent op in local log
                    2) more()?
                 */
-                if ( !oplogReader.more() ) {
+
+                bool moreInitialSyncsPending = !addDbNextPass.empty() && n; // we need "&& n" to assure we actually process at least one op to get a sync point recorded in the first place.
+
+                if ( moreInitialSyncsPending || !oplogReader.more() ) {
                     dblock lk;
                     OpTime nextLastSaved = nextLastSavedLocalTs();
                     {
                         dbtemprelease t;
-                        if ( oplogReader.more() ) {
+                        if ( !moreInitialSyncsPending && oplogReader.more() ) {
                             if ( getInitialSyncCompleted() ) { // if initial sync hasn't completed, break out of loop so we can set to completed or clone more dbs
                                 continue;
                             }
@@ -1323,6 +1329,8 @@ namespace mongo {
                     log() << "repl:  end sync_pullOpLog syncedTo: " << syncedTo.toStringLong() << endl;
                     break;
                 }
+                else {
+                }
 
                 OCCASIONALLY if( n > 0 && ( n > 100000 || time(0) - saveLast > 60 ) ) { 
 					// periodically note our progress, in case we are doing a lot of work and crash
@@ -1681,6 +1689,7 @@ namespace mongo {
     void replSlaveThread() {
         sleepsecs(1);
         Client::initThread("replslave");
+        cc().iAmSyncThread();
             
         {
             dblock lk;

db/repl/connections.h

@@ -44,19 +44,36 @@ namespace mongo {
         /** throws assertions if connect failure etc. */
         ScopedConn(string hostport);
         ~ScopedConn();
-        DBClientConnection* operator->();
+
+        /* If we were to run a query and not exhaust the cursor, future use of the connection would be problematic.
+           So here what we do is wrapper known safe methods and not allow cursor-style queries at all.  This makes 
+           ScopedConn limited in functionality but very safe.  More non-cursor wrappers can be added here if needed.
+           */
+
+        bool runCommand(const string &dbname, const BSONObj& cmd, BSONObj &info, int options=0) {
+            return conn()->runCommand(dbname, cmd, info, options);
+        }
+        unsigned long long count(const string &ns) { 
+            return conn()->count(ns); 
+        }
+        BSONObj findOne(const string &ns, const Query& q, const BSONObj *fieldsToReturn = 0, int queryOptions = 0) { 
+            return conn()->findOne(ns, q, fieldsToReturn, queryOptions);
+        }
+
     private:
         auto_ptr<scoped_lock> connLock;
         static mutex mapMutex;
         struct X { 
             mutex z;
             DBClientConnection cc;
-            X() : z("X"), cc(/*reconnect*/true, 0, /*timeout*/10) { 
+            X() : z("X"), cc(/*reconnect*/ true, 0, 
+                             /*timeout*/ theReplSet ? theReplSet->config().ho.heartbeatTimeoutMillis/1000.0 : 10.0) { 
                 cc._logLevel = 2;
             }
         } *x;
         typedef map<string,ScopedConn::X*> M;
         static M& _map;
+        DBClientConnection* conn() { return &x->cc; }
     };
 
     inline ScopedConn::ScopedConn(string hostport) {
@@ -84,8 +101,8 @@ namespace mongo {
         // conLock releases...
     }
 
-    inline DBClientConnection* ScopedConn::operator->() { 
+    /*inline DBClientConnection* ScopedConn::operator->() { 
         return &x->cc; 
-    }
+    }*/
 
 }

db/repl/consensus.cpp

@@ -83,6 +83,17 @@ namespace mongo {
         return vUp * 2 > totalVotes();
     }
 
+    bool Consensus::shouldRelinquish() const {
+        int vUp = rs._self->config().votes;
+        const long long T = rs.config().ho.heartbeatTimeoutMillis * rs.config().ho.heartbeatConnRetries;
+        for( Member *m = rs.head(); m; m=m->next() ) {
+            long long dt = m->hbinfo().timeDown();
+            if( dt < T )
+                vUp += m->config().votes;
+        }
+        return !( vUp * 2 > totalVotes() );
+    }
+
     static const int VETO = -10000;
 
     const time_t LeaseTime = 30;
@@ -123,6 +134,9 @@ namespace mongo {
         OID round = cmd["round"].OID();
         int myver = rs.config().version;
 
+        const Member* primary = rs.box.getPrimary();
+        const Member* hopeful = rs.findById(whoid);
+
         int vote = 0;
         if( set != rs.name() ) { 
             log() << "replSet error received an elect request for '" << set << "' but our set name is '" << rs.name() << "'" << rsLog;
@@ -136,6 +150,16 @@ namespace mongo {
             log() << "replSet info got stale version # during election" << rsLog;
             vote = -10000;
         }
+        else if( !hopeful ) {
+            log() << "couldn't find member with id " << whoid << rsLog;
+            vote = -10000;
+        }
+        else if( primary && primary->hbinfo().opTime > hopeful->hbinfo().opTime ) {
+            // other members might be aware of more up-to-date nodes
+            log() << hopeful->fullName() << " is trying to elect itself but " <<
+                primary->fullName() << " is already primary and more up-to-date" << rsLog;
+            vote = -10000;
+        }
         else {
             try {
                 vote = yea(whoid);
@@ -154,7 +178,7 @@ namespace mongo {
     void ReplSetImpl::_getTargets(list<Target>& L, int& configVersion) {
         configVersion = config().version;
         for( Member *m = head(); m; m=m->next() )
-            if( m->hbinfo().up() )
+            if( m->hbinfo().maybeUp() )
                 L.push_back( Target(m->fullName()) );
     }
 
@@ -322,6 +346,7 @@ namespace mongo {
     void Consensus::electSelf() {
         assert( !rs.lockedByMe() );
         assert( !rs.myConfig().arbiterOnly );
+        assert( rs.myConfig().slaveDelay == 0 );
         try { 
             _electSelf(); 
         } 

db/repl/health.cpp

@@ -19,6 +19,7 @@
 #include "health.h"
 #include "../../util/background.h"
 #include "../../client/dbclient.h"
+#include "../../client/connpool.h"
 #include "../commands.h"
 #include "../../util/concurrency/value.h"
 #include "../../util/concurrency/task.h"
@@ -89,11 +90,13 @@ namespace mongo {
         }
         s << td(config().votes);
         { 
-            string stateText = ReplSet::stateAsStr(state());
+            string stateText = state().toString();
+            if( _config.hidden )
+                stateText += " (hidden)";
             if( ok || stateText.empty() ) 
                 s << td(stateText); // text blank if we've never connected
             else
-                s << td( grey(str::stream() << "(was " << ReplSet::stateAsStr(state()) << ')', true) );
+                s << td( grey(str::stream() << "(was " << state().toString() << ')', true) );
         }
         s << td( grey(hbinfo().lastHeartbeatMsg,!ok) );
         stringstream q;
@@ -105,28 +108,30 @@ namespace mongo {
             s << td("");
         s << _tr();
     }
-    
+   
     string ReplSetImpl::stateAsHtml(MemberState s) { 
         if( s.s == MemberState::RS_STARTUP ) return a("", "serving still starting up, or still trying to initiate the set", "STARTUP");
         if( s.s == MemberState::RS_PRIMARY ) return a("", "this server thinks it is primary", "PRIMARY");
         if( s.s == MemberState::RS_SECONDARY ) return a("", "this server thinks it is a secondary (slave mode)", "SECONDARY");
         if( s.s == MemberState::RS_RECOVERING ) return a("", "recovering/resyncing; after recovery usually auto-transitions to secondary", "RECOVERING");
-        if( s.s == MemberState::RS_FATAL ) return a("", "something bad has occurred and server is not completely offline with regard to the replica set.  fatal error.", "RS_FATAL");
-        if( s.s == MemberState::RS_STARTUP2 ) return a("", "loaded config, still determining who is primary", "RS_STARTUP2");
+        if( s.s == MemberState::RS_FATAL ) return a("", "something bad has occurred and server is not completely offline with regard to the replica set.  fatal error.", "FATAL");
+        if( s.s == MemberState::RS_STARTUP2 ) return a("", "loaded config, still determining who is primary", "STARTUP2");
         if( s.s == MemberState::RS_ARBITER ) return a("", "this server is an arbiter only", "ARBITER");
         if( s.s == MemberState::RS_DOWN ) return a("", "member is down, slow, or unreachable", "DOWN");
+        if( s.s == MemberState::RS_ROLLBACK ) return a("", "rolling back operations to get in sync", "ROLLBACK");
         return "";
     }
 
-    string ReplSetImpl::stateAsStr(MemberState s) { 
-        if( s.s == MemberState::RS_STARTUP ) return "STARTUP";
-        if( s.s == MemberState::RS_PRIMARY ) return "PRIMARY";
-        if( s.s == MemberState::RS_SECONDARY ) return "SECONDARY";
-        if( s.s == MemberState::RS_RECOVERING ) return "RECOVERING";
-        if( s.s == MemberState::RS_FATAL ) return "FATAL";
-        if( s.s == MemberState::RS_STARTUP2 ) return "STARTUP2";
-        if( s.s == MemberState::RS_ARBITER ) return "ARBITER";
-        if( s.s == MemberState::RS_DOWN ) return "DOWN";
+    string MemberState::toString() const { 
+        if( s == MemberState::RS_STARTUP ) return "STARTUP";
+        if( s == MemberState::RS_PRIMARY ) return "PRIMARY";
+        if( s == MemberState::RS_SECONDARY ) return "SECONDARY";
+        if( s == MemberState::RS_RECOVERING ) return "RECOVERING";
+        if( s == MemberState::RS_FATAL ) return "FATAL";
+        if( s == MemberState::RS_STARTUP2 ) return "STARTUP2";
+        if( s == MemberState::RS_ARBITER ) return "ARBITER";
+        if( s == MemberState::RS_DOWN ) return "DOWN";
+        if( s == MemberState::RS_ROLLBACK ) return "ROLLBACK";
         return "";
     }
 
@@ -171,7 +176,7 @@ namespace mongo {
     }
 
     void ReplSetImpl::_getOplogDiagsAsHtml(unsigned server_id, stringstream& ss) const { 
-        Member *m = findById(server_id);
+        const Member *m = findById(server_id);
         if( m == 0 ) { 
             ss << "Error : can't find a member with id: " << server_id << '\n';
             return;
@@ -182,7 +187,7 @@ namespace mongo {
         //const bo fields = BSON( "o" << false << "o2" << false );
         const bo fields;
 
-        ScopedConn conn(m->fullName());        
+        ScopedDbConnection conn(m->fullName());
 
         auto_ptr<DBClientCursor> c = conn->query(rsoplog, Query().sort("$natural",1), 20, 0, &fields);
         if( c.get() == 0 ) { 
@@ -241,8 +246,6 @@ namespace mongo {
         ss << _table();
         ss << p(time_t_to_String_short(time(0)) + " current time");
 
-        //ss << "</pre>\n";
-
         if( !otEnd.isNull() ) {
             ss << "<p>Log length in time: ";
             unsigned d = otEnd.getSecs() - otFirst.getSecs();
@@ -255,6 +258,7 @@ namespace mongo {
             ss << "</p>\n";
         }
 
+        conn.done();
     }
 
     void ReplSetImpl::_summarizeAsHtml(stringstream& s) const { 
@@ -302,7 +306,7 @@ namespace mongo {
                 td(ago(started)) << 
 	        td("") << // last heartbeat
                 td(ToString(_self->config().votes)) << 
-                td(stateAsHtml(box.getState()));
+                td( stateAsHtml(box.getState()) + (_self->config().hidden?" (hidden)":"") );
             s << td( _hbmsg );
             stringstream q;
             q << "/_replSetOplog?" << _self->id();
@@ -329,7 +333,7 @@ namespace mongo {
         _rsLog.toHTML( s );
     }
 
-    Member* ReplSetImpl::findById(unsigned id) const { 
+    const Member* ReplSetImpl::findById(unsigned id) const { 
         if( id == _self->id() ) return _self;
         for( Member *m = head(); m; m = m->next() )
             if( m->id() == id ) 
@@ -371,6 +375,7 @@ namespace mongo {
             v.push_back(bb.obj());
             m = m->next();
         }
+        sort(v.begin(), v.end());
         b.append("set", name());
         b.appendTimeT("date", time(0));
         b.append("myState", box.getState().s);

db/repl/health.h

@@ -27,7 +27,7 @@ namespace mongo {
         HealthOptions() { 
             heartbeatSleepMillis = 2000;
             heartbeatTimeoutMillis = 10000;
-            heartbeatConnRetries  = 3;
+            heartbeatConnRetries  = 2;
         }
 
         bool isDefault() const { return *this == HealthOptions(); }

db/repl/heartbeat.cpp

@@ -40,6 +40,13 @@ namespace mongo {
     // hacky
     string *discoveredSeed = 0;
 
+    long long HeartbeatInfo::timeDown() const {
+        if( up() ) return 0;
+        if( downSince == 0 ) 
+            return 0; // still waiting on first heartbeat
+        return jsTime() - downSince;
+    }
+
     /* { replSetHeartbeat : <setname> } */
     class CmdReplSetHeartbeat : public ReplSetCommand {
     public:
@@ -55,6 +62,14 @@ namespace mongo {
                 errmsg = "not running with --replSet";
                 return false;
             }
+
+            /* we want to keep heartbeat connections open when relinquishing primary.  tag them here. */
+            {
+                MessagingPort *mp = cc()._mp;
+                if( mp ) 
+                    mp->tag |= 1;
+            }
+
             if( cmdObj["pv"].Int() != 1 ) { 
                 errmsg = "incompatible replset protocol version";
                 return false;
@@ -91,7 +106,7 @@ namespace mongo {
             result.append("set", theReplSet->name());
             result.append("state", theReplSet->state().s);
             result.append("hbmsg", theReplSet->hbmsg());
-            result.append("time", (int) time(0));
+            result.append("time", (long long) time(0));
             result.appendDate("opTime", theReplSet->lastOpTimeWritten.asDate());
             int v = theReplSet->config().version;
             result.append("v", v);
@@ -119,7 +134,7 @@ namespace mongo {
         assert( theReplSet == 0 || !theReplSet->lockedByMe() );
 
         ScopedConn conn(memberFullName);
-        return conn->runCommand("admin", cmd, result);
+        return conn.runCommand("admin", cmd, result, 0);
     }
 
     /* poll every other set member to check its status */
@@ -196,7 +211,12 @@ namespace mongo {
 
             static time_t last = 0;
             time_t now = time(0);
-            if( mem.changed(old) || now-last>4 ) {
+            bool changed = mem.changed(old);
+            if( changed ) { 
+                if( old.hbstate != mem.hbstate ) 
+                    log() << "replSet " << h.toString() << ' ' << mem.hbstate.toString() << rsLog;
+            }
+            if( changed || now-last>4 ) {
                 last = now;
                 theReplSet->mgr->send( boost::bind(&Manager::msgCheckNewState, theReplSet->mgr) );
             }
@@ -205,8 +225,9 @@ namespace mongo {
     private:
         void down(HeartbeatInfo& mem, string msg) {
             mem.health = 0.0;
-            if( mem.upSince ) {
+            if( mem.upSince || mem.downSince == 0 ) {
                 mem.upSince = 0;
+                mem.downSince = jsTime();
                 log() << "replSet info " << h.toString() << " is now down (or slow to respond)" << rsLog;
             }
             mem.lastHeartbeatMsg = msg;

db/repl/manager.cpp

@@ -29,12 +29,17 @@ namespace mongo {
     };
 
     /* check members OTHER THAN US to see if they think they are primary */
-    const Member * Manager::findOtherPrimary() { 
+    const Member * Manager::findOtherPrimary(bool& two) { 
+        two = false;
         Member *m = rs->head();
         Member *p = 0;
         while( m ) {
+            DEV assert( m != rs->_self );
             if( m->state().primary() && m->hbinfo().up() ) {
-                if( p ) throw "twomasters"; // our polling is asynchronous, so this is often ok.
+                if( p ) { 
+                    two = true;
+                    return 0;
+                }
                 p = m;
             }
             m = m->next();
@@ -63,7 +68,11 @@ namespace mongo {
         if( rs->box.getPrimary() == m )
             return;
         rs->_self->lhb() = "";
-        rs->box.set(rs->iAmArbiterOnly() ? MemberState::RS_ARBITER : MemberState::RS_RECOVERING, m);
+        if( rs->iAmArbiterOnly() ) {
+            rs->box.set(MemberState::RS_ARBITER, m);
+        } else {
+            rs->box.noteRemoteIsPrimary(m);
+        }
     }
 
     /** called as the health threads get new results */
@@ -87,11 +96,14 @@ namespace mongo {
             }
 
             const Member *p2;
-            try { p2 = findOtherPrimary(); }
-            catch(string s) { 
-                /* two other nodes think they are primary (asynchronously polled) -- wait for things to settle down. */
-                log() << "replSet warning DIAG 2 primary" << s << rsLog;
-                return;
+            {
+                bool two;
+                p2 = findOtherPrimary(two);
+                if( two ) {
+                    /* two other nodes think they are primary (asynchronously polled) -- wait for things to settle down. */
+                    log() << "replSet warning DIAG two primaries (transiently)" << rsLog;
+                    return;
+                }
             }
 
             if( p2 ) {
@@ -136,7 +148,7 @@ namespace mongo {
                     return;
                 }
 
-                if( !rs->elect.aMajoritySeemsToBeUp() ) { 
+                if( rs->elect.shouldRelinquish() ) { 
                     log() << "replSet can't see a majority of the set, relinquishing primary" << rsLog;
                     rs->relinquish();
                 }

db/repl/multicmd.h

@@ -43,7 +43,7 @@ namespace mongo {
         void run() {
             try { 
                 ScopedConn c(d.toHost);
-                d.ok = c->runCommand("admin", cmd, d.result);
+                d.ok = c.runCommand("admin", cmd, d.result);
             }
             catch(DBException&) { 
                 DEV log() << "dev caught dbexception on multiCommand " << d.toHost << rsLog;

db/repl/replset_commands.cpp

@@ -26,7 +26,7 @@
 
 namespace mongo { 
 
-    void checkAllMembersUpForConfigChange(const ReplSetConfig& cfg, bool initial);
+    void checkMembersUpForConfigChange(const ReplSetConfig& cfg, bool initial);
 
     /* commands in other files:
          replSetHeartbeat - health.cpp
@@ -34,19 +34,27 @@ namespace mongo {
     */
 
     bool replSetBlind = false;
+    unsigned replSetForceInitialSyncFailure = 0;
 
     class CmdReplSetTest : public ReplSetCommand {
     public:
         virtual void help( stringstream &help ) const {
-            help << "Just for testing : do not use.\n";
+            help << "Just for regression tests.\n";
         }
         CmdReplSetTest() : ReplSetCommand("replSetTest") { }
         virtual bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+            log() << "replSet replSetTest command received: " << cmdObj.toString() << rsLog;
+            if( cmdObj.hasElement("forceInitialSyncFailure") ) {
+                replSetForceInitialSyncFailure = (unsigned) cmdObj["forceInitialSyncFailure"].Number();
+                return true;
+            }
+
+            // may not need this, but if removed check all tests still work:
             if( !check(errmsg, result) ) 
                 return false;
+
             if( cmdObj.hasElement("blind") ) {
                 replSetBlind = cmdObj.getBoolField("blind");
-                log() << "replSet info replSetTest command received, replSetBlind=" << replSetBlind << rsLog;
                 return true;
             }
             return false;
@@ -55,6 +63,7 @@ namespace mongo {
 
     class CmdReplSetGetRBID : public ReplSetCommand {
     public:
+        /* todo: ideally this should only change on rollbacks NOT on mongod restarts also. fix... */
         int rbid;
         virtual void help( stringstream &help ) const {
             help << "internal";
@@ -65,12 +74,15 @@ namespace mongo {
         virtual bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
             if( !check(errmsg, result) ) 
                 return false;
-	    result.append("rbid",rbid);
+            result.append("rbid",rbid);
             return true;
         }
     } cmdReplSetRBID;
 
     using namespace bson;
+    void incRBID() { 
+        cmdReplSetRBID.rbid++;
+    }
     int getRBID(DBClientConnection *c) { 
         bo info;
         c->simpleCommand("admin", &info, "replSetGetRBID");
@@ -151,9 +163,9 @@ namespace mongo {
                     return false;
                 }
 
-                checkAllMembersUpForConfigChange(newConfig,false);
+                checkMembersUpForConfigChange(newConfig,false);
 
-                log() << "replSet replSetReconfig all members seem up" << rsLog;
+                log() << "replSet replSetReconfig [2]" << rsLog;
 
                 theReplSet->haveNewConfig(newConfig, true);
                 ReplSet::startupStatusMsg = "replSetReconfig'd";

db/repl/rs.cpp

@@ -31,39 +31,48 @@ namespace mongo {
     extern string *discoveredSeed;
 
     void ReplSetImpl::sethbmsg(string s, int logLevel) { 
-      static time_t lastLogged;
-      if( s == _hbmsg ) { 
-	// unchanged
-	if( time(0)-lastLogged < 60 )
-	  return;
-      }
+        static time_t lastLogged;
+        _hbmsgTime = time(0);
 
-      unsigned sz = s.size();
-      if( sz >= 256 ) 
-	memcpy(_hbmsg, s.c_str(), 255);
-      else {
-	_hbmsg[sz] = 0;
-	memcpy(_hbmsg, s.c_str(), sz);
-      }
-      if( !s.empty() ) {
-	lastLogged = time(0);
-	log(logLevel) << "replSet " << s << rsLog;
-      }
+        if( s == _hbmsg ) { 
+            // unchanged
+            if( _hbmsgTime - lastLogged < 60 )
+                return;
+        }
+
+        unsigned sz = s.size();
+        if( sz >= 256 ) 
+            memcpy(_hbmsg, s.c_str(), 255);
+        else {
+            _hbmsg[sz] = 0;
+            memcpy(_hbmsg, s.c_str(), sz);
+        }
+        if( !s.empty() ) {
+            lastLogged = _hbmsgTime;
+            log(logLevel) << "replSet " << s << rsLog;
+        }
     }
 
     void ReplSetImpl::assumePrimary() { 
         assert( iAmPotentiallyHot() );
         writelock lk("admin."); // so we are synchronized with _logOp()
         box.setSelfPrimary(_self);
-        log() << "replSet PRIMARY" << rsLog; // self (" << _self->id() << ") is now primary" << rsLog;
+        //log() << "replSet PRIMARY" << rsLog; // self (" << _self->id() << ") is now primary" << rsLog;
     }
 
     void ReplSetImpl::changeState(MemberState s) { box.change(s, _self); }
 
     void ReplSetImpl::relinquish() { 
         if( box.getState().primary() ) {
+            log() << "replSet relinquishing primary state" << rsLog;
             changeState(MemberState::RS_RECOVERING);
-            log() << "replSet info relinquished primary state" << rsLog;
+            
+            /* close sockets that were talking to us */
+            /*log() << "replSet closing sockets after reqlinquishing primary" << rsLog;
+            MessagingPort::closeAllSockets(1);*/
+
+            // todo: >
+            //changeState(MemberState::RS_SECONDARY);
         }
         else if( box.getState().startup2() ) {
             // ? add comment
@@ -109,11 +118,18 @@ namespace mongo {
     }
 
     void ReplSetImpl::_fillIsMasterHost(const Member *m, vector<string>& hosts, vector<string>& passives, vector<string>& arbiters) {
+        if( m->config().hidden )
+            return;
+
         if( m->potentiallyHot() ) {
             hosts.push_back(m->h().toString());
         }
         else if( !m->config().arbiterOnly ) {
-            passives.push_back(m->h().toString());
+            if( m->config().slaveDelay ) {
+                /* hmmm - we don't list these as they are stale. */   
+            } else {
+                passives.push_back(m->h().toString());
+            }
         }
         else {
             arbiters.push_back(m->h().toString());
@@ -123,6 +139,7 @@ namespace mongo {
     void ReplSetImpl::_fillIsMaster(BSONObjBuilder& b) {
         const StateBox::SP sp = box.get();
         bool isp = sp.state.primary();
+        b.append("setName", name());
         b.append("ismaster", isp);
         b.append("secondary", sp.state.secondary());
         {
@@ -151,6 +168,10 @@ namespace mongo {
         }
         if( myConfig().arbiterOnly )
             b.append("arbiterOnly", true);
+        if( myConfig().slaveDelay )
+            b.append("slaveDelay", myConfig().slaveDelay);
+        if( myConfig().hidden )
+            b.append("hidden", true);
     }
 
     /** @param cfgString <setname>/<seedhost1>,<seedhost2> */
@@ -199,6 +220,7 @@ namespace mongo {
         _self(0), 
         mgr( new Manager(this) )
     {
+        _cfg = 0;
         memset(_hbmsg, 0, sizeof(_hbmsg));
         *_hbmsg = '.'; // temp...just to see
         lastH = 0;
@@ -244,7 +266,7 @@ namespace mongo {
             loadLastOpTimeWritten();
         }
         catch(std::exception& e) { 
-            log() << "replSet ERROR FATAL couldn't query the local " << rsoplog << " collection.  Terminating mongod after 30 seconds." << rsLog;
+            log() << "replSet error fatal couldn't query the local " << rsoplog << " collection.  Terminating mongod after 30 seconds." << rsLog;
             log() << e.what() << rsLog;
             sleepsecs(30);
             dbexit( EXIT_REPLICATION_ERROR );
@@ -259,25 +281,64 @@ namespace mongo {
     ReplSetImpl::StartupStatus ReplSetImpl::startupStatus = PRESTART;
     string ReplSetImpl::startupStatusMsg;
 
-    // true if ok; throws if config really bad; false if config doesn't include self
-    bool ReplSetImpl::initFromConfig(ReplSetConfig& c) {
+    extern BSONObj *getLastErrorDefault;
+
+    /** @param reconf true if this is a reconfiguration and not an initial load of the configuration.
+        @return true if ok; throws if config really bad; false if config doesn't include self
+    */
+    bool ReplSetImpl::initFromConfig(ReplSetConfig& c, bool reconf) {
+        /* NOTE: haveNewConfig() writes the new config to disk before we get here.  So 
+                 we cannot error out at this point, except fatally.  Check errors earlier.
+                 */
         lock lk(this);
 
+        if( getLastErrorDefault || !c.getLastErrorDefaults.isEmpty() ) {
+            // see comment in dbcommands.cpp for getlasterrordefault
+            getLastErrorDefault = new BSONObj( c.getLastErrorDefaults );
+        }
+
+        list<const ReplSetConfig::MemberCfg*> newOnes;
+        bool additive = reconf;
         {
+            unsigned nfound = 0;
             int me = 0;
             for( vector<ReplSetConfig::MemberCfg>::iterator i = c.members.begin(); i != c.members.end(); i++ ) { 
                 const ReplSetConfig::MemberCfg& m = *i;
                 if( m.h.isSelf() ) {
+                    nfound++;
                     me++;
+
+                    if( !reconf || (_self && _self->id() == (unsigned) m._id) )
+                        ;
+                    else { 
+                        log() << "replSet " << _self->id() << ' ' << m._id << rsLog;
+                        assert(false);
+                    }
+                }
+                else if( reconf ) { 
+                    const Member *old = findById(m._id);
+                    if( old ) { 
+                        nfound++;
+                        assert( (int) old->id() == m._id );
+                        if( old->config() == m ) { 
+                            additive = false;
+                        }
+                    }
+                    else {
+                        newOnes.push_back(&m);
+                    }
                 }
             }
             if( me == 0 ) {
                 // log() << "replSet config : " << _cfg->toString() << rsLog;
-                log() << "replSet warning can't find self in the repl set configuration:" << rsLog;
+                log() << "replSet error can't find self in the repl set configuration:" << rsLog;
                 log() << c.toString() << rsLog;
-                return false;
+                assert(false);
             }
             uassert( 13302, "replSet error self appears twice in the repl set configuration", me<=1 );
+
+            if( reconf && config().members.size() != nfound ) 
+                additive = false;
         }
 
         _cfg = new ReplSetConfig(c);
@@ -286,6 +347,24 @@ namespace mongo {
         _name = _cfg->_id;
         assert( !_name.empty() );
 
+        if( additive ) { 
+            log() << "replSet info : additive change to configuration" << rsLog;
+            for( list<const ReplSetConfig::MemberCfg*>::const_iterator i = newOnes.begin(); i != newOnes.end(); i++ ) {
+                const ReplSetConfig::MemberCfg* m = *i;
+                Member *mi = new Member(m->h, m->_id, m, false);
+
+                /** we will indicate that new members are up() initially so that we don't relinquish our 
+                    primary state because we can't (transiently) see a majority.  they should be up as we 
+                    check that new members are up before getting here on reconfig anyway.
+                    */
+                mi->get_hbinfo().health = 0.1;
+
+                _members.push(mi);
+                startHealthTaskFor(mi);
+            }
+            return true;
+        }
+
         // start with no members.  if this is a reconfig, drop the old ones.
         _members.orphanAll();
 
@@ -390,9 +469,9 @@ namespace mongo {
                         startupStatus = EMPTYCONFIG;
                         startupStatusMsg = "can't get " + rsConfigNs + " config from self or any seed (EMPTYCONFIG)";
                         log() << "replSet can't get " << rsConfigNs << " config from self or any seed (EMPTYCONFIG)" << rsLog;
-                        log() << "replSet   have you ran replSetInitiate yet?" << rsLog;
+                        log(1) << "replSet have you ran replSetInitiate yet?" << rsLog;
                         if( _seeds->size() == 0 )
-                            log() << "replSet   no seed hosts were specified on the --replSet command line - that might be the issue" << rsLog;
+                            log(1) << "replSet info no seed hosts were specified on the --replSet command line" << rsLog;
                     }
                     else {
                         startupStatus = EMPTYUNREACHABLE;
@@ -415,6 +494,7 @@ namespace mongo {
                 startupStatusMsg = "replSet error loading set config (BADCONFIG)";
                 log() << "replSet error loading configurations " << e.toString() << rsLog;
                 log() << "replSet error replication will not start" << rsLog;
+                sethbmsg("error loading set config");
                 _fatal();
                 throw;
             }
@@ -428,11 +508,10 @@ namespace mongo {
     { 
         //lock l(this);
         box.set(MemberState::RS_FATAL, 0);
-        sethbmsg("fatal error");
-        log() << "replSet error fatal error, stopping replication" << rsLog; 
+        //sethbmsg("fatal error");
+        log() << "replSet error fatal, stopping replication" << rsLog; 
     }
 
-
     void ReplSet::haveNewConfig(ReplSetConfig& newConfig, bool addComment) { 
         lock l(this); // convention is to lock replset before taking the db rwlock
         writelock lk("");
@@ -441,7 +520,7 @@ namespace mongo {
             comment = BSON( "msg" << "Reconfig set" << "version" << newConfig.version );
         newConfig.saveConfigLocally(comment);
         try { 
-            initFromConfig(newConfig);
+            initFromConfig(newConfig, true);
             log() << "replSet replSetReconfig new config saved locally" << rsLog;
         }
         catch(DBException& e) { 

db/repl/rs.h

@@ -44,19 +44,19 @@ namespace mongo {
     public:
         Member(HostAndPort h, unsigned ord, const ReplSetConfig::MemberCfg *c, bool self);
         string fullName() const { return h().toString(); }
-        const ReplSetConfig::MemberCfg& config() const { return *_config; }
+        const ReplSetConfig::MemberCfg& config() const { return _config; }
         const HeartbeatInfo& hbinfo() const { return _hbinfo; }
-        string lhb() { return _hbinfo.lastHeartbeatMsg; }
+        HeartbeatInfo& get_hbinfo() { return _hbinfo; }
+        string lhb() const { return _hbinfo.lastHeartbeatMsg; }
         MemberState state() const { return _hbinfo.hbstate; }
         const HostAndPort& h() const { return _h; }
         unsigned id() const { return _hbinfo.id(); }
-        bool potentiallyHot() const { return _config->potentiallyHot(); } // not arbiter, not priority 0
-
+        bool potentiallyHot() const { return _config.potentiallyHot(); } // not arbiter, not priority 0
         void summarizeMember(stringstream& s) const;
         friend class ReplSetImpl;
     private:
-        const ReplSetConfig::MemberCfg *_config; /* todo: when this changes??? */
-        HostAndPort _h;
+        const ReplSetConfig::MemberCfg _config;
+        const HostAndPort _h;
         HeartbeatInfo _hbinfo;
     };
 
@@ -64,7 +64,13 @@ namespace mongo {
         ReplSetImpl *rs;
         bool busyWithElectSelf;
         int _primary;
-        const Member* findOtherPrimary();
+
+        /** @param two - if true two primaries were seen.  this can happen transiently, in addition to our 
+                         polling being only occasional.  in this case null is returned, but the caller should 
+                         not assume primary itself in that situation.
+        */
+        const Member* findOtherPrimary(bool& two);
+
         void noteARemoteIsPrimary(const Member *);
         virtual void starting();
     public:
@@ -102,6 +108,7 @@ namespace mongo {
 
         int totalVotes() const;
         bool aMajoritySeemsToBeUp() const;
+        bool shouldRelinquish() const;
         void electSelf();
         void electCmdReceived(BSONObj, BSONObjBuilder*);
         void multiCommand(BSONObj cmd, list<Target>& L);
@@ -119,8 +126,8 @@ namespace mongo {
     protected:
         RSBase() : magic(0x12345677), m("RSBase"), _locked(0) { }
         ~RSBase() { 
-            log() << "~RSBase should never be called?" << rsLog;
-            assert(false);
+            /* this can happen if we throw in the constructor; otherwise never happens.  thus we log it as it is quite unusual. */
+            log() << "replSet ~RSBase called" << rsLog;
         }
 
         class lock { 
@@ -175,6 +182,9 @@ namespace mongo {
         const Member* getPrimary() const { return sp.primary; }
         void change(MemberState s, const Member *self) { 
             scoped_lock lk(m);
+            if( sp.state != s ) { 
+                log() << "replSet " << s.toString() << rsLog;
+            }
             sp.state = s;
             if( s.primary() ) { 
                 sp.primary = self;
@@ -194,6 +204,12 @@ namespace mongo {
             assert( !sp.state.primary() );
             sp.primary = mem;
         }
+        void noteRemoteIsPrimary(const Member *remote) { 
+            scoped_lock lk(m);
+            if( !sp.state.secondary() && !sp.state.fatal() )
+                sp.state = MemberState::RS_RECOVERING;
+            sp.primary = remote;
+        }
         StateBox() : m("StateBox") { }
     private:
         mutex m;
@@ -226,7 +242,6 @@ namespace mongo {
         };
         static StartupStatus startupStatus;
         static string startupStatusMsg;
-        static string stateAsStr(MemberState state);
         static string stateAsHtml(MemberState state);
 
         /* todo thread */
@@ -241,28 +256,24 @@ namespace mongo {
         void endOldHealthTasks();
         void startHealthTaskFor(Member *m);
 
-    private:
         Consensus elect;
-        bool ok() const { return !box.getState().fatal(); }
-
         void relinquish();
         void forgetPrimary();
-
     protected:
         bool _stepDown();
     private:
         void assumePrimary();
         void loadLastOpTimeWritten();
         void changeState(MemberState s);
-
     protected:
         // "heartbeat message"
         // sent in requestHeartbeat respond in field "hbm" 
         char _hbmsg[256]; // we change this unlocked, thus not an stl::string
+        time_t _hbmsgTime; // when it was logged
     public:
         void sethbmsg(string s, int logLevel = 0); 
     protected:
-        bool initFromConfig(ReplSetConfig& c); // true if ok; throws if config really bad; false if config doesn't include self
+        bool initFromConfig(ReplSetConfig& c, bool reconf=false); // true if ok; throws if config really bad; false if config doesn't include self
         void _fillIsMaster(BSONObjBuilder&);
         void _fillIsMasterHost(const Member*, vector<string>&, vector<string>&, vector<string>&);
         const ReplSetConfig& config() { return *_cfg; }
@@ -305,7 +316,9 @@ namespace mongo {
 
     private:
         Member* head() const { return _members.head(); }
-        Member* findById(unsigned id) const;
+    public:
+        const Member* findById(unsigned id) const;
+    private:
         void _getTargets(list<Target>&, int &configVersion);
         void getTargets(list<Target>&, int &configVersion);
         void startThreads();
@@ -321,8 +334,10 @@ namespace mongo {
         void _syncDoInitialSync();
         void syncDoInitialSync();
         void _syncThread();
+        bool tryToGoLiveAsASecondary(OpTime&); // readlocks
         void syncTail();
         void syncApply(const BSONObj &o);
+        unsigned _syncRollback(OplogReader& r);
         void syncRollback(OplogReader& r);
         void syncFixUp(HowToFixUp& h, OplogReader& r);
     public:
@@ -342,9 +357,10 @@ namespace mongo {
 
         /* call after constructing to start - returns fairly quickly after la[unching its threads */
         void go() { _go(); }
+
         void fatal() { _fatal(); }
-        bool isPrimary();
-        bool isSecondary();
+        bool isPrimary() { return box.getState().primary(); }
+        bool isSecondary() {  return box.getState().secondary(); }
         MemberState state() const { return ReplSetImpl::state(); }
         string name() const { return ReplSetImpl::name(); }
         const ReplSetConfig& config() { return ReplSetImpl::config(); }
@@ -364,7 +380,10 @@ namespace mongo {
         bool lockedByMe() { return RSBase::lockedByMe(); }
 
         // heartbeat msg to send to others; descriptive diagnostic info
-        string hbmsg() const { return _hbmsg; }
+        string hbmsg() const { 
+            if( time(0)-_hbmsgTime > 120 ) return "";
+            return _hbmsg; 
+        }
     };
 
     /** base class for repl set commands.  checks basic things such as in rs mode before the command 
@@ -386,6 +405,8 @@ namespace mongo {
             if( theReplSet == 0 ) {
                 result.append("startupStatus", ReplSet::startupStatus);
                 errmsg = ReplSet::startupStatusMsg.empty() ? "replset unknown error 2" : ReplSet::startupStatusMsg;
+                if( ReplSet::startupStatus == 3 ) 
+                    result.append("info", "run rs.initiate(...) if not yet done for the set");
                 return false;
             }
             return true;
@@ -395,19 +416,10 @@ namespace mongo {
     /** inlines ----------------- */
 
     inline Member::Member(HostAndPort h, unsigned ord, const ReplSetConfig::MemberCfg *c, bool self) : 
-        _config(c), _h(h), _hbinfo(ord) { 
-            if( self ) { 
-                _hbinfo.health = 1.0;
-            }
-    }
-
-    inline bool ReplSet::isPrimary() {         
-        /* todo replset */
-        return box.getState().primary();
-    }
-
-    inline bool ReplSet::isSecondary() { 
-      return box.getState().secondary();
+        _config(*c), _h(h), _hbinfo(ord) 
+    { 
+        if( self )
+            _hbinfo.health = 1.0;
     }
 
 }

db/repl/rs_config.cpp

@@ -31,6 +31,16 @@ namespace mongo {
 
     void logOpInitiate(const bo&);
 
+    void assertOnlyHas(BSONObj o, const set<string>& fields) { 
+        BSONObj::iterator i(o);
+        while( i.more() ) {
+            BSONElement e = i.next();
+            if( !fields.count( e.fieldName() ) ) {
+                uasserted(13434, str::stream() << "unexpected field '" << e.fieldName() << "'in object");
+            }
+        }
+    }
+
     list<HostAndPort> ReplSetConfig::otherMemberHostnames() const { 
         list<HostAndPort> L;
         for( vector<MemberCfg>::const_iterator i = members.begin(); i != members.end(); i++ ) {
@@ -42,7 +52,7 @@ namespace mongo {
     
     /* comment MUST only be set when initiating the set by the initiator */
     void ReplSetConfig::saveConfigLocally(bo comment) { 
-        check();
+        checkRsConfig();
         log() << "replSet info saving a newer config version to local.system.replset" << rsLog;
         { 
             writelock lk("");
@@ -81,6 +91,8 @@ namespace mongo {
         if( votes != 1 ) b << "votes" << votes;
         if( priority != 1.0 ) b << "priority" << priority;
         if( arbiterOnly ) b << "arbiterOnly" << true;
+        if( slaveDelay ) b << "slaveDelay" << slaveDelay;
+        if( hidden ) b << "hidden" << hidden;
         return b.obj();
     }
 
@@ -114,10 +126,18 @@ namespace mongo {
         mchk(_id >= 0 && _id <= 255);
         mchk(priority >= 0 && priority <= 1000);
         mchk(votes >= 0 && votes <= 100);
-        uassert(13419, "replica set config : this version of mongo only supports priorities 0 and 1", priority == 0 || priority == 1);
+        uassert(13419, "this version of mongod only supports priorities 0 and 1", priority == 0 || priority == 1);
+        uassert(13437, "slaveDelay requires priority be zero", slaveDelay == 0 || priority == 0);
+        uassert(13438, "bad slaveDelay value", slaveDelay >= 0 && slaveDelay <= 3600 * 24 * 366);
+        uassert(13439, "priority must be 0 when hidden=true", priority == 0 || !hidden);
     }
 
+    /** @param o old config
+        @param n new config 
+        */
     /*static*/ bool ReplSetConfig::legalChange(const ReplSetConfig& o, const ReplSetConfig& n, string& errmsg) { 
+        assert( theReplSet );
+
         if( o._id != n._id ) { 
             errmsg = "set name may not change"; 
             return false;
@@ -131,9 +151,28 @@ namespace mongo {
             return false;
         }
 
+        map<HostAndPort,const ReplSetConfig::MemberCfg*> old;
+        for( vector<ReplSetConfig::MemberCfg>::const_iterator i = o.members.begin(); i != o.members.end(); i++ ) { 
+            old[i->h] = &(*i);
+        }
+        int me = 0;
+        for( vector<ReplSetConfig::MemberCfg>::const_iterator i = n.members.begin(); i != n.members.end(); i++ ) { 
+            const ReplSetConfig::MemberCfg& m = *i;
+            if( old.count(m.h) ) { 
+                if( old[m.h]->_id != m._id ) { 
+                    log() << "replSet reconfig error with member: " << m.h.toString() << rsLog;
+                    uasserted(13432, "_id may not change for members");
+                }
+            }
+            if( m.h.isSelf() ) 
+                me++;
+        }
+
+        uassert(13433, "can't find self in new replset config", me == 1);
+
         /* TODO : MORE CHECKS HERE */
 
-        cout << "TODO : don't allow removal of a node until we handle it at the removed node end." << endl;
+        log() << "replSet TODO : don't allow removal of a node until we handle it at the removed node end?" << endl;
         // we could change its votes to zero perhaps instead as a short term...
 
         return true;
@@ -144,7 +183,7 @@ namespace mongo {
         _ok = false;
     }
 
-    void ReplSetConfig::check() const { 
+    void ReplSetConfig::checkRsConfig() const { 
         uassert(13132,
             "nonmatching repl set name in _id field; check --replSet command line",
             _id == cmdLine.ourSetName());
@@ -154,6 +193,10 @@ namespace mongo {
     }
 
     void ReplSetConfig::from(BSONObj o) {
+        static const string legal[] = {"_id","version", "members","settings"};
+        static const set<string> legals(legal, legal + 4);
+        assertOnlyHas(o, legals);
+
         md5 = o.md5();
         _id = o["_id"].String();
         if( o["version"].ok() ) {
@@ -170,7 +213,7 @@ namespace mongo {
             if( settings["heartbeatTimeout"].ok() )
                 ho.heartbeatTimeoutMillis = (unsigned) (settings["heartbeatTimeout"].Number() * 1000);
             ho.check();
-            try { getLastErrorDefaults = settings["getLastErrorDefaults"].Obj(); } catch(...) { }
+            try { getLastErrorDefaults = settings["getLastErrorDefaults"].Obj().copy(); } catch(...) { }
         }
 
         set<string> hosts;
@@ -188,6 +231,10 @@ namespace mongo {
             BSONObj mobj = members[i].Obj();
             MemberCfg m;
             try {
+                static const string legal[] = {"_id","votes","priority","host","hidden","slaveDelay","arbiterOnly"};
+                static const set<string> legals(legal, legal + 7);
+                assertOnlyHas(mobj, legals);
+
                 try { 
                     m._id = (int) mobj["_id"].Number();
                 } catch(...) { 
@@ -205,6 +252,9 @@ namespace mongo {
                 if( m.h.isLocalHost() ) 
                     localhosts++;
                 m.arbiterOnly = mobj.getBoolField("arbiterOnly");
+                m.slaveDelay = mobj["slaveDelay"].numberInt();
+                if( mobj.hasElement("hidden") )
+                    m.hidden = mobj.getBoolField("hidden");
                 if( mobj.hasElement("priority") )
                     m.priority = mobj["priority"].Number();
                 if( mobj.hasElement("votes") )
@@ -220,7 +270,7 @@ namespace mongo {
             catch(DBException& e) { 
                 log() << "replSet cfg parsing exception for members[" << i << "] " << e.what() << rsLog;
                 stringstream ss;
-                ss << "replSet members[" << i << "] bad config object";
+                ss << "bad config for member[" << i << "] " << e.what();
                 uassert(13135, ss.str(), false);
             }
             if( !(ords.count(m._id) == 0 && hosts.count(m.h.toString()) == 0) ) {
@@ -252,9 +302,8 @@ namespace mongo {
         clear();
         int level = 2;
         DEV level = 0;
-        //log(0) << "replSet load config from: " << h.toString() << rsLog;
 
-        auto_ptr<DBClientCursor> c;
+        BSONObj cfg;
         int v = -5;
         try {
             if( h.isSelf() ) {
@@ -287,13 +336,28 @@ namespace mongo {
             }
 
             v = -4;
-            ScopedConn conn(h.toString());
-            v = -3;
-            c = conn->query(rsConfigNs);
-            if( c.get() == 0 ) {
-                version = v; return;
+            unsigned long long count = 0;
+            try {
+                ScopedConn conn(h.toString());
+                v = -3;
+                cfg = conn.findOne(rsConfigNs, Query()).getOwned();
+                count = conn.count(rsConfigNs);
             }
-            if( !c->more() ) {
+            catch ( DBException& e) {
+                if ( !h.isSelf() ) {
+                    throw;
+                }
+
+                // on startup, socket is not listening yet
+                DBDirectClient cli;
+                cfg = cli.findOne( rsConfigNs, Query() ).getOwned();
+                count = cli.count(rsConfigNs);
+            }
+
+            if( count > 1 )
+                uasserted(13109, str::stream() << "multiple rows in " << rsConfigNs << " not supported host: " << h.toString());
+            
+            if( cfg.isEmpty() ) {
                 version = EMPTYCONFIG;
                 return;
             }
@@ -305,9 +369,8 @@ namespace mongo {
             return;
         }
 
-        BSONObj o = c->nextSafe();
-        uassert(13109, "multiple rows in " + rsConfigNs + " not supported", !c->more());
-        from(o);
+        from(cfg);
+        checkRsConfig();
         _ok = true;
         log(level) << "replSet load config ok from " << (h.isSelf() ? "self" : h.toString()) << rsLog;
     }

db/repl/rs_config.h

@@ -41,18 +41,27 @@ namespace mongo {
         bool ok() const { return _ok; }
 
         struct MemberCfg {
-            MemberCfg() : _id(-1), votes(1), priority(1.0), arbiterOnly(false) { }
+            MemberCfg() : _id(-1), votes(1), priority(1.0), arbiterOnly(false), slaveDelay(0), hidden(false) { }
             int _id;              /* ordinal */
             unsigned votes;       /* how many votes this node gets. default 1. */
             HostAndPort h;
             double priority;      /* 0 means can never be primary */
             bool arbiterOnly;
+            int slaveDelay;       /* seconds.  int rather than unsigned for convenient to/front bson conversion. */
+            bool hidden;          /* if set, don't advertise to drives in isMaster. for non-primaries (priority 0) */
+
             void check() const;   /* check validity, assert if not. */
             BSONObj asBson() const;
             bool potentiallyHot() const { 
                 return !arbiterOnly && priority > 0;
             }
+            bool operator==(const MemberCfg& r) const { 
+                return _id==r._id && votes == r.votes && h == r.h && priority == r.priority && 
+                    arbiterOnly == r.arbiterOnly && slaveDelay == r.slaveDelay && hidden == r.hidden;
+            }
+            bool operator!=(const MemberCfg& r) const { return !(*this == r); }
         };
+
         vector<MemberCfg> members;
         string _id;
         int version;
@@ -68,7 +77,7 @@ namespace mongo {
         string toString() const { return asBson().toString(); }
 
         /** validate the settings. does not call check() on each member, you have to do that separately. */
-        void check() const;
+        void checkRsConfig() const;
 
         /** check if modification makes sense */
         static bool legalChange(const ReplSetConfig& old, const ReplSetConfig& n, string& errmsg);

db/repl/rs_initialsync.cpp

@@ -66,7 +66,7 @@ namespace mongo {
 
     void _logOpObjRS(const BSONObj& op);
 
-    bool copyCollectionFromRemote(const string& host, const string& ns, const BSONObj& query, string errmsg);
+    bool copyCollectionFromRemote(const string& host, const string& ns, const BSONObj& query, string &errmsg, bool logforrepl);
 
     static void emptyOplog() {
         writelock lk(rsoplog);

db/repl/rs_initiate.cpp

@@ -32,10 +32,12 @@ using namespace mongoutils;
 
 namespace mongo { 
 
-    /* throws 
-       @param initial - 
+    /* called on a reconfig AND on initiate
+       throws 
+       @param initial true when initiating
     */
-    void checkAllMembersUpForConfigChange(const ReplSetConfig& cfg, bool initial) {
+    void checkMembersUpForConfigChange(const ReplSetConfig& cfg, bool initial) {
+        int failures = 0;
         int me = 0;
         for( vector<ReplSetConfig::MemberCfg>::const_iterator i = cfg.members.begin(); i != cfg.members.end(); i++ ) {
             if( i->h.isSelf() ) {
@@ -45,7 +47,7 @@ namespace mongo {
                 }
             }
         }
-        uassert(13278, "bad config?", me <= 1);
+        uassert(13278, "bad config - dups?", me <= 1); // dups?
         uassert(13279, "can't find self in the replset config", me == 1);
 
         for( vector<ReplSetConfig::MemberCfg>::const_iterator i = cfg.members.begin(); i != cfg.members.end(); i++ ) {
@@ -62,12 +64,11 @@ namespace mongo {
                     }
                 }
                 catch(DBException& e) { 
-                    log() << "replSet requestHeartbeat " << i->h.toString() << " : " << e.toString() << rsLog;
+                    log() << "replSet cmufcc requestHeartbeat " << i->h.toString() << " : " << e.toString() << rsLog;
                 }
                 catch(...) { 
-                    log() << "replSet error exception in requestHeartbeat?" << rsLog;
+                    log() << "replSet cmufcc error exception in requestHeartbeat?" << rsLog;
                 }
-                cout << "TEMP hb res cfg change:" << res.toString() << endl;
                 if( res.getBoolField("mismatch") )
                     uasserted(13145, "set name does not match the set name host " + i->h.toString() + " expects");
                 if( *res.getStringField("set") ) {
@@ -83,9 +84,31 @@ namespace mongo {
                     }
                 }
                 if( !ok && !res["rs"].trueValue() ) {
-                    if( !res.isEmpty() )
+                    if( !res.isEmpty() ) {
+                        /* strange.  got a response, but not "ok". log it. */
                         log() << "replSet warning " << i->h.toString() << " replied: " << res.toString() << rsLog;
-                    uasserted(13144, "need members up to initiate/reconfig, not ok: " + i->h.toString());
+                    }
+
+                    bool allowFailure = false;
+                    failures++;
+                    if( res.isEmpty() && !initial && failures == 1 ) {
+                        /* for now we are only allowing 1 node to be down on a reconfig.  this can be made to be a minority
+                           trying to keep change small as release is near.
+                           */
+                        const Member* m = theReplSet->findById( i->_id );
+                        if( m ) { 
+                            // ok, so this was an existing member (wouldn't make sense to add to config a new member that is down)
+                            assert( m->h().toString() == i->h.toString() );
+                            allowFailure = true;
+                        }
+                    }
+
+                    if( !allowFailure ) {
+                        string msg = string("need members up to initiate, not ok : ") + i->h.toString();
+                        if( !initial )
+                            msg = string("need most members up to reconfigure, not ok : ") + i->h.toString();
+                        uasserted(13144, msg);
+                    }
                 }
             }
             if( initial ) {
@@ -175,8 +198,10 @@ namespace mongo {
                 configObj = cmdObj["replSetInitiate"].Obj();
             }
 
+            bool parsed = false;
             try {
                 ReplSetConfig newConfig(configObj);
+                parsed = true;
 
                 if( newConfig.version > 1 ) { 
                     errmsg = "can't initiate with a version number greater than 1";
@@ -185,7 +210,7 @@ namespace mongo {
 
                 log() << "replSet replSetInitiate config object parses ok, " << newConfig.members.size() << " members specified" << rsLog;
 
-                checkAllMembersUpForConfigChange(newConfig, true);
+                checkMembersUpForConfigChange(newConfig, true);
 
                 log() << "replSet replSetInitiate all members seem up" << rsLog;
 
@@ -199,7 +224,11 @@ namespace mongo {
             }
             catch( DBException& e ) { 
                 log() << "replSet replSetInitiate exception: " << e.what() << rsLog;
-                throw;
+                if( !parsed ) 
+                    errmsg = string("couldn't parse cfg object ") + e.what();
+                else
+                    errmsg = string("couldn't initiate : ") + e.what();
+                return false;
             }
 
             return true;

db/repl/rs_member.h

@@ -40,7 +40,8 @@ namespace mongo {
             RS_STARTUP2,
             RS_UNKNOWN, /* remote node not yet reached */
             RS_ARBITER,
-            RS_DOWN /* node not reachable for a report */
+            RS_DOWN, /* node not reachable for a report */
+            RS_ROLLBACK
         } s;
 
         MemberState(MS ms = RS_UNKNOWN) : s(ms) { }
@@ -51,6 +52,9 @@ namespace mongo {
         bool recovering() const { return s == RS_RECOVERING; }
         bool startup2() const { return s == RS_STARTUP2; }
         bool fatal() const { return s == RS_FATAL; }
+        bool rollback() const { return s == RS_ROLLBACK; }
+
+        string toString() const;
 
         bool operator==(const MemberState& r) const { return s == r.s; }
         bool operator!=(const MemberState& r) const { return s != r.s; }
@@ -61,26 +65,35 @@ namespace mongo {
     class HeartbeatInfo { 
         unsigned _id;
     public:
-        HeartbeatInfo() : _id(0xffffffff),skew(INT_MIN) { }
+        HeartbeatInfo() : _id(0xffffffff),hbstate(MemberState::RS_UNKNOWN),health(-1.0),downSince(0),skew(INT_MIN) { }
         HeartbeatInfo(unsigned id);
-        bool up() const { return health > 0; }
         unsigned id() const { return _id; }
         MemberState hbstate;
         double health;
         time_t upSince;
+        long long downSince;
         time_t lastHeartbeat;
         string lastHeartbeatMsg;
         OpTime opTime;
         int skew;
 
+        bool up() const { return health > 0; }
+
+        /** health is set to -1 on startup.  that means we haven't even checked yet.  0 means we checked and it failed. */
+        bool maybeUp() const { return health != 0; }
+
+        long long timeDown() const; // ms
+
         /* true if changed in a way of interest to the repl set manager. */
         bool changed(const HeartbeatInfo& old) const;
     };
 
     inline HeartbeatInfo::HeartbeatInfo(unsigned id) : _id(id) { 
-          health = -1.0;
-          lastHeartbeat = upSince = 0; 
-          skew = INT_MIN;
+        hbstate = MemberState::RS_UNKNOWN;
+        health = -1.0;
+        downSince = 0;
+        lastHeartbeat = upSince = 0; 
+        skew = INT_MIN;
     }
 
     inline bool HeartbeatInfo::changed(const HeartbeatInfo& old) const { 

db/repl/rs_rollback.cpp

@@ -62,6 +62,14 @@ namespace mongo {
 
     using namespace bson;
 
+    bool copyCollectionFromRemote(const string& host, const string& ns, const BSONObj& query, string& errmsg, bool logforrepl);
+    void incRBID();
+
+    class rsfatal : public std::exception { 
+    public:
+        virtual const char* what() const throw(){ return "replica set fatal exception"; }
+    };
+
     struct DocID {
         const char *ns;
         be _id;
@@ -81,6 +89,8 @@ namespace mongo {
         /* collections to drop */
         set<string> toDrop;
 
+        set<string> collectionsToResync;
+
         OpTime commonPoint;
         DiskLoc commonPointOurDiskloc;
 
@@ -113,17 +123,59 @@ namespace mongo {
         if( *op == 'c' ) { 
             be first = o.firstElement();
             NamespaceString s(d.ns); // foo.$cmd
-
-            if( string("create") == first.fieldName() ) {
-                /* Create collection operation 
-                   { ts: ..., h: ..., op: "c", ns: "foo.$cmd", o: { create: "abc", ... } }
-                */
-                string ns = s.db + '.' + o["create"].String(); // -> foo.abc
-                h.toDrop.insert(ns);
+            string cmdname = first.fieldName();
+            Command *cmd = Command::findCommand(cmdname.c_str());
+            if( cmd == 0 ) { 
+                log() << "replSet warning rollback no suchcommand " << first.fieldName() << " - different mongod versions perhaps?" << rsLog;
                 return;
             }
-            else { 
-                log() << "replSet WARNING can't roll back this command yet: " << o.toString() << rsLog;
+            else {
+                /* findandmodify - tranlated?
+                   godinsert?,  
+                   renamecollection a->b.  just resync a & b
+                */
+                if( cmdname == "create" ) {
+                    /* Create collection operation 
+                       { ts: ..., h: ..., op: "c", ns: "foo.$cmd", o: { create: "abc", ... } }
+                    */
+                    string ns = s.db + '.' + o["create"].String(); // -> foo.abc
+                    h.toDrop.insert(ns);
+                    return;
+                }
+                else if( cmdname == "drop" ) { 
+                    string ns = s.db + '.' + first.valuestr();
+                    h.collectionsToResync.insert(ns);
+                    return;
+                }
+                else if( cmdname == "dropIndexes" || cmdname == "deleteIndexes" ) { 
+                    /* TODO: this is bad.  we simply full resync the collection here, which could be very slow. */
+                    log() << "replSet info rollback of dropIndexes is slow in this version of mongod" << rsLog;
+                    string ns = s.db + '.' + first.valuestr();
+                    h.collectionsToResync.insert(ns);
+                    return;
+                }
+                else if( cmdname == "renameCollection" ) { 
+                    /* TODO: slow. */
+                    log() << "replSet info rollback of renameCollection is slow in this version of mongod" << rsLog;
+                    string from = first.valuestr();
+                    string to = o["to"].String();
+                    h.collectionsToResync.insert(from);
+                    h.collectionsToResync.insert(to);
+                    return;
+                }
+                else if( cmdname == "reIndex" ) { 
+                    return;
+                }
+                else if( cmdname == "dropDatabase" ) { 
+                    log() << "replSet error rollback : can't rollback drop database full resync will be required" << rsLog;
+                    log() << "replSet " << o.toString() << rsLog;
+                    throw rsfatal();
+                }
+                else { 
+                    log() << "replSet error can't rollback this command yet: " << o.toString() << rsLog;
+                    log() << "replSet cmdname=" << cmdname << rsLog;
+                    throw rsfatal();
+                }
             }
         }
 
@@ -141,8 +193,6 @@ namespace mongo {
     static void syncRollbackFindCommonPoint(DBClientConnection *them, HowToFixUp& h) { 
         static time_t last;
         if( time(0)-last < 60 ) { 
-            // this could put a lot of load on someone else, don't repeat too often
-            sleepsecs(10);
             throw "findcommonpoint waiting a while before trying again";
         }
         last = time(0);
@@ -170,12 +220,14 @@ namespace mongo {
         BSONObj theirObj = t->nextSafe();
         OpTime theirTime = theirObj["ts"]._opTime();
 
-        if( 1 ) {
+        {
             long long diff = (long long) ourTime.getSecs() - ((long long) theirTime.getSecs());
             /* diff could be positive, negative, or zero */
-            log() << "replSet info syncRollback diff in end of log times : " << diff << " seconds" << rsLog;
+            log() << "replSet info rollback our last optime:   " << ourTime.toStringPretty() << rsLog;
+            log() << "replSet info rollback their last optime: " << theirTime.toStringPretty() << rsLog;
+            log() << "replSet info rollback diff in end of log times: " << diff << " seconds" << rsLog;
             if( diff > 3600 ) { 
-                log() << "replSet syncRollback too long a time period for a rollback." << rsLog;
+                log() << "replSet rollback too long a time period for a rollback." << rsLog;
                 throw "error not willing to roll back more than one hour of data";
             }
         }
@@ -197,16 +249,35 @@ namespace mongo {
 
                 refetch(h, ourObj);
 
+                if( !t->more() ) { 
+                    log() << "replSet rollback error RS100 reached beginning of remote oplog" << rsLog;
+                    log() << "replSet   them:      " << them->toString() << " scanned: " << scanned << rsLog;
+                    log() << "replSet   theirTime: " << theirTime.toStringLong() << rsLog;
+                    log() << "replSet   ourTime:   " << ourTime.toStringLong() << rsLog;
+                    throw "RS100 reached beginning of remote oplog [2]";
+                }
                 theirObj = t->nextSafe();
                 theirTime = theirObj["ts"]._opTime();
 
                 u.advance();
-                if( !u.ok() ) throw "reached beginning of local oplog";
+                if( !u.ok() ) {
+                    log() << "replSet rollback error RS101 reached beginning of local oplog" << rsLog;
+                    log() << "replSet   them:      " << them->toString() << " scanned: " << scanned << rsLog;
+                    log() << "replSet   theirTime: " << theirTime.toStringLong() << rsLog;
+                    log() << "replSet   ourTime:   " << ourTime.toStringLong() << rsLog;
+                    throw "RS101 reached beginning of local oplog [1]";
+                }
                 ourObj = u.current();
                 ourTime = ourObj["ts"]._opTime();
             }
             else if( theirTime > ourTime ) { 
-                /* todo: we could hit beginning of log here.  exception thrown is ok but not descriptive, so fix up */
+                if( !t->more() ) { 
+                    log() << "replSet rollback error RS100 reached beginning of remote oplog" << rsLog;
+                    log() << "replSet   them:      " << them->toString() << " scanned: " << scanned << rsLog;
+                    log() << "replSet   theirTime: " << theirTime.toStringLong() << rsLog;
+                    log() << "replSet   ourTime:   " << ourTime.toStringLong() << rsLog;
+                    throw "RS100 reached beginning of remote oplog [1]";
+                }
                 theirObj = t->nextSafe();
                 theirTime = theirObj["ts"]._opTime();
             }
@@ -214,7 +285,13 @@ namespace mongo {
                 // theirTime < ourTime
                 refetch(h, ourObj);
                 u.advance();
-                if( !u.ok() ) throw "reached beginning of local oplog";
+                if( !u.ok() ) { 
+                    log() << "replSet rollback error RS101 reached beginning of local oplog" << rsLog;
+                    log() << "replSet   them:      " << them->toString() << " scanned: " << scanned << rsLog;
+                    log() << "replSet   theirTime: " << theirTime.toStringLong() << rsLog;
+                    log() << "replSet   ourTime:   " << ourTime.toStringLong() << rsLog;
+                    throw "RS101 reached beginning of local oplog [2]";
+                }
                 ourObj = u.current();
                 ourTime = ourObj["ts"]._opTime();
             }
@@ -226,7 +303,19 @@ namespace mongo {
         bson::bo goodVersionOfObject;
     };
 
-   void ReplSetImpl::syncFixUp(HowToFixUp& h, OplogReader& r) {
+    static void setMinValid(bo newMinValid) { 
+       try {
+           log() << "replSet minvalid=" << newMinValid["ts"]._opTime().toStringLong() << rsLog;
+       }
+       catch(...) { }
+       {
+           Helpers::putSingleton("local.replset.minvalid", newMinValid);
+           Client::Context cx( "local." );
+           cx.db()->flushFiles(true);            
+       }
+    }
+
+    void ReplSetImpl::syncFixUp(HowToFixUp& h, OplogReader& r) {
        DBClientConnection *them = r.conn();
 
        // fetch all first so we needn't handle interruption in a fancy way
@@ -237,6 +326,7 @@ namespace mongo {
 
        bo newMinValid;
 
+       /* fetch all the goodVersions of each document from current primary */
        DocID d;
        unsigned long long n = 0;
        try {
@@ -258,43 +348,98 @@ namespace mongo {
            }
            newMinValid = r.getLastOp(rsoplog);
            if( newMinValid.isEmpty() ) { 
-               sethbmsg("syncRollback error newMinValid empty?");
+               sethbmsg("rollback error newMinValid empty?");
                return;
            }
        }
        catch(DBException& e) {
-           sethbmsg(str::stream() << "syncRollback re-get objects: " << e.toString(),0);
-           log() << "syncRollback couldn't re-get ns:" << d.ns << " _id:" << d._id << ' ' << n << '/' << h.toRefetch.size() << rsLog;
+           sethbmsg(str::stream() << "rollback re-get objects: " << e.toString(),0);
+           log() << "rollback couldn't re-get ns:" << d.ns << " _id:" << d._id << ' ' << n << '/' << h.toRefetch.size() << rsLog;
            throw e;
        }
 
-       sethbmsg("syncRollback 3.5");
+       MemoryMappedFile::flushAll(true);
+
+       sethbmsg("rollback 3.5");
        if( h.rbid != getRBID(r.conn()) ) { 
            // our source rolled back itself.  so the data we received isn't necessarily consistent.
-           sethbmsg("syncRollback rbid on source changed during rollback, cancelling this attempt");
+           sethbmsg("rollback rbid on source changed during rollback, cancelling this attempt");
            return;
        }
 
        // update them
-       sethbmsg(str::stream() << "syncRollback 4 n:" << goodVersions.size());
+       sethbmsg(str::stream() << "rollback 4 n:" << goodVersions.size());
 
        bool warn = false;
 
        assert( !h.commonPointOurDiskloc.isNull() );
 
-       MemoryMappedFile::flushAll(true);
-
        dbMutex.assertWriteLocked();
 
        /* we have items we are writing that aren't from a point-in-time.  thus best not to come online 
 	      until we get to that point in freshness. */
-       try {
-           log() << "replSet set minvalid=" << newMinValid["ts"]._opTime().toString() << rsLog;
-       }
-       catch(...){}
-       Helpers::putSingleton("local.replset.minvalid", newMinValid);
+       setMinValid(newMinValid);
+       
+       /** any full collection resyncs required? */
+       if( !h.collectionsToResync.empty() ) {
+           for( set<string>::iterator i = h.collectionsToResync.begin(); i != h.collectionsToResync.end(); i++ ) { 
+               string ns = *i;
+               sethbmsg(str::stream() << "rollback 4.1 coll resync " << ns);
+               Client::Context c(*i, dbpath, 0, /*doauth*/false);
+               try {
+                   bob res;
+                   string errmsg;
+                   dropCollection(ns, errmsg, res);
+                   {
+                       dbtemprelease r;
+                       bool ok = copyCollectionFromRemote(them->getServerAddress(), ns, bo(), errmsg, false);
+                       if( !ok ) { 
+                           log() << "replSet rollback error resyncing collection " << ns << ' ' << errmsg << rsLog;
+                           throw "rollback error resyncing rollection [1]";
+                       }
+                   }
+               }
+               catch(...) { 
+                   log() << "replset rollback error resyncing collection " << ns << rsLog;
+                   throw "rollback error resyncing rollection [2]";
+               }
+           }
 
-       /** first drop collections to drop - that might make things faster below actually if there were subsequent inserts */
+           /* we did more reading from primary, so check it again for a rollback (which would mess us up), and 
+              make minValid newer. 
+              */
+           sethbmsg("rollback 4.2");
+           { 
+               string err;
+               try {
+                   newMinValid = r.getLastOp(rsoplog);
+                   if( newMinValid.isEmpty() ) {
+                       err = "can't get minvalid from primary";
+                   } else { 
+                       setMinValid(newMinValid);
+                   }
+               }
+               catch(...) { 
+                   err = "can't get/set minvalid"; 
+               }
+               if( h.rbid != getRBID(r.conn()) ) {
+                   // our source rolled back itself.  so the data we received isn't necessarily consistent.
+                   // however, we've now done writes.  thus we have a problem.
+                   err += "rbid at primary changed during resync/rollback";
+               }
+               if( !err.empty() ) {
+                   log() << "replSet error rolling back : " << err << ". A full resync will be necessary." << rsLog;
+                   /* todo: reset minvalid so that we are permanently in fatal state */
+                   /* todo: don't be fatal, but rather, get all the data first. */
+                   sethbmsg("rollback error");
+                   throw rsfatal();
+               }
+           }
+           sethbmsg("rollback 4.3");
+       }
+
+       sethbmsg("rollback 4.6");
+       /** drop collections to drop before doing individual fixups - that might make things faster below actually if there were subsequent inserts to rollback */
        for( set<string>::iterator i = h.toDrop.begin(); i != h.toDrop.end(); i++ ) { 
            Client::Context c(*i, dbpath, 0, /*doauth*/false);
            try {
@@ -308,6 +453,7 @@ namespace mongo {
            }
        }
 
+       sethbmsg("rollback 4.7");
        Client::Context c(rsoplog, dbpath, 0, /*doauth*/false);
        NamespaceDetails *oplogDetails = nsdetails(rsoplog);
        uassert(13423, str::stream() << "replSet error in rollback can't find " << rsoplog, oplogDetails);
@@ -320,7 +466,12 @@ namespace mongo {
            bo pattern = d._id.wrap(); // { _id : ... }
            try { 
                assert( d.ns && *d.ns );
-               
+               if( h.collectionsToResync.count(d.ns) ) {
+                   /* we just synced this entire collection */
+                   continue;
+               }
+
+               /* keep an archive of items rolled back */
                shared_ptr<RemoveSaver>& rs = removeSavers[d.ns];
                if ( ! rs )
                    rs.reset( new RemoveSaver( "rollback" , "" , d.ns ) );
@@ -343,7 +494,7 @@ namespace mongo {
                                long long start = Listener::getElapsedTimeMillis();
                                DiskLoc loc = Helpers::findOne(d.ns, pattern, false);
                                if( Listener::getElapsedTimeMillis() - start > 200 ) 
-                                   log() << "replSet warning roll back slow no _id index for " << d.ns << rsLog; 
+                                   log() << "replSet warning roll back slow no _id index for " << d.ns << " perhaps?" << rsLog; 
                                //would be faster but requires index: DiskLoc loc = Helpers::findById(nsd, pattern);
                                if( !loc.isNull() ) {
                                    try {
@@ -411,9 +562,9 @@ namespace mongo {
 
        removeSavers.clear(); // this effectively closes all of them
 
-       sethbmsg(str::stream() << "syncRollback 5 d:" << deletes << " u:" << updates);
+       sethbmsg(str::stream() << "rollback 5 d:" << deletes << " u:" << updates);
        MemoryMappedFile::flushAll(true);
-       sethbmsg("syncRollback 6");
+       sethbmsg("rollback 6");
 
        // clean up oplog
        log(2) << "replSet rollback truncate oplog after " << h.commonPoint.toStringPretty() << rsLog;
@@ -423,59 +574,99 @@ namespace mongo {
        /* reset cached lastoptimewritten and h value */
        loadLastOpTimeWritten();
 
-       sethbmsg("syncRollback 7");
+       sethbmsg("rollback 7");
        MemoryMappedFile::flushAll(true);
 
        // done
        if( warn ) 
            sethbmsg("issues during syncRollback, see log");
        else
-           sethbmsg("syncRollback done");
+           sethbmsg("rollback done");
    }
 
     void ReplSetImpl::syncRollback(OplogReader&r) { 
+        unsigned s = _syncRollback(r);
+        if( s ) 
+            sleepsecs(s);
+    }
+
+    unsigned ReplSetImpl::_syncRollback(OplogReader&r) { 
         assert( !lockedByMe() );
         assert( !dbMutex.atLeastReadLocked() );
 
-        sethbmsg("syncRollback 0");
+        sethbmsg("rollback 0");
 
         writelocktry lk(rsoplog, 20000);
         if( !lk.got() ) {
-            sethbmsg("syncRollback couldn't get write lock in a reasonable time");
-            sleepsecs(2);
-            return;
+            sethbmsg("rollback couldn't get write lock in a reasonable time");
+            return 2;
+        }
+
+        if( box.getState().secondary() ) {
+            /* by doing this, we will not service reads (return an error as we aren't in secondary staate.
+               that perhaps is moot becasue of the write lock above, but that write lock probably gets deferred 
+               or removed or yielded later anyway.
+
+               also, this is better for status reporting - we know what is happening.
+               */
+            box.change(MemberState::RS_ROLLBACK, _self);
         }
 
         HowToFixUp how;
-        sethbmsg("syncRollback 1");
+        sethbmsg("rollback 1");
         {
             r.resetCursor();
             /*DBClientConnection us(false, 0, 0);
             string errmsg;
             if( !us.connect(HostAndPort::me().toString(),errmsg) ) { 
-                sethbmsg("syncRollback connect to self failure" + errmsg);
+                sethbmsg("rollback connect to self failure" + errmsg);
                 return;
             }*/
 
-            sethbmsg("syncRollback 2 FindCommonPoint");
+            sethbmsg("rollback 2 FindCommonPoint");
             try {
                 syncRollbackFindCommonPoint(r.conn(), how);
             }
             catch( const char *p ) { 
-                sethbmsg(string("syncRollback 2 error ") + p);
-                sleepsecs(10);
-                return;
+                sethbmsg(string("rollback 2 error ") + p);
+                return 10;
+            }
+            catch( rsfatal& ) { 
+                _fatal();
+                return 2;
             }
             catch( DBException& e ) { 
-                sethbmsg(string("syncRollback 2 exception ") + e.toString() + "; sleeping 1 min");
+                sethbmsg(string("rollback 2 exception ") + e.toString() + "; sleeping 1 min");
+                dbtemprelease r;
                 sleepsecs(60);
                 throw;
             }
         }
 
-        sethbmsg("replSet syncRollback 3 fixup");
+        sethbmsg("replSet rollback 3 fixup");
 
-        syncFixUp(how, r);
+        {
+            incRBID();
+            try { 
+                syncFixUp(how, r);
+            }
+            catch( rsfatal& ) { 
+                sethbmsg("rollback fixup error");
+                _fatal();
+                return 2;
+            }
+            catch(...) { 
+                incRBID(); throw;
+            }
+            incRBID();
+
+            /* success - leave "ROLLBACK" state 
+               can go to SECONDARY once minvalid is achieved
+            */
+            box.change(MemberState::RS_RECOVERING, _self);
+        }
+
+        return 0;
     }
 
 }

db/repl/rs_sync.cpp

@@ -24,8 +24,11 @@ namespace mongo {
 
     using namespace bson;
 
+    extern unsigned replSetForceInitialSyncFailure;
+
     void startSyncThread() { 
         Client::initThread("rs_sync");
+        cc().iAmSyncThread();
         theReplSet->syncThread();
         cc().shutdown();
     }
@@ -67,7 +70,14 @@ namespace mongo {
                 return false;
             }
 
-            r.query(rsoplog, bo());
+            {
+                BSONObjBuilder q;
+                q.appendDate("$gte", applyGTE.asDate());
+                BSONObjBuilder query;
+                query.append("ts", q.done());
+                BSONObj queryObj = query.done();
+                r.query(rsoplog, queryObj);
+            }
             assert( r.haveCursor() );
 
             /* we lock outside the loop to avoid the overhead of locking on every operation.  server isn't usable yet anyway! */
@@ -91,6 +101,7 @@ namespace mongo {
             // todo : use exhaust
             unsigned long long n = 0;
             while( 1 ) { 
+
                 if( !r.more() )
                     break;
                 BSONObj o = r.nextSafe(); /* note we might get "not master" at some point */
@@ -103,9 +114,14 @@ namespace mongo {
                         anymore. assumePrimary is in the db lock so we are safe as long as 
                         we check after we locked above. */
 					const Member *p1 = box.getPrimary();
-                    if( p1 != primary ) {
-					  log() << "replSet primary was:" << primary->fullName() << " now:" << 
-						(p1 != 0 ? p1->fullName() : "none") << rsLog;
+                    if( p1 != primary || replSetForceInitialSyncFailure ) {
+                        int f = replSetForceInitialSyncFailure;
+                        if( f > 0 ) {
+                            replSetForceInitialSyncFailure = f-1;
+                            log() << "replSet test code invoked, replSetForceInitialSyncFailure" << rsLog;
+                        }
+                        log() << "replSet primary was:" << primary->fullName() << " now:" << 
+                            (p1 != 0 ? p1->fullName() : "none") << rsLog;
                         throw DBException("primary changed",0);
                     }
 
@@ -131,6 +147,32 @@ namespace mongo {
         return true;
     }
 
+    /* should be in RECOVERING state on arrival here.  
+       readlocks
+       @return true if transitioned to SECONDARY
+    */
+    bool ReplSetImpl::tryToGoLiveAsASecondary(OpTime& /*out*/ minvalid) { 
+        bool golive = false;			
+        {
+            readlock lk("local.replset.minvalid");
+            BSONObj mv;
+            if( Helpers::getSingleton("local.replset.minvalid", mv) ) { 
+                minvalid = mv["ts"]._opTime();
+                if( minvalid <= lastOpTimeWritten ) { 
+                    golive=true;
+                }
+            }
+            else 
+                golive = true; /* must have been the original member */
+        }
+        if( golive ) {
+            sethbmsg("");
+            changeState(MemberState::RS_SECONDARY);
+        }
+        return golive;
+    }
+
+    /* tail the primary's oplog.  ok to return, will be re-called. */
     void ReplSetImpl::syncTail() { 
         // todo : locking vis a vis the mgr...
 
@@ -147,14 +189,19 @@ namespace mongo {
         {
             BSONObj remoteOldestOp = r.findOne(rsoplog, Query());
             OpTime ts = remoteOldestOp["ts"]._opTime();
-            DEV log() << "remoteOldestOp: " << ts.toStringPretty() << endl;
-            else log(3) << "remoteOldestOp: " << ts.toStringPretty() << endl;
+            DEV log() << "replSet remoteOldestOp:    " << ts.toStringLong() << rsLog;
+            else log(3) << "replSet remoteOldestOp: " << ts.toStringLong() << rsLog;
+            DEV { 
+                // debugging sync1.js...
+                log() << "replSet lastOpTimeWritten: " << lastOpTimeWritten.toStringLong() << rsLog;
+                log() << "replSet our state: " << state().toString() << rsLog;
+            }
             if( lastOpTimeWritten < ts ) { 
-                log() << "replSet error too stale to catch up, at least from primary " << hn << rsLog;
-                log() << "replSet our last optime : " << lastOpTimeWritten.toStringPretty() << rsLog;
-                log() << "replSet oldest at " << hn << " : " << ts.toStringPretty() << rsLog;
+                log() << "replSet error RS102 too stale to catch up, at least from primary: " << hn << rsLog;
+                log() << "replSet our last optime : " << lastOpTimeWritten.toStringLong() << rsLog;
+                log() << "replSet oldest at " << hn << " : " << ts.toStringLong() << rsLog;
                 log() << "replSet See http://www.mongodb.org/display/DOCS/Resyncing+a+Very+Stale+Replica+Set+Member" << rsLog;
-                sethbmsg("error too stale to catch up");
+                sethbmsg("error RS102 too stale to catch up");
                 sleepsecs(120);
                 return;
             }
@@ -203,8 +250,8 @@ namespace mongo {
             OpTime ts = o["ts"]._opTime();
             long long h = o["h"].numberLong();
             if( ts != lastOpTimeWritten || h != lastH ) { 
-                log() << "TEMP our last op time written: " << lastOpTimeWritten.toStringPretty() << endl;
-                log() << "TEMP primary's GTE: " << ts.toStringPretty() << endl;
+                log(1) << "TEMP our last op time written: " << lastOpTimeWritten.toStringPretty() << endl;
+                log(1) << "TEMP primary's GTE: " << ts.toStringPretty() << endl;
                 /*
                 }*/
 
@@ -213,7 +260,13 @@ namespace mongo {
             }
         }
 
-        while( 1 ) { 
+        /* we have now checked if we need to rollback and we either don't have to or did it. */
+        {
+            OpTime minvalid;
+            tryToGoLiveAsASecondary(minvalid);
+        }
+
+        while( 1 ) {
             while( 1 ) {
                 if( !r.moreInCurrentBatch() ) { 
                     /* we need to occasionally check some things. between 
@@ -222,28 +275,13 @@ namespace mongo {
                     /* perhaps we should check this earlier? but not before the rollback checks. */
                     if( state().recovering() ) { 
                         /* can we go to RS_SECONDARY state?  we can if not too old and if minvalid achieved */
-                        bool golive = false;			
                         OpTime minvalid;
-                        {
-                            readlock lk("local.replset.minvalid");
-                            BSONObj mv;
-                            if( Helpers::getSingleton("local.replset.minvalid", mv) ) { 
-                                minvalid = mv["ts"]._opTime();
-                                if( minvalid <= lastOpTimeWritten ) { 
-                                    golive=true;
-                                }
-                            }
-                            else 
-                                golive = true; /* must have been the original member */
-                        }
+                        bool golive = ReplSetImpl::tryToGoLiveAsASecondary(minvalid);
                         if( golive ) {
-                            sethbmsg("");
-                            log() << "replSet SECONDARY" << rsLog;
-                            changeState(MemberState::RS_SECONDARY);
+                            ;
                         }
                         else { 
-                            sethbmsg(str::stream() << "still syncing, not yet to minValid optime " << minvalid.toString());
-                            //log() << "TEMP " << lastOpTimeWritten.toString() << rsLog;
+                            sethbmsg(str::stream() << "still syncing, not yet to minValid optime" << minvalid.toString());
                         }
 
                         /* todo: too stale capability */
@@ -271,12 +309,40 @@ namespace mongo {
                         syncApply(o);
                         _logOpObjRS(o);   /* with repl sets we write the ops to our oplog too: */                   
                     }
+                    int sd = myConfig().slaveDelay;
+                    if( sd ) { 
+                        const OpTime ts = o["ts"]._opTime();
+                        long long a = ts.getSecs();
+                        long long b = time(0);
+                        long long lag = b - a;
+                        long long sleeptime = sd - lag;
+                        if( sleeptime > 0 ) {
+                            uassert(12000, "rs slaveDelay differential too big check clocks and systems", sleeptime < 0x40000000);
+                            log() << "replSet temp slavedelay sleep:" << sleeptime << rsLog;
+                            if( sleeptime < 60 ) {
+                                sleepsecs((int) sleeptime);
+                            }
+                            else {
+                                // sleep(hours) would prevent reconfigs from taking effect & such!
+                                long long waitUntil = b + sleeptime;
+                                while( 1 ) {
+                                    sleepsecs(6);
+                                    if( time(0) >= waitUntil )
+                                        break;
+                                    if( box.getPrimary() != primary )
+                                        break;
+                                    if( myConfig().slaveDelay != sd ) // reconf
+                                        break;
+                                }
+                            }
+                        }
+                    }
                 }
             }
             r.tailCheck();
             if( !r.haveCursor() ) {
-                log() << "replSet TEMP end syncTail pass with " << hn << rsLog;
-                // TODO : reuse our cnonection to the primary.
+                log(1) << "replSet end syncTail pass with " << hn << rsLog;
+                // TODO : reuse our connection to the primary.
                 return;
             }
             if( box.getPrimary() != primary )
@@ -291,6 +357,10 @@ namespace mongo {
             sleepsecs(1);
             return;
         }
+        if( sp.state.fatal() ) { 
+            sleepsecs(5);
+            return;
+        }
 
         /* later, we can sync from up secondaries if we want. tbd. */
         if( sp.primary == 0 )

db/replpair.h

@@ -164,7 +164,7 @@ namespace mongo {
         if( replSet ) {
             /* todo: speed up the secondary case.  as written here there are 2 mutex entries, it can be 1. */
             if( isMaster() ) return;
-            notMasterUnless( pq.hasOption(QueryOption_SlaveOk) && theReplSet->isSecondary() );
+            notMasterUnless( pq.hasOption(QueryOption_SlaveOk) && theReplSet && theReplSet->isSecondary() );
         } else {
             notMasterUnless(isMaster() || pq.hasOption(QueryOption_SlaveOk) || replSettings.slave == SimpleSlave );
         }

db/restapi.cpp

@@ -266,8 +266,8 @@ namespace mongo {
             
             ss << "# databases: " << dbHolder.size() << '\n';
             
-            if( ClientCursor::byLocSize()>500 )
-                ss << "Cursors byLoc.size(): " << ClientCursor::byLocSize() << '\n';
+            if( ClientCursor::numCursors()>500 )
+                ss << "# Cursors: " << ClientCursor::numCursors() << '\n';
             
             ss << "\nreplication: ";
             if( *replInfo )

db/scanandorder.h

@@ -88,16 +88,17 @@ namespace mongo {
 
         void _add(BSONObj& k, BSONObj o, DiskLoc* loc) {
             if (!loc){
-                best.insert(make_pair(k,o));
+                best.insert(make_pair(k.getOwned(),o.getOwned()));
             } else {
                 BSONObjBuilder b;
                 b.appendElements(o);
                 b.append("$diskLoc", loc->toBSONObj());
-                best.insert(make_pair(k, b.obj()));
+                best.insert(make_pair(k.getOwned(), b.obj().getOwned()));
             }
         }
 
         void _addIfBetter(BSONObj& k, BSONObj o, BestMap::iterator i, DiskLoc* loc) {
+            /* todo : we don't correct approxSize here. */
             const BSONObj& worstBestKey = i->first;
             int c = worstBestKey.woCompare(k, order.pattern);
             if ( c > 0 ) {
@@ -124,7 +125,11 @@ namespace mongo {
             BSONObj k = order.getKeyFromObject(o);
             if ( (int) best.size() < limit ) {
                 approxSize += k.objsize();
-                uassert( 10128 ,  "too much key data for sort() with no index.  add an index or specify a smaller limit", approxSize < 1 * 1024 * 1024 );
+                approxSize += o.objsize();
+
+                /* note : adjust when bson return limit adjusts. note this limit should be a bit higher. */
+                uassert( 10128 ,  "too much data for sort() with no index.  add an index or specify a smaller limit", approxSize < 32 * 1024 * 1024 );
+
                 _add(k, o, loc);
                 return;
             }

db/security_commands.cpp

@@ -49,9 +49,7 @@ namespace mongo {
     class CmdGetNonce : public Command {
     public:
         virtual bool requiresAuth() { return false; }
-        virtual bool logTheOp() {
-            return false;
-        }
+        virtual bool logTheOp() { return false; }
         virtual bool slaveOk() const {
             return true;
         }

db/stats/snapshots.cpp

@@ -163,7 +163,7 @@ namespace mongo {
             ss << usage.count;
             ss << "</td><td>";
             double per = 100 * ((double)usage.time)/elapsed;
-            ss << setprecision(4) << fixed << per << "%";
+            ss << setprecision(1) << fixed << per << "%";
             ss << "</td>";
         }
 

db/update.cpp

@@ -517,17 +517,19 @@ namespace mongo {
         DEBUGUPDATE( "\t\t createNewFromMods root: " << root );
         BSONObjIteratorSorted es( obj );
         BSONElement e = es.next();
-
+        
         ModStateHolder::iterator m = _mods.lower_bound( root );
-        ModStateHolder::iterator mend = _mods.lower_bound( root + '{' );
-
+        StringBuilder buf(root.size() + 2 );
+        buf << root << (char)255;
+        ModStateHolder::iterator mend = _mods.lower_bound( buf.str() );
+        
         set<string> onedownseen;
         
         while ( e.type() && m != mend ){
             string field = root + e.fieldName();
             FieldCompareResult cmp = compareDottedFieldNames( m->second.m->fieldName , field );
 
-            DEBUGUPDATE( "\t\t\t field:" << field << "\t mod:" << m->second.m->fieldName << "\t cmp:" << cmp );
+            DEBUGUPDATE( "\t\t\t field:" << field << "\t mod:" << m->second.m->fieldName << "\t cmp:" << cmp << "\t short: " << e.fieldName() );
             
             switch ( cmp ){
                 
@@ -550,6 +552,13 @@ namespace mongo {
                     e = es.next();
                     m++;
                 }
+                else {
+                    // this is a very weird case
+                    // have seen it in production, but can't reproduce
+                    // this assert prevents an inf. loop
+                    // but likely isn't the correct solution
+                    assert(0);
+                }
                 continue;
             }
             case LEFT_BEFORE: // Mod on a field that doesn't exist
@@ -613,6 +622,8 @@ namespace mongo {
             BSONObjIteratorSorted i( query );
             while ( i.more() ){
                 BSONElement e = i.next();
+                if ( e.fieldName()[0] == '$' ) // for $atomic and anything else we add
+                    continue;
 
                 if ( e.type() == Object && e.embeddedObject().firstElement().fieldName()[0] == '$' ){
                     // this means this is a $gt type filter, so don't make part of the new object

dbtests/basictests.cpp

@@ -234,7 +234,7 @@ namespace BasicTests {
                     if ( y < 1000 || y > 2500 ){
                         cout << "sleeptest y: " << y << endl;
                         ASSERT( y >= 1000 );
-                        ASSERT( y <= 100000 );
+                        /* ASSERT( y <= 100000 ); */
                     }
                 }
             }
@@ -319,6 +319,9 @@ namespace BasicTests {
     class LexNumCmp {
     public:
         void run() {
+            
+            ASSERT( ! isNumber( (char)255 ) );
+
             ASSERT_EQUALS( 0, lexNumCmp( "a", "a" ) );
             ASSERT_EQUALS( -1, lexNumCmp( "a", "aa" ) );
             ASSERT_EQUALS( 1, lexNumCmp( "aa", "a" ) );
@@ -346,7 +349,7 @@ namespace BasicTests {
             ASSERT_EQUALS( -1, lexNumCmp( "f12f", "f12g" ) );
             ASSERT_EQUALS( 1, lexNumCmp( "f12g", "f12f" ) );
             ASSERT_EQUALS( 1, lexNumCmp( "aa{", "aab" ) );
-            ASSERT_EQUALS( 1, lexNumCmp( "aa{", "aa1" ) );
+            ASSERT_EQUALS( -1, lexNumCmp( "aa{", "aa1" ) );
             ASSERT_EQUALS( -1, lexNumCmp( "a1{", "a11" ) );
             ASSERT_EQUALS( 1, lexNumCmp( "a1{a", "a1{" ) );
             ASSERT_EQUALS( -1, lexNumCmp( "a1{", "a1{a" ) );
@@ -355,6 +358,38 @@ namespace BasicTests {
             
             ASSERT_EQUALS( -1 , lexNumCmp( "a.0" , "a.1" ) );
             ASSERT_EQUALS( -1 , lexNumCmp( "a.0.b" , "a.1" ) );
+
+            ASSERT_EQUALS( -1 , lexNumCmp( "b." , "b.|" ) );
+            ASSERT_EQUALS( -1 , lexNumCmp( "b.0e" , (string("b.") + (char)255).c_str() ) );
+            ASSERT_EQUALS( -1 , lexNumCmp( "b." , "b.0e" ) );
+
+            ASSERT_EQUALS( 0, lexNumCmp( "238947219478347782934718234", "238947219478347782934718234")); 
+            ASSERT_EQUALS( 0, lexNumCmp( "000238947219478347782934718234", "238947219478347782934718234")); 
+            ASSERT_EQUALS( 1, lexNumCmp( "000238947219478347782934718235", "238947219478347782934718234")); 
+            ASSERT_EQUALS( -1, lexNumCmp( "238947219478347782934718234", "238947219478347782934718234.1")); 
+            ASSERT_EQUALS( 0, lexNumCmp( "238", "000238")); 
+            ASSERT_EQUALS( 0, lexNumCmp( "002384", "0002384")); 
+            ASSERT_EQUALS( 0, lexNumCmp( "00002384", "0002384")); 
+            ASSERT_EQUALS( 0, lexNumCmp( "0", "0")); 
+            ASSERT_EQUALS( 0, lexNumCmp( "0000", "0")); 
+            ASSERT_EQUALS( 0, lexNumCmp( "0", "000"));
+            ASSERT_EQUALS( -1, lexNumCmp( "0000", "0.0"));
+            ASSERT_EQUALS( 1, lexNumCmp( "2380", "238")); 
+            ASSERT_EQUALS( 1, lexNumCmp( "2385", "2384")); 
+            ASSERT_EQUALS( 1, lexNumCmp( "2385", "02384")); 
+            ASSERT_EQUALS( 1, lexNumCmp( "2385", "002384")); 
+            ASSERT_EQUALS( -1, lexNumCmp( "123.234.4567", "00238")); 
+            ASSERT_EQUALS( 0, lexNumCmp( "123.234", "00123.234")); 
+            ASSERT_EQUALS( 0, lexNumCmp( "a.123.b", "a.00123.b")); 
+            ASSERT_EQUALS( 1, lexNumCmp( "a.123.b", "a.b.00123.b")); 
+            ASSERT_EQUALS( -1, lexNumCmp( "a.00.0", "a.0.1")); 
+            ASSERT_EQUALS( 0, lexNumCmp( "01.003.02", "1.3.2")); 
+            ASSERT_EQUALS( -1, lexNumCmp( "1.3.2", "10.300.20")); 
+            ASSERT_EQUALS( 0, lexNumCmp( "10.300.20", "000000000000010.0000300.000000020")); 
+            ASSERT_EQUALS( 0, lexNumCmp( "0000a", "0a")); 
+            ASSERT_EQUALS( -1, lexNumCmp( "a", "0a")); 
+            ASSERT_EQUALS( -1, lexNumCmp( "000a", "001a")); 
+            ASSERT_EQUALS( 0, lexNumCmp( "010a", "0010a")); 
         }
     };
 

dbtests/jsobjtests.cpp

@@ -389,6 +389,35 @@ namespace JsobjTests {
             }
         };
 
+        class ToStringNumber {
+        public:
+            
+            void run(){
+                BSONObjBuilder b;
+                b.append( "a" , (int)4 );
+                b.append( "b" , (double)5 );
+                b.append( "c" , (long long)6 );
+                
+                b.append( "d" , 123.456789123456789123456789123456789 );
+                b.append( "e" , 123456789.123456789123456789123456789 );
+                b.append( "f" , 1234567891234567891234.56789123456789 );
+
+                b.append( "g" , -123.456 );
+                
+                BSONObj x = b.obj();
+                ASSERT_EQUALS( "4", x["a"].toString( false , true ) );
+                ASSERT_EQUALS( "5.0", x["b"].toString( false , true ) );
+                ASSERT_EQUALS( "6", x["c"].toString( false , true ) );                
+
+                ASSERT_EQUALS( "123.4567891234568" , x["d"].toString( false , true ) );
+                ASSERT_EQUALS( "123456789.1234568" , x["e"].toString( false , true ) );
+                // ASSERT_EQUALS( "1.234567891234568e+21" , x["f"].toString( false , true ) ); // windows and *nix are different - TODO, work around for test or not bother?
+                
+                ASSERT_EQUALS( "-123.456" , x["g"].toString( false , true ) );
+
+            }
+        };
+
         class NullString {
         public:
             void run() {
@@ -1693,6 +1722,7 @@ namespace JsobjTests {
             add< BSONObjTests::AppendIntOrLL >();
             add< BSONObjTests::AppendNumber >();
             add< BSONObjTests::ToStringArray >();
+            add< BSONObjTests::ToStringNumber >();
             add< BSONObjTests::NullString >();
             add< BSONObjTests::Validation::BadType >();
             add< BSONObjTests::Validation::EooBeforeEnd >();

dbtests/jstests.cpp

@@ -541,7 +541,7 @@ namespace JSTests {
             ASSERT( s->exec( "c = {c:a.a.toString()}", "foo", false, true, false ) );
             out = s->getObject( "c" );
             stringstream ss;
-            ss << "NumberLong( \"" << val << "\" )";
+            ss << "NumberLong(\"" << val << "\")";
             ASSERT_EQUALS( ss.str(), out.firstElement().valuestr() );
 
             ASSERT( s->exec( "d = {d:a.a.toNumber()}", "foo", false, true, false ) );

dbtests/pdfiletests.cpp

@@ -25,6 +25,12 @@
 
 #include "dbtests.h"
 
+namespace mongo {
+    // here because we don't nesc. want to expose yet
+    int initialExtentSize(int len);
+    int followupExtentSize(int len, int lastExtentLen);
+}
+
 namespace PdfileTests {
 
     namespace ScanCapped {
@@ -301,6 +307,31 @@ namespace PdfileTests {
             }
         };
     } // namespace Insert
+
+    class ExtentSizing {
+    public:
+        struct SmallFilesControl {
+            SmallFilesControl(){
+                old = cmdLine.smallfiles;
+                cmdLine.smallfiles = false;
+            }
+            ~SmallFilesControl(){
+                cmdLine.smallfiles = old;
+            }
+            bool old;
+        };
+        void run(){
+            SmallFilesControl c;
+            // test that no matter what we start with, we always get to max extent size
+            for ( int obj=16; obj<(4*1024*1024); obj += 111 ){
+                int sz = initialExtentSize( obj );
+                for ( int i=0; i<100; i++ ){
+                    sz = followupExtentSize( obj , sz );
+                }
+                ASSERT_EQUALS( Extent::maxSize() , sz );
+            }
+        }
+    };
     
     class All : public Suite {
     public:
@@ -321,6 +352,7 @@ namespace PdfileTests {
             add< ScanCapped::FirstInExtent >();
             add< ScanCapped::LastInExtent >();
             add< Insert::UpdateDate >();
+            add< ExtentSizing >();
         }
     } myall;
 

debian/changelog

@@ -1,8 +1,55 @@
+mongodb (1.6.5) unstable; urgency=low
+
+  * full change log http://jira.mongodb.org/browse/SERVER/fixforversion/10207
+
+  -- Richard Kreuter <richard@10gen.com>  Tue, 7 Dec 2010 16:56:28 -0500
+
+mongodb (1.6.4) unstable; urgency=low
+
+  * replica_sets shell helpers
+  * sharding chunk safety, yielding during migrate cleanup
+  * full change log http://jira.mongodb.org/browse/SERVER/fixforversion/10191
+
+  -- Richard Kreuter <richard@10gen.com>  Tue, 26 Oct 2010 16:56:28 -0500
+
+mongodb (1.6.3) unstable; urgency=low
+
+  * replica_sets slavedelay, rollback
+  * sharding optimization for larger than ram data sets
+  * full change log http://jira.mongodb.org/browse/SERVER/fixforversion/10190
+
+  -- Richard Kreuter <richard@10gen.com>  Thu, 23 Sep 2010 16:56:28 -0500
+
+mongodb (1.6.2) unstable; urgency=low
+
+  * replica_sets some fixes
+  * sharding some fixes with rs
+  * full change log http://jira.mongodb.org/browse/SERVER/fixforversion/10187
+
+  -- Richard Kreuter <richard@10gen.com>  Wed, 1 Sep 2010 16:56:28 -0500
+
+
+mongodb (1.6.1) unstable; urgency=low
+
+  * replica_sets some fixes
+  * sharding some fixes with rs
+  * full change log http://jira.mongodb.org/browse/SERVER/fixforversion/10183
+
+  -- Richard Kreuter <richard@10gen.com>  Tue, 17 Aug 2010 16:56:28 -0500
+
+mongodb (1.6.0) unstable; urgency=low
+
+  * sharding stable
+  * replica_sets stable
+
+  -- Richard Kreuter <richard@10gen.com>  Thu, 05 Aug 2010 16:56:28 -0500
+
 mongodb (1.5.8) unstable; urgency=low
 
   * sharding lots of changes
   * replica_sets lots of changes
 
+  -- Richard Kreuter <richard@10gen.com>  Tue, 03 Aug 2010 16:56:28 -0500
 
 mongodb (1.5.7) unstable; urgency=low
 

doxygenConfig

@@ -3,7 +3,7 @@
 #---------------------------------------------------------------------------
 DOXYFILE_ENCODING      = UTF-8
 PROJECT_NAME           = MongoDB
-PROJECT_NUMBER         = 1.5.8
+PROJECT_NUMBER         = 1.6.6-pre-
 OUTPUT_DIRECTORY       = docs/doxygen
 CREATE_SUBDIRS         = NO
 OUTPUT_LANGUAGE        = English

jstests/basic3.js

@@ -1,5 +1,5 @@
 
-t = db.getCollection( "foo" );
+t = db.getCollection( "foo_basic3" );
 
 t.find( { "a.b" : 1 } ).toArray();
 

jstests/basic9.js

@@ -1,5 +1,5 @@
 
-t = db.getCollection( "foo" );
+t = db.getCollection( "foo_basic9" );
 
 t.save( { "foo$bar" : 5 } );
 

jstests/datasize3.js

@@ -0,0 +1,32 @@
+
+t = db.datasize3;
+t.drop()
+
+function run( options ){
+    var c = { dataSize : "test.datasize3" };
+    if ( options )
+        Object.extend( c , options );
+    return db.runCommand( c );
+}
+
+t.insert( { x : 1 } )
+
+a = run()
+b = run( { estimate : true } )
+
+assert.eq( a.size , b.size );
+
+
+t.ensureIndex( { x : 1 } )
+
+for ( i=2; i<100; i++ )
+    t.insert( { x : i } )
+
+a = run( { min : { x : 20 } , max : { x : 50 } } )
+b = run( { min : { x : 20 } , max : { x : 50 } , estimate : true } )
+
+assert.eq( a.size , b.size );
+
+
+
+

jstests/dbadmin.js

@@ -21,4 +21,10 @@ assert( res.databases && res.databases.length > 0 , "listDatabases 1 " + tojson(
 x = db._adminCommand( "ismaster" );
 assert( x.ismaster , "ismaster failed: " + tojson( x ) )
 
+before = db.runCommand( "serverStatus" )
+sleep( 5000 )
+after = db.runCommand( "serverStatus" )
+assert.lt( 3 , after.uptimeEstimate , "up1" )
+assert.gt( after.uptimeEstimate , before.uptimeEstimate , "up2" )
+
 // TODO: add more tests here

jstests/dbcase.js

@@ -1,6 +1,6 @@
 
-a = db.getSisterDB( "test_dbnamea" )
-b = db.getSisterDB( "test_dbnameA" )
+a = db.getSisterDB( "dbcasetest_dbnamea" )
+b = db.getSisterDB( "dbcasetest_dbnameA" )
 
 a.dropDatabase();
 b.dropDatabase();

jstests/delx.js

@@ -0,0 +1,31 @@
+
+a = db.getSisterDB("delxa" )
+b = db.getSisterDB("delxb" )
+
+function setup( mydb ){
+    mydb.dropDatabase();
+    for ( i=0; i<100; i++ ){
+        mydb.foo.insert( { _id : i } );
+    }
+    mydb.getLastError();
+}
+
+setup( a );
+setup( b );
+
+assert.eq( 100 , a.foo.find().itcount() , "A1" )
+assert.eq( 100 , b.foo.find().itcount() , "A2" )
+
+x = a.foo.find().sort( { _id : 1 } ).batchSize( 60 )
+y = b.foo.find().sort( { _id : 1 } ).batchSize( 60 )
+
+x.next();
+y.next();
+
+a.foo.remove( { _id : { $gt : 50 } } );
+
+assert.eq( 51 , a.foo.find().itcount() , "B1" )
+assert.eq( 100 , b.foo.find().itcount() , "B2" )
+
+assert.eq( 59 , x.itcount() , "C1" ) 
+assert.eq( 99 , y.itcount() , "C2" );  // this was asserting because ClientCursor byLoc doesn't take db into consideration

jstests/disk/diskfull.js

@@ -14,9 +14,10 @@ if ( !doIt ) {
 if ( doIt ) {
     port = allocatePorts( 1 )[ 0 ];
     m = startMongoProgram( "mongod", "--port", port, "--dbpath", "/data/db/diskfulltest", "--nohttpinterface", "--bind_ip", "127.0.0.1" );
-    m.getDB( "diskfulltest" ).getCollection( "diskfulltest" ).save( { a: 6 } );
+    c = m.getDB( "diskfulltest" ).getCollection( "diskfulltest" )
+    c.save( { a: 6 } );
     assert.soon( function() { return rawMongoProgramOutput().match( /file allocation failure/ ); }, "didn't see 'file allocation failure'" );
-    assert.soon( function() { return rawMongoProgramOutput().match( /Caught Assertion in insert , continuing/ ); }, "didn't see 'Caught Assertion...'" );
+    assert.isnull( c.findOne() , "shouldn't exist" );
     sleep( 3000 );
     m2 = new Mongo( m.host );
     printjson( m2.getDBs() );

jstests/evalc.js

@@ -0,0 +1,32 @@
+t = db.jstests_evalc;
+t.drop();
+
+for( i = 0; i < 10; ++i ) {
+    t.save( {i:i} );
+}
+
+// SERVER-1610
+
+function op() {
+    uri = db.runCommand( "whatsmyuri" ).you;
+    printjson( uri );
+    p = db.currentOp().inprog;
+    for ( var i in p ) {
+        var o = p[ i ];
+        if ( o.client == uri ) {
+            print( "found it" );
+            return o.opid;
+        }
+    }
+    return -1;
+}
+
+s = startParallelShell( "print( 'starting forked:' + Date() ); for ( i=0; i<500000; i++ ){ db.currentOp(); } print( 'ending forked:' + Date() ); " )
+
+print( "starting eval: " + Date() )
+for ( i=0; i<20000; i++ ){
+    db.eval( "db.jstests_evalc.count( {i:10} );" );
+}
+print( "end eval: " + Date() )
+
+s();

jstests/geo_circle1.js

@@ -36,7 +36,7 @@ for ( i=0; i<searches.length; i++ ){
     //printjson( Array.sort( t.find(q).map( function(z){ return z._id; } ) ) )
     
     assert.eq( correct[i].length , t.find( q ).itcount() , "itcount : " + tojson( searches[i] ) );
-    assert.eq( correct[i].length , t.find( q ).itcount() , "count : " + tojson( searches[i] ) );
+    assert.eq( correct[i].length , t.find( q ).count() , "count : " + tojson( searches[i] ) );
     assert.gt( correct[i].length * 2 , t.find(q).explain().nscanned , "nscanned : " + tojson( searches[i] ) )
 }
 

jstests/geo_queryoptimizer.js

@@ -0,0 +1,27 @@
+
+t = db.geo_qo1;
+t.drop()
+
+t.ensureIndex({loc:"2d"}) 
+
+t.insert({'issue':0}) 
+t.insert({'issue':1}) 
+t.insert({'issue':2}) 
+t.insert({'issue':2, 'loc':[30.12,-118]}) 
+t.insert({'issue':1, 'loc':[30.12,-118]}) 
+t.insert({'issue':0, 'loc':[30.12,-118]}) 
+
+assert.eq( 6 , t.find().itcount() , "A1" )
+
+assert.eq( 2 , t.find({'issue':0}).itcount() , "A2" )
+
+assert.eq( 1 , t.find({'issue':0,'loc':{$near:[30.12,-118]}}).itcount() , "A3" )
+
+assert.eq( 2 , t.find({'issue':0}).itcount() , "B1" )
+
+assert.eq( 6 , t.find().itcount() , "B2" )
+
+assert.eq( 2 , t.find({'issue':0}).itcount() , "B3" )
+
+assert.eq( 1 , t.find({'issue':0,'loc':{$near:[30.12,-118]}}).itcount() , "B4" )
+

jstests/mr_sort.js

@@ -0,0 +1,44 @@
+
+t = db.mr_sort;
+t.drop()
+
+t.ensureIndex( { x : 1 } )
+
+t.insert( { x : 1 } )
+t.insert( { x : 10 } )
+t.insert( { x : 2 } )
+t.insert( { x : 9 } )
+t.insert( { x : 3 } )
+t.insert( { x : 8 } )
+t.insert( { x : 4 } )
+t.insert( { x : 7 } )
+t.insert( { x : 5 } )
+t.insert( { x : 6 } )
+
+m = function(){
+    emit( "a" , this.x )
+}
+
+r = function( k , v ){
+    return Array.sum( v )
+}
+
+
+res = t.mapReduce( m , r );
+x = res.convertToSingleObject();
+res.drop();
+assert.eq( { "a" : 55 } , x , "A1" )
+
+res = t.mapReduce( m , r , { query : { x : { $lt : 3 } } } )
+x = res.convertToSingleObject();
+res.drop();
+assert.eq( { "a" : 3 } , x , "A2" )
+
+res = t.mapReduce( m , r , { sort : { x : 1 } , limit : 2 } );
+x = res.convertToSingleObject();
+res.drop();
+assert.eq( { "a" : 3 } , x , "A3" )
+
+
+
+

jstests/numberlong.js

@@ -4,50 +4,50 @@ n = new NumberLong( 4 );
 assert.eq.automsg( "4", "n" );
 assert.eq.automsg( "4", "n.toNumber()" );
 assert.eq.automsg( "8", "n + 4" );
-assert.eq.automsg( "'NumberLong( 4 )'", "n.toString()" );
-assert.eq.automsg( "'NumberLong( 4 )'", "tojson( n )" );
+assert.eq.automsg( "'NumberLong(4)'", "n.toString()" );
+assert.eq.automsg( "'NumberLong(4)'", "tojson( n )" );
 a = {}
 a.a = n;
 p = tojson( a );
-assert.eq.automsg( "'{ \"a\" : NumberLong( 4 ) }'", "p" );
+assert.eq.automsg( "'{ \"a\" : NumberLong(4) }'", "p" );
 
-assert.eq.automsg( "NumberLong( 4 )", "eval( tojson( NumberLong( 4 ) ) )" );
+assert.eq.automsg( "NumberLong(4 )", "eval( tojson( NumberLong( 4 ) ) )" );
 assert.eq.automsg( "a", "eval( tojson( a ) )" );
 
 n = new NumberLong( -4 );
 assert.eq.automsg( "-4", "n" );
 assert.eq.automsg( "-4", "n.toNumber()" );
 assert.eq.automsg( "0", "n + 4" );
-assert.eq.automsg( "'NumberLong( -4 )'", "n.toString()" );
-assert.eq.automsg( "'NumberLong( -4 )'", "tojson( n )" );
+assert.eq.automsg( "'NumberLong(-4)'", "n.toString()" );
+assert.eq.automsg( "'NumberLong(-4)'", "tojson( n )" );
 a = {}
 a.a = n;
 p = tojson( a );
-assert.eq.automsg( "'{ \"a\" : NumberLong( -4 ) }'", "p" );
+assert.eq.automsg( "'{ \"a\" : NumberLong(-4) }'", "p" );
 
 // too big to fit in double
 n = new NumberLong( "11111111111111111" );
 assert.eq.automsg( "11111111111111112", "n.toNumber()" );
 assert.eq.automsg( "11111111111111116", "n + 4" );
-assert.eq.automsg( "'NumberLong( \"11111111111111111\" )'", "n.toString()" );
-assert.eq.automsg( "'NumberLong( \"11111111111111111\" )'", "tojson( n )" );
+assert.eq.automsg( "'NumberLong(\"11111111111111111\")'", "n.toString()" );
+assert.eq.automsg( "'NumberLong(\"11111111111111111\")'", "tojson( n )" );
 a = {}
 a.a = n;
 p = tojson( a );
-assert.eq.automsg( "'{ \"a\" : NumberLong( \"11111111111111111\" ) }'", "p" );
+assert.eq.automsg( "'{ \"a\" : NumberLong(\"11111111111111111\") }'", "p" );
 
-assert.eq.automsg( "NumberLong( '11111111111111111' )", "eval( tojson( NumberLong( '11111111111111111' ) ) )" );
+assert.eq.automsg( "NumberLong('11111111111111111' )", "eval( tojson( NumberLong( '11111111111111111' ) ) )" );
 assert.eq.automsg( "a", "eval( tojson( a ) )" );
 
 n = new NumberLong( "-11111111111111111" );
 assert.eq.automsg( "-11111111111111112", "n.toNumber()" );
 assert.eq.automsg( "-11111111111111108", "n + 4" );
-assert.eq.automsg( "'NumberLong( \"-11111111111111111\" )'", "n.toString()" );
-assert.eq.automsg( "'NumberLong( \"-11111111111111111\" )'", "tojson( n )" );
+assert.eq.automsg( "'NumberLong(\"-11111111111111111\")'", "n.toString()" );
+assert.eq.automsg( "'NumberLong(\"-11111111111111111\")'", "tojson( n )" );
 a = {}
 a.a = n;
 p = tojson( a );
-assert.eq.automsg( "'{ \"a\" : NumberLong( \"-11111111111111111\" ) }'", "p" );
+assert.eq.automsg( "'{ \"a\" : NumberLong(\"-11111111111111111\") }'", "p" );
 
 // parsing
 assert.throws.automsg( function() { new NumberLong( "" ); } );

jstests/or8.js

@@ -14,3 +14,15 @@ t.find({ $or: [ { a: {$in:[]} } ] } ).toArray();
 assert.eq.automsg( "2", "t.find({ $or: [ { a: {$in:[]} }, {a:1}, {a:3} ] } ).toArray().length" );
 assert.eq.automsg( "2", "t.find({ $or: [ {a:1}, { a: {$in:[]} }, {a:3} ] } ).toArray().length" );
 assert.eq.automsg( "2", "t.find({ $or: [ {a:1}, {a:3}, { a: {$in:[]} } ] } ).toArray().length" );
+
+// nested negate field
+
+t.drop();
+t.save( {a:{b:1,c:1}} );
+t.ensureIndex( { 'a.b':1 } );
+t.ensureIndex( { 'a.c':1 } );
+assert.eq( 1, t.find( {$or: [ { 'a.b':1 }, { 'a.c':1 } ] } ).itcount() );
+
+t.remove();
+t.save( {a:[{b:1,c:1},{b:2,c:1}]} );
+assert.eq( 1, t.find( {$or: [ { 'a.b':2 }, { 'a.c':1 } ] } ).itcount() );

jstests/remove3.js

@@ -14,5 +14,5 @@ assert.eq( 4 , t.count() , "B" );
 t.remove( { _id : 5 } );
 assert.eq( 3 , t.count() , "C" );
 
-t.remove( { _id : { $lt : 8 } } , "D" );
+t.remove( { _id : { $lt : 8 } } );
 assert.eq( 1 , t.count() , "D" );

jstests/remove_justone.js

@@ -0,0 +1,16 @@
+
+t = db.remove_justone
+t.drop()
+
+t.insert( { x : 1 } )
+t.insert( { x : 1 } )
+t.insert( { x : 1 } )
+t.insert( { x : 1 } )
+
+assert.eq( 4 , t.count() )
+
+t.remove( { x : 1 } , true )
+assert.eq( 3 , t.count() )
+
+t.remove( { x : 1 } )
+assert.eq( 0 , t.count() )

jstests/repl/basic1.js

@@ -141,6 +141,14 @@ assert.soon( function(){ return am.lotOfIndexes.getIndexes().length == as.lotOfI
 
 assert.eq( am.lotOfIndexes.getIndexes().length , as.lotOfIndexes.getIndexes().length , "lots of indexes b" )
 
+// multi-update with $inc
+
+am.mu1.update( { _id : 1 , $atomic : 1 } , { $inc : { x : 1 } } , true , true )
+x = { _id : 1 , x : 1 }
+assert.eq( x , am.mu1.findOne() , "mu1" );
+assert.soon( function(){ z = as.mu1.findOne(); printjson( z ); return friendlyEqual( x , z ); } , "mu2" )
+
+
 
 rt.stop();
 

jstests/repl/mastermaster1.js

@@ -16,9 +16,13 @@ ldb = left.getDB( "test" )
 rdb = right.getDB( "test" )
 
 ldb.foo.insert( { _id : 1 , x : "eliot" } )
-ldb.runCommand( { getlasterror : 1 , w : 2 } )
+var result = ldb.runCommand( { getlasterror : 1 , w : 2 , wtimeout : 20000 } );
+printjson(result);
 rdb.foo.insert( { _id : 2 , x : "sara" } )
-rdb.runCommand( { getlasterror : 1 , w : 2 } )
+result = rdb.runCommand( { getlasterror : 1 , w : 2 , wtimeout : 20000 } )
+printjson(result);
+
+print( "check 3" )
 
 assert.eq( 2 , ldb.foo.count() , "B1" )
 assert.eq( 2 , rdb.foo.count() , "B2" )

jstests/repl/pair3.js

@@ -12,32 +12,40 @@ ismaster = function( n ) {
 // bring up node connections before arbiter connections so that arb can forward to node when expected
 connect = function() {
     if ( lp == null ) {
+        print("connecting lp");
         lp = startMongoProgram( "mongobridge", "--port", lpPort, "--dest", "localhost:" + lPort );
     }
     if ( rp == null ) {
+        print("connecting rp");
         rp = startMongoProgram( "mongobridge", "--port", rpPort, "--dest", "localhost:" + rPort );
     }
     if ( al == null ) {
+        print("connecting al");
         al = startMongoProgram( "mongobridge", "--port", alPort, "--dest", "localhost:" + aPort );
     }
     if ( ar == null ) {
+        print("connecting ar");
         ar = startMongoProgram( "mongobridge", "--port", arPort, "--dest", "localhost:" + aPort );
     }
 }
 
-disconnectNode = function( mongo ) {
+disconnectNode = function( mongo ) {    
     if ( lp ) {
+        print("disconnecting lp: "+lpPort);
         stopMongoProgram( lpPort );
         lp = null;
     }
     if ( rp ) {
+        print("disconnecting rp: "+rpPort);
         stopMongoProgram( rpPort );
         rp = null;
     }
     if ( mongo.host.match( new RegExp( "^127.0.0.1:" + lPort + "$" ) ) ) {
+        print("disconnecting al: "+alPort);
         stopMongoProgram( alPort );
         al = null;
     } else if ( mongo.host.match( new RegExp( "^127.0.0.1:" + rPort + "$" ) ) ) {
+        print("disconnecting ar: "+arPort);
         stopMongoProgram( arPort );
         ar = null;
     } else {
@@ -64,47 +72,47 @@ doTest1 = function() {
 
     pair = new ReplPair( l, r, a );
 
-    // normal startup
+    print("normal startup");
     pair.start();
     pair.waitForSteadyState();
     
-    // disconnect slave
+    print("disconnect slave");
     disconnectNode( pair.slave() );
     pair.waitForSteadyState( [ 1, -3 ], pair.master().host );
     
-    // disconnect master
+    print("disconnect master");
     disconnectNode( pair.master() );
     pair.waitForSteadyState( [ -3, -3 ] );
     
-    // reconnect
+    print("reconnect");
     connect();
     pair.waitForSteadyState();
 
-    // disconnect master
+    print("disconnect master");
     disconnectNode( pair.master() );
     pair.waitForSteadyState( [ 1, -3 ], pair.slave().host, true );
     
-    // disconnect new master
+    print("disconnect new master");
     disconnectNode( pair.master() );
     pair.waitForSteadyState( [ -3, -3 ] );
     
-    // reconnect
+    print("reconnect");
     connect();
     pair.waitForSteadyState();
     
-    // disconnect slave
+    print("disconnect slave");
     disconnectNode( pair.slave() );
     pair.waitForSteadyState( [ 1, -3 ], pair.master().host );
     
-    // reconnect slave
+    print("reconnect slave");
     connect();
     pair.waitForSteadyState( [ 1, 0 ], pair.master().host );
     
-    // disconnect master
+    print("disconnect master");
     disconnectNode( pair.master() );
     pair.waitForSteadyState( [ 1, -3 ], pair.slave().host, true );
     
-    // reconnect old master
+    print("reconnect old master");
     connect();
     pair.waitForSteadyState( [ 1, 0 ], pair.master().host );
 

jstests/repl/repl10.js

@@ -26,13 +26,15 @@ doTest = function( signal ) {
     am.save( {i:2} );
     assert.eq( 2, am.count() );
     sleep( 3000 );
-    
-    rt.stop( true, signal );
-    sleep( 3000 );
     assert.eq( 1, s.getDB( baseName ).a.count() );
 
+    soonCount( 2 );
+
     rt.stop();
 }
 
-doTest( 15 ); // SIGTERM
-doTest( 9 );  // SIGKILL
+print("repl10.js dotest(15)");
+doTest(15); // SIGTERM
+print("repl10.js dotest(15)");
+doTest(9);  // SIGKILL
+print("repl10.js SUCCESS");

jstests/repl/repl12.js

@@ -0,0 +1,47 @@
+// SERVER-1626
+// check for initial sync of multiple db's
+
+function debug( x ) {
+    print( "DEBUG:" + tojson( x ) );
+}
+
+rt = new ReplTest( "repl12tests" );
+
+m = rt.start( true );
+
+usedDBs = []
+
+a = "a"
+for( i = 0; i < 3; ++i ) {
+    usedDBs.push( a )
+    m.getDB( a ).c.save( {} );
+    a += "a";
+}
+m.getDB(a).getLastError();
+
+//print("\n\n\n DB NAMES MASTER:");
+//printjson(m.getDBNames());
+
+var z = 10500;
+print("sleeping " + z + "ms");
+sleep(z);
+
+s = rt.start(false);
+
+function countHave(){
+    var have = 0;
+    for ( var i=0; i<usedDBs.length; i++ ){
+        if ( s.getDB( usedDBs[i] ).c.findOne() )
+            have++;
+    }
+    return have;
+}
+
+assert.soon( 
+    function() { 
+        var c = countHave();
+        debug( "count: " + c ); 
+        return c == 3; } 
+);
+
+//printjson(s.getDBNames());

jstests/repl/replacePeer2.js

@@ -44,15 +44,8 @@ doTest = function( signal ) {
     checkWrite( rp.master(), rp.slave() );
 
     // allow slave to finish initial sync
-    assert.soon( 
-        function() { 
-            var res = rp.slave().getDB( "admin" ).runCommand( {replacepeer:1} );
-            if ( res.ok == 1 )
-                return true;
-            printjson( res );
-            return false;
-        } 
-    );
+    var res = rp.slave().getDB( "admin" ).runCommand( {replacepeer:1} );
+    assert( res.ok , "replacepeer didn't finish: " + tojson( res ) );
     
     // Should not be saved to slave.
     writeOne( rp.master() );

jstests/repl/snapshot2.js

@@ -1,4 +1,6 @@
-// Test SERVER-623 - starting repl peer from a new snapshot of master
+// Test SERVER-623 - starting repl peer from a new snapshot of master
+
+print("snapshot2.js 1 -----------------------------------------------------------");
 
 ports = allocatePorts( 3 );
 
@@ -7,21 +9,37 @@ var basePath = "/data/db/" + baseName;
 
 a = new MongodRunner( ports[ 0 ], basePath + "-arbiter" );
 l = new MongodRunner( ports[ 1 ], basePath + "-left", "127.0.0.1:" + ports[ 2 ], "127.0.0.1:" + ports[ 0 ] );
-r = new MongodRunner( ports[ 2 ], basePath + "-right", "127.0.0.1:" + ports[ 1 ], "127.0.0.1:" + ports[ 0 ] ); 
+r = new MongodRunner( ports[ 2 ], basePath + "-right", "127.0.0.1:" + ports[ 1 ], "127.0.0.1:" + ports[ 0 ] );
+
+print("snapshot2.js 2 -----------------------------------------------------------");
 
-rp = new ReplPair( l, r, a );
-rp.start();
-rp.waitForSteadyState();
+rp = new ReplPair(l, r, a);
+rp.start();
+print("snapshot2.js 3 -----------------------------------------------------------");
+rp.waitForSteadyState();
+
+print("snapshot2.js 4 -----------------------------------------------------------");
 
 big = new Array( 2000 ).toString(); // overflow oplog, so test can't pass supriously
-rp.slave().setSlaveOk();
-for( i = 0; i < 500; ++i ) {
-    rp.master().getDB( baseName )[ baseName ].save( { _id: new ObjectId(), i: i, b: big } );
-    if ( i % 250 == 249 ) {
-        assert.soon( function() { return i+1 == rp.slave().getDB( baseName )[ baseName ].count(); } );    
+rp.slave().setSlaveOk();
+print("snapshot2.js 5 -----------------------------------------------------------");
+for (i = 0; i < 500; ++i) {
+    rp.master().getDB( baseName )[ baseName ].save( { _id: new ObjectId(), i: i, b: big } );
+    if (i % 250 == 249) {
+        function p() { return i + 1 == rp.slave().getDB(baseName)[baseName].count(); }
+        try {
+            assert.soon(p);
+        } catch (e) {
+            print("\n\n\nsnapshot2.js\ni+1:" + (i + 1));
+            print("slave count:" + rp.slave().getDB(baseName)[baseName].count());
+            sleep(2000);
+            print(p());
+            throw (e);
+        } 
         sleep( 10 ); // give master a chance to grab a sync point - have such small oplogs the master log might overflow otherwise
     }
-}
+}
+print("snapshot2.js 6 -----------------------------------------------------------");
 
 rp.master().getDB( "admin" ).runCommand( {fsync:1,lock:1} );
 leftMaster = ( rp.master().host == rp.left().host );
@@ -47,5 +65,8 @@ assert.eq( 500, rp.slave().getDB( baseName )[ baseName ].count() );
 rp.master().getDB( baseName )[ baseName ].save( {i:500} );
 assert.soon( function() { return 501 == rp.slave().getDB( baseName )[ baseName ].count(); } );
 
-assert( !rawMongoProgramOutput().match( /resync/ ) );
-assert( !rawMongoProgramOutput().match( /SyncException/ ) );
+assert( !rawMongoProgramOutput().match( /resync/ ) );
+assert(!rawMongoProgramOutput().match(/SyncException/));
+
+print("snapshot2.js SUCCESS ----------------");
+