Backport of fix from SERVER-3824.

Conflicts:

	dbtests/repltests.cpp

SConstruct

@@ -114,6 +114,7 @@ add_option( "64" , "whether to force 64 bit" , 0 , True , "force64" )
 add_option( "32" , "whether to force 32 bit" , 0 , True , "force32" )
 
 add_option( "cxx", "compiler to use" , 1 , True )
+add_option( "cc", "compiler to use for c" , 1 , True )
 
 add_option( "cpppath", "Include path if you have headers in a nonstandard directory" , 1 , True )
 add_option( "libpath", "Library path if you have libraries in a nonstandard directory" , 1 , True )
@@ -211,6 +212,10 @@ env = Environment( MSVS_ARCH=msarch , tools = ["default", "gch"], toolpath = '.'
 if has_option( "cxx" ):
     env["CC"] = get_option( "cxx" )
     env["CXX"] = get_option( "cxx" )
+
+if has_option( "cc" ):
+    env["CC"] = get_option( "cc" )
+
 env["LIBPATH"] = []
 
 if has_option( "libpath" ):
@@ -750,6 +755,27 @@ def add_exe(target):
         return target + ".exe"
     return target
 
+def smoke_python_name():
+    # if this script is being run by py2.5 or greater,
+    # then we assume that "python" points to a 2.5 or
+    # greater python VM. otherwise, explicitly use 2.5
+    # which we assume to be installed.
+    import subprocess
+    version = re.compile(r'[Pp]ython ([\d\.]+)', re.MULTILINE)
+    binaries = ['python', 'python2.5', 'python2.6', 'python2.7', 'python25', 'python26', 'python27']
+    for binary in binaries:
+        try:
+            # py-2.4 compatible replacement for shell backticks
+            output = subprocess.Popen([binary, '--version'], stdout=subprocess.PIPE).communicate()[0]
+            match = version.search(output)
+            if match and float(match.group(1)) >= 2.5:
+                return binary
+        except:
+            pass
+
+    # if that all fails, fall back to "python"
+    return "python"
+
 def setupBuildInfoFile( outFile ):
     version = utils.getGitVersion()
     if len(moduleNames) > 0:
@@ -1245,7 +1271,7 @@ def addSmoketest( name, deps ):
         else:
             target = name[5].lower() + name[6:]
 
-    addTest(name, deps, [ "python buildscripts/smoke.py " + " ".join(smokeFlags) + ' ' + target ])
+    addTest(name, deps, [ smoke_python_name() + " buildscripts/smoke.py " + " ".join(smokeFlags) + ' ' + target ])
 
 addSmoketest( "smoke", [ add_exe( "test" ) ] )
 addSmoketest( "smokePerf", [ "perftest" ]  )
@@ -1643,7 +1669,7 @@ def build_and_test_client(env, target, source):
 
     call(scons_command + ["libmongoclient.a", "clientTests"], cwd=installDir)
 
-    return bool(call(["python", "buildscripts/smoke.py",
+    return bool(call([smoke_python_name(), "buildscripts/smoke.py",
                       "--test-path", installDir, "client"]))
 env.Alias("clientBuild", [mongod, installDir], [build_and_test_client])
 env.AlwaysBuild("clientBuild")

client/clientOnly.cpp

@@ -68,5 +68,9 @@ namespace mongo {
         return false;
     }
 
+    string prettyHostName() {
+        assert(0);
+        return "";
+    }
 
 }

client/connpool.cpp

@@ -37,7 +37,12 @@ namespace mongo {
     }
 
     void PoolForHost::done( DBClientBase * c ) {
-        _pool.push(c);
+        if ( _pool.size() >= _maxPerHost ) {
+            delete c;
+        }
+        else {
+            _pool.push(c);
+        }
     }
 
     DBClientBase * PoolForHost::get() {
@@ -86,6 +91,8 @@ namespace mongo {
         _created++;
     }
 
+    unsigned PoolForHost::_maxPerHost = 50;
+
     // ------ DBConnectionPool ------
 
     DBConnectionPool pool;
@@ -185,6 +192,9 @@ namespace mongo {
         {
             scoped_lock lk( _mutex );
             for ( PoolMap::iterator i=_pools.begin(); i!=_pools.end(); ++i ) {
+                if ( i->second.numCreated() == 0 )
+                    continue;
+
                 string s = i->first;
                 BSONObjBuilder temp( bb.subobjStart( s ) );
                 temp.append( "available" , i->second.numAvailable() );

client/connpool.h

@@ -57,6 +57,9 @@ namespace mongo {
         void done( DBClientBase * c );
 
         void flush();
+
+        static void setMaxPerHost( unsigned max ) { _maxPerHost = max; }
+        static unsigned getMaxPerHost() { return _maxPerHost; }
     private:
 
         struct StoredConnection {
@@ -71,6 +74,8 @@ namespace mongo {
         std::stack<StoredConnection> _pool;
         long long _created;
         ConnectionString::ConnectionType _type;
+
+        static unsigned _maxPerHost;
     };
 
     class DBConnectionHook {

client/dbclient.cpp

@@ -442,15 +442,16 @@ namespace mongo {
         return false;
     }
 
-    BSONObj DBClientWithCommands::mapreduce(const string &ns, const string &jsmapf, const string &jsreducef, BSONObj query, const string& outputcolname) {
+    DBClientWithCommands::MROutput DBClientWithCommands::MRInline (BSON("inline" << 1));
+
+    BSONObj DBClientWithCommands::mapreduce(const string &ns, const string &jsmapf, const string &jsreducef, BSONObj query, MROutput output) {
         BSONObjBuilder b;
         b.append("mapreduce", nsGetCollection(ns));
         b.appendCode("map", jsmapf);
         b.appendCode("reduce", jsreducef);
         if( !query.isEmpty() )
             b.append("query", query);
-        if( !outputcolname.empty() )
-            b.append("out", outputcolname);
+        b.append("out", output.out);
         BSONObj info;
         runCommand(nsGetDB(ns), b.done(), info);
         return info;
@@ -915,13 +916,14 @@ namespace mongo {
 
     void DBClientConnection::checkResponse( const char *data, int nReturned ) {
         /* check for errors.  the only one we really care about at
-         this stage is "not master" */
+         * this stage is "not master" 
+        */
+        
         if ( clientSet && nReturned ) {
             assert(data);
             BSONObj o(data);
-            BSONElement e = o.firstElement();
-            if ( strcmp(e.fieldName(), "$err") == 0 &&
-                    e.type() == String && strncmp(e.valuestr(), "not master", 10) == 0 ) {
+            BSONElement e = o["$err"];
+            if ( e.type() == String && str::contains( e.valuestr() , "not master" ) ) {
                 clientSet->isntMaster();
             }
         }

client/dbclient.h

@@ -528,6 +528,19 @@ namespace mongo {
         bool setDbProfilingLevel(const string &dbname, ProfilingLevel level, BSONObj *info = 0);
         bool getDbProfilingLevel(const string &dbname, ProfilingLevel& level, BSONObj *info = 0);
 
+
+        /** This implicitly converts from char*, string, and BSONObj to be an argument to mapreduce
+            You shouldn't need to explicitly construct this
+         */
+        struct MROutput {
+            MROutput(const char* collection) : out(BSON("replace" << collection)) {}
+            MROutput(const string& collection) : out(BSON("replace" << collection)) {}
+            MROutput(const BSONObj& obj) : out(obj) {}
+
+            BSONObj out;
+        };
+        static MROutput MRInline;
+
         /** Run a map/reduce job on the server.
 
             See http://www.mongodb.org/display/DOCS/MapReduce
@@ -536,8 +549,8 @@ namespace mongo {
             jsmapf    javascript map function code
             jsreducef javascript reduce function code.
             query     optional query filter for the input
-            output    optional permanent output collection name.  if not specified server will
-                      generate a temporary collection and return its name.
+            output    either a string collection name or an object representing output type
+                      if not specified uses inline output type
 
             returns a result object which contains:
              { result : <collection_name>,
@@ -551,7 +564,7 @@ namespace mongo {
                result.getField("ok").trueValue()
              on the result to check if ok.
         */
-        BSONObj mapreduce(const string &ns, const string &jsmapf, const string &jsreducef, BSONObj query = BSONObj(), const string& output = "");
+        BSONObj mapreduce(const string &ns, const string &jsmapf, const string &jsreducef, BSONObj query = BSONObj(), MROutput output = MRInline);
 
         /** Run javascript code on the database server.
            dbname    database SavedContext in which the code runs. The javascript variable 'db' will be assigned

client/dbclient_rs.cpp

@@ -52,13 +52,17 @@ namespace mongo {
         }
     protected:
         void run() {
+            log() << "starting" << endl;
             while ( ! inShutdown() ) {
                 sleepsecs( 20 );
                 try {
                     ReplicaSetMonitor::checkAll();
                 }
                 catch ( std::exception& e ) {
-                    error() << "ReplicaSetMonitorWatcher: check failed: " << e.what() << endl;
+                    error() << "check failed: " << e.what() << endl;
+                }
+                catch ( ... ) {
+                    error() << "unkown error" << endl;
                 }
             }
         }
@@ -70,7 +74,7 @@ namespace mongo {
 
 
     ReplicaSetMonitor::ReplicaSetMonitor( const string& name , const vector<HostAndPort>& servers )
-        : _lock( "ReplicaSetMonitor instance" ) , _checkConnectionLock( "ReplicaSetMonitor check connection lock" ), _name( name ) , _master(-1) {
+        : _lock( "ReplicaSetMonitor instance" ) , _checkConnectionLock( "ReplicaSetMonitor check connection lock" ), _name( name ) , _master(-1), _nextSlave(0) {
         
         uassert( 13642 , "need at least 1 node for a replica set" , servers.size() > 0 );
 
@@ -81,6 +85,12 @@ namespace mongo {
         string errmsg;
 
         for ( unsigned i=0; i<servers.size(); i++ ) {
+
+            bool haveAlready = false;
+            for ( unsigned n = 0; n < _nodes.size() && ! haveAlready; n++ )
+                haveAlready = ( _nodes[n].addr == servers[i] );
+            if( haveAlready ) continue;
+
             auto_ptr<DBClientConnection> conn( new DBClientConnection( true , 0, 5.0 ) );
             if (!conn->connect( servers[i] , errmsg ) ) {
                 log(1) << "error connecting to seed " << servers[i] << ": " << errmsg << endl;
@@ -121,6 +131,7 @@ namespace mongo {
         while ( true ) {
             ReplicaSetMonitorPtr m;
             {
+                scoped_lock lk( _setsLock );
                 for ( map<string,ReplicaSetMonitorPtr>::iterator i=_sets.begin(); i!=_sets.end(); ++i ) {
                     string name = i->first;
                     if ( seen.count( name ) )
@@ -175,26 +186,26 @@ namespace mongo {
     void ReplicaSetMonitor::notifyFailure( const HostAndPort& server ) {
         scoped_lock lk( _lock );
         if ( _master >= 0 && _master < (int)_nodes.size() ) {
-            if ( server == _nodes[_master].addr )
+            if ( server == _nodes[_master].addr ) {
+                _nodes[_master].ok = false; 
                 _master = -1;
+            }
         }
     }
 
 
 
     HostAndPort ReplicaSetMonitor::getMaster() {
-        bool good = false;
         {
             scoped_lock lk( _lock );
-            good = _master >= 0 && _nodes[_master].ok;
+            if ( _master >= 0 && _nodes[_master].ok )
+                return _nodes[_master].addr;
         }
-
-        if ( ! good )
-            _check();
-
-        uassert( 10009 , str::stream() << "ReplicaSetMonitor no master found for set: " << _name , _master >= 0 );
+        
+        _check();
 
         scoped_lock lk( _lock );
+        uassert( 10009 , str::stream() << "ReplicaSetMonitor no master found for set: " << _name , _master >= 0 );
         return _nodes[_master].addr;
     }
     
@@ -216,19 +227,17 @@ namespace mongo {
     }
 
     HostAndPort ReplicaSetMonitor::getSlave() {
-        int x = rand() % _nodes.size();
-        {
-            scoped_lock lk( _lock );
-            for ( unsigned i=0; i<_nodes.size(); i++ ) {
-                int p = ( i + x ) % _nodes.size();
-                if ( p == _master )
-                    continue;
-                if ( _nodes[p].ok )
-                    return _nodes[p].addr;
-            }
+
+        scoped_lock lk( _lock );
+        for ( unsigned i=0; i<_nodes.size(); i++ ) {
+            _nextSlave = ( _nextSlave + 1 ) % _nodes.size();
+            if ( _nextSlave == _master )
+                continue;
+            if ( _nodes[ _nextSlave ].ok )
+                return _nodes[ _nextSlave ].addr;
         }
 
-        return _nodes[0].addr;
+        return _nodes[ 0 ].addr;
     }
 
     /**
@@ -257,7 +266,7 @@ namespace mongo {
             string host = member["name"].String();
 
             int m = -1;
-            if ((m = _find(host)) <= 0) {
+            if ((m = _find(host)) < 0) {
                 continue;
             }
 
@@ -287,6 +296,10 @@ namespace mongo {
             newConn->connect( h , temp );
             {
                 scoped_lock lk( _lock );
+                if ( _find_inlock( toCheck ) >= 0 ) {
+                    // we need this check inside the lock so there isn't thread contention on adding to vector
+                    continue;
+                }
                 _nodes.push_back( Node( h , newConn ) );
             }
             log() << "updated set (" << _name << ") to: " << getServerAddress() << endl;
@@ -304,10 +317,9 @@ namespace mongo {
             BSONObj o;
             c->isMaster(isMaster, &o);
 
-            log( ! verbose ) << "ReplicaSetMonitor::_checkConnection: " << c->toString() << ' ' << o << '\n';
+            log( ! verbose ) << "ReplicaSetMonitor::_checkConnection: " << c->toString() << ' ' << o << endl;
 
             // add other nodes
-            string maybePrimary;
             if ( o["hosts"].type() == Array ) {
                 if ( o["primary"].type() == String )
                     maybePrimary = o["primary"].String();
@@ -389,12 +401,17 @@ namespace mongo {
 
     int ReplicaSetMonitor::_find( const string& server ) const {
         scoped_lock lk( _lock );
+        return _find_inlock( server );
+    }
+
+    int ReplicaSetMonitor::_find_inlock( const string& server ) const {
         for ( unsigned i=0; i<_nodes.size(); i++ )
             if ( _nodes[i].addr == server )
                 return i;
         return -1;
     }
 
+
     int ReplicaSetMonitor::_find( const HostAndPort& server ) const {
         scoped_lock lk( _lock );
         for ( unsigned i=0; i<_nodes.size(); i++ )
@@ -421,7 +438,7 @@ namespace mongo {
     DBClientConnection * DBClientReplicaSet::checkMaster() {
         HostAndPort h = _monitor->getMaster();
 
-        if ( h == _masterHost ) {
+        if ( h == _masterHost && _master ) {
             // a master is selected.  let's just make sure connection didn't die
             if ( ! _master->isFailed() )
                 return _master.get();
@@ -429,7 +446,7 @@ namespace mongo {
         }
 
         _masterHost = _monitor->getMaster();
-        _master.reset( new DBClientConnection( true ) );
+        _master.reset( new DBClientConnection( true , this ) );
         string errmsg;
         if ( ! _master->connect( _masterHost , errmsg ) ) {
             _monitor->notifyFailure( _masterHost );
@@ -442,14 +459,17 @@ namespace mongo {
     DBClientConnection * DBClientReplicaSet::checkSlave() {
         HostAndPort h = _monitor->getSlave( _slaveHost );
 
-        if ( h == _slaveHost ) {
+        if ( h == _slaveHost && _slave ) {
             if ( ! _slave->isFailed() )
                 return _slave.get();
             _monitor->notifySlaveFailure( _slaveHost );
+            _slaveHost = _monitor->getSlave();
         }
-        
-        _slaveHost = _monitor->getSlave();
-        _slave.reset( new DBClientConnection( true ) );
+        else {
+            _slaveHost = h;
+        }
+
+        _slave.reset( new DBClientConnection( true , this ) );
         _slave->connect( _slaveHost );
         _auth( _slave.get() );
         return _slave.get();
@@ -527,10 +547,10 @@ namespace mongo {
             // checkSlave will try a different slave automatically after a failure
             for ( int i=0; i<2; i++ ) {
                 try {
-                    return checkSlave()->query(ns,query,nToReturn,nToSkip,fieldsToReturn,queryOptions,batchSize);
+                    return checkSlaveQueryResult( checkSlave()->query(ns,query,nToReturn,nToSkip,fieldsToReturn,queryOptions,batchSize) );
                 }
-                catch ( DBException & ) {
-                    LOG(1) << "can't query replica set slave: " << _slaveHost << endl;
+                catch ( DBException &e ) {
+                    log() << "can't query replica set slave " << i << " : " << _slaveHost << e.what() << endl;
                 }
             }
         }
@@ -547,8 +567,8 @@ namespace mongo {
                 try {
                     return checkSlave()->findOne(ns,query,fieldsToReturn,queryOptions);
                 }
-                catch ( DBException & ) {
-                    LOG(1) << "can't query replica set slave: " << _slaveHost << endl;
+                catch ( DBException &e ) {
+                	LOG(1) << "can't findone replica set slave " << i << " : " << _slaveHost << e.what() << endl;
                 }
             }
         }
@@ -564,6 +584,43 @@ namespace mongo {
         assert(0);
     }
 
+    auto_ptr<DBClientCursor> DBClientReplicaSet::checkSlaveQueryResult( auto_ptr<DBClientCursor> result ){
+
+        bool isError = result->hasResultFlag( ResultFlag_ErrSet );
+        if( ! isError ) return result;
+
+        BSONObj error = result->peekOne();
+
+        BSONElement code = error["code"];
+        if( code.eoo() || ! code.isNumber() ){
+            warning() << "no code for error from secondary host " << _slaveHost << ", error was " << error << endl;
+            return result;
+        }
+
+        // We only check for "not master or secondary" errors here
+
+        // If the error code here ever changes, we need to change this code also
+        if( code.Int() == 13436 /* not master or secondary */ ){
+            isntSecondary();
+            throw DBException( str::stream() << "slave " << _slaveHost.toString() << " is no longer secondary", 14812 );
+        }
+
+        return result;
+    }
+
+    void DBClientReplicaSet::isntSecondary() {
+        log() << "slave no longer has secondary status: " << _slaveHost << endl;
+        // Failover to next slave
+        _monitor->notifySlaveFailure( _slaveHost );
+        _slave.reset();
+    }
+
+
+    void DBClientReplicaSet::isntMaster() { 
+        log() << "got not master for: " << _masterHost << endl;
+        _monitor->notifyFailure( _masterHost );
+        _master.reset(); 
+    }
 
     bool DBClientReplicaSet::call( Message &toSend, Message &response, bool assertOk , string * actualServer ) {
         if ( toSend.operation() == dbQuery ) {
@@ -578,8 +635,8 @@ namespace mongo {
                             *actualServer = s->getServerAddress();
                         return s->call( toSend , response , assertOk );
                     }
-                    catch ( DBException & ) {
-                        log(1) << "can't query replica set slave: " << _slaveHost << endl;
+                    catch ( DBException &e ) {
+                    	LOG(1) << "can't call replica set slave " << i << " : " << _slaveHost << e.what() << endl;
                         if ( actualServer )
                             *actualServer = "";
                     }

client/dbclient_rs.h

@@ -124,6 +124,7 @@ namespace mongo {
         bool _checkConnection( DBClientConnection * c , string& maybePrimary , bool verbose );
 
         int _find( const string& server ) const ;
+        int _find_inlock( const string& server ) const ;
         int _find( const HostAndPort& server ) const ;
 
         mutable mongo::mutex _lock; // protects _nodes
@@ -147,7 +148,7 @@ namespace mongo {
         vector<Node> _nodes;
 
         int _master; // which node is the current master.  -1 means no master is known
-
+        int _nextSlave; // which node is the current slave
 
         static mongo::mutex _setsLock; // protects _sets
         static map<string,ReplicaSetMonitorPtr> _sets; // set name to Monitor
@@ -213,7 +214,10 @@ namespace mongo {
 
         /* this is the callback from our underlying connections to notify us that we got a "not master" error.
          */
-        void isntMaster() { _master.reset(); }
+        void isntMaster();
+        /* this is used to indicate we got a "not master or secondary" error from a secondary.
+         */
+        void isntSecondary();
 
         // ----- status ------
 
@@ -239,6 +243,9 @@ namespace mongo {
 
     private:
 
+        // Used to simplify slave-handling logic on errors
+        auto_ptr<DBClientCursor> checkSlaveQueryResult( auto_ptr<DBClientCursor> result );
+
         DBClientConnection * checkMaster();
         DBClientConnection * checkSlave();
 

client/dbclientcursor.h

@@ -86,6 +86,11 @@ namespace mongo {
             WARNING: no support for _putBack yet!
         */
         void peek(vector<BSONObj>&, int atMost);
+        BSONObj peekOne(){
+            vector<BSONObj> v;
+            peek( v, 1 );
+            return v.size() > 0 ? v[0] : BSONObj();
+        }
 
         /**
            iterate the rest of the cursor and return the number if items

client/examples/clientTest.cpp

@@ -224,5 +224,27 @@ int main( int argc, const char **argv ) {
         }
     }
 
+    {
+        //Map Reduce (this mostly just tests that it compiles with all output types)
+        const string ns = "test.mr";
+        conn.insert(ns, BSON("a" << 1));
+        conn.insert(ns, BSON("a" << 1));
+
+        const char* map = "function() { emit(this.a, 1); }";
+        const char* reduce = "function(key, values) { return Array.sum(values); }";
+
+        const string outcoll = ns + ".out";
+
+        BSONObj out;
+        out = conn.mapreduce(ns, map, reduce, BSONObj()); // default to inline
+        //MONGO_PRINT(out);
+        out = conn.mapreduce(ns, map, reduce, BSONObj(), outcoll);
+        //MONGO_PRINT(out);
+        out = conn.mapreduce(ns, map, reduce, BSONObj(), outcoll.c_str());
+        //MONGO_PRINT(out);
+        out = conn.mapreduce(ns, map, reduce, BSONObj(), BSON("reduce" << outcoll));
+        //MONGO_PRINT(out);
+    }
+
     cout << "client test finished!" << endl;
 }

db/btree.cpp

@@ -1114,16 +1114,17 @@ namespace mongo {
 
     /** remove a key from the index */
     bool BtreeBucket::unindex(const DiskLoc thisLoc, IndexDetails& id, const BSONObj& key, const DiskLoc recordLoc ) const {
-        if ( key.objsize() > KeyMax ) {
-            OCCASIONALLY problem() << "unindex: key too large to index, skipping " << id.indexNamespace() << /* ' ' << key.toString() << */ endl;
-            return false;
-        }
-
         int pos;
         bool found;
         DiskLoc loc = locate(id, thisLoc, key, Ordering::make(id.keyPattern()), pos, found, recordLoc, 1);
         if ( found ) {
+
+            if ( key.objsize() > KeyMax ) {
+                OCCASIONALLY problem() << "unindex: key too large to index but was found for " << id.indexNamespace() << " reIndex suggested" << endl;
+            }
+            
             loc.btreemod()->delKeyAtPos(loc, id, pos, Ordering::make(id.keyPattern()));
+            
             return true;
         }
         return false;
@@ -1187,8 +1188,6 @@ namespace mongo {
             out() << "   " << thisLoc.toString() << ".insertHere " << key.toString() << '/' << recordLoc.toString() << ' '
                   << lchild.toString() << ' ' << rchild.toString() << " keypos:" << keypos << endl;
 
-        DiskLoc oldLoc = thisLoc;
-
         if ( !basicInsert(thisLoc, keypos, recordLoc, key, order) ) {
             thisLoc.btreemod()->split(thisLoc, keypos, recordLoc, key, order, lchild, rchild, idx);
             return;
@@ -1643,6 +1642,7 @@ namespace mongo {
     }
 
     DiskLoc BtreeBucket::findSingle( const IndexDetails& indexdetails , const DiskLoc& thisLoc, const BSONObj& key ) const {
+        indexdetails.checkVersion();
         int pos;
         bool found;
         // TODO: is it really ok here that the order is a default?
@@ -1740,13 +1740,18 @@ namespace mongo {
     }
 
     void BtreeBuilder::mayCommitProgressDurably() {
-        RARELY {
-            getDur().commitIfNeeded();
+        if ( getDur().commitIfNeeded() ) {
             b = cur.btreemod();
         }
     }
 
     void BtreeBuilder::addKey(BSONObj& key, DiskLoc loc) {
+        if ( key.objsize() > KeyMax ) {
+            problem() << "Btree::insert: key too large to index, skipping " << idx.indexNamespace() 
+                      << ' ' << key.objsize() << ' ' << key.toString() << endl;
+            return;
+        }
+
         if( !dupsAllowed ) {
             if( n > 0 ) {
                 int cmp = keyLast.woCompare(key, order);
@@ -1760,15 +1765,9 @@ namespace mongo {
         }
 
         if ( ! b->_pushBack(loc, key, ordering, DiskLoc()) ) {
-            // no room
-            if ( key.objsize() > KeyMax ) {
-                problem() << "Btree::insert: key too large to index, skipping " << idx.indexNamespace() << ' ' << key.objsize() << ' ' << key.toString() << endl;
-            }
-            else {
-                // bucket was full
-                newBucket();
-                b->pushBack(loc, key, ordering, DiskLoc());
-            }
+            // bucket was full
+            newBucket();
+            b->pushBack(loc, key, ordering, DiskLoc());
         }
         n++;
         mayCommitProgressDurably();
@@ -1790,6 +1789,11 @@ namespace mongo {
 
             DiskLoc xloc = loc;
             while( !xloc.isNull() ) {
+                if ( getDur().commitIfNeeded() ) {
+                    b = cur.btreemod();
+                    up = upLoc.btreemod();
+                }
+
                 BtreeBucket *x = xloc.btreemod();
                 BSONObj k;
                 DiskLoc r;
@@ -1833,18 +1837,21 @@ namespace mongo {
     }
 
     BtreeBuilder::~BtreeBuilder() {
-        if( !committed ) {
-            log(2) << "Rolling back partially built index space" << endl;
-            DiskLoc x = first;
-            while( !x.isNull() ) {
-                DiskLoc next = x.btree()->tempNext();
-                string ns = idx.indexNamespace();
-                theDataFileMgr._deleteRecord(nsdetails(ns.c_str()), ns.c_str(), x.rec(), x);
-                x = next;
+        DESTRUCTOR_GUARD(
+            if( !committed ) {
+                log(2) << "Rolling back partially built index space" << endl;
+                DiskLoc x = first;
+                while( !x.isNull() ) {
+                    DiskLoc next = x.btree()->tempNext();
+                    string ns = idx.indexNamespace();
+                    theDataFileMgr._deleteRecord(nsdetails(ns.c_str()), ns.c_str(), x.rec(), x);
+                    x = next;
+                    getDur().commitIfNeeded();
+                }
+                assert( idx.head.isNull() );
+                log(2) << "done rollback" << endl;
             }
-            assert( idx.head.isNull() );
-            log(2) << "done rollback" << endl;
-        }
+        )
     }
 
 }

db/btreecursor.cpp

@@ -73,6 +73,7 @@ namespace mongo {
     }
 
     void BtreeCursor::audit() {
+        indexDetails.checkVersion();
         dassert( d->idxNo((IndexDetails&) indexDetails) == idxNo );
 
         if ( otherTraceLevel >= 12 ) {

db/cap.cpp

@@ -319,6 +319,7 @@ namespace mongo {
                 // 'end' has been found and removed, so break.
                 break;
             }
+            getDur().commitIfNeeded();
             // 'curr' will point to the newest document in the collection.
             DiskLoc curr = theCapExtent()->lastRecord;
             assert( !curr.isNull() );

db/client.cpp

@@ -169,11 +169,6 @@ namespace mongo {
             uassert( 13005 , "can't create db, keeps getting closed" , _db );
         }
 
-        _client->_context = this;
-        _client->_curOp->enter( this );
-        if ( doauth )
-            _auth( lockState );
-
         switch ( _client->_curOp->getOp() ) {
         case dbGetMore: // getMore's are special and should be handled else where
         case dbUpdate: // update & delete check shard version in instance.cpp, so don't check here as well
@@ -188,6 +183,11 @@ namespace mongo {
             }
         }
         }
+
+        _client->_context = this;
+        _client->_curOp->enter( this );
+        if ( doauth )
+            _auth( lockState );
     }
 
     void Client::Context::_auth( int lockState ) {

db/clientcursor.h

@@ -91,6 +91,13 @@ namespace mongo {
                     _c = 0;
                 }
             }
+            /**
+             * call this if during a yield, the cursor got deleted
+             * if so, we don't want to use the point address
+             */
+            void deleted() {
+                _c = 0;
+            }
             ~Pointer() { release(); }
             Pointer(long long cursorid) {
                 recursive_scoped_lock lock(ccmutex);

db/cloner.cpp

@@ -244,6 +244,7 @@ namespace mongo {
             string temp = ctx.db()->name + ".system.indexes";
             copy( temp.c_str() , temp.c_str() , /*isindex*/true , logForRepl , false , true , BSON( "ns" << ns ) );
         }
+        getDur().commitIfNeeded();
         return true;
     }
 
@@ -623,6 +624,8 @@ namespace mongo {
                 nsToDatabase( target.c_str(), to );
                 if ( strcmp( from, to ) == 0 ) {
                     renameNamespace( source.c_str(), target.c_str() );
+                    // make sure we drop counters etc
+                    Top::global.collectionDropped( source );
                     return true;
                 }
             }

db/cmdline.cpp

@@ -20,6 +20,7 @@
 #include "cmdline.h"
 #include "commands.h"
 #include "../util/processinfo.h"
+#include "../util/message.h"
 #include "security_key.h"
 
 #ifdef _WIN32
@@ -50,6 +51,7 @@ namespace mongo {
         ("quiet", "quieter output")
         ("port", po::value<int>(&cmdLine.port), "specify port number")
         ("bind_ip", po::value<string>(&cmdLine.bind_ip), "comma separated list of ip addresses to listen on - all local ips by default")
+        ("maxConns",po::value<int>(), "max number of simultaneous connections")
         ("logpath", po::value<string>() , "log file to send write to instead of stdout - has to be a file, not directory" )
         ("logappend" , "append to logpath instead of over-writing" )
         ("pidfilepath", po::value<string>(), "full path to pidfile (if not set, no pidfile is created)")
@@ -163,6 +165,19 @@ namespace mongo {
             cmdLine.quiet = true;
         }
 
+        if ( params.count( "maxConns" ) ) {
+            int newSize = params["maxConns"].as<int>();
+            if ( newSize < 5 ) {
+                out() << "maxConns has to be at least 5" << endl;
+                dbexit( EXIT_BADOPTIONS );
+            }
+            else if ( newSize >= 10000000 ) {
+                out() << "maxConns can't be greater than 10000000" << endl;
+                dbexit( EXIT_BADOPTIONS );
+            }
+            connTicketHolder.resize( newSize );
+        }
+
         string logpath;
 
 #ifndef _WIN32
@@ -257,8 +272,12 @@ namespace mongo {
                 dbexit(EXIT_BADOPTIONS);
             }
 
+            cmdLine.keyFile = true;
             noauth = false;
         }
+        else {
+            cmdLine.keyFile = false;
+        }
 
 
         {

db/cmdline.h

@@ -100,6 +100,8 @@ namespace mongo {
 
         string socket;         // UNIX domain socket directory
 
+        bool keyFile;
+
         static void addGlobalOptions( boost::program_options::options_description& general ,
                                       boost::program_options::options_description& hidden );
 

db/commands.cpp

@@ -127,7 +127,6 @@ namespace mongo {
         if ( strcmp(p, ".$cmd") != 0 ) return false;
 
         bool ok = false;
-        bool valid = false;
 
         BSONElement e = jsobj.firstElement();
         map<string,Command*>::iterator i;
@@ -138,7 +137,6 @@ namespace mongo {
            migrated over to the command object format.
            */
         else if ( (i = _commands->find(e.fieldName())) != _commands->end() ) {
-            valid = true;
             string errmsg;
             Command *c = i->second;
             if ( c->adminOnly() && !startsWith(ns, "admin.") ) {

db/commands/distinct.cpp

@@ -109,7 +109,7 @@ namespace mongo {
 
                         int now = bb.len();
 
-                        uassert(10044,  "distinct too big, 4mb cap", ( now + e.size() + 1024 ) < bufSize );
+                        uassert(10044,  "distinct too big, 16mb cap", ( now + e.size() + 1024 ) < bufSize );
 
                         arr.append( e );
                         BSONElement x( start + now );

db/commands/mr.cpp

@@ -508,7 +508,7 @@ namespace mongo {
 
             _scope->injectNative( "emit" , fast_emit );
 
-            if ( _onDisk ) {
+            if ( _onDisk && _config.incLong != _config.tempLong ) {
                 // clear temp collections
                 _db.dropCollection( _config.tempLong );
                 _db.dropCollection( _config.incLong );
@@ -758,7 +758,18 @@ namespace mongo {
         BSONObj fast_emit( const BSONObj& args ) {
             uassert( 10077 , "fast_emit takes 2 args" , args.nFields() == 2 );
             uassert( 13069 , "an emit can't be more than half max bson size" , args.objsize() < ( BSONObjMaxUserSize / 2 ) );
-            (*_tl)->emit( args );
+            
+            if ( args.firstElement().type() == Undefined ) {
+                BSONObjBuilder b( args.objsize() );
+                b.appendNull( "" );
+                BSONObjIterator i( args );
+                i.next();
+                b.append( i.next() );
+                (*_tl)->emit( b.obj() );
+            }
+            else {
+                (*_tl)->emit( args );
+            }
             return BSONObj();
         }
 

db/cursor.h

@@ -113,6 +113,8 @@ namespace mongo {
         // The implementation may return different matchers depending on the
         // position of the cursor.  If matcher() is nonzero at the start,
         // matcher() should be checked each time advance() is called.
+        // Implementations which generate their own matcher should return this
+        // to avoid a matcher being set manually.
         virtual CoveredIndexMatcher *matcher() const { return 0; }
 
         // A convenience function for setting the value of matcher() manually

db/db.cpp

@@ -46,6 +46,7 @@
 # include "../util/ntservice.h"
 #else
 # include <sys/file.h>
+# include <sys/resource.h>
 #endif
 
 namespace mongo {
@@ -108,7 +109,36 @@ namespace mongo {
             }
 
             try {
+#ifndef __linux__  // TODO: consider making this ifdef _WIN32
                 boost::thread thr(boost::bind(&connThread,mp));
+#else
+                pthread_attr_t attrs;
+                pthread_attr_init(&attrs);
+                pthread_attr_setdetachstate(&attrs, PTHREAD_CREATE_DETACHED);
+
+                static const size_t STACK_SIZE = 4*1024*1024;
+
+                struct rlimit limits;
+                assert(getrlimit(RLIMIT_STACK, &limits) == 0);
+                if (limits.rlim_cur > STACK_SIZE) {
+                    pthread_attr_setstacksize(&attrs, (DEBUG_BUILD
+                                                        ? (STACK_SIZE / 2)
+                                                        : STACK_SIZE));
+                }
+                else if (limits.rlim_cur < 1024*1024) {
+                    warning() << "Stack size set to " << (limits.rlim_cur/1024) << "KB. We suggest at least 1MB" << endl;
+                }
+
+                pthread_t thread;
+                int failed = pthread_create(&thread, &attrs, (void*(*)(void*)) &connThread, mp);
+
+                pthread_attr_destroy(&attrs);
+
+                if (failed) {
+                    log() << "pthread_create failed: " << errnoWithDescription(failed) << endl;
+                    throw boost::thread_resource_error(); // for consistency with boost::thread
+                }
+#endif
             }
             catch ( boost::thread_resource_error& ) {
                 log() << "can't create new thread, closing connection" << endl;
@@ -635,14 +665,11 @@ int main(int argc, char* argv[]) {
     ("dbpath", po::value<string>() , "directory for datafiles")
     ("diaglog", po::value<int>(), "0=off 1=W 2=R 3=both 7=W+some reads")
     ("directoryperdb", "each database will be stored in a separate directory")
-    ("dur", "enable journaling")
-    ("durOptions", po::value<int>(), "durability diagnostic options")
+    ("journal", "enable journaling")
+    ("journalOptions", po::value<int>(), "journal diagnostic options")
     ("ipv6", "enable IPv6 support (disabled by default)")
     ("jsonp","allow JSONP access via http (has security implications)")
-    ("maxConns",po::value<int>(), "max number of simultaneous connections")
     ("noauth", "run without security")
-    ("nocursors", "diagnostic/debugging option")
-    ("nohints", "ignore query hints")
     ("nohttpinterface", "disable http interface")
     ("noprealloc", "disable data file preallocation - will often hurt performance")
     ("noscripting", "disable scripting engine")
@@ -702,7 +729,12 @@ int main(int argc, char* argv[]) {
     ("pairwith", po::value<string>(), "address of server to pair with DEPRECATED")
     ("arbiter", po::value<string>(), "address of replica pair arbiter server DEPRECATED")
     ("nodur", "disable journaling (currently the default)")
+    ("nojournal", "disable journaling (currently the default)")
     ("appsrvpath", po::value<string>(), "root directory for the babble app server")
+    ("nocursors", "diagnostic/debugging option that turns off cursors DO NOT USE IN PRODUCTION")
+    ("nohints", "ignore query hints")
+    ("dur", "enable journaling") // deprecated version
+    ("durOptions", po::value<int>(), "durability diagnostic options") // deprecated version
     ;
 
 
@@ -723,7 +755,11 @@ int main(int argc, char* argv[]) {
     dbExecCommand = argv[0];
 
     srand(curTimeMicros());
+#if( BOOST_VERSION >= 104500 )
+    boost::filesystem::path::default_name_check( boost::filesystem2::no_check );
+#else
     boost::filesystem::path::default_name_check( boost::filesystem::no_check );
+#endif
 
     {
         unsigned x = 0x12345678;
@@ -795,12 +831,18 @@ int main(int argc, char* argv[]) {
         if( params.count("nodur") ) {
             cmdLine.dur = false;
         }
-        if( params.count("dur") ) {
+        if( params.count("nojournal") ) {
+            cmdLine.dur = false;
+        }
+        if( params.count("dur") || params.count( "journal" ) ) {
             cmdLine.dur = true;
         }
         if (params.count("durOptions")) {
             cmdLine.durOptions = params["durOptions"].as<int>();
         }
+        if (params.count("journalOptions")) {
+            cmdLine.durOptions = params["journalOptions"].as<int>();
+        }
         if (params.count("objcheck")) {
             objcheck = true;
         }
@@ -987,18 +1029,6 @@ int main(int argc, char* argv[]) {
         if ( params.count( "profile" ) ) {
             cmdLine.defaultProfile = params["profile"].as<int>();
         }
-        if ( params.count( "maxConns" ) ) {
-            int newSize = params["maxConns"].as<int>();
-            if ( newSize < 5 ) {
-                out() << "maxConns has to be at least 5" << endl;
-                dbexit( EXIT_BADOPTIONS );
-            }
-            else if ( newSize >= 10000000 ) {
-                out() << "maxConns can't be greater than 10000000" << endl;
-                dbexit( EXIT_BADOPTIONS );
-            }
-            connTicketHolder.resize( newSize );
-        }
         if (params.count("nounixsocket")) {
             noUnixSocket = true;
         }
@@ -1099,9 +1129,27 @@ namespace mongo {
         oss << "Backtrace:" << endl;
         printStackTrace( oss );
         rawOut( oss.str() );
+
+        if( cmdLine.dur ) { 
+            ::exit(EXIT_ABRUPT);
+        }
+
         dbexit( EXIT_ABRUPT );
     }
 
+    void abruptQuitWithAddrSignal( int signal, siginfo_t *siginfo, void * ) {
+        ostringstream oss;
+        oss << "Invalid";
+        if ( signal == SIGSEGV || signal == SIGBUS ) {
+            oss << " access";
+        } else {
+            oss << " operation";   
+        }
+        oss << " at address: " << siginfo->si_addr << endl;
+        rawOut( oss.str() );
+        abruptQuit( signal );   
+    }
+        
     sigset_t asyncSignals;
     // The above signals will be processed by this thread only, in order to
     // ensure the db and log mutexes aren't held.
@@ -1124,10 +1172,18 @@ namespace mongo {
     void setupSignals_ignoreHelper( int signal ) {}
 
     void setupSignals( bool inFork ) {
-        assert( signal(SIGSEGV, abruptQuit) != SIG_ERR );
-        assert( signal(SIGFPE, abruptQuit) != SIG_ERR );
+        struct sigaction addrSignals;
+        memset( &addrSignals, 0, sizeof( struct sigaction ) );
+        addrSignals.sa_sigaction = abruptQuitWithAddrSignal;
+        sigemptyset( &addrSignals.sa_mask );
+        addrSignals.sa_flags = SA_SIGINFO;
+       
+        assert( sigaction(SIGSEGV, &addrSignals, 0) == 0 );
+        assert( sigaction(SIGBUS, &addrSignals, 0) == 0 );
+        assert( sigaction(SIGILL, &addrSignals, 0) == 0 );
+        assert( sigaction(SIGFPE, &addrSignals, 0) == 0 );
+        
         assert( signal(SIGABRT, abruptQuit) != SIG_ERR );
-        assert( signal(SIGBUS, abruptQuit) != SIG_ERR );
         assert( signal(SIGQUIT, abruptQuit) != SIG_ERR );
         assert( signal(SIGPIPE, pipeSigHandler) != SIG_ERR );
 

db/dbcommands.cpp

@@ -94,9 +94,10 @@ namespace mongo {
         virtual void help( stringstream& help ) const {
             help << "return error status of the last operation on this connection\n"
                  << "options:\n"
-                 << "  fsync - fsync before returning, or wait for journal commit if running with --dur\n"
-                 << "  w - await replication to w servers (including self) before returning\n"
-                 << "  wtimeout - timeout for w in milliseconds";
+                 << "  { fsync:true } - fsync before returning, or wait for journal commit if running with --journal\n"
+                 << "  { j:true } - wait for journal commit if running with --journal\n"
+                 << "  { w:n } - await replication to n servers (including self) before returning\n"
+                 << "  { wtimeout:m} - timeout for w in m milliseconds";
         }
         bool run(const string& dbname, BSONObj& _cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
             LastError *le = lastError.disableForCommand();
@@ -125,7 +126,17 @@ namespace mongo {
                 }
             }
 
-            if ( cmdObj["fsync"].trueValue() ) {
+            if ( cmdObj["j"].trueValue() ) { 
+                if( !getDur().awaitCommit() ) {
+                    // --journal is off
+                    result.append("jnote", "journaling not enabled on this server");
+                }
+                if( cmdObj["fsync"].trueValue() ) { 
+                    errmsg = "fsync and j options are not used together";
+                    return false;
+                }
+            }
+            else if ( cmdObj["fsync"].trueValue() ) {
                 Timer t;
                 if( !getDur().awaitCommit() ) {
                     // if get here, not running with --dur
@@ -406,6 +417,8 @@ namespace mongo {
                     t.appendBool( "supported" , false );
                 }
 
+                timeBuilder.appendNumber( "middle of mem" , Listener::getElapsedTimeMillis() - start );
+
                 t.appendNumber( "mapped" , MemoryMappedFile::totalMappedLength() / ( 1024 * 1024 ) );
 
                 t.done();
@@ -821,7 +834,7 @@ namespace mongo {
             }
 
             list<BSONObj> all;
-            auto_ptr<DBClientCursor> i = db.getIndexes( toDeleteNs );
+            auto_ptr<DBClientCursor> i = db.query( dbname + ".system.indexes" , BSON( "ns" << toDeleteNs ) , 0 , 0 , 0 , QueryOption_SlaveOk );
             BSONObjBuilder b;
             while ( i->more() ) {
                 BSONObj o = i->next().getOwned();
@@ -838,6 +851,17 @@ namespace mongo {
 
             for ( list<BSONObj>::iterator i=all.begin(); i!=all.end(); i++ ) {
                 BSONObj o = *i;
+                if ( o.getIntField("v") > 0 ) {
+                    BSONObjBuilder b;
+                    BSONObjIterator i( o );
+                    while ( i.more() ) {
+                        BSONElement e = i.next();
+                        if ( str::equals( e.fieldName() , "v" ) )
+                            continue;
+                        b.append( e );
+                    }
+                    o = b.obj();
+                }
                 theDataFileMgr.insertWithObjMod( Namespace( toDeleteNs.c_str() ).getSisterNS( "system.indexes" ).c_str() , o , true );
             }
 
@@ -1540,7 +1564,7 @@ namespace mongo {
             uassert( 13049, "godinsert must specify a collection", !coll.empty() );
             string ns = dbname + "." + coll;
             BSONObj obj = cmdObj[ "obj" ].embeddedObjectUserCheck();
-            DiskLoc loc = theDataFileMgr.insertWithObjMod( ns.c_str(), obj, true );
+            theDataFileMgr.insertWithObjMod( ns.c_str(), obj, true );
             return true;
         }
     } cmdGodInsert;
@@ -1740,6 +1764,7 @@ namespace mongo {
         }
 
         if ( cmdObj["help"].trueValue() ) {
+            client.curop()->ensureStarted();
             stringstream ss;
             ss << "help for: " << c->name << " ";
             c->help( ss );
@@ -1764,6 +1789,7 @@ namespace mongo {
 
         if ( c->locktype() == Command::NONE ) {
             // we also trust that this won't crash
+            client.curop()->ensureStarted();
             string errmsg;
             int ok = c->run( dbname , cmdObj , errmsg , result , fromRepl );
             if ( ! ok )
@@ -1778,6 +1804,7 @@ namespace mongo {
         }
 
         mongolock lk( needWriteLock );
+        client.curop()->ensureStarted();
         Client::Context ctx( dbname , dbpath , &lk , c->requiresAuth() );
 
         try {
@@ -1811,7 +1838,6 @@ namespace mongo {
        returns true if ran a cmd
     */
     bool _runCommands(const char *ns, BSONObj& _cmdobj, BufBuilder &b, BSONObjBuilder& anObjBuilder, bool fromRepl, int queryOptions) {
-        cc().curop()->ensureStarted();
         string dbname = nsToDatabase( ns );
 
         if( logLevel >= 1 )

db/dbhelpers.cpp

@@ -85,6 +85,7 @@ namespace mongo {
             }
         }
         virtual long long nscanned() {
+            // We don't support yielding, so will always have c_.
             assert( c_.get() );
             return c_->nscanned();
         }
@@ -268,7 +269,7 @@ namespace mongo {
 
             getDur().commitIfNeeded();
 
-            if ( yield && ! cc->yieldSometimes() ) {
+            if ( yield && ! cc->yield() ) {
                 // cursor got finished by someone else, so we're done
                 cc.release(); // if the collection/db is dropped, cc may be deleted
                 break;

db/dur.cpp

@@ -211,14 +211,14 @@ namespace mongo {
             return p;
         }
 
-        void DurableImpl::commitIfNeeded() {
-#if defined(_DEBUG)
-            commitJob._nSinceCommitIfNeededCall = 0;
-#endif
+        bool DurableImpl::commitIfNeeded() {
+            DEV commitJob._nSinceCommitIfNeededCall = 0;
             if (commitJob.bytes() > UncommittedBytesLimit) { // should this also fire if CmdLine::DurAlwaysCommit?
                 stats.curr->_earlyCommits++;
                 groupCommit();
+                return true;
             }
+            return false;
         }
 
         /** Used in _DEBUG builds to check that we didn't overwrite the last intent
@@ -369,7 +369,7 @@ namespace mongo {
             // faults after remapping, so doing a little bit at a time will avoid big load spikes on
             // remapping.
             unsigned long long now = curTimeMicros64();
-            double fraction = (now-lastRemap)/20000000.0;
+            double fraction = (now-lastRemap)/2000000.0;
             lastRemap = now;
 
             rwlock lk(MongoFile::mmmutex, false);
@@ -522,7 +522,9 @@ namespace mongo {
             }
 
             // starvation on read locks could occur.  so if read lock acquisition is slow, try to get a
-            // write lock instead.  otherwise writes could use too much RAM.
+            // write lock instead.  otherwise journaling could be delayed too long (too much data will 
+            // not accumulate though, as commitIfNeeded logic will have executed in the meantime if there 
+            // has been writes)
             writelock lk;
             groupCommit();
         }
@@ -587,18 +589,11 @@ namespace mongo {
         void recover();
 
         void releasingWriteLock() {
-            try {
-#if defined(_DEBUG)
-                commitJob._nSinceCommitIfNeededCall = 0; // implicit commit if needed
-#endif
-                if (commitJob.bytes() > UncommittedBytesLimit || cmdLine.durOptions & CmdLine::DurAlwaysCommit) {
-                    stats.curr->_earlyCommits++;
-                    groupCommit();
-                }
-            }
-            catch(std::exception& e) {
-                log() << "exception in dur::releasingWriteLock causing immediate shutdown: " << e.what() << endl;
-                abort(); // based on myTerminate()
+            // implicit commitIfNeeded check on each write unlock
+            DEV commitJob._nSinceCommitIfNeededCall = 0; // implicit commit if needed
+            if( commitJob.bytes() > UncommittedBytesLimit || cmdLine.durOptions & CmdLine::DurAlwaysCommit ) {
+                stats.curr->_earlyCommits++;
+                groupCommit();
             }
         }
 

db/dur.h

@@ -96,8 +96,9 @@ namespace mongo {
                 (like creating an index or update with $atomic) can call this
                 whenever the db is in a sane state and it will prevent commits
                 from growing too large.
+                @return true if commited
             */
-            virtual void commitIfNeeded() = 0;
+            virtual bool commitIfNeeded() = 0;
 
             /** Declare write intent for a DiskLoc.  @see DiskLoc::writing() */
             inline DiskLoc& writingDiskLoc(DiskLoc& d) { return *((DiskLoc*) writingPtr(&d, sizeof(d))); }
@@ -172,7 +173,7 @@ namespace mongo {
             void createdFile(string filename, unsigned long long len) { }
             bool awaitCommit() { return false; }
             bool commitNow() { return false; }
-            void commitIfNeeded() { }
+            bool commitIfNeeded() { return false; }
             void setNoJournal(void *dst, void *src, unsigned len);
             void syncDataAndTruncateJournal() {}
         };
@@ -185,7 +186,7 @@ namespace mongo {
             void createdFile(string filename, unsigned long long len);
             bool awaitCommit();
             bool commitNow();
-            void commitIfNeeded();
+            bool commitIfNeeded();
             void setNoJournal(void *dst, void *src, unsigned len);
             void syncDataAndTruncateJournal();
         };

db/dur_commitjob.cpp

@@ -183,7 +183,7 @@ namespace mongo {
                 // remember intent. we will journal it in a bit
                 _wi.insertWriteIntent(p, len);
                 wassert( _wi._writes.size() <  2000000 );
-                assert(  _wi._writes.size() < 20000000 );
+                //assert(  _wi._writes.size() < 20000000 );
 
                 {
                     // a bit over conservative in counting pagebytes used
@@ -200,7 +200,19 @@ namespace mongo {
                                 log() << "debug nsincecommitifneeded:" << _nSinceCommitIfNeededCall << " bytes:" << _bytes << endl;
                         }
 #endif
-                        uassert(13623, "DR102 too much data written uncommitted", _bytes < UncommittedBytesLimit * 3);
+                        if (_bytes > UncommittedBytesLimit * 3) {
+                            static time_t lastComplain;
+                            static unsigned nComplains;
+                            // throttle logging
+                            if( ++nComplains < 100 || time(0) - lastComplain >= 60 ) {
+                                lastComplain = time(0);
+                                warning() << "DR102 too much data written uncommitted " << _bytes/1000000.0 << "MB" << endl;
+                                if( nComplains < 10 || nComplains % 10 == 0 ) {
+                                    // wassert makes getLastError show an error, so we just print stack trace
+                                    printStackTrace();
+                                }
+                            }
+                        }
                     }
                 }
             }

db/dur_journal.cpp

@@ -262,6 +262,7 @@ namespace mongo {
                         log() << "warning caught exception in preallocateFiles, continuing" << endl;
                     }
             }
+            j.open();
         }
 
         void removeOldJournalFile(path p) { 
@@ -515,7 +516,7 @@ namespace mongo {
 
             scoped_lock lk(_curLogFileMutex);
 
-            if ( inShutdown() )
+            if ( inShutdown() || !_curLogFile )
                 return;
 
             j.updateLSNFile();

db/dur_journalimpl.h

@@ -63,10 +63,10 @@ namespace mongo {
                     _open();
             }
 
-        private:
             /** open a journal file to journal operations to. */
             void open();
 
+        private:
             void _open();
             void closeCurrentJournalFile();
             void removeUnneededJournalFiles();

db/dur_recover.cpp

@@ -73,16 +73,17 @@ namespace mongo {
                     if( m.count(u) ) {
                         uasserted(13531, str::stream() << "unexpected files in journal directory " << dir.string() << " : " << fileName);
                     }
-                    if( !m.empty() && !m.count(u-1) ) {
-                        uasserted(13532,
-                                  str::stream() << "unexpected file in journal directory " << dir.string()
-                                  << " : " << fileName << " : can't find its preceeding file");
-                    }
                     m.insert( pair<unsigned,path>(u,filepath) );
                 }
             }
-            for( map<unsigned,path>::iterator i = m.begin(); i != m.end(); ++i )
+            for( map<unsigned,path>::iterator i = m.begin(); i != m.end(); ++i ) {
+                if( i != m.begin() && m.count(i->first - 1) == 0 ) {
+                    uasserted(13532,
+                    str::stream() << "unexpected file in journal directory " << dir.string()
+                      << " : " << filesystem::path(i->second).leaf() << " : can't find its preceeding file");
+                }
                 files.push_back(i->second);
+            }
         }
 
         /** read through the memory mapped data of a journal file (journal/j._<n> file)
@@ -186,8 +187,10 @@ namespace mongo {
         }
 
         RecoveryJob::~RecoveryJob() {
-            if( !_mmfs.empty() )
-                close();
+            DESTRUCTOR_GUARD(
+                if( !_mmfs.empty() )
+                    close();
+            )
         }
 
         void RecoveryJob::close() {
@@ -445,8 +448,8 @@ namespace mongo {
             }
         } brunittest;
 
-
-        RecoveryJob RecoveryJob::_instance;
+        // can't free at termination because order of destruction of global vars is arbitrary
+        RecoveryJob &RecoveryJob::_instance = *(new RecoveryJob());
 
     } // namespace dur
 

db/dur_recover.h

@@ -15,7 +15,7 @@ namespace mongo {
          */
         class RecoveryJob : boost::noncopyable {
         public:
-            RecoveryJob() :_lastDataSyncedFromLastRun(0), _mx("recovery") { _lastSeqMentionedInConsoleLog = 1; }
+            RecoveryJob() :_lastDataSyncedFromLastRun(0), _mx("recovery"), _recovering(false) { _lastSeqMentionedInConsoleLog = 1; }
             void go(vector<path>& files);
             ~RecoveryJob();
             void processSection(const void *, unsigned len);
@@ -39,7 +39,7 @@ namespace mongo {
 
             bool _recovering; // are we in recovery or WRITETODATAFILES
 
-            static RecoveryJob _instance;
+            static RecoveryJob &_instance;
         };
     }
 }

db/geo/2d.cpp

@@ -1125,7 +1125,14 @@ namespace mongo {
         virtual Record* _current() { assert(ok()); return _cur->_loc.rec(); }
         virtual BSONObj current() { assert(ok()); return _cur->_o; }
         virtual DiskLoc currLoc() { assert(ok()); return _cur->_loc; }
-        virtual bool advance() { _cur++; incNscanned(); return ok(); }
+        virtual bool advance() {
+            if( ok() ){
+                _cur++;
+                incNscanned();
+                return ok();
+            }
+            return false;
+        }
         virtual BSONObj currKey() const { return _cur->_key; }
 
         virtual string toString() {
@@ -1144,6 +1151,10 @@ namespace mongo {
 
         virtual long long nscanned() { return _nscanned; }
 
+        virtual CoveredIndexMatcher *matcher() const {
+        	return _s->_hopper->_matcher.get();
+        }
+
         shared_ptr<GeoSearch> _s;
         GeoHopper::Holder::iterator _cur;
         GeoHopper::Holder::iterator _end;
@@ -1212,6 +1223,9 @@ namespace mongo {
         virtual DiskLoc currLoc() { assert(ok()); return _cur._loc; }
         virtual BSONObj currKey() const { return _cur._key; }
 
+        virtual CoveredIndexMatcher *matcher() const {
+        	return _matcher.get();
+        }
 
         virtual bool moreToDo() = 0;
         virtual void fillStack() = 0;

db/index.cpp

@@ -127,7 +127,6 @@ namespace mongo {
     void getIndexChanges(vector<IndexChanges>& v, NamespaceDetails& d, BSONObj newObj, BSONObj oldObj, bool &changedId) {
         int z = d.nIndexesBeingBuilt();
         v.resize(z);
-        NamespaceDetails::IndexIterator i = d.ii();
         for( int i = 0; i < z; i++ ) {
             IndexDetails& idx = d.idx(i);
             BSONObj idxKey = idx.info.obj().getObjectField("key"); // eg { ts : 1 }

db/index.h

@@ -145,6 +145,13 @@ namespace mongo {
 
         const IndexSpec& getSpec() const;
 
+        void checkVersion() const {
+            // TODO: cache?
+            massert( 13658 , 
+                     str::stream() << "using a newer index version: " << info.obj() << " v: " << info.obj().getIntField("v" ) , 
+                     info.obj().getIntField("v") <= 0 );
+        }
+
         string toString() const {
             return info.obj().toString();
         }

db/instance.cpp

@@ -488,7 +488,7 @@ namespace mongo {
 
         writelock lk(ns);
         // if this ever moves to outside of lock, need to adjust check Client::Context::_finishInit
-        if ( ! broadcast & handlePossibleShardedMessage( m , 0 ) )
+        if ( ! broadcast && handlePossibleShardedMessage( m , 0 ) )
             return;
 
         Client::Context ctx(ns);

db/lasterror.cpp

@@ -28,7 +28,6 @@ namespace mongo {
 
     LastError LastError::noError;
     LastErrorHolder lastError;
-    mongo::mutex LastErrorHolder::_idsmutex("LastErrorHolder");
 
     bool isShell = false;
     void raiseError(int code , const char *msg) {
@@ -71,30 +70,19 @@ namespace mongo {
             b.appendBool( "updatedExisting", updatedExisting == True );
         if ( upsertedId.isSet() )
             b.append( "upserted" , upsertedId );
-        if ( writebackId.isSet() )
+        if ( writebackId.isSet() ) {
             b.append( "writeback" , writebackId );
+            b.append( "instanceIdent" , prettyHostName() ); // this can be any unique string
+        }
         b.appendNumber( "n", nObjects );
 
         return ! msg.empty();
     }
 
     LastErrorHolder::~LastErrorHolder() {
-        for ( IDMap::iterator i = _ids.begin(); i != _ids.end(); ++i ) {
-            delete i->second.lerr;
-            i->second.lerr = 0;
-        }
-        _ids.clear();
     }
 
 
-    void LastErrorHolder::setID( int id ) {
-        _id.set( id );
-    }
-
-    int LastErrorHolder::getID() {
-        return _id.get();
-    }
-
     LastError * LastErrorHolder::disableForCommand() {
         LastError *le = _get();
         assert( le );
@@ -111,77 +99,31 @@ namespace mongo {
     }
 
     LastError * LastErrorHolder::_get( bool create ) {
-        int id = _id.get();
-        if ( id == 0 ) {
-            LastError * le = _tl.get();
-            if ( ! le && create ) {
-                le = new LastError();
-                _tl.reset( le );
-            }
-            return le;
+        LastError * le = _tl.get();
+        if ( ! le && create ) {
+            le = new LastError();
+            _tl.reset( le );
         }
-
-        scoped_lock lock(_idsmutex);
-        map<int,Status>::iterator i = _ids.find( id );
-        if ( i == _ids.end() ) {
-            if ( ! create )
-                return 0;
-
-            LastError * le = new LastError();
-            Status s;
-            s.time = time(0);
-            s.lerr = le;
-            _ids[id] = s;
-            return le;
-        }
-
-        Status &status = i->second;
-        status.time = time(0);
-        return status.lerr;
-    }
-
-    void LastErrorHolder::remove( int id ) {
-        scoped_lock lock(_idsmutex);
-        map<int,Status>::iterator i = _ids.find( id );
-        if ( i == _ids.end() )
-            return;
-
-        delete i->second.lerr;
-        _ids.erase( i );
+        return le;
     }
 
     void LastErrorHolder::release() {
-        int id = _id.get();
-        if ( id == 0 ) {
-            _tl.release();
-            return;
-        }
-
-        remove( id );
+        _tl.release();
     }
 
     /** ok to call more than once. */
     void LastErrorHolder::initThread() {
-        if( _tl.get() ) return;
-        assert( _id.get() == 0 );
-        _tl.reset( new LastError() );
+        if( ! _tl.get() ) 
+            _tl.reset( new LastError() );
     }
 
     void LastErrorHolder::reset( LastError * le ) {
-        int id = _id.get();
-        if ( id == 0 ) {
-            _tl.reset( le );
-            return;
-        }
-
-        scoped_lock lock(_idsmutex);
-        Status & status = _ids[id];
-        status.time = time(0);
-        status.lerr = le;
+        _tl.reset( le );
     }
 
     void prepareErrForNewRequest( Message &m, LastError * err ) {
         // a killCursors message shouldn't affect last error
+        assert( err );
         if ( m.operation() == dbKillCursors ) {
             err->disabled = true;
         }
@@ -191,60 +133,10 @@ namespace mongo {
         }
     }
 
-    LastError * LastErrorHolder::startRequest( Message& m , int clientId ) {
-        assert( clientId );
-        setID( clientId );
-
-        LastError * le = _get( true );
+    LastError * LastErrorHolder::startRequest( Message& m , LastError * le ) {
+        assert( le );
         prepareErrForNewRequest( m, le );
         return le;
     }
 
-    void LastErrorHolder::startRequest( Message& m , LastError * connectionOwned ) {
-        prepareErrForNewRequest( m, connectionOwned );
-    }
-
-    void LastErrorHolder::disconnect( int clientId ) {
-        if ( clientId )
-            remove(clientId);
-    }
-
-    struct LastErrorHolderTest : public UnitTest {
-    public:
-
-        void test( int i ) {
-            _tl.set( i );
-            assert( _tl.get() == i );
-        }
-
-        void tlmaptest() {
-            test( 1 );
-            test( 12123123 );
-            test( -123123 );
-            test( numeric_limits<int>::min() );
-            test( numeric_limits<int>::max() );
-        }
-
-        void run() {
-            tlmaptest();
-
-            LastError * a = new LastError();
-            LastError * b = new LastError();
-
-            LastErrorHolder holder;
-            holder.reset( a );
-            assert( a == holder.get() );
-            holder.setID( 1 );
-            assert( 0 == holder.get() );
-            holder.reset( b );
-            assert( b == holder.get() );
-            holder.setID( 0 );
-            assert( a == holder.get() );
-
-            holder.remove( 1 );
-        }
-
-        ThreadLocalValue<int> _tl;
-    } lastErrorHolderTest;
-
 } // namespace mongo

db/lasterror.h

@@ -100,14 +100,14 @@ namespace mongo {
 
     extern class LastErrorHolder {
     public:
-        LastErrorHolder() : _id( 0 ) {}
+        LastErrorHolder(){}
         ~LastErrorHolder();
 
         LastError * get( bool create = false );
         LastError * getSafe() {
             LastError * le = get(false);
             if ( ! le ) {
-                log( LL_ERROR ) << " no LastError!  id: " << getID() << endl;
+                error() << " no LastError!" << endl;
                 assert( le );
             }
             return le;
@@ -120,18 +120,12 @@ namespace mongo {
         /** ok to call more than once. */
         void initThread();
 
-        /**
-         * id of 0 means should use thread local management
-         */
-        void setID( int id );
         int getID();
-
-        void remove( int id );
+        
         void release();
 
         /** when db receives a message/request, call this */
-        void startRequest( Message& m , LastError * connectionOwned );
-        LastError * startRequest( Message& m , int clientId );
+        LastError * startRequest( Message& m , LastError * connectionOwned );
 
         void disconnect( int clientId );
 
@@ -139,17 +133,12 @@ namespace mongo {
         // disable causes get() to return 0.
         LastError *disableForCommand(); // only call once per command invocation!
     private:
-        ThreadLocalValue<int> _id;
         boost::thread_specific_ptr<LastError> _tl;
 
         struct Status {
             time_t time;
             LastError *lerr;
         };
-        typedef map<int,Status> IDMap;
-
-        static mongo::mutex _idsmutex;
-        IDMap _ids;
     } lastError;
 
     void raiseError(int code , const char *msg);

db/mongommf.cpp

@@ -342,11 +342,7 @@ namespace mongo {
 
     bool MongoMMF::create(string fname, unsigned long long& len, bool sequentialHint) {
         setPath(fname);
-        bool preExisting = MemoryMappedFile::exists(fname.c_str());
         _view_write = map(fname.c_str(), len, sequentialHint ? SEQUENTIAL : 0);
-        if( cmdLine.dur && _view_write && !preExisting ) {
-            getDur().createdFile(fname, len);
-        }
         return finishOpening();
     }
 

db/namespace.cpp

@@ -182,6 +182,7 @@ namespace mongo {
             maybeMkdir();
             unsigned long long l = lenForNewNsFiles;
             if( f.create(pathString, l, true) ) {
+                getDur().createdFile(pathString, l); // always a new file
                 len = l;
                 assert( len == lenForNewNsFiles );
                 p = f.getView();
@@ -194,6 +195,7 @@ namespace mongo {
             dbexit( EXIT_FS );
         }
 
+
         assert( len <= 0x7fffffff );
         ht = new HashTable<Namespace,NamespaceDetails>(p, (int) len, "namespace index");
         if( checkNsFilesOnLoad )
@@ -596,6 +598,17 @@ namespace mongo {
         }
     }
 
+    void NamespaceDetailsTransient::eraseForPrefix(const char *prefix) {
+        assertInWriteLock();
+        vector< string > found;
+        for( ouriter i = _map.begin(); i != _map.end(); ++i )
+            if ( strncmp( i->first.c_str(), prefix, strlen( prefix ) ) == 0 )
+                found.push_back( i->first );
+        for( vector< string >::iterator i = found.begin(); i != found.end(); ++i ) {
+            _map.erase(*i);
+        }
+    }
+
     void NamespaceDetailsTransient::computeIndexKeys() {
         _keysComputed = true;
         _indexKeys.clear();
@@ -646,7 +659,7 @@ namespace mongo {
         // index details across commands are in cursors and nsd
         // transient (including query cache) so clear these.
         ClientCursor::invalidate( from );
-        NamespaceDetailsTransient::clearForPrefix( from );
+        NamespaceDetailsTransient::eraseForPrefix( from );
 
         NamespaceDetails *details = ni->details( from );
         ni->add_ns( to, *details );

db/namespace.h

@@ -425,6 +425,7 @@ namespace mongo {
            Can be useful as index namespaces share the same start as the regular collection.
            SLOW - sequential scan of all NamespaceDetailsTransient objects */
         static void clearForPrefix(const char *prefix);
+        static void eraseForPrefix(const char *prefix);
 
         /* indexKeys() cache ---------------------------------------------------- */
         /* assumed to be in write lock for this */

db/oplog.cpp

@@ -480,7 +480,48 @@ namespace mongo {
         }
     }
 
-    void applyOperation_inlock(const BSONObj& op , bool fromRepl ) {
+    bool shouldRetry(const BSONObj& o, const string& hn) {
+        OplogReader missingObjReader;
+
+        // we don't have the object yet, which is possible on initial sync.  get it.
+        log() << "replication info adding missing object" << endl; // rare enough we can log
+        uassert(15916, str::stream() << "Can no longer connect to initial sync source: " << hn, missingObjReader.connect(hn));
+
+        const char *ns = o.getStringField("ns");
+        // might be more than just _id in the update criteria
+        BSONObj query = BSONObjBuilder().append(o.getObjectField("o2")["_id"]).obj();
+        BSONObj missingObj;
+        try {
+            missingObj = missingObjReader.findOne(ns, query);
+        } catch(DBException& e) {
+            log() << "replication assertion fetching missing object: " << e.what() << endl;
+            throw;
+        }
+
+        if( missingObj.isEmpty() ) {
+            log() << "replication missing object not found on source. presumably deleted later in oplog" << endl;
+            log() << "replication object: " << o.getObjectField("o2").toString() << endl;
+            log() << "replication o object: " << o.getObjectField("o").toString() << endl;
+
+            return false;
+        }
+        else {
+            Client::Context ctx(ns);
+            DiskLoc d = theDataFileMgr.insert(ns, (void*) missingObj.objdata(), missingObj.objsize());
+            uassert(15917, "Got bad disk location when attempting to insert", !d.isNull());
+
+            return true;
+        }
+    }
+
+    /** @param fromRepl false if from ApplyOpsCmd
+        @return true if was and update should have happened and the document DNE.  see replset initial sync code.
+     */
+    bool applyOperation_inlock(const BSONObj& op , bool fromRepl ) {
+        assertInWriteLock();
+        LOG(6) << "applying op: " << op << endl;
+        bool failedUpdate = false;
+
         OpCounters * opCounters = fromRepl ? &replOpCounters : &globalOpCounters;
 
         if( logLevel >= 6 )
@@ -530,9 +571,45 @@ namespace mongo {
         }
         else if ( *opType == 'u' ) {
             opCounters->gotUpdate();
-
+            // dm do we create this for a capped collection?
+            //  - if not, updates would be slow
+            //    - but if were by id would be slow on primary too so maybe ok
+            //    - if on primary was by another key and there are other indexes, this could be very bad w/out an index
+            //  - if do create, odd to have on secondary but not primary.  also can cause secondary to block for
+            //    quite a while on creation.
             RARELY ensureHaveIdIndex(ns); // otherwise updates will be super slow
-            updateObjects(ns, o, op.getObjectField("o2"), /*upsert*/ op.getBoolField("b"), /*multi*/ false, /*logop*/ false , debug );
+            OpDebug debug;
+            BSONObj updateCriteria = op.getObjectField("o2");
+            bool upsert = op.getBoolField("b");
+            UpdateResult ur = updateObjects(ns, o, updateCriteria, upsert, /*multi*/ false, /*logop*/ false , debug );
+            if( ur.num == 0 ) {
+                if( ur.mod ) {
+                    if( updateCriteria.nFields() == 1 ) {
+                        // was a simple { _id : ... } update criteria
+                        failedUpdate = true;
+                        // todo: probably should assert in these failedUpdate cases if not in initialSync
+                    }
+                    // need to check to see if it isn't present so we can set failedUpdate correctly.
+                    // note that adds some overhead for this extra check in some cases, such as an updateCriteria
+                    // of the form
+                    //   { _id:..., { x : {$size:...} }
+                    // thus this is not ideal.
+                    else if( Helpers::findById(nsdetails(ns), updateCriteria).isNull() ) {
+                        failedUpdate = true;
+                    }
+                    else {
+                        // it's present; zero objects were updated because of additional specifiers in the query for idempotence
+                    }
+                }
+                else {
+                    // this could happen benignly on an oplog duplicate replay of an upsert
+                    // (because we are idempotent),
+                    // if an regular non-mod update fails the item is (presumably) missing.
+                    if( !upsert ) {
+                        failedUpdate = true;
+                    }
+                }
+            }
         }
         else if ( *opType == 'd' ) {
             opCounters->gotDelete();
@@ -557,7 +634,7 @@ namespace mongo {
             ss << "unknown opType [" << opType << "]";
             throw MsgAssertionException( 13141 , ss.str() );
         }
-
+        return failedUpdate;
     }
 
     class ApplyOpsCmd : public Command {

db/oplog.h

@@ -212,6 +212,12 @@ namespace mongo {
      * take an op and apply locally
      * used for applying from an oplog
      * @param fromRepl really from replication or for testing/internal/command/etc...
+     * Returns if the op was an update that could not be applied (true on failure)
      */
-    void applyOperation_inlock(const BSONObj& op , bool fromRepl = true );
+    bool applyOperation_inlock(const BSONObj& op , bool fromRepl = true );
+
+    /**
+     * If applyOperation_inlock should be called again after an update fails.
+     */
+    bool shouldRetry(const BSONObj& op , const string& hn);
 }

db/pdfile.cpp

@@ -252,6 +252,10 @@ namespace mongo {
             while ( size > 0 ) {
                 int max = MongoDataFile::maxSize() - DataFileHeader::HeaderSize;
                 int desiredExtentSize = (int) (size > max ? max : size);
+                if ( desiredExtentSize < Extent::minSize() ) {
+                    desiredExtentSize = Extent::minSize();
+                }
+                desiredExtentSize &= 0xffffff00;
                 Extent *e = database->allocExtent( ns, desiredExtentSize, newCapped );
                 size -= e->length;
             }
@@ -403,7 +407,7 @@ namespace mongo {
             uassert( 10084 , "can't map file memory - mongo requires 64 bit build for larger datasets", _mb != 0);
         else
             uassert( 10085 , "can't map file memory", _mb != 0);
-        header()->init(fileNo, size);
+        header()->init(fileNo, size, filename);
     }
 
     void MongoDataFile::flush( bool sync ) {
@@ -435,11 +439,11 @@ namespace mongo {
 
     Extent* MongoDataFile::createExtent(const char *ns, int approxSize, bool newCapped, int loops) {
         massert( 10357 ,  "shutdown in progress", ! inShutdown() );
-        massert( 10358 ,  "bad new extent size", approxSize >= 0 && approxSize <= Extent::maxSize() );
+        massert( 10358 ,  "bad new extent size", approxSize >= Extent::minSize() && approxSize <= Extent::maxSize() );
         massert( 10359 ,  "header==0 on new extent: 32 bit mmap space exceeded?", header() ); // null if file open failed
         int ExtentSize = approxSize <= header()->unusedLength ? approxSize : header()->unusedLength;
         DiskLoc loc;
-        if ( ExtentSize <= 0 ) {
+        if ( ExtentSize < Extent::minSize() ) {
             /* not there could be a lot of looping here is db just started and
                no files are open yet.  we might want to do something about that. */
             if ( loops > 8 ) {
@@ -802,6 +806,7 @@ namespace mongo {
         result.append("ns", name.c_str());
         ClientCursor::invalidate(name.c_str());
         Top::global.collectionDropped( name );
+        NamespaceDetailsTransient::eraseForPrefix( name.c_str() );
         dropNS(name);
     }
 
@@ -1187,13 +1192,17 @@ namespace mongo {
             op->setMessage( "index: (3/3) btree-middle" );
             log(t.seconds() > 10 ? 0 : 1 ) << "\t done building bottom layer, going to commit" << endl;
             btBuilder.commit();
-            wassert( btBuilder.getn() == nkeys || dropDups );
+            if ( btBuilder.getn() != nkeys && ! dropDups ) {
+                warning() << "not all entries were added to the index, probably some keys were too large" << endl;
+            }
         }
 
         log(1) << "\t fastBuildIndex dupsToDrop:" << dupsToDrop.size() << endl;
 
-        for( list<DiskLoc>::iterator i = dupsToDrop.begin(); i != dupsToDrop.end(); i++ )
+        for( list<DiskLoc>::iterator i = dupsToDrop.begin(); i != dupsToDrop.end(); i++ ){
             theDataFileMgr.deleteRecord( ns, i->rec(), *i, false, true );
+            getDur().commitIfNeeded();
+        }
 
         return n;
     }
@@ -1248,6 +1257,8 @@ namespace mongo {
                 n++;
                 progress.hit();
 
+                getDur().commitIfNeeded();
+
                 if ( n % 128 == 0 && !cc->yield() ) {
                     cc.release();
                     uasserted(12584, "cursor gone during bg index");
@@ -1281,7 +1292,7 @@ namespace mongo {
             prep(ns.c_str(), d);
             assert( idxNo == d->nIndexes );
             try {
-                idx.head = BtreeBucket::addBucket(idx);
+                idx.head.writing() = BtreeBucket::addBucket(idx);
                 n = addExistingToIndex(ns.c_str(), d, idx, idxNo);
             }
             catch(...) {

db/pdfile.h

@@ -272,6 +272,7 @@ namespace mongo {
         Extent* getPrevExtent() { return xprev.isNull() ? 0 : DataFileMgr::getExtent(xprev); }
 
         static int maxSize();
+        static int minSize() { return 0x100; }
         /**
          * @param len lengt of record we need
          * @param lastRecord size of last extent which is a factor in next extent size
@@ -325,11 +326,12 @@ namespace mongo {
 
         bool uninitialized() const { return version == 0; }
 
-        void init(int fileno, int filelength) {
+        void init(int fileno, int filelength, const char* filename) {
             if ( uninitialized() ) {
                 if( !(filelength > 32768 ) ) { 
                     massert(13640, str::stream() << "DataFileHeader looks corrupt at file open filelength:" << filelength << " fileno:" << fileno, false);
                 }
+                getDur().createdFile(filename, filelength);
                 assert( HeaderSize == 8192 );
                 DataFileHeader *h = getDur().writing(this);
                 h->fileLength = filelength;

db/query.cpp

@@ -76,8 +76,7 @@ namespace mongo {
             }
         }
         virtual long long nscanned() {
-            assert( c_.get() );
-            return c_->nscanned();
+            return c_.get() ? c_->nscanned() : _nscanned;
         }
         virtual void next() {
             if ( !c_->ok() ) {
@@ -380,8 +379,16 @@ namespace mongo {
                     }
                 }
                 c->advance();
-            }
 
+                if ( ! cc->yieldSometimes() ) {
+                    ClientCursor::erase(cursorid);
+                    cursorid = 0;
+                    cc = 0;
+                    p.deleted();
+                    break;
+                }
+            }
+            
             if ( cc ) {
                 cc->updateLocation();
                 cc->mayUpgradeStorage();
@@ -408,6 +415,7 @@ namespace mongo {
             _ns(ns), _capped(false), _count(), _myCount(),
             _skip( spec["skip"].numberLong() ),
             _limit( spec["limit"].numberLong() ),
+            _nscanned(),
             _bc() {
         }
 
@@ -422,19 +430,22 @@ namespace mongo {
         }
 
         virtual long long nscanned() {
-            assert( _c.get() );
-            return _c->nscanned();
+            return _c.get() ? _c->nscanned() : _nscanned;
         }
 
         virtual bool prepareToYield() {
-            if ( ! _cc ) {
+            if ( _c && !_cc ) {
                 _cc.reset( new ClientCursor( QueryOption_NoCursorTimeout , _c , _ns.c_str() ) );
             }
-            return _cc->prepareToYield( _yieldData );
+            if ( _cc ) {
+	            return _cc->prepareToYield( _yieldData );
+            }
+            // no active cursor - ok to yield
+            return true;
         }
 
         virtual void recoverFromYield() {
-            if ( !ClientCursor::recoverFromYield( _yieldData ) ) {
+            if ( _cc && !ClientCursor::recoverFromYield( _yieldData ) ) {
                 _c.reset();
                 _cc.reset();
 
@@ -453,6 +464,7 @@ namespace mongo {
                 return;
             }
 
+            _nscanned = _c->nscanned();
             if ( _bc ) {
                 if ( _firstMatch.isEmpty() ) {
                     _firstMatch = _bc->currKeyNode().key.copy();
@@ -515,6 +527,7 @@ namespace mongo {
         long long _myCount;
         long long _skip;
         long long _limit;
+        long long _nscanned;
         shared_ptr<Cursor> _c;
         BSONObj _query;
         BtreeCursor * _bc;
@@ -690,11 +703,15 @@ namespace mongo {
                 return _findingStartCursor->prepareToYield();
             }
             else {
-                if ( ! _cc ) {
+                if ( _c && !_cc ) {
                     _cc.reset( new ClientCursor( QueryOption_NoCursorTimeout , _c , _pq.ns() ) );
                 }
-                return _cc->prepareToYield( _yieldData );
+                if ( _cc ) {
+	                return _cc->prepareToYield( _yieldData );
+                }
             }
+            // no active cursor - ok to yield
+            return true;
         }
 
         virtual void recoverFromYield() {
@@ -703,7 +720,7 @@ namespace mongo {
             if ( _findingStartCursor.get() ) {
                 _findingStartCursor->recoverFromYield();
             }
-            else if ( ! ClientCursor::recoverFromYield( _yieldData ) ) {
+            else if ( _cc && !ClientCursor::recoverFromYield( _yieldData ) ) {
                 _c.reset();
                 _cc.reset();
                 _so.reset();
@@ -724,8 +741,7 @@ namespace mongo {
             if ( _findingStartCursor.get() ) {
                 return 0; // should only be one query plan, so value doesn't really matter.
             }
-            assert( _c.get() );
-            return _c->nscanned();
+            return _c.get() ? _c->nscanned() : _nscanned;
         }
 
         virtual void next() {
@@ -842,6 +858,7 @@ namespace mongo {
 
         // this plan won, so set data for response broadly
         void finish( bool stop ) {
+            massert( 13638, "client cursor dropped during explain query yield", !_pq.isExplain() || _c.get() );
 
             if ( _pq.isExplain() ) {
                 _n = _inMemSort ? _so->size() : _n;
@@ -863,7 +880,6 @@ namespace mongo {
             }
 
             if ( _pq.isExplain() ) {
-                massert( 13638, "client cursor dropped during explain query yield", _c.get() );
                 _eb.noteScan( _c.get(), _nscanned, _nscannedObjects, _n, scanAndOrderRequired(),
                               _curop.elapsedMillis(), useHints && !_pq.getHint().eoo(), _nYields ,
                               _nChunkSkips, _keyFieldsOnly.get() > 0 );
@@ -1147,7 +1163,7 @@ namespace mongo {
                 cc = new ClientCursor(queryOptions, multi, ns, jsobj.getOwned());
             }
             else {
-                cursor->setMatcher( dqo.matcher() );
+                if( ! cursor->matcher() ) cursor->setMatcher( dqo.matcher() );
                 cc = new ClientCursor( queryOptions, cursor, ns, jsobj.getOwned() );
             }
             cursorid = cc->cursorid();

db/queryoptimizer.cpp

@@ -914,7 +914,8 @@ doneCheckOrder:
         }
 
         if ( !id ) {
-            errmsg = (string)"no index found for specified keyPattern: " + keyPattern.toString();
+            errmsg = str::stream() << "no index found for specified keyPattern: " << keyPattern.toString() 
+                                   << " min: " << min << " max: " << max;
             return 0;
         }
 

db/queryoptimizer.h

@@ -449,7 +449,8 @@ namespace mongo {
             auto_ptr< FieldRangeSet > frs( new FieldRangeSet( ns, query ) );
             auto_ptr< FieldRangeSet > origFrs( new FieldRangeSet( *frs ) );
             shared_ptr< Cursor > ret = QueryPlanSet( ns, frs, origFrs, query, sort ).getBestGuess()->newCursor();
-            if ( !query.isEmpty() ) {
+            // If we don't already have a matcher, supply one.
+            if ( !query.isEmpty() && ! ret->matcher() ) {
                 shared_ptr< CoveredIndexMatcher > matcher( new CoveredIndexMatcher( query, ret->indexKeyPattern() ) );
                 ret->setMatcher( matcher );
             }

db/repl.cpp

@@ -851,7 +851,10 @@ namespace mongo {
 
     void ReplSource::applyOperation(const BSONObj& op) {
         try {
-            applyOperation_inlock( op );
+            bool failedUpdate = applyOperation_inlock( op );
+            if (failedUpdate && shouldRetry(op, hostName)) {
+                uassert(15914, "Failure retrying initial sync update", applyOperation_inlock(op));
+            }
         }
         catch ( UserException& e ) {
             log() << "sync: caught user assertion " << e << " while applying op: " << op << endl;;
@@ -1351,6 +1354,7 @@ namespace mongo {
                             setLastSavedLocalTs( nextLastSaved );
                         }
                     }
+
                     if( oplogReader.awaitCapable() && tailing )
                         okResultCode = 0; // don't sleep
                     syncedTo = nextOpTime;
@@ -1426,6 +1430,8 @@ namespace mongo {
                         break;
                     }
                     op = oplogReader.next();
+
+                    getDur().commitIfNeeded();
                 }
             }
         }

db/repl.h

@@ -134,7 +134,7 @@ namespace mongo {
     public:
         OplogReader oplogReader;
 
-        static void applyOperation(const BSONObj& op);
+        void applyOperation(const BSONObj& op);
         bool replacing; // in "replace mode" -- see CmdReplacePeer
         bool paired; // --pair in use
         string hostName;    // ip addr or hostname plus optionally, ":<port>"

db/repl/consensus.cpp

@@ -154,6 +154,12 @@ namespace mongo {
             log() << "couldn't find member with id " << whoid << rsLog;
             vote = -10000;
         }
+        else if( primary && primary == rs._self && rs.lastOpTimeWritten >= hopeful->hbinfo().opTime ) {
+            // hbinfo is not updated, so we have to check the primary's last optime separately
+            log() << "I am already primary, " << hopeful->fullName()
+                  << " can try again once I've stepped down" << rsLog;
+            vote = -10000;
+        }
         else if( primary && primary->hbinfo().opTime >= hopeful->hbinfo().opTime ) {
             // other members might be aware of more up-to-date nodes
             log() << hopeful->fullName() << " is trying to elect itself but " <<

db/repl/manager.cpp

@@ -64,6 +64,9 @@ namespace mongo {
 
     void Manager::starting() {
         Client::initThread("rs Manager");
+        if (!noauth) {
+            cc().getAuthenticationInfo()->authorize("local");
+        }
     }
 
     void Manager::noteARemoteIsPrimary(const Member *m) {

db/repl/replset_commands.cpp

@@ -274,7 +274,7 @@ namespace mongo {
                     s << p("Not using --replSet");
                 else  {
                     s << p("Still starting up, or else set is not yet " + a("http://www.mongodb.org/display/DOCS/Replica+Set+Configuration#InitialSetup", "", "initiated")
-                           + ".<br>" + ReplSet::startupStatusMsg);
+                           + ".<br>" + ReplSet::startupStatusMsg.get());
                 }
             }
             else {
@@ -305,7 +305,7 @@ namespace mongo {
                     s << p("Not using --replSet");
                 else  {
                     s << p("Still starting up, or else set is not yet " + a("http://www.mongodb.org/display/DOCS/Replica+Set+Configuration#InitialSetup", "", "initiated")
-                           + ".<br>" + ReplSet::startupStatusMsg);
+                           + ".<br>" + ReplSet::startupStatusMsg.get());
                 }
             }
             else {

db/repl/rs.cpp

@@ -23,6 +23,7 @@
 #include "../../s/d_logic.h"
 #include "rs.h"
 #include "connections.h"
+#include "../repl.h"
 
 namespace mongo {
 
@@ -320,7 +321,7 @@ namespace mongo {
     }
 
     ReplSetImpl::StartupStatus ReplSetImpl::startupStatus = PRESTART;
-    string ReplSetImpl::startupStatusMsg;
+    DiagStr ReplSetImpl::startupStatusMsg;
 
     extern BSONObj *getLastErrorDefault;
 
@@ -380,6 +381,10 @@ namespace mongo {
                 conn.setTimeout(c.ho.heartbeatTimeoutMillis/1000.0);
             }
             if( me == 0 ) {
+                // initial startup with fastsync
+                if (!reconf && replSettings.fastsync) {
+                    return false;
+                }
                 // log() << "replSet config : " << _cfg->toString() << rsLog;
                 log() << "replSet error self not present in the repl set configuration:" << rsLog;
                 log() << c.toString() << rsLog;

db/repl/rs.h

@@ -244,7 +244,7 @@ namespace mongo {
             EMPTYUNREACHABLE=4, STARTED=5, SOON=6
         };
         static StartupStatus startupStatus;
-        static string startupStatusMsg;
+        static DiagStr startupStatusMsg;
         static string stateAsHtml(MemberState state);
 
         /* todo thread */
@@ -344,7 +344,7 @@ namespace mongo {
         void _syncThread();
         bool tryToGoLiveAsASecondary(OpTime&); // readlocks
         void syncTail();
-        void syncApply(const BSONObj &o);
+        bool syncApply(const BSONObj &o);
         unsigned _syncRollback(OplogReader& r);
         void syncRollback(OplogReader& r);
         void syncFixUp(HowToFixUp& h, OplogReader& r);
@@ -420,7 +420,7 @@ namespace mongo {
             }
             if( theReplSet == 0 ) {
                 result.append("startupStatus", ReplSet::startupStatus);
-                errmsg = ReplSet::startupStatusMsg.empty() ? "replset unknown error 2" : ReplSet::startupStatusMsg;
+                errmsg = ReplSet::startupStatusMsg.empty() ? "replset unknown error 2" : ReplSet::startupStatusMsg.get();
                 if( ReplSet::startupStatus == 3 )
                     result.append("info", "run rs.initiate(...) if not yet done for the set");
                 return false;

db/repl/rs_config.cpp

@@ -314,12 +314,12 @@ namespace mongo {
                 }
                 if( m.h.isLocalHost() )
                     localhosts++;
-                m.arbiterOnly = mobj.getBoolField("arbiterOnly");
+                m.arbiterOnly = mobj["arbiterOnly"].trueValue();
                 m.slaveDelay = mobj["slaveDelay"].numberInt();
                 if( mobj.hasElement("hidden") )
-                    m.hidden = mobj.getBoolField("hidden");
+                    m.hidden = mobj["hidden"].trueValue();
                 if( mobj.hasElement("buildIndexes") )
-                    m.buildIndexes = mobj.getBoolField("buildIndexes");
+                    m.buildIndexes = mobj["buildIndexes"].trueValue();
                 if( mobj.hasElement("priority") )
                     m.priority = mobj["priority"].Number();
                 if( mobj.hasElement("votes") )

db/repl/rs_initialsync.cpp

@@ -158,6 +158,7 @@ namespace mongo {
 
         for( Member *m = head(); m; m = m->next() ) {
             if (!m->hbinfo().up() ||
+                !m->config().buildIndexes ||
                     (m->state() != MemberState::RS_SECONDARY &&
                      m->state() != MemberState::RS_PRIMARY) ||
                     (secondaryOnly && m->state() != MemberState::RS_SECONDARY) ||

db/repl/rs_member.h

@@ -19,6 +19,8 @@
 
 #pragma once
 
+#include "../../util/concurrency/value.h"
+
 namespace mongo {
 
 
@@ -74,7 +76,7 @@ namespace mongo {
         time_t upSince;
         long long downSince;
         time_t lastHeartbeat;
-        string lastHeartbeatMsg;
+        DiagStr lastHeartbeatMsg;
         OpTime opTime;
         int skew;
 

db/repl/rs_rollback.cpp

@@ -473,6 +473,8 @@ namespace mongo {
                     continue;
                 }
 
+                getDur().commitIfNeeded();
+
                 /* keep an archive of items rolled back */
                 shared_ptr<RemoveSaver>& rs = removeSavers[d.ns];
                 if ( ! rs )

db/repl/rs_sync.cpp

@@ -25,24 +25,25 @@ namespace mongo {
     using namespace bson;
     extern unsigned replSetForceInitialSyncFailure;
 
-    /* apply the log op that is in param o */
-    void ReplSetImpl::syncApply(const BSONObj &o) {
-        char db[MaxDatabaseNameLen];
-        const char *ns = o.getStringField("ns");
-        nsToDatabase(ns, db);
+    void NOINLINE_DECL blank(const BSONObj& o) {
+        if( *o.getStringField("op") != 'n' ) {
+            log() << "replSet skipping bad op in oplog: " << o.toString() << rsLog;
+        }
+    }
 
+    /* apply the log op that is in param o
+       @return bool failedUpdate
+    */
+    bool ReplSetImpl::syncApply(const BSONObj &o) {
+        const char *ns = o.getStringField("ns");
         if ( *ns == '.' || *ns == 0 ) {
-            if( *o.getStringField("op") == 'n' )
-                return;
-            log() << "replSet skipping bad op in oplog: " << o.toString() << endl;
-            return;
+            blank(o);
+            return false;
         }
 
         Client::Context ctx(ns);
         ctx.getClient()->curop()->reset();
-
-        /* todo : if this asserts, do we want to ignore or not? */
-        applyOperation_inlock(o);
+        return applyOperation_inlock(o);
     }
 
     /* initial oplog application, during initial sync, after cloning.
@@ -133,9 +134,11 @@ namespace mongo {
                         throw DBException("primary changed",0);
                     }
 
-                    if( ts >= applyGTE ) {
-                        // optimes before we started copying need not be applied.
-                        syncApply(o);
+                    if( ts >= applyGTE ) { // optimes before we started copying need not be applied.
+                        bool failedUpdate = syncApply(o);
+                        if( failedUpdate && shouldRetry(o, hn)) {
+                            uassert(15915, "replSet update still fails after adding missing object", syncApply(o));
+                        }
                     }
                     _logOpObjRS(o);   /* with repl sets we write the ops to our oplog too */
                 }
@@ -143,6 +146,8 @@ namespace mongo {
                     // simple progress metering
                     log() << "replSet initialSyncOplogApplication " << n << rsLog;
                 }
+
+                getDur().commitIfNeeded();
             }
             catch (DBException& e) {
                 if( e.getCode() == 11000 || e.getCode() == 11001 ) {
@@ -362,7 +367,7 @@ namespace mongo {
 
                     {
                         const Member *primary = box.getPrimary();
-                        
+
                         if( !target->hbinfo().hbstate.readable() ||
                             // if we are not syncing from the primary, return (if
                             // it's up) so that we can try accessing it again

db/security_commands.cpp

@@ -139,6 +139,7 @@ namespace mongo {
             string pwd;
 
             if (user == internalSecurity.user) {
+                uassert(15889, "key file must be used to log in with internal user", cmdLine.keyFile);
                 pwd = internalSecurity.pwd;
             }
             else {

db/stats/top.cpp

@@ -44,6 +44,9 @@ namespace mongo {
     }
 
     void Top::record( const string& ns , int op , int lockType , long long micros , bool command ) {
+        if ( ns[0] == '?' )
+            return;
+
         //cout << "record: " << ns << "\t" << op << "\t" << command << endl;
         scoped_lock lk(_lock);
 

db/update.cpp

@@ -913,15 +913,14 @@ namespace mongo {
             }
         }
         virtual long long nscanned() {
-            assert( _c.get() );
-            return _c->nscanned();
+            return _c.get() ? _c->nscanned() : _nscanned;
         }
         virtual void next() {
             if ( ! _c->ok() ) {
                 setComplete();
                 return;
             }
-            _nscanned++;
+            _nscanned = _c->nscanned();
             if ( matcher()->matches(_c->currKey(), _c->currLoc(), &_details ) ) {
                 setComplete();
                 return;
@@ -988,7 +987,7 @@ namespace mongo {
                 BSONObj newObj = mss->createNewFromMods();
                 checkTooLarge(newObj);
                 assert(nsdt);
-                DiskLoc newLoc = theDataFileMgr.updateRecord(ns, d, nsdt, r, loc , newObj.objdata(), newObj.objsize(), debug);
+                theDataFileMgr.updateRecord(ns, d, nsdt, r, loc , newObj.objdata(), newObj.objsize(), debug);
             }
 
             if ( logop ) {
@@ -1226,8 +1225,7 @@ namespace mongo {
                     }
                 }
 
-                if (atomic)
-                    getDur().commitIfNeeded();
+                getDur().commitIfNeeded();
 
                 continue;
             }
@@ -1273,7 +1271,8 @@ namespace mongo {
                 logOp( "i", ns, no );
             return UpdateResult( 0 , 0 , 1 , no );
         }
-        return UpdateResult( 0 , 0 , 0 );
+
+        return UpdateResult( 0 , isOperatorUpdate , 0 );
     }
 
     UpdateResult updateObjects(const char *ns, const BSONObj& updateobj, BSONObj patternOrig, bool upsert, bool multi, bool logop , OpDebug& debug ) {

dbtests/basictests.cpp

@@ -399,6 +399,27 @@ namespace BasicTests {
             ASSERT_EQUALS( -1, lexNumCmp( "a", "0a"));
             ASSERT_EQUALS( -1, lexNumCmp( "000a", "001a"));
             ASSERT_EQUALS( 0, lexNumCmp( "010a", "0010a"));
+
+            ASSERT_EQUALS( -1 , lexNumCmp( "a0" , "a00" ) );
+            ASSERT_EQUALS( 0 , lexNumCmp( "a.0" , "a.00" ) );
+            ASSERT_EQUALS( -1 , lexNumCmp( "a.b.c.d0" , "a.b.c.d00" ) );
+            ASSERT_EQUALS( 1 , lexNumCmp( "a.b.c.0.y" , "a.b.c.00.x" ) );
+
+            ASSERT_EQUALS( -1, lexNumCmp( "a", "a-" ) );
+            ASSERT_EQUALS( 1, lexNumCmp( "a-", "a" ) );
+            ASSERT_EQUALS( 0, lexNumCmp( "a-", "a-" ) );
+
+            ASSERT_EQUALS( -1, lexNumCmp( "a", "a-c" ) );
+            ASSERT_EQUALS( 1, lexNumCmp( "a-c", "a" ) );
+            ASSERT_EQUALS( 0, lexNumCmp( "a-c", "a-c" ) );
+
+            ASSERT_EQUALS( 1, lexNumCmp( "a-c.t", "a.t" ) );
+            ASSERT_EQUALS( -1, lexNumCmp( "a.t", "a-c.t" ) );
+            ASSERT_EQUALS( 0, lexNumCmp( "a-c.t", "a-c.t" ) );
+
+            ASSERT_EQUALS( 1, lexNumCmp( "ac.t", "a.t" ) );
+            ASSERT_EQUALS( -1, lexNumCmp( "a.t", "ac.t" ) );
+            ASSERT_EQUALS( 0, lexNumCmp( "ac.t", "ac.t" ) );
         }
     };
 

dbtests/queryoptimizertests.cpp

@@ -1558,7 +1558,6 @@ namespace QueryOptimizerTests {
                     theDataFileMgr.insertWithObjMod( ns(), temp );
                 }
                 BSONObj hint = fromjson( "{$hint:{a:1,b:1}}" );
-                BSONElement hintElt = hint.firstElement();
                 auto_ptr< FieldRangeSet > frs( new FieldRangeSet( ns(), fromjson( "{a:5,b:{$in:[2,3,6,9,11]}}" ) ) );
                 QueryPlan qp( nsd(), 1, *frs, *frs, fromjson( "{a:5,b:{$in:[2,3,6,9,11]}}" ), BSONObj() );
                 boost::shared_ptr<Cursor> c = qp.newCursor();
@@ -1581,7 +1580,6 @@ namespace QueryOptimizerTests {
                     theDataFileMgr.insertWithObjMod( ns(), temp );
                 }
                 BSONObj hint = fromjson( "{$hint:{a:1,b:1}}" );
-                BSONElement hintElt = hint.firstElement();
                 auto_ptr< FieldRangeSet > frs( new FieldRangeSet( ns(), fromjson( "{a:{$gte:5},b:{$in:[2,3,6,9,11]}}" ) ) );
                 QueryPlan qp( nsd(), 1, *frs, *frs, fromjson( "{a:{$gte:5},b:{$in:[2,3,6,9,11]}}" ), BSONObj() );
                 boost::shared_ptr<Cursor> c = qp.newCursor();

dbtests/repltests.cpp

@@ -105,12 +105,6 @@ namespace ReplTests {
             return count;
         }
         static void applyAllOperations() {
-            class Applier : public ReplSource {
-            public:
-                static void apply( const BSONObj &op ) {
-                    ReplSource::applyOperation( op );
-                }
-            };
             dblock lk;
             vector< BSONObj > ops;
             {
@@ -120,8 +114,13 @@ namespace ReplTests {
             }
             {
                 Client::Context ctx( ns() );
-                for( vector< BSONObj >::iterator i = ops.begin(); i != ops.end(); ++i )
-                    Applier::apply( *i );
+                BSONObjBuilder b;
+                b.append("host", "localhost");
+                b.appendTimestamp("syncedTo", 0);
+                ReplSource a(b.obj());
+                for( vector< BSONObj >::iterator i = ops.begin(); i != ops.end(); ++i ) {
+                    a.applyOperation( *i );
+                }
             }
         }
         static void printAll( const char *ns ) {

dbtests/spin_lock_test.cpp

@@ -70,7 +70,7 @@ namespace {
     public:
         void run() {
 
-#if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4)
+#if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) || defined(_WIN32)
 
             SpinLock spin;
             int counter = 0;
@@ -93,10 +93,12 @@ namespace {
 
             ASSERT_EQUALS( counter, threads*incs );
 #else
-
-            // WARNING "TODO Missing spin lock in this platform."
-            ASSERT( true );
-
+            warning() << "spin lock slow on this platform" << endl;
+            
+#if defined(__linux__)
+            // we don't want to have linux binaries without a fast spinlock
+            //ASSERT( false ); TODO SERVER-3075
+#endif
 
 #endif
 

debian/changelog

@@ -1,3 +1,81 @@
+mongodb (1.8.5) unstable; urgency=low
+
+  * see http://jira.mongodb.org/browse/SERVER/fixforversion/10988
+
+ -- Richard Kreuter <richard@10gen.com>  Wed, 01 Feb 2012 16:56:28 -0500
+
+mongodb (1.8.4) unstable; urgency=low
+
+  * see http://jira.mongodb.org/browse/SERVER/fixforversion/10595
+
+ -- Richard Kreuter <richard@10gen.com>  Mon, 24 Oct 2011 16:56:28 -0500
+
+mongodb (1.8.3) unstable; urgency=low
+
+  * see http://jira.mongodb.org/browse/SERVER/fixforversion/10380
+
+ -- Richard Kreuter <richard@10gen.com>  Fri, 19 Aug 2011 16:56:28 -0500
+
+mongodb (1.8.2) unstable; urgency=low
+
+  * see http://jira.mongodb.org/browse/SERVER/fixforversion/10263
+
+ -- Richard Kreuter <richard@10gen.com>  Mon, 06 Jun 2011 16:56:28 -0500
+
+mongodb (1.8.2-rc3) unstable; urgency=low
+
+  * see http://jira.mongodb.org/browse/SERVER/fixforversion/10263
+
+ -- Richard Kreuter <richard@10gen.com>  Wed, 01 Jun 2011 16:56:28 -0500
+
+mongodb (1.8.2-rc2) unstable; urgency=low
+
+  * see http://jira.mongodb.org/browse/SERVER/fixforversion/10263
+
+ -- Richard Kreuter <richard@10gen.com>  Fri, 20 May 2011 16:56:28 -0500
+
+mongodb (1.8.2-rc1) unstable; urgency=low
+
+  * see http://jira.mongodb.org/browse/SERVER/fixforversion/10263
+
+ -- Richard Kreuter <richard@10gen.com>  Wed, 11 May 2011 16:56:28 -0500
+
+mongodb (1.8.2-rc0) unstable; urgency=low
+
+  * see http://jira.mongodb.org/browse/SERVER/fixforversion/10263
+
+ -- Richard Kreuter <richard@10gen.com>  Wed, 4 May 2011 16:56:28 -0500
+
+mongodb (1.8.1) unstable; urgency=low
+
+  * see http://jira.mongodb.org/browse/SERVER/fixforversion/10260
+
+ -- Richard Kreuter <richard@10gen.com>  Thu, 6 Apr 2011 16:56:28 -0500
+
+mongodb (1.8.1-rc1) unstable; urgency=low
+
+  * see http://jira.mongodb.org/browse/SERVER/fixforversion/10260
+
+ -- Richard Kreuter <richard@10gen.com>  Thu, 31 Mar 2011 16:56:28 -0500
+
+mongodb (1.8.1-rc0) unstable; urgency=low
+
+  * see http://jira.mongodb.org/browse/SERVER/fixforversion/10260
+
+ -- Richard Kreuter <richard@10gen.com>  Thu, 24 Mar 2011 16:56:28 -0500
+
+mongodb (1.8.0) unstable; urgency=low
+
+  * see http://jira.mongodb.org/browse/SERVER/fixforversion/10256
+
+ -- Richard Kreuter <richard@10gen.com>  Wed, 16 Mar 2011 16:56:28 -0500
+
+mongodb (1.8.0-rc1) unstable; urgency=low
+
+  * see http://jira.mongodb.org/browse/SERVER/fixforversion/10246
+
+ -- Richard Kreuter <richard@10gen.com>  Tue, 1 Mar 2011 16:56:28 -0500
+
 mongodb (1.8.0-rc0) unstable; urgency=low
 
   * see http://jira.mongodb.org/browse/SERVER/fixforversion/10245

debian/init.d

@@ -127,6 +127,7 @@ start_server() {
 stop_server() {
 # Stop the process using the wrapper
             start-stop-daemon --stop --quiet --pidfile $PIDFILE \
+                        --retry 300 \
                         --user $DAEMONUSER \
                         --exec $DAEMON
             errcode=$?

debian/mongodb.upstart

@@ -2,6 +2,8 @@
 
 limit nofile 20000 20000
 
+kill timeout 300 # wait 300s between SIGTERM and SIGKILL.
+
 pre-start script
     mkdir -p /var/lib/mongodb/
     mkdir -p /var/log/mongodb/

doxygenConfig

@@ -3,7 +3,7 @@
 #---------------------------------------------------------------------------
 DOXYFILE_ENCODING      = UTF-8
 PROJECT_NAME           = MongoDB
-PROJECT_NUMBER         = 1.8.0-rc0
+PROJECT_NUMBER         = 1.8.6-pre-
 OUTPUT_DIRECTORY       = docs/doxygen
 CREATE_SUBDIRS         = NO
 OUTPUT_LANGUAGE        = English

jstests/check_shard_index.js

@@ -32,9 +32,19 @@ f.ensureIndex( { x: 1 , y: 1 } );
 assert.eq( 0 , f.count() , "2. initial count should be zero" );
 
 f.save( { x: 1 , y : 1 } );
-f.save( { y: 2 } );
-assert.eq( 2 , f.count() , "2. count after initial insert should be 2" );
+f.save( { x: null , y : 1 } );
+
 res = db.runCommand( { checkShardingIndex: "test.jstests_shardingindex" , keyPattern: {x:1, y:1} , force: true });
-assert.eq( false , res.ok , "2a" );
+assert.eq( true , res.ok , "2a " + tojson(res) );
+
+f.save( { y: 2 } );
+assert.eq( 3 , f.count() , "2. count after initial insert should be 3" );
+res = db.runCommand( { checkShardingIndex: "test.jstests_shardingindex" , keyPattern: {x:1, y:1} , force: true });
+assert.eq( false , res.ok , "2b " + tojson(res) );
+
+// 
+res = db.runCommand( { checkShardingIndex: "test.jstests_shardingindex" , keyPattern: {_id:1} , force: true });
+assert.eq( true , res.ok , "3a " + tojson(res) );
+assert( res.idskip , "3b " + tojson(res) )
 
 print("PASSED");

jstests/drop2.js

@@ -2,7 +2,7 @@ t = db.jstests_drop2;
 t.drop();
 
 function debug( x ) {
-//    printjson( x );
+    //printjson( x );
 }
 
 t.save( {} );
@@ -14,11 +14,11 @@ function op( drop ) {
     for ( var i in p ) {
         var o = p[ i ];
         if ( drop ) {
-            if ( o.active && o.query && o.query.drop && o.query.drop == "jstests_drop2" ) {
+            if (  o.query && o.query.drop && o.query.drop == "jstests_drop2" ) {
                 return o.opid;
             }
         } else {
-            if ( o.active && o.query && o.query.query && o.query.query.$where && o.ns == "test.jstests_drop2" ) {
+            if (  o.query && o.query.query && o.query.query.$where && o.ns == "test.jstests_drop2" ) {
                 return o.opid;
             }
         }

jstests/geo_update.js

@@ -0,0 +1,37 @@
+// Tests geo queries w/ update & upsert
+// from SERVER-3428
+
+var coll = db.testGeoUpdate
+coll.drop()
+
+coll.ensureIndex({ loc : "2d" })
+
+// Test normal update
+print( "Updating..." )
+
+coll.insert({ loc : [1.0, 2.0] })
+
+coll.update({ loc : { $near : [1.0, 2.0] } },
+            { x : true, loc : [1.0, 2.0] })
+
+// Test upsert
+print( "Upserting..." )
+            
+coll.update({ loc : { $within : { $center : [[10, 20], 1] } } },
+            { x : true },
+            true)
+            
+coll.update({ loc : { $near : [10.0, 20.0], $maxDistance : 1 } },
+            { x : true },
+            true)
+           
+            
+coll.update({ loc : { $near : [100, 100], $maxDistance : 1 } }, 
+            { $set : { loc : [100, 100] }, $push : { people : "chris" } },
+            true)
+            
+coll.update({ loc : { $near : [100, 100], $maxDistance : 1 } }, 
+            { $set : { loc : [100, 100] }, $push : { people : "john" } },
+            true)
+
+assert.eq( 4, coll.find().itcount() )

jstests/index11.js

@@ -5,12 +5,9 @@ coll.drop();
 
 coll.ensureIndex({"k": 1, "v": 1});
 coll.insert({k: "x", v: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"});
-/* TODO inconsistent behavior w/ too large to index - this fails but the next assert works.
- * SERVER-1716
- * assert.eq(1, coll.find({"k": "x"}).count());
- */
+assert.eq(0, coll.find({"k": "x"}).count()); // SERVER-1716
 
 coll.dropIndexes();
 coll.ensureIndex({"k": 1, "v": 1});
 
-assert.eq(1, coll.find({"k": "x"}).count());
+assert.eq(0, coll.find({"k": "x"}).count());

jstests/index_fornew.js

@@ -0,0 +1,13 @@
+
+t = db.index_fornew;
+t.drop();
+
+t.insert( { x : 1 } )
+t.ensureIndex( { x : 1 } , { v : 1 } )
+assert.eq( 1 , t.getIndexes()[1].v , tojson( t.getIndexes() ) );
+
+assert.throws( function(){ t.findOne( { x : 1 } ); } )
+
+t.reIndex();
+assert.eq( 0 , t.getIndexes()[1].v , tojson( t.getIndexes() ) );
+assert( t.findOne( { x : 1 } ) );

jstests/mr_undef.js

@@ -0,0 +1,22 @@
+
+t = db.mr_undef
+t.drop()
+
+outname = "mr_undef_out"
+out = db[outname]
+out.drop()
+
+t.insert({x : 0}) 
+
+var m = function() { emit(this.mod, this.x); } 
+var r = function(k,v) { total = 0; for(i in v) { total+= v[i]; } return total; } 
+
+res = t.mapReduce(m, r, {out : outname } )
+
+assert.eq( 0 , out.find( { _id : { $type : 6 } } ).itcount() , "A1" )
+assert.eq( 1 , out.find( { _id : { $type : 10 } } ).itcount() , "A2" )
+
+x = out.findOne()
+assert.eq( x , out.findOne( { _id : x["_id"] } ) , "A3" )
+
+

jstests/or5.js

@@ -70,6 +70,7 @@ assert.eq.automsg( "6", "t.find( {$or:[{a:2},{b:3},{c:4}]} ).batchSize( 2 ).itco
 c = t.find( {$or:[{a:2},{b:3},{c:4}]} ).batchSize( 2 );
 c.next();
 t.remove( {b:3} );
+db.getLastError();
 assert.eq.automsg( "3", c.itcount() );
 
 reset();
@@ -78,6 +79,7 @@ c = t.find( {$or:[{a:2},{b:3},{c:4}]} ).batchSize( 2 );
 c.next();
 c.next();
 t.remove( {b:3} );
+db.getLastError();
 assert.eq.automsg( "2", c.itcount() );
 
 reset();
@@ -87,6 +89,7 @@ c.next();
 c.next();
 c.next();
 t.remove( {b:3} );
+db.getLastError();
 assert.eq.automsg( "3", c.itcount() );
 
 reset();
@@ -97,6 +100,7 @@ c.next();
 c.next();
 c.next();
 t.remove( {b:3} );
+db.getLastError();
 assert.eq.automsg( "2", c.itcount() );
 
 t.drop();

jstests/replsets/auth1.js

@@ -182,3 +182,15 @@ wait(function() {
         return results.members[3].state == 2;
     });
 
+print("make sure it has the config, too");
+assert.soon(function() {
+        for (var i in rs.nodes) {
+            rs.nodes[i].setSlaveOk();
+            rs.nodes[i].getDB("admin").auth("foo","bar");
+            config = rs.nodes[i].getDB("local").system.replset.findOne();
+            if (config.version != 2) {
+                return false;
+            }
+        }
+        return true;
+    });

jstests/replsets/fastsync.js

@@ -73,21 +73,7 @@ print("total in foo: "+foo.bar.count());
 
 print("2");
 admin.runCommand( {fsync:1,lock:1} );
-
 copyDbpath( basePath + "-p", basePath + "-s" );
-
-print("remove local files or slave will get confused");
-var files = listFiles(basePath+"-s");
-
-for (var i in files) {
-  var filename = files[i].name;
-  if (filename.match("/local")) {
-    print("removing "+filename);
-    removeFile(filename);
-  }
-}
-//run("rm", basePath+"-s/local.*");
-    
 admin.$cmd.sys.unlock.findOne();
 
 

jstests/replsets/initial_sync1.js

@@ -114,9 +114,6 @@ wait(function() {
   });
 
 
-/**
- * TODO: this fails on buildbot
- * see SERVER-2550
 print("10. Insert some stuff");
 master = replTest.getMaster();
 for (var i=0; i<10000; i++) {
@@ -126,4 +123,4 @@ for (var i=0; i<10000; i++) {
 
 print("11. Everyone happy eventually");
 replTest.awaitReplication();
-*/
+

jstests/replsets/replset5.js

@@ -49,6 +49,23 @@ doTest = function (signal) {
     var master1count = master.getDB(testDB).foo.count();
     assert( master1count == docNum, "Master has " + master1count + " of " + docNum + " documents!");
 
+    print("reconfigure with hidden=1");
+    config = master.getDB("local").system.replset.findOne();
+    config.version++;
+    config.members[2].priority = 0;
+    config.members[2].hidden = 1;
+
+    try {
+        master.adminCommand({replSetReconfig : config});
+    }
+    catch(e) {
+        print(e);
+    }
+
+    config = master.getDB("local").system.replset.findOne();
+    printjson(config);
+    assert.eq(config.members[2].hidden, true);
+
     replTest.stopSet(signal);
 }
 

jstests/sharding/bigMapReduce.js

@@ -67,6 +67,10 @@ for (iter = 0; iter < 5; iter++) {
     print("checking result field");
     assert.eq(res.result.collection, outCollStr, "Wrong collection " + res.result.collection);
     assert.eq(res.result.db, outDbStr, "Wrong db " + res.result.db);
+
+    // make sure final collection has index on _id
+    assert(outDb.system.indexes.count({ns: outDbStr + "." + outCollStr, key: {_id: 1}}) > 0, "No index on _id")
+    assert( gotAGoodOne , "no good for out db" )
 }
 
 assert( gotAGoodOne , "no good for out db" )

jstests/sharding/features1.js

@@ -175,7 +175,7 @@ assert( ! s.admin.runCommand( { shardcollection : "test.foo9" , key : { a : 1 }
 // --- listDatabases ---
 
 r = db.getMongo().getDBs()
-assert.eq( 4 , r.databases.length , "listDatabases 1 : " + tojson( r ) )
+assert.eq( 3 , r.databases.length , "listDatabases 1 : " + tojson( r ) )
 assert.lt( 10000 , r.totalSize , "listDatabases 2 : " + tojson( r ) );
 
 s.stop()

jstests/sharding/multi_mongos2a.js

@@ -0,0 +1,31 @@
+// multi_mongos2.js
+// This tests sharding an existing collection that both shards are aware of (SERVER-2828)
+
+
+// setup sharding with two mongos, s1 and s2
+s1 = new ShardingTest( "multi_mongos1" , 2 , 1 , 2 );
+s2 = s1._mongos[1];
+
+s1.adminCommand( { enablesharding : "test" } );
+s1.adminCommand( { shardcollection : "test.foo" , key : { num : 1 } } );
+
+s1.config.databases.find().forEach( printjson )
+
+s1.getDB('test').existing.insert({_id:1})
+assert.eq(1, s1.getDB('test').existing.count({_id:1}));
+assert.eq(1, s2.getDB('test').existing.count({_id:1}));
+
+s2.adminCommand( { shardcollection : "test.existing" , key : { _id : 1 } } );
+assert.eq(true, s2.getDB('test').existing.stats().sharded);
+
+
+res = s2.getDB( "admin" ).runCommand( { moveChunk: "test.existing" , find : { _id : 1 } , to : s1.getOther( s1.getServer( "test" ) ).name } );
+
+assert.eq(1 , res.ok, tojson(res));
+
+s1.adminCommand( { flushRouterConfig : 1 } )
+
+assert.eq(1, s1.getDB('test').existing.count({_id:1})); // SERVER-2828
+assert.eq(1, s2.getDB('test').existing.count({_id:1}));
+
+s1.stop();

jstests/sharding/shard_insert_getlasterror_w2.js

@@ -51,7 +51,7 @@ function go() {
     db.foo.insert({_id:'a', x:1});
     db.foo.insert({_id:'a', x:1});
     var x = db.getLastErrorObj(2, 30000)
-    assert.neq(x.err, null, tojson(x));
+    assert.neq(x.err, null, "C1 " + tojson(x));
 
     // Add more data
     for (var i = N; i < 2*N; i++) {
@@ -59,7 +59,7 @@ function go() {
         var x = db.getLastErrorObj(2, 30000)  // wait to be copied to at least one secondary
             if (i % 30 == 0) print(i)
         if (i % 100 == 0 || x.err != null) printjson(x);
-        assert.eq(x.err, null, tojson(x));
+        assert.eq(x.err, null, "C2 " + tojson(x));
     }
 
     // take down the slave and make sure it fails over

jstests/sharding/writeback_shard_version.js

@@ -0,0 +1,64 @@
+// Tests whether a newly sharded collection can be handled by the wbl
+
+var jsTestLog = function( msg ){ print( "\n\n****\n" + msg + "\n****\n\n" ) }
+var jsTestName = function(){ return "writeback_shard_version" }
+
+jsTestLog( "Starting sharded cluster..." )
+
+// Need to start as a replica set here, just because there's no other way to trigger separate configs,
+// See SERVER-4222
+var st = new ShardingTest( name = jsTestName(), shards = 1, verbose = 2, mongos = 2, other = { rs : true } )
+
+var mongosA = st._mongos[0]
+var mongosB = st._mongos[1]
+
+var config = mongosA.getDB("config")
+config.settings.update({ _id : "balancer" }, { $set : { stopped : true } }, true )
+
+jsTestLog( "Adding new collections...")
+
+var collA = mongosA.getCollection( jsTestName() + ".coll" )
+collA.insert({ hello : "world" })
+assert.eq( null, collA.getDB().getLastError() )
+
+var collB = mongosB.getCollection( "" + collA )
+collB.findOne()
+collB.insert({ hello : "world" })
+assert.eq( null, collB.getDB().getLastError() )
+
+jsTestLog( "Enabling sharding..." )
+
+printjson( mongosA.getDB( "admin" ).runCommand({ enableSharding : "" + collA.getDB() }) )
+
+otherCollA = mongosA.getCollection( jsTestName() + ".otherColl" )
+otherCollB = mongosB.getCollection( "" + otherCollA )
+printjson( mongosA.getDB( "admin" ).runCommand({ shardCollection : "" + otherCollA, key : { _id : 1 } }) )
+
+// Make sure mongosB knows about the collC sharding
+mongosA.getDB("admin").runCommand({ flushRouterConfig : 1 })
+mongosB.getDB("admin").runCommand({ flushRouterConfig : 1 })
+otherCollA.findOne()
+otherCollB.findOne()
+
+printjson( mongosA.getDB( "admin" ).runCommand({ shardCollection : "" + collA, key : { _id : 1 } }) )
+
+// Make sure mongosA knows about the collA sharding
+mongosA.getDB("admin").runCommand({ flushRouterConfig : 1 })
+collA.findOne();
+
+printjson( config.collections.find().toArray() )
+
+sleep( 3000 )
+
+jsTestLog( "Trigger wbl..." )
+
+//collB.findOne()
+collB.insert({ goodbye : "world" })
+assert.eq( null, collB.getDB().getLastError() )
+
+print( "Inserted..." )
+
+assert.eq( 3, collA.find().itcount() )
+assert.eq( 3, collB.find().itcount() )
+
+st.stop()

jstests/slowNightly/explain1.js

@@ -0,0 +1,17 @@
+// SERVER-2662 - drop client cursor in a context where query will yield frequently
+
+t = db.jstests_slowNightly_explain1;
+t.drop();
+
+// Periodically drops the collection, invalidating client cursors for s2's operations.
+s1 = startParallelShell( "t = db.jstests_slowNightly_explain1; for( var i = 0; i < 80; ++i ) { t.drop(); t.ensureIndex({x:1}); for( var j = 0; j < 1000; ++j ) { t.save( {x:j,y:1} ) }; sleep( 100 ); }" );
+
+// Query repeatedly.
+s2 = startParallelShell( "t = db.jstests_slowNightly_explain1; for( var i = 0; i < 500; ++i ) { try { z = t.find( {x:{$gt:0},y:1} ).explain(); t.count( {x:{$gt:0},y:1} ); } catch( e ) {} }" );
+
+// Put pressure on s2 to yield more often.
+s3 = startParallelShell( "t = db.jstests_slowNightly_explain1; for( var i = 0; i < 200; ++i ) { t.validate({scandata:true}); }" );
+
+s1();
+s2();
+s3();

jstests/slowNightly/explain2.js

@@ -0,0 +1,18 @@
+// Test for race condition SERVER-2807.  One cursor is dropped and another is not.
+
+collName = 'jstests_slowNightly_explain2';
+
+t = db[ collName ];
+t.drop();
+
+db.createCollection( collName, {capped:true,size:100000} );
+t = db[ collName ];
+t.ensureIndex( {x:1} );
+
+a = startParallelShell( 'for( i = 0; i < 50000; ++i ) { db.' + collName + '.insert( {x:i,y:1} ); }' );
+
+for( i = 0; i < 800; ++i ) {
+    t.find( {x:{$gt:-1},y:1} ).sort({x:-1}).explain();
+}
+
+a();

jstests/slowNightly/explain3.js

@@ -0,0 +1,17 @@
+// SERVER-2810 - similar to explain1 test, but with a scan and order find
+
+t = db.jstests_slowNightly_explain3;
+t.drop();
+
+// Periodically drops the collection, invalidating client cursors for s2's operations.
+s1 = startParallelShell( "t = db.jstests_slowNightly_explain1; for( var i = 0; i < 80; ++i ) { t.drop(); t.ensureIndex({x:1}); for( var j = 0; j < 1000; ++j ) { t.save( {x:j,y:1} ) }; sleep( 100 ); }" );
+
+// Query repeatedly.
+s2 = startParallelShell( "t = db.jstests_slowNightly_explain1; for( var i = 0; i < 500; ++i ) { try { z = t.find( {x:{$gt:0},y:1} ).sort({x:1}).explain(); } catch( e ) {} }" );
+
+// Put pressure on s2 to yield more often.
+s3 = startParallelShell( "t = db.jstests_slowNightly_explain1; for( var i = 0; i < 200; ++i ) { t.validate({scandata:true}); }" );
+
+s1();
+s2();
+s3();

jstests/slowNightly/newcollection2.js

@@ -0,0 +1,11 @@
+// Alocate collection forcing just a small size remainder in 2nd extent
+
+port = allocatePorts( 1 )[ 0 ]
+var baseName = "jstests_disk_newcollection2";
+var m = startMongod( "--noprealloc", "--smallfiles", "--port", port, "--dbpath", "/data/db/" + baseName );
+db = m.getDB( "test" );
+
+db.createCollection( baseName, {size:0x1FFC0000-0x10-8192} );
+var v = db[ baseName ].validate();
+printjson( v );
+assert( v.valid );

jstests/slowNightly/sharding_balance4.js

@@ -90,8 +90,8 @@ function diff(){
     if ( le.err )
         print( "ELIOT ELIOT : " + tojson( le ) + "\t" + myid );
 
-    assert( le.updatedExisting , "GLE diff 1: " + tojson(le) )
-    assert.eq( 1 , le.n , "GLE diff 2: " + tojson(le) )
+    assert( le.updatedExisting , "GLE diff 1 myid: " + myid + " " + tojson(le) )
+    assert.eq( 1 , le.n , "GLE diff 2 myid: " + myid + " " + tojson(le) )
 
 
     if ( Math.random() > .99 ){

jstests/slowNightly/sharding_rs1.js

@@ -43,6 +43,15 @@ function diff(){
 assert.lt( 20 , diff() , "big differential here" );
 print( diff() )
 
+{
+    // quick test for SERVER-2686
+    var mydbs = db.getMongo().getDBs().databases;
+    for ( var i=0; i<mydbs.length; i++ ) {
+        assert( mydbs[i].name != "local" , "mongos listDatabases can't return local" );
+    }
+}
+
+
 assert.soon( function(){
     var d = diff();
     return d < 5;

jstests/slowWeekly/rollback4.js

@@ -0,0 +1,50 @@
+// Test a large rollback SERVER-2737
+
+var replTest = new ReplSetTest({ name: 'unicomplex', nodes: 3, oplogSize: 2000 });
+var nodes = replTest.nodeList();
+
+var conns = replTest.startSet();
+var r = replTest.initiate({ "_id": "unicomplex",
+                          "members": [
+                                      { "_id": 0, "host": nodes[0] },
+                                      { "_id": 1, "host": nodes[1] },
+                                      { "_id": 2, "host": nodes[2], arbiterOnly: true}]
+                          });
+
+// Make sure we have a master
+var master = replTest.getMaster();
+b_conn = conns[1];
+b_conn.setSlaveOk();
+B = b_conn.getDB("admin");
+
+// Make sure we have an arbiter
+assert.soon(function () {
+            res = conns[2].getDB("admin").runCommand({ replSetGetStatus: 1 });
+            return res.myState == 7;
+            }, "Arbiter failed to initialize.");
+
+// Wait for initial replication
+replTest.awaitReplication();
+
+// Insert into master
+var big = { b:new Array( 1000 ).toString() };
+for( var i = 0; i < 1000000; ++i ) {
+    if ( i % 10000 == 0 ) {
+        print( i );
+    }
+	master.getDB( 'db' ).c.insert( big );
+}
+
+// Stop master
+replTest.stop( 0 );
+
+// Wait for slave to take over
+assert.soon(function () { return B.isMaster().ismaster; });
+master = replTest.getMaster();
+
+// Save to new master, forcing rollback of old master
+master.getDB( 'db' ).c.save( big );
+
+// Restart old master
+replTest.restart( 0 );
+replTest.awaitReplication();

jstests/updatea.js

@@ -46,5 +46,9 @@ assert.eq( orig , t.findOne() , "C2" );
 t.update( {} , { $inc: { "a.10" : 1 } } );
 orig.a[10]++;
 
+// SERVER-3218
+t.drop()
+t.insert({"a":{"c00":1}, 'c':2})
+t.update({"c":2}, {'$inc':{'a.c000':1}})
 
-
+assert.eq( { "c00" : 1 , "c000" : 1 } , t.findOne().a , "D1" )

jstests/updateg.js

@@ -0,0 +1,17 @@
+// SERVER-3370 check modifiers with field name characters comparing less than '.' character.
+
+t = db.jstests_updateg;
+
+t.drop();
+t.update({}, { '$inc' : { 'all.t' : 1, 'all-copy.t' : 1 }}, true);
+assert.eq( 1, t.count( {all:{t:1},'all-copy':{t:1}} ) );
+
+t.drop();
+t.save({ 'all' : {}, 'all-copy' : {}});
+t.update({}, { '$inc' : { 'all.t' : 1, 'all-copy.t' : 1 }});
+assert.eq( 1, t.count( {all:{t:1},'all-copy':{t:1}} ) );
+
+t.drop();
+t.save({ 'all11' : {}, 'all2' : {}});
+t.update({}, { '$inc' : { 'all11.t' : 1, 'all2.t' : 1 }});
+assert.eq( 1, t.count( {all11:{t:1},'all2':{t:1}} ) );

pch.h

@@ -72,6 +72,7 @@
 #include <boost/any.hpp>
 #include "boost/thread/once.hpp"
 #include <boost/archive/iterators/transform_width.hpp>
+#define BOOST_FILESYSTEM_VERSION 2
 #include <boost/filesystem/convenience.hpp>
 #include <boost/filesystem/exception.hpp>
 #include <boost/filesystem/operations.hpp>

rpm/init.d-mongod

@@ -13,10 +13,16 @@
 # things from mongod.conf get there by mongod reading it
 
 
-
-OPTIONS=" -f /etc/mongod.conf"
+# NOTE: if you change any OPTIONS here, you get what you pay for:
+# this script assumes all options are in the config file.
+CONFIGFILE="/etc/mongod.conf"
+OPTIONS=" -f $CONFIGFILE"
 SYSCONFIG="/etc/sysconfig/mongod"
 
+# FIXME: 1.9.x has a --shutdown flag that parses the config file and
+# shuts down the correct running pid, but that's unavailable in 1.8
+# for now.  This can go away when this script stops supporting 1.8.
+DBPATH=`awk -F= '/^dbpath=/{print $2}' "$CONFIGFILE"`
 mongod=${MONGOD-/usr/bin/mongod}
 
 MONGO_USER=mongod
@@ -36,7 +42,7 @@ start()
 stop()
 {
   echo -n $"Stopping mongod: "
-  killproc -p /var/lib/mongo/mongod.lock -t30 -TERM /usr/bin/mongod
+  killproc -p "$DBPATH"/mongod.lock -d 300 /usr/bin/mongod
   RETVAL=$?
   echo
   [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/mongod

rpm/mongo.spec

@@ -1,5 +1,5 @@
 Name: mongo
-Version: 1.8.0-rc0
+Version: 1.8.5
 Release: mongodb_1%{?dist}
 Summary: mongo client shell and tools
 License: AGPL 3.0

s/balance.cpp

@@ -147,7 +147,7 @@ namespace mongo {
         auto_ptr<DBClientCursor> cursor = conn.query( ShardNS::collection , BSONObj() );
         vector< string > collections;
         while ( cursor->more() ) {
-            BSONObj col = cursor->next();
+            BSONObj col = cursor->nextSafe();
 
             // sharded collections will have a shard "key".
             if ( ! col["key"].eoo() )
@@ -199,7 +199,7 @@ namespace mongo {
             map< string,vector<BSONObj> > shardToChunksMap;
             cursor = conn.query( ShardNS::chunk , QUERY( "ns" << ns ).sort( "min" ) );
             while ( cursor->more() ) {
-                BSONObj chunk = cursor->next();
+                BSONObj chunk = cursor->nextSafe();
                 vector<BSONObj>& chunks = shardToChunksMap[chunk["shard"].String()];
                 chunks.push_back( chunk.getOwned() );
             }
@@ -275,12 +275,22 @@ namespace mongo {
         while ( ! inShutdown() ) {
 
             try {
+                
                 ScopedDbConnection conn( config );
-
+                
+                // ping has to be first so we keep things in the config server in sync
                 _ping( conn.conn() );
-                if ( ! _checkOIDs() ) {
-                    uassert( 13258 , "oids broken after resetting!" , _checkOIDs() );
+
+                // now make sure we should even be running
+                if ( ! grid.shouldBalance() ) {
+                    log(1) << "skipping balancing round because balancing is disabled" << endl;
+                    conn.done();
+                    
+                    sleepsecs( 30 );
+                    continue;
                 }
+                
+                uassert( 13258 , "oids broken after resetting!" , _checkOIDs() );
 
                 // use fresh shard state
                 Shard::reloadShardInfo();
@@ -294,14 +304,6 @@ namespace mongo {
                     continue;
                 }
 
-                if ( ! grid.shouldBalance() ) {
-                    log(1) << "skipping balancing round because balancing is disabled" << endl;;
-                    conn.done();
-
-                    sleepsecs( 30 );
-                    continue;
-                }
-
                 log(1) << "*** start balancing round" << endl;
 
                 vector<CandidateChunkPtr> candidateChunks;