post 2.2.7

Conflicts:
	src/mongo/db/ops/update.cpp

SConstruct

@@ -294,6 +294,7 @@ env = Environment( BUILD_DIR=variantDir,
                    CLIENT_SCONSTRUCT='#distsrc/client/SConstruct',
                    DIST_ARCHIVE_SUFFIX='.tgz',
                    EXTRAPATH=get_option("extrapath"),
+                   MODULE_BANNERS=[],
                    MODULETEST_LIST='#build/moduletests.txt',
                    MSVS_ARCH=msarch ,
                    PYTHON=utils.find_python(),
@@ -861,7 +862,7 @@ def doConfigure(myenv):
 
     # discover modules (subdirectories of db/modules/), and
     # load the (python) module for each module's build.py
-    modules = moduleconfig.discover_modules('.')
+    modules = moduleconfig.discover_modules('src/mongo/')
 
     # ask each module to configure itself, and return a
     # dictionary of name => list_of_sources for each module.
@@ -939,6 +940,9 @@ def getSystemInstallName():
     if nix and os.uname()[2].startswith( "8." ):
         n += "-tiger"
 
+    if len(env.get("MONGO_MODULES", None)):
+            n += "-" + "-".join(env["MONGO_MODULES"].keys())
+
     try:
         findSettingsSetup()
         import settings

debian/changelog

@@ -1,3 +1,9 @@
+mongodb (2.2.0) unstable; urgency=low
+
+  * see http://docs.mongodb.org/manual/release-notes/2.2/
+
+ -- Richard Kreuter <richard@10gen.com>  Wed, 29 Aug 2012 16:56:28 -0500
+
 mongodb (2.1.2) unstable; urgency=low
 
   * see http://jira.mongodb.org/browse/SERVER/fixforversion/10894

distsrc/THIRD-PARTY-NOTICES

@@ -2,7 +2,7 @@ MongoDB uses third-party libraries or other resources that may
 be distributed under licenses different than the MongoDB software.
 
 In the event that we accidentally failed to list a required notice,
-please bring it to our attention through any of the ways detailed here : 
+please bring it to our attention through any of the ways detailed here :
 
            mongodb-dev@googlegroups.com
 
@@ -228,6 +228,149 @@ ghost@aladdin.com
  using BMDiff and then compressing the output of BMDiff with
  Snappy.
 
+6) License notice for Google Perftools (TCMalloc utility)
+---------------------------------
+New BSD License
+
+Copyright (c) 1998-2006, Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or
+without modification, are permitted provided that the following
+conditions are met:
+
+    * Redistributions of source code must retain the above
+      copyright notice, this list of conditions and the following
+      disclaimer.
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+    * Neither the name of Google Inc. nor the names of its
+      contributors may be used to endorse or promote products
+      derived from this software without specific prior written
+      permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
-End
+6) License notice for SpiderMonkey 1.7
+---------------------------------
+For applicable files:
+
+  Version: MPL 1.1/GPL 2.0/LGPL 2.1
+
+  The contents of this file are subject to the Mozilla Public License Version
+  1.1 (the "License"); you may not use this file except in compliance with
+  the License. You may obtain a copy of the License at
+  http://www.mozilla.org/MPL/
+
+  Software distributed under the License is distributed on an "AS IS" basis,
+  WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+  for the specific language governing rights and limitations under the
+  License.
+
+  The Original Code is Mozilla Communicator client code, released
+  March 31, 1998.
+
+  The Initial Developer of the Original Code is
+  Netscape Communications Corporation.
+  Portions created by the Initial Developer are Copyright (C) 1998
+  the Initial Developer. All Rights Reserved.
+
+  Contributor(s):
+
+  Alternatively, the contents of this file may be used under the terms of
+  either of the GNU General Public License Version 2 or later (the "GPL"),
+  or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+  in which case the provisions of the GPL or the LGPL are applicable instead
+  of those above. If you wish to allow use of your version of this file only
+  under the terms of either the GPL or the LGPL, and not to allow others to
+  use your version of this file under the terms of the MPL, indicate your
+  decision by deleting the provisions above and replace them with the notice
+  and other provisions required by the GPL or the LGPL. If you do not delete
+  the provisions above, a recipient may use your version of this file under
+  the terms of any one of the MPL, the GPL or the LGPL.
+
+7) License notice for Linenoise
+-------------------------------
+
+  Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com>
+  Copyright (c) 2010, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+    * Neither the name of Redis nor the names of its contributors may be used
+     to endorse or promote products derived from this software without
+     specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+  POSSIBILITY OF SUCH DAMAGE.
+
+8) License notice for S2 Geometry Library
+-----------------------------------------
+  Copyright 2005 Google Inc.  All Rights Reserved.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+
+9) License notice for MurmurHash
+--------------------------------
+
+  Copyright (c) 2010-2012 Austin Appleby
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+
+End

doxygenConfig

@@ -3,7 +3,7 @@
 #---------------------------------------------------------------------------
 DOXYFILE_ENCODING      = UTF-8
 PROJECT_NAME           = MongoDB
-PROJECT_NUMBER         = 2.2.0-rc1-pre-
+PROJECT_NUMBER         = 2.2.8-pre-
 OUTPUT_DIRECTORY       = docs/doxygen
 CREATE_SUBDIRS         = NO
 OUTPUT_LANGUAGE        = English

jstests/aggregation/bugs/server6779.js

@@ -0,0 +1,18 @@
+// server 6779: serializing ExpressionCoerceToBool
+// This test only fails in debug mode with the bug since that tests round-tripping
+function test(op, val) {
+    t = db.server6779;
+    t.drop();
+
+    t.insert({a:true});
+    t.insert({a:false});
+
+    obj = {};
+    obj[op] = ['$a', val];
+    result = t.aggregate({$project: {_id: 0, bool: obj}});
+
+    assert.commandWorked(result);
+    assert.eq(result.result, [{bool:true}, {bool:false}]);
+}
+test('$and', true);
+test('$or', false);

jstests/aggregation/testshard1.js

@@ -131,5 +131,27 @@ for(i = 0; i < 6; ++i) {
            'agg sharded test simple match failed');
 }
 
+function testSkipLimit(ops, expectedCount) {
+    if (expectedCount > 10) {
+        // make shard -> mongos intermediate results less than 16MB
+        ops.unshift({$project: {_id:1}})
+    }
+
+    ops.push({$group: {_id:1, count: {$sum: 1}}});
+
+    var out = db.runCommand({aggregate:"ts1", pipeline:ops});
+    assert.commandWorked(out);
+    assert.eq(out.result[0].count, expectedCount);
+}
+
+testSkipLimit([], nItems); // control
+testSkipLimit([{$skip:10}], nItems - 10);
+testSkipLimit([{$limit:10}], 10);
+testSkipLimit([{$skip:5}, {$limit:10}], 10);
+testSkipLimit([{$limit:10}, {$skip:5}], 10 - 5);
+testSkipLimit([{$skip:5}, {$skip: 3}, {$limit:10}], 10);
+testSkipLimit([{$skip:5}, {$limit:10}, {$skip: 3}], 10 - 3);
+testSkipLimit([{$limit:10}, {$skip:5}, {$skip: 3}], 10 - 3 - 5);
+
 // shut everything down
 shardedAggTest.stop();

jstests/auth/localhostAuthBypass.js

@@ -0,0 +1,103 @@
+//SERVER-6591: Localhost authentication exception doesn't work right on sharded cluster
+//
+//This test is to ensure that localhost authentication works correctly against a standalone 
+//mongod whether it is hosted with "localhost" or a hostname.
+
+var baseName = "auth_server-6591";
+var dbpath = "/data/db/" + baseName;
+var username = "foo";
+var password = "bar";
+var port = allocatePorts(1)[0];
+var host = "localhost:" + port;
+
+var addUser = function(mongo) {
+    print("============ adding a user.");
+    mongo.getDB("admin").addUser(username, password);
+};
+
+var assertCannotRunCommands = function(mongo) {
+    print("============ ensuring that commands cannot be run.");
+
+    var test = mongo.getDB("test");
+    assert.throws( function() { test.system.users.findOne(); });
+
+    test.foo.save({_id:0});
+    test.foo.update({_id:0}, {$set:{x:20}});
+    test.foo.remove({_id:0});
+    assert.throws(function() { test.getLastError(); });
+    
+    assert.throws( function() { test.foo.findOne({_id:0}); });
+
+    assert.throws(function() { 
+        test.foo.mapReduce(
+            function() { emit(1, 1); }, 
+            function(id, count) { return Array.sum(count); },
+            { out: "other" });
+    });
+};
+
+var assertCanRunCommands = function(mongo) {
+    print("============ ensuring that commands can be run.");
+
+    var test = mongo.getDB("test");
+    // will throw on failure
+    test.system.users.findOne();
+
+    test.foo.save({_id: 0});
+    assert(test.getLastError() == null);
+    
+    test.foo.update({_id: 0}, {$set:{x:20}});
+    assert(test.getLastError() == null);
+    
+    test.foo.remove({_id: 0});
+    assert(test.getLastError() == null);
+    
+    test.foo.mapReduce(
+        function() { emit(1, 1); }, 
+        function(id, count) { return Array.sum(count); },
+        { out: "other" }
+    );
+};
+
+var authenticate = function(mongo) {
+    print("============ authenticating user.");
+    mongo.getDB("admin").auth(username, password);
+};
+
+var shutdown = function(mongo) {
+    print("============ shutting down.");
+    MongoRunner.stopMongod(port, /*signal*/false, { auth: { user: username, pwd: password}});
+};
+
+var runTest = function(useHostName) {
+    print("==========================");
+    print("starting mongod: useHostName=" + useHostName);
+    print("==========================");
+    MongoRunner.runMongod({auth: "", port: port, dbpath: dbpath, useHostName: useHostName});
+
+    var mongo = new Mongo(host);
+
+    assertCanRunCommands(mongo);
+
+    addUser(mongo);
+
+    assertCannotRunCommands(mongo);
+
+    authenticate(mongo);
+
+    assertCanRunCommands(mongo);
+
+    print("============ reconnecting with new client.");
+    mongo = new Mongo(host);
+
+    assertCannotRunCommands(mongo);
+
+    authenticate(mongo);
+
+    assertCanRunCommands(mongo);
+
+    shutdown(mongo);
+};
+
+runTest(false);
+runTest(true);

jstests/cursorb.js

@@ -0,0 +1,17 @@
+// The 'cursor not found in map -1' warning is not logged when get more exhausts a client cursor.
+// SERVER-6931
+
+t = db.jstests_cursorb;
+t.drop();
+
+// Exhaust a client cursor in get more.
+for( i = 0; i < 200; ++i ) {
+    t.save( { a:i } );
+}
+t.find().itcount();
+
+// Check that the 'cursor not found in map -1' message is not printed.  This message indicates an
+// attempt to look up a cursor with an invalid id and should never appear in the log.
+log = db.adminCommand( { getLog:'global' } ).log
+log.forEach( function( line ) { assert( !line.match( /cursor not found in map -1 / ),
+                                        'Cursor map lookup with id -1.' ); } );

jstests/distinct3.js

@@ -16,8 +16,15 @@ for( i = 0; i < 1000; ++i ) {
 }
 db.getLastError();
 
-// The idea here is to try and remove the last match for the {a:1} index scan while distinct is yielding.
-p = startParallelShell( 'for( i = 0; i < 2500; ++i ) { db.jstests_distinct3.remove({a:49}); for( j = 0; j < 20; ++j ) { db.jstests_distinct3.save({a:49,c:49,d:j}) } }' );
+// Attempt to remove the last match for the {a:1} index scan while distinct is yielding.
+p = startParallelShell( 'for( i = 0; i < 2500; ++i ) {                             ' +
+                        '    db.jstests_distinct3.remove( { a:49 } );              ' +
+                        '    for( j = 0; j < 20; ++j ) {                           ' +
+                        '        db.jstests_distinct3.save( { a:49, c:49, d:j } ); ' +
+                        '    }                                                     ' +
+                        '}                                                         ' +
+                        '// Wait for the above writes to complete.                 ' +
+                        'db.getLastError();                                        ' );
 
 for( i = 0; i < 100; ++i ) {
     count = t.distinct( 'c', {$or:[{a:{$gte:0},d:0},{b:{$gte:0}}]} ).length;

jstests/evalb.js

@@ -1,17 +1,40 @@
+// Check the return value of a db.eval function running a database query, and ensure the function's
+// contents are logged in the profile log.
 
-t = db.evalb;
-t.drop();
+// Use a reserved database name to avoid a conflict in the parallel test suite.
+var stddb = db;
+var db = db.getSisterDB( 'evalb' );
 
-t.save( { x : 3 } );
+function profileCursor() {
+    return db.system.profile.find( { user:username } );
+}
 
-assert.eq( 3, db.eval( function(){ return db.evalb.findOne().x; } ) , "A" );
+function lastOp() {
+    return profileCursor().sort( { $natural:-1 } ).next();
+}
 
-db.setProfilingLevel( 2 );
+try {
 
-assert.eq( 3, db.eval( function(){ return db.evalb.findOne().x; } ) , "B" );
+    username = 'jstests_evalb_user';
+    db.addUser( username, 'password', false, 1 );
+    db.auth( username, 'password' );
 
-o = db.system.profile.find( { "command.$eval" : { $exists : true } } ).sort( { $natural : -1 } ).limit(1).next();
-assert( tojson(o).indexOf( "findOne().x" ) > 0 , "C : " + tojson( o ) )
+    t = db.evalb;
+    t.drop();
 
-db.setProfilingLevel( 0 );
+    t.save( { x:3 } );
 
+    assert.eq( 3, db.eval( function() { return db.evalb.findOne().x; } ), 'A' );
+
+    db.setProfilingLevel( 2 );
+
+    assert.eq( 3, db.eval( function() { return db.evalb.findOne().x; } ), 'B' );
+
+    o = lastOp();
+    assert( tojson( o ).indexOf( 'findOne().x' ) > 0, 'C : ' + tojson( o ) );
+}
+finally {
+
+    db.setProfilingLevel(0);
+    db = stddb;
+}

jstests/find_and_modify_server6909.js

@@ -0,0 +1,21 @@
+c = db.find_and_modify_server6906;
+
+
+c.drop();
+
+c.insert( { _id : 5 , a:{ b:1 } } );
+ret = c.findAndModify( { query:{ 'a.b':1 },
+                         update:{ $set:{ 'a.b':2 } }, // Ensure the query on 'a.b' no longer matches.          
+                         new:true } );
+assert.eq( 5, ret._id );
+assert.eq( 2, ret.a.b );
+
+
+c.drop();
+
+c.insert( { _id : null , a:{ b:1 } } );
+ret = c.findAndModify( { query:{ 'a.b':1 },
+                         update:{ $set:{ 'a.b':2 } }, // Ensure the query on 'a.b' no longer matches.          
+                         new:true } );
+assert.eq( 2, ret.a.b );
+

jstests/find_and_modify_server6993.js

@@ -0,0 +1,9 @@
+
+c = db.find_and_modify_server6993;
+c.drop();
+ 
+c.insert( { a:[ 1, 2 ] } );
+ 
+c.findAndModify( { query:{ a:1 }, update:{ $set:{ 'a.$':5 } } } );
+ 
+assert.eq( 5, c.findOne().a[ 0 ] );

jstests/find_and_modify_server7660.js

@@ -0,0 +1,18 @@
+
+t = db.find_and_modify_server7660;
+t.drop();
+
+a = t.findAndModify({
+    query : { foo : 'bar' },
+    update : { $set : { bob : 'john' } },
+    sort: { foo : 1},
+    upsert: true,
+    new : true
+});
+
+b = t.findOne();
+assert.eq( a, b );
+assert.eq( "bar", a.foo );
+assert.eq( "john", a.bob )
+
+

jstests/geo_box1_noindex.js

@@ -0,0 +1,33 @@
+// SERVER-7343: allow $within without a geo index.
+t = db.geo_box1_noindex;
+t.drop();
+
+num = 0;
+for ( x=0; x<=20; x++ ){
+    for ( y=0; y<=20; y++ ){
+        o = { _id : num++ , loc : [ x , y ] } 
+        t.save( o )
+    }
+}
+
+searches = [
+    [ [ 1 , 2 ] , [ 4 , 5 ] ] ,
+    [ [ 1 , 1 ] , [ 2 , 2 ] ] ,
+    [ [ 0 , 2 ] , [ 4 , 5 ] ] ,
+    [ [ 1 , 1 ] , [ 2 , 8 ] ] ,
+];
+
+for ( i=0; i<searches.length; i++ ){
+    b = searches[i];
+    
+    q = { loc : { $within : { $box : b } } }
+    numWanted = ( 1 + b[1][0] - b[0][0] ) * ( 1 + b[1][1] - b[0][1] );
+    assert.eq( numWanted  , t.find(q).itcount() , "itcount: " + tojson( q ) );
+    printjson( t.find(q).explain() )
+}
+
+assert.eq( 0 , t.find( { loc : { $within : { $box : [ [100 , 100 ] , [ 110 , 110 ] ] } } } ).itcount() , "E1" )
+assert.eq( 0 , t.find( { loc : { $within : { $box : [ [100 , 100 ] , [ 110 , 110 ] ] } } } ).count() , "E2" )
+assert.eq( num , t.find( { loc : { $within : { $box : [ [ 0 , 0 ] , [ 110 , 110 ] ] } } } ).count() , "E3" )
+assert.eq( num , t.find( { loc : { $within : { $box : [ [ 0 , 0 ] , [ 110 , 110 ] ] } } } ).itcount() , "E4" )
+assert.eq( 57 , t.find( { loc : { $within : { $box : [ [ 0 , 0 ] , [ 110 , 110 ] ] } } } ).limit(57).itcount() , "E5" )

jstests/geo_circle1_noindex.js

@@ -0,0 +1,29 @@
+// SERVER-7343: allow $within without a geo index.
+t = db.geo_circle1_noindex;
+t.drop();
+
+searches = [ 
+    [ [ 5 , 5 ] , 3 ] ,
+    [ [ 5 , 5 ] , 1 ] ,
+    [ [ 5 , 5 ] , 5 ] ,
+    [ [ 0 , 5 ] , 5 ] ,
+];
+correct = searches.map( function(z){ return []; } );
+
+num = 0;
+
+for ( x=0; x<=20; x++ ){
+    for ( y=0; y<=20; y++ ){
+        o = { _id : num++ , loc : [ x , y ] } 
+        t.save( o )
+        for ( i=0; i<searches.length; i++ )
+            if ( Geo.distance( [ x , y ] , searches[i][0] ) <= searches[i][1] )
+                correct[i].push( o );
+    }
+}
+
+for ( i=0; i<searches.length; i++ ){
+    q = { loc : { $within : { $center : searches[i] } } }
+    assert.eq( correct[i].length , t.find( q ).itcount() , "itcount : " + tojson( searches[i] ) );
+    assert.eq( correct[i].length , t.find( q ).count() , "count : " + tojson( searches[i] ) );
+}

jstests/geo_polygon1_noindex.js

@@ -0,0 +1,47 @@
+// SERVER-7343: allow $within without a geo index.
+
+t = db.geo_polygon1_noindex;
+t.drop();
+
+num = 0;
+for ( x=1; x < 9; x++ ){
+    for ( y= 1; y < 9; y++ ){
+        o = { _id : num++ , loc : [ x , y ] };
+        t.save( o );
+    }
+}
+
+triangle = [[0,0], [1,1], [0,2]];
+
+// Look at only a small slice of the data within a triangle
+assert.eq( 1 , t.find( { loc: { "$within": { "$polygon" : triangle }}} ).count() , "Triangle Test" );
+
+boxBounds = [ [0,0], [0,10], [10,10], [10,0] ];
+
+assert.eq( num , t.find( { loc : { "$within" : { "$polygon" : boxBounds } } } ).count() , "Bounding Box Test" );
+
+//Make sure we can add object-based polygons
+assert.eq( num, t.find( { loc : { $within : { $polygon : { a : [-10, -10], b : [-10, 10], c : [10, 10], d : [10, -10] } } } } ).count() )
+
+// Look in a box much bigger than the one we have data in
+boxBounds = [[-100,-100], [-100, 100], [100,100], [100,-100]];
+assert.eq( num , t.find( { loc : { "$within" : { "$polygon" : boxBounds } } } ).count() , "Big Bounding Box Test" );
+
+t.drop();
+
+pacman = [
+           [0,2], [0,4], [2,6], [4,6], // Head
+           [6,4], [4,3], [6,2], // Mouth
+           [4,0], [2,0] // Bottom
+         ];
+
+t.save({loc: [1,3] }); // Add a point that's in
+assert.isnull( db.getLastError() )
+
+assert.eq( 1 , t.find({loc : { $within : { $polygon : pacman }}} ).count() , "Pacman single point" );
+
+t.save({ loc : [5, 3] })  // Add a point that's out right in the mouth opening
+t.save({ loc : [3, 7] })  // Add a point above the center of the head
+t.save({ loc : [3,-1] })  // Add a point below the center of the bottom
+
+assert.eq( 1 , t.find({loc : { $within : { $polygon : pacman }}} ).count() , "Pacman double point" );

jstests/geo_withinquery.js

@@ -0,0 +1,15 @@
+// SERVER-7343: allow $within without a geo index.
+t = db.geo_withinquery;
+t.drop();
+
+num = 0;
+for ( x=0; x<=20; x++ ){
+    for ( y=0; y<=20; y++ ){
+        o = { _id : num++ , loc : [ x , y ] } 
+        t.save( o )
+    }
+}
+
+assert.eq(21 * 21 - 1, t.find({ $and: [ {loc: {$ne:[0,0]}},
+                                        {loc: {$within: {$box: [[0,0], [100,100]]}}},
+                                      ]}).itcount(), "UHOH!")

jstests/replsets/localhostAuthBypass.js

@@ -0,0 +1,120 @@
+//SERVER-6591: Localhost authentication exception doesn't work right on sharded cluster
+//
+//This test is to ensure that localhost authentication works correctly against a replica set 
+//whether they are hosted with "localhost" or a hostname.
+
+var replSetName = "replsets_server-6591";
+var keyfile = "jstests/libs/key1";
+var memberCount = 3;
+var username = "foo";
+var password = "bar";
+
+var addUser = function(mongo) {
+    print("============ adding a user.");
+    mongo.getDB("admin").addUser(username, password);
+};
+
+var assertCannotRunCommands = function(mongo) {
+    print("============ ensuring that commands cannot be run.");
+
+    var test = mongo.getDB("test");
+    assert.throws( function() { test.system.users.findOne(); });
+
+    test.foo.save({_id:0});
+    test.foo.update({_id:0}, {$set:{x:20}});
+    test.foo.remove({_id:0});
+    assert.throws(function() { test.getLastError(); });
+
+    assert.throws( function() { test.foo.findOne({_id:0}); });
+
+    assert.throws(function() { 
+        test.foo.mapReduce(
+            function() { emit(1, 1); }, 
+            function(id, count) { return Array.sum(count); },
+            { out: "other" });
+    });
+};
+
+var assertCanRunCommands = function(mongo) {
+    print("============ ensuring that commands can be run.");
+
+    var test = mongo.getDB("test");
+    // will throw on failure
+    test.system.users.findOne();
+
+    test.foo.save({_id: 0});
+    assert(test.getLastError() == null);
+    
+    test.foo.update({_id: 0}, {$set:{x:20}});
+    assert(test.getLastError() == null);
+    
+    test.foo.remove({_id: 0});
+    assert(test.getLastError() == null);
+    
+    test.foo.mapReduce(
+        function() { emit(1, 1); }, 
+        function(id, count) { return Array.sum(count); },
+        { out: "other" }
+    );
+};
+
+var authenticate = function(mongo) {
+    print("============ authenticating user.");
+    mongo.getDB("admin").auth(username, password);
+};
+
+var start = function(useHostName) {
+    var rs = new ReplSetTest({name: replSetName, 
+        nodes : 3, 
+        keyFile : keyfile,  
+        useHostName: useHostName});
+
+    rs.startSet();
+    rs.initiate();
+    return rs;
+};
+
+var shutdown = function(rs) {
+    print("============ shutting down.");
+    rs.stopSet(/*signal*/false, 
+        /*forRestart*/false, 
+        { auth: { user: username, pwd: password}});
+};
+
+var runTest = function(useHostName) {
+    print("=====================");
+    print("starting replica set: useHostName=" + useHostName);
+    print("=====================");
+    var rs = start(useHostName);
+    var port = rs.getPort(rs.getPrimary());
+    var host = "localhost:" + port;
+
+    var mongo = new Mongo(host);
+
+    assertCanRunCommands(mongo);
+
+    addUser(mongo);
+
+    assertCannotRunCommands(mongo);
+
+    authenticate(mongo);
+
+    assertCanRunCommands(mongo);
+
+    print("===============================");
+    print("reconnecting with a new client.");
+    print("===============================");
+
+    mongo = new Mongo(host);
+
+    assertCannotRunCommands(mongo);
+
+    authenticate(mongo);
+
+    assertCanRunCommands(mongo);
+
+    shutdown(rs);
+}
+
+runTest(false);
+runTest(true);

jstests/replsets/no_chaining.js

@@ -0,0 +1,74 @@
+
+function myprint( x ) {
+    print( "chaining output: " + x );
+}
+
+var replTest = new ReplSetTest({name: 'testSet', nodes: 3});
+var nodes = replTest.startSet();
+var hostnames = replTest.nodeList();
+replTest.initiate(
+    {
+        "_id" : "testSet",
+        "members" : [
+            {"_id" : 0, "host" : hostnames[0], "priority" : 2},
+            {"_id" : 1, "host" : hostnames[1]},
+            {"_id" : 2, "host" : hostnames[2]}
+        ],
+        "settings" : {
+            "chainingAllowed" : false
+        }
+    }
+);
+
+var master = replTest.getMaster();
+replTest.awaitReplication();
+
+
+var breakNetwork = function() {
+    replTest.bridge();
+    replTest.partition(0, 2);
+    master = replTest.getMaster();
+};
+
+var checkNoChaining = function() {
+    master.getDB("test").foo.insert({x:1});
+
+    assert.soon(
+        function() {
+            return nodes[1].getDB("test").foo.findOne() != null;
+        }
+    );
+
+    var endTime = (new Date()).getTime()+10000;
+    while ((new Date()).getTime() < endTime) {
+        print('CHAINING IS NOT HAPPENING');
+        assert(nodes[2].getDB("test").foo.findOne() == null,
+               'Check that 2 does not catch up');
+    }
+};
+
+var forceSync = function() {
+    assert.soon(
+        function() {
+            var config = nodes[2].getDB("local").system.replset.findOne();
+            var targetHost = config.members[1].host;
+            printjson(nodes[2].getDB("admin").runCommand({replSetSyncFrom : targetHost}));
+            return nodes[2].getDB("test").foo.findOne() != null;
+        },
+        'Check force sync still works'
+    );
+};
+
+if (!_isWindows()) {
+    print("break the network so that node 2 cannot replicate");
+    breakNetwork();
+
+    print("make sure chaining is not happening");
+    checkNoChaining();
+
+    print("check that forcing sync target still works");
+    forceSync();
+
+    var config = master.getDB("local").system.replset.findOne();
+    assert.eq(false, config.settings.chainingAllowed, tojson(config));
+}

jstests/replsets/replset9.js

@@ -0,0 +1,52 @@
+
+
+var rt = new ReplSetTest( { name : "replset9tests" , nodes: 1, oplogSize: 400 } );
+
+var nodes = rt.startSet();
+rt.initiate();
+var master = rt.getMaster();
+var bigstring = "a";
+var md = master.getDB( 'd' );
+var mdc = md[ 'c' ];
+
+// idea: while cloner is running, update some docs and then immediately remove them.
+// oplog will have ops referencing docs that no longer exist.
+
+var doccount = 20000;
+// Avoid empty extent issues
+mdc.insert( { _id:-1, x:"dummy" } );
+
+// Make this db big so that cloner takes a while.
+print ("inserting bigstrings");
+for( i = 0; i < doccount; ++i ) {
+    mdc.insert( { _id:i, x:bigstring } );
+    bigstring += "a";
+}
+md.getLastError();
+
+// Insert some docs to update and remove
+print ("inserting x");
+for( i = doccount; i < doccount*2; ++i ) {
+    mdc.insert( { _id:i, bs:bigstring, x:i } );
+}
+md.getLastError();
+
+// add a secondary; start cloning
+var slave = rt.add();
+rt.reInitiate();
+print ("initiation complete!");
+var sc = slave.getDB( 'd' )[ 'c' ];
+slave.setSlaveOk();
+
+print ("updating and deleting documents");
+for (i = doccount*2; i > doccount; --i) {
+    mdc.update( { _id:i }, { $inc: { x : 1 } } );
+    md.getLastError();
+    mdc.remove( { _id:i } );
+    md.getLastError();
+    mdc.insert( { bs:bigstring } );
+    md.getLastError();
+}
+print ("finished");
+// Wait for replication to catch up.
+rt.awaitReplication(640000);

jstests/replsets/rollback5.js

@@ -0,0 +1,102 @@
+// test that a rollback directory is created during a replica set rollback
+// this also tests that updates are recorded in the rollback file
+//  (this test does no delete rollbacks)
+
+var replTest = new ReplSetTest({ name: 'rollback5', nodes: 3 });
+var nodes = replTest.nodeList();
+
+var conns = replTest.startSet();
+var r = replTest.initiate({ "_id": "rollback5",
+                            "members": [
+                                { "_id": 0, "host": nodes[0] },
+                                { "_id": 1, "host": nodes[1] },
+                                { "_id": 2, "host": nodes[2], arbiterOnly: true}]
+                          });
+
+// Make sure we have a master
+var master = replTest.getMaster();
+var a_conn = conns[0];
+var b_conn = conns[1];
+a_conn.setSlaveOk();
+b_conn.setSlaveOk();
+var A = a_conn.getDB("test");
+var B = b_conn.getDB("test");
+var AID = replTest.getNodeId(a_conn);
+var BID = replTest.getNodeId(b_conn);
+var Apath = "/data/db/rollback5-0/";
+var Bpath = "/data/db/rollback5-1/";
+assert(master == conns[0], "conns[0] assumed to be master");
+assert(a_conn.host == master.host);
+
+// Make sure we have an arbiter
+assert.soon(function () {
+    res = conns[2].getDB("admin").runCommand({ replSetGetStatus: 1 });
+    return res.myState == 7;
+}, "Arbiter failed to initialize.");
+
+A.foo.update({key:'value1'}, {$set: {req: 'req'}}, true);
+A.foo.runCommand({getLastError : 1, w : 2, wtimeout : 60000});
+replTest.stop(AID);
+
+master = replTest.getMaster();
+assert(b_conn.host == master.host);
+B.foo.update({key:'value1'}, {$set: {res: 'res'}}, true);
+B.foo.runCommand({getLastError : 1, w : 1, wtimeout : 60000});
+replTest.stop(BID);
+replTest.restart(AID);
+master = replTest.getMaster();
+assert(a_conn.host == master.host);
+A.foo.update({key:'value2'}, {$set: {req: 'req'}}, true);
+A.foo.runCommand({getLastError : 1, w : 1, wtimeout : 60000});
+replTest.restart(BID); // should rollback
+reconnect(B);
+
+print("BEFORE------------------");
+printjson(A.foo.find().toArray());
+
+replTest.awaitReplication();
+replTest.awaitSecondaryNodes();
+
+print("AFTER------------------");
+printjson(A.foo.find().toArray());
+
+assert.eq(2, A.foo.count());
+assert.eq('req', A.foo.findOne({key:'value1'}).req);
+assert.eq(null, A.foo.findOne({key:'value1'}).res);
+assert.eq(2, B.foo.count());
+assert.eq('req', B.foo.findOne({key:'value1'}).req);
+assert.eq(null, B.foo.findOne({key:'value1'}).res);
+
+// check here for rollback files
+var rollbackDir = Bpath + "rollback/";
+assert(pathExists(rollbackDir), "rollback directory was not created!");
+
+print("rollback5.js SUCCESS");
+replTest.stopSet(15);
+
+
+function wait(f) {
+    var n = 0;
+    while (!f()) {
+        if (n % 4 == 0)
+            print("rollback5.js waiting");
+        if (++n == 4) {
+            print("" + f);
+        }
+        assert(n < 200, 'tried 200 times, giving up');
+        sleep(1000);
+    }
+}
+
+function reconnect(a) {
+  wait(function() {
+      try {
+        a.bar.stats();
+        return true;
+      } catch(e) {
+        print(e);
+        return false;
+      }
+    });
+};
+

jstests/replsets/stepdown3.js

@@ -0,0 +1,48 @@
+// Test that GLE asserts when the primary steps down while we're waiting for w:
+
+var replTest = new ReplSetTest({ name: 'testSet', nodes: 2 });
+var nodes = replTest.startSet();
+replTest.initiate();
+var master = replTest.getMaster();
+
+// do a write to allow stepping down of the primary;
+// otherwise, the primary will refuse to step down
+print("\ndo a write");
+master.getDB("test").foo.insert({x:1});
+replTest.awaitReplication();
+
+// do another write, because the first one might be longer than 10 seconds ago
+// on the secondary (due to starting up), and we need to be within 10 seconds
+// to step down.
+master.getDB("test").foo.insert({x:2});
+master.getDB("test").runCommand({getLastError : 1, w : 2, wtimeout : 30000 });
+// lock secondary, to pause replication
+print("\nlock secondary");
+var locked = replTest.liveNodes.slaves[0];
+printjson( locked.getDB("admin").runCommand({fsync : 1, lock : 1}) );
+
+// do a write
+print("\ndo a write");
+master.getDB("test").foo.insert({x:3});
+
+// step down the primary asyncronously
+print("stepdown");
+var command = "sleep(4000); tojson(db.adminCommand( { replSetStepDown : 60, force : 1 } ));"
+var waitfunc = startParallelShell(command, master.port);
+
+print("getlasterror; should assert or return an error, depending on timing");
+var gleFunction = function() { 
+    var result = master.getDB("test").runCommand({getLastError : 1, w: 2 , wtimeout :30000 });
+    if (result.errmsg === "not master") {
+        throw new Error("satisfy assert.throws()");
+    }
+    print("failed to throw exception; GLE returned: ");
+    printjson(result);
+};
+var result = assert.throws(gleFunction);
+print("result of gle:");
+printjson(result);
+
+// unlock and shut down
+printjson(locked.getDB("admin").$cmd.sys.unlock.findOne());
+replTest.stopSet();

jstests/sharding/addshard5.js

@@ -36,8 +36,8 @@ coll.insert({ hello : "world" })
 assert.eq( null, coll.getDB().getLastError() )
 
 // Migrate the collection to and from shard2 so shard1 loads the shard2 host
-printjson( admin.runCommand({ moveChunk : coll + "", find : { _id : 0 }, to : shards[1]._id }) )
-printjson( admin.runCommand({ moveChunk : coll + "", find : { _id : 0 }, to : shards[0]._id }) )
+printjson( admin.runCommand({ moveChunk : coll + "", find : { _id : 0 }, to : shards[1]._id, _waitForDelete : true }) )
+printjson( admin.runCommand({ moveChunk : coll + "", find : { _id : 0 }, to : shards[0]._id, _waitForDelete : true }) )
 
 //
 // Drop and re-add shard with last shard's host

jstests/sharding/auth.js

@@ -132,6 +132,9 @@ s.getDB("test").foo.insert({x:1});
 login(testUser);
 assert.eq(s.getDB("test").foo.findOne(), null);
 
+print("Test that reading system.users as read-write user works");
+s.getDB("test").system.users.findOne();
+
 print("insert try 2");
 s.getDB("test").foo.insert({x:1});
 result = s.getDB("test").getLastErrorObj();
@@ -261,6 +264,9 @@ login( testUserReadOnly , readOnlyS );
 print( "   testing find that should work" );
 readOnlyDB.foo.findOne();
 
+print("   testing read of system.users that should fail");
+assert.throws(function() {readOnlyDB.system.users.findOne();});
+
 print( "   testing write that should fail" );
 readOnlyDB.foo.insert( { eliot : 1 } );
 result = readOnlyDB.getLastError();

jstests/sharding/auth2.js

@@ -8,29 +8,7 @@ var mongos = st.s;
 var adminDB = mongos.getDB('admin');
 var db = mongos.getDB('test')
 
-
-// SERVER-6591: can't add first admin user even when connected to mongos on localhost.
-var addUser = function( db, username, password ) {
-    var conn = db.getMongo();
-    // Get a connection over localhost so that the first user can be added.
-    if ( conn.host.indexOf('localhost') != 0 && conn.host.split(',').length > 1 ) {
-        print( 'Getting locahost connection instead of ' + conn + ' to add user' );
-        var hosts = conn.host.split(',');
-        for ( var i = 0; i < hosts.length; i++ ) {
-            conn = new Mongo( 'localhost:' + hosts[i].split(':')[1] );
-            print( "Adding user on connection: " + conn );
-            if ( !conn.getDB('admin').addUser( username, password ) ) {
-                return false;
-            }
-        }
-        return true;
-    } else {
-        return conn.getDB('admin').addUser( username, password );
-    }
-}
-
-
-addUser( st._configConnection.getDB('admin'), 'admin', 'password' );
+adminDB.addUser('admin', 'password');
 
 jsTestLog( "Add user was successful" );
 

jstests/sharding/authCommands.js

@@ -16,9 +16,12 @@ assert.eq(0, db.runCommand({dbStats : 1}).ok);
 
 assert( db.getSiblingDB('local').auth('__system', 'foopdedoop'), "Failed to authenticate as system user" );
 
-assert.eq(0, db.runCommand({dbStats : 1}).ok);
+// Because of SERVER-6897, commands sent without an $auth table are assumed to have full access
+// to preserve compatibility with 2.0
+// assert.eq(0, db.runCommand({dbStats : 1}).ok); // SERVER-6897
 assert.eq(1, db.runCommand({dbStats : 1, $auth : { test : { userName : NumberInt(1) } } } ).ok );
-assert.eq(0, db.runCommand({dbStats : 1}).ok); // Make sure the credentials are temporary.
+ // SERVER-6897
+// assert.eq(0, db.runCommand({dbStats : 1}).ok); // Make sure the credentials are temporary.
 assert.eq(0, db.runCommand({dropDatabase : 1, $auth : { test : { userName : NumberInt(1) } } } ).ok );
 assert.eq(1, db.runCommand({dropDatabase : 1, $auth : { test : { userName : NumberInt(2) } } } ).ok );
 

jstests/sharding/authConnectionHook.js

@@ -0,0 +1,46 @@
+// Test for SERVER-8786 - if the first operation on an authenticated shard is moveChunk, it breaks the cluster.
+var st = new ShardingTest({ keyFile : 'jstests/libs/key1', shards : 2, chunksize : 1, config : 3, verbose : 2,
+                            other : { nopreallocj : 1, verbose : 2, useHostname : true,
+                                      configOptions : { verbose : 2 }}});
+
+var mongos = st.s;
+var adminDB = mongos.getDB('admin');
+var db = mongos.getDB('test')
+
+adminDB.addUser('admin', 'password');
+
+adminDB.auth('admin', 'password');
+
+adminDB.runCommand({enableSharding : "test"});
+adminDB.runCommand({shardCollection : "test.foo", key : {x : 1}});
+st.stopBalancer();
+
+for (var i = 0; i < 100; i++) {
+    db.foo.insert({x:i});
+}
+
+adminDB.runCommand({split: "test.foo", middle: {x:50}});
+var curShard = st.getShard("test.foo", {x:75});
+var otherShard = st.getOther(curShard).name;
+adminDB.runCommand({moveChunk: "test.foo", find: {x:25}, to: otherShard});
+assert.soon( function() { return !st.isAnyBalanceInFlight(); });
+
+st.printShardingStatus();
+
+MongoRunner.stopMongod(st.shard0);
+st.shard0 = MongoRunner.runMongod({restart : st.shard0});
+
+// May fail the first couple times due to socket exceptions
+assert.soon( function() {
+                 var res = adminDB.runCommand({moveChunk: "test.foo",
+                                               find: {x:75},
+                                               to: otherShard});
+                 printjson(res);
+                 return res.ok;
+             });
+
+
+printjson(db.foo.findOne({x:25}));
+printjson(db.foo.findOne({x:75}));
+
+st.stop();

jstests/sharding/auth_slaveok_routing.js

@@ -84,6 +84,12 @@ coll.setSlaveOk( true );
  */
 ReplSetTest.awaitRSClientHosts( mongos, replTest.getSecondaries(),
     { ok : true, secondary : true });
+//
+// We also need to wait for the primary, it's possible that the mongos may think a node is a 
+// secondary but it actually changed to a primary before we send our final query.
+//
+ReplSetTest.awaitRSClientHosts( mongos, replTest.getPrimary(),
+    { ok : true, ismaster : true });
 
 // Recheck if we can still query secondaries after refreshing connections.
 jsTest.log( 'Final query to SEC' );

jstests/sharding/count1.js

@@ -63,7 +63,7 @@ assert.eq( 6 , db.foo.find().count() , "basic count after split " );
 assert.eq( 6 , db.foo.find().sort( { name : 1 } ).count() , "basic count after split sorted " );
 
 // part 4
-s.adminCommand( { movechunk : "test.foo" , find : { name : "allan" } , to : secondary.getMongo().name } );
+s.adminCommand( { movechunk : "test.foo" , find : { name : "allan" } , to : secondary.getMongo().name , _waitForDelete : true } );
 
 assert.eq( 3 , primary.foo.find().toArray().length , "primary count" );
 assert.eq( 3 , secondary.foo.find().toArray().length , "secondary count" );

jstests/sharding/count2.js

@@ -26,7 +26,7 @@ assert.eq( 3, db2.count( { name : { $gte: "aaa" , $lt: "ddd" } } ) , "initial co
 
 s1.printChunks( "test.foo" )
 
-s1.adminCommand( { movechunk : "test.foo" , find : { name : "aaa" } , to : s1.getOther( s1.getServer( "test" ) ).name } );
+s1.adminCommand( { movechunk : "test.foo" , find : { name : "aaa" } , to : s1.getOther( s1.getServer( "test" ) ).name, _waitForDelete : true });
 
 assert.eq( 3, db1.count( { name : { $gte: "aaa" , $lt: "ddd" } } ) , "post count mongos1" );
 

jstests/sharding/delete_during_migrate.js

@@ -0,0 +1,42 @@
+// Test migrating a big chunk while deletions are happening within that chunk.
+// Test is slightly non-deterministic, since removes could happen before migrate
+// starts. Protect against that by making chunk very large.
+
+// start up a new sharded cluster
+var st = new ShardingTest({ shards : 2, mongos : 1 });
+
+// stop balancer since we want manual control for this
+st.stopBalancer();
+
+var dbname = "testDB";
+var coll = "foo";
+var ns = dbname + "." + coll;
+var s = st.s0;
+var t = s.getDB( dbname ).getCollection( coll );
+
+// Create fresh collection with lots of docs
+t.drop();
+for ( i=0; i<200000; i++ ){
+    t.insert( { a : i  } );
+}
+
+// enable sharding of the collection. Only 1 chunk.
+t.ensureIndex( { a : 1 } );
+s.adminCommand( { enablesharding : dbname } );
+s.adminCommand( { shardcollection : ns , key: { a : 1 } } );
+
+// start a parallel shell that deletes things
+startMongoProgramNoConnect( "mongo" ,
+                            "--host" , getHostName() ,
+                            "--port" , st.s0.port ,
+                            "--eval" , "db." + coll + ".remove({});" ,
+                            dbname );
+
+// migrate while deletions are happening
+var moveResult =  s.adminCommand( { moveChunk : ns ,
+                                    find : { a : 1 } ,
+                                    to : st.getOther( st.getServer( dbname ) ).name } );
+// check if migration worked
+assert( moveResult.ok , "migration didn't work while doing deletes" );
+
+st.stop();

jstests/sharding/deletion_range.js

@@ -0,0 +1,43 @@
+//
+// Tests deletion ranges for a sharded system when using prefix shard key
+//
+
+var st = new ShardingTest({ shards : 2, mongos : 2 });
+
+st.stopBalancer();
+
+var mongos = st.s0;
+var config = mongos.getDB( "config" );
+var admin = mongos.getDB( "admin" );
+var shards = config.shards.find().toArray();
+var shard0 = new Mongo( shards[0].host );
+var shard1 = new Mongo( shards[1].host );
+
+var coll = mongos.getCollection( "foo.bar" );
+
+printjson( admin.runCommand({ enableSharding : coll.getDB() + "" }) );
+printjson( admin.runCommand({ movePrimary : coll.getDB() + "", to : shards[0]._id }) );
+printjson( coll.ensureIndex({ skey : 1, extra : 1 }) );
+printjson( admin.runCommand({ shardCollection : coll + "", key : { skey : 1 } }) );
+
+for( var i = 0; i < 5; i++ ){
+    coll.insert({ skey : 0, extra : i });
+}
+assert.eq( null, coll.getDB().getLastError() );
+
+printjson( admin.runCommand({ split : coll + "", middle : { skey : 0 } }) );
+printjson( admin.runCommand({ moveChunk : coll + "", find : { skey : 0 }, to : shards[1]._id }) );
+
+printjson( shard0.getCollection( coll + "" ).find().toArray() );
+printjson( shard1.getCollection( coll + "" ).find().toArray() );
+
+assert( coll.find().itcount() == 5 );
+
+printjson( admin.runCommand({ moveChunk : coll + "", find : { skey : -1 }, to : shards[1]._id }) );
+
+assert.eq( 0 , shard0.getCollection( coll + "" ).find().itcount() );
+assert.eq( 5 , shard1.getCollection( coll + "" ).find().itcount() );
+
+assert( coll.find().itcount() == 5 );
+
+st.stop()

jstests/sharding/error1.js

@@ -24,7 +24,7 @@ db.foo2.save( { _id : 3 , num : 15 } );
 db.foo2.save( { _id : 4 , num : 20 } );
 
 s.adminCommand( { split : "test.foo2" , middle : { num : 10 } } );
-s.adminCommand( { movechunk : "test.foo2" , find : { num : 20 } , to : s.getOther( s.getServer( "test" ) ).name } );
+s.adminCommand( { movechunk : "test.foo2" , find : { num : 20 } , to : s.getOther( s.getServer( "test" ) ).name, _waitForDelete : true } );
 
 print( "a: " + a.foo2.count() );
 print( "b: " + b.foo2.count() );

jstests/sharding/features2.js

@@ -176,7 +176,9 @@ catch ( e ){
     y = e;
 }
 
-assert.eq( x , y , "assert format" )
+assert.eq( x.code , y.code , "assert format" )
+assert.eq( x.errmsg , y.errmsg , "assert format" )
+assert.eq( x.ok , y.ok , "assert format" )
 
 // isMaster and query-wrapped-command
 isMaster = db.runCommand({isMaster:1});

jstests/sharding/findandmodify2.js

@@ -103,10 +103,10 @@ s.printChunks();
 
 
 print("---------- Verifying that both codepaths resulted in splits...");
-assert.gt( s.config.chunks.count({ "ns": "test." + col_fam }), minChunks, "findAndModify update code path didn't result in splits" );
-assert.gt( s.config.chunks.count({ "ns": "test." + col_fam_upsert }), minChunks, "findAndModify upsert code path didn't result in splits" );
-assert.gt( s.config.chunks.count({ "ns": "test." + col_update }), minChunks, "update code path didn't result in splits" );
-assert.gt( s.config.chunks.count({ "ns": "test." + col_update_upsert }), minChunks, "upsert code path didn't result in splits" );
+assert.gte( s.config.chunks.count({ "ns": "test." + col_fam }), minChunks, "findAndModify update code path didn't result in splits" );
+assert.gte( s.config.chunks.count({ "ns": "test." + col_fam_upsert }), minChunks, "findAndModify upsert code path didn't result in splits" );
+assert.gte( s.config.chunks.count({ "ns": "test." + col_update }), minChunks, "update code path didn't result in splits" );
+assert.gte( s.config.chunks.count({ "ns": "test." + col_update_upsert }), minChunks, "upsert code path didn't result in splits" );
 
 printjson( db[col_update].stats() );
 

jstests/sharding/geo_near_random2.js

@@ -18,7 +18,7 @@ test.insertPts(5000);
 for (var i = (test.nPts/10); i < test.nPts; i+= (test.nPts/10)){
     s.adminCommand({split: ('test.' + testName), middle: {_id: i} });
     try {
-        s.adminCommand({moveChunk: ('test.' + testName), find: {_id: i-1}, to: ('shard000' + (i%3))});
+        s.adminCommand({moveChunk: ('test.' + testName), find: {_id: i-1}, to: ('shard000' + (i%3)), _waitForDelete : true });
     } catch (e) {
         // ignore this error
         if (! e.match(/that chunk is already on that shard/)){

jstests/sharding/gle_error_message.js

@@ -111,7 +111,11 @@ jsTest.log( "Testing stale version GLE when host goes down..." )
 var staleColl = st.s1.getCollection( coll + "" )
 staleColl.findOne()
 
-printjson( admin.runCommand({ moveChunk : "" + coll, find : { _id : 0 }, to : shards[2]._id }) )
+// As it turns out, on the *second* auto-reconnect attempt we need to wait at least 2 secs,
+// otherwise reconnect fails with FAILED_STATE 
+sleep( 2000 );
+
+assert( admin.runCommand({ moveChunk : "" + coll, find : { _id : 0 }, to : shards[2]._id }).ok );
 
 MongoRunner.stopMongod( st.shard2 )
 
@@ -124,4 +128,4 @@ assert.neq( null, staleColl.getDB().getLastError() )
 
 jsTest.log( "Done!" )
 
-st.stop()
+st.stop()

jstests/sharding/gle_with_conf_servers.js

@@ -11,14 +11,6 @@ function writeToConfigTest(){
 
     assert( gleObj.ok );
 
-    printjson( gleObj );
-    assert( gleObj.hasOwnProperty( 'shardRawGLE' ),
-        'missing shardRawGLE from get last error fields!' );
-    
-    var shardGLE = gleObj.shardRawGLE[ st.config0.host ];
-    assert( shardGLE.ok );
-    assert.neq( null, shardGLE.err );
-
     st.stop();
 }
 

jstests/sharding/key_many.js

@@ -98,7 +98,7 @@ for ( var i=0; i<types.length; i++ ){
     s.adminCommand( { split : longName , find : makeObjectDotted( curT.values[3] ) } );
     s.adminCommand( { split : longName , find : makeObjectDotted( curT.values[3] ) } );
 
-    s.adminCommand( { movechunk : longName , find : makeObjectDotted( curT.values[0] ) , to : secondary.getMongo().name } );
+    s.adminCommand( { movechunk : longName , find : makeObjectDotted( curT.values[0] ) , to : secondary.getMongo().name, _waitForDelete : true } );
     
     s.printChunks();
     

jstests/sharding/key_string.js

@@ -24,7 +24,7 @@ s.adminCommand( { split : "test.foo" , find : { name : "joe" } } ); // [Minkey -
 s.adminCommand( { split : "test.foo" , find : { name : "joe" } } ); // * [allan -> sara) , [sara -> Maxkey)
 s.adminCommand( { split : "test.foo" , find : { name : "joe" } } ); // [alan -> joe) , [joe -> sara]
 
-s.adminCommand( { movechunk : "test.foo" , find : { name : "allan" } , to : seconday.getMongo().name } );
+s.adminCommand( { movechunk : "test.foo" , find : { name : "allan" } , to : seconday.getMongo().name, _waitForDelete : true } );
 
 s.printChunks();
 

jstests/sharding/limit_push.js

@@ -3,6 +3,9 @@
 
 s = new ShardingTest( "limit_push", 2, 1, 1 );
 
+// Stop balancer since we do manual moves.
+s.stopBalancer();
+
 db = s.getDB( "test" );
 
 // Create some data
@@ -16,7 +19,7 @@ s.adminCommand( { shardcollection : "test.limit_push" , key : { x : 1 } } );
 
 // Now split the and move the data between the shards
 s.adminCommand( { split : "test.limit_push", middle : { x : 50 }} ); 
-s.adminCommand( { moveChunk: "test.limit_push", find : { x : 51}, to : "shard0000" }) 
+s.adminCommand( { moveChunk: "test.limit_push", find : { x : 51}, to : "shard0000", _waitForDelete : true }) 
 
 // Check that the chunck have split correctly
 assert.eq( 2 , s.config.chunks.count() , "wrong number of chunks");

jstests/sharding/listDatabases.js

@@ -0,0 +1,46 @@
+// tests that listDatabases doesn't show config db on a shard, even if it is there
+
+var test = new ShardingTest({shards: 1, mongos: 1, config: 1, other: {chunksize:1, separateConfig:true}})
+
+var mongos = test.s0
+var mongod = test.shard0;
+
+//grab the config db instance by name
+var getDBSection = function (dbsArray, dbToFind) {
+    for(var pos in dbsArray) {
+        if (dbsArray[pos].name && dbsArray[pos].name === dbToFind)
+            return dbsArray[pos];
+    }
+    return null;
+}
+
+mongos.getDB("blah").foo.insert({_id:1})
+mongos.getDB("foo").foo.insert({_id:1})
+mongos.getDB("raw").foo.insert({_id:1})
+//wait for writes to finish
+mongos.getDB("raw").getLastError()
+
+//verify that the config db is not on a shard
+var res = mongos.adminCommand("listDatabases");
+var dbArray = res.databases;
+assert(getDBSection(dbArray, "config"), "config db not found! 1")
+assert(!getDBSection(dbArray, "config").shards, "config db is on a shard! 1")
+
+//add doc in config/admin db on the shard
+mongod.getDB("config").foo.insert({_id:1})
+mongod.getDB("admin").foo.insert({_id:1})
+
+//add doc in admin db (via mongos)
+mongos.getDB("admin").foo.insert({_id:1})
+
+//verify that the config db is not on a shard
+var res = mongos.adminCommand("listDatabases");
+var dbArray = res.databases;
+//check config db
+assert(getDBSection(dbArray, "config"), "config db not found! 2")
+assert(!getDBSection(dbArray, "config").shards, "config db is on a shard! 2")
+//check admin db
+assert(getDBSection(dbArray, "admin"), "admin db not found! 2")
+assert(!getDBSection(dbArray, "admin").shards, "admin db is on a shard! 2")
+
+test.stop()

jstests/sharding/localhostAuthBypass.js

@@ -0,0 +1,226 @@
+//SERVER-6591: Localhost authentication exception doesn't work right on sharded cluster
+//
+//This test is to ensure that localhost authentication works correctly against a sharded
+//cluster whether they are hosted with "localhost" or a hostname.
+
+var replSetName = "replsets_server-6591";
+var keyfile = "jstests/libs/key1";
+var numShards = 2;
+var numConfigs = 3;
+var username = "foo";
+var password = "bar";
+
+var addUser = function(mongo) {
+    print("============ adding a user.");
+    mongo.getDB("admin").addUser(username, password);
+};
+
+var addUsersToEachShard = function(st) {
+    for(i = 0; i < numShards; i++) {
+        print("============ adding a user to shard " + i);
+        var d = st["shard" + i];
+        d.getDB("admin").addUser(username, password);
+    }
+};
+
+var findEmptyShard = function(st, ns) {
+    var counts = st.chunkCounts( "foo" )
+
+    for(shard in counts){
+        if(counts[shard] == 0) {
+            return shard;
+        }
+    }
+
+    return null;
+};
+
+var assertCannotRunCommands = function(mongo, st) {
+    print("============ ensuring that commands cannot be run.");
+
+    // CRUD
+    var test = mongo.getDB("test");
+    assert.throws( function() { test.system.users.findOne(); });
+
+    test.foo.save({_id:0});
+    test.foo.update({_id:0}, {$set:{x:20}});
+    test.foo.remove({_id:0});
+    assert.throws(function() { test.getLastError(); });
+
+    assert.throws( function() { test.foo.findOne({_id:0}); });
+
+    // Multi-shard
+    assert.throws(function() { 
+        test.foo.mapReduce(
+            function() { emit(1, 1); }, 
+            function(id, count) { return Array.sum(count); },
+            { out: "other" });
+    });
+
+    // Config
+    assert.throws(function() {
+        mongo.getDB("config").shards.findOne();
+    });
+
+    var to = findEmptyShard(st, "test.foo");
+
+    var res = mongo.getDB("admin").runCommand({
+        moveChunk: "test.foo",
+        find: {_id: 1},
+        to: to
+    });
+    assert.commandFailed(res);
+};
+
+var assertCanRunCommands = function(mongo, st) {
+    print("============ ensuring that commands can be run.");
+
+    // CRUD
+    var test = mongo.getDB("test");
+
+    // this will throw if it fails
+    test.system.users.findOne();
+
+    test.foo.save({_id: 0});
+    assert(test.getLastError() == null);
+    
+    test.foo.update({_id: 0}, {$set:{x:20}});
+    assert(test.getLastError() == null);
+    
+    test.foo.remove({_id: 0});
+    assert(test.getLastError() == null);
+    
+    // Multi-shard
+    test.foo.mapReduce(
+        function() { emit(1, 1); }, 
+        function(id, count) { return Array.sum(count); },
+        { out: "other" }
+    );
+
+    // Config
+    // this will throw if it fails
+    mongo.getDB("config").shards.findOne();
+
+    to = findEmptyShard(st, "test.foo");
+    var res = mongo.getDB("admin").runCommand({
+        moveChunk: "test.foo",
+        find: {_id: 1},
+        to: to
+    });
+    assert.commandWorked(res);
+};
+
+var authenticate = function(mongo) {
+    print("============ authenticating user.");
+    mongo.getDB("admin").auth(username, password);
+};
+
+var setupSharding = function(mongo) {
+    print("============ enabling sharding on test.foo.");
+    mongo.getDB("admin").runCommand({enableSharding : "test"});
+    mongo.getDB("admin").runCommand({shardCollection : "test.foo", key : {_id : 1}});
+
+    var test = mongo.getDB("test");
+    for(i = 1; i < 40; i++) {
+        test.foo.insert({_id: i});
+    }
+};
+
+var start = function(useHostName) {
+    return new ShardingTest({
+        keyFile: keyfile, 
+        shards: numShards, 
+        chunksize: 1, 
+        config: numConfigs, 
+        separateConfig: true,
+        other : { 
+            nopreallocj: 1, 
+            useHostName: useHostName 
+        } 
+    });
+};
+
+var shutdown = function(st) {
+    print("============ shutting down.");
+
+    // SERVER-8445
+    // Unlike MongoRunner.stopMongod and ReplSetTest.stopSet,
+    // ShardingTest.stop does not have a way to provide auth
+    // information.  Therefore, we'll do this manually for now.
+
+    for(i = 0; i < st._mongos.length; i++) {
+        var port = st["s" + i].port;
+        MongoRunner.stopMongos(
+            port,
+            /*signal*/false,
+            { auth : { user: username, pwd: password }}
+        );
+    }
+
+    for(i = 0; i < st._connections.length; i++) {
+        var port = st["shard" + i].port;
+        MongoRunner.stopMongod(
+            port, 
+            /*signal*/false, 
+            { auth : { user: username, pwd: password }}
+        );
+    }
+
+    for(i = 0; i < st._configServers.length; i++) {
+        var c = st["config" + i].port;
+        MongoRunner.stopMongod(
+            port, 
+            /*signal*/false, 
+            { auth : { user: username, pwd: password }}
+        );
+    }
+
+    st.stop();
+};
+
+var runTest = function(useHostName) {
+    print("=====================");
+    print("starting shards: useHostName=" + useHostName);
+    print("=====================");
+    var st = start(useHostName);
+    var host = st.s.host;
+
+    var mongo = new Mongo(host);
+
+    setupSharding(mongo);
+
+    assertCanRunCommands(mongo, st);
+
+    addUsersToEachShard(st);
+
+    assertCanRunCommands(mongo, st);
+
+    addUser(mongo);
+
+    // now that we have a user, we need to make sure 
+    // helper functions on st work for the rest of the script.
+    authenticate(st.s); 
+
+    assertCannotRunCommands(mongo, st);
+
+    authenticate(mongo);
+
+    assertCanRunCommands(mongo, st);
+
+    print("===============================");
+    print("reconnecting with a new client.");
+    print("===============================");
+
+    mongo = new Mongo(host);
+
+    assertCannotRunCommands(mongo, st);
+
+    authenticate(mongo);
+
+    assertCanRunCommands(mongo, st);
+
+    shutdown(st);
+}
+
+runTest(false);
+runTest(true);

jstests/sharding/migrateBig.js

@@ -54,6 +54,8 @@ s.config.settings.update( { _id: "balancer" }, { $set : { stopped: false } } , t
 
 assert.soon( function(){ var x = s.chunkDiff( "foo" , "test" ); print( "chunk diff: " + x ); return x < 2; } , "no balance happened" , 8 * 60 * 1000 , 2000 ) 
 
+assert.soon( function(){ return !s.isAnyBalanceInFlight(); } );
+
 assert.eq( coll.count() , coll.find().itcount() );
 
 s.stop()

jstests/sharding/mongos_validate_backoff.js

@@ -39,7 +39,7 @@ for( var test = 0; test < 3; test++ ){
     
     // Kind a heuristic test, we want to make sure that the error wait after sleeping is much less
     // than the error wait after a lot of errors
-    assert.gt( lastWait, firstWait * 2 * 2 * 2 * 2 )
+    assert.gt( lastWait, firstWait * 2 * 2 )
     
     // Sleeping for long enough to reset our exponential counter
     sleep( 3000 )

jstests/sharding/mrShardedOutput.js

@@ -21,16 +21,59 @@ function reduce2(key, values) { return values[0]; }
 var numdocs = 0;
 var numbatch = 100000;
 var nchunks = 0;
-for ( iter=0; iter<2; iter++ ){
+
+var numIterations = 2;
+
+for (var it = 0; it < numIterations; it++) {
+    
+    jsTest.log("Starting new insert batch...");
+    
     // add some more data for input so that chunks will get split further
-    for (i=0; i<numbatch; i++){ db.foo.save({a: Math.random() * 1000, y:str})}
-    db.getLastError();
+    for (i=0; i<numbatch; i++){ db.foo.save({a: Math.random() * 1000, y:str, i : numdocs + i})}
+    
+    assert.eq(null, db.getLastError());
+    
+    jsTest.log("No errors on insert batch.")
+    
     numdocs += numbatch
     
     var isBad = db.foo.find().itcount() != numdocs
     
+    if (isBad) jsTest.log("Insert count is smaller than full count!")
+    
+    if (isBad) {
+        
+        jsTest.log( "Showing document distribution because documents missed..." )
+        
+        // Stop balancing
+        s.stopBalancer();
+        
+        // Wait for writebacks
+        sleep( 10000 );
+        
+        s.printShardingStatus(true);
+        
+        var shards = config.shards.find().toArray();
+        
+        for (var i = 0; i < shards.length; i++){
+            
+            var shard = new Mongo(shards[i].host)
+            
+            var partialColl = shard.getCollection(db.foo + "").find();
+            
+            while (partialColl.hasNext()) {
+                var obj = partialColl.next();
+                delete obj.y;
+                print(tojson(obj));
+            }
+        }
+        
+        jsTest.log( "End document distribution." )
+    }
+    
     // Verify that wbl weirdness isn't causing this
     assert.soon( function(){ var c = db.foo.find().itcount(); print( "Count is " + c ); return c == numdocs } )
+   
     assert( ! isBad )
     //assert.eq( numdocs, db.foo.find().itcount(), "Not all data was saved!" )
     

jstests/sharding/multi_mongos1.js

@@ -43,11 +43,11 @@ s1.adminCommand( { split : "test.foo" , middle : { num : 1 } } );
 s1.adminCommand( { split : "test.foo" , middle : { num : N } } );
 
 // s2 is now stale w.r.t boundaires around { num: 1 }
-res = s2.getDB( "admin" ).runCommand( { movechunk : "test.foo" , find : { num : 1 } , to : s1.getOther( s1.getServer( "test" ) ).name } );
+res = s2.getDB( "admin" ).runCommand( { movechunk : "test.foo" , find : { num : 1 } , to : s1.getOther( s1.getServer( "test" ) ).name, _waitForDelete : true } );
 assert.eq( 0 , res.ok , "a move with stale boundaries should not have succeeded" + tojson(res) ); 
 
 // s2 must have reloaded as a result of a failed move; retrying should work
-res = s2.getDB( "admin" ).runCommand( { movechunk : "test.foo" , find : { num : 1 } , to : s1.getOther( s1.getServer( "test" ) ).name } );
+res = s2.getDB( "admin" ).runCommand( { movechunk : "test.foo" , find : { num : 1 } , to : s1.getOther( s1.getServer( "test" ) ).name, _waitForDelete : true } );
 assert.eq( 1 , res.ok , "mongos did not reload after a failed migrate" + tojson(res) );
 
 // s1 is not stale about the boundaries of [MinKey->1) 

jstests/sharding/noUpdateButN1inAnotherCollection.js

@@ -0,0 +1,60 @@
+
+function debug( str ) {
+    print( "---\n" + str + "\n-----" );
+}
+
+var name = "badNonUpdate";
+debug("Starting sharded cluster test stuff");
+
+s = new ShardingTest( {name: name, shards : 2, mongos : 2, separateConfig : true, verbose:5, nopreallocj : true });
+
+var mongosA=s.s0;
+var mongosB=s.s1;
+
+ns = "test.coll";
+ns2 = "test.coll2";
+
+adminSA = mongosA.getDB( "admin" );
+adminSA.runCommand({ enableSharding : "test"});
+
+adminSA.runCommand( { moveprimary : "test", to : "shard0000" } );
+adminSA.runCommand( { moveprimary : "test2", to : "shard0001" } );
+
+adminSA.runCommand({ shardCollection : ns, key : { _id : 1 } });
+
+try {
+   s.stopBalancer();
+} catch (e) { 
+   print("coundn't stop balancer via command"); 
+}
+
+adminSA.settings.update({ _id: 'balancer' }, { $set: { stopped: true }});
+
+var db = mongosA.getDB( "test" );
+var coll = db.coll;
+var coll2 = db.coll2;
+
+numDocs = 10;
+for (var i = 1; i < numDocs; i++) {
+    coll.insert({_id:i, control:0});
+    coll2.insert({_id:i, control:0});
+}
+
+debug("Inserted docs, now split chunks");
+
+adminSA.runCommand( { split: ns, find : { _id : 3} });
+adminSA.runCommand( { movechunk: ns, find : { _id : 10}, to: "shard0001" });
+
+var command = 'db.coll.update({_id:9},{$set:{"a":"9"}},true);printjson(db.getLastErrorObj())';
+
+// without this first query through mongo, the second time doesn't "fail"
+debug("Try query first time");
+var GLE2=runMongoProgram( "mongo", "--quiet", "--port", "" + s._mongos[1].port, "--eval", command );
+
+mongosB.getDB("test").coll2.update({_id:0}, {$set:{"c":"333"}});
+var GLE3=mongosB.getDB("test").getLastErrorObj();
+assert.eq( 0, GLE3.n );
+
+
+s.stop();
+

jstests/sharding/no_empty_reset.js

@@ -29,7 +29,7 @@ var fullShard = st.getShard( coll, { _id : 1 } )
 var emptyShard = st.getShard( coll, { _id : -1 } )
 
 var admin = st.s.getDB( "admin" )
-printjson( admin.runCommand({ moveChunk : "" + coll, find : { _id : -1 }, to : fullShard.shardName }) )
+printjson( admin.runCommand({ moveChunk : "" + coll, find : { _id : -1 }, to : fullShard.shardName, _waitForDelete : true }) )
 
 jsTestLog( "Resetting shard version via first mongos..." )
 

jstests/sharding/read_pref.js

@@ -112,12 +112,14 @@ assert.eq( primaryNode.name, explain.server );
 assert.eq( 1, explain.n );
 
 // Kill all members except one
+var stoppedNodes = [];
 for ( var x = 0; x < NODES - 1; x++ ){
     replTest.stop( x );
+    stoppedNodes.push( replTest.nodes[x] );
 }
 
 // Wait for ReplicaSetMonitor to realize nodes are down
-ReplSetTest.awaitRSClientHosts( conn, replTest.nodes[0], { ok: false }, replTest.name );
+ReplSetTest.awaitRSClientHosts( conn, stoppedNodes, { ok: false }, replTest.name );
 
 // Wait for the last node to be in steady state -> secondary (not recovering)
 var lastNode = replTest.nodes[NODES - 1];

jstests/sharding/read_pref_multi_mongos_stale_config.js

@@ -0,0 +1,34 @@
+var st = new ShardingTest({ shards: { rs0: { quiet: '' }, rs1: { quiet: '' }}, mongos: 2 });
+
+var testDB1 = st.s0.getDB('test');
+var testDB2 = st.s1.getDB('test');
+
+// Trigger a query on mongos 1 so it will have a view of test.user as being unsharded.
+testDB1.user.findOne();
+
+testDB2.adminCommand({ enableSharding: 'test' });
+testDB2.adminCommand({ shardCollection: 'test.user', key: { x: 1 }});
+
+testDB2.adminCommand({ split: 'test.user', middle: { x: 100 }});
+
+var configDB2 = st.s1.getDB('config');
+var chunkToMove = configDB2.chunks.find().sort({ min: 1 }).next();
+var toShard = configDB2.shards.findOne({ _id: { $ne: chunkToMove.shard }})._id;
+testDB2.adminCommand({ moveChunk: 'test.user', to: toShard, find: { x: 50 }});
+
+for (var x = 0; x < 200; x++) {
+    testDB2.user.insert({ x: x });
+}
+
+testDB2.runCommand({ getLastError: 1 });
+
+var cursor = testDB1.user.find({ x: 30 }).readPref('primary');
+assert(cursor.hasNext());
+assert.eq(30, cursor.next().x);
+
+cursor = testDB1.user.find({ x: 130 }).readPref('primary');
+assert(cursor.hasNext());
+assert.eq(130, cursor.next().x);
+
+st.stop();
+

jstests/sharding/read_pref_rs_client.js

@@ -180,12 +180,7 @@ function noPriSecOkTest() {
         coll.find().readPref('primary').explain();
     });
 
-    // Needs to restart server, otherwise the js Mongo constructor
-    // would throw because it can't find the primary
-    replTest.start(0, {}, true);
-    replTest.awaitSecondaryNodes();
     replConn = new Mongo(replTest.getURL());
-    replTest.stop(0);
     coll = replConn.getDB('test').user;
     var dest = coll.find().readPref('primaryPreferred').explain().server;
     assert.eq(SEC_HOST, dest);

jstests/sharding/remove2.js

@@ -6,24 +6,36 @@ seedString = function(replTest) {
 };
 
 removeShard = function(st, replTest) {
+
+
+
     print( "Removing shard with name: " + replTest.name );
     res = st.admin.runCommand( { removeshard: replTest.name } )
     printjson(res);
     assert( res.ok , "failed to start draining shard" );
+
     checkRemoveShard = function() {
         res = st.admin.runCommand( { removeshard: replTest.name } );
         printjson(res);
         return res.ok && res.msg == 'removeshard completed successfully';
     }
     assert.soon( checkRemoveShard , "failed to remove shard" );
-    
+
     // Need to wait for migration to be over... only works for inline deletes
     checkNSLock = function() {
         printjson( st.s.getDB( "config" ).locks.find().toArray() )
-        return st.s.getDB( "config" ).locks.find({ _id : { $ne : "balancer" }, state : 2 })
-                   .count() == 0
+        return !st.isAnyBalanceInFlight();
     }
     assert.soon( checkNSLock, "migrations did not end?" )
+
+    sleep( 2000 );
+
+    var directdb = replTest.getPrimary().getDB( "admin" );
+    assert.soon( function(){
+           var res = directdb.currentOp( { desc: /^clean/ } );
+           print( "eliot: " + replTest.getPrimary() + "\t" + tojson(res) );
+           return res.inprog.length == 0;
+       }, "never clean", 60000, 1000 );
     
     replTest.getPrimary().getDB( coll.getDB().getName() ).dropDatabase();
     print( "Shard removed successfully" );
@@ -49,6 +61,11 @@ addShard = function(st, replTest) {
         return x < 2;
     } , "no balance happened", 60000 );
 
+    // NOTE: To avoid getMore problems stranding cursors b/c of our current conn
+    // pool behavior, this needs to be run.
+    // TODO: Remove once conn pooling behavior is better
+    printjson( coll.getMongo().getDB("admin").runCommand({ connPoolSync : true }) );
+
     try {
         assert.eq( 300, coll.find().itcount() );
     } catch (e) {
@@ -96,13 +113,14 @@ for( var i = 0; i < 300; i++ ){
     coll.insert( { i : i % 10, str : str } );
 }
 
+coll.getDB().getLastError();
+
+assert.eq( 300, coll.find().itcount() );
+
 assert.soon( function() {
     var x = st.chunkDiff( 'remove2' , "test" ); print( "chunk diff: " + x ); return x < 2;
 } , "no balance happened" );
 
-printjson(res);
-assert(res.ok);
-
 assert.eq( 300, coll.find().itcount() );
 
 st.admin.printShardingStatus();

jstests/sharding/repl_monitor_refresh.js

@@ -5,7 +5,7 @@
 
 var NODE_COUNT = 3;
 var st = new ShardingTest({ shards: { rs0: { nodes: NODE_COUNT, oplogSize: 10 }},
-    separateConfig: true });
+    separateConfig: true, config : 3 });
 var replTest = st.rs0;
 var mongos = st.s;
 

jstests/sharding/return_partial_shards_down.js

@@ -0,0 +1,96 @@
+//
+// Tests that zero results are correctly returned with returnPartial and shards down
+//
+
+var st = new ShardingTest({shards : 3,
+                           mongos : 1,
+                           other : {mongosOptions : {verbose : 2},
+                                    separateConfig : true}});
+
+var mongos = st.s;
+var config = mongos.getDB("config");
+var admin = mongos.getDB("admin");
+var shards = config.shards.find().toArray();
+
+for ( var i = 0; i < shards.length; i++) {
+    shards[i].conn = new Mongo(shards[i].host);
+}
+
+var collOneShard = mongos.getCollection("foo.collOneShard");
+var collAllShards = mongos.getCollection("foo.collAllShards");
+
+printjson(admin.runCommand({enableSharding : collOneShard.getDB() + ""}))
+printjson(admin.runCommand({movePrimary : collOneShard.getDB() + "",
+                            to : shards[0]._id}));
+
+printjson(admin.runCommand({shardCollection : collOneShard + "",
+                            key : {_id : 1}}));
+printjson(admin.runCommand({shardCollection : collAllShards + "",
+                            key : {_id : 1}}));
+
+// Split and move the "both shard" collection to both shards
+
+printjson(admin.runCommand({split : collAllShards + "",
+                            middle : {_id : 0}}));
+printjson(admin.runCommand({split : collAllShards + "",
+                            middle : {_id : 1000}}));
+printjson(admin.runCommand({moveChunk : collAllShards + "",
+                            find : {_id : 0},
+                            to : shards[1]._id}));
+printjson(admin.runCommand({moveChunk : collAllShards + "",
+                            find : {_id : 1000},
+                            to : shards[2]._id}));
+
+// Collections are now distributed correctly
+jsTest.log("Collections now distributed correctly.");
+st.printShardingStatus();
+
+var inserts = [{_id : -1},
+               {_id : 1},
+               {_id : 1000}];
+
+collOneShard.insert(inserts);
+collAllShards.insert(inserts);
+
+assert.eq(null, collOneShard.getDB().getLastError());
+
+var returnPartialFlag = 1 << 7;
+
+jsTest.log("All shards up!");
+
+assert.eq(3, collOneShard.find().itcount());
+assert.eq(3, collAllShards.find().itcount());
+
+assert.eq(3, collOneShard.find({}, {}, 0, 0, 0, returnPartialFlag).itcount());
+assert.eq(3, collAllShards.find({}, {}, 0, 0, 0, returnPartialFlag).itcount());
+
+jsTest.log("One shard down!")
+
+MongoRunner.stopMongod(st.shard2)
+
+jsTest.log("done.")
+
+assert.eq(3, collOneShard.find({}, {}, 0, 0, 0, returnPartialFlag).itcount());
+assert.eq(2, collAllShards.find({}, {}, 0, 0, 0, returnPartialFlag).itcount());
+
+jsTest.log("Two shards down!")
+
+MongoRunner.stopMongod(st.shard1)
+
+jsTest.log("done.")
+
+assert.eq(3, collOneShard.find({}, {}, 0, 0, 0, returnPartialFlag).itcount());
+assert.eq(1, collAllShards.find({}, {}, 0, 0, 0, returnPartialFlag).itcount());
+
+jsTest.log("All shards down!")
+
+MongoRunner.stopMongod(st.shard0)
+
+jsTest.log("done.")
+
+assert.eq(0, collOneShard.find({}, {}, 0, 0, 0, returnPartialFlag).itcount());
+assert.eq(0, collAllShards.find({}, {}, 0, 0, 0, returnPartialFlag).itcount());
+
+jsTest.log("DONE!");
+
+st.stop();

jstests/sharding/shard2.js

@@ -57,10 +57,10 @@ placeCheck( 2 );
 // NOTE: at this point we have 2 shard on 1 server
 
 // test move shard
-assert.throws( function(){ s.adminCommand( { movechunk : "test.foo" , find : { num : 1 } , to : primary.getMongo().name } ); } );
-assert.throws( function(){ s.adminCommand( { movechunk : "test.foo" , find : { num : 1 } , to : "adasd" } ) } );
+assert.throws( function(){ s.adminCommand( { movechunk : "test.foo" , find : { num : 1 } , to : primary.getMongo().name, _waitForDelete : true } ); } );
+assert.throws( function(){ s.adminCommand( { movechunk : "test.foo" , find : { num : 1 } , to : "adasd", _waitForDelete : true } ) } );
 
-s.adminCommand( { movechunk : "test.foo" , find : { num : 1 } , to : secondary.getMongo().name } );
+s.adminCommand( { movechunk : "test.foo" , find : { num : 1 } , to : secondary.getMongo().name, _waitForDelete : true } );
 assert.eq( 2 , secondary.foo.find().length() , "secondary should have 2 after move shard" );
 assert.eq( 1 , primary.foo.find().length() , "primary should only have 1 after move shard" );
 
@@ -221,10 +221,10 @@ assert.eq( 2 , s.onNumShards( "foo" ) , "on 2 shards" );
 
 secondary.foo.insert( { num : -3 } );
 
-s.adminCommand( { movechunk : "test.foo" , find : { num : -2 } , to : secondary.getMongo().name } );
+s.adminCommand( { movechunk : "test.foo" , find : { num : -2 } , to : secondary.getMongo().name, _waitForDelete : true } );
 assert.eq( 1 , s.onNumShards( "foo" ) , "on 1 shards" );
 
-s.adminCommand( { movechunk : "test.foo" , find : { num : -2 } , to : primary.getMongo().name } );
+s.adminCommand( { movechunk : "test.foo" , find : { num : -2 } , to : primary.getMongo().name, _waitForDelete : true } );
 assert.eq( 2 , s.onNumShards( "foo" ) , "on 2 shards again" );
 assert.eq( 3 , s.config.chunks.count() , "only 3 chunks" );
 

jstests/sharding/shard3.js

@@ -37,7 +37,7 @@ assert.eq( 0 , secondary.count() , "s1" )
 assert.eq( 1 , s.onNumShards( "foo" ) , "on 1 shards" );
 
 s.adminCommand( { split : "test.foo" , middle : { num : 2 } } );
-s.adminCommand( { movechunk : "test.foo" , find : { num : 3 } , to : s.getOther( s.getServer( "test" ) ).name } );
+s.adminCommand( { movechunk : "test.foo" , find : { num : 3 } , to : s.getOther( s.getServer( "test" ) ).name, _waitForDelete : true } );
 
 assert( primary.find().toArray().length > 0 , "blah 1" );
 assert( secondary.find().toArray().length > 0 , "blah 2" );
@@ -89,7 +89,7 @@ assert( a.findOne( { num : 1 } ) , "pre move 1" )
 s.printCollectionInfo( "test.foo" );
 myto = s.getOther( s.getServer( "test" ) ).name
 print( "counts before move: " + tojson( s.shardCounts( "foo" ) ) );
-s.adminCommand( { movechunk : "test.foo" , find : { num : 1 } , to : myto } )
+s.adminCommand( { movechunk : "test.foo" , find : { num : 1 } , to : myto, _waitForDelete : true } )
 print( "counts after move: " + tojson( s.shardCounts( "foo" ) ) );
 s.printCollectionInfo( "test.foo" );
 assert.eq( 1 , s.onNumShards( "foo" ) , "on 1 shard again" );
@@ -132,7 +132,7 @@ s.adminCommand( { shardcollection : "test.foo" , key : { num : 1 } } );
 a.save( { num : 2 } );
 a.save( { num : 3 } );
 s.adminCommand( { split : "test.foo" , middle : { num : 2 } } );
-s.adminCommand( { movechunk : "test.foo" , find : { num : 3 } , to : s.getOther( s.getServer( "test" ) ).name } );
+s.adminCommand( { movechunk : "test.foo" , find : { num : 3 } , to : s.getOther( s.getServer( "test" ) ).name, _waitForDelete : true } );
 s.printShardingStatus();
 
 s.printCollectionInfo( "test.foo" , "after dropDatabase setup" );
@@ -165,7 +165,7 @@ assert.eq( 3 , dba.foo.count() , "Ba" );
 assert.eq( 3 , dbb.foo.count() , "Bb" );
 
 s.adminCommand( { split : "test2.foo" , middle : { num : 2 } } );
-s.adminCommand( { movechunk : "test2.foo" , find : { num : 3 } , to : s.getOther( s.getServer( "test2" ) ).name } );
+s.adminCommand( { movechunk : "test2.foo" , find : { num : 3 } , to : s.getOther( s.getServer( "test2" ) ).name, _waitForDelete : true } );
 
 assert.eq( 2 , s.onNumShards( "foo" , "test2" ) , "B on 2 shards" );
 

jstests/sharding/shard4.js

@@ -1,6 +1,7 @@
 // shard4.js
 
 s = new ShardingTest( "shard4" , 2 , 50 , 2 );
+s.stopBalancer()
 
 s2 = s._mongos[1];
 
@@ -19,7 +20,7 @@ assert.eq( 7 , s.getDB( "test" ).foo.find().toArray().length , "normal A" );
 assert.eq( 7 , s2.getDB( "test" ).foo.find().toArray().length , "other A" );
 
 s.adminCommand( { split : "test.foo" , middle : { num : 4 } } );
-s.adminCommand( { movechunk : "test.foo" , find : { num : 3 } , to : s.getOther( s.getServer( "test" ) ).name } );
+s.adminCommand( { movechunk : "test.foo" , find : { num : 3 } , to : s.getOther( s.getServer( "test" ) ).name, _waitForDelete : true } );
 
 assert( s._connections[0].getDB( "test" ).foo.find().toArray().length > 0 , "blah 1" );
 assert( s._connections[1].getDB( "test" ).foo.find().toArray().length > 0 , "blah 2" );

jstests/sharding/shard5.js

@@ -21,7 +21,7 @@ assert.eq( 7 , s.getDB( "test" ).foo.find().toArray().length , "normal A" );
 assert.eq( 7 , s2.getDB( "test" ).foo.find().toArray().length , "other A" );
 
 s.adminCommand( { split : "test.foo" , middle : { num : 4 } } );
-s.adminCommand( { movechunk : "test.foo" , find : { num : 3 } , to : s.getOther( s.getServer( "test" ) ).name } );
+s.adminCommand( { movechunk : "test.foo" , find : { num : 3 } , to : s.getOther( s.getServer( "test" ) ).name, _waitForDelete : true } );
 
 assert( s._connections[0].getDB( "test" ).foo.find().toArray().length > 0 , "blah 1" );
 assert( s._connections[1].getDB( "test" ).foo.find().toArray().length > 0 , "blah 2" );

jstests/sharding/shard6.js

@@ -2,7 +2,7 @@
 
 summary = "";
 
-s = new ShardingTest( "shard6" , 2 , 0 , 2 );
+s = new ShardingTest( "shard6" , 2 , 0 , 2, { separateConfig : true } );
 
 s.config.settings.update( { _id: "balancer" }, { $set : { stopped: true } } , true );
 

jstests/sharding/shard_insert_getlasterror_w2.js

@@ -87,4 +87,4 @@ function go() {
 }
 
 //Uncomment below to execute
-go()
+//go()

jstests/sharding/wbl_not_cleared.js

@@ -0,0 +1,63 @@
+/**
+ * Tests whether the WBL gets reset without subsequent events.
+ */
+
+var st = new ShardingTest({ shards : 2, mongos : 2, other : { mongosOptions : { verbose : 5 } } });
+
+st.stopBalancer();
+
+var mongos = st.s0;
+var staleMongos = st.s1;
+var admin = mongos.getDB("admin");
+var config = mongos.getDB("config");
+var shards = config.shards.find().toArray();
+var coll = mongos.getCollection("foo.bar");
+
+jsTest.log("Sharding collection...");
+
+printjson(admin.runCommand({ enableSharding : coll.getDB() + "" }));
+printjson(admin.runCommand({ movePrimary : coll.getDB() + "", to : shards[0]._id }));
+printjson(admin.runCommand({ shardCollection : coll + "", key : { _id : 1 } }));
+printjson(admin.runCommand({ split : coll + "", middle : { _id : 0 } }));
+printjson(admin.runCommand({ moveChunk : coll + "", find : { _id : 0 }, to : shards[1]._id }));
+
+jsTest.log("Collection now sharded...");
+st.printShardingStatus();
+
+jsTest.log("Making mongos stale...");
+
+coll.insert({ _id : 0 });
+coll.getDB().getLastErrorObj();
+
+// Make sure the stale mongos knows about the collection at the original version
+assert.neq(null, staleMongos.getCollection(coll + "").findOne());
+
+printjson(admin.runCommand({ moveChunk : coll + "", find : { _id : 0 }, to : shards[0]._id, _waitForDelete : true }));
+printjson(admin.runCommand({ moveChunk : coll + "", find : { _id : 0 }, to : shards[1]._id, _waitForDelete : true }));
+
+jsTest.log("Running a stale insert...");
+
+staleMongos.getCollection(coll + "").insert({ _id : 0, dup : "key" });
+
+jsTest.log("Getting initial GLE result...");
+
+printjson(staleMongos.getDB(coll.getDB() + "").getLastErrorObj());
+printjson(staleMongos.getDB(coll.getDB() + "").getLastErrorObj());
+st.printShardingStatus();
+
+jsTest.log("Performing insert op on the same shard...");
+
+staleMongos.getCollection(coll + "").insert({ _id : 1, key : "isOk" })
+
+jsTest.log("Getting GLE result...");
+
+printjson(staleMongos.getDB(coll.getDB() + "").getLastErrorObj());
+assert.eq(null, staleMongos.getDB(coll.getDB() + "").getLastError());
+
+jsTest.log("DONE!");
+
+st.stop();
+
+
+
+

jstests/sharding/writeback_bulk_insert.js

@@ -0,0 +1,116 @@
+//
+// Tests whether a writeback error during bulk insert hangs GLE
+//
+
+jsTest.log("Starting sharded cluster...")
+
+var st = new ShardingTest({shards : 1,
+                           mongos : 3,
+                           verbose : 2,
+                           other : {separateConfig : true,
+                                    mongosOptions : {noAutoSplit : ""}}})
+
+st.stopBalancer()
+
+var mongosA = st.s0
+var mongosB = st.s1
+var mongosC = st.s2
+
+jsTest.log("Adding new collection...")
+
+var collA = mongosA.getCollection(jsTestName() + ".coll")
+collA.insert({hello : "world"})
+assert.eq(null, collA.getDB().getLastError())
+
+var collB = mongosB.getCollection("" + collA)
+collB.insert({hello : "world"})
+assert.eq(null, collB.getDB().getLastError())
+
+var collC = mongosC.getCollection("" + collA)
+collC.insert({hello : "world"})
+assert.eq(null, collC.getDB().getLastError())
+
+jsTest.log("Enabling sharding...")
+
+printjson(mongosA.getDB("admin").runCommand({enableSharding : collA.getDB()
+                                                              + ""}))
+printjson(mongosA.getDB("admin").runCommand({shardCollection : collA + "",
+                                             key : {_id : 1}}))
+
+// MongoD doesn't know about the config shard version *until* MongoS tells it
+collA.findOne()
+
+jsTest.log("Preparing bulk insert...")
+
+var data1MB = "x"
+while (data1MB.length < 1024 * 1024)
+    data1MB += data1MB;
+
+var data7MB = ""
+// Data now at 7MB
+for ( var i = 0; i < 7; i++)
+    data7MB += data1MB;
+
+print("7MB object size is : " + Object.bsonsize({_id : 0,
+                                                 d : data7MB}))
+
+var dataCloseTo8MB = data7MB;
+// WARNING - MAGIC NUMBERS HERE
+// The idea is to exceed the 16MB limit by just enough so that the message gets
+// passed in the
+// shell, but adding additional writeback information fails.
+for ( var i = 0; i < 1031 * 1024 + 862; i++) {
+    dataCloseTo8MB += "x"
+}
+
+print("Object size is: " + Object.bsonsize([{_id : 0,
+                                             d : dataCloseTo8MB},
+                                            {_id : 1,
+                                             d : dataCloseTo8MB}]))
+
+jsTest.log("Trigger wbl for mongosB...")
+
+collB.insert([{_id : 0,
+               d : dataCloseTo8MB},
+              {_id : 1,
+               d : dataCloseTo8MB}])
+
+// Will hang if overflow is not detected correctly
+jsTest.log("Waiting for GLE...")
+
+assert.neq(null, collB.getDB().getLastError())
+
+print("GLE correctly returned error...")
+
+assert.eq(3, collA.find().itcount())
+assert.eq(3, collB.find().itcount())
+
+var data8MB = "";
+for ( var i = 0; i < 8; i++) {
+    data8MB += data1MB;
+}
+
+print("Object size is: " + Object.bsonsize([{_id : 0,
+                                             d : data8MB},
+                                            {_id : 1,
+                                             d : data8MB}]))
+
+jsTest.log("Trigger wbl for mongosC...")
+
+collC.insert([{_id : 0,
+               d : data8MB},
+              {_id : 1,
+               d : data8MB}])
+
+// Should succeed since our insert size is 16MB (plus very small overhead)
+jsTest.log("Waiting for GLE...")
+
+assert.eq(null, collC.getDB().getLastError())
+
+print("GLE Successful...")
+
+assert.eq(5, collA.find().itcount())
+assert.eq(5, collB.find().itcount())
+assert.eq(5, collC.find().itcount())
+
+st.stop()

jstests/sharding/writeback_server7958.js

@@ -0,0 +1,94 @@
+jsTest.log("Starting sharded cluster for wrong duplicate error setup");
+
+s = new ShardingTest( name="writeback_server7958", shards = 2, verbose=0, mongos = 4 );
+
+var mongosA=s.s0;
+var mongosB=s.s1;
+var mongosC=s.s2;
+var mongosD=s.s3;
+
+ns1 = "test.trans";
+ns2 = "test.node";
+
+adminSA = mongosA.getDB( "admin" );
+adminSB = mongosB.getDB( "admin" );
+adminSD = mongosD.getDB( "admin" );
+adminSA.runCommand({ enableSharding : "test"});
+adminSA.runCommand({ shardCollection : ns1, key : { owner : 1 }, unique: true });
+//adminSA.runCommand({ shardCollection : ns1, key : { owner : 1 } });
+
+try {
+   s.stopBalancer();
+} catch (e) {
+   print("coundn't stop balancer via command");
+}
+
+adminSA.settings.update({ _id: 'balancer' }, { $set: { stopped: true }});
+
+var db = mongosA.getDB( "test" );
+var dbB = mongosB.getDB( "test" );
+var dbC = mongosC.getDB( "test" );
+var dbD = mongosD.getDB( "test" );
+var trans = db.trans;
+var node = db.node;
+var transB = dbB.trans;
+var nodeB = dbB.node;
+var transC = dbC.trans;
+var nodeC = dbC.node;
+var transD = dbD.trans;
+var nodeD = dbD.node;
+
+var primary = s.getServerName("test");
+var shard1 = s._shardNames[0];
+var shard2 = s._shardNames[1];
+if (primary == shard1) {
+    other = shard2;
+} else {
+    other = shard1;
+}
+
+
+trans.insert({"owner":NumberLong("1234567890"),"tid":"2c4ba280-450a-11e2-bcfd-0800200c9a66"});
+db.runCommand({getLastError:1, j:1});
+
+node.insert({"owner":NumberLong("1234567890"),"parent":NumberLong("0"),_id:NumberLong("1234567890"), "counts":0});
+db.runCommand({getLastError:1, j:1});
+for (var i=0; i<1000; i++) {
+   trans.insert({"owner":NumberLong(i),"tid":"2c4ba280-450a-11e2-bcfd-0800200c9a66"});
+   node.insert({"owner":NumberLong(i),"parent":NumberLong(i+1000),_id:NumberLong(i+1234567890), "counts":0});
+}
+
+transB.insert({"owner":NumberLong("1234567890"),"tid":"2c4ba280-450a-11e2-bcfd-0800200c9a66"});
+var r1=dbB.runCommand( { getLastError: 1, w: 1 } );
+assert( r1.n == 0 && r1.err.length > 0 && r1.hasOwnProperty("code"), tojson( r1 ) );
+
+jsTest.log("Inserted dup (failed), now split chunks and move data");
+
+adminSD.runCommand( { split: ns1, middle : { owner : 100} });
+adminSD.runCommand( { movechunk: ns1, find : { owner : 105}, to: other});
+
+jsTest.log("Kicking off dup inserts and updates");
+
+errors=[];
+i=0;
+trans.insert({"owner":NumberLong("1234567890"),"tid":"2c4ba280-450a-11e2-bcfd-0800200c9a66"});
+var r1=db.runCommand( { getLastError: 1, w: 1 } );
+assert( r1.n == 0 && r1.err.length > 0 && r1.hasOwnProperty("code"), tojson( r1 ) );
+transB.insert({"owner":NumberLong("1234567890"),"tid":"2c4ba280-450a-11e2-bcfd-0800200c9a66"});
+var rB1=dbB.runCommand( { getLastError: 1, w: 1 } );
+assert( rB1.n == 0 && rB1.err.length > 0 && rB1.hasOwnProperty("code"), tojson( r1 ) );
+
+nodeB.update({"owner":NumberLong("1234567890"),"parent":NumberLong("0"),_id:NumberLong("1234567890")},{"$inc":{"counts":1}});
+var resultB = dbB.runCommand( { getLastError: 1, w: 1 } )
+node.update({"owner":NumberLong("1234567890"),"parent":NumberLong("0"),_id:NumberLong("1234567890")},{"$inc":{"counts":1}});
+var result = db.runCommand( { getLastError: 1, w: 1 } )
+
+assert.eq( 2, node.findOne().counts );
+
+printjson( result )
+printjson( resultB )
+
+assert( result.n==1 && result.updatedExisting==true && result.err == null, "update succeeded on collection node on mongos A but GLE was\nn=" + result.n + ",\nupdatedExisting=" + result.updatedExisting + ",\nerr=" + result.err);
+assert( resultB.n==1 && resultB.updatedExisting==true && resultB.err == null, "update succeeded on collection node on mongos B but GLE was\nn=" + resultB.n + ",\nupdatedExisting=" + resultB.updatedExisting + ",\nerr=" + resultB.err);
+
+s.stop();

jstests/slowNightly/autosplit_heuristics.js

@@ -0,0 +1,72 @@
+//
+// Tests autosplitting heuristics, and that the heuristic counting of chunk sizes
+// works as expected even after splitting.
+//
+
+var st = new ShardingTest({ shards : 1, 
+                            mongos : 1, 
+                            other : { mongosOptions : { chunkSize : 1, verbose : 1 }, 
+                            separateConfig : true } });
+
+// The balancer may interfere unpredictably with the chunk moves/splits depending on timing.
+st.stopBalancer();
+
+var mongos = st.s0;
+var config = mongos.getDB("config");
+var admin = mongos.getDB("admin");
+var coll = mongos.getCollection("foo.hashBar");
+
+printjson(admin.runCommand({ enableSharding : coll.getDB() + "" }));
+printjson(admin.runCommand({ shardCollection : coll + "", key : { _id : 1 } }));
+
+var numChunks = 10;
+
+// Split off the low and high chunks, to get non-special-case behavior
+printjson(admin.runCommand({ split : coll + "", middle : { _id : 0 } }));
+printjson(admin.runCommand({ split : coll + "", middle : { _id : numChunks } }));
+
+// Split all the other chunks
+for (var i = 1; i < numChunks; i++) {
+    printjson(admin.runCommand({ split : coll + "", middle : { _id : i } }));
+}
+
+jsTest.log("Setup collection...");
+st.printShardingStatus(true);
+
+var approxSize = Object.bsonsize({ _id : 0.0 });
+
+jsTest.log("Starting inserts of approx size: " + approxSize + "...");
+
+var chunkSizeBytes = 1024 * 1024;
+
+// We insert slightly more than the max number of docs per chunk, to test
+// if resetting the chunk size happens during reloads.  If the size is 
+// reset, we'd expect to split less, since the first split would then
+// disable further splits (statistically, since the decision is randomized).
+var insertsForSplit = Math.ceil((chunkSizeBytes * 1.25) / approxSize);
+var totalInserts = insertsForSplit * numChunks;
+
+printjson({ chunkSizeBytes : chunkSizeBytes,
+            insertsForSplit : insertsForSplit,
+            totalInserts : totalInserts });
+
+// Insert enough docs to trigger splits into all chunks
+for (var i = 0; i < totalInserts; i++) {
+    coll.insert({ _id : i % numChunks + (i / totalInserts) });
+}
+
+assert.eq(null, coll.getDB().getLastError());
+
+jsTest.log("Inserts completed...");
+
+st.printShardingStatus(true);
+printjson(coll.stats());
+
+// Check that all chunks (except the two extreme chunks)
+// have been split at least once.
+assert.gte(config.chunks.count(), numChunks * 2 + 2);
+
+jsTest.log("DONE!");
+
+st.stop();
+

jstests/slowNightly/balance_tags1.js

@@ -1,5 +1,5 @@
 
-s = new ShardingTest( "balance_tags1" , 3 , 1 , 1 , { chunksize : 1 , nopreallocj : true } )
+s = new ShardingTest( "balance_tags1" , 3 , 1 , 1 , { sync:true, chunksize : 1 , nopreallocj : true } )
 s.config.settings.update( { _id: "balancer" }, { $set : { stopped: false, _nosleep: true } } , true );
 
 db = s.getDB( "test" );
@@ -11,9 +11,13 @@ db.getLastError();
 s.adminCommand( { enablesharding : "test" } )
 s.adminCommand( { shardcollection : "test.foo" , key : { _id : 1 } } );
 
+s.stopBalancer();
+
 for ( i=0; i<20; i++ )
     s.adminCommand( { split : "test.foo" , middle : { _id : i } } );
 
+s.startBalancer();
+
 sh.status( true )
 assert.soon( function() {
     counts = s.chunkCounts( "foo" );
@@ -42,6 +46,7 @@ assert.soon( function() {
     return counts["shard0002"] == 0;
 } , "balance 2 didn't happen" , 1000 * 60 * 10 , 1000 )
 
+printjson(sh.status());
 
 s.stop();
 

jstests/slowNightly/no_balance_collection.js

@@ -60,6 +60,10 @@ jsTest.log( "Chunks for " + collB + " are balanced." )
 
 // Re-disable balancing for collB
 sh.disableBalancing( collB )
+// Wait for the balancer to fully finish the last migration and write the changelog
+// MUST set db var here, ugly but necessary
+db = st.s0.getDB("config")
+sh.waitForBalancer(true)
 
 // Make sure auto-migrates on insert don't move chunks
 var lastMigration = sh._lastMigration( collB )

jstests/slowNightly/replReads.js

@@ -119,4 +119,4 @@ function testReadLoadBalancing(numReplicas) {
 //}
 
 // Is there a way that this can be run multiple times with different values?
-testReadLoadBalancing(3)
+//testReadLoadBalancing(3)

jstests/slowNightly/server7428.js

@@ -0,0 +1,16 @@
+// Regression test for SERVER-7428.
+//
+// Verify that the copyDatabase command works appropriately when the
+// target mongo instance has authentication enabled.
+
+// Setup fromDb with no auth
+var fromDb = MongoRunner.runMongod({ port: 29000 });
+ 
+// Setup toDb with auth
+var toDb = MongoRunner.runMongod({auth : "", port : 31001});
+var admin = toDb.getDB("admin");
+admin.addUser("foo","bar");
+admin.auth("foo","bar");
+  
+admin.copyDatabase('test', 'test', fromDb.host)
+

jstests/slowNightly/sharding_balance4.js

@@ -3,6 +3,7 @@
 // check that doing updates done during a migrate all go to the right place
 
 s = new ShardingTest( "slow_sharding_balance4" , 2 , 1 , 1 , { chunksize : 1 } )
+s.stopBalancer();
 
 s.adminCommand( { enablesharding : "test" } );
 s.adminCommand( { shardcollection : "test.foo" , key : { _id : 1 } } );
@@ -22,19 +23,37 @@ num = 0;
 
 counts = {}
 
-function doUpdate( includeString ){
+//
+// TODO: Rewrite to make much clearer.
+//
+// The core behavior of this test is to add a bunch of documents to a sharded collection, then
+// incrementally update each document and make sure the counts in the document match our update
+// counts while balancing occurs (doUpdate()).  Every once in a while we also check (check())
+// our counts via a query.
+//
+// If during a chunk migration an update is missed, we trigger an assertion and fail.
+//
+
+
+function doUpdate( includeString, optionalId ){
     var up = { $inc : { x : 1 } }
     if ( includeString )
         up["$set"] = { s : bigString };
-    var myid = Random.randInt( N )
+    var myid = optionalId == undefined ? Random.randInt( N ) : optionalId
     db.foo.update( { _id : myid } , up , true );
 
     counts[myid] = ( counts[myid] ? counts[myid] : 0 ) + 1;
     return myid;
 }
 
-for ( i=0; i<N*10; i++ ){
-    doUpdate( true )
+// Initially update all documents from 1 to N, otherwise later checks can fail because no document
+// previously existed
+for ( i = 0; i < N; i++ ){
+    doUpdate( true, i )
+}
+
+for ( i=0; i<N*9; i++ ){
+    doUpdate( false )
 }
 db.getLastError();
 
@@ -63,7 +82,11 @@ function check( msg , dontAssert ){
             print( "not asserting for key failure: " + x + " want: " + e + " got: " + tojson(z) )
             return false;
         }
-
+        
+        s.s.getDB("admin").runCommand({ setParameter : 1, logLevel : 2 })
+        
+        printjson( db.foo.findOne( { _id : parseInt( x ) } ) )
+        
         // we will assert past this point but wait a bit to see if it is because the missing update
         // was being held in the writeback roundtrip
         sleep( 10000 );
@@ -84,6 +107,9 @@ function check( msg , dontAssert ){
 }
 
 function diff1(){
+    
+    jsTest.log("Running diff1...")
+    
     var myid = doUpdate( false )
     var le = db.getLastErrorCmd();
 
@@ -109,14 +135,11 @@ function diff1(){
     return Math.max( x.shard0000 , x.shard0001 ) - Math.min( x.shard0000 , x.shard0001 );
 }
 
-function sum(){
-    var x = s.chunkCounts( "foo" )
-    return x.shard0000 + x.shard0001;
-}
-
 assert.lt( 20 , diff1() ,"initial load" );
 print( diff1() )
 
+s.startBalancer();
+
 assert.soon( function(){
     
     var d = diff1();

jstests/slowNightly/sharding_migrate_cursor1.js

@@ -57,6 +57,11 @@ print( "cursor should be gone" )
 
 join();
 
+assert.soon( function(){
+    print( "Waiting for migrate cleanup to complete..." );
+    return t.count() == t.find().itcount();
+})
+
 //assert.soon( function(){ return numDocs == t.count(); } , "at end 1" )
 sleep( 5000 )
 assert.eq( numDocs , t.count() , "at end 2" )

jstests/slowNightly/sharding_migrate_large_docs.js

@@ -0,0 +1,63 @@
+//
+// Tests migration behavior of large documents
+//
+
+var st = new ShardingTest({ shards : 2, mongos : 1,
+                            other : { separateConfig : true,
+                                      mongosOptions : { noAutoSplit : "" },
+                                      shardOptions : { /* binVersion : "latest" */ } } });
+st.stopBalancer()
+
+var mongos = st.s0;
+var coll = mongos.getCollection( "foo.bar" );
+var admin = mongos.getDB( "admin" );
+var shards = mongos.getCollection( "config.shards" ).find().toArray();
+var shardAdmin = st.shard0.getDB( "admin" );
+
+assert( admin.runCommand({ enableSharding : coll.getDB() + "" }).ok );
+printjson( admin.runCommand({ movePrimary : coll.getDB() + "", to : shards[0]._id }) );
+assert( admin.runCommand({ shardCollection : coll + "", key : { _id : 1 } }).ok );
+assert( admin.runCommand({ split : coll + "", middle : { _id : 0 } }).ok );
+
+jsTest.log( "Preparing large insert..." );
+
+var data1MB = "x"
+while ( data1MB.length < 1024 * 1024 )
+    data1MB += data1MB;
+
+var data15MB = "";
+for ( var i = 0; i < 15; i++ ) data15MB += data1MB;
+
+var data15PlusMB = data15MB;
+for ( var i = 0; i < 1023 * 1024; i++ ) data15PlusMB += "x";
+
+print("~15MB object size is : " + Object.bsonsize({ _id : 0, d : data15PlusMB }));
+
+jsTest.log( "Inserting docs of large and small sizes..." );
+
+// Two large docs next to each other
+coll.insert({ _id : -2, d : data15PlusMB });
+coll.insert({ _id : -1, d : data15PlusMB });
+
+// Docs of assorted sizes
+coll.insert({ _id : 0, d : "x" });
+coll.insert({ _id : 1, d : data15PlusMB });
+coll.insert({ _id : 2, d : "x" });
+coll.insert({ _id : 3, d : data15MB });
+coll.insert({ _id : 4, d : "x" });
+coll.insert({ _id : 5, d : data1MB });
+coll.insert({ _id : 6, d : "x" });
+
+assert.eq( null, coll.getDB().getLastError() );
+assert.eq( 9, coll.find().itcount() );
+
+jsTest.log( "Starting migration..." );
+
+assert( admin.runCommand({ moveChunk : coll + "", find : { _id : 0 }, to : shards[1]._id }).ok );
+assert( admin.runCommand({ moveChunk : coll + "", find : { _id : -1 }, to : shards[1]._id }).ok );
+
+assert.eq( 9, coll.find().itcount() );
+
+jsTest.log( "DONE!" );
+
+st.stop();

jstests/slowNightly/sharding_rs2.js

@@ -125,7 +125,7 @@ s.adminCommand( { split : "test.foo" , middle : { x : 50 } } )
 db.printShardingStatus()
 
 other = s.config.shards.findOne( { _id : { $ne : serverName } } );
-s.adminCommand( { moveChunk : "test.foo" , find : { x : 10 } , to : other._id } )
+s.adminCommand( { moveChunk : "test.foo" , find : { x : 10 } , to : other._id, _waitForDelete : true } )
 assert.eq( 100 , t.count() , "C3" )
 
 assert.eq( 50 , rs.test.getMaster().getDB( "test" ).foo.count() , "C4" )

jstests/slowWeekly/minvalid2.js

@@ -0,0 +1,70 @@
+/**
+ * This checks rollback, which shouldn't happen unless we have reached minvalid.
+ *  1. make 3-member set w/arb (2)
+ *  2. shut down 1
+ *  3. do writes to 0
+ *  4. modify 0's minvalid
+ *  5. shut down 0
+ *  6. start up 1
+ *  7. writes on 1
+ *  8. start up 0
+ *  9. check 0 does not rollback
+ */
+
+print("1. make 3-member set w/arb (2)");
+var name = "minvalid"
+var replTest = new ReplSetTest({name: name, nodes: 3, oplogSize:1});
+var host = getHostName();
+
+var nodes = replTest.startSet();
+replTest.initiate({_id : name, members : [
+    {_id : 0, host : host+":"+replTest.ports[0]},
+    {_id : 1, host : host+":"+replTest.ports[1]},
+    {_id : 2, host : host+":"+replTest.ports[2], arbiterOnly : true}
+]});
+var master = replTest.getMaster();
+var mdb = master.getDB("foo");
+
+mdb.foo.save({a: 1000});
+replTest.awaitReplication();
+
+print("2: shut down 1");
+replTest.stop(1);
+
+print("3: do writes to 0");
+mdb.foo.save({a: 1001});
+
+print("4: modify 0's minvalid");
+var local = master.getDB("local");
+var lastOp = local.oplog.rs.find().sort({$natural:-1}).limit(1).next();
+printjson(lastOp);
+
+local.replset.minvalid.insert({ts:new Timestamp(lastOp.ts.t, lastOp.ts.i+1),
+                               h:new NumberLong("1234567890")});
+printjson(local.replset.minvalid.findOne());
+
+print("5: shut down 0");
+replTest.stop(0);
+
+print("6: start up 1");
+replTest.restart(1);
+
+print("7: writes on 1")
+master = replTest.getMaster();
+mdb1 = master.getDB("foo");
+mdb1.foo.save({a:1002});
+
+print("8: start up 0");
+replTest.restart(0);
+
+print("9: check 0 does not rollback");
+assert.soon(function(){
+    var status = master.adminCommand({replSetGetStatus:1});
+    var stateStr = status.members[0].stateStr;
+    assert(stateStr != "ROLLBACK" &&
+           stateStr != "SECONDARY" &&
+           stateStr != "PRIMARY", tojson(status));
+    return stateStr == "FATAL";
+});
+
+replTest.stopSet(15);

jstests/splitvector.js

@@ -28,6 +28,22 @@ assertChunkSizes = function ( splitVec , numDocs , maxChunkSize , msg ){
     }
 }
 
+// Takes two documents and asserts that both contain exactly the same set of field names.
+// This is useful for checking that splitPoints have the same format as the original key pattern,
+// even when sharding on a prefix key.
+// Not very efficient, so only call when # of field names is small
+var assertFieldNamesMatch = function( splitPoint , keyPattern ){
+    for ( var p in splitPoint ) {
+        if( splitPoint.hasOwnProperty( p ) ) {
+            assert( keyPattern.hasOwnProperty( p ) , "property " + p + " not in keyPattern" );
+        }
+    }
+    for ( var p in keyPattern ) {
+        if( keyPattern.hasOwnProperty( p ) ){
+            assert( splitPoint.hasOwnProperty( p ) , "property " + p + " not in splitPoint" );
+        }
+    }
+}
 
 // -------------------------
 //  TESTS START HERE
@@ -84,6 +100,9 @@ var case4 = function() {
     assert.eq( true , res.ok , "4b" );
     assert.close( numDocs*docSize / ((1<<20) * factor), res.splitKeys.length , "num split keys" , -1 );
     assertChunkSizes( res.splitKeys , numDocs, (1<<20) * factor , "4d" );
+    for( i=0; i < res.splitKeys.length; i++ ){
+        assertFieldNamesMatch( res.splitKeys[i] , {x : 1} );
+    }
 }
 case4();
 
@@ -104,6 +123,9 @@ var case5 = function() {
 
     assert.eq( true , res.ok , "5a" );
     assert.eq( 1 , res.splitKeys.length , "5b" );
+    for( i=0; i < res.splitKeys.length; i++ ){
+        assertFieldNamesMatch( res.splitKeys[i] , {x : 1} );
+    }
 }
 case5();
 
@@ -124,6 +146,9 @@ var case6 = function() {
 
     assert.eq( true , res.ok , "6a" );
     assert.eq( 19 , res.splitKeys.length , "6b" );
+    for( i=0; i < res.splitKeys.length; i++ ){
+        assertFieldNamesMatch( res.splitKeys[i] , {x : 1} );
+    }
 }
 case6();
 
@@ -149,6 +174,9 @@ var case7 = function() {
 
     assert.eq( true , res.ok , "7a" );
     assert.eq( 2 , res.splitKeys[0].x, "7b");
+    for( i=0; i < res.splitKeys.length; i++ ){
+        assertFieldNamesMatch( res.splitKeys[i] , {x : 1} );
+    }
 }
 case7();
 
@@ -180,6 +208,9 @@ var case8 = function() {
     assert.eq( 2 , res.splitKeys.length , "8b" );
     assert.eq( 2 , res.splitKeys[0].x , "8c" );
     assert.eq( 3 , res.splitKeys[1].x , "8d" );
+    for( i=0; i < res.splitKeys.length; i++ ){
+        assertFieldNamesMatch( res.splitKeys[i] , {x : 1} );
+    }
 }
 case8();
 
@@ -211,6 +242,9 @@ var case9 = function() {
         assert.eq( true , res.ok , "9a: " + tojson(res) );
         assert.eq( 1 , res.splitKeys.length , "9b: " + tojson(res) );
         assert.eq( 2 , res.splitKeys[0].x , "9c: " + tojson(res) );
+        for( i=0; i < res.splitKeys.length; i++ ){
+            assertFieldNamesMatch( res.splitKeys[i] , {x : 1} );
+        }
     }
 }
 case9();

jstests/tool/dumprestore_auth.js

@@ -0,0 +1,28 @@
+// dumprestore_auth.js
+
+t = new ToolTest("dumprestore_auth", { auth : "" });
+
+c = t.startDB("foo");
+
+adminDB = c.getDB().getSiblingDB('admin');
+adminDB.addUser('admin', 'password');
+adminDB.auth('admin','password');
+
+assert.eq(0 , c.count() , "setup1");
+c.save({ a : 22 });
+assert.eq(1 , c.count() , "setup2");
+
+t.runTool("dump" , "--out" , t.ext, "--username", "admin", "--password", "password");
+
+c.drop();
+assert.eq(0 , c.count() , "after drop");
+
+t.runTool("restore" , "--dir" , t.ext); // Should fail
+assert.eq(0 , c.count() , "after restore without auth");
+
+t.runTool("restore" , "--dir" , t.ext, "--username", "admin", "--password", "password");
+assert.soon("c.findOne()" , "no data after sleep");
+assert.eq(1 , c.count() , "after restore 2");
+assert.eq(22 , c.findOne().a , "after restore 2");
+
+t.stop();

jstests/tool/exportimport1.js

@@ -19,7 +19,11 @@ assert.eq( 1 , c.count() , "after restore 2" );
 var doc = c.findOne();
 assert.eq( 22 , doc.a , "after restore 2" );
 for (var i=0; i<arr.length; i++) {
-    assert.eq( arr[i], doc.b[i] , "after restore array: "+i );
+    if (typeof arr[i] == 'undefined') {
+        assert.eq( { "$undefined" : true }, doc.b[i] , "after restore array: "+i );
+    } else {
+        assert.eq( arr[i], doc.b[i] , "after restore array: "+i );
+    }
 }
 
 // now with --jsonArray
@@ -49,7 +53,11 @@ assert.soon( "c.findOne()" , "no data after sleep" );
 assert.eq( 1 , c.count() , "after restore 2" );
 var doc = c.findOne();
 for (var i=0; i<arr.length; i++) {
-    assert.eq( arr[i], doc.a[i] , "after restore array: "+i );
+    if (typeof arr[i] == 'undefined') {
+        assert.eq( { "$undefined" : true }, doc.a[i] , "after restore array: "+i );
+    } else {
+        assert.eq( arr[i], doc.a[i] , "after restore array: "+i );
+    }
 }
 
 

jstests/tool/restorewithauth.js

@@ -23,22 +23,25 @@ var conn = startMongod( "--port", port, "--dbpath", "/data/db/" + baseName, "--n
 var foo = conn.getDB( "foo" );
 for( var i = 0; i < 4; i++ ) {
     foo["bar"].save( { "x": i } );
+    foo["baz"].save({"x": i});
 }
 
 // make sure the collection exists
 assert.eq( foo.system.namespaces.count({name: "foo.bar"}), 1 )
 
 //make sure it has no index except _id
-assert.eq( foo.system.indexes.count(), 1 )
+assert.eq(foo.system.indexes.count(), 2);
+
+foo.bar.createIndex({x:1});
+assert.eq(foo.system.indexes.count(), 3);
 
 // get data dump
 var dumpdir = "/data/db/restorewithauth-dump1/";
 resetDbpath( dumpdir );
-x = runMongoProgram( "mongodump", "--db", "foo", "-h", "127.0.0.1:"+port, "--collection", "bar", 
-                     "--out", dumpdir );
+x = runMongoProgram("mongodump", "--db", "foo", "-h", "127.0.0.1:"+port, "--out", dumpdir);
 
-// now drop the collection
-foo.bar.drop();
+// now drop the db
+foo.dropDatabase();
 
 // stop mongod
 stopMongod( port );
@@ -55,22 +58,45 @@ admin.auth( "admin" , "admin" );
 var foo = conn.getDB( "foo" )
 
 // make sure no collection with the same name exists
-assert.eq( foo.system.namespaces.count( {name: "foo.bar"}), 0 )
+assert.eq(foo.system.namespaces.count( {name: "foo.bar"}), 0);
+assert.eq(foo.system.namespaces.count( {name: "foo.baz"}), 0);
 
 // now try to restore dump
 x = runMongoProgram( "mongorestore", "-h", "127.0.0.1:" + port,  "--dir" , dumpdir, "-vvvvv" );
 
 // make sure that the collection isn't restored
-assert.eq( foo.system.namespaces.count({name: "foo.bar"}), 0 )
+assert.eq(foo.system.namespaces.count({name: "foo.bar"}), 0);
+assert.eq(foo.system.namespaces.count({name: "foo.baz"}), 0);
 
 // now try to restore dump with correct credentials
 x = runMongoProgram( "mongorestore", "-h", "127.0.0.1:" + port, "-d", "foo", "-u", "admin", "-p",
                      "admin", "--dir", dumpdir + "foo/", "-vvvvv");
 
 // make sure that the collection was restored
-assert.eq( foo.system.namespaces.count({name: "foo.bar"}), 1 )
+assert.eq(foo.system.namespaces.count({name: "foo.bar"}), 1);
+assert.eq(foo.system.namespaces.count({name: "foo.baz"}), 1);
 
 // make sure the collection has 4 documents
-assert.eq( foo.bar.count(), 4 )
+assert.eq(foo.bar.count(), 4);
+assert.eq(foo.baz.count(), 4);
+
+foo.dropDatabase();
+
+// make sure that the collection is empty
+assert.eq(foo.system.namespaces.count({name: "foo.bar"}), 0);
+assert.eq(foo.system.namespaces.count({name: "foo.baz"}), 0);
+
+foo.addUser('user', 'password');
+
+// now try to restore dump with foo database credentials
+x = runMongoProgram("mongorestore", "-h", "127.0.0.1:" + port, "-d", "foo", "-u", "user", "-p",
+                    "password", "--dir", dumpdir + "foo/", "-vvvvv");
+
+// make sure that the collection was restored
+assert.eq(foo.system.namespaces.count({name: "foo.bar"}), 1);
+assert.eq(foo.system.namespaces.count({name: "foo.baz"}), 1);
+assert.eq(foo.bar.count(), 4);
+assert.eq(foo.baz.count(), 4);
+assert.eq(foo.system.indexes.count(), 4); // _id on foo, _id on bar, x on foo, _id on system.users
 
 stopMongod( port );

jstests/tool/tool_replset.js

@@ -0,0 +1,79 @@
+/*
+ * Test to ensure that (dump/restore/export/import/oplog) works with a replica set connection string
+ * 1. Start a replica set.
+ * 2. Add data to a collection.
+ * 3. Take a dump of the database.
+ * 4. Drop the db.
+ * 5. Restore the db.
+ * 6. Export a collection.
+ * 7. Drop the collection.
+ * 8. Import the collection.
+ * 9. Add data to the oplog.rs collection.
+ * 10. Ensure that the document doesn't exist yet.
+ * 11. Now play the mongooplog tool.
+ * 12. Make sure that the oplog was played
+*/
+
+// Load utility methods for replica set tests
+load("jstests/replsets/rslib.js");
+
+var replTest = new ReplSetTest({ name: 'rs', nodes: 2, oplogSize: 5 });
+var nodes = replTest.startSet();
+replTest.initiate();
+var master = replTest.getMaster();
+var docNum = 100;
+for (var i = 0; i < docNum; i++) {
+    master.getDB("foo").bar.insert({ a: i });
+}
+replTest.awaitReplication();
+
+var replSetConnString = "rs/127.0.0.1:" + replTest.ports[0] + ",127.0.0.1:" + replTest.ports[1];
+
+// Test with mongodump/mongorestore
+print("dump the db");
+var data = "/data/db/dumprestore11-dump1/";
+runMongoProgram("mongodump", "--host", replSetConnString, "--out", data);
+
+master.getDB("foo").dropDatabase();
+replTest.awaitReplication();
+
+print("restore the db");
+runMongoProgram("mongorestore", "--host", replSetConnString, "--dir", data);
+
+var x = master.getDB("foo").getCollection("bar").count();
+assert.eq(x, docNum, "mongorestore should have successfully restored the collection" + docNum);
+
+replTest.awaitReplication();
+
+// Test with mongoexport/mongoimport
+print("export the collection");
+var extFile = "/data/db/exportimport_replSet/export";
+runMongoProgram("mongoexport", "--host", replSetConnString, "--out", extFile,
+                "-d", "foo", "-c", "bar");
+
+master.getDB("foo").getCollection("bar").drop();
+replTest.awaitReplication();
+
+print("import the collection");
+runMongoProgram("mongoimport", "--host", replSetConnString, "--file", extFile,
+                "-d", "foo", "-c", "bar");
+
+var x = master.getDB("foo").getCollection("bar").count();
+assert.eq(x, docNum, "mongoimport should have successfully imported the collection" + docNum);
+
+master.getDB("foo").getCollection("bar").drop();
+
+// Test with mongooplog
+var doc = { _id : 5, x : 17 };
+master.getDB("local").oplog.rs.insert({ ts : new Timestamp(), "op" : "i", "ns" : "foo.bar",
+                                         "o" : doc });
+
+assert.eq(0, master.getDB("foo").getCollection("bar").count(), "before");
+
+var replSetConnString = "rs/127.0.0.1:" + replTest.ports[0] + ",127.0.0.1:" + replTest.ports[1];
+runMongoProgram("mongooplog" , "--from", "127.0.0.1:" + replTest.ports[0],
+                               "--host", replSetConnString);
+
+assert.eq(101, master.getDB("foo").getCollection("bar").count(), "after")
+
+replTest.stopSet();

jstests/update_arraymatch8.js

@@ -0,0 +1,113 @@
+// Checking for positional array updates with either .$ or .0 at the end
+// SERVER-7511
+
+// array.$.name
+t = db.jstests_update_arraymatch8;
+t.drop();
+t.ensureIndex( {'array.name': 1} );
+t.insert( {'array': [{'name': 'old'}]} );
+assert( t.findOne({'array.name': 'old'}) );
+t.update( {'array.name': 'old'}, {$set: {'array.$.name': 'new'}} );
+assert( t.findOne({'array.name': 'new'}) );
+assert( !t.findOne({'array.name': 'old'}) );
+
+// array.$   (failed in 2.2.2)
+t = db.jstests_update_arraymatch8;
+t.drop();
+t.ensureIndex( {'array.name': 1} );
+t.insert( {'array': [{'name': 'old'}]} );
+assert( t.findOne({'array.name': 'old'}) );
+t.update( {'array.name': 'old'}, {$set: {'array.$': {'name':'new'}}} );
+assert( t.findOne({'array.name': 'new'}) );
+assert( !t.findOne({'array.name': 'old'}) );
+
+// array.0.name
+t = db.jstests_update_arraymatch8;
+t.drop();
+t.ensureIndex( {'array.name': 1} );
+t.insert( {'array': [{'name': 'old'}]} );
+assert( t.findOne({'array.name': 'old'}) );
+t.update( {'array.name': 'old'}, {$set: {'array.0.name': 'new'}} );
+assert( t.findOne({'array.name': 'new'}) );
+assert( !t.findOne({'array.name': 'old'}) );
+
+// array.0   (failed in 2.2.2)
+t = db.jstests_update_arraymatch8;
+t.drop();
+t.ensureIndex( {'array.name': 1} );
+t.insert( {'array': [{'name': 'old'}]} );
+assert( t.findOne({'array.name': 'old'}) );
+t.update( {'array.name': 'old'}, {$set: {'array.0': {'name':'new'}}} );
+assert( t.findOne({'array.name': 'new'}) );
+assert( !t.findOne({'array.name': 'old'}) );
+
+// // array.12.name
+t = db.jstests_update_arraymatch8;
+t.drop();
+arr = new Array();
+for (var i=0; i<20; i++) {
+    arr.push({'name': 'old'});
+}
+t.ensureIndex( {'array.name': 1} );
+t.insert( {_id:0, 'array': arr} );
+assert( t.findOne({'array.name': 'old'}) );
+t.update( {_id:0}, {$set: {'array.12.name': 'new'}} );
+// note: both documents now have to be in the array
+assert( t.findOne({'array.name': 'new'}) );
+assert( t.findOne({'array.name': 'old'}) );
+
+// array.12   (failed in 2.2.2)
+t = db.jstests_update_arraymatch8;
+t.drop();
+arr = new Array();
+for (var i=0; i<20; i++) {
+    arr.push({'name': 'old'});
+}
+t.ensureIndex( {'array.name': 1} );
+t.insert( {_id:0, 'array': arr} );
+assert( t.findOne({'array.name': 'old'}) );
+t.update( {_id:0}, {$set: {'array.12': {'name':'new'}}} );
+// note: both documents now have to be in the array
+assert( t.findOne({'array.name': 'new'}) );
+assert( t.findOne({'array.name': 'old'}) );
+
+// array.$.123a.name
+t = db.jstests_update_arraymatch8;
+t.drop();
+t.ensureIndex( {'array.123a.name': 1} );
+t.insert( {'array': [{'123a':{'name': 'old'}}]} );
+assert( t.findOne({'array.123a.name': 'old'}) );
+t.update( {'array.123a.name': 'old'}, {$set: {'array.$.123a.name': 'new'}} );
+assert( t.findOne({'array.123a.name': 'new'}) );
+assert( !t.findOne({'array.123a.name': 'old'}) );
+
+// array.$.123a
+t = db.jstests_update_arraymatch8;
+t.drop();
+t.ensureIndex( {'array.name': 1} );
+t.insert( {'array': [{'123a':{'name': 'old'}}]} );
+assert( t.findOne({'array.123a.name': 'old'}) );
+t.update( {'array.123a.name': 'old'}, {$set: {'array.$.123a': {'name': 'new'}}} );
+assert( t.findOne({'array.123a.name': 'new'}) );
+assert( !t.findOne({'array.123a.name': 'old'}) );
+
+// array.0.123a.name
+t = db.jstests_update_arraymatch8;
+t.drop();
+t.ensureIndex( {'array.123a.name': 1} );
+t.insert( {'array': [{'123a':{'name': 'old'}}]} );
+assert( t.findOne({'array.123a.name': 'old'}) );
+t.update( {'array.123a.name': 'old'}, {$set: {'array.0.123a.name': 'new'}} );
+assert( t.findOne({'array.123a.name': 'new'}) );
+assert( !t.findOne({'array.123a.name': 'old'}) );
+
+// array.0.123a
+t = db.jstests_update_arraymatch8;
+t.drop();
+t.ensureIndex( {'array.name': 1} );
+t.insert( {'array': [{'123a':{'name': 'old'}}]} );
+assert( t.findOne({'array.123a.name': 'old'}) );
+t.update( {'array.123a.name': 'old'}, {$set: {'array.0.123a': {'name': 'new'}}} );
+assert( t.findOne({'array.123a.name': 'new'}) );
+assert( !t.findOne({'array.123a.name': 'old'}) );
+

jstests/upsert2.js

@@ -0,0 +1,20 @@
+// A query field with a $not operator should be excluded when constructing the object to which mods
+// will be applied when performing an upsert.  SERVER-8178
+
+t = db.jstests_upsert2;
+
+// The a:$not query operator does not cause an 'a' field to be added to the upsert document.
+t.drop();
+t.update( { a:{ $not:{ $lt:1 } } }, { $set:{ b:1 } }, true );
+assert( !t.findOne().a );
+
+// The a:$not query operator does not cause an 'a' field to be added to the upsert document.
+t.drop();
+t.update( { a:{ $not:{ $elemMatch:{ a:1 } } } }, { $set:{ b:1 } }, true );
+assert( !t.findOne().a );
+
+// The a:$not query operator does not cause an 'a' field to be added to the upsert document, and as
+// a result $push can be applied to the (missing) 'a' field.
+t.drop();
+t.update( { a:{ $not:{ $elemMatch:{ a:1 } } } }, { $push:{ a:{ b:1, c:0 } } }, true );
+assert.eq( [ { b:1, c:0 } ], t.findOne().a );

rpm/mongo.spec

@@ -1,7 +1,7 @@
 Name: mongo-10gen
 Conflicts: mongo, mongo-10gen-unstable
 Obsoletes: mongo-stable
-Version: 2.1.2
+Version: 2.2.6
 Release: mongodb_1%{?dist}
 Summary: mongodb client shell and tools
 License: AGPL 3.0

site_scons/libdeps.py

@@ -51,6 +51,9 @@ import SCons.Errors
 import SCons.Scanner
 import SCons.Util
 
+libdeps_env_var = 'LIBDEPS'
+syslibdeps_env_var = 'SYSLIBDEPS'
+
 def sorted_by_str(iterable):
     """Shorthand for sorting an iterable according to its string representation.
 
@@ -71,19 +74,19 @@ class DependencyCycleError(SCons.Errors.UserError):
     def __str__(self):
         return " => ".join(str(n) for n in self.cycle_nodes)
 
-def __get_libdeps(node, env_var):
+def __get_libdeps(node):
     """Given a SCons Node, return its library dependencies.
 
     Computes the dependencies if they're not already cached.
     """
 
-    cached_var_name = env_var + '_cached'
+    cached_var_name = libdeps_env_var + '_cached'
 
     if not hasattr(node.attributes, cached_var_name):
-        setattr(node.attributes, cached_var_name, __compute_libdeps(node, env_var))
+        setattr(node.attributes, cached_var_name, __compute_libdeps(node))
     return getattr(node.attributes, cached_var_name)
 
-def __compute_libdeps(node, env_var):
+def __compute_libdeps(node):
     """Recursively identify all library dependencies for a node."""
 
     if getattr(node.attributes, 'libdeps_exploring', False):
@@ -94,11 +97,11 @@ def __compute_libdeps(node, env_var):
     node.attributes.libdeps_exploring = True
     try:
         try:
-            for child in env.Flatten(env.get(env_var, [])):
+            for child in env.Flatten(env.get(libdeps_env_var, [])):
                 if not child:
                     continue
                 deps.add(child)
-                deps.update(__get_libdeps(child, env_var))
+                deps.update(__get_libdeps(child))
 
         except DependencyCycleError, e:
             if len(e.cycle_nodes) == 1 or e.cycle_nodes[0] != e.cycle_nodes[-1]:
@@ -109,6 +112,20 @@ def __compute_libdeps(node, env_var):
 
     return deps
 
+def __get_syslibdeps(node):
+    """ Given a SCons Node, return its system library dependencies.
+
+    These are the depencencies listed with SYSLIBDEPS, and are linked using -l.
+    """
+    cached_var_name = syslibdeps_env_var + '_cached'
+    if not hasattr(node.attributes, cached_var_name):
+        syslibdeps = []
+        for lib in __get_libdeps(node):
+            for syslib in lib.get_env().get(syslibdeps_env_var, []):
+                syslibdeps.append(syslib)
+        setattr(node.attributes, cached_var_name, sorted(syslibdeps))
+    return getattr(node.attributes, cached_var_name)
+
 def update_scanner(builder):
     """Update the scanner for "builder" to also scan library dependencies."""
 
@@ -118,14 +135,12 @@ def update_scanner(builder):
         path_function = old_scanner.path_function
         def new_scanner(node, env, path=()):
             result = set(old_scanner.function(node, env, path))
-            result.update(__get_libdeps(node, 'LIBDEPS'))
-            result.update(__get_libdeps(node, 'SYSLIBDEPS'))
+            result.update(__get_libdeps(node))
             return sorted_by_str(result)
     else:
         path_function = None
         def new_scanner(node, env, path=()):
-            result = set(__get_libdeps(node, 'LIBDEPS'))
-            result.update(__get_libdeps(node, 'SYSLIBDEPS'))
+            result = set(__get_libdeps(node))
             return sorted_by_str(result)
 
     builder.target_scanner = SCons.Scanner.Scanner(function=new_scanner,
@@ -138,21 +153,24 @@ def get_libdeps(source, target, env, for_signature):
     """
 
     target = env.Flatten([target])
-    return list(__get_libdeps(target[0], 'LIBDEPS'))
+    return sorted_by_str(__get_libdeps(target[0]))
 
 def get_libdeps_objs(source, target, env, for_signature):
     objs = set()
     for lib in get_libdeps(source, target, env, for_signature):
         objs.update(lib.sources_set)
-    return list(objs)
+    return sorted_by_str(objs)
 
 def get_libdeps_special_sun(source, target, env, for_signature):
     x = get_libdeps(source, target, env, for_signature )
     return x + x + x
 
 def get_syslibdeps(source, target, env, for_signature):
-    deps = list(__get_libdeps(target[0], 'SYSLIBDEPS'))
-    return deps
+    deps = __get_syslibdeps(target[0])
+    lib_link_prefix = env.subst('$LIBLINKPREFIX')
+    lib_link_suffix = env.subst('$LIBLINKSUFFIX')
+    result = ''.join([' %s%s%s' % (lib_link_prefix, d, lib_link_suffix) for d in deps])
+    return result
 
 def libdeps_emitter(target, source, env):
     """SCons emitter that takes values from the LIBDEPS environment variable and
@@ -172,7 +190,7 @@ def libdeps_emitter(target, source, env):
     libdep_files = []
     lib_suffix = env.subst('$LIBSUFFIX', target=target, source=source)
     lib_prefix = env.subst('$LIBPREFIX', target=target, source=source)
-    for dep in env.Flatten([env.get('LIBDEPS', [])]):
+    for dep in env.Flatten([env.get(libdeps_env_var, [])]):
         full_path = env.subst(str(dep), target=target, source=source)
         dir_name = os.path.dirname(full_path)
         file_name = os.path.basename(full_path)
@@ -182,7 +200,7 @@ def libdeps_emitter(target, source, env):
             file_name += '${LIBSUFFIX}'
         libdep_files.append(env.File(os.path.join(dir_name, file_name)))
 
-    env['LIBDEPS'] = libdep_files
+    env[libdeps_env_var] = libdep_files
 
     return target, source
 
@@ -203,11 +221,11 @@ def setup_environment(env):
         env['_LIBDEPS_LIBS'] = get_libdeps
 
     env['_LIBDEPS_OBJS'] = get_libdeps_objs
-    env['_SYSLIBDEPS'] = ' ${_stripixes(LIBLINKPREFIX, SYSLIBDEPS, LIBLINKSUFFIX, LIBPREFIXES, LIBSUFFIXES, __env__)} '
+    env['_SYSLIBDEPS'] = get_syslibdeps
     env['_SHLIBDEPS'] = '$SHLIBDEP_GROUP_START ${_concat(SHLIBDEPPREFIX, __env__.subst(_LIBDEPS, target=TARGET, source=SOURCE), SHLIBDEPSUFFIX, __env__, target=TARGET, source=SOURCE)} $SHLIBDEP_GROUP_END'
 
-    env['LIBDEPS'] = SCons.Util.CLVar()
-    env['SYSLIBDEPS'] = SCons.Util.CLVar()
+    env[libdeps_env_var] = SCons.Util.CLVar()
+    env[syslibdeps_env_var] = SCons.Util.CLVar()
     env.Append(LIBEMITTER=libdeps_emitter,
                PROGEMITTER=libdeps_emitter,
                SHLIBEMITTER=libdeps_emitter)

src/SConscript.client

@@ -8,7 +8,8 @@ Import('env clientEnv')
 clientSource = [
     'mongo/bson/oid.cpp',
     'mongo/buildinfo.cpp',
-    "mongo/client/authentication_table.cpp",
+    "mongo/client/authentication_table_common.cpp",
+    "mongo/client/authentication_table_client.cpp",
     'mongo/client/clientAndShell.cpp',
     'mongo/client/clientOnly.cpp',
     'mongo/client/connection_factory.cpp',

src/mongo/SConscript

@@ -3,6 +3,7 @@
 # This SConscript describes build rules for the "mongo" project.
 
 import os
+import itertools
 from buildscripts import utils
 
 Import("env")
@@ -80,7 +81,7 @@ commonFiles = [ "pch.cpp",
                 "util/net/message_port.cpp",
                 "util/net/listen.cpp",
                 "util/startup_test.cpp",
-                "client/authentication_table.cpp",
+                "client/authentication_table_common.cpp",
                 "client/connpool.cpp",
                 "client/dbclient.cpp",
                 "client/dbclient_rs.cpp",
@@ -101,6 +102,7 @@ env.StaticLibrary('mongocommon', commonFiles,
                            '$BUILD_DIR/third_party/shim_boost'],)
 
 env.StaticLibrary("coredb", [
+        "client/authentication_table_server.cpp",
         "client/parallel.cpp",
         "db/commands.cpp",
         "db/commands/hashcmd.cpp",
@@ -343,6 +345,16 @@ env.CppUnitTest( "balancer_policy_test" , [ "s/balancer_policy_tests.cpp" ] ,
                  LIBDEPS=["mongoscore", "coreshard","mongocommon","coreserver","coredb","dbcmdline","mongodandmongos"] ,
                  NO_CRUTCH=True)
 
+env.CppUnitTest("shard_test", [ "s/shard_test.cpp" ],
+                LIBDEPS=[ "mongoscore",
+                          "coreshard",
+                          "mongocommon",
+                          "coreserver",
+                          "coredb",
+                          "dbcmdline",
+                          "mongodandmongos"],
+                NO_CRUTCH=True)
+
 serverOnlyFiles += [ "s/d_logic.cpp",
                      "s/d_writeback.cpp",
                      "s/d_migrate.cpp",
@@ -432,7 +444,12 @@ mongos = env.Program(
              "mongodandmongos"])
 env.Install( '#/', mongos )
 
-env.Library("clientandshell", ["client/clientAndShell.cpp"], LIBDEPS=["mongocommon", "defaultversion", "gridfs", "notmongodormongos"])
+env.Library("clientandshell", ["client/authentication_table_client.cpp",
+                               "client/clientAndShell.cpp"],
+                              LIBDEPS=["mongocommon",
+                                       "defaultversion",
+                                       "gridfs",
+                                       "notmongodormongos"])
 env.Library("allclient", "client/clientOnly.cpp", LIBDEPS=["clientandshell"])
 
 if has_option( "sharedclient" ):
@@ -564,18 +581,36 @@ if installSetup.libraries:
     if has_option( "sharedclient" ):
         env.Install( "$INSTALL_DIR/$NIX_LIB_DIR",  '#${SHLIBPREFIX}mongoclient${SHLIBSUFFIX}')
 
+# Stage the top-level mongodb banners
+distsrc = env.Dir('#distsrc')
+env.Append(MODULE_BANNERS = [distsrc.File('README'),
+                             distsrc.File('THIRD-PARTY-NOTICES')])
+
+# If no module has introduced a file named LICENSE.txt, then inject the AGPL.
+if sum(itertools.imap(lambda x: x.name == "LICENSE.txt", env['MODULE_BANNERS'])) == 0:
+    env.Append(MODULE_BANNERS = [distsrc.File('GNU-AGPL-3.0')])
+
+# All module banners get staged to the top level of the tarfile, so we
+# need to fail if we are going to have a name collision.
+module_banner_filenames = set([f.name for f in env['MODULE_BANNERS']])
+if not len(module_banner_filenames) == len(env['MODULE_BANNERS']):
+    # TODO: Be nice and identify conflicts in error.
+    print "ERROR: Filename conflicts exist in module banners."
+    Exit(-1)
+
+# Build a set of directories containing module banners, and use that
+# to build a --transform option for each directory so that the files
+# are tar'ed up to the proper location.
+module_banner_dirs = set([Dir('#').rel_path(f.get_dir()) for f in env['MODULE_BANNERS']])
+module_banner_transforms = ["--transform %s=$SERVER_DIST_BASENAME" % d for d in module_banner_dirs]
+
 env.Command(
     '#/${SERVER_ARCHIVE}',
-    ['#buildscripts/make_archive.py',
-     '#distsrc/GNU-AGPL-3.0',
-     '#distsrc/README',
-     '#distsrc/THIRD-PARTY-NOTICES',
-     distBinaries],
-    '$PYTHON ${SOURCES[0]} -o $TARGET '
-    '--transform distsrc=$SERVER_DIST_BASENAME '
-    '--transform ${str(Dir(BUILD_DIR))}/mongo/stripped=$SERVER_DIST_BASENAME/bin '
-    '--transform ${str(Dir(BUILD_DIR))}/mongo=$SERVER_DIST_BASENAME/bin '
-    '${TEMPFILE(SOURCES[1:])}')
+    ['#buildscripts/make_archive.py'] + env["MODULE_BANNERS"] + distBinaries,
+    ' '.join(['$PYTHON ${SOURCES[0]} -o $TARGET'] + module_banner_transforms + [
+            '--transform ${str(Dir(BUILD_DIR))}/mongo/stripped=$SERVER_DIST_BASENAME/bin',
+            '--transform ${str(Dir(BUILD_DIR))}/mongo=$SERVER_DIST_BASENAME/bin',
+            '${TEMPFILE(SOURCES[1:])}']))
 
 #final alias
 env.Alias( "install", "$INSTALL_DIR" )

src/mongo/bson/bsonelement.h

@@ -17,6 +17,7 @@
 
 #pragma once
 
+#include <boost/cstdint.hpp>
 #include <string.h> // strlen
 #include <string>
 #include <vector>
@@ -240,8 +241,8 @@ namespace mongo {
         }
 
         // for objects the size *includes* the size of the size field
-        int objsize() const {
-            return *reinterpret_cast< const int* >( value() );
+        size_t objsize() const {
+            return static_cast< const size_t >( *reinterpret_cast< const uint32_t* >( value() ) );
         }
 
         /** Get a string's value.  Also gives you start of the real data for an embedded object.

src/mongo/bson/util/builder.h

@@ -247,13 +247,10 @@ namespace mongo {
         void decouple(); // not allowed. not implemented.
     };
 
-    namespace {
 #if defined(_WIN32)
-        int (*mongo_snprintf)(char *str, size_t size, const char *format, ...) = &sprintf_s;
-#else
-        int (*mongo_snprintf)(char *str, size_t size, const char *format, ...) = &snprintf;
+#pragma push_macro("snprintf")
+#define snprintf _snprintf
 #endif
-    }
 
     /** stringstream deals with locale so this is a lot faster than std::stringstream for UTF8 */
     template <typename Allocator>
@@ -301,7 +298,7 @@ namespace mongo {
             const int prev = _buf.l;
             const int maxSize = 32; 
             char * start = _buf.grow( maxSize );
-            int z = mongo_snprintf( start , maxSize , "%.16g" , x );
+            int z = snprintf( start , maxSize , "%.16g" , x );
             verify( z >= 0 );
             verify( z < maxSize );
             _buf.l = prev + z;
@@ -335,7 +332,7 @@ namespace mongo {
         template <typename T>
         StringBuilderImpl& SBNUM(T val,int maxSize,const char *macro)  {
             int prev = _buf.l;
-            int z = mongo_snprintf( _buf.grow(maxSize) , maxSize , macro , (val) );
+            int z = snprintf( _buf.grow(maxSize) , maxSize , macro , (val) );
             verify( z >= 0 );
             verify( z < maxSize );
             _buf.l = prev + z;
@@ -346,4 +343,8 @@ namespace mongo {
     typedef StringBuilderImpl<TrivialAllocator> StringBuilder;
     typedef StringBuilderImpl<StackAllocator> StackStringBuilder;
 
+#if defined(_WIN32)
+#undef snprintf
+#pragma pop_macro("snprintf")
+#endif
 } // namespace mongo

src/mongo/client/authentication_table.h

@@ -59,6 +59,9 @@ namespace mongo {
 
         static const string fieldName;
     private:
+
+        bool _shouldSendInternalSecurityTable() const;
+
         typedef map<std::string,Auth> DBAuthMap;
         DBAuthMap _dbs; // dbname -> auth
     };

src/mongo/client/authentication_table_client.cpp

@@ -0,0 +1,27 @@
+/*    Copyright 2012 10gen Inc.
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+
+#include "mongo/client/authentication_table.h"
+#include "mongo/db/client_common.h"
+#include "mongo/db/security.h"
+
+namespace mongo {
+
+    bool AuthenticationTable::_shouldSendInternalSecurityTable() const {
+        return false;
+    }
+
+}

src/mongo/client/authentication_table_common.cpp

@@ -93,7 +93,12 @@ namespace mongo {
             cmdWithAuth.append(e);
         }
 
-        cmdWithAuth.append( fieldName, toBSON() );
+        if (_shouldSendInternalSecurityTable()) {
+            cmdWithAuth.append(fieldName, internalSecurityAuthenticationTable.toBSON());
+        }
+        else {
+            cmdWithAuth.append(fieldName, toBSON());
+        }
         return cmdWithAuth.obj();
     }
 

src/mongo/client/authentication_table_server.cpp

@@ -0,0 +1,30 @@
+/*    Copyright 2012 10gen Inc.
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+
+#include "mongo/client/authentication_table.h"
+#include "mongo/db/client_common.h"
+#include "mongo/db/security.h"
+
+namespace mongo {
+
+    bool AuthenticationTable::_shouldSendInternalSecurityTable() const {
+        if (!ClientBasic::hasCurrent() || !ClientBasic::getCurrent()->getAuthenticationInfo()) {
+            return false;
+        }
+        return ClientBasic::getCurrent()->getAuthenticationInfo()->isSpecialLocalhostAdmin();
+    }
+
+}

src/mongo/client/clientAndShell.cpp

@@ -87,4 +87,8 @@ namespace mongo {
     ClientBasic* ClientBasic::getCurrent() {
         return 0;
     }
+
+    bool ClientBasic::hasCurrent() {
+        return false;
+    }
 }

src/mongo/client/dbclient.cpp

@@ -36,6 +36,8 @@
 
 namespace mongo {
 
+    AtomicInt64 DBClientBase::ConnectionIdSequence;
+
     void ConnectionString::_fillServers( string s ) {
         
         //
@@ -94,12 +96,12 @@ namespace mongo {
         case MASTER: {
             DBClientConnection * c = new DBClientConnection(true);
             c->setSoTimeout( socketTimeout );
-            log(1) << "creating new connection to:" << _servers[0] << endl;
+            LOG(1) << "creating new connection to:" << _servers[0] << endl;
             if ( ! c->connect( _servers[0] , errmsg ) ) {
                 delete c;
                 return 0;
             }
-            log(1) << "connected connection!" << endl;
+            LOG(1) << "connected connection!" << endl;
             return c;
         }
 
@@ -152,6 +154,45 @@ namespace mongo {
         return 0;
     }
 
+    bool ConnectionString::sameLogicalEndpoint( const ConnectionString& other ) const {
+        if ( _type != other._type )
+            return false;
+
+        switch ( _type ) {
+        case INVALID:
+            return true;
+        case MASTER:
+            return _servers[0] == other._servers[0];
+        case PAIR:
+            if ( _servers[0] == other._servers[0] )
+                return _servers[1] == other._servers[1];
+            return
+                ( _servers[0] == other._servers[1] ) &&
+                ( _servers[1] == other._servers[0] );
+        case SET:
+            return _setName == other._setName;
+        case SYNC:
+            // The servers all have to be the same in each, but not in the same order.
+            if ( _servers.size() != other._servers.size() )
+                return false;
+            for ( unsigned i = 0; i < _servers.size(); i++ ) {
+                bool found = false;
+                for ( unsigned j = 0; j < other._servers.size(); j++ ) {
+                    if ( _servers[i] == other._servers[j] ) {
+                        found = true;
+                        break;
+                    }
+                }
+                if ( ! found )
+                    return false;
+            }
+            return true;
+        case CUSTOM:
+            return _string == other._string;
+        }
+        verify( false );
+    }
+
     ConnectionString ConnectionString::parse( const string& host , string& errmsg ) {
 
         string::size_type i = host.find( '/' );
@@ -389,6 +430,14 @@ namespace mongo {
     }
 
     BSONObj DBClientWithCommands::getLastErrorDetailed(bool fsync, bool j, int w, int wtimeout) {
+        return getLastErrorDetailed("admin", fsync, j, w, wtimeout);
+    }
+
+    BSONObj DBClientWithCommands::getLastErrorDetailed(const std::string& db,
+                                                       bool fsync,
+                                                       bool j,
+                                                       int w,
+                                                       int wtimeout) {
         BSONObj info;
         BSONObjBuilder b;
         b.append( "getlasterror", 1 );
@@ -407,21 +456,37 @@ namespace mongo {
         if ( wtimeout > 0 )
             b.append( "wtimeout", wtimeout );
 
-        runCommand("admin", b.obj(), info);
+        runCommand(db, b.obj(), info);
 
         return info;
     }
 
     string DBClientWithCommands::getLastError(bool fsync, bool j, int w, int wtimeout) {
-        BSONObj info = getLastErrorDetailed(fsync, j, w, wtimeout);
+        return getLastError("admin", fsync, j, w, wtimeout);
+    }
+
+    string DBClientWithCommands::getLastError(const std::string& db,
+                                              bool fsync,
+                                              bool j,
+                                              int w,
+                                              int wtimeout) {
+        BSONObj info = getLastErrorDetailed(db, fsync, j, w, wtimeout);
         return getLastErrorString( info );
     }
 
-    string DBClientWithCommands::getLastErrorString( const BSONObj& info ) {
-        BSONElement e = info["err"];
-        if( e.eoo() ) return "";
-        if( e.type() == Object ) return e.toString();
-        return e.str();
+    string DBClientWithCommands::getLastErrorString(const BSONObj& info) {
+        if (info["ok"].trueValue()) {
+            BSONElement e = info["err"];
+            if (e.eoo()) return "";
+            if (e.type() == Object) return e.toString();
+            return e.str();
+        } else {
+            // command failure
+            BSONElement e = info["errmsg"];
+            if (e.eoo()) return "";
+            if (e.type() == Object) return "getLastError command failed: " + e.toString();
+            return "getLastError command failed: " + e.str();
+        }
     }
 
     const BSONObj getpreverrorcmdobj = fromjson("{getpreverror:1}");
@@ -458,7 +523,7 @@ namespace mongo {
         BSONObj info;
         string nonce;
         if( !runCommand(dbname, getnoncecmdobj, info) ) {
-            errmsg = "getnonce fails - connection problem?";
+            errmsg = "getnonce failed: " + info.toString();
             return false;
         }
         {
@@ -753,22 +818,22 @@ namespace mongo {
             throw SocketException( SocketException::FAILED_STATE , toString() );
 
         lastReconnectTry = time(0);
-        log(_logLevel) << "trying reconnect to " << _serverString << endl;
+        LOG(_logLevel) << "trying reconnect to " << _serverString << endl;
         string errmsg;
         _failed = false;
         if ( ! _connect(errmsg) ) {
             _failed = true;
-            log(_logLevel) << "reconnect " << _serverString << " failed " << errmsg << endl;
+            LOG(_logLevel) << "reconnect " << _serverString << " failed " << errmsg << endl;
             throw SocketException( SocketException::CONNECT_ERROR , toString() );
         }
 
-        log(_logLevel) << "reconnect " << _serverString << " ok" << endl;
+        LOG(_logLevel) << "reconnect " << _serverString << " ok" << endl;
         for( map< string, pair<string,string> >::iterator i = authCache.begin(); i != authCache.end(); i++ ) {
             const char *dbname = i->first.c_str();
             const char *username = i->second.first.c_str();
             const char *password = i->second.second.c_str();
             if( !DBClientBase::auth(dbname, username, password, errmsg, false) )
-                log(_logLevel) << "reconnect: auth failed db:" << dbname << " user:" << username << ' ' << errmsg << '\n';
+                LOG(_logLevel) << "reconnect: auth failed db:" << dbname << " user:" << username << ' ' << errmsg << '\n';
         }
     }
 
@@ -992,7 +1057,7 @@ namespace mongo {
         if ( ! runCommand( nsToDatabase( ns.c_str() ) ,
                            BSON( "deleteIndexes" << NamespaceString( ns ).coll << "index" << indexName ) ,
                            info ) ) {
-            log(_logLevel) << "dropIndex failed: " << info << endl;
+            LOG(_logLevel) << "dropIndex failed: " << info << endl;
             uassert( 10007 ,  "dropIndex failed" , 0 );
         }
         resetIndexCache();

src/mongo/client/dbclient_rs.cpp

@@ -46,15 +46,19 @@ namespace mongo {
      * @param lastHost the last host returned (mainly used for doing round-robin).
      *     Will be overwritten with the newly returned host if not empty. Should
      *     never be NULL.
+     * @param isPrimarySelected out parameter that is set to true if the returned host
+     *     is a primary.
      *
      * @return the host object of the node selected. If none of the nodes are
-     *     eligible, returns an empty host.
+     *     eligible, returns an empty host. Cannot be NULL and valid only if returned
+     *     host is not empty.
      */
     HostAndPort _selectNode(const vector<ReplicaSetMonitor::Node>& nodes,
                             const BSONObj& readPreferenceTag,
                             bool secOnly,
                             int localThresholdMillis,
-                            HostAndPort* lastHost /* in/out */) {
+                            HostAndPort* lastHost /* in/out */,
+                            bool* isPrimarySelected) {
         HostAndPort fallbackHost;
 
         // Implicit: start from index 0 if lastHost doesn't exist anymore
@@ -74,7 +78,7 @@ namespace mongo {
             const ReplicaSetMonitor::Node& node = nodes[nextNodeIndex];
 
             if (!node.ok) {
-                log(2) << "dbclient_rs not selecting " << node << ", not currently ok" << endl;
+                LOG(2) << "dbclient_rs not selecting " << node << ", not currently ok" << endl;
                 continue;
             }
 
@@ -85,10 +89,11 @@ namespace mongo {
             if (node.matchesTag(readPreferenceTag)) {
                 // found an ok candidate; may not be local.
                 fallbackHost = node.addr;
+                *isPrimarySelected = node.ismaster;
 
                 if (node.isLocalSecondary(localThresholdMillis)) {
                     // found a local node.  return early.
-                    log(2) << "dbclient_rs getSlave found local secondary for queries: "
+                    LOG(2) << "dbclient_rs getSlave found local secondary for queries: "
                            << nextNodeIndex << ", ping time: " << node.pingTimeMillis << endl;
                     *lastHost = fallbackHost;
                     return fallbackHost;
@@ -282,7 +287,7 @@ namespace mongo {
         if ( createFromSeed ) {
             map<string,vector<HostAndPort> >::const_iterator j = _seedServers.find( name );
             if ( j != _seedServers.end() ) {
-                log(4) << "Creating ReplicaSetMonitor from cached address" << endl;
+                LOG(4) << "Creating ReplicaSetMonitor from cached address" << endl;
                 ReplicaSetMonitorPtr& m = _sets[name];
                 verify( !m );
                 m.reset( new ReplicaSetMonitor( name, j->second ) );
@@ -334,7 +339,7 @@ namespace mongo {
     }
 
     void ReplicaSetMonitor::_remove_inlock( const string& name, bool clearSeedCache ) {
-        log(2) << "Removing ReplicaSetMonitor for " << name << " from replica set table" << endl;
+        LOG(2) << "Removing ReplicaSetMonitor for " << name << " from replica set table" << endl;
         _sets.erase( name );
         if ( clearSeedCache ) {
             _seedServers.erase( name );
@@ -364,7 +369,7 @@ namespace mongo {
         for ( unsigned i=0; i<_nodes.size(); i++ ) {
             if ( i > 0 )
                 ss << ",";
-            ss << _nodes[i].addr.toString();
+            _nodes[i].addr.append( ss );
         }
 
         return ss.str();
@@ -459,21 +464,21 @@ namespace mongo {
                         return fallbackNode;
                     else if ( _nodes[ _nextSlave ].isLocalSecondary( _localThresholdMillis ) ) {
                         // found a local slave.  return early.
-                        log(2) << "dbclient_rs getSlave found local secondary for queries: "
+                        LOG(2) << "dbclient_rs getSlave found local secondary for queries: "
                                << _nextSlave << ", ping time: "
                                << _nodes[ _nextSlave ].pingTimeMillis << endl;
                         return fallbackNode;
                     }
                 }
                 else
-                    log(2) << "dbclient_rs getSlave not selecting " << _nodes[_nextSlave]
+                    LOG(2) << "dbclient_rs getSlave not selecting " << _nodes[_nextSlave]
                            << ", not currently okForSecondaryQueries" << endl;
             }
         }
 
         if ( ! fallbackNode.empty() ) {
             // use a non-local secondary, even if local was preferred
-            log(1) << "dbclient_rs getSlave falling back to a non-local secondary node" << endl;
+            LOG(1) << "dbclient_rs getSlave falling back to a non-local secondary node" << endl;
             return fallbackNode;
         }
 
@@ -482,7 +487,7 @@ namespace mongo {
                 _master < static_cast<int>(_nodes.size()) && _nodes[_master].ok);
 
         // Fall back to primary
-        log(1) << "dbclient_rs getSlave no member in secondary state found, "
+        LOG(1) << "dbclient_rs getSlave no member in secondary state found, "
                   "returning primary " << _nodes[ _master ] << endl;
         return _nodes[_master].addr;
     }
@@ -505,7 +510,7 @@ namespace mongo {
          * and tell it to use the internal credentials.
          */
         scoped_ptr<ScopedDbConnection> authenticatedConn(
-                ScopedDbConnection::getInternalScopedDbConnection( hostAddr ) );
+                ScopedDbConnection::getInternalScopedDbConnection( hostAddr, 5.0 ) );
 
         if ( !authenticatedConn->get()->runCommand( "admin",
                                                     BSON( "replSetGetStatus" << 1 ),
@@ -734,7 +739,7 @@ namespace mongo {
                 node.lastIsMaster = o.copy();
             }
 
-            log( ! verbose ) << "ReplicaSetMonitor::_checkConnection: " << conn->toString()
+            LOG( verbose ? 0 : 1 ) << "ReplicaSetMonitor::_checkConnection: " << conn->toString()
                              << ' ' << o << endl;
             
             // add other nodes
@@ -757,7 +762,7 @@ namespace mongo {
 
         }
         catch ( std::exception& e ) {
-            log( ! verbose ) << "ReplicaSetMonitor::_checkConnection: caught exception "
+            LOG( verbose ? 0 : 1 ) << "ReplicaSetMonitor::_checkConnection: caught exception "
                              << conn->toString() << ' ' << e.what() << endl;
 
             errorOccured = true;
@@ -1006,14 +1011,15 @@ namespace mongo {
     }
 
     HostAndPort ReplicaSetMonitor::selectAndCheckNode(ReadPreference preference,
-                                                      TagSet* tags) {
+                                                      TagSet* tags,
+                                                      bool* isPrimarySelected) {
 
         HostAndPort candidate;
 
         {
             scoped_lock lk(_lock);
             candidate = ReplicaSetMonitor::selectNode(_nodes, preference, tags,
-                    _localThresholdMillis, &_lastReadPrefHost);
+                    _localThresholdMillis, &_lastReadPrefHost, isPrimarySelected);
         }
 
         if (candidate.empty()) {
@@ -1022,7 +1028,7 @@ namespace mongo {
 
             scoped_lock lk(_lock);
             return ReplicaSetMonitor::selectNode(_nodes, preference, tags, _localThresholdMillis,
-                    &_lastReadPrefHost);
+                    &_lastReadPrefHost, isPrimarySelected);
         }
 
         return candidate;
@@ -1033,11 +1039,15 @@ namespace mongo {
                                               ReadPreference preference,
                                               TagSet* tags,
                                               int localThresholdMillis,
-                                              HostAndPort* lastHost) {
+                                              HostAndPort* lastHost,
+                                              bool* isPrimarySelected) {
+        *isPrimarySelected = false;
+
         switch (preference) {
         case ReadPreference_PrimaryOnly:
             for (vector<Node>::const_iterator iter = nodes.begin(); iter != nodes.end(); ++iter) {
                 if (iter->ismaster && iter->ok) {
+                    *isPrimarySelected = true;
                     return iter->addr;
                 }
             }
@@ -1047,14 +1057,14 @@ namespace mongo {
         case ReadPreference_PrimaryPreferred:
         {
             HostAndPort candidatePri = selectNode(nodes, ReadPreference_PrimaryOnly, tags,
-                                                  localThresholdMillis, lastHost);
+                    localThresholdMillis, lastHost, isPrimarySelected);
 
             if (!candidatePri.empty()) {
                 return candidatePri;
             }
 
             return selectNode(nodes, ReadPreference_SecondaryOnly, tags,
-                              localThresholdMillis, lastHost);
+                              localThresholdMillis, lastHost, isPrimarySelected);
         }
 
         case ReadPreference_SecondaryOnly:
@@ -1063,7 +1073,7 @@ namespace mongo {
 
             while (!tags->isExhausted()) {
                 candidate = _selectNode(nodes, tags->getCurrentTag(), true, localThresholdMillis,
-                        lastHost);
+                        lastHost, isPrimarySelected);
 
                 if (candidate.empty()) {
                     tags->next();
@@ -1079,14 +1089,14 @@ namespace mongo {
         case ReadPreference_SecondaryPreferred:
         {
             HostAndPort candidateSec = selectNode(nodes, ReadPreference_SecondaryOnly, tags,
-                                                  localThresholdMillis, lastHost);
+                    localThresholdMillis, lastHost, isPrimarySelected);
 
             if (!candidateSec.empty()) {
                 return candidateSec;
             }
 
             return selectNode(nodes, ReadPreference_PrimaryOnly, tags,
-                              localThresholdMillis, lastHost);
+                    localThresholdMillis, lastHost, isPrimarySelected);
         }
 
         case ReadPreference_Nearest:
@@ -1095,7 +1105,7 @@ namespace mongo {
 
             while (!tags->isExhausted()) {
                 candidate = _selectNode(nodes, tags->getCurrentTag(), false, localThresholdMillis,
-                        lastHost);
+                        lastHost, isPrimarySelected);
 
                 if (candidate.empty()) {
                     tags->next();
@@ -1163,6 +1173,19 @@ namespace mongo {
         _check(true);
     }
 
+    bool ReplicaSetMonitor::isAnyNodeOk() const {
+        scoped_lock lock(_lock);
+
+        for (vector<Node>::const_iterator iter = _nodes.begin();
+                iter != _nodes.end(); ++iter) {
+            if (iter->ok) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
     bool ReplicaSetMonitor::Node::matchesTag(const BSONObj& tag) const {
         if (tag.isEmpty()) {
             return true;
@@ -1347,19 +1370,7 @@ namespace mongo {
     }
 
     bool DBClientReplicaSet::connect() {
-        try {
-            checkMaster();
-        }
-        catch (AssertionException&) {
-            // Can't use _getMonitor because that will create a new monitor from the cached seed if
-            // the monitor doesn't exist.
-            ReplicaSetMonitorPtr monitor = ReplicaSetMonitor::get(_setName);
-            if (_master && monitor ) {
-                monitor->notifyFailure(_masterHost);
-            }
-            return false;
-        }
-        return true;
+        return _getMonitor()->isAnyNodeOk();
     }
 
     bool DBClientReplicaSet::auth(const string &dbname, const string &username, const string &pwd, string& errmsg, bool digestPassword, Auth::Level * level) {
@@ -1558,12 +1569,24 @@ namespace mongo {
         }
 
         ReplicaSetMonitorPtr monitor = _getMonitor();
-        _lastSlaveOkHost = monitor->selectAndCheckNode(preference, tags);
+        bool isPrimarySelected = false;
+        _lastSlaveOkHost = monitor->selectAndCheckNode(preference, tags, &isPrimarySelected);
 
         if ( _lastSlaveOkHost.empty() ){
             return NULL;
         }
 
+        // Primary connection is special because it is the only connection that is
+        // versioned in mongos. Therefore, we have to make sure that this object
+        // maintains only one connection to the primary and use that connection
+        // every time we need to talk to the primary.
+        if (isPrimarySelected) {
+            checkMaster();
+            _lastSlaveOkConn = _master;
+            _lastSlaveOkHost = _masterHost; // implied, but still assign just to be safe
+            return _master.get();
+        }
+
         _lastSlaveOkConn.reset(new DBClientConnection(true , this , _so_timeout));
         _lastSlaveOkConn->connect(_lastSlaveOkHost);
 

src/mongo/client/dbclient_rs.h

@@ -147,6 +147,8 @@ namespace mongo {
          *     robin, starting from the node next to this lastHost. This will be overwritten
          *     with the newly chosen host if not empty, not primary and when preference
          *     is not Nearest.
+         * @param isPrimarySelected out parameter that is set to true if the returned host
+         *     is a primary. Cannot be NULL and valid only if returned host is not empty.
          *
          * @return the host object of the node selected. If none of the nodes are
          *     eligible, returns an empty host.
@@ -155,7 +157,8 @@ namespace mongo {
                                       ReadPreference preference,
                                       TagSet* tags,
                                       int localThresholdMillis,
-                                      HostAndPort* lastHost);
+                                      HostAndPort* lastHost,
+                                      bool* isPrimarySelected);
 
         /**
          * Selects the right node given the nodes to pick from and the preference. This
@@ -163,14 +166,17 @@ namespace mongo {
          * if the primary node needs to be returned but is not currently available (except
          * for ReadPrefrence_Nearest).
          *
-         * @param preference the read mode to use
-         * @param tags the tags used for filtering nodes
+         * @param preference the read mode to use.
+         * @param tags the tags used for filtering nodes.
+         * @param isPrimarySelected out parameter that is set to true if the returned host
+         *     is a primary. Cannot be NULL and valid only if returned host is not empty.
          *
          * @return the host object of the node selected. If none of the nodes are
          *     eligible, returns an empty host.
          */
         HostAndPort selectAndCheckNode(ReadPreference preference,
-                                       TagSet* tags);
+                                       TagSet* tags,
+                                       bool* isPrimarySelected);
 
         /**
          * Creates a new ReplicaSetMonitor, if it doesn't already exist.
@@ -262,6 +268,14 @@ namespace mongo {
         bool isHostCompatible(const HostAndPort& host, ReadPreference readPreference,
                 const TagSet* tagSet) const;
 
+        /**
+         * Performs a quick check if at least one node is up based on the cached
+         * view of the set.
+         *
+         * @return true if any node is ok
+         */
+        bool isAnyNodeOk() const;
+
     private:
         /**
          * This populates a list of hosts from the list of seeds (discarding the
@@ -405,10 +419,10 @@ namespace mongo {
         DBClientReplicaSet( const string& name , const vector<HostAndPort>& servers, double so_timeout=0 );
         virtual ~DBClientReplicaSet();
 
-        /** Returns false if nomember of the set were reachable, or neither is
-         * master, although,
-         * when false returned, you can still try to use this connection object, it will
-         * try reconnects.
+        /**
+         * Returns false if no member of the set were reachable. This object
+         * can still be used even when false was returned as it will try to
+         * reconnect when you use it later.
          */
         bool connect();
 
@@ -448,6 +462,12 @@ namespace mongo {
 
         // ---- access raw connections ----
 
+        /**
+         * WARNING: this method is very dangerous - this object can decide to free the
+         *     returned master connection any time.
+         *
+         * @return the reference to the address that points to the master connection.
+         */
         DBClientConnection& masterConn();
         DBClientConnection& slaveConn();
 
@@ -542,12 +562,18 @@ namespace mongo {
         string _setName;
 
         HostAndPort _masterHost;
-        scoped_ptr<DBClientConnection> _master;
+        // Note: reason why this is a shared_ptr is because we want _lastSlaveOkConn to
+        // keep a reference of the _master connection when it selected a primary node.
+        // This is because the primary connection is special in mongos - it is the only
+        // connection that is versioned.
+        // WARNING: do not assign this variable (which will increment the internal ref
+        // counter) to any other variable other than _lastSlaveOkConn.
+        boost::shared_ptr<DBClientConnection> _master;
 
         // Last used host in a slaveOk query (can be a primary)
         HostAndPort _lastSlaveOkHost;
         // Last used connection in a slaveOk query (can be a primary)
-        scoped_ptr<DBClientConnection> _lastSlaveOkConn;
+        boost::shared_ptr<DBClientConnection> _lastSlaveOkConn;
         
         double _so_timeout;
 

src/mongo/client/dbclientcursor.cpp

@@ -157,10 +157,11 @@ namespace mongo {
         verify( !haveLimit );
         auto_ptr<Message> response(new Message());
         verify( _client );
-        if ( _client->recv(*response) ) {
-            batch.m = response;
-            dataReceived();
+        if (!_client->recv(*response)) {
+            uasserted(16465, "recv failed while exhausting cursor");
         }
+        batch.m = response;
+        dataReceived();
     }
 
     void DBClientCursor::dataReceived( bool& retry, string& host ) {

src/mongo/client/dbclientinterface.h

@@ -25,6 +25,7 @@
 #include "mongo/client/authlevel.h"
 #include "mongo/client/authentication_table.h"
 #include "mongo/db/jsobj.h"
+#include "mongo/platform/atomic_word.h"
 #include "mongo/util/net/message.h"
 #include "mongo/util/net/message_port.h"
 
@@ -247,6 +248,14 @@ namespace mongo {
         
         ConnectionType type() const { return _type; }
 
+        /**
+         * This returns true if this and other point to the same logical entity.
+         * For single nodes, thats the same address.
+         * For replica sets, thats just the same replica set name.
+         * For pair (deprecated) or sync cluster connections, that's the same hosts in any ordering.
+         */
+        bool sameLogicalEndpoint( const ConnectionString& other ) const;
+
         static ConnectionString parse( const string& url , string& errmsg );
 
         static string typeToString( ConnectionType type );
@@ -631,16 +640,30 @@ namespace mongo {
         bool createCollection(const string &ns, long long size = 0, bool capped = false, int max = 0, BSONObj *info = 0);
 
         /** Get error result from the last write operation (insert/update/delete) on this connection.
+            db doesn't change the command's behavior - it is just for auth checks.
             @return error message text, or empty string if no error.
         */
+        string getLastError(const std::string& db,
+                            bool fsync = false,
+                            bool j = false,
+                            int w = 0,
+                            int wtimeout = 0);
+        // Same as above but defaults to using admin DB
         string getLastError(bool fsync = false, bool j = false, int w = 0, int wtimeout = 0);
 
         /** Get error result from the last write operation (insert/update/delete) on this connection.
+            db doesn't change the command's behavior - it is just for auth checks.
             @return full error object.
 
             If "w" is -1, wait for propagation to majority of nodes.
             If "wtimeout" is 0, the operation will block indefinitely if needed.
         */
+        virtual BSONObj getLastErrorDetailed(const std::string& db,
+                                             bool fsync = false,
+                                             bool j = false,
+                                             int w = 0,
+                                             int wtimeout = 0);
+        // Same as above but defaults to using admin DB
         virtual BSONObj getLastErrorDetailed(bool fsync = false, bool j = false, int w = 0, int wtimeout = 0);
 
         /** Can be called with the returned value from getLastErrorDetailed to extract an error string. 
@@ -907,13 +930,17 @@ namespace mongo {
      */
     class DBClientBase : public DBClientWithCommands, public DBConnector {
     protected:
+        static AtomicInt64 ConnectionIdSequence;
+        long long _connectionId; // unique connection id for this connection
         WriteConcern _writeConcern;
-
     public:
         DBClientBase() {
             _writeConcern = W_NORMAL;
+            _connectionId = ConnectionIdSequence.fetchAndAdd(1);
         }
 
+        long long getConnectionId() const { return _connectionId; }
+
         WriteConcern getWriteConcern() const { return _writeConcern; }
         void setWriteConcern( WriteConcern w ) { _writeConcern = w; }
 

src/mongo/client/distlock.cpp

@@ -85,14 +85,14 @@ namespace mongo {
 
             string pingId = pingThreadId( addr, process );
 
-            log( DistributedLock::logLvl - 1 ) << "creating distributed lock ping thread for " << addr
+            LOG( DistributedLock::logLvl - 1 ) << "creating distributed lock ping thread for " << addr
                                                << " and process " << process
                                                << " (sleeping for " << sleepTime << "ms)" << endl;
 
             static int loops = 0;
             while( ! inShutdown() && ! shouldKill( addr, process ) ) {
 
-                log( DistributedLock::logLvl + 2 ) << "distributed lock pinger '" << pingId << "' about to ping." << endl;
+                LOG( DistributedLock::logLvl + 2 ) << "distributed lock pinger '" << pingId << "' about to ping." << endl;
 
                 Date_t pingTime;
 
@@ -155,7 +155,7 @@ namespace mongo {
                         conn->ensureIndex( DistributedLock::lockPingNS , BSON( "ping" << 1 ) );
                     }
 
-                    log( DistributedLock::logLvl - ( loops % 10 == 0 ? 1 : 0 ) ) << "cluster " << addr << " pinged successfully at " << pingTime
+                    LOG( DistributedLock::logLvl - ( loops % 10 == 0 ? 1 : 0 ) ) << "cluster " << addr << " pinged successfully at " << pingTime
                             << " by distributed lock pinger '" << pingId
                             << "', sleeping for " << sleepTime << "ms" << endl;
 
@@ -165,7 +165,7 @@ namespace mongo {
 
                     int numOldLocks = _oldLockOIDs.size();
                     if( numOldLocks > 0 )
-                        log( DistributedLock::logLvl - 1 ) << "trying to delete " << _oldLockOIDs.size() << " old lock entries for process " << process << endl;
+                        LOG( DistributedLock::logLvl - 1 ) << "trying to delete " << _oldLockOIDs.size() << " old lock entries for process " << process << endl;
 
                     bool removed = false;
                     for( list<OID>::iterator i = _oldLockOIDs.begin(); i != _oldLockOIDs.end();
@@ -179,11 +179,11 @@ namespace mongo {
 
                             // Either the update went through or it didn't, either way we're done trying to
                             // unlock
-                            log( DistributedLock::logLvl - 1 ) << "handled late remove of old distributed lock with ts " << *i << endl;
+                            LOG( DistributedLock::logLvl - 1 ) << "handled late remove of old distributed lock with ts " << *i << endl;
                             removed = true;
                         }
                         catch( UpdateNotTheSame& ) {
-                            log( DistributedLock::logLvl - 1 ) << "partially removed old distributed lock with ts " << *i << endl;
+                            LOG( DistributedLock::logLvl - 1 ) << "partially removed old distributed lock with ts " << *i << endl;
                             removed = true;
                         }
                         catch ( std::exception& e) {
@@ -194,7 +194,7 @@ namespace mongo {
                     }
 
                     if( numOldLocks > 0 && _oldLockOIDs.size() > 0 ){
-                        log( DistributedLock::logLvl - 1 ) << "not all old lock entries could be removed for process " << process << endl;
+                        LOG( DistributedLock::logLvl - 1 ) << "not all old lock entries could be removed for process " << process << endl;
                     }
 
                     conn.done();
@@ -319,9 +319,9 @@ namespace mongo {
           _lockTimeout( lockTimeout == 0 ? LOCK_TIMEOUT : lockTimeout ), _maxClockSkew( _lockTimeout / LOCK_SKEW_FACTOR ), _maxNetSkew( _maxClockSkew ), _lockPing( _maxClockSkew ),
           _mutex( "DistributedLock" )
     {
-        log( logLvl - 1 ) << "created new distributed lock for " << name << " on " << conn
-                          << " ( lock timeout : " << _lockTimeout
-                          << ", ping interval : " << _lockPing << ", process : " << asProcess << " )" << endl;
+        LOG( logLvl ) << "created new distributed lock for " << name << " on " << conn
+                      << " ( lock timeout : " << _lockTimeout
+                      << ", ping interval : " << _lockPing << ", process : " << asProcess << " )" << endl;
 
 
     }
@@ -427,7 +427,7 @@ namespace mongo {
                 // Skew is how much time we'd have to add to local to get to remote
                 avgSkews[s] += (long long) (remote - local);
 
-                log( logLvl + 1 ) << "skew from remote server " << server << " found: " << (long long) (remote - local) << endl;
+                LOG( logLvl + 1 ) << "skew from remote server " << server << " found: " << (long long) (remote - local) << endl;
 
             }
         }
@@ -459,11 +459,11 @@ namespace mongo {
 
         // Make sure our max skew is not more than our pre-set limit
         if(totalSkew > (long long) maxClockSkew) {
-            log( logLvl + 1 ) << "total clock skew of " << totalSkew << "ms for servers " << cluster << " is out of " << maxClockSkew << "ms bounds." << endl;
+            LOG( logLvl + 1 ) << "total clock skew of " << totalSkew << "ms for servers " << cluster << " is out of " << maxClockSkew << "ms bounds." << endl;
             return false;
         }
 
-        log( logLvl + 1 ) << "total clock skew of " << totalSkew << "ms for servers " << cluster << " is in " << maxClockSkew << "ms bounds." << endl;
+        LOG( logLvl + 1 ) << "total clock skew of " << totalSkew << "ms for servers " << cluster << " is in " << maxClockSkew << "ms bounds." << endl;
         return true;
     }
 
@@ -492,6 +492,11 @@ namespace mongo {
         // This should always be true, if not, we are using the lock incorrectly.
         verify( _name != "" );
 
+        LOG( logLvl ) << "trying to acquire new distributed lock for " << _name << " on " << _conn
+                      << " ( lock timeout : " << _lockTimeout
+                      << ", ping interval : " << _lockPing << ", process : " << _processId << " )"
+                      << endl;
+
         // write to dummy if 'other' is null
         BSONObj dummyOther;
         if ( other == NULL )
@@ -512,7 +517,7 @@ namespace mongo {
             // Case 1: No locks
             if ( o.isEmpty() ) {
                 try {
-                    log( logLvl ) << "inserting initial doc in " << locksNS << " for lock " << _name << endl;
+                    LOG( logLvl ) << "inserting initial doc in " << locksNS << " for lock " << _name << endl;
                     conn->insert( locksNS , BSON( "_id" << _name << "state" << 0 << "who" << "" ) );
                 }
                 catch ( UserException& e ) {
@@ -527,10 +532,10 @@ namespace mongo {
 
                 bool canReenter = reenter && o["process"].String() == _processId && ! distLockPinger.willUnlockOID( o["ts"].OID() ) && o["state"].numberInt() == 2;
                 if( reenter && ! canReenter ) {
-                    log( logLvl - 1 ) << "not re-entering distributed lock " << lockName;
-                    if( o["process"].String() != _processId ) log( logLvl - 1 ) << ", different process " << _processId << endl;
-                    else if( o["state"].numberInt() == 2 ) log( logLvl - 1 ) << ", state not finalized" << endl;
-                    else log( logLvl - 1 ) << ", ts " << o["ts"].OID() << " scheduled for late unlock" << endl;
+                    LOG( logLvl - 1 ) << "not re-entering distributed lock " << lockName;
+                    if( o["process"].String() != _processId ) LOG( logLvl - 1 ) << ", different process " << _processId << endl;
+                    else if( o["state"].numberInt() == 2 ) LOG( logLvl - 1 ) << ", state not finalized" << endl;
+                    else LOG( logLvl - 1 ) << ", ts " << o["ts"].OID() << " scheduled for late unlock" << endl;
 
                     // reset since we've been bounced by a previous lock not being where we thought it was,
                     // and should go through full forcing process if required.
@@ -541,7 +546,7 @@ namespace mongo {
 
                 BSONObj lastPing = conn->findOne( lockPingNS , o["process"].wrap( "_id" ) );
                 if ( lastPing.isEmpty() ) {
-                    log( logLvl ) << "empty ping found for process in lock '" << lockName << "'" << endl;
+                    LOG( logLvl ) << "empty ping found for process in lock '" << lockName << "'" << endl;
                     // TODO:  Using 0 as a "no time found" value Will fail if dates roll over, but then, so will a lot.
                     lastPing = BSON( "_id" << o["process"].String() << "ping" << (Date_t) 0 );
                 }
@@ -550,7 +555,7 @@ namespace mongo {
                 unsigned long long takeover = _lockTimeout;
                 PingData _lastPingCheck = getLastPing();
 
-                log( logLvl ) << "checking last ping for lock '" << lockName << "'" << " against process " << _lastPingCheck.id << " and ping " << _lastPingCheck.lastPing << endl;
+                LOG( logLvl ) << "checking last ping for lock '" << lockName << "'" << " against process " << _lastPingCheck.id << " and ping " << _lastPingCheck.lastPing << endl;
 
                 try {
 
@@ -587,17 +592,17 @@ namespace mongo {
                 }
 
                 if ( elapsed <= takeover && ! canReenter ) {
-                    log( logLvl ) << "could not force lock '" << lockName << "' because elapsed time " << elapsed << " <= takeover time " << takeover << endl;
+                    LOG( logLvl ) << "could not force lock '" << lockName << "' because elapsed time " << elapsed << " <= takeover time " << takeover << endl;
                     *other = o; other->getOwned(); conn.done();
                     return false;
                 }
                 else if( elapsed > takeover && canReenter ) {
-                    log( logLvl - 1 ) << "not re-entering distributed lock " << lockName << "' because elapsed time " << elapsed << " > takeover time " << takeover << endl;
+                    LOG( logLvl - 1 ) << "not re-entering distributed lock " << lockName << "' because elapsed time " << elapsed << " > takeover time " << takeover << endl;
                     *other = o; other->getOwned(); conn.done();
                     return false;
                 }
 
-                log( logLvl - 1 ) << ( canReenter ? "re-entering" : "forcing" ) << " lock '" << lockName << "' because "
+                LOG( logLvl - 1 ) << ( canReenter ? "re-entering" : "forcing" ) << " lock '" << lockName << "' because "
                                   << ( canReenter ? "re-entering is allowed, " : "" )
                                   << "elapsed time " << elapsed << " > takeover time " << takeover << endl;
 
@@ -626,7 +631,7 @@ namespace mongo {
 
                         // TODO: Clean up all the extra code to exit this method, probably with a refactor
                         if ( !errMsg.empty() || !err["n"].type() || err["n"].numberInt() < 1 ) {
-                            ( errMsg.empty() ? log( logLvl - 1 ) : warning() ) << "Could not force lock '" << lockName << "' "
+                            ( errMsg.empty() ? LOG( logLvl - 1 ) : warning() ) << "Could not force lock '" << lockName << "' "
                                     << ( !errMsg.empty() ? causedBy(errMsg) : string("(another force won)") ) << endl;
                             *other = o; other->getOwned(); conn.done();
                             return false;
@@ -668,7 +673,7 @@ namespace mongo {
 
                         // TODO: Clean up all the extra code to exit this method, probably with a refactor
                         if ( ! errMsg.empty() || ! err["n"].type() || err["n"].numberInt() < 1 ) {
-                            ( errMsg.empty() ? log( logLvl - 1 ) : warning() ) << "Could not re-enter lock '" << lockName << "' "
+                            ( errMsg.empty() ? LOG( logLvl - 1 ) : warning() ) << "Could not re-enter lock '" << lockName << "' "
                                                                                << ( !errMsg.empty() ? causedBy(errMsg) : string("(not sure lock is held)") ) 
                                                                                << " gle: " << err
                                                                                << endl;
@@ -689,14 +694,14 @@ namespace mongo {
                                              << lockName << causedBy( e ), 13660);
                     }
 
-                    log( logLvl - 1 ) << "re-entered distributed lock '" << lockName << "'" << endl;
+                    LOG( logLvl - 1 ) << "re-entered distributed lock '" << lockName << "'" << endl;
                     *other = o.getOwned(); 
                     conn.done();
                     return true;
 
                 }
 
-                log( logLvl - 1 ) << "lock '" << lockName << "' successfully forced" << endl;
+                LOG( logLvl - 1 ) << "lock '" << lockName << "' successfully forced" << endl;
 
                 // We don't need the ts value in the query, since we will only ever replace locks with state=0.
             }
@@ -725,7 +730,7 @@ namespace mongo {
 
             // Main codepath to acquire lock
 
-            log( logLvl ) << "about to acquire distributed lock '" << lockName << ":\n"
+            LOG( logLvl ) << "about to acquire distributed lock '" << lockName << ":\n"
                           <<  lockDetails.jsonString(Strict, true) << "\n"
                           << query.jsonString(Strict, true) << endl;
 
@@ -737,7 +742,7 @@ namespace mongo {
             currLock = conn->findOne( locksNS , _id );
 
             if ( !errMsg.empty() || !err["n"].type() || err["n"].numberInt() < 1 ) {
-                ( errMsg.empty() ? log( logLvl - 1 ) : warning() ) << "could not acquire lock '" << lockName << "' "
+                ( errMsg.empty() ? LOG( logLvl - 1 ) : warning() ) << "could not acquire lock '" << lockName << "' "
                         << ( !errMsg.empty() ? causedBy( errMsg ) : string("(another update won)") ) << endl;
                 *other = currLock;
                 other->getOwned();
@@ -816,11 +821,11 @@ namespace mongo {
             // Locks on all servers are now set and safe until forcing
 
             if ( currLock["ts"] == lockDetails["ts"] ) {
-                log( logLvl - 1 ) << "lock update won, completing lock propagation for '" << lockName << "'" << endl;
+                LOG( logLvl - 1 ) << "lock update won, completing lock propagation for '" << lockName << "'" << endl;
                 gotLock = true;
             }
             else {
-                log( logLvl - 1 ) << "lock update lost, lock '" << lockName << "' not propagated." << endl;
+                LOG( logLvl - 1 ) << "lock update lost, lock '" << lockName << "' not propagated." << endl;
 
                 // Register the lock for deletion, to speed up failover
                 // Not strictly necessary, but helpful
@@ -889,9 +894,9 @@ namespace mongo {
 
         // Log our lock results
         if(gotLock)
-            log( logLvl - 1 ) << "distributed lock '" << lockName << "' acquired, ts : " << currLock["ts"].OID() << endl;
+            LOG( logLvl - 1 ) << "distributed lock '" << lockName << "' acquired, ts : " << currLock["ts"].OID() << endl;
         else
-            log( logLvl - 1 ) << "distributed lock '" << lockName << "' was not acquired." << endl;
+            LOG( logLvl - 1 ) << "distributed lock '" << lockName << "' was not acquired." << endl;
 
         conn.done();
 
@@ -946,12 +951,12 @@ namespace mongo {
                     continue;
                 }
 
-                log( logLvl - 1 ) << "distributed lock '" << lockName << "' unlocked. " << endl;
+                LOG( logLvl - 1 ) << "distributed lock '" << lockName << "' unlocked. " << endl;
                 conn.done();
                 return;
             }
             catch( UpdateNotTheSame& ) {
-                log( logLvl - 1 ) << "distributed lock '" << lockName << "' unlocked (messily). " << endl;
+                LOG( logLvl - 1 ) << "distributed lock '" << lockName << "' unlocked (messily). " << endl;
                 conn.done();
                 break;
             }
@@ -967,7 +972,7 @@ namespace mongo {
 
         if( attempted > maxAttempts && ! oldLock.isEmpty() && ! oldLock["ts"].eoo() ) {
 
-            log( logLvl - 1 ) << "could not unlock distributed lock with ts " << oldLock["ts"].OID()
+            LOG( logLvl - 1 ) << "could not unlock distributed lock with ts " << oldLock["ts"].OID()
                               << ", will attempt again later" << endl;
 
             // We couldn't unlock the lock at all, so try again later in the pinging thread...

src/mongo/client/distlock_test.cpp

@@ -363,11 +363,11 @@ namespace mongo {
                 bsonArrToNumVector<long long>(cmdObj["skewHosts"], skew);
             }
             else {
-                log( logLvl ) << "No host clocks to skew." << endl;
+                LOG( logLvl ) << "No host clocks to skew." << endl;
                 return;
             }
 
-            log( logLvl ) << "Skewing clocks of hosts " << cluster << endl;
+            LOG( logLvl ) << "Skewing clocks of hosts " << cluster << endl;
 
             unsigned s = 0;
             for(vector<long long>::iterator i = skew.begin(); i != skew.end(); ++i,s++) {
@@ -385,7 +385,7 @@ namespace mongo {
 
                     uassert(13678, str::stream() << "Could not communicate with server " << server.toString() << " in cluster " << cluster.toString() << " to change skew by " << *i, success );
 
-                    log( logLvl + 1 ) << " Skewed host " << server << " clock by " << *i << endl;
+                    LOG( logLvl + 1 ) << " Skewed host " << server << " clock by " << *i << endl;
                 }
                 catch(...) {
                     conn->done();

src/mongo/client/model.cpp

@@ -94,7 +94,7 @@ namespace mongo {
             conn->get()->insert( getNS() , o );
             _id = o["_id"].wrap().getOwned();
 
-            log(4) << "inserted new model " << getNS() << "  " << o << endl;
+            LOG(4) << "inserted new model " << getNS() << "  " << o << endl;
         }
         else {
             if ( myId.eoo() ) {
@@ -110,7 +110,7 @@ namespace mongo {
             BSONObj q = qb.obj();
             BSONObj o = b.obj();
 
-            log(4) << "updated model" << getNS() << "  " << q << " " << o << endl;
+            LOG(4) << "updated model" << getNS() << "  " << q << " " << o << endl;
 
             conn->get()->update( getNS() , q , o , true );