mongodb/mongo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SERVER-13297: convert parallel suite to use jsCore tests

Browse Source 
(cherry picked from commit 835a7cc8ea)
This commit is contained in:
Scott Hernandez 2014-03-25 11:09:37 -04:00
parent 886e1c281d
commit 174f9bea00
14 changed files with 312 additions and 259 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
buildscripts/smoke.py
View File

@ -392,14 +392,14 @@ def ternary( b , l="true", r="false" ):
return l
return r
# Blech.
def skipTest(path):
basename = os.path.basename(path)
parentPath = os.path.dirname(path)
parentDir = os.path.basename(parentPath)
if small_oplog: # For tests running in parallel
if basename in ["cursor8.js", "indexh.js", "dropdb.js", "connections_opened.js", "opcounters.js", "dbadmin.js"]:
if basename in ["cursor8.js", "indexh.js", "dropdb.js",
"connections_opened.js", "opcounters.js", "dbadmin.js"]:
return True
if use_ssl:
# Skip tests using mongobridge since it does not support SSL
@ -442,6 +442,14 @@ def skipTest(path):
return False
def setShellWriteModeForTest(path, argv):
forceCommandsForSuite = ["aggregation", "replsets", "parallel"]
swm = shell_write_mode;
if swm == "legacy": # change when the default changes to "commands"
if use_write_commands or [s for s in forceCommandsForSuite if s in path]:
swm = "commands"
argv += ["--writeMode", swm]
def runTest(test, result):
# result is a map containing test result details, like result["url"]
@ -468,10 +476,9 @@ def runTest(test, result):
path = argv[1]
elif ext == ".js":
argv = [shell_executable, "--port", mongod_port, '--authenticationMechanism', authMechanism]
if use_write_commands or "aggregation" in path or "replsets" in path:
argv += ["--writeMode", "commands"]
else:
argv += ["--writeMode", shell_write_mode]
setShellWriteModeForTest(path, argv)
if not usedb:
argv += ["--nodb"]
if small_oplog or small_oplog_rs:
@ -524,8 +531,7 @@ def runTest(test, result):
'TestData.authMechanism = ' + ternary( authMechanism,
'"' + str(authMechanism) + '"', 'null') + ";" + \
'TestData.useSSL = ' + ternary( use_ssl ) + ";" + \
'TestData.useX509 = ' + ternary( use_x509 ) + ";" + \
'TestData.useWriteCommands = ' + ternary( use_write_commands ) + ";"
'TestData.useX509 = ' + ternary( use_x509 ) + ";"
# this updates the default data directory for mongod processes started through shell (src/mongo/shell/servers.js)
evalString += 'MongoRunner.dataDir = "' + os.path.abspath(smoke_db_prefix + '/data/db') + '";'
evalString += 'MongoRunner.dataPath = MongoRunner.dataDir + "/";'
@ -890,7 +896,19 @@ def expand_suites(suites,expandUseDB=True):
module_suites = get_module_suites()
for suite in suites:
if suite == 'all':
return expand_suites(['test', 'perf', 'js', 'jsPerf', 'jsSlowNightly', 'jsSlowWeekly', 'clone', 'parallel', 'repl', 'auth', 'sharding', 'tool'],expandUseDB=expandUseDB)
return expand_suites(['test',
'perf',
'js',
'jsPerf',
'jsSlowNightly',
'jsSlowWeekly',
'clone',
'parallel',
'repl',
'auth',
'sharding',
'tool'],
expandUseDB=expandUseDB)
if suite == 'test':
if os.sys.platform == "win32":
program = 'test.exe'
@ -951,8 +969,10 @@ def add_exe(e):
return e
def set_globals(options, tests):
global mongod_executable, mongod_port, shell_executable, continue_on_failure, small_oplog, small_oplog_rs
global no_journal, set_parameters, set_parameters_mongos, no_preallocj, auth, authMechanism, keyFile, keyFileData, smoke_db_prefix, test_path, start_mongod
global mongod_executable, mongod_port, shell_executable, continue_on_failure
global small_oplog, small_oplog_rs
global no_journal, set_parameters, set_parameters_mongos, no_preallocj
global auth, authMechanism, keyFile, keyFileData, smoke_db_prefix, test_path, start_mongod
global use_ssl, use_x509
global file_of_commands_mode
global report_file, shell_write_mode, use_write_commands
@ -1093,8 +1113,8 @@ def add_to_failfile(tests, options):
def main():
global mongod_executable, mongod_port, shell_executable, continue_on_failure, small_oplog
global no_journal, set_parameters, set_parameters_mongos, no_preallocj, auth, keyFile, smoke_db_prefix, test_path
global use_write_commands
global no_journal, set_parameters, set_parameters_mongos, no_preallocj, auth
global keyFile, smoke_db_prefix, test_path, use_write_commands
parser = OptionParser(usage="usage: smoke.py [OPTIONS] ARGS*")
parser.add_option('--mode', dest='mode', default='suite',

16
jstests/core/explain7.js
View File

@ -170,12 +170,24 @@ for( i = 30; i < 150; ++i ) {
t.save( { a:i, b:i } );
}
explain = assertUnhintedExplain( { n:150},
// Non-covered $or query.
explain = assertUnhintedExplain( { n:150, nscannedObjects:300 },
t.find( { $or:[ { a:{ $gte:-1, $lte:200 },
b:{ $gte:0, $lte:201 } },
{ a:{ $gte:0, $lte:201 },
b:{ $gte:-1, $lte:200 } } ] },
{ _id:1, a:1, b:1 } ).hint( { a:1, b:1 } ) );
printjson(explain);
assert.eq( 150, explain.clauses[ 0 ].nscannedObjects );
assert.eq( 150, explain.clauses[ 1 ].nscannedObjects );
// Covered $or query.
explain = assertUnhintedExplain( { n:150, nscannedObjects:0 },
t.find( { $or:[ { a:{ $gte:-1, $lte:200 },
b:{ $gte:0, $lte:201 } },
{ a:{ $gte:0, $lte:201 },
b:{ $gte:-1, $lte:200 } } ] },
{ _id:0, a:1, b:1 } ).hint( { a:1, b:1 } ) );
printjson(explain);
// Check nscannedObjects for each clause.
assert.eq( 0, explain.clauses[ 0 ].nscannedObjects );
assert.eq( 0, explain.clauses[ 1 ].nscannedObjects );

8
jstests/core/mr_drop.js
View File

@ -1,6 +1,10 @@
// Drop a collection while a map/reduce job is running against it. SERVER-6757
t = db.jstests_mr_drop;
// Use a different DB to prevent other tests from breaking this race-y test when run in parallel.
// Formerly another test could take out a write lock on the DB, preventing map/reduce and drop()
// from running until the other action completed. When the other action completed, the drop thread
// was finished sleeping and ready to drop and would drop before map/reduce was able to start.
t = db.getSiblingDB('MrDrop').jstests_mr_drop;
t.drop();
Random.setRandomSeed();
@ -20,7 +24,7 @@ for( i = 0; i < 10000; ++i ) {
}
// Schedule a collection drop two seconds in the future.
s = startParallelShell( "sleep( 2000 ); db.jstests_mr_drop.drop();" );
s = startParallelShell( "sleep( 2000 ); db.getSiblingDB('MrDrop').jstests_mr_drop.drop();" );
// Run the map/reduce job. Check for command failure internally. The job succeeds even if the
// source collection is dropped in progress.

248
jstests/libs/parallelTester.js Normal file
View File

@ -0,0 +1,248 @@
/**
* The ParallelTester class is used to test more than one test concurrently
*/
if ( typeof _threadInject != "undefined" ){
//print( "fork() available!" );
Thread = function(){
this.init.apply( this, arguments );
}
_threadInject( Thread.prototype );
ScopedThread = function() {
this.init.apply( this, arguments );
}
ScopedThread.prototype = new Thread( function() {} );
_scopedThreadInject( ScopedThread.prototype );
fork = function() {
var t = new Thread( function() {} );
Thread.apply( t, arguments );
return t;
}
// Helper class to generate a list of events which may be executed by a ParallelTester
EventGenerator = function( me, collectionName, mean, host ) {
this.mean = mean;
if (host == undefined) host = db.getMongo().host;
this.events = new Array( me, collectionName, host );
}
EventGenerator.prototype._add = function( action ) {
this.events.push( [ Random.genExp( this.mean ), action ] );
}
EventGenerator.prototype.addInsert = function( obj ) {
this._add( "t.insert( " + tojson( obj ) + " )" );
}
EventGenerator.prototype.addRemove = function( obj ) {
this._add( "t.remove( " + tojson( obj ) + " )" );
}
EventGenerator.prototype.addUpdate = function( objOld, objNew ) {
this._add( "t.update( " + tojson( objOld ) + ", " + tojson( objNew ) + " )" );
}
EventGenerator.prototype.addCheckCount = function( count, query, shouldPrint, checkQuery ) {
query = query || {};
shouldPrint = shouldPrint || false;
checkQuery = checkQuery || false;
var action = "assert.eq( " + count + ", t.count( " + tojson( query ) + " ) );"
if ( checkQuery ) {
action += " assert.eq( " + count + ", t.find( " + tojson( query ) + " ).toArray().length );"
}
if ( shouldPrint ) {
action += " print( me + ' ' + " + count + " );";
}
this._add( action );
}
EventGenerator.prototype.getEvents = function() {
return this.events;
}
EventGenerator.dispatch = function() {
var args = argumentsToArray( arguments );
var me = args.shift();
var collectionName = args.shift();
var host = args.shift();
var m = new Mongo( host );
var t = m.getDB( "test" )[ collectionName ];
for( var i in args ) {
sleep( args[ i ][ 0 ] );
eval( args[ i ][ 1 ] );
}
}
// Helper class for running tests in parallel. It assembles a set of tests
// and then calls assert.parallelests to run them.
ParallelTester = function() {
assert.neq(db.getMongo().writeMode(), "legacy", "wrong shell write mode")
this.params = new Array();
}
ParallelTester.prototype.add = function( fun, args ) {
args = args || [];
args.unshift( fun );
this.params.push( args );
}
ParallelTester.prototype.run = function( msg, newScopes ) {
newScopes = newScopes || false;
assert.parallelTests( this.params, msg, newScopes );
}
// creates lists of tests from jstests dir in a format suitable for use by
// ParallelTester.fileTester. The lists will be in random order.
// n: number of lists to split these tests into
ParallelTester.createJstestsLists = function( n ) {
var params = new Array();
for( var i = 0; i < n; ++i ) {
params.push( [] );
}
var makeKeys = function( a ) {
var ret = {};
for( var i in a ) {
ret[ a[ i ] ] = 1;
}
return ret;
}
// some tests can't run in parallel with most others
var skipTests = makeKeys([ "dbadmin.js",
"repair.js",
"cursor8.js",
"recstore.js",
"extent.js",
"indexb.js",
"profile1.js",
"profile3.js",
"profile4.js",
"mr3.js",
"indexh.js",
"apitest_db.js",
"evalb.js",
"evald.js",
"evalf.js",
"killop.js",
"run_program1.js",
"notablescan.js",
"drop2.js",
"dropdb_race.js",
"fsync2.js", // May be placed in serialTestsArr once SERVER-4243 is fixed.
"bench_test1.js",
"padding.js",
"queryoptimizera.js",
"loglong.js",// log might overflow before
// this has a chance to see the message
"connections_opened.js", // counts connections, globally
"opcounters.js",
"currentop.js", // SERVER-8673, plus rwlock yielding issues
"set_param1.js", // changes global state
"geo_update_btree2.js", // SERVER-11132 test disables table scans
"update_setOnInsert.js", // SERVER-9982
] );
var parallelFilesDir = "jstests/core";
// some tests can't be run in parallel with each other
var serialTestsArr = [ parallelFilesDir + "/fsync.js",
parallelFilesDir + "/auth1.js"
];
var serialTests = makeKeys( serialTestsArr );
// prefix the first thread with the serialTests
// (which we will exclude from the rest of the threads below)
params[ 0 ] = serialTestsArr;
var files = listFiles( parallelFilesDir );
files = Array.shuffle( files );
var i = 0;
files.forEach(
function(x) {
if ( ( /[\/\\]_/.test(x.name) ) ||
( ! /\.js$/.test(x.name) ) ||
( x.name.match(parallelFilesDir + "/(.*\.js)")[1] in skipTests ) || //
( x.name in serialTests )) {
print(" >>>>>>>>>>>>>>> skipping " + x.name);
return;
}
// add the test to run in one of the threads.
params[ i % n ].push( x.name );
++i;
}
);
// randomize ordering of the serialTests
params[ 0 ] = Array.shuffle( params[ 0 ] );
for( var i in params ) {
params[ i ].unshift( i );
}
return params;
}
// runs a set of test files
// first argument is an identifier for this tester, remaining arguments are file names
ParallelTester.fileTester = function() {
var args = argumentsToArray( arguments );
var suite = args.shift();
args.forEach(
function( x ) {
print(" S" + suite + " Test : " + x + " ...");
var time = Date.timeFunc( function() { load(x); }, 1);
print(" S" + suite + " Test : " + x + " " + time + "ms" );
}
);
}
// params: array of arrays, each element of which consists of a function followed
// by zero or more arguments to that function. Each function and its arguments will
// be called in a separate thread.
// msg: failure message
// newScopes: if true, each thread starts in a fresh scope
assert.parallelTests = function( params, msg, newScopes ) {
newScopes = newScopes || false;
var wrapper = function( fun, argv ) {
eval (
"var z = function() {" +
"var __parallelTests__fun = " + fun.toString() + ";" +
"var __parallelTests__argv = " + tojson( argv ) + ";" +
"var __parallelTests__passed = false;" +
"try {" +
"__parallelTests__fun.apply( 0, __parallelTests__argv );" +
"__parallelTests__passed = true;" +
"} catch ( e ) {" +
"print('');" +
"print( '********** Parallel Test FAILED: ' + tojson(e) );" +
"print('');" +
"}" +
"return __parallelTests__passed;" +
"}"
);
return z;
}
var runners = new Array();
for( var i in params ) {
var param = params[ i ];
var test = param.shift();
var t;
if ( newScopes )
t = new ScopedThread( wrapper( test, param ) );
else
t = new Thread( wrapper( test, param ) );
runners.push( t );
}
runners.forEach( function( x ) { x.start(); } );
var nFailed = 0;
// v8 doesn't like it if we exit before all threads are joined (SERVER-529)
runners.forEach( function( x ) { if( !x.returnData() ) { ++nFailed; } } );
assert.eq( 0, nFailed, msg );
}
}

1
jstests/parallel/allops.js
View File

@ -1,4 +1,5 @@
// test all operations in parallel
load('jstests/libs/parallelTester.js')
f = db.jstests_parallel_allops;
f.drop();

1
jstests/parallel/basic.js
View File

@ -1,4 +1,5 @@
// perform basic js tests in parallel
load('jstests/libs/parallelTester.js')
Random.setRandomSeed();

1
jstests/parallel/basicPlus.js
View File

@ -1,4 +1,5 @@
// perform basic js tests in parallel & some other tasks as well
load('jstests/libs/parallelTester.js')
var c = db.jstests_parallel_basicPlus;
c.drop();

1
jstests/parallel/checkMultiThread.js
View File

@ -1,3 +1,4 @@
load('jstests/libs/parallelTester.js')
var start = new Date();
print("start: " + start);

1
jstests/parallel/del.js
View File

@ -1,3 +1,4 @@
load('jstests/libs/parallelTester.js')
N = 1000;
HOST = db.getMongo().host

1
jstests/parallel/insert.js
View File

@ -1,4 +1,5 @@
// perform inserts in parallel from several clients
load('jstests/libs/parallelTester.js')
f = db.jstests_parallel_insert;
f.drop();

1
jstests/parallel/manyclients.js
View File

@ -1,4 +1,5 @@
// perform inserts in parallel from a large number of clients
load('jstests/libs/parallelTester.js')
f = db.jstests_parallel_manyclients;
f.drop();

1
jstests/parallel/repl.js
View File

@ -1,4 +1,5 @@
// test basic operations in parallel, with replication
load('jstests/libs/parallelTester.js')
baseName = "parallel_repl"

3
jstests/parallel/shellfork.js
View File

@ -1,3 +1,5 @@
load('jstests/libs/parallelTester.js')
a = fork( function( a, b ) { return a / b; }, 10, 2 );
a.start();
b = fork( function( a, b, c ) { return a + b + c; }, 18, " is a ", "multiple of 3" );
@ -13,6 +15,7 @@ assert.eq( "18 is a multiple of 3", b.returnData() );
assert.eq( "paisley ha ha!", c.returnData() );
z = fork( function( a ) {
load('jstests/libs/parallelTester.js');
var y = fork( function( a ) {
return a + 1; }, 5 );
y.start();

242
src/mongo/shell/utils.js
View File

@ -101,248 +101,6 @@ compareOn = function(field){
return function(l, r) { return compare(l[field], r[field]); }
}
if ( typeof _threadInject != "undefined" ){
//print( "fork() available!" );
Thread = function(){
this.init.apply( this, arguments );
}
_threadInject( Thread.prototype );
ScopedThread = function() {
this.init.apply( this, arguments );
}
ScopedThread.prototype = new Thread( function() {} );
_scopedThreadInject( ScopedThread.prototype );
fork = function() {
var t = new Thread( function() {} );
Thread.apply( t, arguments );
return t;
}
// Helper class to generate a list of events which may be executed by a ParallelTester
EventGenerator = function( me, collectionName, mean, host ) {
this.mean = mean;
if (host == undefined) host = db.getMongo().host;
this.events = new Array( me, collectionName, host );
}
EventGenerator.prototype._add = function( action ) {
this.events.push( [ Random.genExp( this.mean ), action ] );
}
EventGenerator.prototype.addInsert = function( obj ) {
this._add( "t.insert( " + tojson( obj ) + " )" );
}
EventGenerator.prototype.addRemove = function( obj ) {
this._add( "t.remove( " + tojson( obj ) + " )" );
}
EventGenerator.prototype.addUpdate = function( objOld, objNew ) {
this._add( "t.update( " + tojson( objOld ) + ", " + tojson( objNew ) + " )" );
}
EventGenerator.prototype.addCheckCount = function( count, query, shouldPrint, checkQuery ) {
query = query || {};
shouldPrint = shouldPrint || false;
checkQuery = checkQuery || false;
var action = "assert.eq( " + count + ", t.count( " + tojson( query ) + " ) );"
if ( checkQuery ) {
action += " assert.eq( " + count + ", t.find( " + tojson( query ) + " ).toArray().length );"
}
if ( shouldPrint ) {
action += " print( me + ' ' + " + count + " );";
}
this._add( action );
}
EventGenerator.prototype.getEvents = function() {
return this.events;
}
EventGenerator.dispatch = function() {
var args = argumentsToArray( arguments );
var me = args.shift();
var collectionName = args.shift();
var host = args.shift();
var m = new Mongo( host );
var t = m.getDB( "test" )[ collectionName ];
for( var i in args ) {
sleep( args[ i ][ 0 ] );
eval( args[ i ][ 1 ] );
}
}
// Helper class for running tests in parallel. It assembles a set of tests
// and then calls assert.parallelests to run them.
ParallelTester = function() {
this.params = new Array();
}
ParallelTester.prototype.add = function( fun, args ) {
args = args || [];
args.unshift( fun );
this.params.push( args );
}
ParallelTester.prototype.run = function( msg, newScopes ) {
newScopes = newScopes || false;
assert.parallelTests( this.params, msg, newScopes );
}
// creates lists of tests from jstests dir in a format suitable for use by
// ParallelTester.fileTester. The lists will be in random order.
// n: number of lists to split these tests into
ParallelTester.createJstestsLists = function( n ) {
var params = new Array();
for( var i = 0; i < n; ++i ) {
params.push( [] );
}
var makeKeys = function( a ) {
var ret = {};
for( var i in a ) {
ret[ a[ i ] ] = 1;
}
return ret;
}
// some tests can't run in parallel with most others
var skipTests = makeKeys( [ "jstests/dbadmin.js",
"jstests/repair.js",
"jstests/cursor8.js",
"jstests/recstore.js",
"jstests/extent.js",
"jstests/indexb.js",
"jstests/profile1.js",
"jstests/profile3.js",
"jstests/profile4.js",
"jstests/mr3.js",
"jstests/indexh.js",
"jstests/apitest_db.js",
"jstests/evalb.js",
"jstests/evald.js",
"jstests/evalf.js",
"jstests/killop.js",
"jstests/run_program1.js",
"jstests/notablescan.js",
"jstests/drop2.js",
"jstests/dropdb_race.js",
"jstests/fsync2.js", // May be placed in serialTestsArr once SERVER-4243 is fixed.
"jstests/bench_test1.js",
"jstests/padding.js",
"jstests/queryoptimizera.js",
"jstests/loglong.js",// log might overflow before
// this has a chance to see the message
"jstests/connections_opened.js", // counts connections, globally
"jstests/opcounters.js",
"jstests/currentop.js", // SERVER-8673, plus rwlock yielding issues
"jstests/set_param1.js", // changes global state
"jstests/geo_update_btree2.js", // SERVER-11132 test disables table scans
"jstests/update_setOnInsert.js", // SERVER-9982
] );
// some tests can't be run in parallel with each other
var serialTestsArr = [ "jstests/fsync.js",
"jstests/auth1.js",
"jstests/connection_status.js",
"jstests/validate_user_documents.js"
// ,"jstests/fsync2.js" // SERVER-4243
];
var serialTests = makeKeys( serialTestsArr );
params[ 0 ] = serialTestsArr;
var files = listFiles("jstests");
files = Array.shuffle( files );
var i = 0;
files.forEach(
function(x) {
if ( ( /[\/\\]_/.test(x.name) ) ||
( ! /\.js$/.test(x.name ) ) ||
( x.name in skipTests ) ||
( x.name in serialTests ) ||
! /\.js$/.test(x.name ) ){
print(" >>>>>>>>>>>>>>> skipping " + x.name);
return;
}
params[ i % n ].push( x.name );
++i;
}
);
// randomize ordering of the serialTests
params[ 0 ] = Array.shuffle( params[ 0 ] );
for( var i in params ) {
params[ i ].unshift( i );
}
return params;
}
// runs a set of test files
// first argument is an identifier for this tester, remaining arguments are file names
ParallelTester.fileTester = function() {
var args = argumentsToArray( arguments );
var suite = args.shift();
args.forEach(
function( x ) {
print(" S" + suite + " Test : " + x + " ...");
var time = Date.timeFunc( function() { load(x); }, 1);
print(" S" + suite + " Test : " + x + " " + time + "ms" );
}
);
}
// params: array of arrays, each element of which consists of a function followed
// by zero or more arguments to that function. Each function and its arguments will
// be called in a separate thread.
// msg: failure message
// newScopes: if true, each thread starts in a fresh scope
assert.parallelTests = function( params, msg, newScopes ) {
newScopes = newScopes || false;
var wrapper = function( fun, argv ) {
eval (
"var z = function() {" +
"var __parallelTests__fun = " + fun.toString() + ";" +
"var __parallelTests__argv = " + tojson( argv ) + ";" +
"var __parallelTests__passed = false;" +
"try {" +
"__parallelTests__fun.apply( 0, __parallelTests__argv );" +
"__parallelTests__passed = true;" +
"} catch ( e ) {" +
"print( '********** Parallel Test FAILED: ' + tojson(e) );" +
"}" +
"return __parallelTests__passed;" +
"}"
);
return z;
}
var runners = new Array();
for( var i in params ) {
var param = params[ i ];
var test = param.shift();
var t;
if ( newScopes )
t = new ScopedThread( wrapper( test, param ) );
else
t = new Thread( wrapper( test, param ) );
runners.push( t );
}
runners.forEach( function( x ) { x.start(); } );
var nFailed = 0;
// v8 doesn't like it if we exit before all threads are joined (SERVER-529)
runners.forEach( function( x ) { if( !x.returnData() ) { ++nFailed; } } );
assert.eq( 0, nFailed, msg );
}
}
shellPrint = function( x ){
it = x;