SERVER-17755 Remove legacy code from build system

This commit is contained in:
Jonathan Reams 2015-04-16 16:22:56 -04:00
parent 0cb8dc2a3c
commit a64b7af35a
33 changed files with 7 additions and 3031 deletions

View File

@ -13,10 +13,8 @@
# This file, SConstruct, configures the build environment, and then delegates to
# several, subordinate SConscript files, which describe specific build rules.
import buildscripts
import copy
import datetime
import imp
import os
import re
import shlex
@ -24,9 +22,6 @@ import shutil
import stat
import sys
import textwrap
import types
import urllib
import urllib2
import uuid
from buildscripts import utils
from buildscripts import moduleconfig
@ -35,11 +30,6 @@ import libdeps
EnsureSConsVersion( 2, 3, 0 )
def findSettingsSetup():
sys.path.append( "." )
sys.path.append( ".." )
sys.path.append( "../../" )
def versiontuple(v):
return tuple(map(int, (v.split("."))))
@ -581,7 +571,7 @@ if releaseBuild and (debugBuild or not optBuild):
print("Error: A --release build may not have debugging, and must have optimization")
Exit(1)
noshell = has_option( "noshell" )
noshell = has_option( "noshell" )
jsEngine = get_option( "js-engine")
@ -882,12 +872,6 @@ elif env.TargetOSIs('solaris'):
env['LINK_LIBGROUP_END'] = ''
# ---- other build setup -----
dontReplacePackage = False
isBuildingLatest = False
def filterExists(paths):
return filter(os.path.exists, paths)
if debugBuild:
env.SetConfigHeaderDefine("MONGO_CONFIG_DEBUG_BUILD")
@ -1040,6 +1024,10 @@ elif env.TargetOSIs('windows'):
if usev8:
env.Append(LIBS=['winmm.lib'])
# When building on visual studio, this sets the name of the debug symbols file
if env.ToolchainIs('msvc'):
env['PDB'] = '${TARGET.base}.pdb'
env['STATIC_AND_SHARED_OBJECTS_ARE_THE_SAME'] = 1
if env.TargetOSIs('posix'):
@ -1137,11 +1125,7 @@ if env['TARGET_ARCH'] == 'i386':
if env.ToolchainIs('GCC', 'clang'):
env.Append( CCFLAGS=['-march=nocona', '-mtune=generic'] )
try:
umask = os.umask(022)
except OSError:
pass
# Needed for auth tests since key files are stored in git with mode 644.
if not env.TargetOSIs('windows'):
for keysuffix in [ "1" , "2" ]:
keyfile = "jstests/libs/key%s" % keysuffix
@ -1999,8 +1983,6 @@ env = doConfigure( env )
# compilation database entries for the configure tests, which is weird.
env.Tool("compilation_db")
env['PDB'] = '${TARGET.base}.pdb'
def checkErrorCodes():
import buildscripts.errorcodes as x
if x.checkErrorCodes() == False:
@ -2009,32 +1991,8 @@ def checkErrorCodes():
checkErrorCodes()
# ---- astyle ----
def doStyling( env , target , source ):
res = utils.execsys( "astyle --version" )
res = " ".join(res)
if res.count( "2." ) == 0:
print( "astyle 2.x needed, found:" + res )
Exit(-1)
files = utils.getAllSourceFiles()
files = filter( lambda x: not x.endswith( ".c" ) , files )
cmd = "astyle --options=mongo_astyle " + " ".join( files )
res = utils.execsys( cmd )
print( res[0] )
print( res[1] )
env.Alias( "style" , [] , [ doStyling ] )
env.AlwaysBuild( "style" )
# --- lint ----
def doLint( env , target , source ):
import buildscripts.lint
if not buildscripts.lint.run_lint( [ "src/mongo/" ] ):
@ -2052,20 +2010,10 @@ def getSystemInstallName():
n = env.GetTargetOSName() + "-" + arch_name
if has_option("nostrip"):
n += "-debugsymbols"
if env.TargetOSIs('posix') and os.uname()[2].startswith("8."):
n += "-tiger"
if len(mongo_modules):
n += "-" + "-".join(m.name for m in mongo_modules)
try:
findSettingsSetup()
import settings
if "distmod" in dir(settings):
n = n + "-" + str(settings.distmod)
except:
pass
dn = GetOption("distmod")
if dn and len(dn) > 0:
n = n + "-" + dn
@ -2079,96 +2027,13 @@ if mongoCodeVersion == None:
if has_option('distname'):
distName = GetOption( "distname" )
elif mongoCodeVersion[-1] not in ("+", "-"):
dontReplacePackage = True
distName = mongoCodeVersion
else:
isBuildingLatest = True
distName = utils.getGitBranchString("" , "-") + datetime.date.today().strftime("%Y-%m-%d")
env['SERVER_DIST_BASENAME'] = 'mongodb-%s-%s' % (getSystemInstallName(), distName)
distFile = "${SERVER_ARCHIVE}"
# ---- CONVENIENCE ----
def tabs( env, target, source ):
from subprocess import Popen, PIPE
from re import search, match
diff = Popen( [ "git", "diff", "-U0", "origin", "master" ], stdout=PIPE ).communicate()[ 0 ]
sourceFile = False
for line in diff.split( "\n" ):
if match( "diff --git", line ):
sourceFile = not not search( "\.(h|hpp|c|cpp)\s*$", line )
if sourceFile and match( "\+ *\t", line ):
return True
return False
env.Alias( "checkSource", [], [ tabs ] )
env.AlwaysBuild( "checkSource" )
def gitPush( env, target, source ):
import subprocess
return subprocess.call( [ "git", "push" ] )
env.Alias( "push", [ ".", "smoke", "checkSource" ], gitPush )
env.AlwaysBuild( "push" )
# ---- deploying ---
def s3push(localName, remoteName=None, platformDir=True):
localName = str( localName )
if isBuildingLatest:
remotePrefix = utils.getGitBranchString("-") + "-latest"
else:
remotePrefix = "-" + distName
findSettingsSetup()
import simples3
import settings
s = simples3.S3Bucket( settings.bucket , settings.id , settings.key )
if remoteName is None:
remoteName = localName
name = '%s-%s%s' % (remoteName , getSystemInstallName(), remotePrefix)
lastDotIndex = localName.rfind('.')
if lastDotIndex != -1:
name += localName[lastDotIndex:]
name = name.lower()
if platformDir:
name = platform + "/" + name
print( "uploading " + localName + " to http://s3.amazonaws.com/" + s.name + "/" + name )
if dontReplacePackage:
for ( key , modify , etag , size ) in s.listdir( prefix=name ):
print( "error: already a file with that name, not uploading" )
Exit(2)
s.put( name , open( localName , "rb" ).read() , acl="public-read" );
print( " done uploading!" )
def s3shellpush( env , target , source ):
s3push( "mongo" , "mongo-shell" )
env.Alias( "s3shell" , [ "mongo" ] , [ s3shellpush ] )
env.AlwaysBuild( "s3shell" )
def s3dist( env , target , source ):
s3push( str(source[0]) , "mongodb" )
env.AlwaysBuild(env.Alias( "s3dist" , [ '$SERVER_ARCHIVE' ] , [ s3dist ] ))
# --- an uninstall target ---
if len(COMMAND_LINE_TARGETS) > 0 and 'uninstall' in COMMAND_LINE_TARGETS:
SetOption("clean", 1)
# By inspection, changing COMMAND_LINE_TARGETS here doesn't do
# what we want, but changing BUILD_TARGETS does.
BUILD_TARGETS.remove("uninstall")
BUILD_TARGETS.append("install")
module_sconscripts = moduleconfig.get_module_sconscripts(mongo_modules)
# The following symbols are exported for use in subordinate SConscript files.
@ -2188,7 +2053,6 @@ Export("v8version v8suffix")
Export("boostSuffix")
Export('module_sconscripts')
Export("debugBuild optBuild")
Export("s3push")
Export("wiredtiger")
def injectMongoIncludePaths(thisEnv):
@ -2200,19 +2064,4 @@ env.Alias("compiledb", env.CompilationDatabase('compile_commands.json'))
env.SConscript('src/SConscript', variant_dir='$BUILD_DIR', duplicate=False)
env.SConscript('SConscript.smoke')
def clean_old_dist_builds(env, target, source):
prefix = "mongodb-%s-%s" % (platform, processor)
filenames = sorted(os.listdir("."))
filenames = [x for x in filenames if x.startswith(prefix)]
to_keep = [x for x in filenames if x.endswith(".tgz") or x.endswith(".zip")][-2:]
for filename in [x for x in filenames if x not in to_keep]:
print "removing %s" % filename
try:
shutil.rmtree(filename)
except:
os.remove(filename)
env.Alias("dist_clean", [], [clean_old_dist_builds])
env.AlwaysBuild("dist_clean")
env.Alias('all', ['core', 'tools', 'dbtest', 'unittests', 'file_allocator_bench'])
env.Alias('all', ['core', 'tools', 'dbtest', 'unittests'])

View File

@ -1,5 +1 @@
import os;
def findHacks( un ):
return None

View File

@ -1,40 +0,0 @@
import utils
import os
import shutil
import sys
def go( boost_root ):
OUTPUT = "src/third_party/boost"
if os.path.exists( OUTPUT ):
shutil.rmtree( OUTPUT )
cmd = [ "bcp" , "--scan" , "--boost=%s" % boost_root ]
src = utils.getAllSourceFiles()
cmd += src
cmd.append( OUTPUT )
if not os.path.exists( OUTPUT ):
os.makedirs( OUTPUT )
res = utils.execsys( cmd )
out = open( OUTPUT + "/bcp-out.txt" , 'w' )
out.write( res[0] )
out.close()
out = open( OUTPUT + "/notes.txt" , 'w' )
out.write( "command: " + " ".join( cmd ) )
out.close()
print( res[1] )
if __name__ == "__main__":
if len(sys.argv) == 1:
print( "usage: python %s <boost root directory>" % sys.argv[0] )
sys.exit(1)
go( sys.argv[1] )

View File

@ -1,54 +0,0 @@
@echo off
cls
echo This script builds the boost libs that MongoDB requires on Windows.
echo We assume boost source is in machine's \boost directory.
echo You can get boost at www.boost.org.
echo .
echo Note: you will want boost v1.42 or higher with VS2010.
echo .
echo We assume you have bjam. To build bjam:
echo cd tools\jam\src
echo build.bat
echo .
cd \boost
echo bin\bjam --version
bin\bjam --version
echo .
echo .
echo .
echo About to build release libraries
pause
cls
bin\bjam variant=release runtime-link=static link=static --with-filesystem --with-thread --with-date_time --with-program_options --layout=versioned threading=multi toolset=msvc
echo .
echo .
echo .
echo About to try to move libs from /boost/stage/lib to /boost/lib/
pause
cls
rem bjam makes extra copies without the ver #; we kill those:
del stage\lib\*s.lib
move stage\lib\* lib\
echo .
echo .
echo .
echo About to build debug libraries
pause
cls
bin\bjam variant=debug --with-filesystem --with-thread --with-date_time --with-program_options --layout=versioned threading=multi toolset=msvc
echo .
echo .
echo .
echo About to try to move libs from /boost/stage/lib to /boost/lib/
pause
cls
rem bjam makes extra copies without the ver #; we kill those:
del stage\lib\*-gd.lib
move stage\lib\* lib\
echo Done - try running "dir \boost\lib\"

View File

@ -1,61 +0,0 @@
@echo off
rem 64 bit version
rem address-model=64
rem run
rem bin\bjam --clean
rem if you switch compilers etc.
cls
echo This script builds the (64 bit) boost libs that MongoDB requires on Windows.
echo We assume boost source is in machine's \boost directory.
echo You can get boost at www.boost.org.
echo .
echo Note: you will want boost v1.42 or higher with VS2010.
echo .
echo We assume you have bjam. To build bjam:
echo cd tools\jam\src
echo build.bat
echo .
cd \boost
echo bin\bjam --version
bin\bjam --version
echo .
echo .
echo .
echo About to build release libraries
pause
cls
bin\bjam --build-dir=c:\temp\boost64 address-model=64 variant=release runtime-link=static link=static --with-filesystem --with-thread --with-date_time --with-program_options --layout=versioned threading=multi toolset=msvc
echo .
echo .
echo .
echo About to try to move libs from /boost/stage/lib to /boost/lib/
pause
cls
rem bjam makes extra copies without the ver #; we kill those:
del stage\lib\*s.lib
move stage\lib\* lib\
echo .
echo .
echo .
echo About to build debug libraries
pause
cls
bin\bjam --build-dir=c:\temp\boost64 address-model=64 variant=debug --with-filesystem --with-thread --with-date_time --with-program_options --layout=versioned threading=multi toolset=msvc
echo .
echo .
echo .
echo About to try to move libs from /boost/stage/lib to /boost/lib/
pause
cls
rem bjam makes extra copies without the ver #; we kill those:
del stage\lib\*-gd.lib
move stage\lib\* lib\
echo Done - try running "dir \boost\lib\"

View File

@ -1,107 +0,0 @@
// FileLock.java
import java.io.*;
import java.util.*;
import java.util.concurrent.*;
/**
* "locks" a resource by using the file system as storage
* file has 1 line
* <incarnation> <last ping time in millis>
*/
public class FileLock {
public FileLock( String logicalName )
throws IOException {
_file = new File( "/tmp/java-fileLock-" + logicalName );
_incarnation = "xxx" + Math.random() + "yyy";
if ( ! _file.exists() ) {
FileOutputStream fout = new FileOutputStream( _file );
fout.write( "\n".getBytes() );
fout.close();
}
}
/**
* takes lock
* if someone else has it, blocks until the other one finishes
*/
public void lock()
throws IOException {
if ( _lock != null )
throw new IllegalStateException( "can't lock when you're locked" );
try {
_semaphore.acquire();
}
catch ( InterruptedException ie ) {
throw new RuntimeException( "sad" , ie );
}
_raf = new RandomAccessFile( _file , "rw" );
_lock = _raf.getChannel().lock();
}
public void unlock()
throws IOException {
if ( _lock == null )
throw new IllegalStateException( "can't unlock when you're not locked" );
_lock.release();
_semaphore.release();
_locked = false;
}
final File _file;
final String _incarnation;
private RandomAccessFile _raf;
private java.nio.channels.FileLock _lock;
private boolean _locked;
private static Semaphore _semaphore = new Semaphore(1);
public static void main( final String[] args )
throws Exception {
List<Thread> threads = new ArrayList<Thread>();
for ( int i=0; i<3; i++ ) {
threads.add( new Thread() {
public void run() {
try {
FileLock lock = new FileLock( args[0] );
long start = System.currentTimeMillis();
lock.lock();
System.out.println( "time to lock:\t" + (System.currentTimeMillis()-start) );
Thread.sleep( Integer.parseInt( args[1] ) );
lock.unlock();
System.out.println( "total time:\t" + (System.currentTimeMillis()-start) );
}
catch ( Exception e ) {
e.printStackTrace();
}
}
} );
}
for ( Thread t : threads ) {
t.start();
}
for ( Thread t : threads ) {
t.join();
}
}
}

View File

@ -1,156 +0,0 @@
// IOUtil.java
import java.io.*;
import java.net.*;
import java.util.*;
public class IOUtil {
public static String urlFileName( String url ) {
int idx = url.lastIndexOf( "/" );
if ( idx < 0 )
return url;
return url.substring( idx + 1 );
}
public static long pipe( InputStream in , OutputStream out )
throws IOException {
long bytes = 0;
byte[] buf = new byte[2048];
while ( true ) {
int x = in.read( buf );
if ( x < 0 )
break;
bytes += x;
out.write( buf , 0 , x );
}
return bytes;
}
public static class PipingThread extends Thread {
public PipingThread( InputStream in , OutputStream out ) {
_in = in;
_out = out;
_wrote = 0;
}
public void run() {
try {
_wrote = pipe( _in , _out );
}
catch ( IOException ioe ) {
ioe.printStackTrace();
_wrote = -1;
}
}
public long wrote() {
return _wrote;
}
long _wrote;
final InputStream _in;
final OutputStream _out;
}
public static String readStringFully( InputStream in )
throws IOException {
ByteArrayOutputStream bout = new ByteArrayOutputStream();
pipe( in , bout );
return new String( bout.toByteArray() , "UTF8" );
}
public static Map<String,Object> readPythonSettings( File file )
throws IOException {
String all = readStringFully( new FileInputStream( file ) );
Map<String,Object> map = new TreeMap<String,Object>();
for ( String line : all.split( "\n" ) ) {
line = line.trim();
if ( line.length() == 0 )
continue;
String[] pcs = line.split( "=" );
if ( pcs.length != 2 )
continue;
String name = pcs[0].trim();
String value = pcs[1].trim();
if ( value.startsWith( "\"" ) ) {
map.put( name , value.substring( 1 , value.length() - 1 ) );
}
else {
map.put( name , Long.parseLong( value ) );
}
}
return map;
}
public static String[] runCommand( String cmd , File dir )
throws IOException {
Process p = Runtime.getRuntime().exec( cmd.split( " +" ) , new String[]{} , dir );
String[] results = new String[]{ IOUtil.readStringFully( p.getInputStream() ) , IOUtil.readStringFully( p.getErrorStream() ) };
try {
if ( p.waitFor() != 0 )
throw new RuntimeException( "command failed [" + cmd + "]\n" + results[0] + "\n" + results[1] );
}
catch ( InterruptedException ie ) {
throw new RuntimeException( "uh oh" );
}
return results;
}
public static void download( String http , File localDir )
throws IOException {
File f = localDir;
f.mkdirs();
f = new File( f.toString() + File.separator + urlFileName( http ) );
System.out.println( "downloading\n\t" + http + "\n\t" + f );
if ( f.exists() ) {
System.out.println( "\t already exists" );
return;
}
URL url = new URL( http );
InputStream in = url.openConnection().getInputStream();
OutputStream out = new FileOutputStream( f );
pipe( in , out );
out.close();
in.close();
}
public static void main( String[] args )
throws Exception {
byte[] data = new byte[]{ 'e' , 'r' , 'h' , 0 };
System.out.write( data );
System.out.println( "yo" );
}
}

View File

@ -1,2 +0,0 @@
Manifest-Version: 1.0
Main-Class: emr

View File

@ -1,380 +0,0 @@
// emr.java
import java.io.*;
import java.util.*;
import java.net.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapred.*;
import org.apache.hadoop.fs.*;
public class emr {
static class MongoSuite {
String mongo;
String code;
String workingDir;
String suite;
void copy( MongoSuite c ) {
mongo = c.mongo;
code = c.code;
workingDir = c.workingDir;
suite = c.suite;
}
void downloadTo( File localDir )
throws IOException {
IOUtil.download( mongo , localDir );
IOUtil.download( code , localDir );
}
boolean runTest()
throws IOException {
// mkdir
File dir = new File( workingDir , suite );
dir.mkdirs();
// download
System.out.println( "going to download" );
downloadTo( dir );
// explode
System.out.println( "going to explode" );
IOUtil.runCommand( "tar zxvf " + IOUtil.urlFileName( code ) , dir );
String[] res = IOUtil.runCommand( "tar zxvf " + IOUtil.urlFileName( mongo ) , dir );
for ( String x : res[0].split( "\n" ) ) {
if ( x.indexOf( "/bin/" ) < 0 )
continue;
File f = new File( dir.toString() , x );
if ( ! f.renameTo( new File( dir , IOUtil.urlFileName( x ) ) ) )
throw new RuntimeException( "rename failed" );
}
List<String> cmd = new ArrayList<String>();
cmd.add( "/usr/bin/python" );
cmd.add( "buildscripts/smoke.py" );
File log_config = new File( dir , "log_config.py" );
System.out.println( "log_config: " + log_config.exists() );
if ( log_config.exists() ) {
java.util.Map<String,Object> properties = IOUtil.readPythonSettings( log_config );
cmd.add( "--buildlogger-builder" );
cmd.add( properties.get( "name" ).toString() );
cmd.add( "--buildlogger-buildnum" );
cmd.add( properties.get( "number" ).toString() );
cmd.add( "--buildlogger-credentials" );
cmd.add( "log_config.py" );
cmd.add( "--buildlogger-phase" );
{
int idx = suite.lastIndexOf( "/" );
if ( idx < 0 )
cmd.add( suite );
else
cmd.add( suite.substring( 0 , idx ) );
}
}
cmd.add( suite );
System.out.println( cmd );
Process p = Runtime.getRuntime().exec( cmd.toArray( new String[cmd.size()] ) , new String[]{} , dir );
List<Thread> threads = new ArrayList<Thread>();
threads.add( new IOUtil.PipingThread( p.getInputStream() , System.out ) );
threads.add( new IOUtil.PipingThread( p.getErrorStream() , System.out ) );
for ( Thread t : threads )
t.start();
try {
for ( Thread t : threads ) {
t.join();
}
int rc = p.waitFor();
return rc == 0;
}
catch ( InterruptedException ie ) {
ie.printStackTrace();
throw new RuntimeException( "sad" , ie );
}
}
public void readFields( DataInput in )
throws IOException {
mongo = in.readUTF();
code = in.readUTF();
workingDir = in.readUTF();
suite = in.readUTF();
}
public void write( final DataOutput out )
throws IOException {
out.writeUTF( mongo );
out.writeUTF( code );
out.writeUTF( workingDir );
out.writeUTF( suite );
}
public String toString() {
return "mongo: " + mongo + " code: " + code + " suite: " + suite + " workingDir: " + workingDir;
}
}
public static class Map implements Mapper<Text, MongoSuite, Text, IntWritable> {
public void map( Text key, MongoSuite value, OutputCollector<Text,IntWritable> output, Reporter reporter )
throws IOException {
FileLock lock = new FileLock( "mapper" );
try {
lock.lock();
System.out.println( "key: " + key );
System.out.println( "value: " + value );
long start = System.currentTimeMillis();
boolean passed = value.runTest();
long end = System.currentTimeMillis();
output.collect( new Text( passed ? "passed" : "failed" ) , new IntWritable( 1 ) );
output.collect( new Text( key.toString() + "-time-seconds" ) , new IntWritable( (int)((end-start)/(1000)) ) );
output.collect( new Text( key.toString() + "-passed" ) , new IntWritable( passed ? 1 : 0 ) );
String ip = IOUtil.readStringFully( new URL( "http://myip.10gen.com/" ).openConnection().getInputStream() );
ip = ip.substring( ip.indexOf( ":" ) + 1 ).trim();
output.collect( new Text( ip ) , new IntWritable(1) );
}
catch ( RuntimeException re ) {
re.printStackTrace();
throw re;
}
catch ( IOException ioe ) {
ioe.printStackTrace();
throw ioe;
}
finally {
lock.unlock();
}
}
public void configure(JobConf job) {}
public void close(){}
}
public static class Reduce implements Reducer<Text, IntWritable, Text, IntWritable> {
public void reduce( Text key, Iterator<IntWritable> values, OutputCollector<Text,IntWritable> output , Reporter reporter )
throws IOException {
int sum = 0;
while ( values.hasNext() ) {
sum += values.next().get();
}
output.collect( key , new IntWritable( sum ) );
}
public void configure(JobConf job) {}
public void close(){}
}
public static class MySplit implements InputSplit , Writable {
public MySplit(){
}
MySplit( MongoSuite config , int length ) {
_config = config;
_length = length;
}
public long getLength() {
return _length;
}
public String[] getLocations() {
return new String[0];
}
public void readFields( DataInput in )
throws IOException {
_config = new MongoSuite();
_config.readFields( in );
_length = in.readInt();
}
public void write( final DataOutput out )
throws IOException {
_config.write( out );
out.writeInt( _length );
}
MongoSuite _config;
int _length;
}
public static class InputMagic implements InputFormat<Text,MongoSuite> {
public RecordReader<Text,MongoSuite> getRecordReader( InputSplit split, JobConf job , Reporter reporter ){
final MySplit s = (MySplit)split;
return new RecordReader<Text,MongoSuite>() {
public void close(){}
public Text createKey() {
return new Text();
}
public MongoSuite createValue() {
return new MongoSuite();
}
public long getPos() {
return _seen ? 1 : 0;
}
public float getProgress() {
return getPos();
}
public boolean next( Text key , MongoSuite value ) {
key.set( s._config.suite );
value.copy( s._config );
boolean x = _seen;
_seen = true;
return !x;
}
boolean _seen = false;
};
}
public InputSplit[] getSplits( JobConf job , int numSplits ){
String[] pcs = job.get( "suites" ).split(",");
InputSplit[] splits = new InputSplit[pcs.length];
for ( int i=0; i<splits.length; i++ ) {
MongoSuite c = new MongoSuite();
c.suite = pcs[i];
c.mongo = job.get( "mongo" );
c.code = job.get( "code" );
c.workingDir = job.get( "workingDir" );
splits[i] = new MySplit( c , 100 /* XXX */);
}
return splits;
}
public void validateInput(JobConf job){}
}
/**
* args
* mongo tgz
* code tgz
* output path
* tests to run ?
*/
public static void main( String[] args ) throws Exception{
JobConf conf = new JobConf();
conf.setJarByClass(emr.class);
String workingDir = "/data/db/emr/";
// parse args
int pos = 0;
for ( ; pos < args.length; pos++ ) {
if ( ! args[pos].startsWith( "--" ) )
break;
String arg = args[pos].substring(2);
if ( arg.equals( "workingDir" ) ) {
workingDir = args[++pos];
}
else {
System.err.println( "unknown arg: " + arg );
throw new RuntimeException( "unknown arg: " + arg );
}
}
String mongo = args[pos++];
String code = args[pos++];
String output = args[pos++];
String suites = "";
for ( ; pos < args.length; pos++ ) {
if ( suites.length() > 0 )
suites += ",";
suites += args[pos];
}
if ( suites.length() == 0 )
throw new RuntimeException( "no suites" );
System.out.println( "workingDir:\t" + workingDir );
System.out.println( "mongo:\t" + mongo );
System.out.println( "code:\t " + code );
System.out.println( "output\t: " + output );
System.out.println( "suites\t: " + suites );
if ( false ) {
MongoSuite s = new MongoSuite();
s.mongo = mongo;
s.code = code;
s.workingDir = workingDir;
s.suite = suites;
s.runTest();
return;
}
// main hadoop set
conf.set( "mongo" , mongo );
conf.set( "code" , code );
conf.set( "workingDir" , workingDir );
conf.set( "suites" , suites );
conf.set( "mapred.map.tasks" , "1" );
conf.setLong( "mapred.task.timeout" , 4 * 3600 * 1000 /* 4 hours */);
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(IntWritable.class);
conf.setMapperClass(Map.class);
conf.setReducerClass(Reduce.class);
conf.setInputFormat(InputMagic.class);
conf.setOutputFormat(TextOutputFormat.class);
FileOutputFormat.setOutputPath(conf, new Path(output) );
// actually run
JobClient.runJob( conf );
}
}

View File

@ -1,385 +0,0 @@
import os
import sys
import shutil
import datetime
import time
import subprocess
import urllib
import urllib2
import json
import pprint
import boto
import simples3
import pymongo
def findSettingsSetup():
sys.path.append( "./" )
sys.path.append( "../" )
sys.path.append( "../../" )
sys.path.append( "../../../" )
findSettingsSetup()
import settings
import buildscripts.utils as utils
import buildscripts.smoke as smoke
bucket = simples3.S3Bucket( settings.emr_bucket , settings.emr_id , settings.emr_key )
def _get_status():
def gh( cmds ):
txt = ""
for cmd in cmds:
res = utils.execsys( "git " + cmd )
txt = txt + res[0] + res[1]
return utils.md5string( txt )
return "%s-%s" % ( utils.execsys( "git describe" )[0].strip(), gh( [ "diff" , "status" ] ) )
def _get_most_recent_tgz( prefix ):
# this is icky, but works for now
all = []
for x in os.listdir( "." ):
if not x.startswith( prefix ) or not x.endswith( ".tgz" ):
continue
all.append( ( x , os.stat(x).st_mtime ) )
if len(all) == 0:
raise Exception( "can't find file with prefix: " + prefix )
all.sort( lambda x,y: int(y[1] - x[1]) )
return all[0][0]
def get_build_info():
return ( os.environ.get('MONGO_BUILDER_NAME') , os.environ.get('MONGO_BUILD_NUMBER') )
def make_tarball():
m = _get_most_recent_tgz( "mongodb-" )
c = "test-code-emr.tgz"
tar = "tar zcf %s src jstests buildscripts" % c
log_config = "log_config.py"
if os.path.exists( log_config ):
os.unlink( log_config )
credentials = do_credentials()
if credentials:
builder , buildnum = get_build_info()
if builder and buildnum:
file = open( log_config , "wb" )
file.write( 'username="%s"\npassword="%s"\n' % credentials )
file.write( 'name="%s"\nnumber=%s\n'% ( builder , buildnum ) )
file.close()
tar = tar + " " + log_config
utils.execsys( tar )
return ( m , c )
def _put_ine( bucket , local , remote ):
print( "going to put\n\t%s\n\thttp://%s.s3.amazonaws.com/%s" % ( local , settings.emr_bucket , remote ) )
for x in bucket.listdir( prefix=remote ):
print( "\talready existed" )
return remote
bucket.put( remote , open( local , "rb" ).read() , acl="public-read" )
return remote
def build_jar():
root = "build/emrjar"
src = "buildscripts/emr"
if os.path.exists( root ):
shutil.rmtree( root )
os.makedirs( root )
for x in os.listdir( src ):
if not x.endswith( ".java" ):
continue
shutil.copyfile( src + "/" + x , root + "/" + x )
shutil.copyfile( src + "/MANIFEST.MF" , root + "/MANIFEST.FM" )
classpath = os.listdir( src + "/lib" )
for x in classpath:
shutil.copyfile( src + "/lib/" + x , root + "/" + x )
classpath.append( "." )
classpath = ":".join(classpath)
for x in os.listdir( root ):
if x.endswith( ".java" ):
if subprocess.call( [ "javac" , "-cp" , classpath , x ] , cwd=root) != 0:
raise Exception( "compiled failed" )
args = [ "jar" , "-cfm" , "emr.jar" , "MANIFEST.FM" ]
for x in os.listdir( root ):
if x.endswith( ".class" ):
args.append( x )
subprocess.call( args , cwd=root )
shutil.copyfile( root + "/emr.jar" , "emr.jar" )
return "emr.jar"
def push():
mongo , test_code = make_tarball()
print( mongo )
print( test_code )
root = "emr/%s/%s" % ( datetime.date.today().strftime("%Y-%m-%d") , os.uname()[0].lower() )
def make_long_name(local,hash):
pcs = local.rpartition( "." )
h = _get_status()
if hash:
h = utils.md5sum( local )
return "%s/%s-%s.%s" % ( root , pcs[0] , h , pcs[2] )
mongo = _put_ine( bucket , mongo , make_long_name( mongo , False ) )
test_code = _put_ine( bucket , test_code , make_long_name( test_code , True ) )
jar = build_jar()
jar = _put_ine( bucket , jar , make_long_name( jar , False ) )
setup = "buildscripts/emr/emrnodesetup.sh"
setup = _put_ine( bucket , setup , make_long_name( setup , True ) )
return mongo , test_code , jar , setup
def run_tests( things , tests ):
if len(tests) == 0:
raise Exception( "no tests" )
oldNum = len(tests)
tests = fix_suites( tests )
print( "tests expanded from %d to %d" % ( oldNum , len(tests) ) )
print( "things:%s\ntests:%s\n" % ( things , tests ) )
emr = boto.connect_emr( settings.emr_id , settings.emr_key )
def http(path):
return "http://%s.s3.amazonaws.com/%s" % ( settings.emr_bucket , path )
run_s3_path = "emr/%s/%s/%s/" % ( os.getenv( "USER" ) ,
os.getenv( "HOST" ) ,
datetime.datetime.today().strftime( "%Y%m%d-%H%M" ) )
run_s3_root = "s3n://%s/%s/" % ( settings.emr_bucket , run_s3_path )
out = run_s3_root + "out"
logs = run_s3_root + "logs"
jar="s3n://%s/%s" % ( settings.emr_bucket , things[2] )
step_args=[ http(things[0]) , http(things[1]) , out , ",".join(tests) ]
step = boto.emr.step.JarStep( "emr main" , jar=jar,step_args=step_args )
print( "jar:%s\nargs:%s" % ( jar , step_args ) )
setup = boto.emr.BootstrapAction( "setup" , "s3n://%s/%s" % ( settings.emr_bucket , things[3] ) , [] )
jobid = emr.run_jobflow( name = "Mongo EMR for %s from %s" % ( os.getenv( "USER" ) , os.getenv( "HOST" ) ) ,
ec2_keyname = "emr1" ,
slave_instance_type = "m1.large" ,
ami_version = "latest" ,
num_instances=5 ,
log_uri = logs ,
bootstrap_actions = [ setup ] ,
steps = [ step ] )
print( "%s jobid: %s" % ( datetime.datetime.today() , jobid ) )
while ( True ):
flow = emr.describe_jobflow( jobid )
print( "%s status: %s" % ( datetime.datetime.today() , flow.state ) )
if flow.state == "COMPLETED" or flow.state == "FAILED":
break
time.sleep(30)
syncdir = "build/emrout/" + jobid + "/"
sync_s3( run_s3_path , syncdir )
final_out = "build/emrout/" + jobid + "/"
print("output in: " + final_out )
do_output( final_out )
def sync_s3( remote_dir , local_dir ):
for x in bucket.listdir( remote_dir ):
out = local_dir + "/" + x[0]
if os.path.exists( out ) and x[2].find( utils.md5sum( out ) ) >= 0:
continue
dir = out.rpartition( "/" )[0]
if not os.path.exists( dir ):
os.makedirs( dir )
thing = bucket.get( x[0] )
open( out , "wb" ).write( thing.read() )
def fix_suites( suites ):
fixed = []
for name,x in smoke.expand_suites( suites , False ):
idx = name.find( "/jstests" )
if idx >= 0:
name = name[idx+1:]
fixed.append( name )
return fixed
def do_credentials():
root = "buildbot.tac"
while len(root) < 40 :
if os.path.exists( root ):
break
root = "../" + root
if not os.path.exists( root ):
return None
credentials = {}
execfile(root, credentials, credentials)
if "slavename" not in credentials:
return None
if "passwd" not in credentials:
return None
return ( credentials["slavename"] , credentials["passwd"] )
def do_output( dir ):
def go_down( start ):
lst = os.listdir(dir)
if len(lst) != 1:
raise Exception( "sad: " + start )
return start + "/" + lst[0]
while "out" not in os.listdir( dir ):
dir = go_down( dir )
dir = dir + "/out"
pieces = os.listdir(dir)
pieces.sort()
passed = []
failed = []
times = {}
for x in pieces:
if not x.startswith( "part" ):
continue
full = dir + "/" + x
for line in open( full , "rb" ):
if line.find( "-passed" ) >= 0:
passed.append( line.partition( "-passed" )[0] )
continue
if line.find( "-failed" ) >= 0:
failed.append( line.partition( "-failed" )[0] )
continue
if line.find( "-time-seconds" ) >= 0:
p = line.partition( "-time-seconds" )
times[p[0]] = p[2].strip()
continue
print( "\t" + line.strip() )
def print_list(name,lst):
print( name )
for x in lst:
print( "\t%s\t%s" % ( x , times[x] ) )
print_list( "passed" , passed )
print_list( "failed" , failed )
if do_credentials():
builder , buildnum = get_build_info()
if builder and buildnum:
conn = pymongo.Connection( "bbout1.10gen.cc" )
db = conn.buildlogs
q = { "builder" : builder , "buildnum" : int(buildnum) }
doc = db.builds.find_one( q )
if doc:
print( "\nhttp://buildlogs.mongodb.org/build/%s" % doc["_id"] )
if __name__ == "__main__":
if len(sys.argv) == 1:
print( "need an arg" )
elif sys.argv[1] == "tarball":
make_tarball()
elif sys.argv[1] == "jar":
build_jar()
elif sys.argv[1] == "push":
print( push() )
elif sys.argv[1] == "sync":
sync_s3( sys.argv[2] , sys.argv[3] )
elif sys.argv[1] == "fix_suites":
for x in fix_suites( sys.argv[2:] ):
print(x)
elif sys.argv[1] == "credentials":
print( do_credentials() )
elif sys.argv[1] == "test":
m , c = make_tarball()
build_jar()
cmd = [ "java" , "-cp" , os.environ.get( "CLASSPATH" , "." ) + ":emr.jar" , "emr" ]
workingDir = "/data/emr/test"
cmd.append( "--workingDir" )
cmd.append( workingDir )
if os.path.exists( workingDir ):
shutil.rmtree( workingDir )
cmd.append( "file://" + os.getcwd() + "/" + m )
cmd.append( "file://" + os.getcwd() + "/" + c )
out = "/tmp/emrresults"
cmd.append( out )
if os.path.exists( out ):
shutil.rmtree( out )
cmd.append( "jstests/basic1.js" )
subprocess.call( cmd )
for x in os.listdir( out ):
if x.startswith( "." ):
continue
print( x )
for z in open( out + "/" + x ):
print( "\t" + z.strip() )
elif sys.argv[1] == "output":
do_output( sys.argv[2] )
elif sys.argv[1] == "full":
things = push()
run_tests( things , sys.argv[2:] )
else:
things = push()
run_tests( things , sys.argv[1:] )

View File

@ -1,7 +0,0 @@
#!/bin/sh
sudo mkdir /mnt/data
sudo ln -s /mnt/data /data
sudo chown hadoop /mnt/data
sudo easy_install pymongo

View File

@ -1,202 +0,0 @@
#!/bin/sh -e
#
# Copyright (c) 2005-2011 The OpenSSL Project.
#
# Depending on output file name, the script either embeds fingerprint
# into libcrypto.so or static application. "Static" refers to static
# libcrypto.a, not [necessarily] application per se.
#
# Even though this script is called fipsld, it expects C compiler
# command line syntax and $FIPSLD_CC or $CC environment variable set
# and can even be used to compile source files.
#set -x
# Note: edit this to point to your copy of g++ if this is not correct.
CC="/usr/bin/g++"
# Initially -c wasn't intended to be interpreted here, but it might
# make life easier for those who want to build FIPS-ified applications
# with minimal [if any] modifications to their Makefiles...
( while [ "x$1" != "x" -a "x$1" != "x-c" -a "x$1" != "x-E" ]; do shift; done;
[ $# -ge 1 ]
) && exec ${CC} "$@"
TARGET=`(while [ "x$1" != "x" -a "x$1" != "x-o" ]; do shift; done; echo $2)`
# If using an auto-tooled (autoconf/automake/libtool) project,
# configure will fail when testing the compiler or even performing
# simple checks. Pass-through to compiler directly if application is
# is not being linked with libcrypto, allowing auto-tooled applications
# to utilize fipsld (e.g. CC=/usr/local/ssl/bin/fipsld FIPSLD_CC=gcc
# ./configure && make). But keep in mind[!] that if certified code
# resides in a shared library, then fipsld *may not* be used and
# end-developer should not modify application configuration and build
# procedures. This is because in-core fingerprint and associated
# procedures are already embedded into and executed in shared library
# context.
case `basename "${TARGET}"` in
libcrypto*|libfips*|*.dll) ;;
*) case "$*" in
*libcrypto.a*|*-lcrypto*|*fipscanister.o*) ;;
*) exec ${CC} "$@" ;;
esac
esac
[ -n "${TARGET}" ] || { echo 'no -o specified'; exit 1; }
# Turn on debugging output?
( while [ "x$1" != "x" -a "x$1" != "x-DDEBUG_FINGERPRINT_PREMAIN" ]; do shift; done;
[ $# -ge 1 ]
) && set -x
THERE="`echo $0 | sed -e 's|[^/]*$||'`"..
# fipscanister.o can appear in command line
CANISTER_O=`(while [ "x$1" != "x" ]; do case "$1" in *fipscanister.o) echo $1; exit;; esac; shift; done)`
if [ -z "${CANISTER_O}" ]; then
# If set, FIPSLIBDIR is location of installed validated FIPS module
if [ -n "${FIPSLIBDIR}" ]; then
CANISTER_O="${FIPSLIBDIR}/fipscanister.o"
elif [ -f "${THERE}/fips/fipscanister.o" ]; then
CANISTER_O="${THERE}/fips/fipscanister.o"
elif [ -f "${THERE}/lib/fipscanister.o" ]; then
CANISTER_O="${THERE}/lib/fipscanister.o"
fi
CANISTER_O_CMD="${CANISTER_O}"
fi
[ -f ${CANISTER_O} ] || { echo "unable to find ${CANISTER_O}"; exit 1; }
PREMAIN_C=`dirname "${CANISTER_O}"`/fips_premain.c
PREMAIN_O=`dirname "${CANISTER_O}"`/fips_premain.o
HMAC_KEY="etaonrishdlcupfm"
case "`(uname -s) 2>/dev/null`" in
OSF1|IRIX*) _WL_PREMAIN="-Wl,-init,FINGERPRINT_premain" ;;
HP-UX) _WL_PREMAIN="-Wl,+init,FINGERPRINT_premain" ;;
AIX) _WL_PREMAIN="-Wl,-binitfini:FINGERPRINT_premain,-bnoobjreorder";;
Darwin) ( while [ "x$1" != "x" -a "x$1" != "x-dynamiclib" ]; do shift; done;
[ $# -ge 1 ]
) && _WL_PREMAIN="-Wl,-init,_FINGERPRINT_premain" ;;
esac
case "${TARGET}" in
[!/]*) TARGET=./${TARGET} ;;
esac
case `basename "${TARGET}"` in
lib*|*.dll) # must be linking a shared lib...
# Shared lib creation can be taking place in the source
# directory only, but fipscanister.o can reside elsewhere...
if [ -x "${THERE}/fips/fips_standalone_sha1" ]; then
FINGERTYPE="${THERE}/fips/fips_standalone_sha1"
PREMAIN_DSO="${THERE}/fips/fips_premain_dso"
elif [ -x "${THERE}/bin/fips_standalone_sha1" ]; then
FINGERTYPE="${THERE}/bin/fips_standalone_sha1"
PREMAIN_DSO="./fips_premain_dso"
fi
# verify fipspremain.c against its detached signature...
${FINGERTYPE} "${PREMAIN_C}" | sed "s/(.*\//(/" | \
diff -w "${PREMAIN_C}.sha1" - || \
{ echo "${PREMAIN_C} fingerprint mismatch"; exit 1; }
# verify fipscanister.o against its detached signature...
${FINGERTYPE} "${CANISTER_O}" | sed "s/(.*\//(/" | \
diff -w "${CANISTER_O}.sha1" - || \
{ echo "${CANISTER_O} fingerprint mismatch"; exit 1; }
[ -z "${FIPSLD_LIBCRYPTO}" -a -f "${THERE}/libcrypto.a" ] && \
FIPSLD_LIBCRYPTO="${THERE}/libcrypto.a"
# Temporarily remove fipscanister.o from libcrypto.a!
# We are required to use the standalone copy...
if [ -n "${FIPSLD_LIBCRYPTO}" ]; then
if ar d "${FIPSLD_LIBCRYPTO}" fipscanister.o; then
(ranlib "${FIPSLD_LIBCRYPTO}") 2>/dev/null || :
trap 'ar r "${FIPSLD_LIBCRYPTO}" "${CANISTER_O}";
(ranlib "${FIPSLD_LIBCRYPTO}") 2>/dev/null || :;
sleep 1;
touch -c "${TARGET}"' 0
fi
fi
/bin/rm -f "${TARGET}"
${CC} -x c "${PREMAIN_C}" -c -o "${PREMAIN_O}"
${CC} ${CANISTER_O_CMD:+"${CANISTER_O_CMD}"} \
"${PREMAIN_O}" \
${_WL_PREMAIN} "$@"
if [ "x${FIPS_SIG}" != "x" ]; then
# embed signature
"${FIPS_SIG}" "${TARGET}"
[ $? -ne 42 ] && exit $?
fi
# generate signature...
SIG=`"${PREMAIN_DSO}" "${TARGET}"`
/bin/rm -f "${TARGET}"
if [ -z "${SIG}" ]; then
echo "unable to collect signature"; exit 1
fi
# recompile with signature...
${CC} -x c -DHMAC_SHA1_SIG=\"${SIG}\" "${PREMAIN_C}" -c -o "${PREMAIN_O}"
${CC} ${CANISTER_O_CMD:+"${CANISTER_O_CMD}"} \
-DHMAC_SHA1_SIG=\"${SIG}\" "${PREMAIN_O}" \
${_WL_PREMAIN} "$@"
;;
*) # must be linking statically...
# Static linking can be taking place either in the source
# directory or off the installed binary target destination.
if [ -x "${THERE}/fips/fips_standalone_sha1" ]; then
FINGERTYPE="${THERE}/fips/fips_standalone_sha1"
elif [ -x "${THERE}/bin/fips_standalone_sha1" ]; then
FINGERTYPE="${THERE}/bin/fips_standalone_sha1"
else # Installed tree is expected to contain
# lib/fipscanister.o, lib/fipscanister.o.sha1 and
# lib/fips_premain.c [not to mention bin/openssl].
FINGERTYPE="openssl sha1 -hmac ${HMAC_KEY}"
fi
# verify fipscanister.o against its detached signature...
${FINGERTYPE} "${CANISTER_O}" | sed "s/(.*\//(/" | \
diff -w "${CANISTER_O}.sha1" - || \
{ echo "${CANISTER_O} fingerprint mismatch"; exit 1; }
# verify fips_premain.c against its detached signature...
${FINGERTYPE} "${PREMAIN_C}" | sed "s/(.*\//(/" | \
diff -w "${PREMAIN_C}.sha1" - || \
{ echo "${PREMAIN_C} fingerprint mismatch"; exit 1; }
/bin/rm -f "${TARGET}"
${CC} -x c "${PREMAIN_C}" -c -o "${PREMAIN_O}"
${CC} ${CANISTER_O_CMD:+"${CANISTER_O_CMD}"} \
"${PREMAIN_O}" \
${_WL_PREMAIN} "$@"
if [ "x${FIPS_SIG}" != "x" ]; then
# embed signature
"${FIPS_SIG}" "${TARGET}"
[ $? -ne 42 ] && exit $?
fi
# generate signature...
SIG=`"${TARGET}"`
/bin/rm -f "${TARGET}"
if [ -z "${SIG}" ]; then
echo "unable to collect signature"; exit 1
fi
# recompile with signature...
${CC} -x c -DHMAC_SHA1_SIG=\"${SIG}\" "${PREMAIN_C}" -c -o "${PREMAIN_O}"
${CC} ${CANISTER_O_CMD:+"${CANISTER_O_CMD}"} \
-DHMAC_SHA1_SIG=\"${SIG}\" "${PREMAIN_O}" \
${_WL_PREMAIN} "$@"
;;
esac

View File

@ -1,69 +0,0 @@
#!/usr/bin/env python
from __future__ import with_statement
import tempfile
import sys
import re
import os
def opentemp(basename):
# The following doesn't work in python before 2.6
# return tempfile.NamedTemporaryFile('w', -1, ".XXXXXX", basename, '.', False)
fname = basename +".TMP"
if os.path.exists(fname):
raise "not clobbering file %s" % fname
return open(fname, 'w')
def frob_debian_changelog(version):
fname = 'debian/changelog'
with opentemp(fname) as o:
with open(fname) as i:
lineno = 0
for line in i:
if lineno == 0:
newline = re.sub(r'\([^)]*\)', '('+version+')', line)
o.write(newline)
else:
o.write(line)
os.rename(o.name, fname)
def frob_rpm_spec(version):
fname = 'rpm/mongo.spec'
with opentemp(fname) as o:
with open(fname) as i:
frobbed = False
for line in i:
if frobbed:
o.write(line)
else:
if line.find('Version:') == 0:
print >> o, 'Version: ' + version
frobbed = True
else:
o.write(line)
os.rename(o.name, fname)
def frob_stdafx_cpp(version):
fname = 'stdafx.cpp'
with opentemp(fname) as o:
with open(fname) as i:
frobbed = False
for line in i:
if frobbed:
o.write(line)
else:
if re.search(r'const.*char.*versionString\[\].*=', line):
o.write(' const char versionString[] = "%s";' % version)
else:
o.write(line)
os.rename(o.name, fname)
(progname, version) = sys.argv
if version is None:
print >> sys.stderr, 'usage: %s VERSION' % progname
sys.exit(1)
frob_debian_changelog(version)
frob_rpm_spec(version)
## I don't yet know what-all cares about the versionString inside the
## mongo code, so I'm not actually calling this yet.
# frob_stdafx_cpp(version)

View File

@ -1,60 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">x86</Platform>
<ProductVersion>3.5</ProductVersion>
<ProjectGuid>{fc40ea06-5d8c-4edf-9e19-a0bdd9a3a7d5}</ProjectGuid>
<SchemaVersion>2.0</SchemaVersion>
<OutputName>MongoDB_$(Version)_x86_Standard</OutputName>
<OutputType>Package</OutputType>
<DefineSolutionProperties>false</DefineSolutionProperties>
<WixTargetsPath Condition=" '$(WixTargetsPath)' == '' AND '$(MSBuildExtensionsPath32)' != '' ">$(MSBuildExtensionsPath32)\Microsoft\WiX\v3.x\Wix.targets</WixTargetsPath>
<WixTargetsPath Condition=" '$(WixTargetsPath)' == '' ">$(MSBuildExtensionsPath)\Microsoft\WiX\v3.x\Wix.targets</WixTargetsPath>
<ProductId Condition=" '$(ProductId)' == '' ">*</ProductId>
<UpgradeCode Condition=" '$(UpgradeCode)' == '' ">867C1D1D-2040-4E90-B04E-1158F9CBDE96</UpgradeCode>
<Name>MongoDB</Name>
<OutputPath>bin\$(Configuration)\$(Platform)\</OutputPath>
<IntermediateOutputPath>obj\$(Configuration)\$(Platform)\</IntermediateOutputPath>
<TreatWarningsAsErrors>True</TreatWarningsAsErrors>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x86' ">
<VerboseOutput>True</VerboseOutput>
</PropertyGroup>
<PropertyGroup>
<ClientSource Condition=" '$(ClientSource)' == '' ">..\..\..\build\win32\normal\clientlib</ClientSource>
<License Condition=" '$(License)' == '' ">..\..\..\distsrc</License>
<Source Condition=" '$(Source)' == '' ">..\..\..\build\win32\normal\mongo</Source>
<Version Condition=" '$(Version)' == '' ">2.4.0</Version>
<ClientHeaderSource Condition=" '$(ClientHeaderSource)' == '' ">..\..\..\build\win32\normal\clientlib\include</ClientHeaderSource>
<DefineConstants>MongoDBVersion=$(Version);LicenseSource=$(License);BinarySource=$(Source);Edition=Standard;ProductId=$(ProductId);UpgradeCode=$(UpgradeCode);ClientSource=$(ClientSource);ClientHeaderSource=$(ClientHeaderSource)</DefineConstants>
</PropertyGroup>
<ItemGroup>
<Compile Include="..\..\..\src\mongo\installer\msi\wxs\BinaryFragment.wxs" />
<Compile Include="..\..\..\src\mongo\installer\msi\wxs\FeatureFragment.wxs" />
<Compile Include="..\..\..\src\mongo\installer\msi\wxs\LicensingFragment.wxs" />
<Compile Include="..\..\..\src\mongo\installer\msi\wxs\Installer.wxs" />
<Compile Include="$(OutputPath)DriverInclude.wxs" />
</ItemGroup>
<ItemGroup>
<WixExtension Include="WixUIExtension">
<HintPath>$(WixExtDir)\WixUIExtension.dll</HintPath>
<Name>WixUIExtension</Name>
</WixExtension>
</ItemGroup>
<ItemGroup>
<Folder Include="wxs\" />
</ItemGroup>
<Import Project="$(WixTargetsPath)" />
<PropertyGroup>
<PreBuildEvent>"%WIX%\bin\heat.exe" dir $(ClientHeaderSource) -gg -g1 -frag -cg cg_DriverHeaders -nologo -directoryid -out DriverInclude.wxs -dr Header -srd -var var.ClientHeaderSource</PreBuildEvent>
</PropertyGroup>
<!--
To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Wix.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>

View File

@ -1,51 +0,0 @@

Microsoft Visual Studio Solution File, Format Version 11.00
# Visual Studio 2010
Project("{930C7802-8A8C-48F9-8165-68863BCCD9DD}") = "MongoDB", "MongoDB.wixproj", "{FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}"
EndProject
Project("{930C7802-8A8C-48F9-8165-68863BCCD9DD}") = "MongoDB_64", "MongoDB_64.wixproj", "{FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{2E84BEF1-F2D9-4A6B-B102-EC941AF313AF}"
ProjectSection(SolutionItems) = preProject
build32bitmsi.bat = build32bitmsi.bat
build64bit2008R2msi.bat = build64bit2008R2msi.bat
build64bitmsi.bat = build64bitmsi.bat
buildenterprisemsi.bat = buildenterprisemsi.bat
README.md = README.md
EndProjectSection
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Mixed Platforms = Debug|Mixed Platforms
Debug|x64 = Debug|x64
Debug|x86 = Debug|x86
Release|Mixed Platforms = Release|Mixed Platforms
Release|x64 = Release|x64
Release|x86 = Release|x86
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Debug|Mixed Platforms.ActiveCfg = Debug|x86
{FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Debug|Mixed Platforms.Build.0 = Debug|x86
{FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Debug|x64.ActiveCfg = Debug|x86
{FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Debug|x86.ActiveCfg = Debug|x86
{FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Debug|x86.Build.0 = Debug|x86
{FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Release|Mixed Platforms.ActiveCfg = Release|x86
{FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Release|Mixed Platforms.Build.0 = Release|x86
{FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Release|x64.ActiveCfg = Release|x86
{FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Release|x86.ActiveCfg = Release|x86
{FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Release|x86.Build.0 = Release|x86
{FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
{FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Debug|Mixed Platforms.Build.0 = Debug|x64
{FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Debug|x64.ActiveCfg = Debug|x64
{FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Debug|x64.Build.0 = Debug|x64
{FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Debug|x86.ActiveCfg = Debug|x64
{FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Release|Mixed Platforms.ActiveCfg = Release|x64
{FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Release|Mixed Platforms.Build.0 = Release|x64
{FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Release|x64.ActiveCfg = Release|x64
{FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Release|x64.Build.0 = Release|x64
{FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Release|x86.ActiveCfg = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
EndGlobal

View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">x64</Platform>
<ProductVersion>3.5</ProductVersion>
<ProjectGuid>{fa9df7fc-a283-4eb8-b0c6-f9fa31e22cbc}</ProjectGuid>
<SchemaVersion>2.0</SchemaVersion>
<OutputName>MongoDB_$(Version)_x64_$(Flavor)_$(Edition)</OutputName>
<OutputType>Package</OutputType>
<DefineSolutionProperties>false</DefineSolutionProperties>
<WixTargetsPath Condition=" '$(WixTargetsPath)' == '' AND '$(MSBuildExtensionsPath32)' != '' ">$(MSBuildExtensionsPath32)\Microsoft\WiX\v3.x\Wix.targets</WixTargetsPath>
<WixTargetsPath Condition=" '$(WixTargetsPath)' == '' ">$(MSBuildExtensionsPath)\Microsoft\WiX\v3.x\Wix.targets</WixTargetsPath>
<Name>MongoDB</Name>
<OutputPath>bin\$(Configuration)\$(Platform)\$(Flavor)\$(Edition)\</OutputPath>
<IntermediateOutputPath>obj\$(Configuration)\$(Platform)\$(Flavor)\$(Edition)\</IntermediateOutputPath>
<TreatWarningsAsErrors>True</TreatWarningsAsErrors>
<ProductId Condition=" '$(ProductId)' == '' ">*</ProductId>
<UpgradeCode Condition=" '$(UpgradeCode)' == '' ">FCF901F6-E963-40B1-9A17-978242068587</UpgradeCode>
</PropertyGroup>
<PropertyGroup>
<ClientSource Condition=" '$(ClientSource)' == '' ">..\..\..\build\win32\64\client_build</ClientSource>
<Edition Condition=" '$(Edition)' == '' ">Standard</Edition>
<Flavor Condition=" '$(Flavor)' == '' ">2008R2Plus</Flavor>
<License Condition=" '$(License)' == '' ">..\..\..\distsrc</License>
<EnterpriseBase Condition=" '$(EnterpriseBase)' == '' ">..\..\..\src\mongo\db\modules\subscription</EnterpriseBase>
<SaslSource Condition=" '$(SaslSource)' == '' ">..\..\..\build\win32\64\mongo</SaslSource>
<SnmpSource Condition=" '$(SnmpSource)' == '' ">..\..\..\build\win32\64\mongo</SnmpSource>
<Source Condition=" '$(Source)' == '' ">..\..\..\build\win32\64\mongo</Source>
<SslSource Condition=" '$(SslSource)' == '' ">..\..\..\build\win32\64\mongo</SslSource>
<Version Condition=" '$(Version)' == '' ">2.4.0</Version>
<MergeModulesBasePath Condition=" '$(MergeModulesBasePath)' == '' ">C:\Program Files (x86)\Common Files\Merge Modules</MergeModulesBasePath>
<ClientHeaderSource Condition=" '$(ClientHeaderSource)' == '' ">..\..\..\build\win32\normal\clientlib\include</ClientHeaderSource>
<DefineConstants>MongoDBVersion=$(Version);LicenseSource=$(License);BinarySource=$(Source);Edition=$(Edition);SaslSource=$(SaslSource);SslSource=$(SslSource);SnmpSource=$(SnmpSource);ProductId=$(ProductId);UpgradeCode=$(UpgradeCode);Flavor=$(Flavor);ClientSource=$(ClientSource);EnterpriseBase=$(EnterpriseBase);ClientHeaderSource=$(ClientHeaderSource);MergeModulesBasePath=$(MergeModulesBasePath)</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x64' ">
<VerboseOutput>True</VerboseOutput>
</PropertyGroup>
<PropertyGroup Condition=" '$(Edition)' == 'Enterprise' ">
<TreatWarningsAsErrors>False</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<Compile Include="..\..\..\src\mongo\installer\msi\wxs\BinaryFragment.wxs" />
<Compile Include="..\..\..\src\mongo\installer\msi\wxs\FeatureFragment.wxs" />
<Compile Include="..\..\..\src\mongo\installer\msi\wxs\Installer_64.wxs" />
<Compile Include="..\..\..\src\mongo\installer\msi\wxs\LicensingFragment.wxs" />
<Compile Include="$(OutputPath)DriverInclude.wxs" />
</ItemGroup>
<ItemGroup>
<WixExtension Include="WixUIExtension">
<HintPath>$(WixExtDir)\WixUIExtension.dll</HintPath>
<Name>WixUIExtension</Name>
</WixExtension>
</ItemGroup>
<ItemGroup>
<Folder Include="wxs\" />
</ItemGroup>
<Import Project="$(WixTargetsPath)" />
<PropertyGroup>
<PreBuildEvent>"%WIX%\bin\heat.exe" dir "$(ClientHeaderSource)" -gg -g1 -frag -cg cg_DriverHeaders -nologo -directoryid -out DriverInclude.wxs -dr Header -srd -var var.ClientHeaderSource</PreBuildEvent>
</PropertyGroup>
<!--
To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Wix.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>

View File

@ -1,36 +0,0 @@
## Prerequisites
WiX Toolset v3.7.1224.0 from http://wixtoolset.org/
## Features
The following are the installer features with the executables they install.
Each of these features can be installed independently using msiexec /ADDLOCAL
or using the Installer GUI
* Server
* mongod.exe
* mongod.pdb
* Client
* mongo.exe
* MonitoringTools
* mongostat.exe
* mongotop.exe
* ImportExportTools
* mongodump.exe
* mongorestore.exe
* mongoexport.exe
* mongoimport.exe
* Router
* mongos.exe
* mongos.pdb
* MiscellaneousTools
* bsondump.exe
* mongofiles.exe
* mongooplog.exe
* mongoperf.exe
## Typical install
The typical (default) install, installs all except the Router and
MiscellaneousTools features.
## Configuring builds
The version, location of binaries and license file can be configured when
building. Refer to build32bitmsi.bat or build64bitmsi.bat for example

View File

@ -1,100 +0,0 @@
@ECHO OFF
SET VERSION=2.4.0
SET BINDIR=..\..\..\build\win32\normal\mongo
SET CLIENTLIBDIR=..\..\..\build\win32\normal\client_build
SET LICENSEDIR=..\..\..\distsrc
SET CLIENTHEADERDIR=..\..\..\build\win32\normal\client_build\include
SET WIXBINDIR=C:\Program Files (x86)\WiX Toolset v3.7\bin
SET PLATFORM=x86
SET GENERATEDWXSDIR=.\wxs
SET EDITION=Standard
SET CONFIGURATION=Release
SET OUTPUTOBJDIR=obj\%CONFIGURATION%\%PLATFORM%\
SET OUTPUTBINDIR=bin\%CONFIGURATION%\%PLATFORM%\
SET PROJECTDIR=C:\git\sridharn\mongo\buildscripts\packaging\msi\
SET TARGETNAME=MongoDB_%VERSION%_%PLATFORM%_%EDITION%
:loop
IF NOT "%1"=="" (
IF "%1"=="-version" (
SET VERSION=%2
SHIFT
)
IF "%1"=="-bindir" (
SET BINDIR=%2
SHIFT
)
IF "%1"=="-licensedir" (
SET LICENSEDIR=%2
SHIFT
)
IF "%1"=="-clientlibdir" (
SET CLIENTLIBDIR=%2
SHIFT
)
IF "%1"=="-clientheaderdir" (
SET CLIENTHEADERDIR=%2
SHIFT
)
IF "%1"=="-wixbindir" (
SET WIXBINDIR=%2
SHIFT
)
IF "%1"=="-generatedwxsdir" (
SET GENERATEDWXSDIR=%2
SHIFT
)
SHIFT
GOTO :loop
)
REM ECHO Building msi for version %VERSION% with binaries from %BINDIR% and license files from %LICENSEDIR%
REM %WINDIR%\Microsoft.NET\Framework64\v4.0.30319\msbuild /p:Configuration=Release;Version=%VERSION%;License=%LICENSEDIR%;
REM Source=%BINDIR%;ClientSource=%CLIENTLIBDIR%;ClientHeaderSource=%CLIENTHEADERDIR% MongoDB.wixproj
ECHO Generating %GENERATEDWXSDIR%\DriverInclude.wxs from sources at %CLIENTHEADERDIR%
"%WIXBINDIR%\heat.exe" dir %CLIENTHEADERDIR% -gg -g1 -frag -cg cg_DriverHeaders -nologo -directoryid -out %GENERATEDWXSDIR%\DriverInclude.wxs -dr Header -srd -var var.ClientHeaderSource
ECHO Compiling wxs files to obj
"%WIXBINDIR%\candle.exe" -wx^
-dMongoDBVersion=%VERSION%^
-dLicenseSource=%LICENSEDIR%^
-dBinarySource=%BINDIR%^
-dEdition=%EDITION%^
-d"ProductId=*"^
-dUpgradeCode=867C1D1D-2040-4E90-B04E-1158F9CBDE96^
-dClientSource=%CLIENTLIBDIR%^
-dClientHeaderSource=%CLIENTHEADERDIR%^
-dConfiguration=%CONFIGURATION%^
-dOutDir=%OUTPUTBINDIR%^
-dPlatform=%PLATFORM%^
-dProjectDir=%PROJECTDIR%^
-dProjectExt=.wixproj^
-dProjectFileName=MongoDB.wixproj^
-dProjectName=MongoDB^
-dProjectPath=%PROJECTDIR%\MongoDB.wixproj^
-dTargetDir=%OUTPUTBINDIR%^
-dTargetExt=.msi^
-dTargetFileName=%TARGETNAME%.msi^
-dTargetName=%TARGETNAME%^
-dTargetPath=%OUTPUTBINDIR%\%TARGETNAME%.msi^
-out %OUTPUTOBJDIR%^
-arch %PLATFORM%^
-ext "%WIXBINDIR%\WixUIExtension.dll"^
wxs\BinaryFragment.wxs wxs\FeatureFragment.wxs wxs\LicensingFragment.wxs wxs\Installer.wxs %GENERATEDWXSDIR%\DriverInclude.wxs
ECHO Linking to msi
"%WIXBINDIR%\Light.exe"^
-out %OUTPUTBINDIR%\%TARGETNAME%.msi^
-pdbout %OUTPUTBINDIR%\%TARGETNAME%.wixpdb^
-wx -cultures:null^
-ext "%WIXBINDIR%\WixUIExtension.dll"^
-contentsfile %OUTPUTOBJDIR%\MongoDB.wixproj.BindContentsFileListnull.txt^
-outputsfile %OUTPUTOBJDIR%\MongoDB.wixproj.BindOutputsFileListnull.txt^
-builtoutputsfile %OUTPUTOBJDIR%\MongoDB.wixproj.BindBuiltOutputsFileListnull.txt^
-wixprojectfile %PROJECTDIR%\MongoDB.wixproj^
%OUTPUTOBJDIR%\BinaryFragment.wixobj %OUTPUTOBJDIR%\FeatureFragment.wixobj^
%OUTPUTOBJDIR%\\LicensingFragment.wixobj %OUTPUTOBJDIR%\Installer.wixobj^
%OUTPUTOBJDIR%\DriverInclude.wixobj

View File

@ -1,37 +0,0 @@
@ECHO OFF
SET VERSION=2.4.0
SET BINDIR=..\..\..\build\win32\64\mongo
SET CLIENTLIBDIR=..\..\..\build\win32\64\client_build
SET LICENSEDIR=..\..\..\distsrc
SET EDITION=Standard
SET FLAVOR=2008R2Plus
SET CLIENTHEADERDIR=..\..\..\build\win32\normal\client_build\include
:loop
IF NOT "%1"=="" (
IF "%1"=="-version" (
SET VERSION=%2
SHIFT
)
IF "%1"=="-bindir" (
SET BINDIR=%2
SHIFT
)
IF "%1"=="-licensedir" (
SET LICENSEDIR=%2
SHIFT
)
IF "%1"=="-clientlibdir" (
SET CLIENTLIBDIR=%2
SHIFT
)
IF "%1"=="-clientheaderdir" (
SET CLIENTHEADERDIR=%2
SHIFT
)
SHIFT
GOTO :loop
)
ECHO Building msi for version %VERSION% with binaries from %BINDIR% and license files from %LICENSEDIR%
%WINDIR%\Microsoft.NET\Framework64\v4.0.30319\msbuild /p:Configuration=Release;Version=%VERSION%;License=%LICENSEDIR%;Source=%BINDIR%;Edition=%EDITION%;Flavor=%FLAVOR%;ClientSource=%CLIENTLIBDIR%;ClientHeaderSource=%CLIENTHEADERDIR% MongoDB_64.wixproj

View File

@ -1,37 +0,0 @@
@ECHO OFF
SET VERSION=2.4.0
SET BINDIR=..\..\..\build\win32\64\mongo
SET CLIENTLIBDIR=..\..\..\build\win32\64\client_build
SET LICENSEDIR=..\..\..\distsrc
SET EDITION=Standard
SET FLAVOR=2008
SET CLIENTHEADERDIR=..\..\..\build\win32\normal\client_build\include
:loop
IF NOT "%1"=="" (
IF "%1"=="-version" (
SET VERSION=%2
SHIFT
)
IF "%1"=="-bindir" (
SET BINDIR=%2
SHIFT
)
IF "%1"=="-licensedir" (
SET LICENSEDIR=%2
SHIFT
)
IF "%1"=="-clientlibdir" (
SET CLIENTLIBDIR=%2
SHIFT
)
IF "%1"=="-clientheaderdir" (
SET CLIENTHEADERDIR=%2
SHIFT
)
SHIFT
GOTO :loop
)
ECHO Building msi for version %VERSION% with binaries from %BINDIR% and license files from %LICENSEDIR%
%WINDIR%\Microsoft.NET\Framework64\v4.0.30319\msbuild /p:Configuration=Release;Version=%VERSION%;License=%LICENSEDIR%;Source=%BINDIR%;Edition=%EDITION%;Flavor=%FLAVOR%;ClientSource=%CLIENTLIBDIR%;ClientHeaderSource=%CLIENTHEADERDIR% MongoDB_64.wixproj

View File

@ -1,63 +0,0 @@
@ECHO OFF
SET VERSION=2.6.0
SET BINDIR=..\..\..\build\win32\64\dynamic-windows\extrapathdyn_c__Utils_sasl_c__Utils_snmp_c__Utils_ssl\release\ssl\mongo
SET CLIENTLIBDIR=..\..\..\build\win32\64\dynamic-windows\extrapathdyn_c__Utils_sasl_c__Utils_snmp_c__Utils_ssl\release\ssl\client_build
SET LICENSEDIR=..\..\..\distsrc
SET ENTERPRISEBASEDIR=..\..\..\src\mongo\db\modules\subscription
SET EDITION=Enterprise
SET FLAVOR=2008R2Plus
SET SASLDIR=..\..\..\..\..\..\Utils\sasl\bin
SET OPENSSLDIR=..\..\..\..\..\..\Utils\openssl\bin
SET SNMPDIR=..\..\..\..\..\..\Utils\snmp\bin
SET CLIENTHEADERDIR=..\..\..\build\win32\normal\client_build\include
SET MERGEMODULESBASEPATH="C:\Program Files (x86)\Common Files\Merge Modules"
:loop
IF NOT "%1"=="" (
IF "%1"=="-version" (
SET VERSION=%2
SHIFT
)
IF "%1"=="-bindir" (
SET BINDIR=%2
SHIFT
)
IF "%1"=="-licensedir" (
SET LICENSEDIR=%2
SHIFT
)
IF "%1"=="-sasldir" (
SET SASLDIR=%2
SHIFT
)
IF "%1"=="-openssldir" (
SET OPENSSLDIR=%2
SHIFT
)
IF "%1"=="-snmpdir" (
SET SNMPDIR=%2
SHIFT
)
IF "%1"=="-clientlibdir" (
SET CLIENTLIBDIR=%2
SHIFT
)
IF "%1"=="-enterprisebasedir" (
SET ENTERPRISEBASEDIR=%2
SHIFT
)
IF "%1"=="-clientheaderdir" (
SET CLIENTHEADERDIR=%2
SHIFT
)
IF "%1"=="-mergemodulesbasepath" (
SET MERGEMODULESBASEPATH=%2
SHIFT
)
SHIFT
GOTO :loop
)
ECHO Building enterprise msi for version %VERSION% with binaries from %BINDIR%, sasl from %SASLDIR%, ssl from %OPENSSLDIR%, snmp from %SNMPDIR% and license files from %LICENSEDIR%
%WINDIR%\Microsoft.NET\Framework64\v4.0.30319\msbuild /p:Configuration=Release;Version=%VERSION%;License=%LICENSEDIR%;Source=%BINDIR%;SaslSource=%SASLDIR%;SnmpSource=%SNMPDIR%;SslSource=%OPENSSLDIR%;Edition=%EDITION%;Flavor=%FLAVOR%;ClientSource=%CLIENTLIBDIR%;EnterpriseBase=%ENTERPRISEBASEDIR%;ClientHeaderSource=%CLIENTHEADERDIR%;MergeModulesBasePath=%MERGEMODULESBASEPATH% MongoDB_64.wixproj

View File

@ -1,121 +0,0 @@
#!/usr/bin/env python
"""
Copyright 2014 MongoDB Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License, version 3,
as published by the Free Software Foundation.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
As a special exception, the copyright holders give permission to link the
code of portions of this program with the OpenSSL library under certain
conditions as described in each individual source file and distribute
linked combinations including the program with the OpenSSL library. You
must comply with the GNU Affero General Public License in all respects
for all of the code used other than as permitted herein. If you modify
file(s) with this exception, you may extend this exception to your
version of the file(s), but you are not obligated to do so. If you do not
wish to do so, delete this exception statement from your version. If you
delete this exception statement from all source files in the program,
then also delete it in the license file.
"""
import os
import subprocess
import sys
import tempfile
import time
from optparse import OptionParser
class Benchmark(object):
def __init__(self, name, executable, options):
self._name = name
self._executable = executable
self._options = options
self._ran = False
def name(self):
return self._name
def run(self):
self._result = subprocess.check_call([self._executable] + self._options)
self._ran = True
def result(self):
if not self._ran:
raise Exception("Cannot get result of a suite that hasn't ran")
return self._result
def file_allocator_bench(variant, ntrials, megabytes, path, report_dir):
executable = os.path.join("build", "file_allocator_bench")
suitename = "FileAllocatorBenchmark-%s" % variant
opts = []
opts.append("--ntrials=%s" % ntrials)
opts.append("--megabytes=%i" % megabytes)
opts.append("--path=%s" % path)
opts.append("--jsonReport=%s.json" % os.path.join(report_dir, variant))
return Benchmark(suitename, executable, opts)
def configure_parser():
parser = OptionParser()
parser.add_option("-r", "--reportDir", type=str,
help="Where to write the report [default: %default]",
default=os.getcwd())
parser.add_option("-w", "--workDir", type=str,
help="scratch space used by tests [default: %default]",
default=tempfile.gettempdir())
return parser
# at some point this should read from a config file, but at this point
# its not worth overengineering
def make_suites(work_dir, report_dir):
return [
# requires 128GB free in work_dir
# file_allocator_bench("16GB", ntrials=8, megabytes=1024*16,
# path=work_dir, report_dir=report_dir),
# requires 16GB free in work_dir
file_allocator_bench("1GB", ntrials=16, megabytes=1024,
path=work_dir, report_dir=report_dir),
# requires 4GB free in work_dir
file_allocator_bench("128MB", ntrials=32, megabytes=128,
path=work_dir, report_dir=report_dir)
]
def main():
parser = configure_parser()
(options, args) = parser.parse_args()
report_dir = options.reportDir
work_dir = options.workDir
# unique dir for this report
unique_dir = os.path.join(report_dir, "perfsuite-run@%s" % time.time())
os.makedirs(unique_dir)
print "Writing results to %s" % unique_dir
print
for suite in make_suites(work_dir, unique_dir):
print("Running suite - %s ..." % suite.name())
start = time.time()
suite.run()
end = time.time()
print("...Finished suite in %i seconds" % (end - start))
sys.exit(0)
if __name__ == "__main__":
main()

View File

@ -1,55 +0,0 @@
import os
import sys
import time
sys.path.append( "." )
sys.path.append( ".." )
sys.path.append( "../../" )
sys.path.append( "../../../" )
import simples3
import settings
import subprocess
# check s3 for md5 hashes
def check_dir( bucket , prefix , todel ):
deleteAll = False
for ( key , modify , etag , size ) in bucket.listdir( prefix=prefix ):
if key.find( todel ) < 0:
continue
print( key )
if not deleteAll:
val = raw_input( "Delete (Y,y,n,N):" ).strip()
if val == "n":
print( "skipping this one" )
continue
elif val == "N":
break
if val == "Y":
val = "y"
deleteAll = True
if val != "y":
raise Exception( "invalid input :(" )
bucket.delete( key )
def clean( todel ):
bucket = simples3.S3Bucket( settings.bucket , settings.id , settings.key )
for x in [ "osx" , "linux" , "win32" , "sunos5" , "src" ]:
check_dir( bucket , x , todel )
if __name__ == "__main__":
clean( sys.argv[1] )

View File

@ -1,48 +0,0 @@
import os
import sys
sys.path.append( "." )
sys.path.append( ".." )
sys.path.append( "../../" )
sys.path.append( "../../../" )
import simples3
import settings
import subprocess
# check s3 for md5 hashes
def check_dir( bucket , prefix ):
zips = {}
md5s = {}
for ( key , modify , etag , size ) in bucket.listdir( prefix=prefix ):
if key.endswith( ".tgz" ) or key.endswith( ".zip" ) or key.endswith( ".tar.gz" ):
zips[key] = etag.replace( '"' , '' )
elif key.endswith( ".md5" ):
md5s[key] = True
elif key.find( "$folder$" ) > 0:
pass
else:
print( "unknown file type: " + key )
for x in zips:
m = x + ".md5"
if m in md5s:
continue
print( "need to do: " + x + " " + zips[x] + " to " + m )
bucket.put( m , zips[x] , acl="public-read" )
def run():
bucket = simples3.S3Bucket( settings.bucket , settings.id , settings.key )
for x in [ "osx" , "linux" , "win32" , "sunos5" , "src" ]:
check_dir( bucket , x )
if __name__ == "__main__":
run()

View File

@ -1,105 +0,0 @@
#!/usr/bin/env python
#
# Generate and upload detached gpg signatures for archive files in Amazon S3
#
# Requires standard MongoDB settings.py, like so:
# bucket = "downloads.mongodb.org"
# # downloads user
# id = "xxxxx"
# key = "xxxxx"
#
# Usage: s3sign.py [ --dry-run ] [ --bucket <overridden s3 bucket> ] [ --notary-url <notary url> ] [ --key-name <key name passed to notary service> ] [ --filter <filter> ]
#
import argparse
import json
import os
import requests
import sys
sys.path.append("." )
sys.path.append(".." )
sys.path.append("../../" )
sys.path.append("../../../" )
import simples3
import settings
import subprocess
# parse command line
#
parser = argparse.ArgumentParser(description='Sign MongoDB S3 Files')
parser.add_argument('--dry-run', action='store_true', required=False, help='Do not write anything to S3', default = False);
parser.add_argument('--bucket', required = False, help='Override bucket in settings.py', default = settings.bucket);
parser.add_argument('--notary-url', required=False, help='URL base for notary service', default = 'http://localhost:5000');
parser.add_argument('--key-name', required=False, help='Key parameter to notary service', default = 'test');
parser.add_argument('--filter', required=False,
help='Only sign files matching case-insensitive substring filter', default = None);
args = parser.parse_args()
notary_urlbase = args.notary_url
notary_url = notary_urlbase + '/api/sign'
notary_payload = { 'key': args.key_name, 'comment': 'Automatic archive signing'}
# check s3 for pgp signatures
def check_dir( bucket , prefix ):
zips = {}
sigs = {}
for ( key , modify , etag , size ) in bucket.listdir( prefix=prefix ):
# filtered out
if args.filter and args.filter.lower() not in key.lower():
pass
# sign it
elif key.endswith(".tgz" ) or key.endswith(".zip" ) or key.endswith(".tar.gz" ) or key.endswith("md5"):
# generate signature
files = {'file': (key, bucket.get(key))}
response_json = {}
try:
r = requests.post(notary_url, files=files, data=notary_payload,
headers = { "Accept": "application/json" })
# get url for signature file
response_json = json.loads(r.text)
except Exception as e:
print('error contacting signing service for %s:\n%s' % (key, e.message))
continue
if 'permalink' in response_json:
signature_url = response_json['permalink']
try:
signature = requests.get(notary_urlbase + signature_url).text
zips[key] = signature
except Exception as e:
print('error downloading signature from signing service for %s:\n%s' % (key, e.message))
else:
print('error from signing service for %s:\n%s' % (key, response_json.get('message')))
# signatures
elif key.endswith(".sig" ) or key.endswith(".asc" ):
sigs[key] = True
# file types we don't need to sign
elif key.endswith(".msi" ) or key.endswith(".deb") or key.endswith(".rpm"):
pass
else:
print("unknown file type: %s" % key)
for x in zips:
m = x + ".sig"
if m in sigs:
continue
print("need to do: " + x + " to " + m )
if not args.dry_run:
bucket.put( m , zips[x] , acl="public-read" )
def run():
bucket = simples3.S3Bucket( args.bucket , settings.id , settings.key )
for x in [ "osx" , "linux" , "win32" , "sunos5" , "src" ]:
check_dir( bucket , x )
if __name__ == "__main__":
run()

View File

@ -1,239 +0,0 @@
# Copyright 2009 10gen, Inc.
#
# This file is part of MongoDB.
#
# MongoDB is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# MongoDB is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with MongoDB. If not, see <http://www.gnu.org/licenses/>.
"""Tests for the MongoDB shell.
Right now these mostly just test that the shell handles command line arguments
appropriately.
"""
import unittest
import sys
import subprocess
import os
"""Exit codes for MongoDB."""
BADOPTS = 2
NOCONNECT = 255
"""Path to the mongo shell executable to be tested."""
mongo_path = None
class TestShell(unittest.TestCase):
def open_mongo(self, args=[]):
"""Get a subprocess.Popen instance of the shell with the given args.
"""
return subprocess.Popen([mongo_path] + args,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr = subprocess.PIPE)
def setUp(self):
assert mongo_path
def test_help(self):
mongo_h = self.open_mongo(["-h"])
mongo_help = self.open_mongo(["--help"])
out = mongo_h.communicate()
self.assertEqual(out, mongo_help.communicate())
self.assert_("usage:" in out[0])
self.assertEqual(0, mongo_h.returncode)
self.assertEqual(0, mongo_help.returncode)
def test_nodb(self):
mongo = self.open_mongo([])
mongo_nodb = self.open_mongo(["--nodb"])
out = mongo_nodb.communicate()
self.assert_("MongoDB shell version" in out[0])
self.assert_("bye" in out[0])
self.assert_("couldn't connect" not in out[0])
self.assertEqual(0, mongo_nodb.returncode)
out = mongo.communicate()
self.assert_("MongoDB shell version" in out[0])
self.assert_("bye" not in out[0])
self.assert_("couldn't connect" in out[0])
self.assertEqual(NOCONNECT, mongo.returncode)
def test_eval(self):
mongo = self.open_mongo(["--nodb", "--eval", "print('hello world');"])
out = mongo.communicate()
self.assert_("hello world" in out[0])
self.assert_("bye" not in out[0])
self.assertEqual(0, mongo.returncode)
mongo = self.open_mongo(["--eval"])
out = mongo.communicate()
self.assert_("required parameter is missing" in out[0])
self.assertEqual(BADOPTS, mongo.returncode)
def test_shell(self):
mongo = self.open_mongo(["--nodb", "--shell", "--eval", "print('hello world');"])
out = mongo.communicate()
self.assert_("hello world" in out[0])
self.assert_("bye" in out[0]) # the shell started and immediately exited because stdin was empty
self.assertEqual(0, mongo.returncode)
def test_host_port(self):
mongo = self.open_mongo([])
out = mongo.communicate()
self.assert_("url: test" in out[0])
self.assert_("connecting to: test" in out[0])
self.assertEqual(NOCONNECT, mongo.returncode)
mongo = self.open_mongo(["--host", "localhost"])
out = mongo.communicate()
self.assert_("url: test" in out[0])
self.assert_("connecting to: localhost/test" in out[0])
self.assertEqual(NOCONNECT, mongo.returncode)
mongo = self.open_mongo(["--port", "27018"])
out = mongo.communicate()
self.assert_("url: test" in out[0])
self.assert_("connecting to: 127.0.0.1:27018" in out[0])
self.assertEqual(NOCONNECT, mongo.returncode)
mongo = self.open_mongo(["--host", "localhost", "--port", "27018"])
out = mongo.communicate()
self.assert_("url: test" in out[0])
self.assert_("connecting to: localhost:27018/test" in out[0])
self.assertEqual(NOCONNECT, mongo.returncode)
mongo = self.open_mongo(["--host"])
out = mongo.communicate()
self.assert_("required parameter is missing" in out[0])
self.assertEqual(BADOPTS, mongo.returncode)
mongo = self.open_mongo(["--port"])
out = mongo.communicate()
self.assert_("required parameter is missing" in out[0])
self.assertEqual(BADOPTS, mongo.returncode)
def test_positionals(self):
dirname = os.path.dirname(__file__)
test_js = os.path.join(dirname, "testdata/test.js")
test_txt = os.path.join(dirname, "testdata/test.txt")
test = os.path.join(dirname, "testdata/test")
non_exist_js = os.path.join(dirname, "testdata/nonexist.js")
non_exist_txt = os.path.join(dirname, "testdata/nonexist.txt")
mongo = self.open_mongo(["--nodb", test_js])
out = mongo.communicate()
self.assert_("hello world" in out[0])
self.assert_("bye" not in out[0])
self.assertEqual(0, mongo.returncode)
mongo = self.open_mongo(["--nodb", test_txt])
out = mongo.communicate()
self.assert_("foobar" in out[0])
self.assert_("bye" not in out[0])
self.assertEqual(0, mongo.returncode)
mongo = self.open_mongo([test_js, test, test_txt])
out = mongo.communicate()
self.assert_("url: test" in out[0])
self.assert_("connecting to: test" in out[0])
self.assertEqual(NOCONNECT, mongo.returncode)
mongo = self.open_mongo([test_txt, test, test_js])
out = mongo.communicate()
self.assert_("url: test" in out[0])
self.assert_("connecting to: test" in out[0])
self.assertEqual(NOCONNECT, mongo.returncode)
mongo = self.open_mongo([test, test_js, test_txt])
out = mongo.communicate()
self.assert_("url: " + test in out[0])
self.assert_("connecting to: " + test in out[0])
self.assertEqual(NOCONNECT, mongo.returncode)
mongo = self.open_mongo([non_exist_js, test, test_txt])
out = mongo.communicate()
self.assert_("url: test" in out[0])
self.assert_("connecting to: test" in out[0])
self.assertEqual(NOCONNECT, mongo.returncode)
mongo = self.open_mongo([non_exist_txt, test_js, test_txt])
out = mongo.communicate()
self.assert_("url: " + non_exist_txt in out[0])
self.assert_("connecting to: " + non_exist_txt in out[0])
self.assertEqual(NOCONNECT, mongo.returncode)
def test_multiple_files(self):
dirname = os.path.dirname(__file__)
test_js = os.path.join(dirname, "testdata/test.js")
test_txt = os.path.join(dirname, "testdata/test.txt")
mongo = self.open_mongo(["--nodb", test_js, test_txt])
out = mongo.communicate()
self.assert_("hello world" in out[0])
self.assert_("foobar" in out[0])
self.assert_("bye" not in out[0])
self.assertEqual(0, mongo.returncode)
mongo = self.open_mongo(["--shell", "--nodb", test_js, test_txt])
out = mongo.communicate()
self.assert_("hello world" in out[0])
self.assert_("foobar" in out[0])
self.assert_("bye" in out[0])
self.assertEqual(0, mongo.returncode)
# just testing that they don't blow up
def test_username_and_password(self):
mongo = self.open_mongo(["--username", "mike"])
out = mongo.communicate()
self.assertEqual(NOCONNECT, mongo.returncode)
mongo = self.open_mongo(["-u", "mike"])
out = mongo.communicate()
self.assertEqual(NOCONNECT, mongo.returncode)
mongo = self.open_mongo(["--password", "mike"])
out = mongo.communicate()
self.assertEqual(NOCONNECT, mongo.returncode)
mongo = self.open_mongo(["-p", "mike"])
out = mongo.communicate()
self.assertEqual(NOCONNECT, mongo.returncode)
mongo = self.open_mongo(["--username"])
out = mongo.communicate()
self.assert_("required parameter is missing" in out[0])
self.assertEqual(BADOPTS, mongo.returncode)
mongo = self.open_mongo(["--password"])
out = mongo.communicate()
self.assert_("required parameter is missing" in out[0])
self.assertEqual(BADOPTS, mongo.returncode)
def run_tests():
suite = unittest.TestLoader().loadTestsFromTestCase(TestShell)
unittest.TextTestRunner(verbosity=1).run(suite)
if __name__ == "__main__":
if len(sys.argv) != 2:
print "must give the path to shell executable to be tested"
sys.exit()
mongo_path = sys.argv[1]
run_tests()

View File

@ -1,16 +0,0 @@
--indent=spaces=4
--convert-tabs
--lineend=linux
--indent-namespaces
--indent-preprocessor
--keep-one-line-statements
--keep-one-line-blocks
--break-closing-brackets
--brackets=attach

View File

@ -1261,20 +1261,6 @@ test = env.Install(
if len(env.subst('$PROGSUFFIX')):
env.Alias( "dbtest", "#/${PROGPREFIX}dbtest${PROGSUFFIX}" )
env.Install('$BUILD_ROOT/', env.Program('file_allocator_bench',
'util/file_allocator_bench.cpp',
LIBDEPS=[
'mongocommon',
'signal_handlers_synchronous',
'$BUILD_DIR/mongo/util/options_parser/options_parser_init',
'serveronly',
'coredb',
'coreserver',
"ntservice_mock",
]))
env.Alias('file_allocator_bench', "$BUILD_ROOT/" + add_exe("file_allocator_bench"))
# --- sniffer ---
mongosniff_built = False
if env.TargetOSIs('osx') or env["_HAVEPCAP"]:

View File

@ -3,7 +3,6 @@
import os
Import("env")
Import("s3push")
Import("has_option")
env = env.Clone()
@ -146,10 +145,3 @@ env.AlwaysBuild(msi)
env.Alias( "msi" , msi )
def s3msipush( env , target , source ):
s3push( source )
env.Alias( "s3msi" , [ msi ] , [ s3msipush ] )
env.AlwaysBuild( "s3msi" )

View File

@ -1,334 +0,0 @@
/*
* Copyright 2014 MongoDB Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* As a special exception, the copyright holders give permission to link the
* code of portions of this program with the OpenSSL library under certain
* conditions as described in each individual source file and distribute
* linked combinations including the program with the OpenSSL library. You
* must comply with the GNU Affero General Public License in all respects
* for all of the code used other than as permitted herein. If you modify
* file(s) with this exception, you may extend this exception to your
* version of the file(s), but you are not obligated to do so. If you do not
* wish to do so, delete this exception statement from your version. If you
* delete this exception statement from all source files in the program,
* then also delete it in the license file.
*/
#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kDefault
#include "mongo/platform/basic.h"
#include <algorithm>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/filesystem.hpp>
#include <boost/scoped_ptr.hpp>
#include <fstream>
#include <iostream>
#include <numeric>
#include <string>
#include "mongo/base/initializer.h"
#include "mongo/base/status.h"
#include "mongo/bson/bsonobjbuilder.h"
#include "mongo/util/file_allocator.h"
#include "mongo/util/log.h"
#include "mongo/util/mongoutils/str.h"
#include "mongo/util/options_parser/environment.h"
#include "mongo/util/options_parser/option_section.h"
#include "mongo/util/options_parser/options_parser.h"
#include "mongo/util/options_parser/startup_option_init.h"
#include "mongo/util/options_parser/startup_options.h"
#include "mongo/util/quick_exit.h"
#include "mongo/util/scopeguard.h"
#include "mongo/util/signal_handlers_synchronous.h"
using namespace mongo;
namespace {
namespace file = boost::filesystem;
namespace ptime = boost::posix_time;
typedef unsigned long long bytes_t;
typedef long long micros_t;
const long DEFAULT_FILE_SIZE_MB = 128;
const file::path DEFAULT_PATH = file::temp_directory_path();
const int DEFAULT_NTRIALS = 10;
const bool DEFAULT_BSON_OUT = false;
// used to convert B/usec to MB/sec
const double MICROSEC_PER_SEC = 1e6;
const double MB_PER_BYTE = 1.0 / (1 << 20);
const double MB_SEC_CONVERSION_FACTOR = MICROSEC_PER_SEC * MB_PER_BYTE;
double toMbSec(const bytes_t bytes, const micros_t micros) {
return (static_cast<double>(bytes) / static_cast<double>(micros)) *
MB_SEC_CONVERSION_FACTOR;
}
}
struct BenchmarkParams {
bytes_t bytes;
file::path path;
int ntrials;
bool quiet;
bool jsonReportEnabled;
std::string jsonReportOut;
} benchParams;
class FileAllocatorBenchmark {
public:
FileAllocatorBenchmark(const BenchmarkParams& params)
: _fa(FileAllocator::get())
, _params(params) {
_fa->start();
if (!file::create_directory(_params.path)) {
std::cerr << "Error: unable to create temporary directory in "
<< _params.path.parent_path() << std::endl;
quickExit(EXIT_FAILURE);
}
}
// Delete any files we created
~FileAllocatorBenchmark() {
file::remove_all(_params.path);
}
void run() {
if (!_params.quiet) {
std::cout << "Allocating " << _params.ntrials << " files of size "
<< _params.bytes << " bytes in " << _params.path << std::endl;
}
for (int n = 0; n < _params.ntrials; ++n) {
const std::string fileName = str::stream() << "garbage-" << n;
file::path filePath = _params.path / fileName;
_files.push_back(filePath);
bytes_t size_allocated = _params.bytes;
const ptime::ptime start = ptime::microsec_clock::universal_time();
try {
_fa->allocateAsap(filePath.string(), size_allocated);
} catch (const DBException& ex) {
std::cerr << "Exception thrown while allocating file:" << std::endl;
std::cerr << ex.what() << std::endl;
throw; // rethrow so that destructor is called
}
if (size_allocated != static_cast<bytes_t>(_params.bytes)) {
std::cerr << "Allocated " << size_allocated << " bytes but expected "
<< _params.bytes;
}
const ptime::ptime end = ptime::microsec_clock::universal_time();
_results.push_back((end - start).total_microseconds());
}
_fa->waitUntilFinished();
if (!_params.quiet) {
textReport();
}
if (_params.jsonReportEnabled) {
jsonReport(_params.jsonReportOut);
}
}
private:
struct benchResults {
micros_t avg;
micros_t max;
micros_t min;
};
benchResults computeResults() {
benchResults res;
const micros_t total = std::accumulate(_results.begin(), _results.end(), 0L);
res.avg = total / _results.size();
res.max = *std::max_element(_results.begin(), _results.end());
res.min = *std::min_element(_results.begin(), _results.end());
return res;
}
void printResult(const std::string& name, const micros_t duration, const bytes_t bytes) {
std::cout << name << ": " << duration << " usec = "
<< toMbSec(bytes, duration) << " MB/sec" << std::endl;
}
void textReport() {
benchResults results = computeResults();
std::cout << "Results for " << _params.ntrials << " allocations of "
<< _params.bytes << " bytes: " << std::endl;
printResult("avg", results.avg, _params.bytes);
printResult("max", results.max, _params.bytes);
printResult("min", results.min, _params.bytes);
}
void addResult(BSONObjBuilder& obj, const std::string& name,
const micros_t duration, const bytes_t bytes) {
BSONObjBuilder so(obj.subobjStart(name));
so.append("usec", duration);
so.append("MBsec", toMbSec(bytes, duration));
so.done();
}
void jsonReport(const std::string& jsonReportOut) {
benchResults results = computeResults();
BSONObjBuilder obj;
obj.append("bytes", static_cast<long long>(_params.bytes));
addResult(obj, "avg", results.avg, _params.bytes);
addResult(obj, "max", results.max, _params.bytes);
addResult(obj, "min", results.min, _params.bytes);
obj.append("raw", _results);
const std::string outStr = obj.done().toString();
if (jsonReportOut == "-") {
std::cout << outStr << std::endl;
} else {
std::ofstream outfile(jsonReportOut.c_str());
if (!outfile.is_open()) {
std::cerr << "Error: couldn't create output file " << jsonReportOut << std::endl;
return;
}
ON_BLOCK_EXIT(&std::ofstream::close, outfile);
outfile << outStr << std::endl;
}
}
FileAllocator* const _fa;
std::vector<micros_t> _results;
std::vector<file::path> _files;
const BenchmarkParams& _params;
};
namespace moe = mongo::optionenvironment;
Status addFileAllocatorBenchOptions(moe::OptionSection& options) {
options.addOptionChaining("help", "help", moe::Switch, "Display help");
options.addOptionChaining("megabytes", "megabytes", moe::Long,
"The number of megabytes to allocate for each file")
.setDefault(moe::Value(DEFAULT_FILE_SIZE_MB));
options.addOptionChaining("path", "path", moe::String,
str::stream() << "The directory path to allocate the file(s) in "
<< "during testing. Files will be allocated in a "
<< "uniquely named temporary directory within the "
<< "specified path")
.setDefault(moe::Value(DEFAULT_PATH.string()));
options.addOptionChaining("ntrials", "ntrials", moe::Int,
"The number of trials to perform")
.setDefault(moe::Value(DEFAULT_NTRIALS));
options.addOptionChaining("quiet", "quiet", moe::Switch,
"Suppress the plaintext report");
options.addOptionChaining("jsonReport", "jsonReport", moe::String,
str::stream() << "If set, results will be saved as a JSON document to "
<< "the specified file path. If specified with no "
<< "arguments the report will be printed to standard "
<< "out")
.setImplicit(moe::Value(std::string("-")));
return Status::OK();
}
Status validateFileAllocatorBenchOptions(const moe::OptionSection& options,
moe::Environment& env) {
Status ret = env.validate();
if (!ret.isOK()) {
return ret;
}
bool displayHelp = false;
ret = env.get(moe::Key("help"), &displayHelp);
if (displayHelp) {
std::cout << options.helpString() << std::endl;
quickExit(EXIT_SUCCESS);
}
return Status::OK();
}
Status storeFileAllocatorBenchOptions(const moe::Environment& env) {
// don't actually need to check Status since we set default values
long mbytes;
Status ret = env.get(moe::Key("megabytes"), &mbytes);
benchParams.bytes = mbytes * (1 << 20);
std::string path;
ret = env.get(moe::Key("path"), &path);
file::path rootPath = file::path(path);
if (!file::is_directory(rootPath)) {
std::cerr << "Error: path argument must be a directory" << std::endl;
quickExit(EXIT_FAILURE);
}
benchParams.path = rootPath / file::unique_path("allocator-bench-%%%%%%%%");
ret = env.get(moe::Key("ntrials"), &benchParams.ntrials);
ret = env.get(moe::Key("quiet"), &benchParams.quiet);
benchParams.jsonReportEnabled = true;
ret = env.get(moe::Key("jsonReport"), &benchParams.jsonReportOut);
if (!ret.isOK()) {
benchParams.jsonReportEnabled = false;
}
return Status::OK();
}
int main(int argc, char** argv, char** envp) {
::mongo::setupSynchronousSignalHandlers();
::mongo::runGlobalInitializersOrDie(argc, argv, envp);
try {
// this try/catch block needs to exist so that
// std::terminate is not called and the FileAllocatorBenchmark
// destructor actually gets called...
FileAllocatorBenchmark(benchParams).run();
} catch (...) {
std::cerr << "Benchmark ended in failure." << std::endl;
quickExit(EXIT_FAILURE);
}
quickExit(EXIT_SUCCESS);
}
MONGO_INITIALIZER(KillLoggingOutput)(InitializerContext* context) {
// The FileAllocator produces a lot of log noise, so we silence
// all logging output
mongo::logger::globalLogDomain()->clearAppenders();
return Status::OK();
}
MONGO_GENERAL_STARTUP_OPTIONS_REGISTER(FileAllocatorBenchOptions)(InitializerContext* context) {
return addFileAllocatorBenchOptions(moe::startupOptions);
}
MONGO_STARTUP_OPTIONS_VALIDATE(FileAllocatorBenchOptions)(InitializerContext* context) {
return validateFileAllocatorBenchOptions(moe::startupOptions,
moe::startupOptionsParsed);
}
MONGO_STARTUP_OPTIONS_STORE(FileAllocatorBenchOptions)(InitializerContext* context) {
return storeFileAllocatorBenchOptions(moe::startupOptionsParsed);
}

View File

@ -1,12 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ImportGroup Label="PropertySheets" />
<PropertyGroup Label="UserMacros" />
<PropertyGroup />
<ItemDefinitionGroup>
<ClCompile>
<PreprocessorDefinitions>MONGO_USE_SRW_ON_WINDOWS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
</ClCompile>
</ItemDefinitionGroup>
<ItemGroup />
</Project>