SERVER-17755 Remove legacy code from build system
This commit is contained in:
parent
0cb8dc2a3c
commit
a64b7af35a
165
SConstruct
165
SConstruct
@ -13,10 +13,8 @@
|
||||
# This file, SConstruct, configures the build environment, and then delegates to
|
||||
# several, subordinate SConscript files, which describe specific build rules.
|
||||
|
||||
import buildscripts
|
||||
import copy
|
||||
import datetime
|
||||
import imp
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
@ -24,9 +22,6 @@ import shutil
|
||||
import stat
|
||||
import sys
|
||||
import textwrap
|
||||
import types
|
||||
import urllib
|
||||
import urllib2
|
||||
import uuid
|
||||
from buildscripts import utils
|
||||
from buildscripts import moduleconfig
|
||||
@ -35,11 +30,6 @@ import libdeps
|
||||
|
||||
EnsureSConsVersion( 2, 3, 0 )
|
||||
|
||||
def findSettingsSetup():
|
||||
sys.path.append( "." )
|
||||
sys.path.append( ".." )
|
||||
sys.path.append( "../../" )
|
||||
|
||||
def versiontuple(v):
|
||||
return tuple(map(int, (v.split("."))))
|
||||
|
||||
@ -581,7 +571,7 @@ if releaseBuild and (debugBuild or not optBuild):
|
||||
print("Error: A --release build may not have debugging, and must have optimization")
|
||||
Exit(1)
|
||||
|
||||
noshell = has_option( "noshell" )
|
||||
noshell = has_option( "noshell" )
|
||||
|
||||
jsEngine = get_option( "js-engine")
|
||||
|
||||
@ -882,12 +872,6 @@ elif env.TargetOSIs('solaris'):
|
||||
env['LINK_LIBGROUP_END'] = ''
|
||||
|
||||
# ---- other build setup -----
|
||||
dontReplacePackage = False
|
||||
isBuildingLatest = False
|
||||
|
||||
def filterExists(paths):
|
||||
return filter(os.path.exists, paths)
|
||||
|
||||
if debugBuild:
|
||||
env.SetConfigHeaderDefine("MONGO_CONFIG_DEBUG_BUILD")
|
||||
|
||||
@ -1040,6 +1024,10 @@ elif env.TargetOSIs('windows'):
|
||||
if usev8:
|
||||
env.Append(LIBS=['winmm.lib'])
|
||||
|
||||
# When building on visual studio, this sets the name of the debug symbols file
|
||||
if env.ToolchainIs('msvc'):
|
||||
env['PDB'] = '${TARGET.base}.pdb'
|
||||
|
||||
env['STATIC_AND_SHARED_OBJECTS_ARE_THE_SAME'] = 1
|
||||
if env.TargetOSIs('posix'):
|
||||
|
||||
@ -1137,11 +1125,7 @@ if env['TARGET_ARCH'] == 'i386':
|
||||
if env.ToolchainIs('GCC', 'clang'):
|
||||
env.Append( CCFLAGS=['-march=nocona', '-mtune=generic'] )
|
||||
|
||||
try:
|
||||
umask = os.umask(022)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# Needed for auth tests since key files are stored in git with mode 644.
|
||||
if not env.TargetOSIs('windows'):
|
||||
for keysuffix in [ "1" , "2" ]:
|
||||
keyfile = "jstests/libs/key%s" % keysuffix
|
||||
@ -1999,8 +1983,6 @@ env = doConfigure( env )
|
||||
# compilation database entries for the configure tests, which is weird.
|
||||
env.Tool("compilation_db")
|
||||
|
||||
env['PDB'] = '${TARGET.base}.pdb'
|
||||
|
||||
def checkErrorCodes():
|
||||
import buildscripts.errorcodes as x
|
||||
if x.checkErrorCodes() == False:
|
||||
@ -2009,32 +1991,8 @@ def checkErrorCodes():
|
||||
|
||||
checkErrorCodes()
|
||||
|
||||
# ---- astyle ----
|
||||
|
||||
def doStyling( env , target , source ):
|
||||
|
||||
res = utils.execsys( "astyle --version" )
|
||||
res = " ".join(res)
|
||||
if res.count( "2." ) == 0:
|
||||
print( "astyle 2.x needed, found:" + res )
|
||||
Exit(-1)
|
||||
|
||||
files = utils.getAllSourceFiles()
|
||||
files = filter( lambda x: not x.endswith( ".c" ) , files )
|
||||
|
||||
cmd = "astyle --options=mongo_astyle " + " ".join( files )
|
||||
res = utils.execsys( cmd )
|
||||
print( res[0] )
|
||||
print( res[1] )
|
||||
|
||||
|
||||
env.Alias( "style" , [] , [ doStyling ] )
|
||||
env.AlwaysBuild( "style" )
|
||||
|
||||
# --- lint ----
|
||||
|
||||
|
||||
|
||||
def doLint( env , target , source ):
|
||||
import buildscripts.lint
|
||||
if not buildscripts.lint.run_lint( [ "src/mongo/" ] ):
|
||||
@ -2052,20 +2010,10 @@ def getSystemInstallName():
|
||||
n = env.GetTargetOSName() + "-" + arch_name
|
||||
if has_option("nostrip"):
|
||||
n += "-debugsymbols"
|
||||
if env.TargetOSIs('posix') and os.uname()[2].startswith("8."):
|
||||
n += "-tiger"
|
||||
|
||||
if len(mongo_modules):
|
||||
n += "-" + "-".join(m.name for m in mongo_modules)
|
||||
|
||||
try:
|
||||
findSettingsSetup()
|
||||
import settings
|
||||
if "distmod" in dir(settings):
|
||||
n = n + "-" + str(settings.distmod)
|
||||
except:
|
||||
pass
|
||||
|
||||
dn = GetOption("distmod")
|
||||
if dn and len(dn) > 0:
|
||||
n = n + "-" + dn
|
||||
@ -2079,96 +2027,13 @@ if mongoCodeVersion == None:
|
||||
if has_option('distname'):
|
||||
distName = GetOption( "distname" )
|
||||
elif mongoCodeVersion[-1] not in ("+", "-"):
|
||||
dontReplacePackage = True
|
||||
distName = mongoCodeVersion
|
||||
else:
|
||||
isBuildingLatest = True
|
||||
distName = utils.getGitBranchString("" , "-") + datetime.date.today().strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
env['SERVER_DIST_BASENAME'] = 'mongodb-%s-%s' % (getSystemInstallName(), distName)
|
||||
|
||||
distFile = "${SERVER_ARCHIVE}"
|
||||
|
||||
# ---- CONVENIENCE ----
|
||||
|
||||
def tabs( env, target, source ):
|
||||
from subprocess import Popen, PIPE
|
||||
from re import search, match
|
||||
diff = Popen( [ "git", "diff", "-U0", "origin", "master" ], stdout=PIPE ).communicate()[ 0 ]
|
||||
sourceFile = False
|
||||
for line in diff.split( "\n" ):
|
||||
if match( "diff --git", line ):
|
||||
sourceFile = not not search( "\.(h|hpp|c|cpp)\s*$", line )
|
||||
if sourceFile and match( "\+ *\t", line ):
|
||||
return True
|
||||
return False
|
||||
env.Alias( "checkSource", [], [ tabs ] )
|
||||
env.AlwaysBuild( "checkSource" )
|
||||
|
||||
def gitPush( env, target, source ):
|
||||
import subprocess
|
||||
return subprocess.call( [ "git", "push" ] )
|
||||
env.Alias( "push", [ ".", "smoke", "checkSource" ], gitPush )
|
||||
env.AlwaysBuild( "push" )
|
||||
|
||||
|
||||
# ---- deploying ---
|
||||
|
||||
def s3push(localName, remoteName=None, platformDir=True):
|
||||
localName = str( localName )
|
||||
|
||||
if isBuildingLatest:
|
||||
remotePrefix = utils.getGitBranchString("-") + "-latest"
|
||||
else:
|
||||
remotePrefix = "-" + distName
|
||||
|
||||
findSettingsSetup()
|
||||
|
||||
import simples3
|
||||
import settings
|
||||
|
||||
s = simples3.S3Bucket( settings.bucket , settings.id , settings.key )
|
||||
|
||||
if remoteName is None:
|
||||
remoteName = localName
|
||||
|
||||
name = '%s-%s%s' % (remoteName , getSystemInstallName(), remotePrefix)
|
||||
lastDotIndex = localName.rfind('.')
|
||||
if lastDotIndex != -1:
|
||||
name += localName[lastDotIndex:]
|
||||
name = name.lower()
|
||||
|
||||
if platformDir:
|
||||
name = platform + "/" + name
|
||||
|
||||
print( "uploading " + localName + " to http://s3.amazonaws.com/" + s.name + "/" + name )
|
||||
if dontReplacePackage:
|
||||
for ( key , modify , etag , size ) in s.listdir( prefix=name ):
|
||||
print( "error: already a file with that name, not uploading" )
|
||||
Exit(2)
|
||||
s.put( name , open( localName , "rb" ).read() , acl="public-read" );
|
||||
print( " done uploading!" )
|
||||
|
||||
def s3shellpush( env , target , source ):
|
||||
s3push( "mongo" , "mongo-shell" )
|
||||
|
||||
env.Alias( "s3shell" , [ "mongo" ] , [ s3shellpush ] )
|
||||
env.AlwaysBuild( "s3shell" )
|
||||
|
||||
def s3dist( env , target , source ):
|
||||
s3push( str(source[0]) , "mongodb" )
|
||||
|
||||
env.AlwaysBuild(env.Alias( "s3dist" , [ '$SERVER_ARCHIVE' ] , [ s3dist ] ))
|
||||
|
||||
# --- an uninstall target ---
|
||||
if len(COMMAND_LINE_TARGETS) > 0 and 'uninstall' in COMMAND_LINE_TARGETS:
|
||||
SetOption("clean", 1)
|
||||
# By inspection, changing COMMAND_LINE_TARGETS here doesn't do
|
||||
# what we want, but changing BUILD_TARGETS does.
|
||||
BUILD_TARGETS.remove("uninstall")
|
||||
BUILD_TARGETS.append("install")
|
||||
|
||||
module_sconscripts = moduleconfig.get_module_sconscripts(mongo_modules)
|
||||
|
||||
# The following symbols are exported for use in subordinate SConscript files.
|
||||
@ -2188,7 +2053,6 @@ Export("v8version v8suffix")
|
||||
Export("boostSuffix")
|
||||
Export('module_sconscripts')
|
||||
Export("debugBuild optBuild")
|
||||
Export("s3push")
|
||||
Export("wiredtiger")
|
||||
|
||||
def injectMongoIncludePaths(thisEnv):
|
||||
@ -2200,19 +2064,4 @@ env.Alias("compiledb", env.CompilationDatabase('compile_commands.json'))
|
||||
env.SConscript('src/SConscript', variant_dir='$BUILD_DIR', duplicate=False)
|
||||
env.SConscript('SConscript.smoke')
|
||||
|
||||
def clean_old_dist_builds(env, target, source):
|
||||
prefix = "mongodb-%s-%s" % (platform, processor)
|
||||
filenames = sorted(os.listdir("."))
|
||||
filenames = [x for x in filenames if x.startswith(prefix)]
|
||||
to_keep = [x for x in filenames if x.endswith(".tgz") or x.endswith(".zip")][-2:]
|
||||
for filename in [x for x in filenames if x not in to_keep]:
|
||||
print "removing %s" % filename
|
||||
try:
|
||||
shutil.rmtree(filename)
|
||||
except:
|
||||
os.remove(filename)
|
||||
|
||||
env.Alias("dist_clean", [], [clean_old_dist_builds])
|
||||
env.AlwaysBuild("dist_clean")
|
||||
|
||||
env.Alias('all', ['core', 'tools', 'dbtest', 'unittests', 'file_allocator_bench'])
|
||||
env.Alias('all', ['core', 'tools', 'dbtest', 'unittests'])
|
||||
|
||||
@ -1,5 +1 @@
|
||||
|
||||
import os;
|
||||
|
||||
def findHacks( un ):
|
||||
return None
|
||||
|
||||
@ -1,40 +0,0 @@
|
||||
|
||||
import utils
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
def go( boost_root ):
|
||||
|
||||
OUTPUT = "src/third_party/boost"
|
||||
if os.path.exists( OUTPUT ):
|
||||
shutil.rmtree( OUTPUT )
|
||||
|
||||
cmd = [ "bcp" , "--scan" , "--boost=%s" % boost_root ]
|
||||
|
||||
src = utils.getAllSourceFiles()
|
||||
|
||||
cmd += src
|
||||
cmd.append( OUTPUT )
|
||||
|
||||
if not os.path.exists( OUTPUT ):
|
||||
os.makedirs( OUTPUT )
|
||||
|
||||
res = utils.execsys( cmd )
|
||||
|
||||
out = open( OUTPUT + "/bcp-out.txt" , 'w' )
|
||||
out.write( res[0] )
|
||||
out.close()
|
||||
|
||||
out = open( OUTPUT + "/notes.txt" , 'w' )
|
||||
out.write( "command: " + " ".join( cmd ) )
|
||||
out.close()
|
||||
|
||||
print( res[1] )
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) == 1:
|
||||
print( "usage: python %s <boost root directory>" % sys.argv[0] )
|
||||
sys.exit(1)
|
||||
go( sys.argv[1] )
|
||||
|
||||
@ -1,54 +0,0 @@
|
||||
@echo off
|
||||
|
||||
cls
|
||||
echo This script builds the boost libs that MongoDB requires on Windows.
|
||||
echo We assume boost source is in machine's \boost directory.
|
||||
echo You can get boost at www.boost.org.
|
||||
echo .
|
||||
echo Note: you will want boost v1.42 or higher with VS2010.
|
||||
echo .
|
||||
echo We assume you have bjam. To build bjam:
|
||||
echo cd tools\jam\src
|
||||
echo build.bat
|
||||
echo .
|
||||
|
||||
cd \boost
|
||||
echo bin\bjam --version
|
||||
bin\bjam --version
|
||||
|
||||
echo .
|
||||
echo .
|
||||
echo .
|
||||
echo About to build release libraries
|
||||
pause
|
||||
cls
|
||||
bin\bjam variant=release runtime-link=static link=static --with-filesystem --with-thread --with-date_time --with-program_options --layout=versioned threading=multi toolset=msvc
|
||||
echo .
|
||||
echo .
|
||||
echo .
|
||||
echo About to try to move libs from /boost/stage/lib to /boost/lib/
|
||||
pause
|
||||
cls
|
||||
rem bjam makes extra copies without the ver #; we kill those:
|
||||
del stage\lib\*s.lib
|
||||
move stage\lib\* lib\
|
||||
|
||||
echo .
|
||||
echo .
|
||||
echo .
|
||||
echo About to build debug libraries
|
||||
pause
|
||||
cls
|
||||
bin\bjam variant=debug --with-filesystem --with-thread --with-date_time --with-program_options --layout=versioned threading=multi toolset=msvc
|
||||
|
||||
echo .
|
||||
echo .
|
||||
echo .
|
||||
echo About to try to move libs from /boost/stage/lib to /boost/lib/
|
||||
pause
|
||||
cls
|
||||
rem bjam makes extra copies without the ver #; we kill those:
|
||||
del stage\lib\*-gd.lib
|
||||
move stage\lib\* lib\
|
||||
|
||||
echo Done - try running "dir \boost\lib\"
|
||||
@ -1,61 +0,0 @@
|
||||
@echo off
|
||||
|
||||
rem 64 bit version
|
||||
rem address-model=64
|
||||
|
||||
rem run
|
||||
rem bin\bjam --clean
|
||||
rem if you switch compilers etc.
|
||||
|
||||
cls
|
||||
echo This script builds the (64 bit) boost libs that MongoDB requires on Windows.
|
||||
echo We assume boost source is in machine's \boost directory.
|
||||
echo You can get boost at www.boost.org.
|
||||
echo .
|
||||
echo Note: you will want boost v1.42 or higher with VS2010.
|
||||
echo .
|
||||
echo We assume you have bjam. To build bjam:
|
||||
echo cd tools\jam\src
|
||||
echo build.bat
|
||||
echo .
|
||||
|
||||
cd \boost
|
||||
echo bin\bjam --version
|
||||
bin\bjam --version
|
||||
|
||||
echo .
|
||||
echo .
|
||||
echo .
|
||||
echo About to build release libraries
|
||||
pause
|
||||
cls
|
||||
bin\bjam --build-dir=c:\temp\boost64 address-model=64 variant=release runtime-link=static link=static --with-filesystem --with-thread --with-date_time --with-program_options --layout=versioned threading=multi toolset=msvc
|
||||
echo .
|
||||
echo .
|
||||
echo .
|
||||
echo About to try to move libs from /boost/stage/lib to /boost/lib/
|
||||
pause
|
||||
cls
|
||||
rem bjam makes extra copies without the ver #; we kill those:
|
||||
del stage\lib\*s.lib
|
||||
move stage\lib\* lib\
|
||||
|
||||
echo .
|
||||
echo .
|
||||
echo .
|
||||
echo About to build debug libraries
|
||||
pause
|
||||
cls
|
||||
bin\bjam --build-dir=c:\temp\boost64 address-model=64 variant=debug --with-filesystem --with-thread --with-date_time --with-program_options --layout=versioned threading=multi toolset=msvc
|
||||
|
||||
echo .
|
||||
echo .
|
||||
echo .
|
||||
echo About to try to move libs from /boost/stage/lib to /boost/lib/
|
||||
pause
|
||||
cls
|
||||
rem bjam makes extra copies without the ver #; we kill those:
|
||||
del stage\lib\*-gd.lib
|
||||
move stage\lib\* lib\
|
||||
|
||||
echo Done - try running "dir \boost\lib\"
|
||||
@ -1,107 +0,0 @@
|
||||
// FileLock.java
|
||||
|
||||
import java.io.*;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.*;
|
||||
|
||||
/**
|
||||
* "locks" a resource by using the file system as storage
|
||||
* file has 1 line
|
||||
* <incarnation> <last ping time in millis>
|
||||
*/
|
||||
public class FileLock {
|
||||
|
||||
public FileLock( String logicalName )
|
||||
throws IOException {
|
||||
|
||||
_file = new File( "/tmp/java-fileLock-" + logicalName );
|
||||
_incarnation = "xxx" + Math.random() + "yyy";
|
||||
|
||||
if ( ! _file.exists() ) {
|
||||
FileOutputStream fout = new FileOutputStream( _file );
|
||||
fout.write( "\n".getBytes() );
|
||||
fout.close();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* takes lock
|
||||
* if someone else has it, blocks until the other one finishes
|
||||
*/
|
||||
public void lock()
|
||||
throws IOException {
|
||||
if ( _lock != null )
|
||||
throw new IllegalStateException( "can't lock when you're locked" );
|
||||
|
||||
try {
|
||||
_semaphore.acquire();
|
||||
}
|
||||
catch ( InterruptedException ie ) {
|
||||
throw new RuntimeException( "sad" , ie );
|
||||
}
|
||||
|
||||
_raf = new RandomAccessFile( _file , "rw" );
|
||||
_lock = _raf.getChannel().lock();
|
||||
}
|
||||
|
||||
public void unlock()
|
||||
throws IOException {
|
||||
|
||||
if ( _lock == null )
|
||||
throw new IllegalStateException( "can't unlock when you're not locked" );
|
||||
|
||||
_lock.release();
|
||||
_semaphore.release();
|
||||
|
||||
_locked = false;
|
||||
}
|
||||
|
||||
final File _file;
|
||||
final String _incarnation;
|
||||
|
||||
private RandomAccessFile _raf;
|
||||
private java.nio.channels.FileLock _lock;
|
||||
|
||||
private boolean _locked;
|
||||
|
||||
private static Semaphore _semaphore = new Semaphore(1);
|
||||
|
||||
|
||||
public static void main( final String[] args )
|
||||
throws Exception {
|
||||
|
||||
List<Thread> threads = new ArrayList<Thread>();
|
||||
|
||||
for ( int i=0; i<3; i++ ) {
|
||||
|
||||
threads.add( new Thread() {
|
||||
public void run() {
|
||||
try {
|
||||
FileLock lock = new FileLock( args[0] );
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
|
||||
lock.lock();
|
||||
System.out.println( "time to lock:\t" + (System.currentTimeMillis()-start) );
|
||||
Thread.sleep( Integer.parseInt( args[1] ) );
|
||||
lock.unlock();
|
||||
System.out.println( "total time:\t" + (System.currentTimeMillis()-start) );
|
||||
}
|
||||
catch ( Exception e ) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
} );
|
||||
}
|
||||
|
||||
for ( Thread t : threads ) {
|
||||
t.start();
|
||||
}
|
||||
|
||||
for ( Thread t : threads ) {
|
||||
t.join();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@ -1,156 +0,0 @@
|
||||
// IOUtil.java
|
||||
|
||||
import java.io.*;
|
||||
import java.net.*;
|
||||
import java.util.*;
|
||||
|
||||
public class IOUtil {
|
||||
|
||||
public static String urlFileName( String url ) {
|
||||
int idx = url.lastIndexOf( "/" );
|
||||
if ( idx < 0 )
|
||||
return url;
|
||||
return url.substring( idx + 1 );
|
||||
}
|
||||
|
||||
public static long pipe( InputStream in , OutputStream out )
|
||||
throws IOException {
|
||||
|
||||
long bytes = 0;
|
||||
|
||||
byte[] buf = new byte[2048];
|
||||
|
||||
while ( true ) {
|
||||
int x = in.read( buf );
|
||||
if ( x < 0 )
|
||||
break;
|
||||
|
||||
bytes += x;
|
||||
out.write( buf , 0 , x );
|
||||
}
|
||||
|
||||
return bytes;
|
||||
}
|
||||
|
||||
public static class PipingThread extends Thread {
|
||||
public PipingThread( InputStream in , OutputStream out ) {
|
||||
_in = in;
|
||||
_out = out;
|
||||
|
||||
_wrote = 0;
|
||||
}
|
||||
|
||||
public void run() {
|
||||
try {
|
||||
_wrote = pipe( _in , _out );
|
||||
}
|
||||
catch ( IOException ioe ) {
|
||||
ioe.printStackTrace();
|
||||
_wrote = -1;
|
||||
}
|
||||
}
|
||||
|
||||
public long wrote() {
|
||||
return _wrote;
|
||||
}
|
||||
|
||||
long _wrote;
|
||||
|
||||
final InputStream _in;
|
||||
final OutputStream _out;
|
||||
}
|
||||
|
||||
public static String readStringFully( InputStream in )
|
||||
throws IOException {
|
||||
|
||||
ByteArrayOutputStream bout = new ByteArrayOutputStream();
|
||||
pipe( in , bout );
|
||||
return new String( bout.toByteArray() , "UTF8" );
|
||||
|
||||
}
|
||||
|
||||
public static Map<String,Object> readPythonSettings( File file )
|
||||
throws IOException {
|
||||
|
||||
String all = readStringFully( new FileInputStream( file ) );
|
||||
|
||||
Map<String,Object> map = new TreeMap<String,Object>();
|
||||
|
||||
for ( String line : all.split( "\n" ) ) {
|
||||
line = line.trim();
|
||||
if ( line.length() == 0 )
|
||||
continue;
|
||||
|
||||
String[] pcs = line.split( "=" );
|
||||
if ( pcs.length != 2 )
|
||||
continue;
|
||||
|
||||
String name = pcs[0].trim();
|
||||
String value = pcs[1].trim();
|
||||
|
||||
if ( value.startsWith( "\"" ) ) {
|
||||
map.put( name , value.substring( 1 , value.length() - 1 ) );
|
||||
}
|
||||
else {
|
||||
map.put( name , Long.parseLong( value ) );
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
public static String[] runCommand( String cmd , File dir )
|
||||
throws IOException {
|
||||
|
||||
Process p = Runtime.getRuntime().exec( cmd.split( " +" ) , new String[]{} , dir );
|
||||
String[] results = new String[]{ IOUtil.readStringFully( p.getInputStream() ) , IOUtil.readStringFully( p.getErrorStream() ) };
|
||||
try {
|
||||
if ( p.waitFor() != 0 )
|
||||
throw new RuntimeException( "command failed [" + cmd + "]\n" + results[0] + "\n" + results[1] );
|
||||
}
|
||||
catch ( InterruptedException ie ) {
|
||||
throw new RuntimeException( "uh oh" );
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
|
||||
public static void download( String http , File localDir )
|
||||
throws IOException {
|
||||
|
||||
File f = localDir;
|
||||
f.mkdirs();
|
||||
|
||||
f = new File( f.toString() + File.separator + urlFileName( http ) );
|
||||
|
||||
System.out.println( "downloading\n\t" + http + "\n\t" + f );
|
||||
|
||||
if ( f.exists() ) {
|
||||
System.out.println( "\t already exists" );
|
||||
return;
|
||||
}
|
||||
|
||||
URL url = new URL( http );
|
||||
|
||||
InputStream in = url.openConnection().getInputStream();
|
||||
OutputStream out = new FileOutputStream( f );
|
||||
|
||||
pipe( in , out );
|
||||
|
||||
out.close();
|
||||
in.close();
|
||||
|
||||
}
|
||||
|
||||
public static void main( String[] args )
|
||||
throws Exception {
|
||||
|
||||
|
||||
byte[] data = new byte[]{ 'e' , 'r' , 'h' , 0 };
|
||||
System.out.write( data );
|
||||
System.out.println( "yo" );
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,2 +0,0 @@
|
||||
Manifest-Version: 1.0
|
||||
Main-Class: emr
|
||||
@ -1,380 +0,0 @@
|
||||
// emr.java
|
||||
|
||||
import java.io.*;
|
||||
import java.util.*;
|
||||
import java.net.*;
|
||||
|
||||
import org.apache.hadoop.conf.*;
|
||||
import org.apache.hadoop.io.*;
|
||||
import org.apache.hadoop.mapred.*;
|
||||
import org.apache.hadoop.fs.*;
|
||||
|
||||
|
||||
public class emr {
|
||||
|
||||
static class MongoSuite {
|
||||
String mongo;
|
||||
String code;
|
||||
String workingDir;
|
||||
|
||||
String suite;
|
||||
|
||||
void copy( MongoSuite c ) {
|
||||
mongo = c.mongo;
|
||||
code = c.code;
|
||||
workingDir = c.workingDir;
|
||||
|
||||
suite = c.suite;
|
||||
|
||||
}
|
||||
|
||||
void downloadTo( File localDir )
|
||||
throws IOException {
|
||||
IOUtil.download( mongo , localDir );
|
||||
IOUtil.download( code , localDir );
|
||||
}
|
||||
|
||||
boolean runTest()
|
||||
throws IOException {
|
||||
|
||||
// mkdir
|
||||
File dir = new File( workingDir , suite );
|
||||
dir.mkdirs();
|
||||
|
||||
// download
|
||||
System.out.println( "going to download" );
|
||||
downloadTo( dir );
|
||||
|
||||
|
||||
// explode
|
||||
System.out.println( "going to explode" );
|
||||
IOUtil.runCommand( "tar zxvf " + IOUtil.urlFileName( code ) , dir );
|
||||
String[] res = IOUtil.runCommand( "tar zxvf " + IOUtil.urlFileName( mongo ) , dir );
|
||||
for ( String x : res[0].split( "\n" ) ) {
|
||||
if ( x.indexOf( "/bin/" ) < 0 )
|
||||
continue;
|
||||
File f = new File( dir.toString() , x );
|
||||
if ( ! f.renameTo( new File( dir , IOUtil.urlFileName( x ) ) ) )
|
||||
throw new RuntimeException( "rename failed" );
|
||||
}
|
||||
|
||||
List<String> cmd = new ArrayList<String>();
|
||||
cmd.add( "/usr/bin/python" );
|
||||
cmd.add( "buildscripts/smoke.py" );
|
||||
|
||||
File log_config = new File( dir , "log_config.py" );
|
||||
System.out.println( "log_config: " + log_config.exists() );
|
||||
if ( log_config.exists() ) {
|
||||
|
||||
java.util.Map<String,Object> properties = IOUtil.readPythonSettings( log_config );
|
||||
|
||||
cmd.add( "--buildlogger-builder" );
|
||||
cmd.add( properties.get( "name" ).toString() );
|
||||
|
||||
cmd.add( "--buildlogger-buildnum" );
|
||||
cmd.add( properties.get( "number" ).toString() );
|
||||
|
||||
cmd.add( "--buildlogger-credentials" );
|
||||
cmd.add( "log_config.py" );
|
||||
|
||||
cmd.add( "--buildlogger-phase" );
|
||||
{
|
||||
int idx = suite.lastIndexOf( "/" );
|
||||
if ( idx < 0 )
|
||||
cmd.add( suite );
|
||||
else
|
||||
cmd.add( suite.substring( 0 , idx ) );
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
cmd.add( suite );
|
||||
|
||||
System.out.println( cmd );
|
||||
|
||||
Process p = Runtime.getRuntime().exec( cmd.toArray( new String[cmd.size()] ) , new String[]{} , dir );
|
||||
|
||||
List<Thread> threads = new ArrayList<Thread>();
|
||||
threads.add( new IOUtil.PipingThread( p.getInputStream() , System.out ) );
|
||||
threads.add( new IOUtil.PipingThread( p.getErrorStream() , System.out ) );
|
||||
|
||||
for ( Thread t : threads )
|
||||
t.start();
|
||||
|
||||
try {
|
||||
for ( Thread t : threads ) {
|
||||
t.join();
|
||||
}
|
||||
int rc = p.waitFor();
|
||||
return rc == 0;
|
||||
}
|
||||
catch ( InterruptedException ie ) {
|
||||
ie.printStackTrace();
|
||||
throw new RuntimeException( "sad" , ie );
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public void readFields( DataInput in )
|
||||
throws IOException {
|
||||
mongo = in.readUTF();
|
||||
code = in.readUTF();
|
||||
workingDir = in.readUTF();
|
||||
|
||||
suite = in.readUTF();
|
||||
}
|
||||
|
||||
public void write( final DataOutput out )
|
||||
throws IOException {
|
||||
out.writeUTF( mongo );
|
||||
out.writeUTF( code );
|
||||
out.writeUTF( workingDir );
|
||||
|
||||
out.writeUTF( suite );
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
return "mongo: " + mongo + " code: " + code + " suite: " + suite + " workingDir: " + workingDir;
|
||||
}
|
||||
}
|
||||
|
||||
public static class Map implements Mapper<Text, MongoSuite, Text, IntWritable> {
|
||||
|
||||
public void map( Text key, MongoSuite value, OutputCollector<Text,IntWritable> output, Reporter reporter )
|
||||
throws IOException {
|
||||
|
||||
FileLock lock = new FileLock( "mapper" );
|
||||
try {
|
||||
lock.lock();
|
||||
|
||||
System.out.println( "key: " + key );
|
||||
System.out.println( "value: " + value );
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
boolean passed = value.runTest();
|
||||
long end = System.currentTimeMillis();
|
||||
|
||||
output.collect( new Text( passed ? "passed" : "failed" ) , new IntWritable( 1 ) );
|
||||
output.collect( new Text( key.toString() + "-time-seconds" ) , new IntWritable( (int)((end-start)/(1000)) ) );
|
||||
output.collect( new Text( key.toString() + "-passed" ) , new IntWritable( passed ? 1 : 0 ) );
|
||||
|
||||
String ip = IOUtil.readStringFully( new URL( "http://myip.10gen.com/" ).openConnection().getInputStream() );
|
||||
ip = ip.substring( ip.indexOf( ":" ) + 1 ).trim();
|
||||
output.collect( new Text( ip ) , new IntWritable(1) );
|
||||
}
|
||||
catch ( RuntimeException re ) {
|
||||
re.printStackTrace();
|
||||
throw re;
|
||||
}
|
||||
catch ( IOException ioe ) {
|
||||
ioe.printStackTrace();
|
||||
throw ioe;
|
||||
}
|
||||
finally {
|
||||
lock.unlock();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public void configure(JobConf job) {}
|
||||
public void close(){}
|
||||
}
|
||||
|
||||
public static class Reduce implements Reducer<Text, IntWritable, Text, IntWritable> {
|
||||
|
||||
public void reduce( Text key, Iterator<IntWritable> values, OutputCollector<Text,IntWritable> output , Reporter reporter )
|
||||
throws IOException {
|
||||
|
||||
int sum = 0;
|
||||
while ( values.hasNext() ) {
|
||||
sum += values.next().get();
|
||||
}
|
||||
output.collect( key , new IntWritable( sum ) );
|
||||
}
|
||||
|
||||
public void configure(JobConf job) {}
|
||||
public void close(){}
|
||||
}
|
||||
|
||||
public static class MySplit implements InputSplit , Writable {
|
||||
|
||||
public MySplit(){
|
||||
}
|
||||
|
||||
MySplit( MongoSuite config , int length ) {
|
||||
_config = config;
|
||||
_length = length;
|
||||
}
|
||||
|
||||
public long getLength() {
|
||||
return _length;
|
||||
}
|
||||
|
||||
public String[] getLocations() {
|
||||
return new String[0];
|
||||
}
|
||||
|
||||
public void readFields( DataInput in )
|
||||
throws IOException {
|
||||
_config = new MongoSuite();
|
||||
_config.readFields( in );
|
||||
_length = in.readInt();
|
||||
}
|
||||
|
||||
public void write( final DataOutput out )
|
||||
throws IOException {
|
||||
_config.write( out );
|
||||
out.writeInt( _length );
|
||||
}
|
||||
|
||||
MongoSuite _config;
|
||||
int _length;
|
||||
}
|
||||
|
||||
public static class InputMagic implements InputFormat<Text,MongoSuite> {
|
||||
|
||||
public RecordReader<Text,MongoSuite> getRecordReader( InputSplit split, JobConf job , Reporter reporter ){
|
||||
final MySplit s = (MySplit)split;
|
||||
return new RecordReader<Text,MongoSuite>() {
|
||||
|
||||
public void close(){}
|
||||
|
||||
public Text createKey() {
|
||||
return new Text();
|
||||
}
|
||||
|
||||
public MongoSuite createValue() {
|
||||
return new MongoSuite();
|
||||
}
|
||||
|
||||
public long getPos() {
|
||||
return _seen ? 1 : 0;
|
||||
}
|
||||
|
||||
public float getProgress() {
|
||||
return getPos();
|
||||
}
|
||||
|
||||
public boolean next( Text key , MongoSuite value ) {
|
||||
key.set( s._config.suite );
|
||||
value.copy( s._config );
|
||||
|
||||
|
||||
boolean x = _seen;
|
||||
_seen = true;
|
||||
return !x;
|
||||
}
|
||||
|
||||
boolean _seen = false;
|
||||
};
|
||||
}
|
||||
|
||||
public InputSplit[] getSplits( JobConf job , int numSplits ){
|
||||
String[] pcs = job.get( "suites" ).split(",");
|
||||
InputSplit[] splits = new InputSplit[pcs.length];
|
||||
for ( int i=0; i<splits.length; i++ ) {
|
||||
MongoSuite c = new MongoSuite();
|
||||
c.suite = pcs[i];
|
||||
|
||||
c.mongo = job.get( "mongo" );
|
||||
c.code = job.get( "code" );
|
||||
c.workingDir = job.get( "workingDir" );
|
||||
|
||||
splits[i] = new MySplit( c , 100 /* XXX */);
|
||||
}
|
||||
return splits;
|
||||
}
|
||||
|
||||
public void validateInput(JobConf job){}
|
||||
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* args
|
||||
* mongo tgz
|
||||
* code tgz
|
||||
* output path
|
||||
* tests to run ?
|
||||
*/
|
||||
|
||||
public static void main( String[] args ) throws Exception{
|
||||
|
||||
JobConf conf = new JobConf();
|
||||
conf.setJarByClass(emr.class);
|
||||
|
||||
String workingDir = "/data/db/emr/";
|
||||
|
||||
|
||||
// parse args
|
||||
|
||||
int pos = 0;
|
||||
for ( ; pos < args.length; pos++ ) {
|
||||
if ( ! args[pos].startsWith( "--" ) )
|
||||
break;
|
||||
|
||||
String arg = args[pos].substring(2);
|
||||
if ( arg.equals( "workingDir" ) ) {
|
||||
workingDir = args[++pos];
|
||||
}
|
||||
else {
|
||||
System.err.println( "unknown arg: " + arg );
|
||||
throw new RuntimeException( "unknown arg: " + arg );
|
||||
}
|
||||
}
|
||||
|
||||
String mongo = args[pos++];
|
||||
String code = args[pos++];
|
||||
String output = args[pos++];
|
||||
|
||||
String suites = "";
|
||||
for ( ; pos < args.length; pos++ ) {
|
||||
if ( suites.length() > 0 )
|
||||
suites += ",";
|
||||
suites += args[pos];
|
||||
}
|
||||
|
||||
if ( suites.length() == 0 )
|
||||
throw new RuntimeException( "no suites" );
|
||||
|
||||
System.out.println( "workingDir:\t" + workingDir );
|
||||
System.out.println( "mongo:\t" + mongo );
|
||||
System.out.println( "code:\t " + code );
|
||||
System.out.println( "output\t: " + output );
|
||||
System.out.println( "suites\t: " + suites );
|
||||
|
||||
if ( false ) {
|
||||
MongoSuite s = new MongoSuite();
|
||||
s.mongo = mongo;
|
||||
s.code = code;
|
||||
s.workingDir = workingDir;
|
||||
s.suite = suites;
|
||||
s.runTest();
|
||||
return;
|
||||
}
|
||||
|
||||
// main hadoop set
|
||||
conf.set( "mongo" , mongo );
|
||||
conf.set( "code" , code );
|
||||
conf.set( "workingDir" , workingDir );
|
||||
conf.set( "suites" , suites );
|
||||
|
||||
conf.set( "mapred.map.tasks" , "1" );
|
||||
conf.setLong( "mapred.task.timeout" , 4 * 3600 * 1000 /* 4 hours */);
|
||||
|
||||
conf.setOutputKeyClass(Text.class);
|
||||
conf.setOutputValueClass(IntWritable.class);
|
||||
|
||||
conf.setMapperClass(Map.class);
|
||||
conf.setReducerClass(Reduce.class);
|
||||
|
||||
conf.setInputFormat(InputMagic.class);
|
||||
conf.setOutputFormat(TextOutputFormat.class);
|
||||
|
||||
FileOutputFormat.setOutputPath(conf, new Path(output) );
|
||||
|
||||
// actually run
|
||||
|
||||
JobClient.runJob( conf );
|
||||
}
|
||||
}
|
||||
@ -1,385 +0,0 @@
|
||||
|
||||
import os
|
||||
import sys
|
||||
import shutil
|
||||
import datetime
|
||||
import time
|
||||
import subprocess
|
||||
import urllib
|
||||
import urllib2
|
||||
import json
|
||||
import pprint
|
||||
|
||||
import boto
|
||||
import simples3
|
||||
|
||||
import pymongo
|
||||
|
||||
def findSettingsSetup():
|
||||
sys.path.append( "./" )
|
||||
sys.path.append( "../" )
|
||||
sys.path.append( "../../" )
|
||||
sys.path.append( "../../../" )
|
||||
|
||||
findSettingsSetup()
|
||||
import settings
|
||||
import buildscripts.utils as utils
|
||||
import buildscripts.smoke as smoke
|
||||
|
||||
bucket = simples3.S3Bucket( settings.emr_bucket , settings.emr_id , settings.emr_key )
|
||||
|
||||
def _get_status():
|
||||
|
||||
def gh( cmds ):
|
||||
txt = ""
|
||||
for cmd in cmds:
|
||||
res = utils.execsys( "git " + cmd )
|
||||
txt = txt + res[0] + res[1]
|
||||
return utils.md5string( txt )
|
||||
|
||||
return "%s-%s" % ( utils.execsys( "git describe" )[0].strip(), gh( [ "diff" , "status" ] ) )
|
||||
|
||||
def _get_most_recent_tgz( prefix ):
|
||||
# this is icky, but works for now
|
||||
all = []
|
||||
for x in os.listdir( "." ):
|
||||
if not x.startswith( prefix ) or not x.endswith( ".tgz" ):
|
||||
continue
|
||||
all.append( ( x , os.stat(x).st_mtime ) )
|
||||
|
||||
if len(all) == 0:
|
||||
raise Exception( "can't find file with prefix: " + prefix )
|
||||
|
||||
all.sort( lambda x,y: int(y[1] - x[1]) )
|
||||
|
||||
return all[0][0]
|
||||
|
||||
def get_build_info():
|
||||
return ( os.environ.get('MONGO_BUILDER_NAME') , os.environ.get('MONGO_BUILD_NUMBER') )
|
||||
|
||||
def make_tarball():
|
||||
|
||||
m = _get_most_recent_tgz( "mongodb-" )
|
||||
|
||||
c = "test-code-emr.tgz"
|
||||
tar = "tar zcf %s src jstests buildscripts" % c
|
||||
|
||||
log_config = "log_config.py"
|
||||
if os.path.exists( log_config ):
|
||||
os.unlink( log_config )
|
||||
|
||||
credentials = do_credentials()
|
||||
if credentials:
|
||||
|
||||
builder , buildnum = get_build_info()
|
||||
|
||||
if builder and buildnum:
|
||||
|
||||
file = open( log_config , "wb" )
|
||||
file.write( 'username="%s"\npassword="%s"\n' % credentials )
|
||||
file.write( 'name="%s"\nnumber=%s\n'% ( builder , buildnum ) )
|
||||
|
||||
file.close()
|
||||
|
||||
tar = tar + " " + log_config
|
||||
|
||||
utils.execsys( tar )
|
||||
return ( m , c )
|
||||
|
||||
def _put_ine( bucket , local , remote ):
|
||||
print( "going to put\n\t%s\n\thttp://%s.s3.amazonaws.com/%s" % ( local , settings.emr_bucket , remote ) )
|
||||
|
||||
for x in bucket.listdir( prefix=remote ):
|
||||
print( "\talready existed" )
|
||||
return remote
|
||||
|
||||
bucket.put( remote , open( local , "rb" ).read() , acl="public-read" )
|
||||
return remote
|
||||
|
||||
def build_jar():
|
||||
root = "build/emrjar"
|
||||
src = "buildscripts/emr"
|
||||
|
||||
if os.path.exists( root ):
|
||||
shutil.rmtree( root )
|
||||
os.makedirs( root )
|
||||
|
||||
for x in os.listdir( src ):
|
||||
if not x.endswith( ".java" ):
|
||||
continue
|
||||
shutil.copyfile( src + "/" + x , root + "/" + x )
|
||||
shutil.copyfile( src + "/MANIFEST.MF" , root + "/MANIFEST.FM" )
|
||||
|
||||
classpath = os.listdir( src + "/lib" )
|
||||
for x in classpath:
|
||||
shutil.copyfile( src + "/lib/" + x , root + "/" + x )
|
||||
classpath.append( "." )
|
||||
classpath = ":".join(classpath)
|
||||
|
||||
for x in os.listdir( root ):
|
||||
if x.endswith( ".java" ):
|
||||
if subprocess.call( [ "javac" , "-cp" , classpath , x ] , cwd=root) != 0:
|
||||
raise Exception( "compiled failed" )
|
||||
|
||||
args = [ "jar" , "-cfm" , "emr.jar" , "MANIFEST.FM" ]
|
||||
for x in os.listdir( root ):
|
||||
if x.endswith( ".class" ):
|
||||
args.append( x )
|
||||
subprocess.call( args , cwd=root )
|
||||
|
||||
shutil.copyfile( root + "/emr.jar" , "emr.jar" )
|
||||
|
||||
return "emr.jar"
|
||||
|
||||
def push():
|
||||
mongo , test_code = make_tarball()
|
||||
print( mongo )
|
||||
print( test_code )
|
||||
|
||||
root = "emr/%s/%s" % ( datetime.date.today().strftime("%Y-%m-%d") , os.uname()[0].lower() )
|
||||
|
||||
def make_long_name(local,hash):
|
||||
pcs = local.rpartition( "." )
|
||||
h = _get_status()
|
||||
if hash:
|
||||
h = utils.md5sum( local )
|
||||
return "%s/%s-%s.%s" % ( root , pcs[0] , h , pcs[2] )
|
||||
|
||||
mongo = _put_ine( bucket , mongo , make_long_name( mongo , False ) )
|
||||
test_code = _put_ine( bucket , test_code , make_long_name( test_code , True ) )
|
||||
|
||||
jar = build_jar()
|
||||
jar = _put_ine( bucket , jar , make_long_name( jar , False ) )
|
||||
|
||||
setup = "buildscripts/emr/emrnodesetup.sh"
|
||||
setup = _put_ine( bucket , setup , make_long_name( setup , True ) )
|
||||
|
||||
return mongo , test_code , jar , setup
|
||||
|
||||
def run_tests( things , tests ):
|
||||
if len(tests) == 0:
|
||||
raise Exception( "no tests" )
|
||||
oldNum = len(tests)
|
||||
tests = fix_suites( tests )
|
||||
print( "tests expanded from %d to %d" % ( oldNum , len(tests) ) )
|
||||
|
||||
print( "things:%s\ntests:%s\n" % ( things , tests ) )
|
||||
|
||||
emr = boto.connect_emr( settings.emr_id , settings.emr_key )
|
||||
|
||||
def http(path):
|
||||
return "http://%s.s3.amazonaws.com/%s" % ( settings.emr_bucket , path )
|
||||
|
||||
run_s3_path = "emr/%s/%s/%s/" % ( os.getenv( "USER" ) ,
|
||||
os.getenv( "HOST" ) ,
|
||||
datetime.datetime.today().strftime( "%Y%m%d-%H%M" ) )
|
||||
|
||||
run_s3_root = "s3n://%s/%s/" % ( settings.emr_bucket , run_s3_path )
|
||||
|
||||
out = run_s3_root + "out"
|
||||
logs = run_s3_root + "logs"
|
||||
|
||||
jar="s3n://%s/%s" % ( settings.emr_bucket , things[2] )
|
||||
step_args=[ http(things[0]) , http(things[1]) , out , ",".join(tests) ]
|
||||
|
||||
step = boto.emr.step.JarStep( "emr main" , jar=jar,step_args=step_args )
|
||||
print( "jar:%s\nargs:%s" % ( jar , step_args ) )
|
||||
|
||||
setup = boto.emr.BootstrapAction( "setup" , "s3n://%s/%s" % ( settings.emr_bucket , things[3] ) , [] )
|
||||
|
||||
jobid = emr.run_jobflow( name = "Mongo EMR for %s from %s" % ( os.getenv( "USER" ) , os.getenv( "HOST" ) ) ,
|
||||
ec2_keyname = "emr1" ,
|
||||
slave_instance_type = "m1.large" ,
|
||||
ami_version = "latest" ,
|
||||
num_instances=5 ,
|
||||
log_uri = logs ,
|
||||
bootstrap_actions = [ setup ] ,
|
||||
steps = [ step ] )
|
||||
|
||||
|
||||
print( "%s jobid: %s" % ( datetime.datetime.today() , jobid ) )
|
||||
|
||||
while ( True ):
|
||||
flow = emr.describe_jobflow( jobid )
|
||||
print( "%s status: %s" % ( datetime.datetime.today() , flow.state ) )
|
||||
if flow.state == "COMPLETED" or flow.state == "FAILED":
|
||||
break
|
||||
time.sleep(30)
|
||||
|
||||
syncdir = "build/emrout/" + jobid + "/"
|
||||
sync_s3( run_s3_path , syncdir )
|
||||
|
||||
final_out = "build/emrout/" + jobid + "/"
|
||||
|
||||
print("output in: " + final_out )
|
||||
do_output( final_out )
|
||||
|
||||
def sync_s3( remote_dir , local_dir ):
|
||||
for x in bucket.listdir( remote_dir ):
|
||||
out = local_dir + "/" + x[0]
|
||||
|
||||
if os.path.exists( out ) and x[2].find( utils.md5sum( out ) ) >= 0:
|
||||
continue
|
||||
|
||||
dir = out.rpartition( "/" )[0]
|
||||
if not os.path.exists( dir ):
|
||||
os.makedirs( dir )
|
||||
|
||||
thing = bucket.get( x[0] )
|
||||
open( out , "wb" ).write( thing.read() )
|
||||
|
||||
def fix_suites( suites ):
|
||||
fixed = []
|
||||
for name,x in smoke.expand_suites( suites , False ):
|
||||
idx = name.find( "/jstests" )
|
||||
if idx >= 0:
|
||||
name = name[idx+1:]
|
||||
fixed.append( name )
|
||||
return fixed
|
||||
|
||||
def do_credentials():
|
||||
root = "buildbot.tac"
|
||||
|
||||
while len(root) < 40 :
|
||||
if os.path.exists( root ):
|
||||
break
|
||||
root = "../" + root
|
||||
|
||||
if not os.path.exists( root ):
|
||||
return None
|
||||
|
||||
credentials = {}
|
||||
execfile(root, credentials, credentials)
|
||||
|
||||
if "slavename" not in credentials:
|
||||
return None
|
||||
|
||||
if "passwd" not in credentials:
|
||||
return None
|
||||
|
||||
return ( credentials["slavename"] , credentials["passwd"] )
|
||||
|
||||
|
||||
def do_output( dir ):
|
||||
|
||||
def go_down( start ):
|
||||
lst = os.listdir(dir)
|
||||
if len(lst) != 1:
|
||||
raise Exception( "sad: " + start )
|
||||
return start + "/" + lst[0]
|
||||
|
||||
while "out" not in os.listdir( dir ):
|
||||
dir = go_down( dir )
|
||||
|
||||
dir = dir + "/out"
|
||||
|
||||
pieces = os.listdir(dir)
|
||||
pieces.sort()
|
||||
|
||||
passed = []
|
||||
failed = []
|
||||
times = {}
|
||||
|
||||
for x in pieces:
|
||||
if not x.startswith( "part" ):
|
||||
continue
|
||||
full = dir + "/" + x
|
||||
|
||||
for line in open( full , "rb" ):
|
||||
if line.find( "-passed" ) >= 0:
|
||||
passed.append( line.partition( "-passed" )[0] )
|
||||
continue
|
||||
|
||||
if line.find( "-failed" ) >= 0:
|
||||
failed.append( line.partition( "-failed" )[0] )
|
||||
continue
|
||||
|
||||
if line.find( "-time-seconds" ) >= 0:
|
||||
p = line.partition( "-time-seconds" )
|
||||
times[p[0]] = p[2].strip()
|
||||
continue
|
||||
|
||||
print( "\t" + line.strip() )
|
||||
|
||||
def print_list(name,lst):
|
||||
print( name )
|
||||
for x in lst:
|
||||
print( "\t%s\t%s" % ( x , times[x] ) )
|
||||
|
||||
print_list( "passed" , passed )
|
||||
print_list( "failed" , failed )
|
||||
|
||||
if do_credentials():
|
||||
builder , buildnum = get_build_info()
|
||||
if builder and buildnum:
|
||||
conn = pymongo.Connection( "bbout1.10gen.cc" )
|
||||
db = conn.buildlogs
|
||||
q = { "builder" : builder , "buildnum" : int(buildnum) }
|
||||
doc = db.builds.find_one( q )
|
||||
|
||||
if doc:
|
||||
print( "\nhttp://buildlogs.mongodb.org/build/%s" % doc["_id"] )
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) == 1:
|
||||
print( "need an arg" )
|
||||
|
||||
elif sys.argv[1] == "tarball":
|
||||
make_tarball()
|
||||
elif sys.argv[1] == "jar":
|
||||
build_jar()
|
||||
elif sys.argv[1] == "push":
|
||||
print( push() )
|
||||
|
||||
elif sys.argv[1] == "sync":
|
||||
sync_s3( sys.argv[2] , sys.argv[3] )
|
||||
|
||||
elif sys.argv[1] == "fix_suites":
|
||||
for x in fix_suites( sys.argv[2:] ):
|
||||
print(x)
|
||||
|
||||
elif sys.argv[1] == "credentials":
|
||||
print( do_credentials() )
|
||||
|
||||
elif sys.argv[1] == "test":
|
||||
m , c = make_tarball()
|
||||
build_jar()
|
||||
cmd = [ "java" , "-cp" , os.environ.get( "CLASSPATH" , "." ) + ":emr.jar" , "emr" ]
|
||||
|
||||
workingDir = "/data/emr/test"
|
||||
cmd.append( "--workingDir" )
|
||||
cmd.append( workingDir )
|
||||
if os.path.exists( workingDir ):
|
||||
shutil.rmtree( workingDir )
|
||||
|
||||
cmd.append( "file://" + os.getcwd() + "/" + m )
|
||||
cmd.append( "file://" + os.getcwd() + "/" + c )
|
||||
|
||||
out = "/tmp/emrresults"
|
||||
cmd.append( out )
|
||||
if os.path.exists( out ):
|
||||
shutil.rmtree( out )
|
||||
|
||||
cmd.append( "jstests/basic1.js" )
|
||||
|
||||
subprocess.call( cmd )
|
||||
|
||||
for x in os.listdir( out ):
|
||||
if x.startswith( "." ):
|
||||
continue
|
||||
print( x )
|
||||
for z in open( out + "/" + x ):
|
||||
print( "\t" + z.strip() )
|
||||
|
||||
elif sys.argv[1] == "output":
|
||||
do_output( sys.argv[2] )
|
||||
|
||||
elif sys.argv[1] == "full":
|
||||
things = push()
|
||||
run_tests( things , sys.argv[2:] )
|
||||
|
||||
else:
|
||||
things = push()
|
||||
run_tests( things , sys.argv[1:] )
|
||||
|
||||
@ -1,7 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
sudo mkdir /mnt/data
|
||||
sudo ln -s /mnt/data /data
|
||||
sudo chown hadoop /mnt/data
|
||||
|
||||
sudo easy_install pymongo
|
||||
Binary file not shown.
@ -1,202 +0,0 @@
|
||||
#!/bin/sh -e
|
||||
#
|
||||
# Copyright (c) 2005-2011 The OpenSSL Project.
|
||||
#
|
||||
# Depending on output file name, the script either embeds fingerprint
|
||||
# into libcrypto.so or static application. "Static" refers to static
|
||||
# libcrypto.a, not [necessarily] application per se.
|
||||
#
|
||||
# Even though this script is called fipsld, it expects C compiler
|
||||
# command line syntax and $FIPSLD_CC or $CC environment variable set
|
||||
# and can even be used to compile source files.
|
||||
|
||||
#set -x
|
||||
|
||||
# Note: edit this to point to your copy of g++ if this is not correct.
|
||||
CC="/usr/bin/g++"
|
||||
|
||||
# Initially -c wasn't intended to be interpreted here, but it might
|
||||
# make life easier for those who want to build FIPS-ified applications
|
||||
# with minimal [if any] modifications to their Makefiles...
|
||||
( while [ "x$1" != "x" -a "x$1" != "x-c" -a "x$1" != "x-E" ]; do shift; done;
|
||||
[ $# -ge 1 ]
|
||||
) && exec ${CC} "$@"
|
||||
|
||||
TARGET=`(while [ "x$1" != "x" -a "x$1" != "x-o" ]; do shift; done; echo $2)`
|
||||
|
||||
# If using an auto-tooled (autoconf/automake/libtool) project,
|
||||
# configure will fail when testing the compiler or even performing
|
||||
# simple checks. Pass-through to compiler directly if application is
|
||||
# is not being linked with libcrypto, allowing auto-tooled applications
|
||||
# to utilize fipsld (e.g. CC=/usr/local/ssl/bin/fipsld FIPSLD_CC=gcc
|
||||
# ./configure && make). But keep in mind[!] that if certified code
|
||||
# resides in a shared library, then fipsld *may not* be used and
|
||||
# end-developer should not modify application configuration and build
|
||||
# procedures. This is because in-core fingerprint and associated
|
||||
# procedures are already embedded into and executed in shared library
|
||||
# context.
|
||||
case `basename "${TARGET}"` in
|
||||
libcrypto*|libfips*|*.dll) ;;
|
||||
*) case "$*" in
|
||||
*libcrypto.a*|*-lcrypto*|*fipscanister.o*) ;;
|
||||
*) exec ${CC} "$@" ;;
|
||||
esac
|
||||
esac
|
||||
|
||||
[ -n "${TARGET}" ] || { echo 'no -o specified'; exit 1; }
|
||||
|
||||
# Turn on debugging output?
|
||||
( while [ "x$1" != "x" -a "x$1" != "x-DDEBUG_FINGERPRINT_PREMAIN" ]; do shift; done;
|
||||
[ $# -ge 1 ]
|
||||
) && set -x
|
||||
|
||||
THERE="`echo $0 | sed -e 's|[^/]*$||'`"..
|
||||
|
||||
# fipscanister.o can appear in command line
|
||||
CANISTER_O=`(while [ "x$1" != "x" ]; do case "$1" in *fipscanister.o) echo $1; exit;; esac; shift; done)`
|
||||
if [ -z "${CANISTER_O}" ]; then
|
||||
# If set, FIPSLIBDIR is location of installed validated FIPS module
|
||||
if [ -n "${FIPSLIBDIR}" ]; then
|
||||
CANISTER_O="${FIPSLIBDIR}/fipscanister.o"
|
||||
elif [ -f "${THERE}/fips/fipscanister.o" ]; then
|
||||
CANISTER_O="${THERE}/fips/fipscanister.o"
|
||||
elif [ -f "${THERE}/lib/fipscanister.o" ]; then
|
||||
CANISTER_O="${THERE}/lib/fipscanister.o"
|
||||
fi
|
||||
CANISTER_O_CMD="${CANISTER_O}"
|
||||
fi
|
||||
[ -f ${CANISTER_O} ] || { echo "unable to find ${CANISTER_O}"; exit 1; }
|
||||
|
||||
PREMAIN_C=`dirname "${CANISTER_O}"`/fips_premain.c
|
||||
PREMAIN_O=`dirname "${CANISTER_O}"`/fips_premain.o
|
||||
|
||||
HMAC_KEY="etaonrishdlcupfm"
|
||||
|
||||
case "`(uname -s) 2>/dev/null`" in
|
||||
OSF1|IRIX*) _WL_PREMAIN="-Wl,-init,FINGERPRINT_premain" ;;
|
||||
HP-UX) _WL_PREMAIN="-Wl,+init,FINGERPRINT_premain" ;;
|
||||
AIX) _WL_PREMAIN="-Wl,-binitfini:FINGERPRINT_premain,-bnoobjreorder";;
|
||||
Darwin) ( while [ "x$1" != "x" -a "x$1" != "x-dynamiclib" ]; do shift; done;
|
||||
[ $# -ge 1 ]
|
||||
) && _WL_PREMAIN="-Wl,-init,_FINGERPRINT_premain" ;;
|
||||
esac
|
||||
|
||||
case "${TARGET}" in
|
||||
[!/]*) TARGET=./${TARGET} ;;
|
||||
esac
|
||||
|
||||
case `basename "${TARGET}"` in
|
||||
lib*|*.dll) # must be linking a shared lib...
|
||||
# Shared lib creation can be taking place in the source
|
||||
# directory only, but fipscanister.o can reside elsewhere...
|
||||
|
||||
if [ -x "${THERE}/fips/fips_standalone_sha1" ]; then
|
||||
FINGERTYPE="${THERE}/fips/fips_standalone_sha1"
|
||||
PREMAIN_DSO="${THERE}/fips/fips_premain_dso"
|
||||
elif [ -x "${THERE}/bin/fips_standalone_sha1" ]; then
|
||||
FINGERTYPE="${THERE}/bin/fips_standalone_sha1"
|
||||
PREMAIN_DSO="./fips_premain_dso"
|
||||
fi
|
||||
|
||||
# verify fipspremain.c against its detached signature...
|
||||
${FINGERTYPE} "${PREMAIN_C}" | sed "s/(.*\//(/" | \
|
||||
diff -w "${PREMAIN_C}.sha1" - || \
|
||||
{ echo "${PREMAIN_C} fingerprint mismatch"; exit 1; }
|
||||
# verify fipscanister.o against its detached signature...
|
||||
${FINGERTYPE} "${CANISTER_O}" | sed "s/(.*\//(/" | \
|
||||
diff -w "${CANISTER_O}.sha1" - || \
|
||||
{ echo "${CANISTER_O} fingerprint mismatch"; exit 1; }
|
||||
|
||||
[ -z "${FIPSLD_LIBCRYPTO}" -a -f "${THERE}/libcrypto.a" ] && \
|
||||
FIPSLD_LIBCRYPTO="${THERE}/libcrypto.a"
|
||||
|
||||
|
||||
# Temporarily remove fipscanister.o from libcrypto.a!
|
||||
# We are required to use the standalone copy...
|
||||
if [ -n "${FIPSLD_LIBCRYPTO}" ]; then
|
||||
if ar d "${FIPSLD_LIBCRYPTO}" fipscanister.o; then
|
||||
(ranlib "${FIPSLD_LIBCRYPTO}") 2>/dev/null || :
|
||||
trap 'ar r "${FIPSLD_LIBCRYPTO}" "${CANISTER_O}";
|
||||
(ranlib "${FIPSLD_LIBCRYPTO}") 2>/dev/null || :;
|
||||
sleep 1;
|
||||
touch -c "${TARGET}"' 0
|
||||
fi
|
||||
fi
|
||||
|
||||
/bin/rm -f "${TARGET}"
|
||||
${CC} -x c "${PREMAIN_C}" -c -o "${PREMAIN_O}"
|
||||
${CC} ${CANISTER_O_CMD:+"${CANISTER_O_CMD}"} \
|
||||
"${PREMAIN_O}" \
|
||||
${_WL_PREMAIN} "$@"
|
||||
|
||||
if [ "x${FIPS_SIG}" != "x" ]; then
|
||||
# embed signature
|
||||
"${FIPS_SIG}" "${TARGET}"
|
||||
[ $? -ne 42 ] && exit $?
|
||||
fi
|
||||
|
||||
# generate signature...
|
||||
SIG=`"${PREMAIN_DSO}" "${TARGET}"`
|
||||
|
||||
/bin/rm -f "${TARGET}"
|
||||
if [ -z "${SIG}" ]; then
|
||||
echo "unable to collect signature"; exit 1
|
||||
fi
|
||||
|
||||
# recompile with signature...
|
||||
${CC} -x c -DHMAC_SHA1_SIG=\"${SIG}\" "${PREMAIN_C}" -c -o "${PREMAIN_O}"
|
||||
${CC} ${CANISTER_O_CMD:+"${CANISTER_O_CMD}"} \
|
||||
-DHMAC_SHA1_SIG=\"${SIG}\" "${PREMAIN_O}" \
|
||||
${_WL_PREMAIN} "$@"
|
||||
;;
|
||||
|
||||
*) # must be linking statically...
|
||||
# Static linking can be taking place either in the source
|
||||
# directory or off the installed binary target destination.
|
||||
if [ -x "${THERE}/fips/fips_standalone_sha1" ]; then
|
||||
FINGERTYPE="${THERE}/fips/fips_standalone_sha1"
|
||||
elif [ -x "${THERE}/bin/fips_standalone_sha1" ]; then
|
||||
FINGERTYPE="${THERE}/bin/fips_standalone_sha1"
|
||||
else # Installed tree is expected to contain
|
||||
# lib/fipscanister.o, lib/fipscanister.o.sha1 and
|
||||
# lib/fips_premain.c [not to mention bin/openssl].
|
||||
FINGERTYPE="openssl sha1 -hmac ${HMAC_KEY}"
|
||||
fi
|
||||
|
||||
# verify fipscanister.o against its detached signature...
|
||||
${FINGERTYPE} "${CANISTER_O}" | sed "s/(.*\//(/" | \
|
||||
diff -w "${CANISTER_O}.sha1" - || \
|
||||
{ echo "${CANISTER_O} fingerprint mismatch"; exit 1; }
|
||||
|
||||
# verify fips_premain.c against its detached signature...
|
||||
${FINGERTYPE} "${PREMAIN_C}" | sed "s/(.*\//(/" | \
|
||||
diff -w "${PREMAIN_C}.sha1" - || \
|
||||
{ echo "${PREMAIN_C} fingerprint mismatch"; exit 1; }
|
||||
|
||||
/bin/rm -f "${TARGET}"
|
||||
${CC} -x c "${PREMAIN_C}" -c -o "${PREMAIN_O}"
|
||||
${CC} ${CANISTER_O_CMD:+"${CANISTER_O_CMD}"} \
|
||||
"${PREMAIN_O}" \
|
||||
${_WL_PREMAIN} "$@"
|
||||
|
||||
if [ "x${FIPS_SIG}" != "x" ]; then
|
||||
# embed signature
|
||||
"${FIPS_SIG}" "${TARGET}"
|
||||
[ $? -ne 42 ] && exit $?
|
||||
fi
|
||||
|
||||
# generate signature...
|
||||
SIG=`"${TARGET}"`
|
||||
|
||||
/bin/rm -f "${TARGET}"
|
||||
if [ -z "${SIG}" ]; then
|
||||
echo "unable to collect signature"; exit 1
|
||||
fi
|
||||
|
||||
# recompile with signature...
|
||||
${CC} -x c -DHMAC_SHA1_SIG=\"${SIG}\" "${PREMAIN_C}" -c -o "${PREMAIN_O}"
|
||||
${CC} ${CANISTER_O_CMD:+"${CANISTER_O_CMD}"} \
|
||||
-DHMAC_SHA1_SIG=\"${SIG}\" "${PREMAIN_O}" \
|
||||
${_WL_PREMAIN} "$@"
|
||||
;;
|
||||
esac
|
||||
@ -1,69 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
from __future__ import with_statement
|
||||
import tempfile
|
||||
import sys
|
||||
import re
|
||||
import os
|
||||
|
||||
def opentemp(basename):
|
||||
# The following doesn't work in python before 2.6
|
||||
# return tempfile.NamedTemporaryFile('w', -1, ".XXXXXX", basename, '.', False)
|
||||
fname = basename +".TMP"
|
||||
if os.path.exists(fname):
|
||||
raise "not clobbering file %s" % fname
|
||||
return open(fname, 'w')
|
||||
|
||||
def frob_debian_changelog(version):
|
||||
fname = 'debian/changelog'
|
||||
with opentemp(fname) as o:
|
||||
with open(fname) as i:
|
||||
lineno = 0
|
||||
for line in i:
|
||||
if lineno == 0:
|
||||
newline = re.sub(r'\([^)]*\)', '('+version+')', line)
|
||||
o.write(newline)
|
||||
else:
|
||||
o.write(line)
|
||||
os.rename(o.name, fname)
|
||||
|
||||
def frob_rpm_spec(version):
|
||||
fname = 'rpm/mongo.spec'
|
||||
with opentemp(fname) as o:
|
||||
with open(fname) as i:
|
||||
frobbed = False
|
||||
for line in i:
|
||||
if frobbed:
|
||||
o.write(line)
|
||||
else:
|
||||
if line.find('Version:') == 0:
|
||||
print >> o, 'Version: ' + version
|
||||
frobbed = True
|
||||
else:
|
||||
o.write(line)
|
||||
os.rename(o.name, fname)
|
||||
|
||||
def frob_stdafx_cpp(version):
|
||||
fname = 'stdafx.cpp'
|
||||
with opentemp(fname) as o:
|
||||
with open(fname) as i:
|
||||
frobbed = False
|
||||
for line in i:
|
||||
if frobbed:
|
||||
o.write(line)
|
||||
else:
|
||||
if re.search(r'const.*char.*versionString\[\].*=', line):
|
||||
o.write(' const char versionString[] = "%s";' % version)
|
||||
else:
|
||||
o.write(line)
|
||||
os.rename(o.name, fname)
|
||||
|
||||
(progname, version) = sys.argv
|
||||
if version is None:
|
||||
print >> sys.stderr, 'usage: %s VERSION' % progname
|
||||
sys.exit(1)
|
||||
frob_debian_changelog(version)
|
||||
frob_rpm_spec(version)
|
||||
## I don't yet know what-all cares about the versionString inside the
|
||||
## mongo code, so I'm not actually calling this yet.
|
||||
# frob_stdafx_cpp(version)
|
||||
@ -1,60 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<Platform Condition=" '$(Platform)' == '' ">x86</Platform>
|
||||
<ProductVersion>3.5</ProductVersion>
|
||||
<ProjectGuid>{fc40ea06-5d8c-4edf-9e19-a0bdd9a3a7d5}</ProjectGuid>
|
||||
<SchemaVersion>2.0</SchemaVersion>
|
||||
<OutputName>MongoDB_$(Version)_x86_Standard</OutputName>
|
||||
<OutputType>Package</OutputType>
|
||||
<DefineSolutionProperties>false</DefineSolutionProperties>
|
||||
<WixTargetsPath Condition=" '$(WixTargetsPath)' == '' AND '$(MSBuildExtensionsPath32)' != '' ">$(MSBuildExtensionsPath32)\Microsoft\WiX\v3.x\Wix.targets</WixTargetsPath>
|
||||
<WixTargetsPath Condition=" '$(WixTargetsPath)' == '' ">$(MSBuildExtensionsPath)\Microsoft\WiX\v3.x\Wix.targets</WixTargetsPath>
|
||||
<ProductId Condition=" '$(ProductId)' == '' ">*</ProductId>
|
||||
<UpgradeCode Condition=" '$(UpgradeCode)' == '' ">867C1D1D-2040-4E90-B04E-1158F9CBDE96</UpgradeCode>
|
||||
<Name>MongoDB</Name>
|
||||
<OutputPath>bin\$(Configuration)\$(Platform)\</OutputPath>
|
||||
<IntermediateOutputPath>obj\$(Configuration)\$(Platform)\</IntermediateOutputPath>
|
||||
<TreatWarningsAsErrors>True</TreatWarningsAsErrors>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x86' ">
|
||||
<VerboseOutput>True</VerboseOutput>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup>
|
||||
<ClientSource Condition=" '$(ClientSource)' == '' ">..\..\..\build\win32\normal\clientlib</ClientSource>
|
||||
<License Condition=" '$(License)' == '' ">..\..\..\distsrc</License>
|
||||
<Source Condition=" '$(Source)' == '' ">..\..\..\build\win32\normal\mongo</Source>
|
||||
<Version Condition=" '$(Version)' == '' ">2.4.0</Version>
|
||||
<ClientHeaderSource Condition=" '$(ClientHeaderSource)' == '' ">..\..\..\build\win32\normal\clientlib\include</ClientHeaderSource>
|
||||
<DefineConstants>MongoDBVersion=$(Version);LicenseSource=$(License);BinarySource=$(Source);Edition=Standard;ProductId=$(ProductId);UpgradeCode=$(UpgradeCode);ClientSource=$(ClientSource);ClientHeaderSource=$(ClientHeaderSource)</DefineConstants>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="..\..\..\src\mongo\installer\msi\wxs\BinaryFragment.wxs" />
|
||||
<Compile Include="..\..\..\src\mongo\installer\msi\wxs\FeatureFragment.wxs" />
|
||||
<Compile Include="..\..\..\src\mongo\installer\msi\wxs\LicensingFragment.wxs" />
|
||||
<Compile Include="..\..\..\src\mongo\installer\msi\wxs\Installer.wxs" />
|
||||
<Compile Include="$(OutputPath)DriverInclude.wxs" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<WixExtension Include="WixUIExtension">
|
||||
<HintPath>$(WixExtDir)\WixUIExtension.dll</HintPath>
|
||||
<Name>WixUIExtension</Name>
|
||||
</WixExtension>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Folder Include="wxs\" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(WixTargetsPath)" />
|
||||
<PropertyGroup>
|
||||
<PreBuildEvent>"%WIX%\bin\heat.exe" dir $(ClientHeaderSource) -gg -g1 -frag -cg cg_DriverHeaders -nologo -directoryid -out DriverInclude.wxs -dr Header -srd -var var.ClientHeaderSource</PreBuildEvent>
|
||||
</PropertyGroup>
|
||||
<!--
|
||||
To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||
Other similar extension points exist, see Wix.targets.
|
||||
<Target Name="BeforeBuild">
|
||||
</Target>
|
||||
<Target Name="AfterBuild">
|
||||
</Target>
|
||||
-->
|
||||
</Project>
|
||||
@ -1,51 +0,0 @@
|
||||
|
||||
Microsoft Visual Studio Solution File, Format Version 11.00
|
||||
# Visual Studio 2010
|
||||
Project("{930C7802-8A8C-48F9-8165-68863BCCD9DD}") = "MongoDB", "MongoDB.wixproj", "{FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}"
|
||||
EndProject
|
||||
Project("{930C7802-8A8C-48F9-8165-68863BCCD9DD}") = "MongoDB_64", "MongoDB_64.wixproj", "{FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}"
|
||||
EndProject
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{2E84BEF1-F2D9-4A6B-B102-EC941AF313AF}"
|
||||
ProjectSection(SolutionItems) = preProject
|
||||
build32bitmsi.bat = build32bitmsi.bat
|
||||
build64bit2008R2msi.bat = build64bit2008R2msi.bat
|
||||
build64bitmsi.bat = build64bitmsi.bat
|
||||
buildenterprisemsi.bat = buildenterprisemsi.bat
|
||||
README.md = README.md
|
||||
EndProjectSection
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Mixed Platforms = Debug|Mixed Platforms
|
||||
Debug|x64 = Debug|x64
|
||||
Debug|x86 = Debug|x86
|
||||
Release|Mixed Platforms = Release|Mixed Platforms
|
||||
Release|x64 = Release|x64
|
||||
Release|x86 = Release|x86
|
||||
EndGlobalSection
|
||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||
{FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Debug|Mixed Platforms.ActiveCfg = Debug|x86
|
||||
{FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Debug|Mixed Platforms.Build.0 = Debug|x86
|
||||
{FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Debug|x64.ActiveCfg = Debug|x86
|
||||
{FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Debug|x86.ActiveCfg = Debug|x86
|
||||
{FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Debug|x86.Build.0 = Debug|x86
|
||||
{FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Release|Mixed Platforms.ActiveCfg = Release|x86
|
||||
{FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Release|Mixed Platforms.Build.0 = Release|x86
|
||||
{FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Release|x64.ActiveCfg = Release|x86
|
||||
{FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Release|x86.ActiveCfg = Release|x86
|
||||
{FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Release|x86.Build.0 = Release|x86
|
||||
{FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
|
||||
{FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Debug|Mixed Platforms.Build.0 = Debug|x64
|
||||
{FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Debug|x64.Build.0 = Debug|x64
|
||||
{FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Debug|x86.ActiveCfg = Debug|x64
|
||||
{FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Release|Mixed Platforms.ActiveCfg = Release|x64
|
||||
{FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Release|Mixed Platforms.Build.0 = Release|x64
|
||||
{FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Release|x64.ActiveCfg = Release|x64
|
||||
{FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Release|x64.Build.0 = Release|x64
|
||||
{FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Release|x86.ActiveCfg = Release|x64
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
@ -1,70 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<Platform Condition=" '$(Platform)' == '' ">x64</Platform>
|
||||
<ProductVersion>3.5</ProductVersion>
|
||||
<ProjectGuid>{fa9df7fc-a283-4eb8-b0c6-f9fa31e22cbc}</ProjectGuid>
|
||||
<SchemaVersion>2.0</SchemaVersion>
|
||||
<OutputName>MongoDB_$(Version)_x64_$(Flavor)_$(Edition)</OutputName>
|
||||
<OutputType>Package</OutputType>
|
||||
<DefineSolutionProperties>false</DefineSolutionProperties>
|
||||
<WixTargetsPath Condition=" '$(WixTargetsPath)' == '' AND '$(MSBuildExtensionsPath32)' != '' ">$(MSBuildExtensionsPath32)\Microsoft\WiX\v3.x\Wix.targets</WixTargetsPath>
|
||||
<WixTargetsPath Condition=" '$(WixTargetsPath)' == '' ">$(MSBuildExtensionsPath)\Microsoft\WiX\v3.x\Wix.targets</WixTargetsPath>
|
||||
<Name>MongoDB</Name>
|
||||
<OutputPath>bin\$(Configuration)\$(Platform)\$(Flavor)\$(Edition)\</OutputPath>
|
||||
<IntermediateOutputPath>obj\$(Configuration)\$(Platform)\$(Flavor)\$(Edition)\</IntermediateOutputPath>
|
||||
<TreatWarningsAsErrors>True</TreatWarningsAsErrors>
|
||||
<ProductId Condition=" '$(ProductId)' == '' ">*</ProductId>
|
||||
<UpgradeCode Condition=" '$(UpgradeCode)' == '' ">FCF901F6-E963-40B1-9A17-978242068587</UpgradeCode>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup>
|
||||
<ClientSource Condition=" '$(ClientSource)' == '' ">..\..\..\build\win32\64\client_build</ClientSource>
|
||||
<Edition Condition=" '$(Edition)' == '' ">Standard</Edition>
|
||||
<Flavor Condition=" '$(Flavor)' == '' ">2008R2Plus</Flavor>
|
||||
<License Condition=" '$(License)' == '' ">..\..\..\distsrc</License>
|
||||
<EnterpriseBase Condition=" '$(EnterpriseBase)' == '' ">..\..\..\src\mongo\db\modules\subscription</EnterpriseBase>
|
||||
<SaslSource Condition=" '$(SaslSource)' == '' ">..\..\..\build\win32\64\mongo</SaslSource>
|
||||
<SnmpSource Condition=" '$(SnmpSource)' == '' ">..\..\..\build\win32\64\mongo</SnmpSource>
|
||||
<Source Condition=" '$(Source)' == '' ">..\..\..\build\win32\64\mongo</Source>
|
||||
<SslSource Condition=" '$(SslSource)' == '' ">..\..\..\build\win32\64\mongo</SslSource>
|
||||
<Version Condition=" '$(Version)' == '' ">2.4.0</Version>
|
||||
<MergeModulesBasePath Condition=" '$(MergeModulesBasePath)' == '' ">C:\Program Files (x86)\Common Files\Merge Modules</MergeModulesBasePath>
|
||||
<ClientHeaderSource Condition=" '$(ClientHeaderSource)' == '' ">..\..\..\build\win32\normal\clientlib\include</ClientHeaderSource>
|
||||
<DefineConstants>MongoDBVersion=$(Version);LicenseSource=$(License);BinarySource=$(Source);Edition=$(Edition);SaslSource=$(SaslSource);SslSource=$(SslSource);SnmpSource=$(SnmpSource);ProductId=$(ProductId);UpgradeCode=$(UpgradeCode);Flavor=$(Flavor);ClientSource=$(ClientSource);EnterpriseBase=$(EnterpriseBase);ClientHeaderSource=$(ClientHeaderSource);MergeModulesBasePath=$(MergeModulesBasePath)</DefineConstants>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x64' ">
|
||||
<VerboseOutput>True</VerboseOutput>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Edition)' == 'Enterprise' ">
|
||||
<TreatWarningsAsErrors>False</TreatWarningsAsErrors>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="..\..\..\src\mongo\installer\msi\wxs\BinaryFragment.wxs" />
|
||||
<Compile Include="..\..\..\src\mongo\installer\msi\wxs\FeatureFragment.wxs" />
|
||||
<Compile Include="..\..\..\src\mongo\installer\msi\wxs\Installer_64.wxs" />
|
||||
<Compile Include="..\..\..\src\mongo\installer\msi\wxs\LicensingFragment.wxs" />
|
||||
<Compile Include="$(OutputPath)DriverInclude.wxs" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<WixExtension Include="WixUIExtension">
|
||||
<HintPath>$(WixExtDir)\WixUIExtension.dll</HintPath>
|
||||
<Name>WixUIExtension</Name>
|
||||
</WixExtension>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Folder Include="wxs\" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(WixTargetsPath)" />
|
||||
<PropertyGroup>
|
||||
<PreBuildEvent>"%WIX%\bin\heat.exe" dir "$(ClientHeaderSource)" -gg -g1 -frag -cg cg_DriverHeaders -nologo -directoryid -out DriverInclude.wxs -dr Header -srd -var var.ClientHeaderSource</PreBuildEvent>
|
||||
</PropertyGroup>
|
||||
<!--
|
||||
To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||
Other similar extension points exist, see Wix.targets.
|
||||
<Target Name="BeforeBuild">
|
||||
</Target>
|
||||
<Target Name="AfterBuild">
|
||||
</Target>
|
||||
-->
|
||||
</Project>
|
||||
@ -1,36 +0,0 @@
|
||||
## Prerequisites
|
||||
WiX Toolset v3.7.1224.0 from http://wixtoolset.org/
|
||||
|
||||
## Features
|
||||
The following are the installer features with the executables they install.
|
||||
Each of these features can be installed independently using msiexec /ADDLOCAL
|
||||
or using the Installer GUI
|
||||
* Server
|
||||
* mongod.exe
|
||||
* mongod.pdb
|
||||
* Client
|
||||
* mongo.exe
|
||||
* MonitoringTools
|
||||
* mongostat.exe
|
||||
* mongotop.exe
|
||||
* ImportExportTools
|
||||
* mongodump.exe
|
||||
* mongorestore.exe
|
||||
* mongoexport.exe
|
||||
* mongoimport.exe
|
||||
* Router
|
||||
* mongos.exe
|
||||
* mongos.pdb
|
||||
* MiscellaneousTools
|
||||
* bsondump.exe
|
||||
* mongofiles.exe
|
||||
* mongooplog.exe
|
||||
* mongoperf.exe
|
||||
|
||||
## Typical install
|
||||
The typical (default) install, installs all except the Router and
|
||||
MiscellaneousTools features.
|
||||
|
||||
## Configuring builds
|
||||
The version, location of binaries and license file can be configured when
|
||||
building. Refer to build32bitmsi.bat or build64bitmsi.bat for example
|
||||
@ -1,100 +0,0 @@
|
||||
@ECHO OFF
|
||||
|
||||
SET VERSION=2.4.0
|
||||
SET BINDIR=..\..\..\build\win32\normal\mongo
|
||||
SET CLIENTLIBDIR=..\..\..\build\win32\normal\client_build
|
||||
SET LICENSEDIR=..\..\..\distsrc
|
||||
SET CLIENTHEADERDIR=..\..\..\build\win32\normal\client_build\include
|
||||
SET WIXBINDIR=C:\Program Files (x86)\WiX Toolset v3.7\bin
|
||||
|
||||
SET PLATFORM=x86
|
||||
SET GENERATEDWXSDIR=.\wxs
|
||||
SET EDITION=Standard
|
||||
SET CONFIGURATION=Release
|
||||
SET OUTPUTOBJDIR=obj\%CONFIGURATION%\%PLATFORM%\
|
||||
SET OUTPUTBINDIR=bin\%CONFIGURATION%\%PLATFORM%\
|
||||
SET PROJECTDIR=C:\git\sridharn\mongo\buildscripts\packaging\msi\
|
||||
SET TARGETNAME=MongoDB_%VERSION%_%PLATFORM%_%EDITION%
|
||||
|
||||
:loop
|
||||
IF NOT "%1"=="" (
|
||||
IF "%1"=="-version" (
|
||||
SET VERSION=%2
|
||||
SHIFT
|
||||
)
|
||||
IF "%1"=="-bindir" (
|
||||
SET BINDIR=%2
|
||||
SHIFT
|
||||
)
|
||||
IF "%1"=="-licensedir" (
|
||||
SET LICENSEDIR=%2
|
||||
SHIFT
|
||||
)
|
||||
IF "%1"=="-clientlibdir" (
|
||||
SET CLIENTLIBDIR=%2
|
||||
SHIFT
|
||||
)
|
||||
IF "%1"=="-clientheaderdir" (
|
||||
SET CLIENTHEADERDIR=%2
|
||||
SHIFT
|
||||
)
|
||||
IF "%1"=="-wixbindir" (
|
||||
SET WIXBINDIR=%2
|
||||
SHIFT
|
||||
)
|
||||
IF "%1"=="-generatedwxsdir" (
|
||||
SET GENERATEDWXSDIR=%2
|
||||
SHIFT
|
||||
)
|
||||
SHIFT
|
||||
GOTO :loop
|
||||
)
|
||||
|
||||
REM ECHO Building msi for version %VERSION% with binaries from %BINDIR% and license files from %LICENSEDIR%
|
||||
REM %WINDIR%\Microsoft.NET\Framework64\v4.0.30319\msbuild /p:Configuration=Release;Version=%VERSION%;License=%LICENSEDIR%;
|
||||
REM Source=%BINDIR%;ClientSource=%CLIENTLIBDIR%;ClientHeaderSource=%CLIENTHEADERDIR% MongoDB.wixproj
|
||||
|
||||
ECHO Generating %GENERATEDWXSDIR%\DriverInclude.wxs from sources at %CLIENTHEADERDIR%
|
||||
"%WIXBINDIR%\heat.exe" dir %CLIENTHEADERDIR% -gg -g1 -frag -cg cg_DriverHeaders -nologo -directoryid -out %GENERATEDWXSDIR%\DriverInclude.wxs -dr Header -srd -var var.ClientHeaderSource
|
||||
|
||||
ECHO Compiling wxs files to obj
|
||||
"%WIXBINDIR%\candle.exe" -wx^
|
||||
-dMongoDBVersion=%VERSION%^
|
||||
-dLicenseSource=%LICENSEDIR%^
|
||||
-dBinarySource=%BINDIR%^
|
||||
-dEdition=%EDITION%^
|
||||
-d"ProductId=*"^
|
||||
-dUpgradeCode=867C1D1D-2040-4E90-B04E-1158F9CBDE96^
|
||||
-dClientSource=%CLIENTLIBDIR%^
|
||||
-dClientHeaderSource=%CLIENTHEADERDIR%^
|
||||
-dConfiguration=%CONFIGURATION%^
|
||||
-dOutDir=%OUTPUTBINDIR%^
|
||||
-dPlatform=%PLATFORM%^
|
||||
-dProjectDir=%PROJECTDIR%^
|
||||
-dProjectExt=.wixproj^
|
||||
-dProjectFileName=MongoDB.wixproj^
|
||||
-dProjectName=MongoDB^
|
||||
-dProjectPath=%PROJECTDIR%\MongoDB.wixproj^
|
||||
-dTargetDir=%OUTPUTBINDIR%^
|
||||
-dTargetExt=.msi^
|
||||
-dTargetFileName=%TARGETNAME%.msi^
|
||||
-dTargetName=%TARGETNAME%^
|
||||
-dTargetPath=%OUTPUTBINDIR%\%TARGETNAME%.msi^
|
||||
-out %OUTPUTOBJDIR%^
|
||||
-arch %PLATFORM%^
|
||||
-ext "%WIXBINDIR%\WixUIExtension.dll"^
|
||||
wxs\BinaryFragment.wxs wxs\FeatureFragment.wxs wxs\LicensingFragment.wxs wxs\Installer.wxs %GENERATEDWXSDIR%\DriverInclude.wxs
|
||||
|
||||
ECHO Linking to msi
|
||||
"%WIXBINDIR%\Light.exe"^
|
||||
-out %OUTPUTBINDIR%\%TARGETNAME%.msi^
|
||||
-pdbout %OUTPUTBINDIR%\%TARGETNAME%.wixpdb^
|
||||
-wx -cultures:null^
|
||||
-ext "%WIXBINDIR%\WixUIExtension.dll"^
|
||||
-contentsfile %OUTPUTOBJDIR%\MongoDB.wixproj.BindContentsFileListnull.txt^
|
||||
-outputsfile %OUTPUTOBJDIR%\MongoDB.wixproj.BindOutputsFileListnull.txt^
|
||||
-builtoutputsfile %OUTPUTOBJDIR%\MongoDB.wixproj.BindBuiltOutputsFileListnull.txt^
|
||||
-wixprojectfile %PROJECTDIR%\MongoDB.wixproj^
|
||||
%OUTPUTOBJDIR%\BinaryFragment.wixobj %OUTPUTOBJDIR%\FeatureFragment.wixobj^
|
||||
%OUTPUTOBJDIR%\\LicensingFragment.wixobj %OUTPUTOBJDIR%\Installer.wixobj^
|
||||
%OUTPUTOBJDIR%\DriverInclude.wixobj
|
||||
@ -1,37 +0,0 @@
|
||||
@ECHO OFF
|
||||
SET VERSION=2.4.0
|
||||
SET BINDIR=..\..\..\build\win32\64\mongo
|
||||
SET CLIENTLIBDIR=..\..\..\build\win32\64\client_build
|
||||
SET LICENSEDIR=..\..\..\distsrc
|
||||
SET EDITION=Standard
|
||||
SET FLAVOR=2008R2Plus
|
||||
SET CLIENTHEADERDIR=..\..\..\build\win32\normal\client_build\include
|
||||
|
||||
:loop
|
||||
IF NOT "%1"=="" (
|
||||
IF "%1"=="-version" (
|
||||
SET VERSION=%2
|
||||
SHIFT
|
||||
)
|
||||
IF "%1"=="-bindir" (
|
||||
SET BINDIR=%2
|
||||
SHIFT
|
||||
)
|
||||
IF "%1"=="-licensedir" (
|
||||
SET LICENSEDIR=%2
|
||||
SHIFT
|
||||
)
|
||||
IF "%1"=="-clientlibdir" (
|
||||
SET CLIENTLIBDIR=%2
|
||||
SHIFT
|
||||
)
|
||||
IF "%1"=="-clientheaderdir" (
|
||||
SET CLIENTHEADERDIR=%2
|
||||
SHIFT
|
||||
)
|
||||
SHIFT
|
||||
GOTO :loop
|
||||
)
|
||||
|
||||
ECHO Building msi for version %VERSION% with binaries from %BINDIR% and license files from %LICENSEDIR%
|
||||
%WINDIR%\Microsoft.NET\Framework64\v4.0.30319\msbuild /p:Configuration=Release;Version=%VERSION%;License=%LICENSEDIR%;Source=%BINDIR%;Edition=%EDITION%;Flavor=%FLAVOR%;ClientSource=%CLIENTLIBDIR%;ClientHeaderSource=%CLIENTHEADERDIR% MongoDB_64.wixproj
|
||||
@ -1,37 +0,0 @@
|
||||
@ECHO OFF
|
||||
SET VERSION=2.4.0
|
||||
SET BINDIR=..\..\..\build\win32\64\mongo
|
||||
SET CLIENTLIBDIR=..\..\..\build\win32\64\client_build
|
||||
SET LICENSEDIR=..\..\..\distsrc
|
||||
SET EDITION=Standard
|
||||
SET FLAVOR=2008
|
||||
SET CLIENTHEADERDIR=..\..\..\build\win32\normal\client_build\include
|
||||
|
||||
:loop
|
||||
IF NOT "%1"=="" (
|
||||
IF "%1"=="-version" (
|
||||
SET VERSION=%2
|
||||
SHIFT
|
||||
)
|
||||
IF "%1"=="-bindir" (
|
||||
SET BINDIR=%2
|
||||
SHIFT
|
||||
)
|
||||
IF "%1"=="-licensedir" (
|
||||
SET LICENSEDIR=%2
|
||||
SHIFT
|
||||
)
|
||||
IF "%1"=="-clientlibdir" (
|
||||
SET CLIENTLIBDIR=%2
|
||||
SHIFT
|
||||
)
|
||||
IF "%1"=="-clientheaderdir" (
|
||||
SET CLIENTHEADERDIR=%2
|
||||
SHIFT
|
||||
)
|
||||
SHIFT
|
||||
GOTO :loop
|
||||
)
|
||||
|
||||
ECHO Building msi for version %VERSION% with binaries from %BINDIR% and license files from %LICENSEDIR%
|
||||
%WINDIR%\Microsoft.NET\Framework64\v4.0.30319\msbuild /p:Configuration=Release;Version=%VERSION%;License=%LICENSEDIR%;Source=%BINDIR%;Edition=%EDITION%;Flavor=%FLAVOR%;ClientSource=%CLIENTLIBDIR%;ClientHeaderSource=%CLIENTHEADERDIR% MongoDB_64.wixproj
|
||||
@ -1,63 +0,0 @@
|
||||
@ECHO OFF
|
||||
SET VERSION=2.6.0
|
||||
SET BINDIR=..\..\..\build\win32\64\dynamic-windows\extrapathdyn_c__Utils_sasl_c__Utils_snmp_c__Utils_ssl\release\ssl\mongo
|
||||
SET CLIENTLIBDIR=..\..\..\build\win32\64\dynamic-windows\extrapathdyn_c__Utils_sasl_c__Utils_snmp_c__Utils_ssl\release\ssl\client_build
|
||||
SET LICENSEDIR=..\..\..\distsrc
|
||||
SET ENTERPRISEBASEDIR=..\..\..\src\mongo\db\modules\subscription
|
||||
SET EDITION=Enterprise
|
||||
SET FLAVOR=2008R2Plus
|
||||
SET SASLDIR=..\..\..\..\..\..\Utils\sasl\bin
|
||||
SET OPENSSLDIR=..\..\..\..\..\..\Utils\openssl\bin
|
||||
SET SNMPDIR=..\..\..\..\..\..\Utils\snmp\bin
|
||||
SET CLIENTHEADERDIR=..\..\..\build\win32\normal\client_build\include
|
||||
SET MERGEMODULESBASEPATH="C:\Program Files (x86)\Common Files\Merge Modules"
|
||||
|
||||
:loop
|
||||
IF NOT "%1"=="" (
|
||||
IF "%1"=="-version" (
|
||||
SET VERSION=%2
|
||||
SHIFT
|
||||
)
|
||||
IF "%1"=="-bindir" (
|
||||
SET BINDIR=%2
|
||||
SHIFT
|
||||
)
|
||||
IF "%1"=="-licensedir" (
|
||||
SET LICENSEDIR=%2
|
||||
SHIFT
|
||||
)
|
||||
IF "%1"=="-sasldir" (
|
||||
SET SASLDIR=%2
|
||||
SHIFT
|
||||
)
|
||||
IF "%1"=="-openssldir" (
|
||||
SET OPENSSLDIR=%2
|
||||
SHIFT
|
||||
)
|
||||
IF "%1"=="-snmpdir" (
|
||||
SET SNMPDIR=%2
|
||||
SHIFT
|
||||
)
|
||||
IF "%1"=="-clientlibdir" (
|
||||
SET CLIENTLIBDIR=%2
|
||||
SHIFT
|
||||
)
|
||||
IF "%1"=="-enterprisebasedir" (
|
||||
SET ENTERPRISEBASEDIR=%2
|
||||
SHIFT
|
||||
)
|
||||
IF "%1"=="-clientheaderdir" (
|
||||
SET CLIENTHEADERDIR=%2
|
||||
SHIFT
|
||||
)
|
||||
IF "%1"=="-mergemodulesbasepath" (
|
||||
SET MERGEMODULESBASEPATH=%2
|
||||
SHIFT
|
||||
)
|
||||
SHIFT
|
||||
GOTO :loop
|
||||
)
|
||||
|
||||
ECHO Building enterprise msi for version %VERSION% with binaries from %BINDIR%, sasl from %SASLDIR%, ssl from %OPENSSLDIR%, snmp from %SNMPDIR% and license files from %LICENSEDIR%
|
||||
|
||||
%WINDIR%\Microsoft.NET\Framework64\v4.0.30319\msbuild /p:Configuration=Release;Version=%VERSION%;License=%LICENSEDIR%;Source=%BINDIR%;SaslSource=%SASLDIR%;SnmpSource=%SNMPDIR%;SslSource=%OPENSSLDIR%;Edition=%EDITION%;Flavor=%FLAVOR%;ClientSource=%CLIENTLIBDIR%;EnterpriseBase=%ENTERPRISEBASEDIR%;ClientHeaderSource=%CLIENTHEADERDIR%;MergeModulesBasePath=%MERGEMODULESBASEPATH% MongoDB_64.wixproj
|
||||
@ -1,121 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
Copyright 2014 MongoDB Inc.
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License, version 3,
|
||||
as published by the Free Software Foundation.
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
As a special exception, the copyright holders give permission to link the
|
||||
code of portions of this program with the OpenSSL library under certain
|
||||
conditions as described in each individual source file and distribute
|
||||
linked combinations including the program with the OpenSSL library. You
|
||||
must comply with the GNU Affero General Public License in all respects
|
||||
for all of the code used other than as permitted herein. If you modify
|
||||
file(s) with this exception, you may extend this exception to your
|
||||
version of the file(s), but you are not obligated to do so. If you do not
|
||||
wish to do so, delete this exception statement from your version. If you
|
||||
delete this exception statement from all source files in the program,
|
||||
then also delete it in the license file.
|
||||
"""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
from optparse import OptionParser
|
||||
|
||||
class Benchmark(object):
|
||||
def __init__(self, name, executable, options):
|
||||
self._name = name
|
||||
self._executable = executable
|
||||
self._options = options
|
||||
self._ran = False
|
||||
|
||||
def name(self):
|
||||
return self._name
|
||||
|
||||
def run(self):
|
||||
self._result = subprocess.check_call([self._executable] + self._options)
|
||||
self._ran = True
|
||||
|
||||
def result(self):
|
||||
if not self._ran:
|
||||
raise Exception("Cannot get result of a suite that hasn't ran")
|
||||
return self._result
|
||||
|
||||
|
||||
def file_allocator_bench(variant, ntrials, megabytes, path, report_dir):
|
||||
executable = os.path.join("build", "file_allocator_bench")
|
||||
suitename = "FileAllocatorBenchmark-%s" % variant
|
||||
opts = []
|
||||
opts.append("--ntrials=%s" % ntrials)
|
||||
opts.append("--megabytes=%i" % megabytes)
|
||||
opts.append("--path=%s" % path)
|
||||
opts.append("--jsonReport=%s.json" % os.path.join(report_dir, variant))
|
||||
return Benchmark(suitename, executable, opts)
|
||||
|
||||
|
||||
def configure_parser():
|
||||
parser = OptionParser()
|
||||
parser.add_option("-r", "--reportDir", type=str,
|
||||
help="Where to write the report [default: %default]",
|
||||
default=os.getcwd())
|
||||
parser.add_option("-w", "--workDir", type=str,
|
||||
help="scratch space used by tests [default: %default]",
|
||||
default=tempfile.gettempdir())
|
||||
return parser
|
||||
|
||||
|
||||
# at some point this should read from a config file, but at this point
|
||||
# its not worth overengineering
|
||||
def make_suites(work_dir, report_dir):
|
||||
return [
|
||||
# requires 128GB free in work_dir
|
||||
# file_allocator_bench("16GB", ntrials=8, megabytes=1024*16,
|
||||
# path=work_dir, report_dir=report_dir),
|
||||
|
||||
# requires 16GB free in work_dir
|
||||
file_allocator_bench("1GB", ntrials=16, megabytes=1024,
|
||||
path=work_dir, report_dir=report_dir),
|
||||
|
||||
# requires 4GB free in work_dir
|
||||
file_allocator_bench("128MB", ntrials=32, megabytes=128,
|
||||
path=work_dir, report_dir=report_dir)
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
parser = configure_parser()
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
report_dir = options.reportDir
|
||||
work_dir = options.workDir
|
||||
|
||||
# unique dir for this report
|
||||
unique_dir = os.path.join(report_dir, "perfsuite-run@%s" % time.time())
|
||||
os.makedirs(unique_dir)
|
||||
print "Writing results to %s" % unique_dir
|
||||
print
|
||||
|
||||
for suite in make_suites(work_dir, unique_dir):
|
||||
|
||||
print("Running suite - %s ..." % suite.name())
|
||||
start = time.time()
|
||||
suite.run()
|
||||
end = time.time()
|
||||
print("...Finished suite in %i seconds" % (end - start))
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,55 +0,0 @@
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
sys.path.append( "." )
|
||||
sys.path.append( ".." )
|
||||
sys.path.append( "../../" )
|
||||
sys.path.append( "../../../" )
|
||||
|
||||
import simples3
|
||||
import settings
|
||||
import subprocess
|
||||
|
||||
# check s3 for md5 hashes
|
||||
|
||||
def check_dir( bucket , prefix , todel ):
|
||||
|
||||
deleteAll = False
|
||||
|
||||
for ( key , modify , etag , size ) in bucket.listdir( prefix=prefix ):
|
||||
if key.find( todel ) < 0:
|
||||
continue
|
||||
print( key )
|
||||
|
||||
if not deleteAll:
|
||||
|
||||
val = raw_input( "Delete (Y,y,n,N):" ).strip()
|
||||
|
||||
if val == "n":
|
||||
print( "skipping this one" )
|
||||
continue
|
||||
elif val == "N":
|
||||
break
|
||||
|
||||
if val == "Y":
|
||||
val = "y"
|
||||
deleteAll = True
|
||||
|
||||
if val != "y":
|
||||
raise Exception( "invalid input :(" )
|
||||
|
||||
bucket.delete( key )
|
||||
|
||||
def clean( todel ):
|
||||
|
||||
|
||||
bucket = simples3.S3Bucket( settings.bucket , settings.id , settings.key )
|
||||
|
||||
for x in [ "osx" , "linux" , "win32" , "sunos5" , "src" ]:
|
||||
check_dir( bucket , x , todel )
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
clean( sys.argv[1] )
|
||||
@ -1,48 +0,0 @@
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append( "." )
|
||||
sys.path.append( ".." )
|
||||
sys.path.append( "../../" )
|
||||
sys.path.append( "../../../" )
|
||||
|
||||
import simples3
|
||||
import settings
|
||||
import subprocess
|
||||
|
||||
# check s3 for md5 hashes
|
||||
|
||||
def check_dir( bucket , prefix ):
|
||||
|
||||
zips = {}
|
||||
md5s = {}
|
||||
for ( key , modify , etag , size ) in bucket.listdir( prefix=prefix ):
|
||||
if key.endswith( ".tgz" ) or key.endswith( ".zip" ) or key.endswith( ".tar.gz" ):
|
||||
zips[key] = etag.replace( '"' , '' )
|
||||
elif key.endswith( ".md5" ):
|
||||
md5s[key] = True
|
||||
elif key.find( "$folder$" ) > 0:
|
||||
pass
|
||||
else:
|
||||
print( "unknown file type: " + key )
|
||||
|
||||
for x in zips:
|
||||
m = x + ".md5"
|
||||
if m in md5s:
|
||||
continue
|
||||
|
||||
print( "need to do: " + x + " " + zips[x] + " to " + m )
|
||||
bucket.put( m , zips[x] , acl="public-read" )
|
||||
|
||||
|
||||
def run():
|
||||
|
||||
bucket = simples3.S3Bucket( settings.bucket , settings.id , settings.key )
|
||||
|
||||
for x in [ "osx" , "linux" , "win32" , "sunos5" , "src" ]:
|
||||
check_dir( bucket , x )
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
@ -1,105 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# Generate and upload detached gpg signatures for archive files in Amazon S3
|
||||
#
|
||||
# Requires standard MongoDB settings.py, like so:
|
||||
# bucket = "downloads.mongodb.org"
|
||||
# # downloads user
|
||||
# id = "xxxxx"
|
||||
# key = "xxxxx"
|
||||
#
|
||||
# Usage: s3sign.py [ --dry-run ] [ --bucket <overridden s3 bucket> ] [ --notary-url <notary url> ] [ --key-name <key name passed to notary service> ] [ --filter <filter> ]
|
||||
#
|
||||
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import requests
|
||||
import sys
|
||||
|
||||
sys.path.append("." )
|
||||
sys.path.append(".." )
|
||||
sys.path.append("../../" )
|
||||
sys.path.append("../../../" )
|
||||
|
||||
import simples3
|
||||
import settings
|
||||
import subprocess
|
||||
|
||||
# parse command line
|
||||
#
|
||||
parser = argparse.ArgumentParser(description='Sign MongoDB S3 Files')
|
||||
parser.add_argument('--dry-run', action='store_true', required=False, help='Do not write anything to S3', default = False);
|
||||
parser.add_argument('--bucket', required = False, help='Override bucket in settings.py', default = settings.bucket);
|
||||
parser.add_argument('--notary-url', required=False, help='URL base for notary service', default = 'http://localhost:5000');
|
||||
parser.add_argument('--key-name', required=False, help='Key parameter to notary service', default = 'test');
|
||||
parser.add_argument('--filter', required=False,
|
||||
help='Only sign files matching case-insensitive substring filter', default = None);
|
||||
args = parser.parse_args()
|
||||
|
||||
notary_urlbase = args.notary_url
|
||||
notary_url = notary_urlbase + '/api/sign'
|
||||
notary_payload = { 'key': args.key_name, 'comment': 'Automatic archive signing'}
|
||||
|
||||
# check s3 for pgp signatures
|
||||
|
||||
def check_dir( bucket , prefix ):
|
||||
|
||||
zips = {}
|
||||
sigs = {}
|
||||
for ( key , modify , etag , size ) in bucket.listdir( prefix=prefix ):
|
||||
# filtered out
|
||||
if args.filter and args.filter.lower() not in key.lower():
|
||||
pass
|
||||
# sign it
|
||||
elif key.endswith(".tgz" ) or key.endswith(".zip" ) or key.endswith(".tar.gz" ) or key.endswith("md5"):
|
||||
# generate signature
|
||||
files = {'file': (key, bucket.get(key))}
|
||||
response_json = {}
|
||||
try:
|
||||
r = requests.post(notary_url, files=files, data=notary_payload,
|
||||
headers = { "Accept": "application/json" })
|
||||
# get url for signature file
|
||||
response_json = json.loads(r.text)
|
||||
except Exception as e:
|
||||
print('error contacting signing service for %s:\n%s' % (key, e.message))
|
||||
continue
|
||||
if 'permalink' in response_json:
|
||||
signature_url = response_json['permalink']
|
||||
try:
|
||||
signature = requests.get(notary_urlbase + signature_url).text
|
||||
zips[key] = signature
|
||||
except Exception as e:
|
||||
print('error downloading signature from signing service for %s:\n%s' % (key, e.message))
|
||||
else:
|
||||
print('error from signing service for %s:\n%s' % (key, response_json.get('message')))
|
||||
# signatures
|
||||
elif key.endswith(".sig" ) or key.endswith(".asc" ):
|
||||
sigs[key] = True
|
||||
# file types we don't need to sign
|
||||
elif key.endswith(".msi" ) or key.endswith(".deb") or key.endswith(".rpm"):
|
||||
pass
|
||||
else:
|
||||
print("unknown file type: %s" % key)
|
||||
|
||||
for x in zips:
|
||||
m = x + ".sig"
|
||||
if m in sigs:
|
||||
continue
|
||||
|
||||
print("need to do: " + x + " to " + m )
|
||||
if not args.dry_run:
|
||||
bucket.put( m , zips[x] , acl="public-read" )
|
||||
|
||||
|
||||
def run():
|
||||
|
||||
bucket = simples3.S3Bucket( args.bucket , settings.id , settings.key )
|
||||
|
||||
for x in [ "osx" , "linux" , "win32" , "sunos5" , "src" ]:
|
||||
check_dir( bucket , x )
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
@ -1,239 +0,0 @@
|
||||
# Copyright 2009 10gen, Inc.
|
||||
#
|
||||
# This file is part of MongoDB.
|
||||
#
|
||||
# MongoDB is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# MongoDB is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with MongoDB. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
"""Tests for the MongoDB shell.
|
||||
|
||||
Right now these mostly just test that the shell handles command line arguments
|
||||
appropriately.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
import sys
|
||||
import subprocess
|
||||
import os
|
||||
|
||||
"""Exit codes for MongoDB."""
|
||||
BADOPTS = 2
|
||||
NOCONNECT = 255
|
||||
|
||||
"""Path to the mongo shell executable to be tested."""
|
||||
mongo_path = None
|
||||
|
||||
class TestShell(unittest.TestCase):
|
||||
|
||||
def open_mongo(self, args=[]):
|
||||
"""Get a subprocess.Popen instance of the shell with the given args.
|
||||
"""
|
||||
return subprocess.Popen([mongo_path] + args,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr = subprocess.PIPE)
|
||||
|
||||
def setUp(self):
|
||||
assert mongo_path
|
||||
|
||||
def test_help(self):
|
||||
mongo_h = self.open_mongo(["-h"])
|
||||
mongo_help = self.open_mongo(["--help"])
|
||||
|
||||
out = mongo_h.communicate()
|
||||
self.assertEqual(out, mongo_help.communicate())
|
||||
self.assert_("usage:" in out[0])
|
||||
|
||||
self.assertEqual(0, mongo_h.returncode)
|
||||
self.assertEqual(0, mongo_help.returncode)
|
||||
|
||||
def test_nodb(self):
|
||||
mongo = self.open_mongo([])
|
||||
mongo_nodb = self.open_mongo(["--nodb"])
|
||||
|
||||
out = mongo_nodb.communicate()
|
||||
self.assert_("MongoDB shell version" in out[0])
|
||||
self.assert_("bye" in out[0])
|
||||
self.assert_("couldn't connect" not in out[0])
|
||||
self.assertEqual(0, mongo_nodb.returncode)
|
||||
|
||||
out = mongo.communicate()
|
||||
self.assert_("MongoDB shell version" in out[0])
|
||||
self.assert_("bye" not in out[0])
|
||||
self.assert_("couldn't connect" in out[0])
|
||||
self.assertEqual(NOCONNECT, mongo.returncode)
|
||||
|
||||
def test_eval(self):
|
||||
mongo = self.open_mongo(["--nodb", "--eval", "print('hello world');"])
|
||||
out = mongo.communicate()
|
||||
self.assert_("hello world" in out[0])
|
||||
self.assert_("bye" not in out[0])
|
||||
self.assertEqual(0, mongo.returncode)
|
||||
|
||||
mongo = self.open_mongo(["--eval"])
|
||||
out = mongo.communicate()
|
||||
self.assert_("required parameter is missing" in out[0])
|
||||
self.assertEqual(BADOPTS, mongo.returncode)
|
||||
|
||||
def test_shell(self):
|
||||
mongo = self.open_mongo(["--nodb", "--shell", "--eval", "print('hello world');"])
|
||||
out = mongo.communicate()
|
||||
self.assert_("hello world" in out[0])
|
||||
self.assert_("bye" in out[0]) # the shell started and immediately exited because stdin was empty
|
||||
self.assertEqual(0, mongo.returncode)
|
||||
|
||||
def test_host_port(self):
|
||||
mongo = self.open_mongo([])
|
||||
out = mongo.communicate()
|
||||
self.assert_("url: test" in out[0])
|
||||
self.assert_("connecting to: test" in out[0])
|
||||
self.assertEqual(NOCONNECT, mongo.returncode)
|
||||
|
||||
mongo = self.open_mongo(["--host", "localhost"])
|
||||
out = mongo.communicate()
|
||||
self.assert_("url: test" in out[0])
|
||||
self.assert_("connecting to: localhost/test" in out[0])
|
||||
self.assertEqual(NOCONNECT, mongo.returncode)
|
||||
|
||||
mongo = self.open_mongo(["--port", "27018"])
|
||||
out = mongo.communicate()
|
||||
self.assert_("url: test" in out[0])
|
||||
self.assert_("connecting to: 127.0.0.1:27018" in out[0])
|
||||
self.assertEqual(NOCONNECT, mongo.returncode)
|
||||
|
||||
mongo = self.open_mongo(["--host", "localhost", "--port", "27018"])
|
||||
out = mongo.communicate()
|
||||
self.assert_("url: test" in out[0])
|
||||
self.assert_("connecting to: localhost:27018/test" in out[0])
|
||||
self.assertEqual(NOCONNECT, mongo.returncode)
|
||||
|
||||
mongo = self.open_mongo(["--host"])
|
||||
out = mongo.communicate()
|
||||
self.assert_("required parameter is missing" in out[0])
|
||||
self.assertEqual(BADOPTS, mongo.returncode)
|
||||
|
||||
mongo = self.open_mongo(["--port"])
|
||||
out = mongo.communicate()
|
||||
self.assert_("required parameter is missing" in out[0])
|
||||
self.assertEqual(BADOPTS, mongo.returncode)
|
||||
|
||||
def test_positionals(self):
|
||||
dirname = os.path.dirname(__file__)
|
||||
test_js = os.path.join(dirname, "testdata/test.js")
|
||||
test_txt = os.path.join(dirname, "testdata/test.txt")
|
||||
test = os.path.join(dirname, "testdata/test")
|
||||
non_exist_js = os.path.join(dirname, "testdata/nonexist.js")
|
||||
non_exist_txt = os.path.join(dirname, "testdata/nonexist.txt")
|
||||
|
||||
mongo = self.open_mongo(["--nodb", test_js])
|
||||
out = mongo.communicate()
|
||||
self.assert_("hello world" in out[0])
|
||||
self.assert_("bye" not in out[0])
|
||||
self.assertEqual(0, mongo.returncode)
|
||||
|
||||
mongo = self.open_mongo(["--nodb", test_txt])
|
||||
out = mongo.communicate()
|
||||
self.assert_("foobar" in out[0])
|
||||
self.assert_("bye" not in out[0])
|
||||
self.assertEqual(0, mongo.returncode)
|
||||
|
||||
mongo = self.open_mongo([test_js, test, test_txt])
|
||||
out = mongo.communicate()
|
||||
self.assert_("url: test" in out[0])
|
||||
self.assert_("connecting to: test" in out[0])
|
||||
self.assertEqual(NOCONNECT, mongo.returncode)
|
||||
|
||||
mongo = self.open_mongo([test_txt, test, test_js])
|
||||
out = mongo.communicate()
|
||||
self.assert_("url: test" in out[0])
|
||||
self.assert_("connecting to: test" in out[0])
|
||||
self.assertEqual(NOCONNECT, mongo.returncode)
|
||||
|
||||
mongo = self.open_mongo([test, test_js, test_txt])
|
||||
out = mongo.communicate()
|
||||
self.assert_("url: " + test in out[0])
|
||||
self.assert_("connecting to: " + test in out[0])
|
||||
self.assertEqual(NOCONNECT, mongo.returncode)
|
||||
|
||||
mongo = self.open_mongo([non_exist_js, test, test_txt])
|
||||
out = mongo.communicate()
|
||||
self.assert_("url: test" in out[0])
|
||||
self.assert_("connecting to: test" in out[0])
|
||||
self.assertEqual(NOCONNECT, mongo.returncode)
|
||||
|
||||
mongo = self.open_mongo([non_exist_txt, test_js, test_txt])
|
||||
out = mongo.communicate()
|
||||
self.assert_("url: " + non_exist_txt in out[0])
|
||||
self.assert_("connecting to: " + non_exist_txt in out[0])
|
||||
self.assertEqual(NOCONNECT, mongo.returncode)
|
||||
|
||||
def test_multiple_files(self):
|
||||
dirname = os.path.dirname(__file__)
|
||||
test_js = os.path.join(dirname, "testdata/test.js")
|
||||
test_txt = os.path.join(dirname, "testdata/test.txt")
|
||||
|
||||
mongo = self.open_mongo(["--nodb", test_js, test_txt])
|
||||
out = mongo.communicate()
|
||||
self.assert_("hello world" in out[0])
|
||||
self.assert_("foobar" in out[0])
|
||||
self.assert_("bye" not in out[0])
|
||||
self.assertEqual(0, mongo.returncode)
|
||||
|
||||
mongo = self.open_mongo(["--shell", "--nodb", test_js, test_txt])
|
||||
out = mongo.communicate()
|
||||
self.assert_("hello world" in out[0])
|
||||
self.assert_("foobar" in out[0])
|
||||
self.assert_("bye" in out[0])
|
||||
self.assertEqual(0, mongo.returncode)
|
||||
|
||||
# just testing that they don't blow up
|
||||
def test_username_and_password(self):
|
||||
mongo = self.open_mongo(["--username", "mike"])
|
||||
out = mongo.communicate()
|
||||
self.assertEqual(NOCONNECT, mongo.returncode)
|
||||
|
||||
mongo = self.open_mongo(["-u", "mike"])
|
||||
out = mongo.communicate()
|
||||
self.assertEqual(NOCONNECT, mongo.returncode)
|
||||
|
||||
mongo = self.open_mongo(["--password", "mike"])
|
||||
out = mongo.communicate()
|
||||
self.assertEqual(NOCONNECT, mongo.returncode)
|
||||
|
||||
mongo = self.open_mongo(["-p", "mike"])
|
||||
out = mongo.communicate()
|
||||
self.assertEqual(NOCONNECT, mongo.returncode)
|
||||
|
||||
mongo = self.open_mongo(["--username"])
|
||||
out = mongo.communicate()
|
||||
self.assert_("required parameter is missing" in out[0])
|
||||
self.assertEqual(BADOPTS, mongo.returncode)
|
||||
|
||||
mongo = self.open_mongo(["--password"])
|
||||
out = mongo.communicate()
|
||||
self.assert_("required parameter is missing" in out[0])
|
||||
self.assertEqual(BADOPTS, mongo.returncode)
|
||||
|
||||
|
||||
def run_tests():
|
||||
suite = unittest.TestLoader().loadTestsFromTestCase(TestShell)
|
||||
unittest.TextTestRunner(verbosity=1).run(suite)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 2:
|
||||
print "must give the path to shell executable to be tested"
|
||||
sys.exit()
|
||||
|
||||
mongo_path = sys.argv[1]
|
||||
run_tests()
|
||||
16
mongo_astyle
16
mongo_astyle
@ -1,16 +0,0 @@
|
||||
--indent=spaces=4
|
||||
--convert-tabs
|
||||
--lineend=linux
|
||||
|
||||
--indent-namespaces
|
||||
--indent-preprocessor
|
||||
|
||||
|
||||
--keep-one-line-statements
|
||||
--keep-one-line-blocks
|
||||
|
||||
--break-closing-brackets
|
||||
--brackets=attach
|
||||
|
||||
|
||||
|
||||
@ -1261,20 +1261,6 @@ test = env.Install(
|
||||
if len(env.subst('$PROGSUFFIX')):
|
||||
env.Alias( "dbtest", "#/${PROGPREFIX}dbtest${PROGSUFFIX}" )
|
||||
|
||||
env.Install('$BUILD_ROOT/', env.Program('file_allocator_bench',
|
||||
'util/file_allocator_bench.cpp',
|
||||
LIBDEPS=[
|
||||
'mongocommon',
|
||||
'signal_handlers_synchronous',
|
||||
'$BUILD_DIR/mongo/util/options_parser/options_parser_init',
|
||||
'serveronly',
|
||||
'coredb',
|
||||
'coreserver',
|
||||
"ntservice_mock",
|
||||
]))
|
||||
|
||||
env.Alias('file_allocator_bench', "$BUILD_ROOT/" + add_exe("file_allocator_bench"))
|
||||
|
||||
# --- sniffer ---
|
||||
mongosniff_built = False
|
||||
if env.TargetOSIs('osx') or env["_HAVEPCAP"]:
|
||||
|
||||
@ -3,7 +3,6 @@
|
||||
import os
|
||||
|
||||
Import("env")
|
||||
Import("s3push")
|
||||
Import("has_option")
|
||||
|
||||
env = env.Clone()
|
||||
@ -146,10 +145,3 @@ env.AlwaysBuild(msi)
|
||||
|
||||
env.Alias( "msi" , msi )
|
||||
|
||||
def s3msipush( env , target , source ):
|
||||
s3push( source )
|
||||
|
||||
env.Alias( "s3msi" , [ msi ] , [ s3msipush ] )
|
||||
env.AlwaysBuild( "s3msi" )
|
||||
|
||||
|
||||
|
||||
@ -1,334 +0,0 @@
|
||||
/*
|
||||
* Copyright 2014 MongoDB Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3,
|
||||
* as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*
|
||||
* As a special exception, the copyright holders give permission to link the
|
||||
* code of portions of this program with the OpenSSL library under certain
|
||||
* conditions as described in each individual source file and distribute
|
||||
* linked combinations including the program with the OpenSSL library. You
|
||||
* must comply with the GNU Affero General Public License in all respects
|
||||
* for all of the code used other than as permitted herein. If you modify
|
||||
* file(s) with this exception, you may extend this exception to your
|
||||
* version of the file(s), but you are not obligated to do so. If you do not
|
||||
* wish to do so, delete this exception statement from your version. If you
|
||||
* delete this exception statement from all source files in the program,
|
||||
* then also delete it in the license file.
|
||||
*/
|
||||
|
||||
#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kDefault
|
||||
|
||||
#include "mongo/platform/basic.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <boost/date_time/posix_time/posix_time.hpp>
|
||||
#include <boost/filesystem.hpp>
|
||||
#include <boost/scoped_ptr.hpp>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <numeric>
|
||||
#include <string>
|
||||
|
||||
#include "mongo/base/initializer.h"
|
||||
#include "mongo/base/status.h"
|
||||
#include "mongo/bson/bsonobjbuilder.h"
|
||||
#include "mongo/util/file_allocator.h"
|
||||
#include "mongo/util/log.h"
|
||||
#include "mongo/util/mongoutils/str.h"
|
||||
#include "mongo/util/options_parser/environment.h"
|
||||
#include "mongo/util/options_parser/option_section.h"
|
||||
#include "mongo/util/options_parser/options_parser.h"
|
||||
#include "mongo/util/options_parser/startup_option_init.h"
|
||||
#include "mongo/util/options_parser/startup_options.h"
|
||||
#include "mongo/util/quick_exit.h"
|
||||
#include "mongo/util/scopeguard.h"
|
||||
#include "mongo/util/signal_handlers_synchronous.h"
|
||||
|
||||
using namespace mongo;
|
||||
|
||||
namespace {
|
||||
namespace file = boost::filesystem;
|
||||
namespace ptime = boost::posix_time;
|
||||
|
||||
typedef unsigned long long bytes_t;
|
||||
typedef long long micros_t;
|
||||
|
||||
const long DEFAULT_FILE_SIZE_MB = 128;
|
||||
const file::path DEFAULT_PATH = file::temp_directory_path();
|
||||
const int DEFAULT_NTRIALS = 10;
|
||||
const bool DEFAULT_BSON_OUT = false;
|
||||
|
||||
// used to convert B/usec to MB/sec
|
||||
const double MICROSEC_PER_SEC = 1e6;
|
||||
const double MB_PER_BYTE = 1.0 / (1 << 20);
|
||||
const double MB_SEC_CONVERSION_FACTOR = MICROSEC_PER_SEC * MB_PER_BYTE;
|
||||
|
||||
double toMbSec(const bytes_t bytes, const micros_t micros) {
|
||||
return (static_cast<double>(bytes) / static_cast<double>(micros)) *
|
||||
MB_SEC_CONVERSION_FACTOR;
|
||||
}
|
||||
}
|
||||
|
||||
struct BenchmarkParams {
|
||||
bytes_t bytes;
|
||||
file::path path;
|
||||
int ntrials;
|
||||
bool quiet;
|
||||
bool jsonReportEnabled;
|
||||
std::string jsonReportOut;
|
||||
} benchParams;
|
||||
|
||||
class FileAllocatorBenchmark {
|
||||
public:
|
||||
FileAllocatorBenchmark(const BenchmarkParams& params)
|
||||
: _fa(FileAllocator::get())
|
||||
, _params(params) {
|
||||
_fa->start();
|
||||
|
||||
if (!file::create_directory(_params.path)) {
|
||||
std::cerr << "Error: unable to create temporary directory in "
|
||||
<< _params.path.parent_path() << std::endl;
|
||||
quickExit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
// Delete any files we created
|
||||
~FileAllocatorBenchmark() {
|
||||
file::remove_all(_params.path);
|
||||
}
|
||||
|
||||
void run() {
|
||||
if (!_params.quiet) {
|
||||
std::cout << "Allocating " << _params.ntrials << " files of size "
|
||||
<< _params.bytes << " bytes in " << _params.path << std::endl;
|
||||
}
|
||||
|
||||
for (int n = 0; n < _params.ntrials; ++n) {
|
||||
const std::string fileName = str::stream() << "garbage-" << n;
|
||||
file::path filePath = _params.path / fileName;
|
||||
_files.push_back(filePath);
|
||||
bytes_t size_allocated = _params.bytes;
|
||||
const ptime::ptime start = ptime::microsec_clock::universal_time();
|
||||
|
||||
try {
|
||||
_fa->allocateAsap(filePath.string(), size_allocated);
|
||||
} catch (const DBException& ex) {
|
||||
std::cerr << "Exception thrown while allocating file:" << std::endl;
|
||||
std::cerr << ex.what() << std::endl;
|
||||
throw; // rethrow so that destructor is called
|
||||
}
|
||||
|
||||
if (size_allocated != static_cast<bytes_t>(_params.bytes)) {
|
||||
std::cerr << "Allocated " << size_allocated << " bytes but expected "
|
||||
<< _params.bytes;
|
||||
}
|
||||
|
||||
const ptime::ptime end = ptime::microsec_clock::universal_time();
|
||||
_results.push_back((end - start).total_microseconds());
|
||||
}
|
||||
|
||||
_fa->waitUntilFinished();
|
||||
|
||||
if (!_params.quiet) {
|
||||
textReport();
|
||||
}
|
||||
|
||||
if (_params.jsonReportEnabled) {
|
||||
jsonReport(_params.jsonReportOut);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
struct benchResults {
|
||||
micros_t avg;
|
||||
micros_t max;
|
||||
micros_t min;
|
||||
};
|
||||
|
||||
benchResults computeResults() {
|
||||
benchResults res;
|
||||
const micros_t total = std::accumulate(_results.begin(), _results.end(), 0L);
|
||||
res.avg = total / _results.size();
|
||||
res.max = *std::max_element(_results.begin(), _results.end());
|
||||
res.min = *std::min_element(_results.begin(), _results.end());
|
||||
return res;
|
||||
}
|
||||
|
||||
void printResult(const std::string& name, const micros_t duration, const bytes_t bytes) {
|
||||
std::cout << name << ": " << duration << " usec = "
|
||||
<< toMbSec(bytes, duration) << " MB/sec" << std::endl;
|
||||
|
||||
}
|
||||
|
||||
void textReport() {
|
||||
benchResults results = computeResults();
|
||||
|
||||
std::cout << "Results for " << _params.ntrials << " allocations of "
|
||||
<< _params.bytes << " bytes: " << std::endl;
|
||||
|
||||
printResult("avg", results.avg, _params.bytes);
|
||||
printResult("max", results.max, _params.bytes);
|
||||
printResult("min", results.min, _params.bytes);
|
||||
}
|
||||
|
||||
void addResult(BSONObjBuilder& obj, const std::string& name,
|
||||
const micros_t duration, const bytes_t bytes) {
|
||||
BSONObjBuilder so(obj.subobjStart(name));
|
||||
so.append("usec", duration);
|
||||
so.append("MBsec", toMbSec(bytes, duration));
|
||||
so.done();
|
||||
}
|
||||
|
||||
void jsonReport(const std::string& jsonReportOut) {
|
||||
benchResults results = computeResults();
|
||||
BSONObjBuilder obj;
|
||||
|
||||
obj.append("bytes", static_cast<long long>(_params.bytes));
|
||||
addResult(obj, "avg", results.avg, _params.bytes);
|
||||
addResult(obj, "max", results.max, _params.bytes);
|
||||
addResult(obj, "min", results.min, _params.bytes);
|
||||
|
||||
obj.append("raw", _results);
|
||||
|
||||
const std::string outStr = obj.done().toString();
|
||||
|
||||
if (jsonReportOut == "-") {
|
||||
std::cout << outStr << std::endl;
|
||||
} else {
|
||||
std::ofstream outfile(jsonReportOut.c_str());
|
||||
if (!outfile.is_open()) {
|
||||
std::cerr << "Error: couldn't create output file " << jsonReportOut << std::endl;
|
||||
return;
|
||||
}
|
||||
ON_BLOCK_EXIT(&std::ofstream::close, outfile);
|
||||
outfile << outStr << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
FileAllocator* const _fa;
|
||||
std::vector<micros_t> _results;
|
||||
std::vector<file::path> _files;
|
||||
|
||||
const BenchmarkParams& _params;
|
||||
};
|
||||
|
||||
namespace moe = mongo::optionenvironment;
|
||||
|
||||
Status addFileAllocatorBenchOptions(moe::OptionSection& options) {
|
||||
options.addOptionChaining("help", "help", moe::Switch, "Display help");
|
||||
options.addOptionChaining("megabytes", "megabytes", moe::Long,
|
||||
"The number of megabytes to allocate for each file")
|
||||
.setDefault(moe::Value(DEFAULT_FILE_SIZE_MB));
|
||||
|
||||
options.addOptionChaining("path", "path", moe::String,
|
||||
str::stream() << "The directory path to allocate the file(s) in "
|
||||
<< "during testing. Files will be allocated in a "
|
||||
<< "uniquely named temporary directory within the "
|
||||
<< "specified path")
|
||||
.setDefault(moe::Value(DEFAULT_PATH.string()));
|
||||
|
||||
options.addOptionChaining("ntrials", "ntrials", moe::Int,
|
||||
"The number of trials to perform")
|
||||
.setDefault(moe::Value(DEFAULT_NTRIALS));
|
||||
|
||||
options.addOptionChaining("quiet", "quiet", moe::Switch,
|
||||
"Suppress the plaintext report");
|
||||
|
||||
options.addOptionChaining("jsonReport", "jsonReport", moe::String,
|
||||
str::stream() << "If set, results will be saved as a JSON document to "
|
||||
<< "the specified file path. If specified with no "
|
||||
<< "arguments the report will be printed to standard "
|
||||
<< "out")
|
||||
.setImplicit(moe::Value(std::string("-")));
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status validateFileAllocatorBenchOptions(const moe::OptionSection& options,
|
||||
moe::Environment& env) {
|
||||
Status ret = env.validate();
|
||||
if (!ret.isOK()) {
|
||||
return ret;
|
||||
}
|
||||
bool displayHelp = false;
|
||||
ret = env.get(moe::Key("help"), &displayHelp);
|
||||
if (displayHelp) {
|
||||
std::cout << options.helpString() << std::endl;
|
||||
quickExit(EXIT_SUCCESS);
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status storeFileAllocatorBenchOptions(const moe::Environment& env) {
|
||||
// don't actually need to check Status since we set default values
|
||||
long mbytes;
|
||||
Status ret = env.get(moe::Key("megabytes"), &mbytes);
|
||||
benchParams.bytes = mbytes * (1 << 20);
|
||||
|
||||
std::string path;
|
||||
ret = env.get(moe::Key("path"), &path);
|
||||
|
||||
file::path rootPath = file::path(path);
|
||||
|
||||
if (!file::is_directory(rootPath)) {
|
||||
std::cerr << "Error: path argument must be a directory" << std::endl;
|
||||
quickExit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
benchParams.path = rootPath / file::unique_path("allocator-bench-%%%%%%%%");
|
||||
|
||||
ret = env.get(moe::Key("ntrials"), &benchParams.ntrials);
|
||||
ret = env.get(moe::Key("quiet"), &benchParams.quiet);
|
||||
|
||||
benchParams.jsonReportEnabled = true;
|
||||
ret = env.get(moe::Key("jsonReport"), &benchParams.jsonReportOut);
|
||||
if (!ret.isOK()) {
|
||||
benchParams.jsonReportEnabled = false;
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
int main(int argc, char** argv, char** envp) {
|
||||
::mongo::setupSynchronousSignalHandlers();
|
||||
::mongo::runGlobalInitializersOrDie(argc, argv, envp);
|
||||
try {
|
||||
// this try/catch block needs to exist so that
|
||||
// std::terminate is not called and the FileAllocatorBenchmark
|
||||
// destructor actually gets called...
|
||||
FileAllocatorBenchmark(benchParams).run();
|
||||
} catch (...) {
|
||||
std::cerr << "Benchmark ended in failure." << std::endl;
|
||||
quickExit(EXIT_FAILURE);
|
||||
}
|
||||
quickExit(EXIT_SUCCESS);
|
||||
}
|
||||
|
||||
MONGO_INITIALIZER(KillLoggingOutput)(InitializerContext* context) {
|
||||
// The FileAllocator produces a lot of log noise, so we silence
|
||||
// all logging output
|
||||
mongo::logger::globalLogDomain()->clearAppenders();
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
MONGO_GENERAL_STARTUP_OPTIONS_REGISTER(FileAllocatorBenchOptions)(InitializerContext* context) {
|
||||
return addFileAllocatorBenchOptions(moe::startupOptions);
|
||||
}
|
||||
|
||||
MONGO_STARTUP_OPTIONS_VALIDATE(FileAllocatorBenchOptions)(InitializerContext* context) {
|
||||
return validateFileAllocatorBenchOptions(moe::startupOptions,
|
||||
moe::startupOptionsParsed);
|
||||
}
|
||||
|
||||
MONGO_STARTUP_OPTIONS_STORE(FileAllocatorBenchOptions)(InitializerContext* context) {
|
||||
return storeFileAllocatorBenchOptions(moe::startupOptionsParsed);
|
||||
}
|
||||
@ -1,12 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ImportGroup Label="PropertySheets" />
|
||||
<PropertyGroup Label="UserMacros" />
|
||||
<PropertyGroup />
|
||||
<ItemDefinitionGroup>
|
||||
<ClCompile>
|
||||
<PreprocessorDefinitions>MONGO_USE_SRW_ON_WINDOWS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
</ClCompile>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemGroup />
|
||||
</Project>
|
||||
Loading…
Reference in New Issue
Block a user