mongodb/mongo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SERVER-32295 Support Python 3

Browse Source
This commit is contained in:
Mathew Robinson 2019-02-19 10:50:57 -05:00
parent c600aa9d74
commit 8dd6d47557
257 changed files with 2613 additions and 2910 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.pylintrc
View File

@ -22,8 +22,15 @@ variable-rgx=[a-z_][a-z0-9_]{1,50}$
# R1705 - no-else-return - sometimes an unnecessary else helps readability
# W0511 - fixme - ignore TODOs in comments
# W0611 - unused-import - typing module is needed for mypy
# R0205 - useless-object-inheritance - See PM-1380
# W0402 - deprecated-module - See PM-1380
# W1505 - deprecated-method - See PM-1380
# W0107 - unnecessary-pass - See PM-1380
# R1720 - no-else-raise - See PM-1380
# W0122 - exec-used - See PM-1380
# R0801 - duplicate-code - See PM-1380
disable=bad-continuation,fixme,import-error,line-too-long,no-member,locally-disabled,no-else-return,redefined-variable-type,too-few-public-methods,unused-import
disable=bad-continuation,fixme,import-error,line-too-long,no-member,locally-disabled,no-else-return,redefined-variable-type,too-few-public-methods,unused-import,useless-object-inheritance,deprecated-module,unnecessary-pass,duplicate-code,no-else-raise,deprecated-method,exec-used
[IMPORTS]
known-third-party=boto3,botocore,psutil,yaml,xmlrunner

65
SConstruct
View File

@ -27,7 +27,7 @@ import mongo.platform as mongo_platform
import mongo.toolchain as mongo_toolchain
import mongo.generators as mongo_generators
EnsurePythonVersion(2, 7)
EnsurePythonVersion(3, 5)
EnsureSConsVersion(3, 0, 4)
from buildscripts import utils
@ -466,7 +466,7 @@ win_version_min_choices = {
}
add_option('win-version-min',
choices=win_version_min_choices.keys(),
choices=list(win_version_min_choices.keys()),
default=None,
help='minimum Windows version to support',
type='choice',
@ -581,7 +581,7 @@ try:
except IOError as e:
# If the file error wasn't because the file is missing, error out
if e.errno != errno.ENOENT:
print("Error opening version.json: {0}".format(e.strerror))
print(("Error opening version.json: {0}".format(e.strerror)))
Exit(1)
version_data = {
@ -590,14 +590,14 @@ except IOError as e:
}
except ValueError as e:
print("Error decoding version.json: {0}".format(e))
print(("Error decoding version.json: {0}".format(e)))
Exit(1)
# Setup the command-line variables
def variable_shlex_converter(val):
# If the argument is something other than a string, propogate
# it literally.
if not isinstance(val, basestring):
if not isinstance(val, str):
return val
parse_mode = get_option('variable-parse-mode')
if parse_mode == 'auto':
@ -661,7 +661,7 @@ def variable_distsrc_converter(val):
variables_files = variable_shlex_converter(get_option('variables-files'))
for file in variables_files:
print("Using variable customization file %s" % file)
print(("Using variable customization file %s" % file))
env_vars = Variables(
files=variables_files,
@ -670,7 +670,7 @@ env_vars = Variables(
sconsflags = os.environ.get('SCONSFLAGS', None)
if sconsflags:
print("Using SCONSFLAGS environment variable arguments: %s" % sconsflags)
print(("Using SCONSFLAGS environment variable arguments: %s" % sconsflags))
env_vars.Add('ABIDW',
help="Configures the path to the 'abidw' (a libabigail) utility")
@ -800,7 +800,7 @@ env_vars.Add('MONGO_DISTNAME',
def validate_mongo_version(key, val, env):
regex = r'^(\d+)\.(\d+)\.(\d+)-?((?:(rc)(\d+))?.*)?'
if not re.match(regex, val):
print("Invalid MONGO_VERSION '{}', or could not derive from version.json or git metadata. Please add a conforming MONGO_VERSION=x.y.z[-extra] as an argument to SCons".format(val))
print(("Invalid MONGO_VERSION '{}', or could not derive from version.json or git metadata. Please add a conforming MONGO_VERSION=x.y.z[-extra] as an argument to SCons".format(val)))
Exit(1)
env_vars.Add('MONGO_VERSION',
@ -935,12 +935,12 @@ if installDir[0] not in ['$', '#']:
Exit(1)
sconsDataDir = Dir(buildDir).Dir('scons')
SConsignFile(str(sconsDataDir.File('sconsign')))
SConsignFile(str(sconsDataDir.File('sconsign.py3')))
def printLocalInfo():
import sys, SCons
print( "scons version: " + SCons.__version__ )
print( "python version: " + " ".join( [ `i` for i in sys.version_info ] ) )
print(( "scons version: " + SCons.__version__ ))
print(( "python version: " + " ".join( [ repr(i) for i in sys.version_info ] ) ))
printLocalInfo()
@ -1020,12 +1020,12 @@ env.AddMethod(mongo_platform.env_os_is_wrapper, 'TargetOSIs')
env.AddMethod(mongo_platform.env_get_os_name_wrapper, 'GetTargetOSName')
def fatal_error(env, msg, *args):
print(msg.format(*args))
print((msg.format(*args)))
Exit(1)
def conf_error(env, msg, *args):
print(msg.format(*args))
print("See {0} for details".format(env.File('$CONFIGURELOG').abspath))
print((msg.format(*args)))
print(("See {0} for details".format(env.File('$CONFIGURELOG').abspath)))
Exit(1)
env.AddMethod(fatal_error, 'FatalError')
@ -1044,12 +1044,12 @@ else:
env.AddMethod(lambda env: env['VERBOSE'], 'Verbose')
if has_option('variables-help'):
print(env_vars.GenerateHelpText(env))
print((env_vars.GenerateHelpText(env)))
Exit(0)
unknown_vars = env_vars.UnknownVariables()
if unknown_vars:
env.FatalError("Unknown variables specified: {0}", ", ".join(unknown_vars.keys()))
env.FatalError("Unknown variables specified: {0}", ", ".join(list(unknown_vars.keys())))
def set_config_header_define(env, varname, varval = 1):
env['CONFIG_HEADER_DEFINES'][varname] = varval
@ -1135,7 +1135,7 @@ def CheckForProcessor(context, which_arch):
context.Result(ret)
return ret;
for k in processor_macros.keys():
for k in list(processor_macros.keys()):
ret = run_compile_check(k)
if ret:
context.Result('Detected a %s processor' % k)
@ -1258,7 +1258,7 @@ else:
env['TARGET_ARCH'] = detected_processor
if env['TARGET_OS'] not in os_macros:
print("No special config for [{0}] which probably means it won't work".format(env['TARGET_OS']))
print(("No special config for [{0}] which probably means it won't work".format(env['TARGET_OS'])))
elif not detectConf.CheckForOS(env['TARGET_OS']):
env.ConfError("TARGET_OS ({0}) is not supported by compiler", env['TARGET_OS'])
@ -1393,8 +1393,8 @@ if link_model.startswith("dynamic"):
if env.TargetOSIs('darwin'):
if link_model.startswith('dynamic'):
print("WARNING: Building MongoDB server with dynamic linking " +
"on macOS is not supported. Static linking is recommended.")
print(("WARNING: Building MongoDB server with dynamic linking " +
"on macOS is not supported. Static linking is recommended."))
if link_model == "dynamic-strict":
# Darwin is strict by default
@ -2151,7 +2151,7 @@ def doConfigure(myenv):
# form -Wno-xxx (but not -Wno-error=xxx), we also add -Wxxx to the flags. GCC does
# warn on unknown -Wxxx style flags, so this lets us probe for availablity of
# -Wno-xxx.
for kw in test_mutation.keys():
for kw in list(test_mutation.keys()):
test_flags = test_mutation[kw]
for test_flag in test_flags:
if test_flag.startswith("-Wno-") and not test_flag.startswith("-Wno-error="):
@ -2165,7 +2165,7 @@ def doConfigure(myenv):
# to make them real errors.
cloned.Append(CCFLAGS=['-Werror'])
conf = Configure(cloned, help=False, custom_tests = {
'CheckFlag' : lambda(ctx) : CheckFlagTest(ctx, tool, extension, flag)
'CheckFlag' : lambda ctx : CheckFlagTest(ctx, tool, extension, flag)
})
available = conf.CheckFlag()
conf.Finish()
@ -2645,7 +2645,7 @@ def doConfigure(myenv):
# Select those unique black files that are associated with the
# currently enabled sanitizers, but filter out those that are
# zero length.
blackfiles = {v for (k, v) in blackfiles_map.iteritems() if k in sanitizer_list}
blackfiles = {v for (k, v) in blackfiles_map.items() if k in sanitizer_list}
blackfiles = [f for f in blackfiles if os.stat(f.path).st_size != 0]
# Filter out any blacklist options that the toolchain doesn't support.
@ -2676,7 +2676,7 @@ def doConfigure(myenv):
llvm_symbolizer = get_option('llvm-symbolizer')
if os.path.isabs(llvm_symbolizer):
if not myenv.File(llvm_symbolizer).exists():
print("WARNING: Specified symbolizer '%s' not found" % llvm_symbolizer)
print(("WARNING: Specified symbolizer '%s' not found" % llvm_symbolizer))
llvm_symbolizer = None
else:
llvm_symbolizer = myenv.WhereIs(llvm_symbolizer)
@ -2997,7 +2997,7 @@ def doConfigure(myenv):
# TODO: If we could programmatically extract the paths from the info output
# we could give a better message here, but brew info's machine readable output
# doesn't seem to include the whole 'caveats' section.
message = subprocess.check_output([brew, "info", "openssl"])
message = subprocess.check_output([brew, "info", "openssl"]).decode('utf-8')
advice = textwrap.dedent(
"""\
NOTE: HomeBrew installed to {0} appears to have OpenSSL installed.
@ -3159,7 +3159,7 @@ def doConfigure(myenv):
# Either crypto engine is native,
# or it's OpenSSL and has been checked to be working.
conf.env.SetConfigHeaderDefine("MONGO_CONFIG_SSL")
print("Using SSL Provider: {0}".format(ssl_provider))
print(("Using SSL Provider: {0}".format(ssl_provider)))
else:
ssl_provider = "none"
@ -3182,7 +3182,7 @@ def doConfigure(myenv):
files = ['ssleay32.dll', 'libeay32.dll']
for extra_file in files:
if not addOpenSslLibraryToDistArchive(extra_file):
print("WARNING: Cannot find SSL library '%s'" % extra_file)
print(("WARNING: Cannot find SSL library '%s'" % extra_file))
def checkHTTPLib(required=False):
# WinHTTP available on Windows
@ -3558,7 +3558,7 @@ def doConfigure(myenv):
outputIndex = next((idx for idx in [0,1] if conf.CheckAltivecVbpermqOutput(idx)), None)
if outputIndex is not None:
conf.env.SetConfigHeaderDefine("MONGO_CONFIG_ALTIVEC_VEC_VBPERMQ_OUTPUT_INDEX", outputIndex)
conf.env.SetConfigHeaderDefine("MONGO_CONFIG_ALTIVEC_VEC_VBPERMQ_OUTPUT_INDEX", outputIndex)
else:
myenv.ConfError("Running on ppc64le, but can't find a correct vec_vbpermq output index. Compiler or platform not supported")
@ -3658,9 +3658,12 @@ def doLint( env , target , source ):
import buildscripts.pylinters
buildscripts.pylinters.lint_all(None, {}, [])
import buildscripts.lint
if not buildscripts.lint.run_lint( [ "src/mongo/" ] ):
raise Exception( "lint errors" )
env.Command(
target="#run_lint",
source=["buildscripts/lint.py", "src/mongo"],
action="$PYTHON $SOURCES[0] $SOURCES[1]",
)
env.Alias( "lint" , [] , [ doLint ] )
env.AlwaysBuild( "lint" )

2
buildscripts/aggregate_tracefiles.py
View File

@ -20,7 +20,7 @@ def aggregate(inputs, output):
args += ['-o', output]
print ' '.join(args)
print(' '.join(args))
return subprocess.call(args)

72
buildscripts/aws_ec2.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python
"""AWS EC2 instance launcher and controller."""
from __future__ import print_function
import base64
import collections
@ -88,12 +88,13 @@ class AwsEc2(object):
if reached_state:
print(" Instance {}!".format(instance.state["Name"]), file=sys.stdout)
else:
print(" Instance in state '{}', failed to reach state '{}'{}!".format(
instance.state["Name"], state, client_error), file=sys.stdout)
print(
" Instance in state '{}', failed to reach state '{}'{}!".format(
instance.state["Name"], state, client_error), file=sys.stdout)
sys.stdout.flush()
return 0 if reached_state else 1
def control_instance( #pylint: disable=too-many-arguments,too-many-branches
def control_instance( #pylint: disable=too-many-arguments,too-many-branches,too-many-locals
self, mode, image_id, wait_time_secs=0, show_progress=False, console_output_file=None,
console_screenshot_file=None):
"""Control an AMI instance. Returns 0 & status information, if successful."""
@ -238,27 +239,29 @@ def main(): # pylint: disable=too-many-locals,too-many-statements
create_options = optparse.OptionGroup(parser, "Create options")
status_options = optparse.OptionGroup(parser, "Status options")
parser.add_option("--mode", dest="mode", choices=_MODES, default="status",
help=("Operations to perform on an EC2 instance, choose one of"
" '{}', defaults to '%default'.".format(", ".join(_MODES))))
parser.add_option(
"--mode", dest="mode", choices=_MODES, default="status",
help=("Operations to perform on an EC2 instance, choose one of"
" '{}', defaults to '%default'.".format(", ".join(_MODES))))
control_options.add_option("--imageId", dest="image_id", default=None,
help="EC2 image_id to perform operation on [REQUIRED for control].")
control_options.add_option("--waitTimeSecs", dest="wait_time_secs", type=int, default=5 * 60,
help=("Time to wait for EC2 instance to reach it's new state,"
" defaults to '%default'."))
control_options.add_option(
"--waitTimeSecs", dest="wait_time_secs", type=int, default=5 * 60,
help=("Time to wait for EC2 instance to reach it's new state,"
" defaults to '%default'."))
create_options.add_option("--ami", dest="ami", default=None,
help="EC2 AMI to launch [REQUIRED for create].")
create_options.add_option("--blockDevice", dest="block_devices",
metavar="DEVICE-NAME DEVICE-SIZE-GB", action="append", default=[],
nargs=2,
help=("EBS device name and volume size in GiB."
" More than one device can be attached, by specifying"
" this option more than once."
" The device will be deleted on termination of the instance."))
create_options.add_option(
"--blockDevice", dest="block_devices", metavar="DEVICE-NAME DEVICE-SIZE-GB",
action="append", default=[], nargs=2,
help=("EBS device name and volume size in GiB."
" More than one device can be attached, by specifying"
" this option more than once."
" The device will be deleted on termination of the instance."))
create_options.add_option("--instanceType", dest="instance_type", default="t1.micro",
help="EC2 instance type to launch, defaults to '%default'.")
@ -266,15 +269,15 @@ def main(): # pylint: disable=too-many-locals,too-many-statements
create_options.add_option("--keyName", dest="key_name", default=None,
help="EC2 key name [REQUIRED for create].")
create_options.add_option("--securityGroupIds", dest="security_group_ids", action="append",
default=[],
help=("EC2 security group ids. More than one security group id can be"
" added, by specifying this option more than once."))
create_options.add_option(
"--securityGroupIds", dest="security_group_ids", action="append", default=[],
help=("EC2 security group ids. More than one security group id can be"
" added, by specifying this option more than once."))
create_options.add_option("--securityGroup", dest="security_groups", action="append",
default=[],
help=("EC2 security group. More than one security group can be added,"
" by specifying this option more than once."))
create_options.add_option(
"--securityGroup", dest="security_groups", action="append", default=[],
help=("EC2 security group. More than one security group can be added,"
" by specifying this option more than once."))
create_options.add_option("--subnetId", dest="subnet_id", default=None,
help="EC2 subnet id to use in VPC.")
@ -296,14 +299,15 @@ def main(): # pylint: disable=too-many-locals,too-many-statements
status_options.add_option("--yamlFile", dest="yaml_file", default=None,
help="Save the status into the specified YAML file.")
status_options.add_option("--consoleOutputFile", dest="console_output_file", default=None,
help="Save the console output into the specified file, if"
" available.")
status_options.add_option(
"--consoleOutputFile", dest="console_output_file", default=None,
help="Save the console output into the specified file, if"
" available.")
status_options.add_option("--consoleScreenshotFile", dest="console_screenshot_file",
default=None,
help="Save the console screenshot (JPG format) into the specified"
" file, if available.")
status_options.add_option(
"--consoleScreenshotFile", dest="console_screenshot_file", default=None,
help="Save the console screenshot (JPG format) into the specified"
" file, if available.")
parser.add_option_group(control_options)
parser.add_option_group(create_options)
@ -331,8 +335,8 @@ def main(): # pylint: disable=too-many-locals,too-many-statements
# The 'expire-on' key is a UTC time.
expire_dt = datetime.datetime.utcnow() + datetime.timedelta(hours=options.tag_expire_hours)
tags = [{"Key": "expire-on", "Value": expire_dt.strftime("%Y-%m-%d %H:%M:%S")},
{"Key": "Name",
"Value": options.tag_name}, {"Key": "owner", "Value": options.tag_owner}]
{"Key": "Name", "Value": options.tag_name},
{"Key": "owner", "Value": options.tag_owner}]
my_kwargs = {}
if options.extra_args is not None:

70
buildscripts/burn_in_tests.py
View File

@ -1,8 +1,6 @@
#!/usr/bin/env python
"""Command line utility for determining what jstests have been added or modified."""
from __future__ import absolute_import
from __future__ import print_function
import collections
import copy
@ -13,7 +11,7 @@ import subprocess
import re
import shlex
import sys
import urlparse
import urllib.parse
import requests
import yaml
@ -56,13 +54,15 @@ def parse_command_line():
parser = optparse.OptionParser(usage="Usage: %prog [options] [resmoke command]")
parser.add_option("--maxRevisions", dest="max_revisions", type=int, default=25,
help=("Maximum number of revisions to check for changes. Default is"
" %default."))
parser.add_option(
"--maxRevisions", dest="max_revisions", type=int, default=25,
help=("Maximum number of revisions to check for changes. Default is"
" %default."))
parser.add_option("--branch", dest="branch", default="master",
help=("The name of the branch the working branch was based on. Default is"
" '%default'."))
parser.add_option(
"--branch", dest="branch", default="master",
help=("The name of the branch the working branch was based on. Default is"
" '%default'."))
parser.add_option("--baseCommit", dest="base_commit", default=None,
help="The base commit to compare to for determining changes.")
@ -79,14 +79,15 @@ def parse_command_line():
help=("The distro the tasks will execute on. Can only be specified"
" with --generateTasksFile."))
parser.add_option("--checkEvergreen", dest="check_evergreen", default=False,
action="store_true",
help=("Checks Evergreen for the last commit that was scheduled."
" This way all the tests that haven't been burned in will be run."))
parser.add_option(
"--checkEvergreen", dest="check_evergreen", default=False, action="store_true",
help=("Checks Evergreen for the last commit that was scheduled."
" This way all the tests that haven't been burned in will be run."))
parser.add_option("--generateTasksFile", dest="generate_tasks_file", default=None,
help=("Write an Evergreen generate.tasks JSON file. If this option is"
" specified then no tests will be executed."))
parser.add_option(
"--generateTasksFile", dest="generate_tasks_file", default=None,
help=("Write an Evergreen generate.tasks JSON file. If this option is"
" specified then no tests will be executed."))
parser.add_option("--noExec", dest="no_exec", default=False, action="store_true",
help="Do not run resmoke loop on new tests.")
@ -100,21 +101,25 @@ def parse_command_line():
parser.add_option("--testListOutfile", dest="test_list_outfile", default=None,
help="Write a JSON file with test executor information.")
parser.add_option("--repeatTests", dest="repeat_tests_num", default=None, type=int,
help="The number of times to repeat each test. If --repeatTestsSecs is not"
" specified then this will be set to {}.".format(REPEAT_SUITES))
parser.add_option(
"--repeatTests", dest="repeat_tests_num", default=None, type=int,
help="The number of times to repeat each test. If --repeatTestsSecs is not"
" specified then this will be set to {}.".format(REPEAT_SUITES))
parser.add_option("--repeatTestsMin", dest="repeat_tests_min", default=None, type=int,
help="The minimum number of times to repeat each test when --repeatTestsSecs"
" is specified.")
parser.add_option(
"--repeatTestsMin", dest="repeat_tests_min", default=None, type=int,
help="The minimum number of times to repeat each test when --repeatTestsSecs"
" is specified.")
parser.add_option("--repeatTestsMax", dest="repeat_tests_max", default=None, type=int,
help="The maximum number of times to repeat each test when --repeatTestsSecs"
" is specified.")
parser.add_option(
"--repeatTestsMax", dest="repeat_tests_max", default=None, type=int,
help="The maximum number of times to repeat each test when --repeatTestsSecs"
" is specified.")
parser.add_option("--repeatTestsSecs", dest="repeat_tests_secs", default=None, type=float,
help="Time, in seconds, to repeat each test. Note that this option is"
" mutually exclusive with with --repeatTests.")
parser.add_option(
"--repeatTestsSecs", dest="repeat_tests_secs", default=None, type=float,
help="Time, in seconds, to repeat each test. Note that this option is"
" mutually exclusive with with --repeatTests.")
# This disables argument parsing on the first unrecognized parameter. This allows us to pass
# a complete resmoke.py command line without accidentally parsing its options.
@ -169,7 +174,7 @@ def find_last_activated_task(revisions, variant, branch_name):
evg_cfg = evergreen_client.read_evg_config()
if evg_cfg is not None and "api_server_host" in evg_cfg:
api_server = "{url.scheme}://{url.netloc}".format(
url=urlparse.urlparse(evg_cfg["api_server_host"]))
url=urllib.parse.urlparse(evg_cfg["api_server_host"]))
else:
api_server = API_SERVER_DEFAULT
@ -210,6 +215,7 @@ def find_changed_tests( # pylint: disable=too-many-locals
if base_commit is None:
base_commit = repo.get_merge_base([branch_name + "@{upstream}", "HEAD"])
if check_evergreen:
# We're going to check up to 200 commits in Evergreen for the last scheduled one.
# The current commit will be activated in Evergreen; we use --skip to start at the
@ -268,8 +274,8 @@ def find_excludes(selector_file):
try:
js_test = yml["selector"]["js_test"]
except KeyError:
raise Exception(
"The selector file " + selector_file + " is missing the 'selector.js_test' key")
raise Exception("The selector file " + selector_file +
" is missing the 'selector.js_test' key")
return (resmokelib.utils.default_if_none(js_test.get("exclude_suites"), []),
resmokelib.utils.default_if_none(js_test.get("exclude_tasks"), []),
@ -352,7 +358,7 @@ def create_task_list( #pylint: disable=too-many-locals
evg_buildvariant = evergreen_conf.get_variant(buildvariant)
if not evg_buildvariant:
print("Buildvariant '{}' not found".format(buildvariant))
print("Buildvariant '{}' not found in {}".format(buildvariant, evergreen_conf.path))
sys.exit(1)
# Find all the buildvariant tasks.

22
buildscripts/bypass_compile_and_fetch_binaries.py
View File

@ -1,8 +1,8 @@
#!/usr/bin/env python
"""Bypass compile and fetch binaries."""
from __future__ import absolute_import
from __future__ import print_function
import argparse
import json
@ -11,10 +11,10 @@ import re
import sys
import tarfile
import urllib
import urllib.request, urllib.parse, urllib.error
# pylint: disable=ungrouped-imports
try:
from urlparse import urlparse
from urllib.parse import urlparse
except ImportError:
from urllib.parse import urlparse # type: ignore
# pylint: enable=ungrouped-imports
@ -259,8 +259,8 @@ def main(): # pylint: disable=too-many-locals,too-many-statements
print("Retrieving archive {}".format(filename))
# This is the artifacts.tgz as referenced in evergreen.yml.
try:
urllib.urlretrieve(artifact["url"], filename)
except urllib.ContentTooShortError:
urllib.request.urlretrieve(artifact["url"], filename)
except urllib.error.ContentTooShortError:
print("The artifact {} could not be completely downloaded. Default"
" compile bypass to false.".format(filename))
return
@ -287,8 +287,8 @@ def main(): # pylint: disable=too-many-locals,too-many-statements
print("Retrieving mongo source {}".format(filename))
# This is the distsrc.[tgz|zip] as referenced in evergreen.yml.
try:
urllib.urlretrieve(artifact["url"], filename)
except urllib.ContentTooShortError:
urllib.request.urlretrieve(artifact["url"], filename)
except urllib.error.ContentTooShortError:
print("The artifact {} could not be completely downloaded. Default"
" compile bypass to false.".format(filename))
return
@ -310,9 +310,9 @@ def main(): # pylint: disable=too-many-locals,too-many-statements
# SERVER-21492 related issue where without running scons the jstests/libs/key1
# and key2 files are not chmod to 0600. Need to change permissions here since we
# bypass SCons.
os.chmod("jstests/libs/key1", 0600)
os.chmod("jstests/libs/key2", 0600)
os.chmod("jstests/libs/keyForRollover", 0600)
os.chmod("jstests/libs/key1", 0o600)
os.chmod("jstests/libs/key2", 0o600)
os.chmod("jstests/libs/keyForRollover", 0o600)
# This is the artifacts.json file.
write_out_artifacts(args.jsonArtifact, artifacts)

6
buildscripts/ciconfig/evergreen.py
View File

@ -61,7 +61,7 @@ class EvergreenProjectConfig(object): # pylint: disable=too-many-instance-attri
@property
def task_names(self):
"""Get the list of task names."""
return self._tasks_by_name.keys()
return list(self._tasks_by_name.keys())
def get_task(self, task_name):
"""Return the task with the given name as a Task instance."""
@ -70,7 +70,7 @@ class EvergreenProjectConfig(object): # pylint: disable=too-many-instance-attri
@property
def task_group_names(self):
"""Get the list of task_group names."""
return self._task_groups_by_name.keys()
return list(self._task_groups_by_name.keys())
def get_task_group(self, task_group_name):
"""Return the task_group with the given name as a Task instance."""
@ -92,7 +92,7 @@ class EvergreenProjectConfig(object): # pylint: disable=too-many-instance-attri
@property
def variant_names(self):
"""Get the list of build variant names."""
return self._variants_by_name.keys()
return list(self._variants_by_name.keys())
def get_variant(self, variant_name):
"""Return the variant with the given name as a Variant instance."""

19
buildscripts/ciconfig/tags.py
View File

@ -1,7 +1,7 @@
"""Module to access and modify tag configuration files used by resmoke."""
from __future__ import absolute_import
from __future__ import print_function
import collections
import copy
@ -9,10 +9,12 @@ import textwrap
import yaml
from functools import cmp_to_key
# Setup to preserve order in yaml.dump, see https://stackoverflow.com/a/8661021
def _represent_dict_order(self, data):
return self.represent_mapping("tag:yaml.org,2002:map", data.items())
return self.represent_mapping("tag:yaml.org,2002:map", list(data.items()))
yaml.add_representer(collections.OrderedDict, _represent_dict_order)
@ -57,11 +59,11 @@ class TagsConfig(object):
def get_test_kinds(self):
"""List the test kinds."""
return self._conf.keys()
return list(self._conf.keys())
def get_test_patterns(self, test_kind):
"""List the test patterns under 'test_kind'."""
return getdefault(self._conf, test_kind, {}).keys()
return list(getdefault(self._conf, test_kind, {}).keys())
def get_tags(self, test_kind, test_pattern):
"""List the tags under 'test_kind' and 'test_pattern'."""
@ -74,7 +76,7 @@ class TagsConfig(object):
tags = setdefault(patterns, test_pattern, [])
if tag not in tags:
tags.append(tag)
tags.sort(cmp=self._cmp_func)
tags.sort(key=cmp_to_key(self._cmp_func) if self._cmp_func else None)
return True
return False
@ -110,8 +112,9 @@ class TagsConfig(object):
"""
with open(filename, "w") as fstream:
if preamble:
print(textwrap.fill(preamble, width=100, initial_indent="# ",
subsequent_indent="# "), file=fstream)
print(
textwrap.fill(preamble, width=100, initial_indent="# ", subsequent_indent="# "),
file=fstream)
# We use yaml.safe_dump() in order avoid having strings being written to the file as
# "!!python/unicode ..." and instead have them written as plain 'str' instances.

22
buildscripts/clang_format.py
View File

@ -7,7 +7,7 @@
4. Has support for checking which files are to be checked.
5. Supports validating and updating a set of files to the right coding style.
"""
from __future__ import print_function, absolute_import
import difflib
import glob
@ -20,7 +20,7 @@ import sys
import tarfile
import tempfile
import threading
import urllib2
import urllib.request, urllib.error, urllib.parse
from distutils import spawn # pylint: disable=no-name-in-module
from optparse import OptionParser
from multiprocessing import cpu_count
@ -52,14 +52,14 @@ CLANG_FORMAT_HTTP_LINUX_CACHE = "https://s3.amazonaws.com/boxes.10gen.com/build/
CLANG_FORMAT_HTTP_DARWIN_CACHE = "https://s3.amazonaws.com/boxes.10gen.com/build/clang%2Bllvm-3.8.0-x86_64-apple-darwin.tar.xz"
# Path in the tarball to the clang-format binary
CLANG_FORMAT_SOURCE_TAR_BASE = string.Template(
"clang+llvm-$version-$tar_path/bin/" + CLANG_FORMAT_PROGNAME)
CLANG_FORMAT_SOURCE_TAR_BASE = string.Template("clang+llvm-$version-$tar_path/bin/" +
CLANG_FORMAT_PROGNAME)
##############################################################################
def callo(args):
"""Call a program, and capture its output."""
return subprocess.check_output(args)
return subprocess.check_output(args).decode('utf-8')
def get_tar_path(version, tar_path):
@ -96,11 +96,11 @@ def get_clang_format_from_cache_and_extract(url, tarball_ext):
num_tries = 5
for attempt in range(num_tries):
try:
resp = urllib2.urlopen(url)
resp = urllib.request.urlopen(url)
with open(temp_tar_file, 'wb') as fh:
fh.write(resp.read())
break
except urllib2.URLError:
except urllib.error.URLError:
if attempt == num_tries - 1:
raise
continue
@ -229,7 +229,7 @@ class ClangFormat(object):
def _lint(self, file_name, print_diff):
"""Check the specified file has the correct format."""
with open(file_name, 'rb') as original_text:
original_file = original_text.read()
original_file = original_text.read().decode('utf-8')
# Get formatted file as clang-format would format the file
formatted_file = callo([self.path, "--style=file", file_name])
@ -381,9 +381,9 @@ def reformat_branch( # pylint: disable=too-many-branches,too-many-locals,too-ma
"Commit After Reformat '%s' is not a valid commit in this repo" % commit_after_reformat)
if not repo.is_ancestor(commit_prior_to_reformat, commit_after_reformat):
raise ValueError(("Commit Prior to Reformat '%s' is not a valid ancestor of Commit After" +
" Reformat '%s' in this repo") % (commit_prior_to_reformat,
commit_after_reformat))
raise ValueError(
("Commit Prior to Reformat '%s' is not a valid ancestor of Commit After" +
" Reformat '%s' in this repo") % (commit_prior_to_reformat, commit_after_reformat))
# Validate the user is on a local branch that has the right merge base
if repo.is_detached():

2
buildscripts/client/evergreen.py
View File

@ -4,7 +4,7 @@ import os
import time
try:
from urlparse import urlparse
from urllib.parse import urlparse
except ImportError:
from urllib.parse import urlparse # type: ignore

25
buildscripts/collect_resource_info.py
View File

@ -1,8 +1,8 @@
#!/usr/bin/env python
"""Collect system resource information on processes running in Evergreen on a given interval."""
from __future__ import absolute_import
from __future__ import print_function
from datetime import datetime
import optparse
@ -24,13 +24,15 @@ def main():
"""Main."""
usage = "usage: %prog [options]"
parser = optparse.OptionParser(description=__doc__, usage=usage)
parser.add_option("-i", "--interval", dest="interval", default=5, type="int",
help="Collect system resource information every <interval> seconds. "
"Default is every 5 seconds.")
parser.add_option("-o", "--output-file", dest="outfile", default="-",
help="If '-', then the file is written to stdout."
" Any other value is treated as the output file name. By default,"
" output is written to stdout.")
parser.add_option(
"-i", "--interval", dest="interval", default=5, type="int",
help="Collect system resource information every <interval> seconds. "
"Default is every 5 seconds.")
parser.add_option(
"-o", "--output-file", dest="outfile", default="-",
help="If '-', then the file is written to stdout."
" Any other value is treated as the output file name. By default,"
" output is written to stdout.")
(options, _) = parser.parse_args()
@ -40,8 +42,9 @@ def main():
# Requires the Evergreen agent to be running on port 2285.
response = requests.get("http://localhost:2285/status")
if response.status_code != requests.codes.ok:
print("Received a {} HTTP response: {}".format(response.status_code,
response.text), file=sys.stderr)
print(
"Received a {} HTTP response: {}".format(response.status_code,
response.text), file=sys.stderr)
time.sleep(options.interval)
continue

15
buildscripts/combine_reports.py
View File

@ -1,8 +1,8 @@
#!/usr/bin/env python
"""Combine JSON report files used in Evergreen."""
from __future__ import absolute_import
from __future__ import print_function
import errno
import json
@ -45,7 +45,7 @@ def check_error(input_count, output_count):
if (not input_count) and (not output_count):
raise ValueError("None of the input file(s) or output file exists")
elif input_count and output_count:
if input_count and output_count:
raise ValueError("Both input file and output files exist")
@ -53,10 +53,11 @@ def main():
"""Execute Main program."""
usage = "usage: %prog [options] report1.json report2.json ..."
parser = OptionParser(description=__doc__, usage=usage)
parser.add_option("-o", "--output-file", dest="outfile", default="-",
help=("If '-', then the combined report file is written to stdout."
" Any other value is treated as the output file name. By default,"
" output is written to stdout."))
parser.add_option(
"-o", "--output-file", dest="outfile", default="-",
help=("If '-', then the combined report file is written to stdout."
" Any other value is treated as the output file name. By default,"
" output is written to stdout."))
parser.add_option("-x", "--no-report-exit", dest="report_exit", default=True,
action="store_false",
help="Do not exit with a non-zero code if any test in the report fails.")

52
buildscripts/cpplint.py vendored
View File

@ -456,7 +456,7 @@ _ALT_TOKEN_REPLACEMENT = {
# False positives include C-style multi-line comments and multi-line strings
# but those have always been troublesome for cpplint.
_ALT_TOKEN_REPLACEMENT_PATTERN = re.compile(
r'[ =()](' + ('|'.join(_ALT_TOKEN_REPLACEMENT.keys())) + r')(?=[ (]|$)')
r'[ =()](' + ('|'.join(list(_ALT_TOKEN_REPLACEMENT.keys()))) + r')(?=[ (]|$)')
# These constants define types of headers for use with
@ -836,7 +836,7 @@ class _CppLintState(object):
def PrintErrorCounts(self):
"""Print a summary of errors by category, and the total."""
for category, count in self.errors_by_category.iteritems():
for category, count in self.errors_by_category.items():
sys.stderr.write('Category \'%s\' errors found: %d\n' %
(category, count))
sys.stderr.write('Total errors found: %d\n' % self.error_count)
@ -1389,7 +1389,7 @@ def FindEndOfExpressionInLine(line, startpos, stack):
On finding an unclosed expression: (-1, None)
Otherwise: (-1, new stack at end of this line)
"""
for i in xrange(startpos, len(line)):
for i in range(startpos, len(line)):
char = line[i]
if char in '([{':
# Found start of parenthesized expression, push to expression stack
@ -1682,7 +1682,7 @@ def CheckForCopyright(filename, lines, error):
# We'll say it should occur by line 10. Don't forget there's a
# dummy line at the front.
for line in xrange(1, min(len(lines), 11)):
for line in range(1, min(len(lines), 11)):
if re.search(r'Copyright', lines[line], re.I):
CheckForServerSidePublicLicense(line, filename, lines, error)
break
@ -1741,10 +1741,10 @@ def CheckForServerSidePublicLicense(copyright_offset, filename, lines, error):
# We expect the first line of the license header to follow shortly after the
# "Copyright" message.
for line in xrange(copyright_offset, min(len(lines), copyright_offset + 3)):
for line in range(copyright_offset, min(len(lines), copyright_offset + 3)):
if re.search(r'This program is free software', lines[line]):
license_header_start_line = line
for i in xrange(len(license_header)):
for i in range(len(license_header)):
line = i + license_header_start_line
if line >= len(lines) or lines[line] != license_header[i]:
error(filename, 0, 'legal/license', 5,
@ -1904,7 +1904,7 @@ def CheckForBadCharacters(filename, lines, error):
error: The function to call with any errors found.
"""
for linenum, line in enumerate(lines):
if u'\ufffd' in line:
if '\ufffd' in line:
error(filename, linenum, 'readability/utf8', 5,
'Line contains invalid UTF-8 (or Unicode replacement character).')
if '\0' in line:
@ -2950,7 +2950,7 @@ def CheckForFunctionLengths(filename, clean_lines, linenum,
if starting_func:
body_found = False
for start_linenum in xrange(linenum, clean_lines.NumLines()):
for start_linenum in range(linenum, clean_lines.NumLines()):
start_line = lines[start_linenum]
joined_line += ' ' + start_line.lstrip()
if Search(r'(;|})', start_line): # Declarations and trivial functions
@ -3427,7 +3427,7 @@ def CheckBracesSpacing(filename, clean_lines, linenum, error):
trailing_text = ''
if endpos > -1:
trailing_text = endline[endpos:]
for offset in xrange(endlinenum + 1,
for offset in range(endlinenum + 1,
min(endlinenum + 3, clean_lines.NumLines() - 1)):
trailing_text += clean_lines.elided[offset]
if not Match(r'^[\s}]*[{.;,)<>\]:]', trailing_text):
@ -3596,7 +3596,7 @@ def IsRValueType(clean_lines, nesting_state, linenum, column):
# Look for the previous 'for(' in the previous lines.
before_text = match_symbol.group(1)
for i in xrange(start - 1, max(start - 6, 0), -1):
for i in range(start - 1, max(start - 6, 0), -1):
before_text = clean_lines.elided[i] + before_text
if Search(r'for\s*\([^{};]*$', before_text):
# This is the condition inside a for-loop
@ -3723,12 +3723,12 @@ def IsRValueAllowed(clean_lines, linenum):
True if line is within the region where RValue references are allowed.
"""
# Allow region marked by PUSH/POP macros
for i in xrange(linenum, 0, -1):
for i in range(linenum, 0, -1):
line = clean_lines.elided[i]
if Match(r'GOOGLE_ALLOW_RVALUE_REFERENCES_(?:PUSH|POP)', line):
if not line.endswith('PUSH'):
return False
for j in xrange(linenum, clean_lines.NumLines(), 1):
for j in range(linenum, clean_lines.NumLines(), 1):
line = clean_lines.elided[j]
if Match(r'GOOGLE_ALLOW_RVALUE_REFERENCES_(?:PUSH|POP)', line):
return line.endswith('POP')
@ -4208,7 +4208,7 @@ def CheckCheck(filename, clean_lines, linenum, error):
expression = lines[linenum][start_pos + 1:end_pos - 1]
else:
expression = lines[linenum][start_pos + 1:]
for i in xrange(linenum + 1, end_line):
for i in range(linenum + 1, end_line):
expression += lines[i]
expression += last_line[0:end_pos - 1]
@ -4336,7 +4336,7 @@ def GetLineWidth(line):
The width of the line in column positions, accounting for Unicode
combining characters and wide characters.
"""
if isinstance(line, unicode):
if isinstance(line, str):
width = 0
for uc in unicodedata.normalize('NFC', line):
if unicodedata.east_asian_width(uc) in ('W', 'F'):
@ -4689,7 +4689,7 @@ def _GetTextInside(text, start_pattern):
# Give opening punctuations to get the matching close-punctuations.
matching_punctuation = {'(': ')', '{': '}', '[': ']'}
closing_punctuation = set(matching_punctuation.itervalues())
closing_punctuation = set(matching_punctuation.values())
# Find the position to start extracting text.
match = re.search(start_pattern, text, re.M)
@ -5015,7 +5015,7 @@ def IsDerivedFunction(clean_lines, linenum):
virt-specifier.
"""
# Scan back a few lines for start of current function
for i in xrange(linenum, max(-1, linenum - 10), -1):
for i in range(linenum, max(-1, linenum - 10), -1):
match = Match(r'^([^()]*\w+)\(', clean_lines.elided[i])
if match:
# Look for "override" after the matching closing parenthesis
@ -5036,7 +5036,7 @@ def IsInitializerList(clean_lines, linenum):
True if current line appears to be inside constructor initializer
list, False otherwise.
"""
for i in xrange(linenum, 1, -1):
for i in range(linenum, 1, -1):
line = clean_lines.elided[i]
if i == linenum:
remove_function_body = Match(r'^(.*)\{\s*$', line)
@ -5132,7 +5132,7 @@ def CheckForNonConstReference(filename, clean_lines, linenum,
# Found the matching < on an earlier line, collect all
# pieces up to current line.
line = ''
for i in xrange(startline, linenum + 1):
for i in range(startline, linenum + 1):
line += clean_lines.elided[i].strip()
# Check for non-const references in function parameters. A single '&' may
@ -5156,7 +5156,7 @@ def CheckForNonConstReference(filename, clean_lines, linenum,
# appear inside the second set of parentheses on the current line as
# opposed to the first set.
if linenum > 0:
for i in xrange(linenum - 1, max(0, linenum - 10), -1):
for i in range(linenum - 1, max(0, linenum - 10), -1):
previous_line = clean_lines.elided[i]
if not Search(r'[),]\s*$', previous_line):
break
@ -5187,7 +5187,7 @@ def CheckForNonConstReference(filename, clean_lines, linenum,
# Don't see a whitelisted function on this line. Actually we
# didn't see any function name on this line, so this is likely a
# multi-line parameter list. Try a bit harder to catch this case.
for i in xrange(2):
for i in range(2):
if (linenum > i and
Search(whitelisted_functions, clean_lines.elided[linenum - i - 1])):
return
@ -5349,7 +5349,7 @@ def CheckCStyleCast(filename, clean_lines, linenum, cast_type, pattern, error):
# Try expanding current context to see if we one level of
# parentheses inside a macro.
if linenum > 0:
for i in xrange(linenum - 1, max(0, linenum - 5), -1):
for i in range(linenum - 1, max(0, linenum - 5), -1):
context = clean_lines.elided[i] + context
if Match(r'.*\b[_A-Z][_A-Z0-9]*\s*\((?:\([^()]*\)|[^()])*$', context):
return False
@ -5606,7 +5606,7 @@ def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error,
required = {} # A map of header name to linenumber and the template entity.
# Example of required: { '<functional>': (1219, 'less<>') }
for linenum in xrange(clean_lines.NumLines()):
for linenum in range(clean_lines.NumLines()):
line = clean_lines.elided[linenum]
if not line or line[0] == '#':
continue
@ -5655,7 +5655,7 @@ def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error,
# include_dict is modified during iteration, so we iterate over a copy of
# the keys.
header_keys = include_dict.keys()
header_keys = list(include_dict.keys())
for header in header_keys:
(same_module, common_path) = FilesBelongToSameModule(abs_filename, header)
fullpath = common_path + header
@ -5750,7 +5750,7 @@ def CheckRedundantVirtual(filename, clean_lines, linenum, error):
end_col = -1
end_line = -1
start_col = len(virtual.group(1))
for start_line in xrange(linenum, min(linenum + 3, clean_lines.NumLines())):
for start_line in range(linenum, min(linenum + 3, clean_lines.NumLines())):
line = clean_lines.elided[start_line][start_col:]
parameter_list = Match(r'^([^(]*)\(', line)
if parameter_list:
@ -5765,7 +5765,7 @@ def CheckRedundantVirtual(filename, clean_lines, linenum, error):
# Look for "override" or "final" after the parameter list
# (possibly on the next few lines).
for i in xrange(end_line, min(end_line + 3, clean_lines.NumLines())):
for i in range(end_line, min(end_line + 3, clean_lines.NumLines())):
line = clean_lines.elided[i][end_col:]
match = Search(r'\b(override|final)\b', line)
if match:
@ -5992,7 +5992,7 @@ def ProcessFileData(filename, file_extension, lines, error,
RemoveMultiLineComments(filename, lines, error)
clean_lines = CleansedLines(lines)
for line in xrange(clean_lines.NumLines()):
for line in range(clean_lines.NumLines()):
ProcessLine(filename, file_extension, clean_lines, line,
include_state, function_state, nesting_state, error,
extra_check_functions)

19
buildscripts/errorcodes.py
View File

@ -5,14 +5,15 @@ Parses .cpp files for assertions and verifies assertion codes are distinct.
Optionally replaces zero codes in source code with new distinct values.
"""
from __future__ import absolute_import
from __future__ import print_function
import bisect
import os.path
import sys
from collections import defaultdict, namedtuple
from optparse import OptionParser
from functools import reduce
# Get relative imports to work when the package is not installed on the PYTHONPATH.
if __name__ == "__main__" and __package__ is None:
@ -44,21 +45,21 @@ list_files = False # pylint: disable=invalid-name
def parse_source_files(callback):
"""Walk MongoDB sourcefiles and invoke a callback for each AssertLocation found."""
quick = ["assert", "Exception", "ErrorCodes::Error"]
quick = [b"assert", b"Exception", b"ErrorCodes::Error"]
patterns = [
re.compile(r"(?:u|m(?:sg)?)asser(?:t|ted)(?:NoTrace)?\s*\(\s*(\d+)", re.MULTILINE),
re.compile(r"(?:DB|Assertion)Exception\s*[({]\s*(\d+)", re.MULTILINE),
re.compile(r"fassert(?:Failed)?(?:WithStatus)?(?:NoTrace)?(?:StatusOK)?\s*\(\s*(\d+)",
re.compile(b"(?:u|m(?:sg)?)asser(?:t|ted)(?:NoTrace)?\s*\(\s*(\d+)", re.MULTILINE),
re.compile(b"(?:DB|Assertion)Exception\s*[({]\s*(\d+)", re.MULTILINE),
re.compile(b"fassert(?:Failed)?(?:WithStatus)?(?:NoTrace)?(?:StatusOK)?\s*\(\s*(\d+)",
re.MULTILINE),
re.compile(r"ErrorCodes::Error\s*[({]\s*(\d+)", re.MULTILINE)
re.compile(b"ErrorCodes::Error\s*[({]\s*(\d+)", re.MULTILINE)
]
for source_file in utils.get_all_source_files(prefix='src/mongo/'):
if list_files:
print('scanning file: ' + source_file)
with open(source_file) as fh:
with open(source_file, 'rb') as fh:
text = fh.read()
if not any([zz in text for zz in quick]):
@ -168,7 +169,7 @@ def read_error_codes():
print("EXCESSIVE SKIPPING OF ERROR CODES:")
print(" %s:%d:%d:%s" % (loc.sourceFile, line, col, loc.lines))
for code, locations in dups.items():
for code, locations in list(dups.items()):
print("DUPLICATE IDS: %s" % code)
for loc in locations:
line, col = get_line_and_column_for_position(loc)

8
buildscripts/eslint.py
View File

@ -10,7 +10,7 @@ There is also a -d mode that assumes you only want to run one copy of ESLint per
parameter supplied. This lets ESLint search for candidate files to lint.
"""
from __future__ import print_function
import os
import shutil
@ -20,7 +20,7 @@ import sys
import tarfile
import tempfile
import threading
import urllib
import urllib.request, urllib.parse, urllib.error
from distutils import spawn # pylint: disable=no-name-in-module
from optparse import OptionParser
@ -57,7 +57,7 @@ ESLINT_SOURCE_TAR_BASE = string.Template(ESLINT_PROGNAME + "-$platform-$arch")
def callo(args):
"""Call a program, and capture its output."""
return subprocess.check_output(args)
return subprocess.check_output(args).decode('utf-8')
def extract_eslint(tar_path, target_file):
@ -84,7 +84,7 @@ def get_eslint_from_cache(dest_file, platform, arch):
# Download the file
print("Downloading ESLint %s from %s, saving to %s" % (ESLINT_VERSION, url, temp_tar_file))
urllib.urlretrieve(url, temp_tar_file)
urllib.request.urlretrieve(url, temp_tar_file)
eslint_distfile = ESLINT_SOURCE_TAR_BASE.substitute(platform=platform, arch=arch)
extract_eslint(temp_tar_file, eslint_distfile)

6
buildscripts/evergreen_gen_fuzzer_tests.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python
"""Generate fuzzer tests to run in evergreen in parallel."""
from __future__ import absolute_import
import argparse
import math
@ -15,8 +15,8 @@ from shrub.task import TaskDependency
from shrub.variant import DisplayTaskDefinition
from shrub.variant import TaskSpec
import util.read_config as read_config
import util.taskname as taskname
import buildscripts.util.read_config as read_config
import buildscripts.util.taskname as taskname
CONFIG_DIRECTORY = "generated_resmoke_config"

11
buildscripts/evergreen_generate_resmoke_tasks.py
View File

@ -6,7 +6,7 @@ Analyze the evergreen history for tests run under the given task and create new
to attempt to keep the task runtime under a specified amount.
"""
from __future__ import absolute_import
import argparse
import datetime
@ -463,7 +463,7 @@ class TestStats(object):
def get_tests_runtimes(self):
"""Return the list of (test_file, runtime_in_secs) tuples ordered by decreasing runtime."""
tests = []
for test_file, runtime_info in self._runtime_by_test.items():
for test_file, runtime_info in list(self._runtime_by_test.items()):
duration = runtime_info["duration"]
test_name = testname.get_short_name_from_test_file(test_file)
hook_runtime_info = self._hook_runtime_by_test[test_name]
@ -536,9 +536,10 @@ class Main(object):
help="Target execution time (in minutes).")
parser.add_argument("--max-sub-suites", dest="max_sub_suites", type=int,
help="Max number of suites to divide into.")
parser.add_argument("--fallback-num-sub-suites", dest="fallback_num_sub_suites", type=int,
help="The number of suites to divide into if the Evergreen test "
"statistics are not available.")
parser.add_argument(
"--fallback-num-sub-suites", dest="fallback_num_sub_suites", type=int,
help="The number of suites to divide into if the Evergreen test "
"statistics are not available.")
parser.add_argument("--project", dest="project", help="The Evergreen project to analyse.")
parser.add_argument("--resmoke-args", dest="resmoke_args",
help="Arguments to pass to resmoke calls.")

24
buildscripts/evergreen_resmoke_job_count.py
View File

@ -1,9 +1,6 @@
#!/usr/bin/env python
"""Determine the number of resmoke jobs to run."""
from __future__ import division
from __future__ import print_function
import argparse
import platform
import re
@ -76,15 +73,18 @@ def main():
parser.add_argument("--taskName", dest="task", required=True, help="Task being executed.")
parser.add_argument("--buildVariant", dest="variant", required=True,
help="Build variant task is being executed on.")
parser.add_argument("--jobFactor", dest="jobs_factor", type=float, default=1.0,
help=("Job factor to use as a mulitplier with the number of CPUs. Defaults"
" to %(default)s."))
parser.add_argument("--jobsMax", dest="jobs_max", type=int, default=0,
help=("Maximum number of jobs to use. Specify 0 to indicate the number of"
" jobs is determined by --jobFactor and the number of CPUs. Defaults"
" to %(default)s."))
parser.add_argument("--outFile", dest="outfile", help=("File to write configuration to. If"
" unspecified no file is generated."))
parser.add_argument(
"--jobFactor", dest="jobs_factor", type=float, default=1.0,
help=("Job factor to use as a mulitplier with the number of CPUs. Defaults"
" to %(default)s."))
parser.add_argument(
"--jobsMax", dest="jobs_max", type=int, default=0,
help=("Maximum number of jobs to use. Specify 0 to indicate the number of"
" jobs is determined by --jobFactor and the number of CPUs. Defaults"
" to %(default)s."))
parser.add_argument(
"--outFile", dest="outfile", help=("File to write configuration to. If"
" unspecified no file is generated."))
options = parser.parse_args()

2
buildscripts/evergreen_run_tests.py
View File

@ -1,8 +1,6 @@
#!/usr/bin/env python
"""Command line utility for executing MongoDB tests in Evergreen."""
from __future__ import absolute_import
import collections
import os.path
import sys

2
buildscripts/evergreen_task_timeout.py
View File

@ -1,8 +1,6 @@
#!/usr/bin/env python
"""Determine the timeout value a task should use in evergreen."""
from __future__ import absolute_import
import argparse
import sys

55
buildscripts/fetch_test_lifecycle.py
View File

@ -7,9 +7,6 @@ Usage:
python buildscsripts/fetch_test_lifecycle.py evergreen-project revision
"""
from __future__ import absolute_import
from __future__ import print_function
import logging
import optparse
import os
@ -138,38 +135,44 @@ def main():
parser = optparse.OptionParser(
description=textwrap.dedent(main.__doc__), usage="Usage: %prog [options] evergreen-project")
parser.add_option("--revision", dest="revision", metavar="<revision>", default="HEAD",
help=("The project revision for which to retrieve the test lifecycle tags"
" file."))
parser.add_option(
"--revision", dest="revision", metavar="<revision>", default="HEAD",
help=("The project revision for which to retrieve the test lifecycle tags"
" file."))
parser.add_option("--metadataRepo", dest="metadata_repo_url", metavar="<metadata-repo-url>",
default="git@github.com:mongodb/mongo-test-metadata.git",
help=("The URL to the metadata repository that contains the test lifecycle"
" tags file."))
parser.add_option(
"--metadataRepo", dest="metadata_repo_url", metavar="<metadata-repo-url>",
default="git@github.com:mongodb/mongo-test-metadata.git",
help=("The URL to the metadata repository that contains the test lifecycle"
" tags file."))
parser.add_option("--lifecycleFile", dest="lifecycle_file", metavar="<lifecycle-file>",
default="etc/test_lifecycle.yml",
help=("The path to the test lifecycle tags file, relative to the root of the"
" metadata repository. Defaults to '%default'."))
parser.add_option(
"--lifecycleFile", dest="lifecycle_file", metavar="<lifecycle-file>",
default="etc/test_lifecycle.yml",
help=("The path to the test lifecycle tags file, relative to the root of the"
" metadata repository. Defaults to '%default'."))
parser.add_option("--referencesFile", dest="references_file", metavar="<references-file>",
default="references.yml",
help=("The path to the metadata references file, relative to the root of the"
" metadata repository. Defaults to '%default'."))
parser.add_option(
"--referencesFile", dest="references_file", metavar="<references-file>",
default="references.yml",
help=("The path to the metadata references file, relative to the root of the"
" metadata repository. Defaults to '%default'."))
parser.add_option("--destinationFile", dest="destination_file", metavar="<destination-file>",
default="etc/test_lifecycle.yml",
help=("The path where the lifecycle file should be available when this script"
" completes successfully. This path is absolute or relative to the"
" current working directory. Defaults to '%default'."))
parser.add_option(
"--destinationFile", dest="destination_file", metavar="<destination-file>",
default="etc/test_lifecycle.yml",
help=("The path where the lifecycle file should be available when this script"
" completes successfully. This path is absolute or relative to the"
" current working directory. Defaults to '%default'."))
parser.add_option("--logLevel", dest="log_level", metavar="<log-level>",
choices=["DEBUG", "INFO", "WARNING", "ERROR"], default="INFO",
help="The log level: DEBUG, INFO, WARNING or ERROR. Defaults to '%default'.")
parser.add_option("--logFile", dest="log_file", metavar="<log-file>", default=None,
help=("The destination file for the logs. If not set the script will log to"
" the standard output"))
parser.add_option(
"--logFile", dest="log_file", metavar="<log-file>", default=None,
help=("The destination file for the logs. If not set the script will log to"
" the standard output"))
options, args = parser.parse_args()

18
buildscripts/gdb/mongo.py
View File

@ -1,5 +1,4 @@
"""GDB commands for MongoDB."""
from __future__ import print_function
import os
import re
@ -22,12 +21,6 @@ except Exception as e:
print("Failed to load the libstdc++ pretty printers: " + str(e))
# pylint: enable=invalid-name,wildcard-import
if sys.version_info[0] >= 3:
# GDB only permits converting a gdb.Value instance to its numerical address when using the
# long() constructor in Python 2 and not when using the int() constructor. We define the
# 'long' class as an alias for the 'int' class in Python 3 for compatibility.
long = int # pylint: disable=redefined-builtin,invalid-name
def get_process_name():
"""Return the main binary we are attached to."""
@ -104,7 +97,7 @@ def get_decorations(obj):
decorable_t = decorable.type.template_argument(0)
decinfo_t = gdb.lookup_type('mongo::DecorationRegistry<{}>::DecorationInfo'.format(
str(decorable_t).replace("class", "").strip()))
count = long((long(finish) - long(start)) / decinfo_t.sizeof)
count = int((int(finish) - int(start)) / decinfo_t.sizeof)
for i in range(count):
descriptor = start[i]
@ -480,7 +473,7 @@ class MongoDBUniqueStack(gdb.Command):
"""Return the first tid."""
return stack['threads'][0]['gdb_thread_num']
for stack in sorted(stacks.values(), key=first_tid, reverse=True):
for stack in sorted(list(stacks.values()), key=first_tid, reverse=True):
for i, thread in enumerate(stack['threads']):
prefix = '' if i == 0 else 'Duplicate '
print(prefix + thread['header'])
@ -527,9 +520,10 @@ class MongoDBJavaScriptStack(gdb.Command):
if gdb.parse_and_eval(
'mongo::mozjs::kCurrentScope && mongo::mozjs::kCurrentScope->_inOp'):
gdb.execute('thread', from_tty=False, to_string=False)
gdb.execute('printf "%s\\n", ' +
'mongo::mozjs::kCurrentScope->buildStackString().c_str()',
from_tty=False, to_string=False)
gdb.execute(
'printf "%s\\n", ' +
'mongo::mozjs::kCurrentScope->buildStackString().c_str()', from_tty=False,
to_string=False)
except gdb.error as err:
print("Ignoring GDB error '%s' in javascript_stack" % str(err))
continue

18
buildscripts/gdb/mongo_lock.py
View File

@ -1,19 +1,11 @@
"""Mongo lock module."""
from __future__ import print_function
import re
import sys
import gdb
import gdb.printing
if sys.version_info[0] >= 3:
# GDB only permits converting a gdb.Value instance to its numerical address when using the
# long() constructor in Python 2 and not when using the int() constructor. We define the
# 'long' class as an alias for the 'int' class in Python 3 for compatibility.
long = int # pylint: disable=redefined-builtin,invalid-name
class NonExecutingThread(object):
"""NonExecutingThread class.
@ -240,7 +232,7 @@ class Graph(object):
def find_thread(thread_dict, search_thread_id):
"""Find thread."""
for (_, thread) in thread_dict.items():
for (_, thread) in list(thread_dict.items()):
if thread.thread_id == search_thread_id:
return thread
return None
@ -307,8 +299,8 @@ def find_mutex_holder(graph, thread_dict, show):
print("Mutex at {} held by {} waited on by {}".format(mutex_value, mutex_holder,
mutex_waiter))
if graph:
graph.add_edge(mutex_waiter, Lock(long(mutex_value), "Mutex"))
graph.add_edge(Lock(long(mutex_value), "Mutex"), mutex_holder)
graph.add_edge(mutex_waiter, Lock(int(mutex_value), "Mutex"))
graph.add_edge(Lock(int(mutex_value), "Mutex"), mutex_holder)
def find_lock_manager_holders(graph, thread_dict, show): # pylint: disable=too-many-locals
@ -355,8 +347,8 @@ def find_lock_manager_holders(graph, thread_dict, show): # pylint: disable=too-
print("MongoDB Lock at {} held by {} ({}) waited on by {}".format(
lock_head, lock_holder, lock_request["mode"], lock_waiter))
if graph:
graph.add_edge(lock_waiter, Lock(long(lock_head), lock_request["mode"]))
graph.add_edge(Lock(long(lock_head), lock_request["mode"]), lock_holder)
graph.add_edge(lock_waiter, Lock(int(lock_head), lock_request["mode"]))
graph.add_edge(Lock(int(lock_head), lock_request["mode"]), lock_holder)
lock_request_ptr = lock_request["next"]

23
buildscripts/gdb/mongo_printers.py
View File

@ -1,5 +1,4 @@
"""GDB Pretty-printers for MongoDB."""
from __future__ import print_function
import re
import struct
@ -18,12 +17,6 @@ except ImportError as err:
print("Check with the pip command if pymongo 3.x is installed.")
bson = None
if sys.version_info[0] >= 3:
# GDB only permits converting a gdb.Value instance to its numerical address when using the
# long() constructor in Python 2 and not when using the int() constructor. We define the
# 'long' class as an alias for the 'int' class in Python 3 for compatibility.
long = int # pylint: disable=redefined-builtin,invalid-name
def get_unique_ptr(obj):
"""Read the value of a libstdc++ std::unique_ptr."""
@ -132,7 +125,7 @@ class BSONObjPrinter(object):
options = CodecOptions(document_class=collections.OrderedDict)
bsondoc = buf.decode(codec_options=options)
for key, val in bsondoc.items():
for key, val in list(bsondoc.items()):
yield 'key', key
yield 'value', bson.json_util.dumps(val)
@ -187,7 +180,7 @@ class DecorablePrinter(object):
decorable_t = val.type.template_argument(0)
decinfo_t = gdb.lookup_type('mongo::DecorationRegistry<{}>::DecorationInfo'.format(
str(decorable_t).replace("class", "").strip()))
self.count = long((long(finish) - long(self.start)) / decinfo_t.sizeof)
self.count = int((int(finish) - int(self.start)) / decinfo_t.sizeof)
@staticmethod
def display_hint():
@ -388,9 +381,8 @@ class AbslHashSetPrinterBase(object):
def to_string(self):
"""Return absl::[node/flat]_hash_set for printing."""
return "absl::%s_hash_set<%s> with %s elems " % (self.to_str,
self.val.type.template_argument(0),
self.val["size_"])
return "absl::%s_hash_set<%s> with %s elems " % (
self.to_str, self.val.type.template_argument(0), self.val["size_"])
class AbslNodeHashSetPrinter(AbslHashSetPrinterBase):
@ -438,10 +430,9 @@ class AbslHashMapPrinterBase(object):
def to_string(self):
"""Return absl::[node/flat]_hash_map for printing."""
return "absl::%s_hash_map<%s, %s> with %s elems " % (self.to_str,
self.val.type.template_argument(0),
self.val.type.template_argument(1),
self.val["size_"])
return "absl::%s_hash_map<%s, %s> with %s elems " % (
self.to_str, self.val.type.template_argument(0), self.val.type.template_argument(1),
self.val["size_"])
class AbslNodeHashMapPrinter(AbslHashMapPrinterBase):

12
buildscripts/generate-pip-constraints.sh
View File

@ -64,10 +64,8 @@ if [[ -d $WORKING_DIR ]]; then
fi
ABSOLUTE_WORKING_DIR="$(mkdir -p "${WORKING_DIR}" && cd "${WORKING_DIR}" && pwd)"
PIP2_DIR="${ABSOLUTE_WORKING_DIR}/python2"
PIP3_DIR="${ABSOLUTE_WORKING_DIR}/python3"
generateConstraints python2 "${PIP2_DIR}"
generateConstraints python3 "${PIP3_DIR}"
if [[ -z $CON_FILE ]]; then
@ -83,15 +81,7 @@ fi
printf '\n'
printf '\n# Common requirements\n'
comm -12 "${PIP2_DIR}/requirements.txt" "${PIP3_DIR}/requirements.txt"
printf '\n# Python2 requirements\n'
comm -23 "${PIP2_DIR}/requirements.txt" "${PIP3_DIR}/requirements.txt" |
sed -e 's/$/; python_version < "3"/'
printf '\n# Python3 requirements\n'
comm -13 "${PIP2_DIR}/requirements.txt" "${PIP3_DIR}/requirements.txt" |
sed -e 's/$/; python_version > "3"/'
cat "${PIP3_DIR}/requirements.txt"
printf '\n'
cat "${SCRIPT_DIR}/../etc/pip/components/platform.req"

2
buildscripts/generate_compile_expansions.py
View File

@ -6,8 +6,6 @@ Invoke by specifying an output file.
$ python generate_compile_expansions.py --out compile_expansions.yml
"""
from __future__ import print_function
import argparse
import json
import os

2
buildscripts/generate_compile_expansions_shared_cache.py
View File

@ -6,8 +6,6 @@ Invoke by specifying an output file.
$ python generate_compile_expansions.py --out compile_expansions.yml
"""
from __future__ import print_function
import argparse
import json
import os

34
buildscripts/git.py
View File

@ -1,30 +1,9 @@
"""Module to run git commands on a repository."""
from __future__ import absolute_import
import logging
import os
import sys
# The subprocess32 module resolves the thread-safety issues of the subprocess module in Python 2.x
# when the _posixsubprocess C extension module is also available. Additionally, the _posixsubprocess
# C extension module avoids triggering invalid free() calls on Python's internal data structure for
# thread-local storage by skipping the PyOS_AfterFork() call when the 'preexec_fn' parameter isn't
# specified to subprocess.Popen(). See SERVER-22219 for more details.
#
# The subprocess32 module is untested on Windows and thus isn't recommended for use, even when it's
# installed. See https://github.com/google/python-subprocess32/blob/3.2.7/README.md#usage.
if os.name == "posix" and sys.version_info[0] == 2:
try:
import subprocess32 as subprocess
except ImportError:
import warnings
warnings.warn(("Falling back to using the subprocess module because subprocess32 isn't"
" available. When using the subprocess module, a child process may trigger"
" an invalid free(). See SERVER-22219 for more details."), RuntimeWarning)
import subprocess # type: ignore
else:
import subprocess
import subprocess
LOGGER = logging.getLogger(__name__)
@ -203,7 +182,7 @@ class Repository(object): # pylint: disable=too-many-public-methods
params.extend(["rev-parse", "--show-toplevel"])
result = Repository._run_process("rev-parse", params)
result.check_returncode()
return result.stdout.rstrip()
return result.stdout.decode('utf-8').rstrip()
@staticmethod
def current_repository():
@ -214,7 +193,7 @@ class Repository(object): # pylint: disable=too-many-public-methods
"""Call git for this repository, and return the captured output."""
result = self._run_cmd(cmd, args)
result.check_returncode()
return result.stdout
return result.stdout.decode('utf-8')
def _callgit(self, cmd, args, raise_exception=False):
"""
@ -291,6 +270,7 @@ class GitCommandResult(object):
def check_returncode(self):
"""Raise GitException if the exit code is non-zero."""
if self.returncode:
raise GitException("Command '{0}' failed with code '{1}'".format(
" ".join(self.process_args), self.returncode), self.returncode, self.cmd,
self.process_args, self.stdout, self.stderr)
raise GitException(
"Command '{0}' failed with code '{1}'".format(" ".join(self.process_args),
self.returncode), self.returncode,
self.cmd, self.process_args, self.stdout, self.stderr)

65
buildscripts/hang_analyzer.py
View File

@ -11,7 +11,7 @@ A prototype hang analyzer for Evergreen integration to help investigate test tim
Supports Linux, MacOS X, Solaris, and Windows.
"""
import StringIO
import io
import csv
import glob
import itertools
@ -60,7 +60,7 @@ def callo(args, logger):
"""Call subprocess on args string."""
logger.info("%s", str(args))
return subprocess.check_output(args)
return subprocess.check_output(args).decode('utf-8')
def find_program(prog, paths):
@ -186,7 +186,7 @@ class WindowsProcessList(object):
ret = callo([ps, "/FO", "CSV"], logger)
buff = StringIO.StringIO(ret)
buff = io.StringIO(ret)
csv_reader = csv.reader(buff)
return [[int(row[1]), row[0]] for row in csv_reader if row[1] != "PID"]
@ -201,7 +201,7 @@ class LLDBDumper(object):
"""Find the installed debugger."""
return find_program(debugger, ['/usr/bin'])
def dump_info( # pylint: disable=too-many-arguments
def dump_info( # pylint: disable=too-many-arguments,too-many-locals
self, root_logger, logger, pid, process_name, take_dump):
"""Dump info."""
debugger = "lldb"
@ -283,7 +283,7 @@ class DarwinProcessList(object):
ret = callo([ps, "-axco", "pid,comm"], logger)
buff = StringIO.StringIO(ret)
buff = io.StringIO(ret)
csv_reader = csv.reader(buff, delimiter=' ', quoting=csv.QUOTE_NONE, skipinitialspace=True)
return [[int(row[0]), row[1]] for row in csv_reader if row[0] != "PID"]
@ -390,8 +390,8 @@ class GDBDumper(object):
"quit",
]
call([dbg, "--quiet", "--nx"] +
list(itertools.chain.from_iterable([['-ex', b] for b in cmds])), logger)
call([dbg, "--quiet", "--nx"] + list(
itertools.chain.from_iterable([['-ex', b] for b in cmds])), logger)
root_logger.info("Done analyzing %s process with PID %d", process_name, pid)
@ -428,7 +428,7 @@ class LinuxProcessList(object):
ret = callo([ps, "-eo", "pid,args"], logger)
buff = StringIO.StringIO(ret)
buff = io.StringIO(ret)
csv_reader = csv.reader(buff, delimiter=' ', quoting=csv.QUOTE_NONE, skipinitialspace=True)
return [[int(row[0]), os.path.split(row[1])[1]] for row in csv_reader if row[0] != "PID"]
@ -450,7 +450,7 @@ class SolarisProcessList(object):
ret = callo([ps, "-eo", "pid,args"], logger)
buff = StringIO.StringIO(ret)
buff = io.StringIO(ret)
csv_reader = csv.reader(buff, delimiter=' ', quoting=csv.QUOTE_NONE, skipinitialspace=True)
return [[int(row[0]), os.path.split(row[1])[1]] for row in csv_reader if row[0] != "PID"]
@ -562,7 +562,7 @@ def signal_process(logger, pid, signalnum):
logger.info("Waiting for process to report")
time.sleep(5)
except OSError, err:
except OSError as err:
logger.error("Hit OS error trying to signal process: %s", err)
except AttributeError:
@ -619,32 +619,34 @@ def main(): # pylint: disable=too-many-branches,too-many-locals,too-many-statem
process_ids = []
parser = OptionParser(description=__doc__)
parser.add_option('-m', '--process-match', dest='process_match', choices=['contains', 'exact'],
default='contains',
help="Type of match for process names (-p & -g), specify 'contains', or"
" 'exact'. Note that the process name match performs the following"
" conversions: change all process names to lowecase, strip off the file"
" extension, like '.exe' on Windows. Default is 'contains'.")
parser.add_option(
'-m', '--process-match', dest='process_match', choices=['contains', 'exact'],
default='contains', help="Type of match for process names (-p & -g), specify 'contains', or"
" 'exact'. Note that the process name match performs the following"
" conversions: change all process names to lowecase, strip off the file"
" extension, like '.exe' on Windows. Default is 'contains'.")
parser.add_option('-p', '--process-names', dest='process_names',
help='Comma separated list of process names to analyze')
parser.add_option('-g', '--go-process-names', dest='go_process_names',
help='Comma separated list of go process names to analyze')
parser.add_option('-d', '--process-ids', dest='process_ids', default=None,
help='Comma separated list of process ids (PID) to analyze, overrides -p &'
' -g')
parser.add_option(
'-d', '--process-ids', dest='process_ids', default=None,
help='Comma separated list of process ids (PID) to analyze, overrides -p &'
' -g')
parser.add_option('-c', '--dump-core', dest='dump_core', action="store_true", default=False,
help='Dump core file for each analyzed process')
parser.add_option('-s', '--max-core-dumps-size', dest='max_core_dumps_size', default=10000,
help='Maximum total size of core dumps to keep in megabytes')
parser.add_option('-o', '--debugger-output', dest='debugger_output', action="append",
choices=['file', 'stdout'], default=None,
help="If 'stdout', then the debugger's output is written to the Python"
" process's stdout. If 'file', then the debugger's output is written"
" to a file named debugger_<process>_<pid>.log for each process it"
" attaches to. This option can be specified multiple times on the"
" command line to have the debugger's output written to multiple"
" locations. By default, the debugger's output is written only to the"
" Python process's stdout.")
parser.add_option(
'-o', '--debugger-output', dest='debugger_output', action="append",
choices=['file', 'stdout'], default=None,
help="If 'stdout', then the debugger's output is written to the Python"
" process's stdout. If 'file', then the debugger's output is written"
" to a file named debugger_<process>_<pid>.log for each process it"
" attaches to. This option can be specified multiple times on the"
" command line to have the debugger's output written to multiple"
" locations. By default, the debugger's output is written only to the"
" Python process's stdout.")
(options, _) = parser.parse_args()
@ -681,7 +683,7 @@ def main(): # pylint: disable=too-many-branches,too-many-locals,too-many-statem
processes = [(pid, pname) for (pid, pname) in all_processes
if pid in process_ids and pid != os.getpid()]
running_pids = set([pid for (pid, pname) in all_processes])
running_pids = {pid for (pid, pname) in all_processes}
missing_pids = set(process_ids) - running_pids
if missing_pids:
root_logger.warning("The following requested process ids are not running %s",
@ -716,8 +718,9 @@ def main(): # pylint: disable=too-many-branches,too-many-locals,too-many-statem
process_name) in [(p, pn) for (p, pn) in processes if not re.match("^(java|python)", pn)]:
process_logger = get_process_logger(options.debugger_output, pid, process_name)
try:
dbg.dump_info(root_logger, process_logger, pid, process_name, options.dump_core
and check_dump_quota(max_dump_size_bytes, dbg.get_dump_ext()))
dbg.dump_info(
root_logger, process_logger, pid, process_name, options.dump_core
and check_dump_quota(max_dump_size_bytes, dbg.get_dump_ext()))
except Exception as err: # pylint: disable=broad-except
root_logger.info("Error encountered when invoking debugger %s", err)
trapped_exceptions.append(traceback.format_exc())

132
buildscripts/idl/idl/ast.py
View File

@ -34,8 +34,6 @@ This is a lossy translation from the IDL Syntax tree as the IDL AST only contain
the enums and structs that need code generated for them, and just enough information to do that.
"""
from __future__ import absolute_import, print_function, unicode_literals
from typing import List, Union, Any, Optional, Tuple
from . import common
@ -78,10 +76,10 @@ class Global(common.SourceLocation):
"""
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a Global."""
self.cpp_namespace = None # type: unicode
self.cpp_includes = [] # type: List[unicode]
self.cpp_namespace = None # type: str
self.cpp_includes = [] # type: List[str]
self.configs = None # type: ConfigGlobal
super(Global, self).__init__(file_name, line, column)
@ -97,11 +95,11 @@ class Struct(common.SourceLocation):
# pylint: disable=too-many-instance-attributes
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a struct."""
self.name = None # type: unicode
self.cpp_name = None # type: unicode
self.description = None # type: unicode
self.name = None # type: str
self.cpp_name = None # type: str
self.description = None # type: str
self.strict = True # type: bool
self.immutable = False # type: bool
self.inline_chained_structs = False # type: bool
@ -114,9 +112,9 @@ class Expression(common.SourceLocation):
"""Literal of C++ expression representation."""
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct an Expression."""
self.expr = None # type: unicode
self.expr = None # type: str
self.validate_constexpr = True # type: bool
self.export = False # type: bool
@ -134,7 +132,7 @@ class Validator(common.SourceLocation):
# pylint: disable=too-many-instance-attributes
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a Validator."""
# Don't lint gt/lt as bad attribute names.
# pylint: disable=C0103
@ -142,7 +140,7 @@ class Validator(common.SourceLocation):
self.lt = None # type: Expression
self.gte = None # type: Expression
self.lte = None # type: Expression
self.callback = None # type: Optional[unicode]
self.callback = None # type: Optional[str]
super(Validator, self).__init__(file_name, line, column)
@ -159,26 +157,26 @@ class Field(common.SourceLocation):
# pylint: disable=too-many-instance-attributes
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a Field."""
self.name = None # type: unicode
self.description = None # type: unicode
self.cpp_name = None # type: unicode
self.name = None # type: str
self.description = None # type: str
self.cpp_name = None # type: str
self.optional = False # type: bool
self.ignore = False # type: bool
self.chained = False # type: bool
self.comparison_order = -1 # type: int
# Properties specific to fields which are types.
self.cpp_type = None # type: unicode
self.bson_serialization_type = None # type: List[unicode]
self.serializer = None # type: unicode
self.deserializer = None # type: unicode
self.bindata_subtype = None # type: unicode
self.default = None # type: unicode
self.cpp_type = None # type: str
self.bson_serialization_type = None # type: List[str]
self.serializer = None # type: str
self.deserializer = None # type: str
self.bindata_subtype = None # type: str
self.default = None # type: str
# Properties specific to fields which are structs.
self.struct_type = None # type: unicode
self.struct_type = None # type: str
# Properties specific to fields which are arrays.
self.array = False # type: bool
@ -208,9 +206,9 @@ class Command(Struct):
"""
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a command."""
self.namespace = None # type: unicode
self.namespace = None # type: str
self.command_field = None # type: Field
super(Command, self).__init__(file_name, line, column)
@ -223,10 +221,10 @@ class EnumValue(common.SourceLocation):
"""
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct an Enum."""
self.name = None # type: unicode
self.value = None # type: unicode
self.name = None # type: str
self.value = None # type: str
super(EnumValue, self).__init__(file_name, line, column)
@ -239,12 +237,12 @@ class Enum(common.SourceLocation):
"""
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct an Enum."""
self.name = None # type: unicode
self.description = None # type: unicode
self.cpp_namespace = None # type: unicode
self.type = None # type: unicode
self.name = None # type: str
self.description = None # type: str
self.cpp_namespace = None # type: str
self.type = None # type: str
self.values = [] # type: List[EnumValue]
super(Enum, self).__init__(file_name, line, column)
@ -254,11 +252,11 @@ class Condition(common.SourceLocation):
"""Condition(s) for a ServerParameter or ConfigOption."""
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a Condition."""
self.expr = None # type: unicode
self.constexpr = None # type: unicode
self.preprocessor = None # type: unicode
self.expr = None # type: str
self.constexpr = None # type: str
self.preprocessor = None # type: str
super(Condition, self).__init__(file_name, line, column)
@ -267,11 +265,11 @@ class ServerParameterClass(common.SourceLocation):
"""ServerParameter as C++ class specialization."""
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a ServerParameterClass."""
self.name = None # type: unicode
self.data = None # type: unicode
self.name = None # type: str
self.data = None # type: str
self.override_ctor = False # type: bool
self.override_set = False # type: bool
@ -284,23 +282,23 @@ class ServerParameter(common.SourceLocation):
# pylint: disable=too-many-instance-attributes
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a ServerParameter."""
self.name = None # type: unicode
self.set_at = None # type: unicode
self.description = None # type: unicode
self.name = None # type: str
self.set_at = None # type: str
self.description = None # type: str
self.cpp_class = None # type: ServerParameterClass
self.cpp_vartype = None # type: unicode
self.cpp_varname = None # type: unicode
self.cpp_vartype = None # type: str
self.cpp_varname = None # type: str
self.condition = None # type: Condition
self.redact = False # type: bool
self.test_only = False # type: bool
self.deprecated_name = [] # type: List[unicode]
self.deprecated_name = [] # type: List[str]
self.default = None # type: Expression
# Only valid if cpp_varname is specified.
self.validator = None # type: Validator
self.on_update = None # type: unicode
self.on_update = None # type: str
super(ServerParameter, self).__init__(file_name, line, column)
@ -309,12 +307,12 @@ class GlobalInitializer(common.SourceLocation):
"""Initializer details for custom registration/storage."""
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a GlobalInitializer."""
self.name = None # type: unicode
self.register = None # type: unicode
self.store = None # type: unicode
self.name = None # type: str
self.register = None # type: str
self.store = None # type: str
super(GlobalInitializer, self).__init__(file_name, line, column)
@ -323,7 +321,7 @@ class ConfigGlobal(common.SourceLocation):
"""IDL ConfigOption Globals."""
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a ConfigGlobal."""
# Other config globals are consumed in bind phase.
@ -338,28 +336,28 @@ class ConfigOption(common.SourceLocation):
# pylint: disable=too-many-instance-attributes
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a ConfigOption."""
self.name = None # type: unicode
self.short_name = None # type: unicode
self.deprecated_name = [] # type: List[unicode]
self.deprecated_short_name = [] # type: List[unicode]
self.name = None # type: str
self.short_name = None # type: str
self.deprecated_name = [] # type: List[str]
self.deprecated_short_name = [] # type: List[str]
self.description = None # type: Expression
self.section = None # type: unicode
self.arg_vartype = None # type: unicode
self.cpp_vartype = None # type: unicode
self.cpp_varname = None # type: unicode
self.section = None # type: str
self.arg_vartype = None # type: str
self.cpp_vartype = None # type: str
self.cpp_varname = None # type: str
self.condition = None # type: Condition
self.conflicts = [] # type: List[unicode]
self.requires = [] # type: List[unicode]
self.conflicts = [] # type: List[str]
self.requires = [] # type: List[str]
self.hidden = False # type: bool
self.redact = False # type: bool
self.default = None # type: Expression
self.implicit = None # type: Expression
self.source = None # type: unicode
self.canonicalize = None # type: unicode
self.source = None # type: str
self.canonicalize = None # type: str
self.duplicates_append = False # type: bool
self.positional_start = None # type: int

28
buildscripts/idl/idl/binder.py
View File

@ -28,8 +28,6 @@
# pylint: disable=too-many-lines
"""Transform idl.syntax trees from the parser into well-defined idl.ast trees."""
from __future__ import absolute_import, print_function, unicode_literals
import re
from typing import cast, List, Set, Union
@ -43,7 +41,7 @@ from . import syntax
def _validate_single_bson_type(ctxt, idl_type, syntax_type):
# type: (errors.ParserContext, Union[syntax.Type, ast.Field], unicode) -> bool
# type: (errors.ParserContext, Union[syntax.Type, ast.Field], str) -> bool
"""Validate bson serialization type is correct for a type."""
bson_type = idl_type.bson_serialization_type[0]
@ -72,7 +70,7 @@ def _validate_single_bson_type(ctxt, idl_type, syntax_type):
def _validate_bson_types_list(ctxt, idl_type, syntax_type):
# type: (errors.ParserContext, Union[syntax.Type, ast.Field], unicode) -> bool
# type: (errors.ParserContext, Union[syntax.Type, ast.Field], str) -> bool
"""Validate bson serialization type(s) is correct for a type."""
bson_types = idl_type.bson_serialization_type
@ -113,7 +111,7 @@ def _validate_type(ctxt, idl_type):
def _validate_cpp_type(ctxt, idl_type, syntax_type):
# type: (errors.ParserContext, Union[syntax.Type, ast.Field], unicode) -> None
# type: (errors.ParserContext, Union[syntax.Type, ast.Field], str) -> None
"""Validate the cpp_type is correct."""
# Validate cpp_type
@ -158,7 +156,7 @@ def _validate_cpp_type(ctxt, idl_type, syntax_type):
def _validate_chain_type_properties(ctxt, idl_type, syntax_type):
# type: (errors.ParserContext, Union[syntax.Type, ast.Field], unicode) -> None
# type: (errors.ParserContext, Union[syntax.Type, ast.Field], str) -> None
"""Validate a chained type has both a deserializer and serializer."""
assert len(
idl_type.bson_serialization_type) == 1 and idl_type.bson_serialization_type[0] == 'chain'
@ -173,7 +171,7 @@ def _validate_chain_type_properties(ctxt, idl_type, syntax_type):
def _validate_type_properties(ctxt, idl_type, syntax_type):
# type: (errors.ParserContext, Union[syntax.Type, ast.Field], unicode) -> None
# type: (errors.ParserContext, Union[syntax.Type, ast.Field], str) -> None
# pylint: disable=too-many-branches
"""Validate each type or field is correct."""
@ -233,7 +231,7 @@ def _validate_types(ctxt, parsed_spec):
def _is_duplicate_field(ctxt, field_container, fields, ast_field):
# type: (errors.ParserContext, unicode, List[ast.Field], ast.Field) -> bool
# type: (errors.ParserContext, str, List[ast.Field], ast.Field) -> bool
"""Return True if there is a naming conflict for a given field."""
# This is normally tested in the parser as part of duplicate detection in a map
@ -494,7 +492,7 @@ def _validate_doc_sequence_field(ctxt, ast_field):
def _normalize_method_name(cpp_type_name, cpp_method_name):
# type: (unicode, unicode) -> unicode
# type: (str, str) -> str
"""Normalize the method name to be fully-qualified with the type name."""
# Default deserializer
if not cpp_method_name:
@ -540,7 +538,7 @@ def _bind_expression(expr, allow_literal_string=True):
# int32_t
try:
intval = int(expr.literal)
if (intval >= -0x80000000) and (intval <= 0x7FFFFFFF):
if intval >= -0x80000000 and intval <= 0x7FFFFFFF: # pylint: disable=chained-comparison
node.expr = repr(intval)
return node
except ValueError:
@ -847,7 +845,7 @@ def _validate_enum_int(ctxt, idl_enum):
min_value = min(int_values_set)
max_value = max(int_values_set)
valid_int = {x for x in xrange(min_value, max_value + 1)}
valid_int = {x for x in range(min_value, max_value + 1)}
if valid_int != int_values_set:
ctxt.add_enum_non_continuous_range_error(idl_enum, idl_enum.name)
@ -879,7 +877,7 @@ def _bind_enum(ctxt, idl_enum):
ast_enum_value.value = enum_value.value
ast_enum.values.append(ast_enum_value)
values_set = set() # type: Set[unicode]
values_set = set() # type: Set[str]
for enum_value in idl_enum.values:
values_set.add(enum_value.value)
@ -953,7 +951,7 @@ def _bind_server_parameter_with_storage(ctxt, ast_param, param):
def _bind_server_parameter_set_at(ctxt, param):
# type: (errors.ParserContext, syntax.ServerParameter) -> unicode
# type: (errors.ParserContext, syntax.ServerParameter) -> str
"""Translate set_at options to C++ enum value."""
set_at = 0
@ -1003,13 +1001,13 @@ def _bind_server_parameter(ctxt, param):
def _is_invalid_config_short_name(name):
# type: (unicode) -> bool
# type: (str) -> bool
"""Check if a given name is valid as a short name."""
return ('.' in name) or (',' in name)
def _parse_config_option_sources(source_list):
# type: (List[unicode]) -> unicode
# type: (List[str]) -> str
"""Parse source list into enum value used by runtime."""
sources = 0
if not source_list:

4
buildscripts/idl/idl/bson.py
View File

@ -31,8 +31,6 @@ BSON Type Information.
Utilities for validating bson types, etc.
"""
from __future__ import absolute_import, print_function, unicode_literals
from typing import Dict, List
# Dictionary of BSON type Information
@ -100,7 +98,7 @@ def cpp_bson_type_name(name):
def list_valid_types():
# type: () -> List[unicode]
"""Return a list of supported bson types."""
return [a for a in _BSON_TYPE_INFORMATION.iterkeys()]
return [a for a in _BSON_TYPE_INFORMATION.keys()]
def is_valid_bindata_subtype(name):

24
buildscripts/idl/idl/common.py
View File

@ -31,8 +31,6 @@ IDL Common classes.
Classes which are shared among both the IDL idl.syntax and idl.AST trees.
"""
from __future__ import absolute_import, print_function, unicode_literals
import os
import string
from typing import Mapping
@ -44,7 +42,7 @@ COMMAND_NAMESPACE_TYPE = "type"
def title_case(name):
# type: (unicode) -> unicode
# type: (str) -> str
"""Return a CapitalCased version of a string."""
# Only capitalize the last part of a fully-qualified name
@ -56,13 +54,13 @@ def title_case(name):
def camel_case(name):
# type: (unicode) -> unicode
# type: (str) -> str
"""Return a camelCased version of a string."""
return name[0:1].lower() + name[1:]
def qualify_cpp_name(cpp_namespace, cpp_type_name):
# type: (unicode, unicode) -> unicode
# type: (str, str) -> str
"""Preprend a type name with a C++ namespace if cpp_namespace is not None."""
if cpp_namespace:
return cpp_namespace + "::" + cpp_type_name
@ -71,7 +69,7 @@ def qualify_cpp_name(cpp_namespace, cpp_type_name):
def _escape_template_string(template):
# type: (unicode) -> unicode
# type: (str) -> str
"""Escape the '$' in template strings unless followed by '{'."""
# See https://docs.python.org/2/library/string.html#template-strings
template = template.replace('${', '#{')
@ -80,20 +78,20 @@ def _escape_template_string(template):
def template_format(template, template_params=None):
# type: (unicode, Mapping[unicode,unicode]) -> unicode
# type: (str, Mapping[str,str]) -> str
"""Write a template to the stream."""
# Ignore the types since we use unicode literals and this expects str but works fine with
# unicode.
# Ignore the types since we use str literals and this expects str but works fine with
# str.
# See https://docs.python.org/2/library/string.html#template-strings
template = _escape_template_string(template)
return string.Template(template).substitute(template_params) # type: ignore
def template_args(template, **kwargs):
# type: (unicode, **unicode) -> unicode
# type: (str, **str) -> str
"""Write a template to the stream."""
# Ignore the types since we use unicode literals and this expects str but works fine with
# unicode.
# Ignore the types since we use str literals and this expects str but works fine with
# str.
# See https://docs.python.org/2/library/string.html#template-strings
template = _escape_template_string(template)
return string.Template(template).substitute(kwargs) # type: ignore
@ -103,7 +101,7 @@ class SourceLocation(object):
"""Source location information about an idl.syntax or idl.AST object."""
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a source location."""
self.file_name = file_name
self.line = line

24
buildscripts/idl/idl/compiler.py
View File

@ -31,8 +31,6 @@ IDL compiler driver.
Orchestrates the 3 passes (parser, binder, and generator) together.
"""
from __future__ import absolute_import, print_function, unicode_literals
import io
import logging
import os
@ -54,14 +52,14 @@ class CompilerArgs(object):
def __init__(self):
# type: () -> None
"""Create a container for compiler arguments."""
self.import_directories = None # type: List[unicode]
self.input_file = None # type: unicode
self.target_arch = None # type: unicode
self.import_directories = None # type: List[str]
self.input_file = None # type: str
self.target_arch = None # type: str
self.output_source = None # type: unicode
self.output_header = None # type: unicode
self.output_base_dir = None # type: unicode
self.output_suffix = None # type: unicode
self.output_source = None # type: str
self.output_header = None # type: str
self.output_base_dir = None # type: str
self.output_suffix = None # type: str
self.write_dependencies = False # type: bool
self.write_dependencies_inline = False # type: bool
@ -71,14 +69,14 @@ class CompilerImportResolver(parser.ImportResolverBase):
"""Class for the IDL compiler to resolve imported files."""
def __init__(self, import_directories):
# type: (List[unicode]) -> None
# type: (List[str]) -> None
"""Construct a ImportResolver."""
self._import_directories = import_directories
super(CompilerImportResolver, self).__init__()
def resolve(self, base_file, imported_file_name):
# type: (unicode, unicode) -> unicode
# type: (str, str) -> str
"""Return the complete path to an imported file name."""
logging.debug("Resolving imported file '%s' for file '%s'", imported_file_name, base_file)
@ -109,7 +107,7 @@ class CompilerImportResolver(parser.ImportResolverBase):
raise errors.IDLError(msg)
def open(self, resolved_file_name):
# type: (unicode) -> Any
# type: (str) -> Any
"""Return an io.Stream for the requested file."""
return io.open(resolved_file_name, encoding='utf-8')
@ -129,7 +127,7 @@ def _write_dependencies(spec, write_dependencies_inline):
def _update_import_includes(args, spec, header_file_name):
# type: (CompilerArgs, syntax.IDLSpec, unicode) -> None
# type: (CompilerArgs, syntax.IDLSpec, str) -> None
"""Update the list of imports with a list of include files for each import with structs."""
# This function is fragile:
# In order to try to generate headers with an "include what you use" set of headers, the IDL

134
buildscripts/idl/idl/cpp_types.py
View File

@ -27,8 +27,6 @@
#
"""IDL C++ Code Generator."""
from __future__ import absolute_import, print_function, unicode_literals
from abc import ABCMeta, abstractmethod
import string
import textwrap
@ -43,7 +41,7 @@ _STD_ARRAY_UINT8_16 = 'std::array<std::uint8_t,16>'
def is_primitive_scalar_type(cpp_type):
# type: (unicode) -> bool
# type: (str) -> bool
"""
Return True if a cpp_type is a primitive scalar type.
@ -56,7 +54,7 @@ def is_primitive_scalar_type(cpp_type):
def get_primitive_scalar_type_default_value(cpp_type):
# type: (unicode) -> unicode
# type: (str) -> str
"""
Return a default value for a primitive scalar type.
@ -70,26 +68,26 @@ def get_primitive_scalar_type_default_value(cpp_type):
def is_primitive_type(cpp_type):
# type: (unicode) -> bool
# type: (str) -> bool
"""Return True if a cpp_type is a primitive type and should not be returned as reference."""
cpp_type = cpp_type.replace(' ', '')
return is_primitive_scalar_type(cpp_type) or cpp_type == _STD_ARRAY_UINT8_16
def _qualify_optional_type(cpp_type):
# type: (unicode) -> unicode
# type: (str) -> str
"""Qualify the type as optional."""
return 'boost::optional<%s>' % (cpp_type)
def _qualify_array_type(cpp_type):
# type: (unicode) -> unicode
# type: (str) -> str
"""Qualify the type if the field is an array."""
return "std::vector<%s>" % (cpp_type)
def _optionally_make_call(method_name, param):
# type: (unicode, unicode) -> unicode
# type: (str, str) -> str
"""Return a call to method_name if it is not None, otherwise return an empty string."""
if not method_name:
return ''
@ -97,11 +95,9 @@ def _optionally_make_call(method_name, param):
return "%s(%s);" % (method_name, param)
class CppTypeBase(object):
class CppTypeBase(metaclass=ABCMeta):
"""Base type for C++ Type information."""
__metaclass__ = ABCMeta
def __init__(self, field):
# type: (ast.Field) -> None
"""Construct a CppTypeBase."""
@ -109,19 +105,19 @@ class CppTypeBase(object):
@abstractmethod
def get_type_name(self):
# type: () -> unicode
# type: () -> str
"""Get the C++ type name for a field."""
pass
@abstractmethod
def get_storage_type(self):
# type: () -> unicode
# type: () -> str
"""Get the C++ type name for the storage of class member for a field."""
pass
@abstractmethod
def get_getter_setter_type(self):
# type: () -> unicode
# type: () -> str
"""Get the C++ type name for the getter/setter parameter for a field."""
pass
@ -151,25 +147,25 @@ class CppTypeBase(object):
@abstractmethod
def get_getter_body(self, member_name):
# type: (unicode) -> unicode
# type: (str) -> str
"""Get the body of the getter."""
pass
@abstractmethod
def get_setter_body(self, member_name, validator_method_name):
# type: (unicode, unicode) -> unicode
# type: (str, str) -> str
"""Get the body of the setter."""
pass
@abstractmethod
def get_transform_to_getter_type(self, expression):
# type: (unicode) -> Optional[unicode]
# type: (str) -> Optional[str]
"""Get the expression to transform the input expression into the getter type."""
pass
@abstractmethod
def get_transform_to_storage_type(self, expression):
# type: (unicode) -> Optional[unicode]
# type: (str) -> Optional[str]
"""Get the expression to transform the input expression into the setter type."""
pass
@ -178,7 +174,7 @@ class _CppTypeBasic(CppTypeBase):
"""Default class for C++ Type information. Does not handle view types."""
def get_type_name(self):
# type: () -> unicode
# type: () -> str
if self._field.struct_type:
cpp_type = common.title_case(self._field.struct_type)
else:
@ -187,11 +183,11 @@ class _CppTypeBasic(CppTypeBase):
return cpp_type
def get_storage_type(self):
# type: () -> unicode
# type: () -> str
return self.get_type_name()
def get_getter_setter_type(self):
# type: () -> unicode
# type: () -> str
return self.get_type_name()
def is_const_type(self):
@ -225,22 +221,22 @@ class _CppTypeBasic(CppTypeBase):
return False
def get_getter_body(self, member_name):
# type: (unicode) -> unicode
# type: (str) -> str
return common.template_args('return ${member_name};', member_name=member_name)
def get_setter_body(self, member_name, validator_method_name):
# type: (unicode, unicode) -> unicode
# type: (str, str) -> str
return common.template_args(
'${optionally_call_validator} ${member_name} = std::move(value);',
optionally_call_validator=_optionally_make_call(validator_method_name,
'value'), member_name=member_name)
def get_transform_to_getter_type(self, expression):
# type: (unicode) -> Optional[unicode]
# type: (str) -> Optional[str]
return None
def get_transform_to_storage_type(self, expression):
# type: (unicode) -> Optional[unicode]
# type: (str) -> Optional[str]
return None
@ -248,21 +244,21 @@ class _CppTypeView(CppTypeBase):
"""Base type for C++ View Types information."""
def __init__(self, field, storage_type, view_type):
# type: (ast.Field, unicode, unicode) -> None
# type: (ast.Field, str, str) -> None
self._storage_type = storage_type
self._view_type = view_type
super(_CppTypeView, self).__init__(field)
def get_type_name(self):
# type: () -> unicode
# type: () -> str
return self._storage_type
def get_storage_type(self):
# type: () -> unicode
# type: () -> str
return self._storage_type
def get_getter_setter_type(self):
# type: () -> unicode
# type: () -> str
return self._view_type
def is_const_type(self):
@ -282,11 +278,11 @@ class _CppTypeView(CppTypeBase):
return True
def get_getter_body(self, member_name):
# type: (unicode) -> unicode
# type: (str) -> str
return common.template_args('return ${member_name};', member_name=member_name)
def get_setter_body(self, member_name, validator_method_name):
# type: (unicode, unicode) -> unicode
# type: (str, str) -> str
return common.template_args(
'auto _tmpValue = ${value}; ${optionally_call_validator} ${member_name} = std::move(_tmpValue);',
member_name=member_name, optionally_call_validator=_optionally_make_call(
@ -294,11 +290,11 @@ class _CppTypeView(CppTypeBase):
'_tmpValue'), value=self.get_transform_to_storage_type("value"))
def get_transform_to_getter_type(self, expression):
# type: (unicode) -> Optional[unicode]
# type: (str) -> Optional[str]
return None
def get_transform_to_storage_type(self, expression):
# type: (unicode) -> Optional[unicode]
# type: (str) -> Optional[str]
return common.template_args(
'${expression}.toString()',
expression=expression,
@ -309,15 +305,15 @@ class _CppTypeVector(CppTypeBase):
"""Base type for C++ Std::Vector Types information."""
def get_type_name(self):
# type: () -> unicode
# type: () -> str
return 'std::vector<std::uint8_t>'
def get_storage_type(self):
# type: () -> unicode
# type: () -> str
return self.get_type_name()
def get_getter_setter_type(self):
# type: () -> unicode
# type: () -> str
return 'ConstDataRange'
def is_const_type(self):
@ -337,12 +333,12 @@ class _CppTypeVector(CppTypeBase):
return True
def get_getter_body(self, member_name):
# type: (unicode) -> unicode
# type: (str) -> str
return common.template_args('return ConstDataRange(${member_name});',
member_name=member_name)
def get_setter_body(self, member_name, validator_method_name):
# type: (unicode, unicode) -> unicode
# type: (str, str) -> str
return common.template_args(
'auto _tmpValue = ${value}; ${optionally_call_validator} ${member_name} = std::move(_tmpValue);',
member_name=member_name, optionally_call_validator=_optionally_make_call(
@ -350,11 +346,11 @@ class _CppTypeVector(CppTypeBase):
'_tmpValue'), value=self.get_transform_to_storage_type("value"))
def get_transform_to_getter_type(self, expression):
# type: (unicode) -> Optional[unicode]
# type: (str) -> Optional[str]
return common.template_args('ConstDataRange(${expression});', expression=expression)
def get_transform_to_storage_type(self, expression):
# type: (unicode) -> Optional[unicode]
# type: (str) -> Optional[str]
return common.template_args(
'std::vector<std::uint8_t>(reinterpret_cast<const uint8_t*>(${expression}.data()), ' +
'reinterpret_cast<const uint8_t*>(${expression}.data()) + ${expression}.length())',
@ -370,15 +366,15 @@ class _CppTypeDelegating(CppTypeBase):
super(_CppTypeDelegating, self).__init__(field)
def get_type_name(self):
# type: () -> unicode
# type: () -> str
return self._base.get_type_name()
def get_storage_type(self):
# type: () -> unicode
# type: () -> str
return self._base.get_storage_type()
def get_getter_setter_type(self):
# type: () -> unicode
# type: () -> str
return self._base.get_getter_setter_type()
def is_const_type(self):
@ -398,19 +394,19 @@ class _CppTypeDelegating(CppTypeBase):
return self._base.is_view_type()
def get_getter_body(self, member_name):
# type: (unicode) -> unicode
# type: (str) -> str
return self._base.get_getter_body(member_name)
def get_setter_body(self, member_name, validator_method_name):
# type: (unicode, unicode) -> unicode
# type: (str, str) -> str
return self._base.get_setter_body(member_name, validator_method_name)
def get_transform_to_getter_type(self, expression):
# type: (unicode) -> Optional[unicode]
# type: (str) -> Optional[str]
return self._base.get_transform_to_getter_type(expression)
def get_transform_to_storage_type(self, expression):
# type: (unicode) -> Optional[unicode]
# type: (str) -> Optional[str]
return self._base.get_transform_to_storage_type(expression)
@ -418,11 +414,11 @@ class _CppTypeArray(_CppTypeDelegating):
"""C++ Array type for wrapping a base C++ Type information."""
def get_storage_type(self):
# type: () -> unicode
# type: () -> str
return _qualify_array_type(self._base.get_storage_type())
def get_getter_setter_type(self):
# type: () -> unicode
# type: () -> str
return _qualify_array_type(self._base.get_getter_setter_type())
def return_by_reference(self):
@ -436,14 +432,14 @@ class _CppTypeArray(_CppTypeDelegating):
return True
def get_getter_body(self, member_name):
# type: (unicode) -> unicode
# type: (str) -> str
convert = self.get_transform_to_getter_type(member_name)
if convert:
return common.template_args('return ${convert};', convert=convert)
return self._base.get_getter_body(member_name)
def get_setter_body(self, member_name, validator_method_name):
# type: (unicode, unicode) -> unicode
# type: (str, str) -> str
convert = self.get_transform_to_storage_type("value")
if convert:
return common.template_args(
@ -453,7 +449,7 @@ class _CppTypeArray(_CppTypeDelegating):
return self._base.get_setter_body(member_name, validator_method_name)
def get_transform_to_getter_type(self, expression):
# type: (unicode) -> Optional[unicode]
# type: (str) -> Optional[str]
if self._base.get_storage_type() != self._base.get_getter_setter_type():
return common.template_args(
'transformVector(${expression})',
@ -462,7 +458,7 @@ class _CppTypeArray(_CppTypeDelegating):
return None
def get_transform_to_storage_type(self, expression):
# type: (unicode) -> Optional[unicode]
# type: (str) -> Optional[str]
if self._base.get_storage_type() != self._base.get_getter_setter_type():
return common.template_args(
'transformVector(${expression})',
@ -475,11 +471,11 @@ class _CppTypeOptional(_CppTypeDelegating):
"""Base type for Optional C++ Type information which wraps C++ types."""
def get_storage_type(self):
# type: () -> unicode
# type: () -> str
return _qualify_optional_type(self._base.get_storage_type())
def get_getter_setter_type(self):
# type: () -> unicode
# type: () -> str
return _qualify_optional_type(self._base.get_getter_setter_type())
def disable_xvalue(self):
@ -493,7 +489,7 @@ class _CppTypeOptional(_CppTypeDelegating):
return self._base.return_by_reference()
def get_getter_body(self, member_name):
# type: (unicode) -> unicode
# type: (str) -> str
base_expression = common.template_args("${member_name}.get()", member_name=member_name)
convert = self._base.get_transform_to_getter_type(base_expression)
@ -517,7 +513,7 @@ class _CppTypeOptional(_CppTypeDelegating):
return common.template_args('return ${member_name};', member_name=member_name)
def get_setter_body(self, member_name, validator_method_name):
# type: (unicode, unicode) -> unicode
# type: (str, str) -> str
convert = self._base.get_transform_to_storage_type("value.get()")
if convert or validator_method_name:
if not convert:
@ -558,11 +554,9 @@ def get_cpp_type(field):
return cpp_type_info
class BsonCppTypeBase(object):
class BsonCppTypeBase(object, metaclass=ABCMeta):
"""Base type for custom C++ support for BSON Types information."""
__metaclass__ = ABCMeta
def __init__(self, field):
# type: (ast.Field) -> None
"""Construct a BsonCppTypeBase."""
@ -570,7 +564,7 @@ class BsonCppTypeBase(object):
@abstractmethod
def gen_deserializer_expression(self, indented_writer, object_instance):
# type: (writer.IndentedTextWriter, unicode) -> unicode
# type: (writer.IndentedTextWriter, str) -> str
"""Generate code with the text writer and return an expression to deserialize the type."""
pass
@ -582,13 +576,13 @@ class BsonCppTypeBase(object):
@abstractmethod
def gen_serializer_expression(self, indented_writer, expression):
# type: (writer.IndentedTextWriter, unicode) -> unicode
# type: (writer.IndentedTextWriter, str) -> str
"""Generate code with the text writer and return an expression to serialize the type."""
pass
def _call_method_or_global_function(expression, method_name):
# type: (unicode, unicode) -> unicode
# type: (str, str) -> str
"""
Given a fully-qualified method name, call it correctly.
@ -609,12 +603,12 @@ class _CommonBsonCppTypeBase(BsonCppTypeBase):
"""Custom C++ support for basic BSON types."""
def __init__(self, field, deserialize_method_name):
# type: (ast.Field, unicode) -> None
# type: (ast.Field, str) -> None
self._deserialize_method_name = deserialize_method_name
super(_CommonBsonCppTypeBase, self).__init__(field)
def gen_deserializer_expression(self, indented_writer, object_instance):
# type: (writer.IndentedTextWriter, unicode) -> unicode
# type: (writer.IndentedTextWriter, str) -> str
return common.template_args('${object_instance}.${method_name}()',
object_instance=object_instance,
method_name=self._deserialize_method_name)
@ -624,7 +618,7 @@ class _CommonBsonCppTypeBase(BsonCppTypeBase):
return self._field.serializer is not None
def gen_serializer_expression(self, indented_writer, expression):
# type: (writer.IndentedTextWriter, unicode) -> unicode
# type: (writer.IndentedTextWriter, str) -> str
return _call_method_or_global_function(expression, self._field.serializer)
@ -632,7 +626,7 @@ class _ObjectBsonCppTypeBase(BsonCppTypeBase):
"""Custom C++ support for object BSON types."""
def gen_deserializer_expression(self, indented_writer, object_instance):
# type: (writer.IndentedTextWriter, unicode) -> unicode
# type: (writer.IndentedTextWriter, str) -> str
if self._field.deserializer:
# Call a method like: Class::method(const BSONObj& value)
indented_writer.write_line(
@ -648,7 +642,7 @@ class _ObjectBsonCppTypeBase(BsonCppTypeBase):
return self._field.serializer is not None
def gen_serializer_expression(self, indented_writer, expression):
# type: (writer.IndentedTextWriter, unicode) -> unicode
# type: (writer.IndentedTextWriter, str) -> str
method_name = writer.get_method_name(self._field.serializer)
indented_writer.write_line(
common.template_args('const BSONObj localObject = ${expression}.${method_name}();',
@ -660,7 +654,7 @@ class _BinDataBsonCppTypeBase(BsonCppTypeBase):
"""Custom C++ support for all binData BSON types."""
def gen_deserializer_expression(self, indented_writer, object_instance):
# type: (writer.IndentedTextWriter, unicode) -> unicode
# type: (writer.IndentedTextWriter, str) -> str
if self._field.bindata_subtype == 'uuid':
return common.template_args('${object_instance}.uuid()',
object_instance=object_instance)
@ -672,7 +666,7 @@ class _BinDataBsonCppTypeBase(BsonCppTypeBase):
return True
def gen_serializer_expression(self, indented_writer, expression):
# type: (writer.IndentedTextWriter, unicode) -> unicode
# type: (writer.IndentedTextWriter, str) -> str
if self._field.serializer:
method_name = writer.get_method_name(self._field.serializer)
indented_writer.write_line(

53
buildscripts/idl/idl/enum_types.py
View File

@ -31,8 +31,6 @@ IDL Enum type information.
Support the code generation for enums
"""
from __future__ import absolute_import, print_function, unicode_literals
from abc import ABCMeta, abstractmethod
import textwrap
from typing import cast, List, Optional, Union
@ -43,11 +41,9 @@ from . import syntax
from . import writer
class EnumTypeInfoBase(object):
class EnumTypeInfoBase(object, metaclass=ABCMeta):
"""Base type for enumeration type information."""
__metaclass__ = ABCMeta
def __init__(self, idl_enum):
# type: (Union[syntax.Enum,ast.Enum]) -> None
"""Construct a EnumTypeInfoBase."""
@ -73,8 +69,8 @@ class EnumTypeInfoBase(object):
def _get_enum_deserializer_name(self):
# type: () -> unicode
"""Return the name of deserializer function without prefix."""
return common.template_args("${enum_name}_parse", enum_name=common.title_case(
self._enum.name))
return common.template_args("${enum_name}_parse",
enum_name=common.title_case(self._enum.name))
def get_enum_deserializer_name(self):
# type: () -> unicode
@ -85,8 +81,8 @@ class EnumTypeInfoBase(object):
def _get_enum_serializer_name(self):
# type: () -> unicode
"""Return the name of serializer function without prefix."""
return common.template_args("${enum_name}_serializer", enum_name=common.title_case(
self._enum.name))
return common.template_args("${enum_name}_serializer",
enum_name=common.title_case(self._enum.name))
def get_enum_serializer_name(self):
# type: () -> unicode
@ -114,7 +110,7 @@ class EnumTypeInfoBase(object):
@abstractmethod
def get_serializer_declaration(self):
# type: () -> unicode
# type: () -> str
"""Get the serializer function declaration minus trailing semicolon."""
pass
@ -125,11 +121,9 @@ class EnumTypeInfoBase(object):
pass
class _EnumTypeInt(EnumTypeInfoBase):
class _EnumTypeInt(EnumTypeInfoBase, metaclass=ABCMeta):
"""Type information for integer enumerations."""
__metaclass__ = ABCMeta
def get_cpp_type_name(self):
# type: () -> unicode
return common.title_case(self._enum.name)
@ -174,7 +168,7 @@ class _EnumTypeInt(EnumTypeInfoBase):
"""))
def get_serializer_declaration(self):
# type: () -> unicode
# type: () -> str
"""Get the serializer function declaration minus trailing semicolon."""
return common.template_args("std::int32_t ${function_name}(${enum_name} value)",
enum_name=self.get_cpp_type_name(),
@ -200,15 +194,13 @@ def _get_constant_enum_name(idl_enum, enum_value):
name=enum_value.name)
class _EnumTypeString(EnumTypeInfoBase):
class _EnumTypeString(EnumTypeInfoBase, metaclass=ABCMeta):
"""Type information for string enumerations."""
__metaclass__ = ABCMeta
def get_cpp_type_name(self):
# type: () -> unicode
return common.template_args("${enum_name}Enum", enum_name=common.title_case(
self._enum.name))
return common.template_args("${enum_name}Enum",
enum_name=common.title_case(self._enum.name))
def get_bson_types(self):
# type: () -> List[unicode]
@ -236,24 +228,25 @@ class _EnumTypeString(EnumTypeInfoBase):
with writer.NamespaceScopeBlock(indented_writer, ['']):
for enum_value in self._enum.values:
indented_writer.write_line(
common.template_args('constexpr StringData ${constant_name} = "${value}"_sd;',
constant_name=_get_constant_enum_name(
self._enum, enum_value), value=enum_value.value))
common.template_args(
'constexpr StringData ${constant_name} = "${value}"_sd;',
constant_name=_get_constant_enum_name(self._enum,
enum_value), value=enum_value.value))
indented_writer.write_empty_line()
with writer.TemplateContext(indented_writer, template_params):
with writer.IndentedScopedBlock(indented_writer, "${function_name} {", "}"):
for enum_value in self._enum.values:
predicate = 'if (value == %s) {' % (
_get_constant_enum_name(self._enum, enum_value))
predicate = 'if (value == %s) {' % (_get_constant_enum_name(
self._enum, enum_value))
with writer.IndentedScopedBlock(indented_writer, predicate, "}"):
indented_writer.write_template('return ${enum_name}::%s;' %
(enum_value.name))
indented_writer.write_template(
'return ${enum_name}::%s;' % (enum_value.name))
indented_writer.write_line("ctxt.throwBadEnumValue(value);")
def get_serializer_declaration(self):
# type: () -> unicode
# type: () -> str
"""Get the serializer function declaration minus trailing semicolon."""
return common.template_args("StringData ${function_name}(${enum_name} value)",
enum_name=self.get_cpp_type_name(),
@ -270,9 +263,9 @@ class _EnumTypeString(EnumTypeInfoBase):
with writer.TemplateContext(indented_writer, template_params):
with writer.IndentedScopedBlock(indented_writer, "${function_name} {", "}"):
for enum_value in self._enum.values:
with writer.IndentedScopedBlock(indented_writer,
'if (value == ${enum_name}::%s) {' %
(enum_value.name), "}"):
with writer.IndentedScopedBlock(
indented_writer, 'if (value == ${enum_name}::%s) {' % (enum_value.name),
"}"):
indented_writer.write_line(
'return %s;' % (_get_constant_enum_name(self._enum, enum_value)))

241
buildscripts/idl/idl/errors.py
View File

@ -32,8 +32,6 @@ Common error handling code for IDL compiler.
- Error codes used by the IDL compiler.
"""
from __future__ import absolute_import, print_function, unicode_literals
import inspect
import os
import sys
@ -132,7 +130,7 @@ class ParserError(common.SourceLocation):
"""
def __init__(self, error_id, msg, file_name, line, column):
# type: (unicode, unicode, unicode, int, int) -> None
# type: (str, str, str, int, int) -> None
"""Construct a parser error with source location information."""
# pylint: disable=too-many-arguments
self.error_id = error_id
@ -160,7 +158,7 @@ class ParserErrorCollection(object):
self._errors = [] # type: List[ParserError]
def add(self, location, error_id, msg):
# type: (common.SourceLocation, unicode, unicode) -> None
# type: (common.SourceLocation, str, str) -> None
"""Add an error message with file (line, column) information."""
self._errors.append(
ParserError(error_id, msg, location.file_name, location.line, location.column))
@ -171,12 +169,12 @@ class ParserErrorCollection(object):
return len(self._errors) > 0
def contains(self, error_id):
# type: (unicode) -> bool
# type: (str) -> bool
"""Check if the error collection has at least one message of a given error_id."""
return len([a for a in self._errors if a.error_id == error_id]) > 0
def to_list(self):
# type: () -> List[unicode]
# type: () -> List[str]
"""Return a list of formatted error messages."""
return [str(error) for error in self._errors]
@ -211,13 +209,13 @@ class ParserContext(object):
# pylint: disable=too-many-public-methods
def __init__(self, file_name, errors):
# type: (unicode, ParserErrorCollection) -> None
# type: (str, ParserErrorCollection) -> None
"""Construct a new ParserContext."""
self.errors = errors
self.file_name = file_name
def _add_error(self, location, error_id, msg):
# type: (common.SourceLocation, unicode, unicode) -> None
# type: (common.SourceLocation, str, str) -> None
"""
Add an error with a source location information.
@ -226,7 +224,7 @@ class ParserContext(object):
self.errors.add(location, error_id, msg)
def _add_node_error(self, node, error_id, msg):
# type: (yaml.nodes.Node, unicode, unicode) -> None
# type: (yaml.nodes.Node, str, str) -> None
"""Add an error with source location information based on a YAML node."""
self.errors.add(
common.SourceLocation(self.file_name, node.start_mark.line, node.start_mark.column),
@ -235,32 +233,32 @@ class ParserContext(object):
def add_unknown_root_node_error(self, node):
# type: (yaml.nodes.Node) -> None
"""Add an error about an unknown YAML root node."""
self._add_node_error(node, ERROR_ID_UNKNOWN_ROOT,
("Unrecognized IDL specification root level node '%s', only " +
" (global, import, types, commands, and structs) are accepted") %
(node.value))
self._add_node_error(
node, ERROR_ID_UNKNOWN_ROOT,
("Unrecognized IDL specification root level node '%s', only " +
" (global, import, types, commands, and structs) are accepted") % (node.value))
def add_unknown_node_error(self, node, name):
# type: (yaml.nodes.Node, unicode) -> None
# type: (yaml.nodes.Node, str) -> None
"""Add an error about an unknown node."""
self._add_node_error(node, ERROR_ID_UNKNOWN_NODE,
"Unknown IDL node '%s' for YAML entity '%s'" % (node.value, name))
def add_duplicate_symbol_error(self, location, name, duplicate_class_name, original_class_name):
# type: (common.SourceLocation, unicode, unicode, unicode) -> None
# type: (common.SourceLocation, str, str, str) -> None
"""Add an error about a duplicate symbol."""
self._add_error(location, ERROR_ID_DUPLICATE_SYMBOL,
"%s '%s' is a duplicate symbol of an existing %s" %
(duplicate_class_name, name, original_class_name))
self._add_error(
location, ERROR_ID_DUPLICATE_SYMBOL, "%s '%s' is a duplicate symbol of an existing %s" %
(duplicate_class_name, name, original_class_name))
def add_unknown_type_error(self, location, field_name, type_name):
# type: (common.SourceLocation, unicode, unicode) -> None
# type: (common.SourceLocation, str, str) -> None
"""Add an error about an unknown type."""
self._add_error(location, ERROR_ID_UNKNOWN_TYPE,
"'%s' is an unknown type for field '%s'" % (type_name, field_name))
def _is_node_type(self, node, node_name, expected_node_type):
# type: (Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode], unicode, unicode) -> bool
# type: (Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode], str, str) -> bool
"""Return True if the yaml node type is expected, otherwise returns False and logs an error."""
if not node.id == expected_node_type:
self._add_node_error(
@ -271,17 +269,17 @@ class ParserContext(object):
return True
def is_mapping_node(self, node, node_name):
# type: (Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode], unicode) -> bool
# type: (Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode], str) -> bool
"""Return True if this YAML node is a Map."""
return self._is_node_type(node, node_name, "mapping")
def is_scalar_node(self, node, node_name):
# type: (Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode], unicode) -> bool
# type: (Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode], str) -> bool
"""Return True if this YAML node is a Scalar."""
return self._is_node_type(node, node_name, "scalar")
def is_scalar_sequence(self, node, node_name):
# type: (Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode], unicode) -> bool
# type: (Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode], str) -> bool
"""Return True if this YAML node is a Sequence of Scalars."""
if self._is_node_type(node, node_name, "sequence"):
for seq_node in node.value:
@ -291,7 +289,7 @@ class ParserContext(object):
return False
def is_scalar_sequence_or_scalar_node(self, node, node_name):
# type: (Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode], unicode) -> bool
# type: (Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode], str) -> bool
# pylint: disable=invalid-name
"""Return True if the YAML node is a Scalar or Sequence."""
if not node.id == "scalar" and not node.id == "sequence":
@ -307,7 +305,7 @@ class ParserContext(object):
return True
def is_scalar_or_mapping_node(self, node, node_name):
# type: (Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode], unicode) -> bool
# type: (Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode], str) -> bool
# pylint: disable=invalid-name
"""Return True if the YAML node is a Scalar or Mapping."""
if not node.id == "scalar" and not node.id == "mapping":
@ -320,7 +318,7 @@ class ParserContext(object):
return True
def is_scalar_bool_node(self, node, node_name):
# type: (Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode], unicode) -> bool
# type: (Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode], str) -> bool
"""Return True if this YAML node is a Scalar and a valid boolean."""
if not self._is_node_type(node, node_name, "scalar"):
return False
@ -343,7 +341,7 @@ class ParserContext(object):
return False
def get_list(self, node):
# type: (Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> List[unicode]
# type: (Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> List[str]
"""Get a YAML scalar or sequence node as a list of strings."""
assert self.is_scalar_sequence_or_scalar_node(node, "unknown")
if node.id == "scalar":
@ -352,49 +350,49 @@ class ParserContext(object):
return [v.value for v in node.value]
def add_duplicate_error(self, node, node_name):
# type: (yaml.nodes.Node, unicode) -> None
# type: (yaml.nodes.Node, str) -> None
"""Add an error about a duplicate node."""
self._add_node_error(node, ERROR_ID_DUPLICATE_NODE,
"Duplicate node found for '%s'" % (node_name))
def add_empty_struct_error(self, node, name):
# type: (yaml.nodes.Node, unicode) -> None
# type: (yaml.nodes.Node, str) -> None
"""Add an error about a struct without fields."""
self._add_node_error(node, ERROR_ID_EMPTY_FIELDS,
("Struct '%s' must either have fields, chained_types, or " +
"chained_structs specified but neither were found") % (name))
def add_missing_required_field_error(self, node, node_parent, node_name):
# type: (yaml.nodes.Node, unicode, unicode) -> None
# type: (yaml.nodes.Node, str, str) -> None
"""Add an error about a YAML node missing a required child."""
# pylint: disable=invalid-name
self._add_node_error(node, ERROR_ID_MISSING_REQUIRED_FIELD,
"IDL node '%s' is missing required scalar '%s'" % (node_parent,
node_name))
self._add_node_error(
node, ERROR_ID_MISSING_REQUIRED_FIELD,
"IDL node '%s' is missing required scalar '%s'" % (node_parent, node_name))
def add_missing_ast_required_field_error(self, location, ast_type, ast_parent, ast_name):
# type: (common.SourceLocation, unicode, unicode, unicode) -> None
# type: (common.SourceLocation, str, str, str) -> None
"""Add an error about a AST node missing a required child."""
# pylint: disable=invalid-name
self._add_error(location, ERROR_ID_MISSING_AST_REQUIRED_FIELD,
"%s '%s' is missing required scalar '%s'" % (ast_type, ast_parent,
ast_name))
self._add_error(
location, ERROR_ID_MISSING_AST_REQUIRED_FIELD,
"%s '%s' is missing required scalar '%s'" % (ast_type, ast_parent, ast_name))
def add_array_not_valid_error(self, location, ast_type, name):
# type: (common.SourceLocation, unicode, unicode) -> None
# type: (common.SourceLocation, str, str) -> None
"""Add an error about a 'array' not being a valid type name."""
self._add_error(location, ERROR_ID_ARRAY_NOT_VALID_TYPE,
"The %s '%s' cannot be named 'array'" % (ast_type, name))
def add_bad_bson_type_error(self, location, ast_type, ast_parent, bson_type_name):
# type: (common.SourceLocation, unicode, unicode, unicode) -> None
# type: (common.SourceLocation, str, str, str) -> None
"""Add an error about a bad bson type."""
self._add_error(location, ERROR_ID_BAD_BSON_TYPE,
"BSON Type '%s' is not recognized for %s '%s'." % (bson_type_name, ast_type,
ast_parent))
self._add_error(
location, ERROR_ID_BAD_BSON_TYPE, "BSON Type '%s' is not recognized for %s '%s'." %
(bson_type_name, ast_type, ast_parent))
def add_bad_bson_scalar_type_error(self, location, ast_type, ast_parent, bson_type_name):
# type: (common.SourceLocation, unicode, unicode, unicode) -> None
# type: (common.SourceLocation, str, str, str) -> None
"""Add an error about a bad list of bson types."""
self._add_error(location, ERROR_ID_BAD_BSON_TYPE_LIST,
("BSON Type '%s' is not a scalar bson type for %s '%s'" +
@ -402,7 +400,7 @@ class ParserContext(object):
(bson_type_name, ast_type, ast_parent))
def add_bad_bson_bindata_subtype_error(self, location, ast_type, ast_parent, bson_type_name):
# type: (common.SourceLocation, unicode, unicode, unicode) -> None
# type: (common.SourceLocation, str, str, str) -> None
"""Add an error about a bindata_subtype associated with a type that is not bindata."""
# pylint: disable=invalid-name
self._add_error(location, ERROR_ID_BAD_BSON_BINDATA_SUBTYPE_TYPE,
@ -410,7 +408,7 @@ class ParserContext(object):
(ast_type, ast_parent, bson_type_name))
def add_bad_bson_bindata_subtype_value_error(self, location, ast_type, ast_parent, value):
# type: (common.SourceLocation, unicode, unicode, unicode) -> None
# type: (common.SourceLocation, str, str, str) -> None
"""Add an error about a bad value for bindata_subtype."""
# pylint: disable=invalid-name
self._add_error(location, ERROR_ID_BAD_BSON_BINDATA_SUBTYPE_VALUE,
@ -418,22 +416,22 @@ class ParserContext(object):
(value, ast_type, ast_parent))
def add_bad_setat_specifier(self, location, specifier):
# type: (common.SourceLocation, unicode) -> None
# type: (common.SourceLocation, str) -> None
"""Add an error about a bad set_at specifier."""
# pylint: disable=invalid-name
self._add_error(location, ERROR_ID_BAD_SETAT_SPECIFIER,
("Unexpected set_at specifier: '%s', expected 'startup' or 'runtime'") %
(specifier))
self._add_error(
location, ERROR_ID_BAD_SETAT_SPECIFIER,
("Unexpected set_at specifier: '%s', expected 'startup' or 'runtime'") % (specifier))
def add_no_string_data_error(self, location, ast_type, ast_parent):
# type: (common.SourceLocation, unicode, unicode) -> None
# type: (common.SourceLocation, str, str) -> None
"""Add an error about using StringData for cpp_type."""
self._add_error(location, ERROR_ID_NO_STRINGDATA,
("Do not use mongo::StringData for %s '%s', use std::string instead") %
(ast_type, ast_parent))
def add_ignored_field_must_be_empty_error(self, location, name, field_name):
# type: (common.SourceLocation, unicode, unicode) -> None
# type: (common.SourceLocation, str, str) -> None
"""Add an error about field must be empty for ignored fields."""
# pylint: disable=invalid-name
self._add_error(
@ -442,16 +440,17 @@ class ParserContext(object):
) % (name, field_name))
def add_struct_field_must_be_empty_error(self, location, name, field_name):
# type: (common.SourceLocation, unicode, unicode) -> None
# type: (common.SourceLocation, str, str) -> None
"""Add an error about field must be empty for fields of type struct."""
# pylint: disable=invalid-name
self._add_error(location, ERROR_ID_FIELD_MUST_BE_EMPTY_FOR_STRUCT, (
"Field '%s' cannot contain a value for property '%s' when a field's type is a struct") %
(name, field_name))
self._add_error(
location, ERROR_ID_FIELD_MUST_BE_EMPTY_FOR_STRUCT,
("Field '%s' cannot contain a value for property '%s' when a field's type is a struct")
% (name, field_name))
def add_not_custom_scalar_serialization_not_supported_error(self, location, ast_type,
ast_parent, bson_type_name):
# type: (common.SourceLocation, unicode, unicode, unicode) -> None
# type: (common.SourceLocation, str, str, str) -> None
# pylint: disable=invalid-name
"""Add an error about field must be empty for fields of type struct."""
self._add_error(
@ -461,7 +460,7 @@ class ParserContext(object):
(ast_type, ast_parent, bson_type_name))
def add_bad_any_type_use_error(self, location, bson_type, ast_type, ast_parent):
# type: (common.SourceLocation, unicode, unicode, unicode) -> None
# type: (common.SourceLocation, str, str, str) -> None
# pylint: disable=invalid-name
"""Add an error about any being used in a list of bson types."""
self._add_error(
@ -470,25 +469,24 @@ class ParserContext(object):
"%s '%s'. It must be only a single bson type.") % (bson_type, ast_type, ast_parent))
def add_bad_cpp_numeric_type_use_error(self, location, ast_type, ast_parent, cpp_type):
# type: (common.SourceLocation, unicode, unicode, unicode) -> None
# type: (common.SourceLocation, str, str, str) -> None
# pylint: disable=invalid-name
"""Add an error about any being used in a list of bson types."""
self._add_error(
location, ERROR_ID_BAD_NUMERIC_CPP_TYPE,
("The C++ numeric type '%s' is not allowed for %s '%s'. Only 'std::int32_t'," +
" 'std::uint32_t', 'std::uint64_t', and 'std::int64_t' are supported.") % (cpp_type,
ast_type,
ast_parent))
" 'std::uint32_t', 'std::uint64_t', and 'std::int64_t' are supported.") %
(cpp_type, ast_type, ast_parent))
def add_bad_array_type_name_error(self, location, field_name, type_name):
# type: (common.SourceLocation, unicode, unicode) -> None
# type: (common.SourceLocation, str, str) -> None
"""Add an error about a field type having a malformed type name."""
self._add_error(location, ERROR_ID_BAD_ARRAY_TYPE_NAME,
("'%s' is not a valid array type for field '%s'. A valid array type" +
" is in the form 'array<type_name>'.") % (type_name, field_name))
def add_array_no_default_error(self, location, field_name):
# type: (common.SourceLocation, unicode) -> None
# type: (common.SourceLocation, str) -> None
"""Add an error about an array having a type with a default value."""
self._add_error(
location, ERROR_ID_ARRAY_NO_DEFAULT,
@ -496,27 +494,27 @@ class ParserContext(object):
(field_name))
def add_cannot_find_import(self, location, imported_file_name):
# type: (common.SourceLocation, unicode) -> None
# type: (common.SourceLocation, str) -> None
"""Add an error about not being able to find an import."""
self._add_error(location, ERROR_ID_BAD_IMPORT,
"Could not resolve import '%s', file not found" % (imported_file_name))
def add_bindata_no_default(self, location, ast_type, ast_parent):
# type: (common.SourceLocation, unicode, unicode) -> None
# type: (common.SourceLocation, str, str) -> None
# pylint: disable=invalid-name
"""Add an error about 'any' being used in a list of bson types."""
self._add_error(location, ERROR_ID_BAD_BINDATA_DEFAULT,
("Default values are not allowed for %s '%s'") % (ast_type, ast_parent))
def add_chained_type_not_found_error(self, location, type_name):
# type: (common.SourceLocation, unicode) -> None
# type: (common.SourceLocation, str) -> None
# pylint: disable=invalid-name
"""Add an error about a chained_type not found."""
self._add_error(location, ERROR_ID_CHAINED_TYPE_NOT_FOUND,
("Type '%s' is not a valid chained type") % (type_name))
def add_chained_type_wrong_type_error(self, location, type_name, bson_type_name):
# type: (common.SourceLocation, unicode, unicode) -> None
# type: (common.SourceLocation, str, str) -> None
# pylint: disable=invalid-name
"""Add an error about a chained_type being the wrong type."""
self._add_error(location, ERROR_ID_CHAINED_TYPE_WRONG_BSON_TYPE,
@ -524,14 +522,15 @@ class ParserContext(object):
"'chain' is supported for chained types.") % (type_name, bson_type_name))
def add_duplicate_field_error(self, location, field_container, field_name, duplicate_location):
# type: (common.SourceLocation, unicode, unicode, common.SourceLocation) -> None
# type: (common.SourceLocation, str, str, common.SourceLocation) -> None
"""Add an error about duplicate fields as a result of chained structs/types."""
self._add_error(location, ERROR_ID_CHAINED_DUPLICATE_FIELD, (
"Chained Struct or Type '%s' duplicates an existing field '%s' at location" + "'%s'.") %
(field_container, field_name, duplicate_location))
self._add_error(
location, ERROR_ID_CHAINED_DUPLICATE_FIELD,
("Chained Struct or Type '%s' duplicates an existing field '%s' at location" + "'%s'.")
% (field_container, field_name, duplicate_location))
def add_chained_type_no_strict_error(self, location, struct_name):
# type: (common.SourceLocation, unicode) -> None
# type: (common.SourceLocation, str) -> None
# pylint: disable=invalid-name
"""Add an error about strict parser validate and chained types."""
self._add_error(location, ERROR_ID_CHAINED_NO_TYPE_STRICT,
@ -539,14 +538,14 @@ class ParserContext(object):
"struct '%s'. Specify 'strict: false' for this struct.") % (struct_name))
def add_chained_struct_not_found_error(self, location, struct_name):
# type: (common.SourceLocation, unicode) -> None
# type: (common.SourceLocation, str) -> None
# pylint: disable=invalid-name
"""Add an error about a chained_struct not found."""
self._add_error(location, ERROR_ID_CHAINED_STRUCT_NOT_FOUND,
("Type '%s' is not a valid chained struct") % (struct_name))
def add_chained_nested_struct_no_strict_error(self, location, struct_name, nested_struct_name):
# type: (common.SourceLocation, unicode, unicode) -> None
# type: (common.SourceLocation, str, str) -> None
# pylint: disable=invalid-name
"""Add an error about strict parser validate and chained types."""
self._add_error(location, ERROR_ID_CHAINED_NO_NESTED_STRUCT_STRICT,
@ -555,7 +554,7 @@ class ParserContext(object):
(nested_struct_name, struct_name))
def add_chained_nested_struct_no_nested_error(self, location, struct_name, chained_name):
# type: (common.SourceLocation, unicode, unicode) -> None
# type: (common.SourceLocation, str, str) -> None
# pylint: disable=invalid-name
"""Add an error about struct's chaining being a struct with nested chaining."""
self._add_error(location, ERROR_ID_CHAINED_NO_NESTED_CHAINED,
@ -563,39 +562,40 @@ class ParserContext(object):
" structs and/or types.") % (struct_name, chained_name))
def add_empty_enum_error(self, node, name):
# type: (yaml.nodes.Node, unicode) -> None
# type: (yaml.nodes.Node, str) -> None
"""Add an error about an enum without values."""
self._add_node_error(node, ERROR_ID_BAD_EMPTY_ENUM,
"Enum '%s' must have values specified but no values were found" %
(name))
self._add_node_error(
node, ERROR_ID_BAD_EMPTY_ENUM,
"Enum '%s' must have values specified but no values were found" % (name))
def add_array_enum_error(self, location, field_name):
# type: (common.SourceLocation, unicode) -> None
# type: (common.SourceLocation, str) -> None
"""Add an error for a field being an array of enums."""
self._add_error(location, ERROR_ID_NO_ARRAY_ENUM,
"Field '%s' cannot be an array of enums" % (field_name))
def add_enum_bad_type_error(self, location, enum_name, enum_type):
# type: (common.SourceLocation, unicode, unicode) -> None
# type: (common.SourceLocation, str, str) -> None
"""Add an error for an enum having the wrong type."""
self._add_error(location, ERROR_ID_ENUM_BAD_TYPE,
"Enum '%s' type '%s' is not a supported enum type" % (enum_name, enum_type))
def add_enum_value_not_int_error(self, location, enum_name, enum_value, err_msg):
# type: (common.SourceLocation, unicode, unicode, unicode) -> None
# type: (common.SourceLocation, str, str, str) -> None
"""Add an error for an enum value not being an integer."""
self._add_error(location, ERROR_ID_ENUM_BAD_INT_VAUE,
"Enum '%s' value '%s' is not an integer, exception '%s'" %
(enum_name, enum_value, err_msg))
self._add_error(
location, ERROR_ID_ENUM_BAD_INT_VAUE,
"Enum '%s' value '%s' is not an integer, exception '%s'" % (enum_name, enum_value,
err_msg))
def add_enum_value_not_unique_error(self, location, enum_name):
# type: (common.SourceLocation, unicode) -> None
# type: (common.SourceLocation, str) -> None
"""Add an error for an enum having duplicate values."""
self._add_error(location, ERROR_ID_ENUM_NON_UNIQUE_VALUES,
"Enum '%s' has duplicate values, all values must be unique" % (enum_name))
def add_enum_non_continuous_range_error(self, location, enum_name):
# type: (common.SourceLocation, unicode) -> None
# type: (common.SourceLocation, str) -> None
"""Add an error for an enum having duplicate values."""
# pylint: disable=invalid-name
self._add_error(location, ERROR_ID_ENUM_NON_CONTINUOUS_RANGE,
@ -604,7 +604,7 @@ class ParserContext(object):
def add_bad_command_namespace_error(self, location, command_name, command_namespace,
valid_commands):
# type: (common.SourceLocation, unicode, unicode, List[unicode]) -> None
# type: (common.SourceLocation, str, str, List[str]) -> None
"""Add an error about the namespace value not being a valid choice."""
self._add_error(
location, ERROR_ID_BAD_COMMAND_NAMESPACE,
@ -612,21 +612,20 @@ class ParserContext(object):
% (command_namespace, command_name, valid_commands))
def add_bad_command_as_field_error(self, location, command_name):
# type: (common.SourceLocation, unicode) -> None
# type: (common.SourceLocation, str) -> None
"""Add an error about using a command for a field."""
self._add_error(
location, ERROR_ID_FIELD_NO_COMMAND,
("Command '%s' cannot be used as a field type'. Commands must be top-level" +
" types due to their serialization rules.") % (command_name))
self._add_error(location, ERROR_ID_FIELD_NO_COMMAND,
("Command '%s' cannot be used as a field type'. Commands must be top-level"
+ " types due to their serialization rules.") % (command_name))
def add_bad_array_of_chain(self, location, field_name):
# type: (common.SourceLocation, unicode) -> None
# type: (common.SourceLocation, str) -> None
"""Add an error about a field being an array of chain_types."""
self._add_error(location, ERROR_ID_NO_ARRAY_OF_CHAIN,
"Field '%s' cannot be an array of chained types" % (field_name))
def add_bad_field_default_and_optional(self, location, field_name):
# type: (common.SourceLocation, unicode) -> None
# type: (common.SourceLocation, str) -> None
"""Add an error about a field being optional and having a default value."""
# pylint: disable=invalid-name
self._add_error(
@ -635,7 +634,7 @@ class ParserContext(object):
(field_name))
def add_bad_struct_field_as_doc_sequence_error(self, location, struct_name, field_name):
# type: (common.SourceLocation, unicode, unicode) -> None
# type: (common.SourceLocation, str, str) -> None
"""Add an error about using a field in a struct being marked with supports_doc_sequence."""
# pylint: disable=invalid-name
self._add_error(location, ERROR_ID_STRUCT_NO_DOC_SEQUENCE,
@ -643,7 +642,7 @@ class ParserContext(object):
" type. They are only supported in commands.") % (field_name, struct_name))
def add_bad_non_array_as_doc_sequence_error(self, location, struct_name, field_name):
# type: (common.SourceLocation, unicode, unicode) -> None
# type: (common.SourceLocation, str, str) -> None
"""Add an error about using a non-array type field being marked with supports_doc_sequence."""
# pylint: disable=invalid-name
self._add_error(location, ERROR_ID_NO_DOC_SEQUENCE_FOR_NON_ARRAY,
@ -651,7 +650,7 @@ class ParserContext(object):
" type since it is not an array.") % (field_name, struct_name))
def add_bad_non_object_as_doc_sequence_error(self, location, field_name):
# type: (common.SourceLocation, unicode) -> None
# type: (common.SourceLocation, str) -> None
"""Add an error about using a non-struct or BSON object for a doc sequence."""
# pylint: disable=invalid-name
self._add_error(location, ERROR_ID_NO_DOC_SEQUENCE_FOR_NON_OBJECT,
@ -659,14 +658,14 @@ class ParserContext(object):
" type since it is not a BSON object or struct.") % (field_name))
def add_bad_command_name_duplicates_field(self, location, command_name):
# type: (common.SourceLocation, unicode) -> None
# type: (common.SourceLocation, str) -> None
"""Add an error about a command and field having the same name."""
# pylint: disable=invalid-name
self._add_error(location, ERROR_ID_COMMAND_DUPLICATES_FIELD,
("Command '%s' cannot have the same name as a field.") % (command_name))
def is_scalar_non_negative_int_node(self, node, node_name):
# type: (Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode], unicode) -> bool
# type: (Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode], str) -> bool
"""Return True if this YAML node is a Scalar and a valid non-negative int."""
if not self._is_node_type(node, node_name, "scalar"):
return False
@ -681,9 +680,9 @@ class ParserContext(object):
return False
except ValueError as value_error:
self._add_node_error(node, ERROR_ID_IS_NODE_VALID_INT,
"Illegal integer value for '%s', message '%s'." % (node_name,
value_error))
self._add_node_error(
node, ERROR_ID_IS_NODE_VALID_INT,
"Illegal integer value for '%s', message '%s'." % (node_name, value_error))
return False
return True
@ -696,7 +695,7 @@ class ParserContext(object):
return int(node.value)
def add_duplicate_comparison_order_field_error(self, location, struct_name, comparison_order):
# type: (common.SourceLocation, unicode, int) -> None
# type: (common.SourceLocation, str, int) -> None
"""Add an error about fields having duplicate comparison_orders."""
# pylint: disable=invalid-name
self._add_error(
@ -705,7 +704,7 @@ class ParserContext(object):
(struct_name, comparison_order))
def add_extranous_command_type(self, location, command_name):
# type: (common.SourceLocation, unicode) -> None
# type: (common.SourceLocation, str) -> None
"""Add an error about commands having type when not needed."""
# pylint: disable=invalid-name
self._add_error(
@ -714,23 +713,23 @@ class ParserContext(object):
(command_name))
def add_value_not_numeric_error(self, location, attrname, value):
# type: (common.SourceLocation, unicode, unicode) -> None
# type: (common.SourceLocation, str, str) -> None
"""Add an error about non-numeric value where number expected."""
# pylint: disable=invalid-name
self._add_error(location, ERROR_ID_VALUE_NOT_NUMERIC,
("'%s' requires a numeric value, but %s can not be cast") % (attrname,
value))
self._add_error(
location, ERROR_ID_VALUE_NOT_NUMERIC,
("'%s' requires a numeric value, but %s can not be cast") % (attrname, value))
def add_server_parameter_invalid_attr(self, location, attrname, conflicts):
# type: (common.SourceLocation, unicode, unicode) -> None
# type: (common.SourceLocation, str, str) -> None
"""Add an error about invalid fields in a server parameter definition."""
# pylint: disable=invalid-name
self._add_error(location, ERROR_ID_SERVER_PARAMETER_INVALID_ATTR,
("'%s' attribute not permitted with '%s' server parameter") % (attrname,
conflicts))
self._add_error(
location, ERROR_ID_SERVER_PARAMETER_INVALID_ATTR,
("'%s' attribute not permitted with '%s' server parameter") % (attrname, conflicts))
def add_server_parameter_required_attr(self, location, attrname, required, dependant=None):
# type: (common.SourceLocation, unicode, unicode, unicode) -> None
# type: (common.SourceLocation, str, str, str) -> None
"""Add an error about missing fields in a server parameter definition."""
# pylint: disable=invalid-name
qualifier = '' if dependant is None else (" when using '%s' attribute" % (dependant))
@ -739,28 +738,28 @@ class ParserContext(object):
(attrname, qualifier, required))
def add_server_parameter_invalid_method_override(self, location, method):
# type: (common.SourceLocation, unicode) -> None
# type: (common.SourceLocation, str) -> None
"""Add an error about invalid method override in SCP method override."""
# pylint: disable=invalid-name
self._add_error(location, ERROR_ID_SERVER_PARAMETER_INVALID_METHOD_OVERRIDE,
("No such method to override in server parameter class: '%s'") % (method))
def add_bad_source_specifier(self, location, value):
# type: (common.SourceLocation, unicode) -> None
# type: (common.SourceLocation, str) -> None
"""Add an error about invalid source specifier."""
# pylint: disable=invalid-name
self._add_error(location, ERROR_ID_BAD_SOURCE_SPECIFIER,
("'%s' is not a valid source specifier") % (value))
def add_bad_duplicate_behavior(self, location, value):
# type: (common.SourceLocation, unicode) -> None
# type: (common.SourceLocation, str) -> None
"""Add an error about invalid duplicate behavior specifier."""
# pylint: disable=invalid-name
self._add_error(location, ERROR_ID_BAD_DUPLICATE_BEHAVIOR_SPECIFIER,
("'%s' is not a valid duplicate behavior specifier") % (value))
def add_bad_numeric_range(self, location, attrname, value):
# type: (common.SourceLocation, unicode, unicode) -> None
# type: (common.SourceLocation, str, str) -> None
"""Add an error about invalid range specifier."""
# pylint: disable=invalid-name
self._add_error(location, ERROR_ID_BAD_NUMERIC_RANGE,
@ -774,21 +773,21 @@ class ParserContext(object):
"Missing 'short_name' for positional arg")
def add_invalid_short_name(self, location, name):
# type: (common.SourceLocation, unicode) -> None
# type: (common.SourceLocation, str) -> None
"""Add an error about invalid short names."""
# pylint: disable=invalid-name
self._add_error(location, ERROR_ID_INVALID_SHORT_NAME,
("Invalid 'short_name' value '%s'") % (name))
def add_invalid_single_name(self, location, name):
# type: (common.SourceLocation, unicode) -> None
# type: (common.SourceLocation, str) -> None
"""Add an error about invalid single names."""
# pylint: disable=invalid-name
self._add_error(location, ERROR_ID_INVALID_SINGLE_NAME,
("Invalid 'single_name' value '%s'") % (name))
def add_missing_short_name_with_single_name(self, location, name):
# type: (common.SourceLocation, unicode) -> None
# type: (common.SourceLocation, str) -> None
"""Add an error about missing required short name when using single name."""
# pylint: disable=invalid-name
self._add_error(location, ERROR_ID_MISSING_SHORT_NAME_WITH_SINGLE_NAME,

337
buildscripts/idl/idl/generator.py
View File

@ -28,8 +28,6 @@
# pylint: disable=too-many-lines
"""IDL C++ Code Generator."""
from __future__ import absolute_import, print_function, unicode_literals
from abc import ABCMeta, abstractmethod
import copy
import io
@ -51,25 +49,25 @@ from . import writer
def _get_field_member_name(field):
# type: (ast.Field) -> unicode
# type: (ast.Field) -> str
"""Get the C++ class member name for a field."""
return '_%s' % (common.camel_case(field.cpp_name))
def _get_field_member_setter_name(field):
# type: (ast.Field) -> unicode
# type: (ast.Field) -> str
"""Get the C++ class setter name for a field."""
return "set%s" % (common.title_case(field.cpp_name))
def _get_field_member_getter_name(field):
# type: (ast.Field) -> unicode
# type: (ast.Field) -> str
"""Get the C++ class getter name for a field."""
return "get%s" % (common.title_case(field.cpp_name))
def _get_has_field_member_name(field):
# type: (ast.Field) -> unicode
# type: (ast.Field) -> str
"""Get the C++ class member name for bool 'has' member field."""
return '_has%s' % (common.title_case(field.cpp_name))
@ -86,20 +84,20 @@ def _is_required_serializer_field(field):
def _get_field_constant_name(field):
# type: (ast.Field) -> unicode
# type: (ast.Field) -> str
"""Get the C++ string constant name for a field."""
return common.template_args('k${constant_name}FieldName', constant_name=common.title_case(
field.cpp_name))
return common.template_args('k${constant_name}FieldName',
constant_name=common.title_case(field.cpp_name))
def _get_field_member_validator_name(field):
# type (ast.Field) -> unicode
# type (ast.Field) -> str
"""Get the name of the validator method for this field."""
return 'validate%s' % common.title_case(field.cpp_name)
def _access_member(field):
# type: (ast.Field) -> unicode
# type: (ast.Field) -> str
"""Get the declaration to access a member for a field."""
member_name = _get_field_member_name(field)
@ -111,7 +109,7 @@ def _access_member(field):
def _get_bson_type_check(bson_element, ctxt_name, field):
# type: (unicode, unicode, ast.Field) -> unicode
# type: (str, str, ast.Field) -> str
"""Get the C++ bson type check for a field."""
bson_types = field.bson_serialization_type
if len(bson_types) == 1:
@ -133,7 +131,7 @@ def _get_bson_type_check(bson_element, ctxt_name, field):
def _get_comparison(field, rel_op, left, right):
# type: (ast.Field, unicode, unicode, unicode) -> unicode
# type: (ast.Field, str, str, str) -> str
"""Generate a comparison for a field."""
name = _get_field_member_name(field)
if not "BSONObj" in field.cpp_type:
@ -166,7 +164,7 @@ def _get_comparison(field, rel_op, left, right):
def _get_comparison_less(fields):
# type: (List[ast.Field]) -> unicode
# type: (List[ast.Field]) -> str
"""Generate a less than comparison for a list of fields recursively."""
field = fields[0]
if len(fields) == 1:
@ -189,11 +187,9 @@ def _get_all_fields(struct):
return sorted([field for field in all_fields], key=lambda f: f.cpp_name)
class _FieldUsageCheckerBase(object):
class _FieldUsageCheckerBase(object, metaclass=ABCMeta):
"""Check for duplicate fields, and required fields as needed."""
__metaclass__ = ABCMeta
def __init__(self, indented_writer):
# type: (writer.IndentedTextWriter) -> None
"""Create a field usage checker."""
@ -202,13 +198,13 @@ class _FieldUsageCheckerBase(object):
@abstractmethod
def add_store(self, field_name):
# type: (unicode) -> None
# type: (str) -> None
"""Create the C++ field store initialization code."""
pass
@abstractmethod
def add(self, field, bson_element_variable):
# type: (ast.Field, unicode) -> None
# type: (ast.Field, str) -> None
"""Add a field to track."""
pass
@ -235,14 +231,14 @@ class _SlowFieldUsageChecker(_FieldUsageCheckerBase):
self._writer.write_line('std::set<StringData> usedFields;')
def add_store(self, field_name):
# type: (unicode) -> None
# type: (str) -> None
self._writer.write_line('auto push_result = usedFields.insert(%s);' % (field_name))
with writer.IndentedScopedBlock(self._writer,
'if (MONGO_unlikely(push_result.second == false)) {', '}'):
self._writer.write_line('ctxt.throwDuplicateField(%s);' % (field_name))
def add(self, field, bson_element_variable):
# type: (ast.Field, unicode) -> None
# type: (ast.Field, str) -> None
if not field in self._fields:
self._fields.append(field)
@ -255,25 +251,25 @@ class _SlowFieldUsageChecker(_FieldUsageCheckerBase):
with writer.IndentedScopedBlock(self._writer, pred, '}'):
if field.default:
if field.enum_type:
self._writer.write_line('%s = %s::%s;' %
(_get_field_member_name(field), field.cpp_type,
field.default))
self._writer.write_line(
'%s = %s::%s;' % (_get_field_member_name(field), field.cpp_type,
field.default))
else:
self._writer.write_line('%s = %s;' % (_get_field_member_name(field),
field.default))
self._writer.write_line(
'%s = %s;' % (_get_field_member_name(field), field.default))
else:
self._writer.write_line('ctxt.throwMissingField(%s);' %
(_get_field_constant_name(field)))
self._writer.write_line(
'ctxt.throwMissingField(%s);' % (_get_field_constant_name(field)))
def _gen_field_usage_constant(field):
# type: (ast.Field) -> unicode
# type: (ast.Field) -> str
"""Get the name for a bitset constant in field usage checking."""
return "k%sBit" % (common.title_case(field.cpp_name))
def _get_constant(name):
# type: (unicode) -> unicode
# type: (str) -> str
"""Transform an arbitrary label to a constant name."""
return 'k' + re.sub(r'([^a-zA-Z0-9_]+)', '_', common.title_case(name))
@ -298,23 +294,24 @@ class _FastFieldUsageChecker(_FieldUsageCheckerBase):
if field.chained:
continue
self._writer.write_line('const size_t %s = %d;' % (_gen_field_usage_constant(field),
bit_id))
self._writer.write_line(
'const size_t %s = %d;' % (_gen_field_usage_constant(field), bit_id))
bit_id += 1
def add_store(self, field_name):
# type: (unicode) -> None
# type: (str) -> None
"""Create the C++ field store initialization code."""
pass
def add(self, field, bson_element_variable):
# type: (ast.Field, unicode) -> None
# type: (ast.Field, str) -> None
"""Add a field to track."""
if not field in self._fields:
self._fields.append(field)
with writer.IndentedScopedBlock(self._writer, 'if (MONGO_unlikely(usedFields[%s])) {' %
(_gen_field_usage_constant(field)), '}'):
with writer.IndentedScopedBlock(
self._writer,
'if (MONGO_unlikely(usedFields[%s])) {' % (_gen_field_usage_constant(field)), '}'):
self._writer.write_line('ctxt.throwDuplicateField(%s);' % (bson_element_variable))
self._writer.write_empty_line()
@ -328,8 +325,9 @@ class _FastFieldUsageChecker(_FieldUsageCheckerBase):
'}'):
for field in self._fields:
if (not field.optional) and (not field.ignore):
with writer.IndentedScopedBlock(self._writer, 'if (!usedFields[%s]) {' %
(_gen_field_usage_constant(field)), '}'):
with writer.IndentedScopedBlock(
self._writer,
'if (!usedFields[%s]) {' % (_gen_field_usage_constant(field)), '}'):
if field.default:
if field.chained_struct_field:
self._writer.write_line(
@ -337,15 +335,15 @@ class _FastFieldUsageChecker(_FieldUsageCheckerBase):
(_get_field_member_name(field.chained_struct_field),
_get_field_member_setter_name(field), field.default))
elif field.enum_type:
self._writer.write_line('%s = %s::%s;' %
(_get_field_member_name(field),
field.cpp_type, field.default))
self._writer.write_line(
'%s = %s::%s;' % (_get_field_member_name(field), field.cpp_type,
field.default))
else:
self._writer.write_line('%s = %s;' % (_get_field_member_name(field),
field.default))
self._writer.write_line(
'%s = %s;' % (_get_field_member_name(field), field.default))
else:
self._writer.write_line('ctxt.throwMissingField(%s);' %
(_get_field_constant_name(field)))
self._writer.write_line(
'ctxt.throwMissingField(%s);' % (_get_field_constant_name(field)))
def _get_field_usage_checker(indented_writer, struct):
@ -361,7 +359,7 @@ def _get_field_usage_checker(indented_writer, struct):
# Turn a python string into a C++ literal.
def _encaps(val):
# type: (unicode) -> unicode
# type: (str) -> str
if val is None:
return '""'
@ -373,7 +371,7 @@ def _encaps(val):
# Turn a list of pything strings into a C++ initializer list.
def _encaps_list(vals):
# type: (List[unicode]) -> unicode
# type: (List[str]) -> str
if vals is None:
return '{}'
@ -382,7 +380,7 @@ def _encaps_list(vals):
# Translate an ast.Expression into C++ code.
def _get_expression(expr):
# type: (ast.Expression) -> unicode
# type: (ast.Expression) -> str
if not expr.validate_constexpr:
return expr.expr
@ -405,7 +403,7 @@ class _CppFileWriterBase(object):
self._writer = indented_writer # type: writer.IndentedTextWriter
def write_unindented_line(self, msg):
# type: (unicode) -> None
# type: (str) -> None
"""Write an unindented line to the stream."""
self._writer.write_unindented_line(msg)
@ -427,24 +425,24 @@ class _CppFileWriterBase(object):
""" % (" ".join(sys.argv))))
def gen_system_include(self, include):
# type: (unicode) -> None
# type: (str) -> None
"""Generate a system C++ include line."""
self._writer.write_unindented_line('#include <%s>' % (include))
def gen_include(self, include):
# type: (unicode) -> None
# type: (str) -> None
"""Generate a non-system C++ include line."""
self._writer.write_unindented_line('#include "%s"' % (include))
def gen_namespace_block(self, namespace):
# type: (unicode) -> writer.NamespaceScopeBlock
# type: (str) -> writer.NamespaceScopeBlock
"""Generate a namespace block."""
namespace_list = namespace.split("::")
return writer.NamespaceScopeBlock(self._writer, namespace_list)
def get_initializer_lambda(self, decl, unused=False, return_type=None):
# type: (unicode, bool, unicode) -> writer.IndentedScopedBlock
# type: (str, bool, str) -> writer.IndentedScopedBlock
"""Generate an indented block lambda initializing an outer scope variable."""
prefix = 'MONGO_COMPILER_VARIABLE_UNUSED ' if unused else ''
prefix = prefix + decl + ' = ([]'
@ -453,7 +451,7 @@ class _CppFileWriterBase(object):
return writer.IndentedScopedBlock(self._writer, prefix + ' {', '})();')
def gen_description_comment(self, description):
# type: (unicode) -> None
# type: (str) -> None
"""Generate a multiline comment with the description from the IDL."""
self._writer.write_line(
textwrap.dedent("""\
@ -462,12 +460,12 @@ class _CppFileWriterBase(object):
*/""" % (description)))
def _with_template(self, template_params):
# type: (Mapping[unicode,unicode]) -> writer.TemplateContext
# type: (Mapping[str,str]) -> writer.TemplateContext
"""Generate a template context for the current parameters."""
return writer.TemplateContext(self._writer, template_params)
def _block(self, opening, closing):
# type: (unicode, unicode) -> Union[writer.IndentedScopedBlock,writer.EmptyBlock]
# type: (str, str) -> Union[writer.IndentedScopedBlock,writer.EmptyBlock]
"""Generate an indented block if opening is not empty."""
if not opening:
return writer.EmptyBlock()
@ -475,7 +473,7 @@ class _CppFileWriterBase(object):
return writer.IndentedScopedBlock(self._writer, opening, closing)
def _predicate(self, check_str, use_else_if=False, constexpr=False):
# type: (unicode, bool, bool) -> Union[writer.IndentedScopedBlock,writer.EmptyBlock]
# type: (str, bool, bool) -> Union[writer.IndentedScopedBlock,writer.EmptyBlock]
"""
Generate an if block if the condition is not-empty.
@ -524,7 +522,7 @@ class _CppHeaderFileWriter(_CppFileWriterBase):
"""C++ .h File writer."""
def gen_class_declaration_block(self, class_name):
# type: (unicode) -> writer.IndentedScopedBlock
# type: (str) -> writer.IndentedScopedBlock
"""Generate a class declaration block."""
return writer.IndentedScopedBlock(self._writer,
'class %s {' % common.title_case(class_name), '};')
@ -604,9 +602,9 @@ class _CppHeaderFileWriter(_CppFileWriterBase):
if field.chained_struct_field:
self._writer.write_template(
'${const_type} ${param_type} ${method_name}() const { return %s.%s(); }' %
((_get_field_member_name(field.chained_struct_field),
_get_field_member_getter_name(field))))
'${const_type} ${param_type} ${method_name}() const { return %s.%s(); }' % (
(_get_field_member_name(field.chained_struct_field),
_get_field_member_getter_name(field))))
elif cpp_type_info.disable_xvalue():
self._writer.write_template(
@ -684,8 +682,8 @@ class _CppHeaderFileWriter(_CppFileWriterBase):
if field.default and not field.constructed:
if field.enum_type:
self._writer.write_line('%s %s{%s::%s};' % (member_type, member_name,
field.cpp_type, field.default))
self._writer.write_line(
'%s %s{%s::%s};' % (member_type, member_name, field.cpp_type, field.default))
else:
self._writer.write_line('%s %s{%s};' % (member_type, member_name, field.default))
else:
@ -811,19 +809,19 @@ class _CppHeaderFileWriter(_CppFileWriterBase):
self.write_empty_line()
def _gen_exported_constexpr(self, name, suffix, expr, condition):
# type: (unicode, unicode, ast.Expression, ast.Condition) -> None
# type: (str, str, ast.Expression, ast.Condition) -> None
"""Generate exports for default initializer."""
if not (name and expr and expr.export):
return
with self._condition(condition, preprocessor_only=True):
self._writer.write_line('constexpr auto %s%s = %s;' % (_get_constant(name), suffix,
expr.expr))
self._writer.write_line(
'constexpr auto %s%s = %s;' % (_get_constant(name), suffix, expr.expr))
self.write_empty_line()
def _gen_extern_declaration(self, vartype, varname, condition):
# type: (unicode, unicode, ast.Condition) -> None
# type: (str, str, ast.Condition) -> None
"""Generate externs for storage declaration."""
if (vartype is None) or (varname is None):
return
@ -851,11 +849,11 @@ class _CppHeaderFileWriter(_CppFileWriterBase):
return
if initializer.register:
self._writer.write_line('Status %s(optionenvironment::OptionSection*);' %
(initializer.register))
self._writer.write_line(
'Status %s(optionenvironment::OptionSection*);' % (initializer.register))
if initializer.store:
self._writer.write_line('Status %s(const optionenvironment::Environment&);' %
(initializer.store))
self._writer.write_line(
'Status %s(const optionenvironment::Environment&);' % (initializer.store))
if initializer.register or initializer.store:
self.write_empty_line()
@ -871,8 +869,8 @@ class _CppHeaderFileWriter(_CppFileWriterBase):
with self._block('class %s : public ServerParameter {' % (cls.name), '};'):
self._writer.write_unindented_line('public:')
if scp.default is not None:
self._writer.write_line('static constexpr auto kDataDefault = %s;' %
(scp.default.expr))
self._writer.write_line(
'static constexpr auto kDataDefault = %s;' % (scp.default.expr))
if cls.override_ctor:
# Explicit custom constructor.
@ -1048,13 +1046,13 @@ class _CppSourceFileWriter(_CppFileWriterBase):
"""C++ .cpp File writer."""
def __init__(self, indented_writer, target_arch):
# type: (writer.IndentedTextWriter, unicode) -> None
# type: (writer.IndentedTextWriter, str) -> None
"""Create a C++ .cpp file code writer."""
self._target_arch = target_arch
super(_CppSourceFileWriter, self).__init__(indented_writer)
def _gen_field_deserializer_expression(self, element_name, field):
# type: (unicode, ast.Field) -> unicode
# type: (str, ast.Field) -> str
# pylint: disable=invalid-name
"""
Generate the C++ deserializer piece for a field.
@ -1064,8 +1062,8 @@ class _CppSourceFileWriter(_CppFileWriterBase):
"""
if field.struct_type:
self._writer.write_line('IDLParserErrorContext tempContext(%s, &ctxt);' %
(_get_field_constant_name(field)))
self._writer.write_line(
'IDLParserErrorContext tempContext(%s, &ctxt);' % (_get_field_constant_name(field)))
self._writer.write_line('const auto localObject = %s.Obj();' % (element_name))
return '%s::parse(tempContext, localObject)' % (common.title_case(field.struct_type))
elif field.deserializer and 'BSONElement::' in field.deserializer:
@ -1104,14 +1102,14 @@ class _CppSourceFileWriter(_CppFileWriterBase):
return '%s(%s)' % (method_name, element_name)
def _gen_array_deserializer(self, field, bson_element):
# type: (ast.Field, unicode) -> None
# type: (ast.Field, str) -> None
"""Generate the C++ deserializer piece for an array field."""
cpp_type_info = cpp_types.get_cpp_type(field)
cpp_type = cpp_type_info.get_type_name()
self._writer.write_line('std::uint32_t expectedFieldNumber{0};')
self._writer.write_line('const IDLParserErrorContext arrayCtxt(%s, &ctxt);' %
(_get_field_constant_name(field)))
self._writer.write_line(
'const IDLParserErrorContext arrayCtxt(%s, &ctxt);' % (_get_field_constant_name(field)))
self._writer.write_line('std::vector<%s> values;' % (cpp_type))
self._writer.write_empty_line()
@ -1145,14 +1143,13 @@ class _CppSourceFileWriter(_CppFileWriterBase):
self._writer.write_line('++expectedFieldNumber;')
if field.chained_struct_field:
self._writer.write_line('%s.%s(std::move(values));' %
(_get_field_member_name(field.chained_struct_field),
_get_field_member_setter_name(field)))
self._writer.write_line('%s.%s(std::move(values));' % (_get_field_member_name(
field.chained_struct_field), _get_field_member_setter_name(field)))
else:
self._writer.write_line('%s = std::move(values);' % (_get_field_member_name(field)))
def _gen_usage_check(self, field, bson_element, field_usage_check):
# type: (ast.Field, unicode, _FieldUsageCheckerBase) -> None
# type: (ast.Field, str, _FieldUsageCheckerBase) -> None
"""Generate the field usage check and insert the required field check."""
if field_usage_check:
field_usage_check.add(field, bson_element)
@ -1161,7 +1158,7 @@ class _CppSourceFileWriter(_CppFileWriterBase):
self._writer.write_line('%s = true;' % (_get_has_field_member_name(field)))
def gen_field_deserializer(self, field, bson_object, bson_element, field_usage_check):
# type: (ast.Field, unicode, unicode, _FieldUsageCheckerBase) -> None
# type: (ast.Field, str, str, _FieldUsageCheckerBase) -> None
"""Generate the C++ deserializer piece for a field."""
if field.array:
self._gen_usage_check(field, bson_element, field_usage_check)
@ -1170,7 +1167,7 @@ class _CppSourceFileWriter(_CppFileWriterBase):
return
def validate_and_assign_or_uassert(field, expression):
# type: (ast.Field, unicode) -> None
# type: (ast.Field, str) -> None
"""Perform field value validation post-assignment."""
field_name = _get_field_member_name(field)
if field.validator is None:
@ -1208,9 +1205,9 @@ class _CppSourceFileWriter(_CppFileWriterBase):
object_value = self._gen_field_deserializer_expression(bson_element, field)
if field.chained_struct_field:
# No need for explicit validation as setter will throw for us.
self._writer.write_line('%s.%s(%s);' %
(_get_field_member_name(field.chained_struct_field),
_get_field_member_setter_name(field), object_value))
self._writer.write_line(
'%s.%s(%s);' % (_get_field_member_name(field.chained_struct_field),
_get_field_member_setter_name(field), object_value))
else:
validate_and_assign_or_uassert(field, object_value)
@ -1233,8 +1230,8 @@ class _CppSourceFileWriter(_CppFileWriterBase):
if field.struct_type:
self._writer.write_line('IDLParserErrorContext tempContext(%s, &ctxt);' %
(_get_field_constant_name(field)))
array_value = '%s::parse(tempContext, sequenceObject)' % (
common.title_case(field.struct_type))
array_value = '%s::parse(tempContext, sequenceObject)' % (common.title_case(
field.struct_type))
else:
assert field.bson_serialization_type == ['object']
if field.deserializer:
@ -1277,9 +1274,9 @@ class _CppSourceFileWriter(_CppFileWriterBase):
field.cpp_type)
if _is_required_serializer_field(field) and needs_init:
initializers.append(
'%s(%s)' %
(_get_field_member_name(field),
cpp_types.get_primitive_scalar_type_default_value(field.cpp_type)))
'%s(%s)' % (_get_field_member_name(field),
cpp_types.get_primitive_scalar_type_default_value(
field.cpp_type)))
# Serialize the _dbName field second
initializes_db_name = False
@ -1329,7 +1326,7 @@ class _CppSourceFileWriter(_CppFileWriterBase):
self._gen_constructor(struct, required_constructor, False)
def _gen_command_deserializer(self, struct, bson_object):
# type: (ast.Struct, unicode) -> None
# type: (ast.Struct, str) -> None
"""Generate the command field deserializer."""
if isinstance(struct, ast.Command) and struct.command_field:
@ -1343,7 +1340,7 @@ class _CppSourceFileWriter(_CppFileWriterBase):
struct_type_info.gen_namespace_check(self._writer, "_dbName", "commandElement")
def _gen_fields_deserializer_common(self, struct, bson_object):
# type: (ast.Struct, unicode) -> _FieldUsageCheckerBase
# type: (ast.Struct, str) -> _FieldUsageCheckerBase
"""Generate the C++ code to deserialize list of fields."""
# pylint: disable=too-many-branches
field_usage_check = _get_field_usage_checker(self._writer, struct)
@ -1439,15 +1436,15 @@ class _CppSourceFileWriter(_CppFileWriterBase):
cpp_types.get_primitive_scalar_type_default_value(
struct.command_field.cpp_type)))
else:
self._writer.write_line('%s localCmdType;' %
(cpp_type_info.get_storage_type()))
self._writer.write_line('%s object(localCmdType);' %
(common.title_case(struct.cpp_name)))
self._writer.write_line(
'%s localCmdType;' % (cpp_type_info.get_storage_type()))
self._writer.write_line(
'%s object(localCmdType);' % (common.title_case(struct.cpp_name)))
elif struct.namespace in (common.COMMAND_NAMESPACE_CONCATENATE_WITH_DB,
common.COMMAND_NAMESPACE_CONCATENATE_WITH_DB_OR_UUID):
self._writer.write_line('NamespaceString localNS;')
self._writer.write_line('%s object(localNS);' %
(common.title_case(struct.cpp_name)))
self._writer.write_line(
'%s object(localNS);' % (common.title_case(struct.cpp_name)))
else:
assert "Missing case"
else:
@ -1457,15 +1454,15 @@ class _CppSourceFileWriter(_CppFileWriterBase):
self._writer.write_line('return object;')
def _compare_and_return_status(self, op, limit, field, optional_param):
# type: (unicode, ast.Expression, ast.Field, unicode) -> None
# type: (str, ast.Expression, ast.Field, str) -> None
"""Throw an error on comparison failure."""
with self._block('if (!(value %s %s)) {' % (op, _get_expression(limit)), '}'):
self._writer.write_line('throwComparisonError<%s>(%s"%s", "%s"_sd, value, %s);' %
(field.cpp_type, optional_param, field.name, op,
_get_expression(limit)))
self._writer.write_line(
'throwComparisonError<%s>(%s"%s", "%s"_sd, value, %s);' %
(field.cpp_type, optional_param, field.name, op, _get_expression(limit)))
def _gen_field_validator(self, struct, field, optional_params):
# type: (ast.Struct, ast.Field, Tuple[unicode, unicode]) -> None
# type: (ast.Struct, ast.Field, Tuple[str, str]) -> None
"""Generate non-trivial field validators."""
validator = field.validator
@ -1570,8 +1567,8 @@ class _CppSourceFileWriter(_CppFileWriterBase):
field_usage_check.add(field, "sequence.name")
if _is_required_serializer_field(field):
self._writer.write_line('%s = true;' %
(_get_has_field_member_name(field)))
self._writer.write_line(
'%s = true;' % (_get_has_field_member_name(field)))
self.gen_doc_sequence_deserializer(field)
@ -1809,8 +1806,8 @@ class _CppSourceFileWriter(_CppFileWriterBase):
with self._block(optional_block_start, '}'):
self._writer.write_line('OpMsg::DocumentSequence documentSequence;')
self._writer.write_template('documentSequence.name = %s.toString();' %
(_get_field_constant_name(field)))
self._writer.write_template(
'documentSequence.name = %s.toString();' % (_get_field_constant_name(field)))
with self._block('for (const auto& item : %s) {' % (_access_member(field)), '}'):
@ -1839,9 +1836,9 @@ class _CppSourceFileWriter(_CppFileWriterBase):
struct_type_info = struct_types.get_struct_info(struct)
with self._block('%s {' %
(struct_type_info.get_op_msg_request_serializer_method().get_definition()),
'}'):
with self._block(
'%s {' % (struct_type_info.get_op_msg_request_serializer_method().get_definition()),
'}'):
self._writer.write_line('BSONObjBuilder localBuilder;')
with self._block('{', '}'):
@ -1884,7 +1881,7 @@ class _CppSourceFileWriter(_CppFileWriterBase):
self._writer.write_empty_line()
def _gen_known_fields_declaration(self, struct, name, include_op_msg_implicit):
# type: (ast.Struct, unicode, bool) -> None
# type: (ast.Struct, str, bool) -> None
"""Generate the known fields declaration with specified name."""
block_name = common.template_args(
'const std::vector<StringData> ${class_name}::_${name}Fields {', name=name,
@ -1897,13 +1894,13 @@ class _CppSourceFileWriter(_CppFileWriterBase):
for field in sorted_fields:
self._writer.write_line(
common.template_args(
'${class_name}::${constant_name},', class_name=common.title_case(
struct.cpp_name), constant_name=_get_field_constant_name(field)))
common.template_args('${class_name}::${constant_name},',
class_name=common.title_case(struct.cpp_name),
constant_name=_get_field_constant_name(field)))
self._writer.write_line(
common.template_args('${class_name}::kCommandName,', class_name=common.title_case(
struct.cpp_name)))
common.template_args('${class_name}::kCommandName,',
class_name=common.title_case(struct.cpp_name)))
def gen_known_fields_declaration(self, struct):
# type: (ast.Struct) -> None
@ -1917,8 +1914,8 @@ class _CppSourceFileWriter(_CppFileWriterBase):
def _gen_server_parameter_specialized(self, param):
# type: (ast.ServerParameter) -> None
"""Generate a specialized ServerParameter."""
self._writer.write_line('return new %s(%s, %s);' % (param.cpp_class.name,
_encaps(param.name), param.set_at))
self._writer.write_line(
'return new %s(%s, %s);' % (param.cpp_class.name, _encaps(param.name), param.set_at))
def _gen_server_parameter_class_definitions(self, param):
# type: (ast.ServerParameter) -> None
@ -1929,8 +1926,8 @@ class _CppSourceFileWriter(_CppFileWriterBase):
self.gen_description_comment("%s: %s" % (param.name, param.description))
if param.default:
self._writer.write_line('constexpr decltype(%s::kDataDefault) %s::kDataDefault;' %
(cls.name, cls.name))
self._writer.write_line(
'constexpr decltype(%s::kDataDefault) %s::kDataDefault;' % (cls.name, cls.name))
self.write_empty_line()
if param.redact:
@ -1977,8 +1974,8 @@ class _CppSourceFileWriter(_CppFileWriterBase):
if param.default and not (param.cpp_vartype and param.cpp_varname):
# Only need to call setValue() if we haven't in-place initialized the declared var.
self._writer.write_line('uassertStatusOK(ret->setValue(%s));' %
(_get_expression(param.default)))
self._writer.write_line(
'uassertStatusOK(ret->setValue(%s));' % (_get_expression(param.default)))
self._writer.write_line('return ret;')
@ -1998,11 +1995,12 @@ class _CppSourceFileWriter(_CppFileWriterBase):
self._writer.write_line(
common.template_args(
'${unused} auto* ${alias_var} = new IDLServerParameterDeprecatedAlias(${name}, ${param_var});',
unused='MONGO_COMPILER_VARIABLE_UNUSED', alias_var='scp_%d_%d' %
(param_no, alias_no), name=_encaps(alias), param_var='scp_%d' % (param_no)))
unused='MONGO_COMPILER_VARIABLE_UNUSED',
alias_var='scp_%d_%d' % (param_no, alias_no), name=_encaps(alias),
param_var='scp_%d' % (param_no)))
def gen_server_parameters(self, params, header_file_name):
# type: (List[ast.ServerParameter], unicode) -> None
# type: (List[ast.ServerParameter], str) -> None
"""Generate IDLServerParameter instances."""
for param in params:
@ -2014,10 +2012,10 @@ class _CppSourceFileWriter(_CppFileWriterBase):
elif (param.cpp_vartype is not None) and (param.cpp_varname is not None):
with self._condition(param.condition, preprocessor_only=True):
init = ('{%s}' % (param.default.expr)) if param.default else ''
self._writer.write_line('%s %s%s;' % (param.cpp_vartype, param.cpp_varname,
init))
self._writer.write_line(
'%s %s%s;' % (param.cpp_vartype, param.cpp_varname, init))
blockname = 'idl_' + hashlib.sha1(header_file_name).hexdigest()
blockname = 'idl_' + hashlib.sha1(header_file_name.encode()).hexdigest()
with self._block('MONGO_SERVER_PARAMETER_REGISTER(%s)(InitializerContext*) {' % (blockname),
'}'):
# ServerParameter instances.
@ -2038,7 +2036,7 @@ class _CppSourceFileWriter(_CppFileWriterBase):
self._writer.write_line('return Status::OK();')
def gen_config_option(self, opt, section):
# type: (ast.ConfigOption, unicode) -> None
# type: (ast.ConfigOption, str) -> None
"""Generate Config Option instance."""
# Derive cpp_vartype from arg_vartype if needed.
@ -2072,16 +2070,16 @@ class _CppSourceFileWriter(_CppFileWriterBase):
for conflicts in opt.conflicts:
self._writer.write_line('.incompatibleWith(%s)' % (_encaps(conflicts)))
if opt.default:
self._writer.write_line('.setDefault(moe::Value(%s))' %
(_get_expression(opt.default)))
self._writer.write_line(
'.setDefault(moe::Value(%s))' % (_get_expression(opt.default)))
if opt.implicit:
self._writer.write_line('.setImplicit(moe::Value(%s))' %
(_get_expression(opt.implicit)))
self._writer.write_line(
'.setImplicit(moe::Value(%s))' % (_get_expression(opt.implicit)))
if opt.duplicates_append:
self._writer.write_line('.composing()')
if (opt.positional_start is not None) and (opt.positional_end is not None):
self._writer.write_line('.positional(%d, %d)' % (opt.positional_start,
opt.positional_end))
self._writer.write_line(
'.positional(%d, %d)' % (opt.positional_start, opt.positional_end))
if opt.canonicalize:
self._writer.write_line('.canonicalize(%s)' % opt.canonicalize)
@ -2090,8 +2088,8 @@ class _CppSourceFileWriter(_CppFileWriterBase):
self._writer.write_line(
common.template_args(
'.addConstraint(new moe::CallbackKeyConstraint<${argtype}>(${key}, ${callback}))',
argtype=vartype, key=_encaps(
opt.name), callback=opt.validator.callback))
argtype=vartype, key=_encaps(opt.name),
callback=opt.validator.callback))
if (opt.validator.gt is not None) or (opt.validator.lt is not None) or (
opt.validator.gte is not None) or (opt.validator.lte is not None):
@ -2100,12 +2098,11 @@ class _CppSourceFileWriter(_CppFileWriterBase):
'.addConstraint(new moe::BoundaryKeyConstraint<${argtype}>(${key}, ${gt}, ${lt}, ${gte}, ${lte}))',
argtype=vartype, key=_encaps(opt.name), gt='boost::none'
if opt.validator.gt is None else _get_expression(opt.validator.gt),
lt='boost::none' if opt.validator.lt is None else _get_expression(
opt.validator.lt), gte='boost::none'
if opt.validator.gte is None else _get_expression(
opt.validator.gte), lte='boost::none'
if opt.validator.lte is None else _get_expression(
opt.validator.lte)))
lt='boost::none'
if opt.validator.lt is None else _get_expression(opt.validator.lt),
gte='boost::none' if opt.validator.gte is None else _get_expression(
opt.validator.gte), lte='boost::none' if
opt.validator.lte is None else _get_expression(opt.validator.lte)))
self.write_empty_line()
@ -2116,7 +2113,7 @@ class _CppSourceFileWriter(_CppFileWriterBase):
for opt in root_opts:
self.gen_config_option(opt, 'options')
for section_name, section_opts in sections.iteritems():
for section_name, section_opts in sections.items():
with self._block('{', '}'):
self._writer.write_line('moe::OptionSection section(%s);' % (_encaps(section_name)))
self.write_empty_line()
@ -2144,15 +2141,14 @@ class _CppSourceFileWriter(_CppFileWriterBase):
(opt.arg_vartype)) if opt.cpp_vartype is None else opt.cpp_vartype
with self._condition(opt.condition):
with self._block('if (params.count(%s)) {' % (_encaps(opt.name)), '}'):
self._writer.write_line('%s = params[%s].as<%s>();' % (opt.cpp_varname,
_encaps(opt.name),
vartype))
self._writer.write_line(
'%s = params[%s].as<%s>();' % (opt.cpp_varname, _encaps(opt.name), vartype))
self.write_empty_line()
self._writer.write_line('return Status::OK();')
def gen_config_options(self, spec, header_file_name):
# type: (ast.IDLAST, unicode) -> None
# type: (ast.IDLAST, str) -> None
"""Generate Config Option instances."""
# pylint: disable=too-many-branches,too-many-statements
@ -2164,13 +2160,13 @@ class _CppSourceFileWriter(_CppFileWriterBase):
if opt.cpp_vartype is not None:
with self._condition(opt.condition, preprocessor_only=True):
init = ('{%s}' % (opt.default.expr)) if opt.default else ''
self._writer.write_line('%s %s%s;' % (opt.cpp_vartype, opt.cpp_varname,
init))
self._writer.write_line(
'%s %s%s;' % (opt.cpp_vartype, opt.cpp_varname, init))
self.write_empty_line()
root_opts = [] # type: List[ast.ConfigOption]
sections = {} # type: Dict[unicode, List[ast.ConfigOption]]
sections = {} # type: Dict[str, List[ast.ConfigOption]]
for opt in spec.configs:
if opt.section:
try:
@ -2183,12 +2179,13 @@ class _CppSourceFileWriter(_CppFileWriterBase):
initializer = spec.globals.configs and spec.globals.configs.initializer
# pylint: disable=consider-using-ternary
blockname = (initializer
and initializer.name) or ('idl_' + hashlib.sha1(header_file_name).hexdigest())
blockname = (initializer and initializer.name) or (
'idl_' + hashlib.sha1(header_file_name.encode()).hexdigest())
if initializer and initializer.register:
with self._block('Status %s(optionenvironment::OptionSection* options_ptr) {' %
initializer.register, '}'):
with self._block(
'Status %s(optionenvironment::OptionSection* options_ptr) {' %
initializer.register, '}'):
self._writer.write_line('auto& options = *options_ptr;')
self._gen_config_options_register(root_opts, sections)
else:
@ -2203,13 +2200,15 @@ class _CppSourceFileWriter(_CppFileWriterBase):
if has_storage_targets:
if initializer and initializer.store:
with self._block('Status %s(const optionenvironment::Environment& params) {' %
initializer.store, '}'):
with self._block(
'Status %s(const optionenvironment::Environment& params) {' %
initializer.store, '}'):
self._gen_config_options_store(spec.configs)
else:
with self.gen_namespace_block(''):
with self._block('MONGO_STARTUP_OPTIONS_STORE(%s)(InitializerContext*) {' %
(blockname), '}'):
with self._block(
'MONGO_STARTUP_OPTIONS_STORE(%s)(InitializerContext*) {' % (blockname),
'}'):
# If all options are guarded by non-passing #ifdefs, then params will be unused.
self._writer.write_line(
'MONGO_COMPILER_VARIABLE_UNUSED const auto& params = optionenvironment::startupOptionsParsed;'
@ -2219,7 +2218,7 @@ class _CppSourceFileWriter(_CppFileWriterBase):
self.write_empty_line()
def generate(self, spec, header_file_name):
# type: (ast.IDLAST, unicode) -> None
# type: (ast.IDLAST, str) -> None
"""Generate the C++ header to a stream."""
self.gen_file_header()
@ -2315,7 +2314,7 @@ class _CppSourceFileWriter(_CppFileWriterBase):
def generate_header_str(spec):
# type: (ast.IDLAST) -> unicode
# type: (ast.IDLAST) -> str
"""Generate a C++ header in-memory."""
stream = io.StringIO()
text_writer = writer.IndentedTextWriter(stream)
@ -2328,7 +2327,7 @@ def generate_header_str(spec):
def _generate_header(spec, file_name):
# type: (ast.IDLAST, unicode) -> None
# type: (ast.IDLAST, str) -> None
"""Generate a C++ header."""
str_value = generate_header_str(spec)
@ -2339,7 +2338,7 @@ def _generate_header(spec, file_name):
def generate_source_str(spec, target_arch, header_file_name):
# type: (ast.IDLAST, unicode, unicode) -> unicode
# type: (ast.IDLAST, str, str) -> str
"""Generate a C++ source file in-memory."""
stream = io.StringIO()
text_writer = writer.IndentedTextWriter(stream)
@ -2352,7 +2351,7 @@ def generate_source_str(spec, target_arch, header_file_name):
def _generate_source(spec, target_arch, file_name, header_file_name):
# type: (ast.IDLAST, unicode, unicode, unicode) -> None
# type: (ast.IDLAST, str, str, str) -> None
"""Generate a C++ source file."""
str_value = generate_source_str(spec, target_arch, header_file_name)
@ -2362,7 +2361,7 @@ def _generate_source(spec, target_arch, file_name, header_file_name):
def generate_code(spec, target_arch, output_base_dir, header_file_name, source_file_name):
# type: (ast.IDLAST, unicode, unicode, unicode, unicode) -> None
# type: (ast.IDLAST, str, str, str, str) -> None
"""Generate a C++ header and source file from an idl.ast tree."""
_generate_header(spec, header_file_name)

57
buildscripts/idl/idl/parser.py
View File

@ -31,7 +31,6 @@ IDL Parser.
Converts a YAML document to an idl.syntax tree.
Only validates the document is syntatically correct, not semantically.
"""
from __future__ import absolute_import, print_function, unicode_literals
from abc import ABCMeta, abstractmethod
import io
@ -64,11 +63,11 @@ class _RuleDesc(object):
OPTIONAL = 2
def __init__(self, node_type, required=OPTIONAL, mapping_parser_func=None):
# type: (unicode, int, Callable[[errors.ParserContext,yaml.nodes.MappingNode], Any]) -> None
# type: (str, int, Callable[[errors.ParserContext,yaml.nodes.MappingNode], Any]) -> None
"""Construct a parser rule description."""
assert required == _RuleDesc.REQUIRED or required == _RuleDesc.OPTIONAL
assert required in (_RuleDesc.REQUIRED, _RuleDesc.OPTIONAL)
self.node_type = node_type # type: unicode
self.node_type = node_type # type: str
self.required = required # type: int
self.mapping_parser_func = mapping_parser_func # type: Callable[[errors.ParserContext,yaml.nodes.MappingNode], Any]
@ -76,9 +75,9 @@ class _RuleDesc(object):
def _generic_parser(
ctxt, # type: errors.ParserContext
node, # type: Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]
syntax_node_name, # type: unicode
syntax_node_name, # type: str
syntax_node, # type: Any
mapping_rules # type: Dict[unicode, _RuleDesc]
mapping_rules # type: Dict[str, _RuleDesc]
): # type: (...) -> None
# pylint: disable=too-many-branches
field_name_set = set() # type: Set[str]
@ -118,15 +117,15 @@ def _generic_parser(
syntax_node.__dict__[first_name] = rule_desc.mapping_parser_func(
ctxt, second_node)
else:
raise errors.IDLError("Unknown node_type '%s' for parser rule" %
(rule_desc.node_type))
raise errors.IDLError(
"Unknown node_type '%s' for parser rule" % (rule_desc.node_type))
else:
ctxt.add_unknown_node_error(first_node, syntax_node_name)
field_name_set.add(first_name)
# Check for any missing required fields
for name, rule_desc in mapping_rules.items():
for name, rule_desc in list(mapping_rules.items()):
if not rule_desc.required == _RuleDesc.REQUIRED:
continue
@ -137,16 +136,16 @@ def _generic_parser(
if syntax_node.__dict__[name] is None:
ctxt.add_missing_required_field_error(node, syntax_node_name, name)
else:
raise errors.IDLError("Unknown node_type '%s' for parser required rule" %
(rule_desc.node_type))
raise errors.IDLError(
"Unknown node_type '%s' for parser required rule" % (rule_desc.node_type))
def _parse_mapping(
ctxt, # type: errors.ParserContext
spec, # type: syntax.IDLSpec
node, # type: Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]
syntax_node_name, # type: unicode
func # type: Callable[[errors.ParserContext,syntax.IDLSpec,unicode,Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]], None]
syntax_node_name, # type: str
func # type: Callable[[errors.ParserContext,syntax.IDLSpec,str,Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]], None]
): # type: (...) -> None
"""Parse a top-level mapping section in the IDL file."""
if not ctxt.is_mapping_node(node, syntax_node_name):
@ -220,7 +219,7 @@ def _parse_imports(ctxt, spec, node):
def _parse_type(ctxt, spec, name, node):
# type: (errors.ParserContext, syntax.IDLSpec, unicode, Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> None
# type: (errors.ParserContext, syntax.IDLSpec, str, Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> None
"""Parse a type section in the IDL file."""
if not ctxt.is_mapping_node(node, "type"):
return
@ -431,7 +430,7 @@ def _parse_chained_structs(ctxt, node):
def _parse_struct(ctxt, spec, name, node):
# type: (errors.ParserContext, syntax.IDLSpec, unicode, Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> None
# type: (errors.ParserContext, syntax.IDLSpec, str, Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> None
"""Parse a struct section in the IDL file."""
if not ctxt.is_mapping_node(node, "struct"):
return
@ -488,7 +487,7 @@ def _parse_enum_values(ctxt, node):
def _parse_enum(ctxt, spec, name, node):
# type: (errors.ParserContext, syntax.IDLSpec, unicode, Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> None
# type: (errors.ParserContext, syntax.IDLSpec, str, Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> None
"""Parse an enum section in the IDL file."""
if not ctxt.is_mapping_node(node, "struct"):
return
@ -510,7 +509,7 @@ def _parse_enum(ctxt, spec, name, node):
def _parse_command(ctxt, spec, name, node):
# type: (errors.ParserContext, syntax.IDLSpec, unicode, Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> None
# type: (errors.ParserContext, syntax.IDLSpec, str, Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> None
"""Parse a command section in the IDL file."""
if not ctxt.is_mapping_node(node, "command"):
return
@ -578,7 +577,7 @@ def _parse_server_parameter_class(ctxt, node):
def _parse_server_parameter(ctxt, spec, name, node):
# type: (errors.ParserContext, syntax.IDLSpec, unicode, Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> None
# type: (errors.ParserContext, syntax.IDLSpec, str, Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> None
"""Parse a server_parameters section in the IDL file."""
if not ctxt.is_mapping_node(node, "server_parameters"):
return
@ -609,7 +608,7 @@ def _parse_server_parameter(ctxt, spec, name, node):
def _parse_config_option(ctxt, spec, name, node):
# type: (errors.ParserContext, syntax.IDLSpec, unicode, Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> None
# type: (errors.ParserContext, syntax.IDLSpec, str, Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> None
"""Parse a configs section in the IDL file."""
if not ctxt.is_mapping_node(node, "configs"):
return
@ -646,7 +645,7 @@ def _parse_config_option(ctxt, spec, name, node):
def _prefix_with_namespace(cpp_namespace, cpp_name):
# type: (unicode, unicode) -> unicode
# type: (str, str) -> str
"""Preface a C++ type name with a namespace if not already qualified or a primitive type."""
if "::" in cpp_name or cpp_types.is_primitive_scalar_type(cpp_name):
return cpp_name
@ -676,7 +675,7 @@ def _propagate_globals(spec):
def _parse(stream, error_file_name):
# type: (Any, unicode) -> syntax.IDLParsedSpec
# type: (Any, str) -> syntax.IDLParsedSpec
"""
Parse a YAML document into an idl.syntax tree.
@ -740,11 +739,9 @@ def _parse(stream, error_file_name):
return syntax.IDLParsedSpec(spec, None)
class ImportResolverBase(object):
class ImportResolverBase(object, metaclass=ABCMeta):
"""Base class for resolving imported files."""
__metaclass__ = ABCMeta
def __init__(self):
# type: () -> None
"""Construct a ImportResolver."""
@ -752,19 +749,19 @@ class ImportResolverBase(object):
@abstractmethod
def resolve(self, base_file, imported_file_name):
# type: (unicode, unicode) -> unicode
# type: (str, str) -> str
"""Return the complete path to an imported file name."""
pass
@abstractmethod
def open(self, resolved_file_name):
# type: (unicode) -> Any
# type: (str) -> Any
"""Return an io.Stream for the requested file."""
pass
def parse(stream, input_file_name, resolver):
# type: (Any, unicode, ImportResolverBase) -> syntax.IDLParsedSpec
# type: (Any, str, ImportResolverBase) -> syntax.IDLParsedSpec
"""
Parse a YAML document into an idl.syntax tree.
@ -778,13 +775,13 @@ def parse(stream, input_file_name, resolver):
if root_doc.errors:
return root_doc
imports = [] # type: List[Tuple[common.SourceLocation, unicode, unicode]]
needs_include = [] # type: List[unicode]
imports = [] # type: List[Tuple[common.SourceLocation, str, str]]
needs_include = [] # type: List[str]
if root_doc.spec.imports:
imports = [(root_doc.spec.imports, input_file_name, import_file_name)
for import_file_name in root_doc.spec.imports.imports]
resolved_file_names = [] # type: List[unicode]
resolved_file_names = [] # type: List[str]
ctxt = errors.ParserContext(input_file_name, errors.ParserErrorCollection())

42
buildscripts/idl/idl/struct_types.py
View File

@ -27,8 +27,6 @@
#
"""Provide code generation information for structs and commands in a polymorphic way."""
from __future__ import absolute_import, print_function, unicode_literals
from abc import ABCMeta, abstractmethod
from typing import Optional, List
@ -46,7 +44,7 @@ def _is_required_constructor_arg(field):
def _get_arg_for_field(field):
# type: (ast.Field) -> unicode
# type: (ast.Field) -> str
"""Generate a moveable parameter."""
cpp_type_info = cpp_types.get_cpp_type(field)
# Use the storage type for the constructor argument since the generated code will use std::move.
@ -56,7 +54,7 @@ def _get_arg_for_field(field):
def _get_required_parameters(struct):
# type: (ast.Struct) -> List[unicode]
# type: (ast.Struct) -> List[str]
"""Get a list of arguments for required parameters."""
return [
_get_arg_for_field(field) for field in struct.fields if _is_required_constructor_arg(field)
@ -67,7 +65,7 @@ class ArgumentInfo(object):
"""Class that encapsulates information about an argument to a method."""
def __init__(self, arg):
# type: (unicode) -> None
# type: (str) -> None
"""Create a instance of the ArgumentInfo class by parsing the argument string."""
parts = arg.split(' ')
self.type = ' '.join(parts[0:-1])
@ -84,7 +82,7 @@ class MethodInfo(object):
def __init__(self, class_name, method_name, args, return_type=None, static=False, const=False,
explicit=False):
# type: (unicode, unicode, List[unicode], unicode, bool, bool, bool) -> None
# type: (str, str, List[str], str, bool, bool, bool) -> None
# pylint: disable=too-many-arguments
"""Create a MethodInfo instance."""
self.class_name = class_name
@ -96,7 +94,7 @@ class MethodInfo(object):
self.explicit = explicit
def get_declaration(self):
# type: () -> unicode
# type: () -> str
"""Get a declaration for a method."""
pre_modifiers = ''
post_modifiers = ''
@ -120,7 +118,7 @@ class MethodInfo(object):
args=', '.join([str(arg) for arg in self.args]), post_modifiers=post_modifiers)
def get_definition(self):
# type: () -> unicode
# type: () -> str
"""Get a definition for a method."""
pre_modifiers = ''
post_modifiers = ''
@ -139,7 +137,7 @@ class MethodInfo(object):
[str(arg) for arg in self.args]), post_modifiers=post_modifiers)
def get_call(self, obj):
# type: (Optional[unicode]) -> unicode
# type: (Optional[str]) -> str
"""Generate a simply call to the method using the defined args list."""
args = ', '.join([arg.name for arg in self.args])
@ -152,11 +150,9 @@ class MethodInfo(object):
args=args)
class StructTypeInfoBase(object):
class StructTypeInfoBase(object, metaclass=ABCMeta):
"""Base class for struct and command code generation."""
__metaclass__ = ABCMeta
@abstractmethod
def get_constructor_method(self):
# type: () -> MethodInfo
@ -234,7 +230,7 @@ class StructTypeInfoBase(object):
@abstractmethod
def gen_namespace_check(self, indented_writer, db_name, element):
# type: (writer.IndentedTextWriter, unicode, unicode) -> None
# type: (writer.IndentedTextWriter, str, str) -> None
"""Generate the namespace check predicate for a command."""
pass
@ -306,7 +302,7 @@ class _StructTypeInfo(StructTypeInfoBase):
pass
def gen_namespace_check(self, indented_writer, db_name, element):
# type: (writer.IndentedTextWriter, unicode, unicode) -> None
# type: (writer.IndentedTextWriter, str, str) -> None
pass
@ -369,12 +365,12 @@ class _IgnoredCommandTypeInfo(_CommandBaseTypeInfo):
indented_writer.write_line('builder->append("%s"_sd, 1);' % (self._command.name))
def gen_namespace_check(self, indented_writer, db_name, element):
# type: (writer.IndentedTextWriter, unicode, unicode) -> None
# type: (writer.IndentedTextWriter, str, str) -> None
pass
def _get_command_type_parameter(command):
# type: (ast.Command) -> unicode
# type: (ast.Command) -> str
"""Get the parameter for the command type."""
cpp_type_info = cpp_types.get_cpp_type(command.command_field)
# Use the storage type for the constructor argument since the generated code will use std::move.
@ -440,7 +436,7 @@ class _CommandFromType(_CommandBaseTypeInfo):
raise NotImplementedError
def gen_namespace_check(self, indented_writer, db_name, element):
# type: (writer.IndentedTextWriter, unicode, unicode) -> None
# type: (writer.IndentedTextWriter, str, str) -> None
# TODO: should the name of the first element be validated??
raise NotImplementedError
@ -500,11 +496,11 @@ class _CommandWithNamespaceTypeInfo(_CommandBaseTypeInfo):
indented_writer.write_empty_line()
def gen_namespace_check(self, indented_writer, db_name, element):
# type: (writer.IndentedTextWriter, unicode, unicode) -> None
# type: (writer.IndentedTextWriter, str, str) -> None
# TODO: should the name of the first element be validated??
indented_writer.write_line('invariant(_nss.isEmpty());')
indented_writer.write_line('_nss = ctxt.parseNSCollectionRequired(%s, %s);' % (db_name,
element))
indented_writer.write_line(
'_nss = ctxt.parseNSCollectionRequired(%s, %s);' % (db_name, element))
class _CommandWithUUIDNamespaceTypeInfo(_CommandBaseTypeInfo):
@ -566,12 +562,12 @@ class _CommandWithUUIDNamespaceTypeInfo(_CommandBaseTypeInfo):
indented_writer.write_line(
'_nssOrUUID.uuid().get().appendToBuilder(builder, "%s"_sd);' % (self._command.name))
with writer.IndentedScopedBlock(indented_writer, "else {", "}"):
indented_writer.write_line('builder->append("%s"_sd, _nssOrUUID.nss().get().coll());' %
(self._command.name))
indented_writer.write_line(
'builder->append("%s"_sd, _nssOrUUID.nss().get().coll());' % (self._command.name))
indented_writer.write_empty_line()
def gen_namespace_check(self, indented_writer, db_name, element):
# type: (writer.IndentedTextWriter, unicode, unicode) -> None
# type: (writer.IndentedTextWriter, str, str) -> None
indented_writer.write_line('invariant(_nssOrUUID.nss() || _nssOrUUID.uuid());')
indented_writer.write_line('_nssOrUUID = ctxt.parseNsOrUUID(%s, %s);' % (db_name, element))

187
buildscripts/idl/idl/syntax.py
View File

@ -33,8 +33,6 @@ It maps 1-1 to the YAML file, and has not been checked if
it follows the rules of the IDL, etc.
"""
from __future__ import absolute_import, print_function, unicode_literals
import itertools
from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
@ -72,7 +70,7 @@ class IDLSpec(object):
def parse_array_type(name):
# type: (unicode) -> unicode
# type: (str) -> str
"""Parse a type name of the form 'array<type>' and extract type."""
if not name.startswith("array<") and not name.endswith(">"):
return None
@ -96,8 +94,7 @@ def _zip_scalar(items, obj):
def _item_and_type(dic):
# type: (Dict[Any, List[Any]]) -> Iterator[Tuple[Any, Any]]
"""Return an Iterator of (key, value) pairs from a dictionary."""
return itertools.chain.from_iterable(
(_zip_scalar(value, key) for (key, value) in dic.viewitems()))
return itertools.chain.from_iterable((_zip_scalar(value, key) for (key, value) in dic.items()))
class SymbolTable(object):
@ -117,7 +114,7 @@ class SymbolTable(object):
self.types = [] # type: List[Type]
def _is_duplicate(self, ctxt, location, name, duplicate_class_name):
# type: (errors.ParserContext, common.SourceLocation, unicode, unicode) -> bool
# type: (errors.ParserContext, common.SourceLocation, str, str) -> bool
"""Return true if the given item already exist in the symbol table."""
for (item, entity_type) in _item_and_type({
"command": self.commands,
@ -181,12 +178,12 @@ class SymbolTable(object):
self.add_type(ctxt, idltype)
def resolve_field_type(self, ctxt, location, field_name, type_name):
# type: (errors.ParserContext, common.SourceLocation, unicode, unicode) -> Optional[Union[Command, Enum, Struct, Type]]
# type: (errors.ParserContext, common.SourceLocation, str, str) -> Optional[Union[Command, Enum, Struct, Type]]
"""Find the type or struct a field refers to or log an error."""
return self._resolve_field_type(ctxt, location, field_name, type_name)
def _resolve_field_type(self, ctxt, location, field_name, type_name):
# type: (errors.ParserContext, common.SourceLocation, unicode, unicode) -> Optional[Union[Command, Enum, Struct, Type]]
# type: (errors.ParserContext, common.SourceLocation, str, str) -> Optional[Union[Command, Enum, Struct, Type]]
"""Find the type or struct a field refers to or log an error."""
# pylint: disable=too-many-return-statements
@ -228,10 +225,10 @@ class Global(common.SourceLocation):
"""
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a Global."""
self.cpp_namespace = None # type: unicode
self.cpp_includes = [] # type: List[unicode]
self.cpp_namespace = None # type: str
self.cpp_includes = [] # type: List[str]
self.configs = None # type: ConfigGlobal
super(Global, self).__init__(file_name, line, column)
@ -241,15 +238,15 @@ class Import(common.SourceLocation):
"""IDL imports object."""
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct an Imports section."""
self.imports = [] # type: List[unicode]
self.imports = [] # type: List[str]
# These are not part of the IDL syntax but are produced by the parser.
# List of imports with structs.
self.resolved_imports = [] # type: List[unicode]
self.resolved_imports = [] # type: List[str]
# All imports directly or indirectly included
self.dependencies = [] # type: List[unicode]
self.dependencies = [] # type: List[str]
super(Import, self).__init__(file_name, line, column)
@ -266,16 +263,16 @@ class Type(common.SourceLocation):
# pylint: disable=too-many-instance-attributes
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a Type."""
self.name = None # type: unicode
self.description = None # type: unicode
self.cpp_type = None # type: unicode
self.bson_serialization_type = None # type: List[unicode]
self.bindata_subtype = None # type: unicode
self.serializer = None # type: unicode
self.deserializer = None # type: unicode
self.default = None # type: unicode
self.name = None # type: str
self.description = None # type: str
self.cpp_type = None # type: str
self.bson_serialization_type = None # type: List[str]
self.bindata_subtype = None # type: str
self.serializer = None # type: str
self.deserializer = None # type: str
self.default = None # type: str
super(Type, self).__init__(file_name, line, column)
@ -291,7 +288,7 @@ class Validator(common.SourceLocation):
# pylint: disable=too-many-instance-attributes
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a Validator."""
# Don't lint gt/lt as bad attibute names.
# pylint: disable=C0103
@ -299,7 +296,7 @@ class Validator(common.SourceLocation):
self.lt = None # type: Expression
self.gte = None # type: Expression
self.lte = None # type: Expression
self.callback = None # type: unicode
self.callback = None # type: str
super(Validator, self).__init__(file_name, line, column)
@ -316,15 +313,15 @@ class Field(common.SourceLocation):
# pylint: disable=too-many-instance-attributes
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a Field."""
self.name = None # type: unicode
self.cpp_name = None # type: unicode
self.description = None # type: unicode
self.type = None # type: unicode
self.name = None # type: str
self.cpp_name = None # type: str
self.description = None # type: str
self.type = None # type: str
self.ignore = False # type: bool
self.optional = False # type: bool
self.default = None # type: unicode
self.default = None # type: str
self.supports_doc_sequence = False # type: bool
self.comparison_order = -1 # type: int
self.validator = None # type: Validator
@ -344,10 +341,10 @@ class ChainedStruct(common.SourceLocation):
"""
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a Type."""
self.name = None # type: unicode
self.cpp_name = None # type: unicode
self.name = None # type: str
self.cpp_name = None # type: str
super(ChainedStruct, self).__init__(file_name, line, column)
@ -360,10 +357,10 @@ class ChainedType(common.SourceLocation):
"""
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a Type."""
self.name = None # type: unicode
self.cpp_name = None # type: unicode
self.name = None # type: str
self.cpp_name = None # type: str
super(ChainedType, self).__init__(file_name, line, column)
@ -378,10 +375,10 @@ class Struct(common.SourceLocation):
# pylint: disable=too-many-instance-attributes
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a Struct."""
self.name = None # type: unicode
self.description = None # type: unicode
self.name = None # type: str
self.description = None # type: str
self.strict = True # type: bool
self.immutable = False # type: bool
self.inline_chained_structs = True # type: bool
@ -391,14 +388,14 @@ class Struct(common.SourceLocation):
self.fields = None # type: List[Field]
# Command only property
self.cpp_name = None # type: unicode
self.cpp_name = None # type: str
# Internal property that is not represented as syntax. An imported struct is read from an
# imported file, and no code is generated for it.
self.imported = False # type: bool
# Internal property: cpp_namespace from globals section
self.cpp_namespace = None # type: unicode
self.cpp_namespace = None # type: str
super(Struct, self).__init__(file_name, line, column)
@ -411,10 +408,10 @@ class Command(Struct):
"""
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a Command."""
self.namespace = None # type: unicode
self.type = None # type: unicode
self.namespace = None # type: str
self.type = None # type: str
super(Command, self).__init__(file_name, line, column)
@ -427,10 +424,10 @@ class EnumValue(common.SourceLocation):
"""
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct an Enum."""
self.name = None # type: unicode
self.value = None # type: unicode
self.name = None # type: str
self.value = None # type: str
super(EnumValue, self).__init__(file_name, line, column)
@ -443,11 +440,11 @@ class Enum(common.SourceLocation):
"""
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct an Enum."""
self.name = None # type: unicode
self.description = None # type: unicode
self.type = None # type: unicode
self.name = None # type: str
self.description = None # type: str
self.type = None # type: str
self.values = None # type: List[EnumValue]
# Internal property that is not represented as syntax. An imported enum is read from an
@ -455,7 +452,7 @@ class Enum(common.SourceLocation):
self.imported = False # type: bool
# Internal property: cpp_namespace from globals section
self.cpp_namespace = None # type: unicode
self.cpp_namespace = None # type: str
super(Enum, self).__init__(file_name, line, column)
@ -464,11 +461,11 @@ class Condition(common.SourceLocation):
"""Condition(s) for a ServerParameter or ConfigOption."""
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a Condition."""
self.expr = None # type: unicode
self.constexpr = None # type: unicode
self.preprocessor = None # type: unicode
self.expr = None # type: str
self.constexpr = None # type: str
self.preprocessor = None # type: str
super(Condition, self).__init__(file_name, line, column)
@ -477,11 +474,11 @@ class Expression(common.SourceLocation):
"""Description of a valid C++ expression."""
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct an Expression."""
self.literal = None # type: unicode
self.expr = None # type: unicode
self.literal = None # type: str
self.expr = None # type: str
self.is_constexpr = True # type: bool
super(Expression, self).__init__(file_name, line, column)
@ -491,11 +488,11 @@ class ServerParameterClass(common.SourceLocation):
"""ServerParameter as C++ class specialization."""
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a ServerParameterClass."""
self.name = None # type: unicode
self.data = None # type: unicode
self.name = None # type: str
self.data = None # type: str
self.override_ctor = False # type: bool
self.override_set = False # type: bool
@ -508,23 +505,23 @@ class ServerParameter(common.SourceLocation):
# pylint: disable=too-many-instance-attributes
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a ServerParameter."""
self.name = None # type: unicode
self.set_at = None # type: List[unicode]
self.description = None # type: unicode
self.cpp_vartype = None # type: unicode
self.cpp_varname = None # type: unicode
self.name = None # type: str
self.set_at = None # type: List[str]
self.description = None # type: str
self.cpp_vartype = None # type: str
self.cpp_varname = None # type: str
self.cpp_class = None # type: ServerParameterClass
self.condition = None # type: Condition
self.deprecated_name = [] # type: List[unicode]
self.deprecated_name = [] # type: List[str]
self.redact = False # type: bool
self.test_only = False # type: bool
self.default = None # type: Expression
# Only valid if cpp_varname is specified.
self.validator = None # type: Validator
self.on_update = None # type: unicode
self.on_update = None # type: str
super(ServerParameter, self).__init__(file_name, line, column)
@ -533,12 +530,12 @@ class GlobalInitializer(common.SourceLocation):
"""Initializer details for custom registration/storage."""
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a GlobalInitializer."""
self.name = None # type: unicode
self.register = None # type: unicode
self.store = None # type: unicode
self.name = None # type: str
self.register = None # type: str
self.store = None # type: str
super(GlobalInitializer, self).__init__(file_name, line, column)
@ -547,10 +544,10 @@ class ConfigGlobal(common.SourceLocation):
"""Global values to apply to all ConfigOptions."""
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a ConfigGlobal."""
self.section = None # type: unicode
self.source = [] # type: List[unicode]
self.section = None # type: str
self.source = [] # type: List[str]
self.initializer = None # type: GlobalInitializer
super(ConfigGlobal, self).__init__(file_name, line, column)
@ -562,32 +559,32 @@ class ConfigOption(common.SourceLocation):
# pylint: disable=too-many-instance-attributes
def __init__(self, file_name, line, column):
# type: (unicode, int, int) -> None
# type: (str, int, int) -> None
"""Construct a ConfigOption."""
self.name = None # type: unicode
self.deprecated_name = [] # type: List[unicode]
self.short_name = None # type: unicode
self.single_name = None # type: unicode
self.deprecated_short_name = [] # type: List[unicode]
self.name = None # type: str
self.deprecated_name = [] # type: List[str]
self.short_name = None # type: str
self.single_name = None # type: str
self.deprecated_short_name = [] # type: List[str]
self.description = None # type: Expression
self.section = None # type: unicode
self.arg_vartype = None # type: unicode
self.cpp_vartype = None # type: unicode
self.cpp_varname = None # type: unicode
self.section = None # type: str
self.arg_vartype = None # type: str
self.cpp_vartype = None # type: str
self.cpp_varname = None # type: str
self.condition = None # type: Condition
self.conflicts = [] # type: List[unicode]
self.requires = [] # type: List[unicode]
self.conflicts = [] # type: List[str]
self.requires = [] # type: List[str]
self.hidden = False # type: bool
self.redact = False # type: bool
self.default = None # type: Expression
self.implicit = None # type: Expression
self.source = [] # type: List[unicode]
self.canonicalize = None # type: unicode
self.source = [] # type: List[str]
self.canonicalize = None # type: str
self.duplicate_behavior = None # type: unicode
self.positional = None # type unicode
self.duplicate_behavior = None # type: str
self.positional = None # type str
self.validator = None # type: Validator
super(ConfigOption, self).__init__(file_name, line, column)

30
buildscripts/idl/idl/writer.py
View File

@ -27,8 +27,6 @@
#
"""Text Writing Utilites."""
from __future__ import absolute_import, print_function, unicode_literals
import io
import string
from typing import List, Mapping, Union
@ -40,7 +38,7 @@ _INDENT_SPACE_COUNT = 4
def _fill_spaces(count):
# type: (int) -> unicode
# type: (int) -> str
"""Fill a string full of spaces."""
fill = ''
for _ in range(count * _INDENT_SPACE_COUNT):
@ -50,7 +48,7 @@ def _fill_spaces(count):
def _indent_text(count, unindented_text):
# type: (int, unicode) -> unicode
# type: (int, str) -> str
"""Indent each line of a multi-line string."""
lines = unindented_text.splitlines()
fill = _fill_spaces(count)
@ -58,7 +56,7 @@ def _indent_text(count, unindented_text):
def is_function(name):
# type: (unicode) -> bool
# type: (str) -> bool
"""
Return True if a serializer/deserializer is function.
@ -69,7 +67,7 @@ def is_function(name):
def get_method_name(name):
# type: (unicode) -> unicode
# type: (str) -> str
"""Get a method name from a fully qualified method name."""
pos = name.rfind('::')
if pos == -1:
@ -78,7 +76,7 @@ def get_method_name(name):
def get_method_name_from_qualified_method_name(name):
# type: (unicode) -> unicode
# type: (str) -> str
# pylint: disable=invalid-name
"""Get a method name from a fully qualified method name."""
# TODO: in the future, we may want to support full-qualified calls to static methods
@ -108,10 +106,10 @@ class IndentedTextWriter(object):
"""Create an indented text writer."""
self._stream = stream
self._indent = 0
self._template_context = None # type: Mapping[unicode, unicode]
self._template_context = None # type: Mapping[str, str]
def write_unindented_line(self, msg):
# type: (unicode) -> None
# type: (str) -> None
"""Write an unindented line to the stream, no template formattin applied."""
self._stream.write(msg)
self._stream.write("\n")
@ -128,13 +126,13 @@ class IndentedTextWriter(object):
self._indent -= 1
def write_line(self, msg):
# type: (unicode) -> None
# type: (str) -> None
"""Write a line to the stream, no template formattin applied."""
self._stream.write(_indent_text(self._indent, msg))
self._stream.write("\n")
def set_template_mapping(self, template_params):
# type: (Mapping[unicode,unicode]) -> None
# type: (Mapping[str,str]) -> None
"""Set the current template mapping parameters for string.Template formatting."""
assert not self._template_context
self._template_context = template_params
@ -146,7 +144,7 @@ class IndentedTextWriter(object):
self._template_context = None
def write_template(self, template):
# type: (unicode) -> None
# type: (str) -> None
"""Write a template to the stream."""
msg = common.template_format(template, self._template_context)
self._stream.write(_indent_text(self._indent, msg))
@ -162,7 +160,7 @@ class TemplateContext(object):
"""Set the template context for the writer."""
def __init__(self, writer, template_params):
# type: (IndentedTextWriter, Mapping[unicode,unicode]) -> None
# type: (IndentedTextWriter, Mapping[str,str]) -> None
"""Create a template context."""
self._writer = writer
self._template_context = template_params
@ -215,7 +213,7 @@ class IndentedScopedBlock(WriterBlock):
"""Generate a block, template the parameters, and indent the contents."""
def __init__(self, writer, opening, closing):
# type: (IndentedTextWriter, unicode, unicode) -> None
# type: (IndentedTextWriter, str, str) -> None
"""Create a block."""
self._writer = writer
self._opening = opening
@ -238,7 +236,7 @@ class NamespaceScopeBlock(WriterBlock):
"""Generate an unindented blocks for a list of namespaces, and do not indent the contents."""
def __init__(self, indented_writer, namespaces):
# type: (IndentedTextWriter, List[unicode]) -> None
# type: (IndentedTextWriter, List[str]) -> None
"""Create a block."""
self._writer = indented_writer
self._namespaces = namespaces
@ -262,7 +260,7 @@ class UnindentedBlock(WriterBlock):
"""Generate a block without indentation."""
def __init__(self, writer, opening, closing):
# type: (IndentedTextWriter, unicode, unicode) -> None
# type: (IndentedTextWriter, str, str) -> None
"""Create a block."""
self._writer = writer
self._opening = opening

2
buildscripts/idl/idlc.py
View File

@ -29,8 +29,6 @@
#
"""IDL Compiler Driver Main Entry point."""
from __future__ import absolute_import, print_function
import argparse
import logging
import sys

1
buildscripts/idl/run_tests.py
View File

@ -32,7 +32,6 @@ IDL Unit Test runner.
Generates a file called results.xml in the XUnit format.
"""
from __future__ import absolute_import, print_function
import sys
import unittest

155
buildscripts/idl/tests/test_binder.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python2
#!/usr/bin/env python
#
# Copyright (C) 2018-present MongoDB, Inc.
#
@ -30,8 +30,6 @@
# pylint: disable=too-many-lines
"""Test cases for IDL binder."""
from __future__ import absolute_import, print_function, unicode_literals
import textwrap
import unittest
@ -61,7 +59,7 @@ def fill_spaces(count):
def indent_text(count, unindented_text):
# type: (int, unicode) -> unicode
# type: (int, str) -> str
"""Indent each line of a multi-line string."""
lines = unindented_text.splitlines()
fill = fill_spaces(count)
@ -88,7 +86,7 @@ class TestBinder(testcase.IDLTestcase):
cpp_includes:
- 'bar'
- 'foo'"""))
self.assertEquals(spec.globals.cpp_namespace, "something")
self.assertEqual(spec.globals.cpp_namespace, "something")
self.assertListEqual(spec.globals.cpp_includes, ['bar', 'foo'])
def test_type_positive(self):
@ -566,7 +564,8 @@ class TestBinder(testcase.IDLTestcase):
""")
# Test array as name
self.assert_bind_fail(test_preamble + textwrap.dedent("""
self.assert_bind_fail(
test_preamble + textwrap.dedent("""
structs:
array<foo>:
description: foo
@ -675,7 +674,8 @@ class TestBinder(testcase.IDLTestcase):
""")
# Test field of a struct type with a default
self.assert_bind_fail(test_preamble + textwrap.dedent("""
self.assert_bind_fail(
test_preamble + textwrap.dedent("""
structs:
foo:
description: foo
@ -692,7 +692,8 @@ class TestBinder(testcase.IDLTestcase):
"""), idl.errors.ERROR_ID_FIELD_MUST_BE_EMPTY_FOR_STRUCT)
# Test array as field name
self.assert_bind_fail(test_preamble + textwrap.dedent("""
self.assert_bind_fail(
test_preamble + textwrap.dedent("""
structs:
foo:
description: foo
@ -702,7 +703,8 @@ class TestBinder(testcase.IDLTestcase):
"""), idl.errors.ERROR_ID_ARRAY_NOT_VALID_TYPE)
# Test recursive array as field type
self.assert_bind_fail(test_preamble + textwrap.dedent("""
self.assert_bind_fail(
test_preamble + textwrap.dedent("""
structs:
foo:
description: foo
@ -712,7 +714,8 @@ class TestBinder(testcase.IDLTestcase):
"""), idl.errors.ERROR_ID_BAD_ARRAY_TYPE_NAME)
# Test inherited default with array
self.assert_bind_fail(test_preamble + textwrap.dedent("""
self.assert_bind_fail(
test_preamble + textwrap.dedent("""
structs:
foo:
description: foo
@ -743,7 +746,8 @@ class TestBinder(testcase.IDLTestcase):
"""), idl.errors.ERROR_ID_ARRAY_NO_DEFAULT)
# Test bindata with default
self.assert_bind_fail(test_preamble + textwrap.dedent("""
self.assert_bind_fail(
test_preamble + textwrap.dedent("""
structs:
foo:
description: foo
@ -755,7 +759,8 @@ class TestBinder(testcase.IDLTestcase):
"""), idl.errors.ERROR_ID_BAD_BINDATA_DEFAULT)
# Test default and optional for the same field
self.assert_bind_fail(test_preamble + textwrap.dedent("""
self.assert_bind_fail(
test_preamble + textwrap.dedent("""
structs:
foo:
description: foo
@ -768,7 +773,8 @@ class TestBinder(testcase.IDLTestcase):
"""), idl.errors.ERROR_ID_ILLEGAL_FIELD_DEFAULT_AND_OPTIONAL)
# Test duplicate comparison order
self.assert_bind_fail(test_preamble + textwrap.dedent("""
self.assert_bind_fail(
test_preamble + textwrap.dedent("""
structs:
foo:
description: foo
@ -861,7 +867,8 @@ class TestBinder(testcase.IDLTestcase):
""")
# Chaining with strict struct
self.assert_bind_fail(test_preamble + textwrap.dedent("""
self.assert_bind_fail(
test_preamble + textwrap.dedent("""
structs:
bar1:
description: foo
@ -871,7 +878,8 @@ class TestBinder(testcase.IDLTestcase):
"""), idl.errors.ERROR_ID_CHAINED_NO_TYPE_STRICT)
# Non-'any' type as chained type
self.assert_bind_fail(test_preamble + textwrap.dedent("""
self.assert_bind_fail(
test_preamble + textwrap.dedent("""
structs:
bar1:
description: foo
@ -881,7 +889,8 @@ class TestBinder(testcase.IDLTestcase):
"""), idl.errors.ERROR_ID_CHAINED_TYPE_WRONG_BSON_TYPE)
# Chaining and fields only with same name
self.assert_bind_fail(test_preamble + textwrap.dedent("""
self.assert_bind_fail(
test_preamble + textwrap.dedent("""
structs:
bar1:
description: foo
@ -893,7 +902,8 @@ class TestBinder(testcase.IDLTestcase):
"""), idl.errors.ERROR_ID_CHAINED_DUPLICATE_FIELD)
# Non-existent chained type
self.assert_bind_fail(test_preamble + textwrap.dedent("""
self.assert_bind_fail(
test_preamble + textwrap.dedent("""
structs:
bar1:
description: foo
@ -905,7 +915,8 @@ class TestBinder(testcase.IDLTestcase):
"""), idl.errors.ERROR_ID_UNKNOWN_TYPE)
# A regular field as a chained type
self.assert_bind_fail(test_preamble + textwrap.dedent("""
self.assert_bind_fail(
test_preamble + textwrap.dedent("""
structs:
bar1:
description: foo
@ -916,7 +927,8 @@ class TestBinder(testcase.IDLTestcase):
"""), idl.errors.ERROR_ID_UNKNOWN_TYPE)
# Array of chained types
self.assert_bind_fail(test_preamble + textwrap.dedent("""
self.assert_bind_fail(
test_preamble + textwrap.dedent("""
structs:
bar1:
description: foo
@ -963,8 +975,9 @@ class TestBinder(testcase.IDLTestcase):
""")
# A struct with only chaining
self.assert_bind(test_preamble + indent_text(1,
textwrap.dedent("""
self.assert_bind(test_preamble + indent_text(
1,
textwrap.dedent("""
bar1:
description: foo
strict: true
@ -973,8 +986,9 @@ class TestBinder(testcase.IDLTestcase):
""")))
# Chaining struct's fields and explicit fields
self.assert_bind(test_preamble + indent_text(1,
textwrap.dedent("""
self.assert_bind(test_preamble + indent_text(
1,
textwrap.dedent("""
bar1:
description: foo
strict: true
@ -985,8 +999,9 @@ class TestBinder(testcase.IDLTestcase):
""")))
# Chained types and structs
self.assert_bind(test_preamble + indent_text(1,
textwrap.dedent("""
self.assert_bind(test_preamble + indent_text(
1,
textwrap.dedent("""
bar1:
description: foo
strict: false
@ -999,8 +1014,9 @@ class TestBinder(testcase.IDLTestcase):
""")))
# Non-strict chained struct
self.assert_bind(test_preamble + indent_text(1,
textwrap.dedent("""
self.assert_bind(test_preamble + indent_text(
1,
textwrap.dedent("""
bar1:
description: foo
strict: false
@ -1011,8 +1027,9 @@ class TestBinder(testcase.IDLTestcase):
""")))
# Inline Chained struct with strict true
self.assert_bind(test_preamble + indent_text(1,
textwrap.dedent("""
self.assert_bind(test_preamble + indent_text(
1,
textwrap.dedent("""
bar1:
description: foo
strict: true
@ -1031,8 +1048,9 @@ class TestBinder(testcase.IDLTestcase):
""")))
# Inline Chained struct with strict true and inline_chained_structs defaulted
self.assert_bind(test_preamble + indent_text(1,
textwrap.dedent("""
self.assert_bind(test_preamble + indent_text(
1,
textwrap.dedent("""
bar1:
description: foo
strict: true
@ -1086,8 +1104,10 @@ class TestBinder(testcase.IDLTestcase):
""")
# Non-existing chained struct
self.assert_bind_fail(test_preamble + indent_text(1,
textwrap.dedent("""
self.assert_bind_fail(
test_preamble + indent_text(
1,
textwrap.dedent("""
bar1:
description: foo
strict: true
@ -1096,8 +1116,10 @@ class TestBinder(testcase.IDLTestcase):
""")), idl.errors.ERROR_ID_UNKNOWN_TYPE)
# Type as chained struct
self.assert_bind_fail(test_preamble + indent_text(1,
textwrap.dedent("""
self.assert_bind_fail(
test_preamble + indent_text(
1,
textwrap.dedent("""
bar1:
description: foo
strict: true
@ -1106,8 +1128,10 @@ class TestBinder(testcase.IDLTestcase):
""")), idl.errors.ERROR_ID_CHAINED_STRUCT_NOT_FOUND)
# Struct as chained type
self.assert_bind_fail(test_preamble + indent_text(1,
textwrap.dedent("""
self.assert_bind_fail(
test_preamble + indent_text(
1,
textwrap.dedent("""
bar1:
description: foo
strict: false
@ -1116,8 +1140,10 @@ class TestBinder(testcase.IDLTestcase):
""")), idl.errors.ERROR_ID_CHAINED_TYPE_NOT_FOUND)
# Duplicated field names across chained struct's fields and fields
self.assert_bind_fail(test_preamble + indent_text(1,
textwrap.dedent("""
self.assert_bind_fail(
test_preamble + indent_text(
1,
textwrap.dedent("""
bar1:
description: foo
strict: false
@ -1128,8 +1154,10 @@ class TestBinder(testcase.IDLTestcase):
""")), idl.errors.ERROR_ID_CHAINED_DUPLICATE_FIELD)
# Duplicated field names across chained structs
self.assert_bind_fail(test_preamble + indent_text(1,
textwrap.dedent("""
self.assert_bind_fail(
test_preamble + indent_text(
1,
textwrap.dedent("""
bar1:
description: foo
strict: false
@ -1139,8 +1167,10 @@ class TestBinder(testcase.IDLTestcase):
""")), idl.errors.ERROR_ID_CHAINED_DUPLICATE_FIELD)
# Chained struct with strict true
self.assert_bind_fail(test_preamble + indent_text(1,
textwrap.dedent("""
self.assert_bind_fail(
test_preamble + indent_text(
1,
textwrap.dedent("""
bar1:
description: foo
strict: true
@ -1159,8 +1189,10 @@ class TestBinder(testcase.IDLTestcase):
""")), idl.errors.ERROR_ID_CHAINED_NO_NESTED_STRUCT_STRICT)
# Chained struct with nested chained struct
self.assert_bind_fail(test_preamble + indent_text(1,
textwrap.dedent("""
self.assert_bind_fail(
test_preamble + indent_text(
1,
textwrap.dedent("""
bar1:
description: foo
strict: false
@ -1178,8 +1210,10 @@ class TestBinder(testcase.IDLTestcase):
""")), idl.errors.ERROR_ID_CHAINED_NO_NESTED_CHAINED)
# Chained struct with nested chained type
self.assert_bind_fail(test_preamble + indent_text(1,
textwrap.dedent("""
self.assert_bind_fail(
test_preamble + indent_text(
1,
textwrap.dedent("""
bar1:
description: foo
strict: false
@ -1304,7 +1338,8 @@ class TestBinder(testcase.IDLTestcase):
""")
# Test array of enums
self.assert_bind_fail(test_preamble + textwrap.dedent("""
self.assert_bind_fail(
test_preamble + textwrap.dedent("""
structs:
foo1:
description: foo
@ -1355,7 +1390,8 @@ class TestBinder(testcase.IDLTestcase):
""")
# Commands cannot be fields in other commands
self.assert_bind_fail(test_preamble + textwrap.dedent("""
self.assert_bind_fail(
test_preamble + textwrap.dedent("""
commands:
foo:
description: foo
@ -1371,7 +1407,8 @@ class TestBinder(testcase.IDLTestcase):
"""), idl.errors.ERROR_ID_FIELD_NO_COMMAND)
# Commands cannot be fields in structs
self.assert_bind_fail(test_preamble + textwrap.dedent("""
self.assert_bind_fail(
test_preamble + textwrap.dedent("""
commands:
foo:
description: foo
@ -1387,7 +1424,8 @@ class TestBinder(testcase.IDLTestcase):
"""), idl.errors.ERROR_ID_FIELD_NO_COMMAND)
# Commands cannot have a field as the same name
self.assert_bind_fail(test_preamble + textwrap.dedent("""
self.assert_bind_fail(
test_preamble + textwrap.dedent("""
commands:
foo:
description: foo
@ -1488,7 +1526,8 @@ class TestBinder(testcase.IDLTestcase):
""")
# A struct
self.assert_bind_fail(test_preamble + textwrap.dedent("""
self.assert_bind_fail(
test_preamble + textwrap.dedent("""
structs:
foo:
description: foo
@ -1499,7 +1538,8 @@ class TestBinder(testcase.IDLTestcase):
"""), idl.errors.ERROR_ID_STRUCT_NO_DOC_SEQUENCE)
# A non-array type
self.assert_bind_fail(test_preamble + textwrap.dedent("""
self.assert_bind_fail(
test_preamble + textwrap.dedent("""
commands:
foo:
description: foo
@ -1511,7 +1551,8 @@ class TestBinder(testcase.IDLTestcase):
"""), idl.errors.ERROR_ID_NO_DOC_SEQUENCE_FOR_NON_ARRAY)
# An array of a scalar
self.assert_bind_fail(test_preamble2 + textwrap.dedent("""
self.assert_bind_fail(
test_preamble2 + textwrap.dedent("""
commands:
foo:
description: foo
@ -1523,7 +1564,8 @@ class TestBinder(testcase.IDLTestcase):
"""), idl.errors.ERROR_ID_NO_DOC_SEQUENCE_FOR_NON_OBJECT)
# An array of 'any'
self.assert_bind_fail(test_preamble2 + textwrap.dedent("""
self.assert_bind_fail(
test_preamble2 + textwrap.dedent("""
commands:
foo:
description: foo
@ -1585,7 +1627,8 @@ class TestBinder(testcase.IDLTestcase):
""")
# supports_doc_sequence must be a bool
self.assert_bind_fail(test_preamble + textwrap.dedent("""
self.assert_bind_fail(
test_preamble + textwrap.dedent("""
commands:
foo:
description: foo

8
buildscripts/idl/tests/test_generator.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python2
#!/usr/bin/env python
#
# Copyright (C) 2018-present MongoDB, Inc.
#
@ -36,8 +36,6 @@ idl base directory:
$ coverage run run_tests.py && coverage html
"""
from __future__ import absolute_import, print_function, unicode_literals
import os
import unittest
@ -72,8 +70,8 @@ class TestGenerator(testcase.IDLTestcase):
unittest_idl_file = os.path.join(idl_dir, 'unittest.idl')
if not os.path.exists(unittest_idl_file):
unittest.skip("Skipping IDL Generator testing since %s could not be found." %
(unittest_idl_file))
unittest.skip(
"Skipping IDL Generator testing since %s could not be found." % (unittest_idl_file))
return
args.input_file = os.path.join(idl_dir, 'unittest_import.idl')

6
buildscripts/idl/tests/test_import.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python2
#!/usr/bin/env python
#
# Copyright (C) 2018-present MongoDB, Inc.
#
@ -29,8 +29,6 @@
#
"""Test cases for IDL binder."""
from __future__ import absolute_import, print_function, unicode_literals
import io
import textwrap
import unittest
@ -67,7 +65,7 @@ class DictionaryImportResolver(idl.parser.ImportResolverBase):
return "imported_%s" % (imported_file_name)
def open(self, resolved_file_name):
# type: (unicode) -> Any
# type: (str) -> Any
"""Return an io.Stream for the requested file."""
assert resolved_file_name.startswith("imported_")
imported_file_name = resolved_file_name.replace("imported_", "")

10
buildscripts/idl/tests/test_parser.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python2
#!/usr/bin/env python
#
# Copyright (C) 2018-present MongoDB, Inc.
#
@ -30,8 +30,6 @@
"""Test cases for IDL parser."""
# pylint: disable=too-many-lines
from __future__ import absolute_import, print_function, unicode_literals
import textwrap
import unittest
@ -964,7 +962,8 @@ class TestParser(testcase.IDLTestcase):
""")
# Commands and structs with same name
self.assert_parse_fail(test_preamble + textwrap.dedent("""
self.assert_parse_fail(
test_preamble + textwrap.dedent("""
commands:
foo:
description: foo
@ -980,7 +979,8 @@ class TestParser(testcase.IDLTestcase):
"""), idl.errors.ERROR_ID_DUPLICATE_SYMBOL)
# Commands and types with same name
self.assert_parse_fail(test_preamble + textwrap.dedent("""
self.assert_parse_fail(
test_preamble + textwrap.dedent("""
commands:
string:
description: foo

33
buildscripts/idl/tests/testcase.py
View File

@ -27,8 +27,6 @@
#
"""Utility methods and classes for testing IDL passes."""
from __future__ import absolute_import, print_function, unicode_literals
import unittest
from typing import Any, Tuple
@ -53,12 +51,12 @@ class NothingImportResolver(idl.parser.ImportResolverBase):
"""An import resolver that does nothing."""
def resolve(self, base_file, imported_file_name):
# type: (unicode, unicode) -> unicode
# type: (str, str) -> str
"""Return the complete path to an imported file name."""
raise NotImplementedError()
def open(self, imported_file_name):
# type: (unicode) -> Any
# type: (str) -> Any
"""Return an io.Stream for the requested file."""
raise NotImplementedError()
@ -67,7 +65,7 @@ class IDLTestcase(unittest.TestCase):
"""IDL Test case base class."""
def _parse(self, doc_str, resolver):
# type: (unicode, idl.parser.ImportResolverBase) -> idl.syntax.IDLParsedSpec
# type: (str, idl.parser.ImportResolverBase) -> idl.syntax.IDLParsedSpec
"""Parse a document and throw a unittest failure if it fails to parse as a valid YAML document."""
try:
@ -76,22 +74,23 @@ class IDLTestcase(unittest.TestCase):
self.fail("Failed to parse document:\n%s" % (doc_str))
def _assert_parse(self, doc_str, parsed_doc):
# type: (unicode, idl.syntax.IDLParsedSpec) -> None
# type: (str, idl.syntax.IDLParsedSpec) -> None
"""Assert a document parsed correctly by the IDL compiler and returned no errors."""
self.assertIsNone(parsed_doc.errors,
"Expected no parser errors\nFor document:\n%s\nReceived errors:\n\n%s" %
(doc_str, errors_to_str(parsed_doc.errors)))
self.assertIsNone(
parsed_doc.errors,
"Expected no parser errors\nFor document:\n%s\nReceived errors:\n\n%s" %
(doc_str, errors_to_str(parsed_doc.errors)))
self.assertIsNotNone(parsed_doc.spec, "Expected a parsed doc")
def assert_parse(self, doc_str, resolver=NothingImportResolver()):
# type: (unicode, idl.parser.ImportResolverBase) -> None
# type: (str, idl.parser.ImportResolverBase) -> None
"""Assert a document parsed correctly by the IDL compiler and returned no errors."""
parsed_doc = self._parse(doc_str, resolver)
self._assert_parse(doc_str, parsed_doc)
def assert_parse_fail(self, doc_str, error_id, multiple=False,
resolver=NothingImportResolver()):
# type: (unicode, unicode, bool, idl.parser.ImportResolverBase) -> None
# type: (str, str, bool, idl.parser.ImportResolverBase) -> None
"""
Assert a document parsed correctly by the YAML parser, but not the by the IDL compiler.
@ -115,22 +114,22 @@ class IDLTestcase(unittest.TestCase):
(doc_str, error_id, errors_to_str(parsed_doc.errors)))
def assert_bind(self, doc_str, resolver=NothingImportResolver()):
# type: (unicode, idl.parser.ImportResolverBase) -> idl.ast.IDLBoundSpec
# type: (str, idl.parser.ImportResolverBase) -> idl.ast.IDLBoundSpec
"""Assert a document parsed and bound correctly by the IDL compiler and returned no errors."""
parsed_doc = self._parse(doc_str, resolver)
self._assert_parse(doc_str, parsed_doc)
bound_doc = idl.binder.bind(parsed_doc.spec)
self.assertIsNone(bound_doc.errors,
"Expected no binder errors\nFor document:\n%s\nReceived errors:\n\n%s" %
(doc_str, errors_to_str(bound_doc.errors)))
self.assertIsNone(
bound_doc.errors, "Expected no binder errors\nFor document:\n%s\nReceived errors:\n\n%s"
% (doc_str, errors_to_str(bound_doc.errors)))
self.assertIsNotNone(bound_doc.spec, "Expected a bound doc")
return bound_doc.spec
def assert_bind_fail(self, doc_str, error_id, resolver=NothingImportResolver()):
# type: (unicode, unicode, idl.parser.ImportResolverBase) -> None
# type: (str, str, idl.parser.ImportResolverBase) -> None
"""
Assert a document parsed correctly by the YAML parser and IDL parser, but not bound by the IDL binder.
@ -156,7 +155,7 @@ class IDLTestcase(unittest.TestCase):
(doc_str, error_id, errors_to_str(bound_doc.errors)))
def assert_generate(self, doc_str, resolver=NothingImportResolver()):
# type: (unicode, idl.parser.ImportResolverBase) -> Tuple[unicode,unicode]
# type: (str, idl.parser.ImportResolverBase) -> Tuple[str,str]
"""Assert a document parsed, bound, and generated correctly by the IDL compiler."""
spec = self.assert_bind(doc_str, resolver)

2
buildscripts/jiraclient.py
View File

@ -1,7 +1,5 @@
"""Module to access a JIRA server."""
from __future__ import absolute_import
import jira

5
buildscripts/lint.py
View File

@ -1,8 +1,5 @@
"""Lint module."""
from __future__ import absolute_import
from __future__ import print_function
import codecs
import os
import sys
@ -35,7 +32,7 @@ class CheckForConfigH(object):
'MONGO_CONFIG define used without prior inclusion of config.h.')
def run_lint(paths, nudge_on=False):
def run_lint(paths, nudge_on=False): # pylint: disable=too-many-statements
"""Run lint."""
# errors are as of 10/14
# idea is not to let it any new type of error

6
buildscripts/linter/base.py
View File

@ -1,16 +1,12 @@
"""Base class and support functions for linters."""
from __future__ import absolute_import
from __future__ import print_function
from abc import ABCMeta, abstractmethod
from typing import Dict, List, Optional
class LinterBase(object):
class LinterBase(object, metaclass=ABCMeta):
"""Base Class for all linters."""
__metaclass__ = ABCMeta
def __init__(self, cmd_name, required_version, cmd_location=None):
# type: (str, str, Optional[str]) -> None
"""

4
buildscripts/linter/git.py
View File

@ -1,6 +1,4 @@
"""Git Utility functions."""
from __future__ import absolute_import
from __future__ import print_function
import itertools
import os
@ -196,7 +194,7 @@ def get_files_to_check_from_patch(patches, filter_function):
lines = [] # type: List[str]
for patch in patches:
with open(patch, "rb") as infile:
with open(patch, "r") as infile:
lines += infile.readlines()
candidates = [check.match(line).group(1) for line in lines if check.match(line)]

9
buildscripts/linter/mypy.py
View File

@ -1,6 +1,4 @@
"""Mypy linter support module."""
from __future__ import absolute_import
from __future__ import print_function
import os
from typing import List
@ -26,7 +24,12 @@ class MypyLinter(base.LinterBase):
def get_lint_cmd_args(self, file_name):
# type: (str) -> List[str]
"""Get the command to run a linter."""
return [file_name]
# Only idl and linter should be type checked by mypy. Other
# files return errors under python 3 type checking. If we
# return an empty list the runner will skip this file.
if 'idl' in file_name or 'linter' in file_name:
return [file_name]
return []
def ignore_interpreter(self):
# type: () -> bool

8
buildscripts/linter/parallel.py
View File

@ -1,8 +1,6 @@
"""Utility code to execute code in parallel."""
from __future__ import absolute_import
from __future__ import print_function
import Queue
import queue
import threading
import time
from multiprocessing import cpu_count
@ -17,7 +15,7 @@ def parallel_process(items, func):
except NotImplementedError:
cpus = 1
task_queue = Queue.Queue() # type: Queue.Queue
task_queue = queue.Queue() # type: Queue.Queue
# Use a list so that worker function will capture this variable
pp_event = threading.Event()
@ -30,7 +28,7 @@ def parallel_process(items, func):
while not pp_event.is_set():
try:
item = task_queue.get_nowait()
except Queue.Empty:
except queue.Empty:
# if the queue is empty, exit the worker thread
pp_event.set()
return

2
buildscripts/linter/pydocstyle.py
View File

@ -1,6 +1,4 @@
"""PyDocStyle linter support module."""
from __future__ import absolute_import
from __future__ import print_function
from typing import List

4
buildscripts/linter/pylint.py
View File

@ -1,6 +1,4 @@
"""PyLint linter support module."""
from __future__ import absolute_import
from __future__ import print_function
import os
from typing import List
@ -15,7 +13,7 @@ class PyLintLinter(base.LinterBase):
def __init__(self):
# type: () -> None
"""Create a pylint linter."""
super(PyLintLinter, self).__init__("pylint", "pylint 1.9.3")
super(PyLintLinter, self).__init__("pylint", "pylint 2.3.1")
def get_lint_version_cmd_args(self):
# type: () -> List[str]

41
buildscripts/linter/runner.py
View File

@ -1,6 +1,4 @@
"""Class to support running various linters in a common framework."""
from __future__ import absolute_import
from __future__ import print_function
import difflib
import logging
@ -23,19 +21,22 @@ def _check_version(linter, cmd_path, args):
logging.info(str(cmd))
process_handle = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, stderr = process_handle.communicate()
output = output.decode('utf-8')
if process_handle.returncode:
logging.info("Version check failed for [%s], return code '%d'." +
"Standard Output:\n%s\nStandard Error:\n%s", cmd,
process_handle.returncode, output, stderr)
logging.info(
"Version check failed for [%s], return code '%d'."
"Standard Output:\n%s\nStandard Error:\n%s", cmd, process_handle.returncode, output,
stderr)
required_version = re.escape(linter.required_version)
pattern = r"\b%s\b" % (required_version)
if not re.search(pattern, output):
logging.info("Linter %s has wrong version for '%s'. Expected '%s'," +
"Standard Output:\n'%s'\nStandard Error:\n%s", linter.cmd_name, cmd,
required_version, output, stderr)
logging.info(
"Linter %s has wrong version for '%s'. Expected '%s',"
"Standard Output:\n'%s'\nStandard Error:\n%s", linter.cmd_name, cmd,
required_version, output, stderr)
return False
except OSError as os_error:
@ -126,7 +127,8 @@ def find_linters(linter_list, config_dict):
for linter in linter_list:
linter_instance = _find_linter(linter, config_dict)
if not linter_instance:
logging.error("""\
logging.error(
"""\
Could not find the correct version of linter '%s', expected '%s'. Check your
PATH environment variable or re-run with --verbose for more information.
@ -166,16 +168,25 @@ class LintRunner(object):
# type: (base.LinterInstance, str) -> bool
"""Run the specified linter for the file."""
cmd = linter.cmd_path + linter.linter.get_lint_cmd_args(file_name)
cmd = linter.cmd_path
cmd += linter.linter.get_lint_cmd_args(file_name)
if cmd == linter.cmd_path:
# If args is empty it means we didn't get a valid command
# to run and so should skip this file.
#
# For example the MyPy linter class will return empty args
# for non-idl files since they shouldn't be type checked.
return True
logging.debug(str(cmd))
try:
if linter.linter.needs_file_diff():
# Need a file diff
with open(file_name, 'rb') as original_text:
original_file = original_text.read()
original_file = original_text.read().decode('utf-8')
formatted_file = subprocess.check_output(cmd)
formatted_file = subprocess.check_output(cmd).decode('utf-8')
if original_file != formatted_file:
original_lines = original_file.splitlines()
formatted_lines = formatted_file.splitlines()
@ -196,7 +207,7 @@ class LintRunner(object):
return False
else:
output = subprocess.check_output(cmd)
output = subprocess.check_output(cmd).decode('utf-8')
# On Windows, mypy.bat returns 0 even if there are length failures so we need to
# check if there was any output
@ -205,7 +216,7 @@ class LintRunner(object):
return False
except subprocess.CalledProcessError as cpe:
self._safe_print("CMD [%s] failed:\n%s" % (cmd, cpe.output))
self._safe_print("CMD [%s] failed:\n%s" % (cmd, cpe.output.decode('utf-8')))
return False
return True
@ -217,7 +228,7 @@ class LintRunner(object):
logging.debug(str(cmd))
try:
subprocess.check_output(cmd)
subprocess.check_output(cmd).decode('utf-8')
except subprocess.CalledProcessError as cpe:
self._safe_print("CMD [%s] failed:\n%s" % (cmd, cpe.output))
return False

4
buildscripts/linter/yapf.py
View File

@ -1,6 +1,4 @@
"""YAPF linter support module."""
from __future__ import absolute_import
from __future__ import print_function
from typing import List
@ -13,7 +11,7 @@ class YapfLinter(base.LinterBase):
def __init__(self):
# type: () -> None
"""Create a yapf linter."""
super(YapfLinter, self).__init__("yapf", "yapf 0.21.0")
super(YapfLinter, self).__init__("yapf", "yapf 0.26.0")
def get_lint_version_cmd_args(self):
# type: () -> List[str]

14
buildscripts/make_archive.py
View File

@ -96,14 +96,14 @@ def make_tar_archive(opts):
enclosing_file_directory = os.path.dirname(temp_file_location)
if not os.path.exists(enclosing_file_directory):
os.makedirs(enclosing_file_directory)
print "copying %s => %s" % (input_filename, temp_file_location)
print("copying %s => %s" % (input_filename, temp_file_location))
if os.path.isdir(input_filename):
shutil.copytree(input_filename, temp_file_location)
else:
shutil.copy2(input_filename, temp_file_location)
tar_command.append(preferred_filename)
print " ".join(tar_command)
print(" ".join(tar_command))
# execute the full tar command
run_directory = os.path.join(os.getcwd(), enclosing_archive_directory)
proc = Popen(tar_command, stdout=PIPE, stderr=STDOUT, bufsize=0, cwd=run_directory)
@ -137,10 +137,10 @@ def parse_options(args):
parser = optparse.OptionParser()
parser.add_option('-o', dest='output_filename', default=None,
help='Name of the archive to output.', metavar='FILE')
parser.add_option('--format', dest='archive_format', default=None, choices=('zip', 'tar',
'tgz'),
help=('Format of archive to create. '
'If omitted, use the suffix of the output filename to decide.'))
parser.add_option(
'--format', dest='archive_format', default=None, choices=('zip', 'tar', 'tgz'),
help=('Format of archive to create. '
'If omitted, use the suffix of the output filename to decide.'))
parser.add_option('--transform', action='append', dest='transformations', default=[])
(opts, input_filenames) = parser.parse_args(args)
@ -171,7 +171,7 @@ def parse_options(args):
xform.replace(os.path.altsep or os.path.sep, os.path.sep).split('=', 1)
for xform in opts.transformations
]
except Exception, err: # pylint: disable=broad-except
except Exception as err: # pylint: disable=broad-except
parser.error(err)
return opts

13
buildscripts/make_vcxproj.py
View File

@ -12,13 +12,12 @@ To build mongodb, you must use scons. You can use this project to navigate code
where FILE_NAME is the of the file to generate e.g., "mongod"
"""
from __future__ import absolute_import, print_function
import io
import json
import os
import re
import StringIO
import io
import sys
import uuid
import xml.etree.ElementTree as ET
@ -111,7 +110,7 @@ def _replace_vcxproj(file_name, restore_elements):
saved_value = restore_elements[(parent.tag, child.tag, cond)]
child.text = saved_value
stream = StringIO.StringIO()
stream = io.StringIO()
tree.write(stream)
@ -173,10 +172,10 @@ class ProjFileGenerator(object): # pylint: disable=too-many-instance-attributes
for command in self.compiles:
defines = command["defines"].difference(common_defines)
if defines:
self.vcxproj.write(
" <ClCompile Include=\"" + command["file"] + "\"><PreprocessorDefinitions>" +
';'.join(defines) + ";%(PreprocessorDefinitions)" +
"</PreprocessorDefinitions></ClCompile>\n")
self.vcxproj.write(" <ClCompile Include=\"" + command["file"] +
"\"><PreprocessorDefinitions>" + ';'.join(defines) +
";%(PreprocessorDefinitions)" +
"</PreprocessorDefinitions></ClCompile>\n")
else:
self.vcxproj.write(" <ClCompile Include=\"" + command["file"] + "\" />\n")
self.vcxproj.write(" </ItemGroup>\n")

9
buildscripts/mobile/adb_monitor.py
View File

@ -132,14 +132,14 @@ class Adb(object):
def systrace_stop(self, output_file=None):
"""Stop the systrace.py script."""
self._cmd.send_to_process("bye")
self._cmd.send_to_process(b"bye")
with open(self._tempfile) as fh:
buff = fh.read()
os.remove(self._tempfile)
self.logger.debug("systrace_stop: %s", buff)
if "Wrote trace" not in buff:
self.logger.error("CPU file not saved: %s", buff)
if os.path.isfile(output_file):
if output_file and os.path.isfile(output_file):
os.remove(output_file)
@ -410,8 +410,9 @@ def main(): #pylint: disable=too-many-statements
output_files[options.cpu_file] = fileops.getmtime(options.cpu_file)
LOGGER.setLevel(options.log_level.upper())
LOGGER.info("This program can be cleanly terminated by issuing the following command:"
"\n\t\t'kill -INT %d'", os.getpid())
LOGGER.info(
"This program can be cleanly terminated by issuing the following command:"
"\n\t\t'kill -INT %d'", os.getpid())
adb = Adb(options.adb_binary)
LOGGER.info("Detected devices by adb:\n%s%s", adb.devices(), adb.device_available())

28
buildscripts/mobile/benchrun_embedded_setup_android.py
View File

@ -1,8 +1,6 @@
#!/usr/bin/env python
"""Setup an Android device to run the benchrun_embedded test suite."""
from __future__ import print_function
import glob
import logging
import optparse
@ -13,7 +11,7 @@ import sys
import tarfile
import tempfile
import time
import urllib
import urllib.request, urllib.parse, urllib.error
# pylint: disable=wrong-import-position
# Get relative imports to work when the package is not installed on the PYTHONPATH.
@ -30,7 +28,7 @@ def download_and_untar(url, root_dir):
"""Download url and untar into root_dir."""
temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".tgz").name
LOGGER.info("Downloading %s", url)
urllib.urlretrieve(url, temp_file)
urllib.request.urlretrieve(url, temp_file)
with tarfile.open(temp_file, "r:gz") as tar:
tar.extractall(root_dir)
os.remove(temp_file)
@ -130,10 +128,10 @@ def main():
help="The remote directory to store the embedded SDK files. Defaults to '%default'.",
default=posixpath.join(benchrun_root, "sdk"))
device_options.add_option("--benchrunJsonRemoteDir", dest="json_remote_dir",
help="The remote directory to store the benchrun JSON files."
" Defaults to '%default'.", default=posixpath.join(
benchrun_root, "testcases"))
device_options.add_option(
"--benchrunJsonRemoteDir", dest="json_remote_dir",
help="The remote directory to store the benchrun JSON files."
" Defaults to '%default'.", default=posixpath.join(benchrun_root, "testcases"))
sdk_url = "https://s3.amazonaws.com/mciuploads/mongodb-mongo-master/embedded-sdk-test/embedded-sdk-android-arm64-latest.tgz"
sdk_options.add_option(
@ -142,9 +140,10 @@ def main():
" any required shared object (.so) libraries. Defaults to '%default'."),
default=sdk_url)
sdk_options.add_option("--sdkLocalDir", dest="sdk_local_dir",
help="The local directory of embedded SDK files to be copied."
"If specified, overrides --sdkUrl.", default=None)
sdk_options.add_option(
"--sdkLocalDir", dest="sdk_local_dir",
help="The local directory of embedded SDK files to be copied."
"If specified, overrides --sdkUrl.", default=None)
sdk_options.add_option(
"--sdkSaveLocalDir", dest="sdk_save_local_dir",
@ -159,9 +158,10 @@ def main():
" files to be used in the benchrun embedded test."
" Defaults to '%default'."), default=json_url)
json_options.add_option("--benchrunJsonLocalDir", dest="json_local_dir",
help="The local directory of benchrun JSON files to be copied."
"If specified, overrides --benchrunJsonUrl.", default=None)
json_options.add_option(
"--benchrunJsonLocalDir", dest="json_local_dir",
help="The local directory of benchrun JSON files to be copied."
"If specified, overrides --benchrunJsonUrl.", default=None)
json_options.add_option(
"--benchrunJsonSaveLocalDir", dest="json_save_local_dir",

1
buildscripts/moduleconfig.py
View File

@ -23,7 +23,6 @@ alter those programs' behavior.
MongoDB module SConscript files can describe libraries, programs and unit tests, just as other
MongoDB SConscript files do.
"""
from __future__ import print_function
__all__ = ('discover_modules', 'discover_module_directories', 'configure_modules',
'register_module_test') # pylint: disable=undefined-all-variable

8
buildscripts/mongosymb.py
View File

@ -38,7 +38,7 @@ def symbolize_frames( # pylint: disable=too-many-locals
The somap_list is a list of dictionaries describing individual loaded libraries.
"""
return {so_entry["b"]: so_entry for so_entry in somap_list if so_entry.has_key("b")}
return {so_entry["b"]: so_entry for so_entry in somap_list if "b" in so_entry}
base_addr_map = make_base_addr_map(trace_doc["processInfo"]["somap"])
@ -52,7 +52,7 @@ def symbolize_frames( # pylint: disable=too-many-locals
addr_base = frame["b"]
else:
addr_base = soinfo.get("vmaddr", "0")
addr = long(addr_base, 16) + long(frame["o"], 16)
addr = int(addr_base, 16) + int(frame["o"], 16)
# addr currently points to the return address which is the one *after* the call. x86 is
# variable length so going backwards is difficult. However llvm-symbolizer seems to do the
# right thing if we just subtract 1 byte here. This has the downside of also adjusting the
@ -152,8 +152,8 @@ class S3BuildidDbgFileResolver(object):
"""Download debug symbols from S3."""
subprocess.check_call(
['wget',
'https://s3.amazonaws.com/%s/%s.debug.gz' %
(self._s3_bucket, build_id)], cwd=self._cache_dir)
'https://s3.amazonaws.com/%s/%s.debug.gz' % (self._s3_bucket, build_id)],
cwd=self._cache_dir)
subprocess.check_call(['gunzip', build_id + ".debug.gz"], cwd=self._cache_dir)

2
buildscripts/msitrim.py
View File

@ -1,7 +1,5 @@
"""Script to fix up our MSI files."""
from __future__ import print_function
import argparse
import shutil

53
buildscripts/packager.py
View File

@ -249,14 +249,11 @@ class Distro(object):
self.dname, self.repo_os_version(build_os), repo_directory, self.repo_component(),
self.archname(arch))
elif re.search("(redhat|fedora|centos|amazon)", self.dname):
return "repo/yum/%s/%s/mongodb-org/%s/%s/RPMS/" % (self.dname,
self.repo_os_version(build_os),
repo_directory, self.archname(arch))
return "repo/yum/%s/%s/mongodb-org/%s/%s/RPMS/" % (
self.dname, self.repo_os_version(build_os), repo_directory, self.archname(arch))
elif re.search("(suse)", self.dname):
return "repo/zypper/%s/%s/mongodb-org/%s/%s/RPMS/" % (self.dname,
self.repo_os_version(build_os),
repo_directory,
self.archname(arch))
return "repo/zypper/%s/%s/mongodb-org/%s/%s/RPMS/" % (
self.dname, self.repo_os_version(build_os), repo_directory, self.archname(arch))
else:
raise Exception("BUG: unsupported platform?")
@ -409,7 +406,7 @@ def main():
prefix = args.prefix
if prefix is None:
prefix = tempfile.mkdtemp()
print "Working in directory %s" % prefix
print("Working in directory %s" % prefix)
os.chdir(prefix)
try:
@ -449,7 +446,7 @@ def crossproduct(*seqs):
def sysassert(argv):
"""Run argv and assert that it exited with status 0."""
print "In %s, running %s" % (os.getcwd(), " ".join(argv))
print("In %s, running %s" % (os.getcwd(), " ".join(argv)))
sys.stdout.flush()
sys.stderr.flush()
assert subprocess.Popen(argv).wait() == 0
@ -457,7 +454,7 @@ def sysassert(argv):
def backtick(argv):
"""Run argv and return its output string."""
print "In %s, running %s" % (os.getcwd(), " ".join(argv))
print("In %s, running %s" % (os.getcwd(), " ".join(argv)))
sys.stdout.flush()
sys.stderr.flush()
return subprocess.Popen(argv, stdout=subprocess.PIPE).communicate()[0]
@ -493,11 +490,11 @@ def unpack_binaries_into(build_os, arch, spec, where):
sysassert(["tar", "xvzf", rootdir + "/" + tarfile(build_os, arch, spec)])
release_dir = glob('mongodb-linux-*')[0]
for releasefile in "bin", "LICENSE-Community.txt", "README", "THIRD-PARTY-NOTICES", "THIRD-PARTY-NOTICES.gotools", "MPL-2":
print "moving file: %s/%s" % (release_dir, releasefile)
print("moving file: %s/%s" % (release_dir, releasefile))
os.rename("%s/%s" % (release_dir, releasefile), releasefile)
os.rmdir(release_dir)
except Exception:
exc = sys.exc_value
exc = sys.exc_info()[1]
os.chdir(rootdir)
raise exc
os.chdir(rootdir)
@ -515,7 +512,7 @@ def make_package(distro, build_os, arch, spec, srcdir):
# directory, so the debian directory is needed in all cases (and
# innocuous in the debianoids' sdirs).
for pkgdir in ["debian", "rpm"]:
print "Copying packaging files from %s to %s" % ("%s/%s" % (srcdir, pkgdir), sdir)
print("Copying packaging files from %s to %s" % ("%s/%s" % (srcdir, pkgdir), sdir))
# FIXME: sh-dash-cee is bad. See if tarfile can do this.
sysassert([
"sh", "-c",
@ -609,11 +606,13 @@ def make_deb_repo(repo, distro, build_os):
oldpwd = os.getcwd()
os.chdir(repo + "../../../../../../")
try:
dirs = set(
[os.path.dirname(deb)[2:] for deb in backtick(["find", ".", "-name", "*.deb"]).split()])
dirs = set([
os.path.dirname(deb)[2:]
for deb in backtick(["find", ".", "-name", "*.deb"]).decode('utf-8').split()
])
for directory in dirs:
st = backtick(["dpkg-scanpackages", directory, "/dev/null"])
with open(directory + "/Packages", "w") as fh:
with open(directory + "/Packages", "wb") as fh:
fh.write(st)
bt = backtick(["gzip", "-9c", directory + "/Packages"])
with open(directory + "/Packages.gz", "wb") as fh:
@ -639,8 +638,8 @@ Description: MongoDB packages
os.chdir(repo + "../../")
s2 = backtick(["apt-ftparchive", "release", "."])
try:
with open("Release", 'w') as fh:
fh.write(s1)
with open("Release", 'wb') as fh:
fh.write(s1.encode('utf-8'))
fh.write(s2)
finally:
os.chdir(oldpwd)
@ -662,7 +661,7 @@ def move_repos_into_place(src, dst): # pylint: disable=too-many-branches
os.mkdir(dname)
break
except OSError:
exc = sys.exc_value
exc = sys.exc_info()[1]
if exc.errno == errno.EEXIST:
pass
else:
@ -682,7 +681,7 @@ def move_repos_into_place(src, dst): # pylint: disable=too-many-branches
os.symlink(dname, tmpnam)
break
except OSError: # as exc: # Python >2.5
exc = sys.exc_value
exc = sys.exc_info()[1]
if exc.errno == errno.EEXIST:
pass
else:
@ -700,7 +699,7 @@ def move_repos_into_place(src, dst): # pylint: disable=too-many-branches
os.symlink(os.readlink(dst), oldnam)
break
except OSError: # as exc: # Python >2.5
exc = sys.exc_value
exc = sys.exc_info()[1]
if exc.errno == errno.EEXIST:
pass
else:
@ -717,9 +716,10 @@ def write_debian_changelog(path, spec, srcdir):
os.chdir(srcdir)
preamble = ""
try:
sb = preamble + backtick(
["sh", "-c",
"git archive %s debian/changelog | tar xOf -" % spec.metadata_gitspec()])
sb = preamble + backtick([
"sh", "-c",
"git archive %s debian/changelog | tar xOf -" % spec.metadata_gitspec()
]).decode('utf-8')
finally:
os.chdir(oldcwd)
lines = sb.split("\n")
@ -789,7 +789,8 @@ def make_rpm(distro, build_os, arch, spec, srcdir): # pylint: disable=too-many-
# --macros will be used in Ubuntu.
#
macrofiles = [
l for l in backtick(["rpm", "--showrc"]).split("\n") if l.startswith("macrofiles")
l for l in backtick(["rpm", "--showrc"]).decode('utf-8').split("\n")
if l.startswith("macrofiles")
]
flags = []
macropath = os.getcwd() + "/macros"
@ -877,7 +878,7 @@ def ensure_dir(filename):
try:
os.makedirs(dirpart)
except OSError: # as exc: # Python >2.5
exc = sys.exc_value
exc = sys.exc_info()[1]
if exc.errno == errno.EEXIST:
pass
else:

24
buildscripts/packager_enterprise.py
View File

@ -36,7 +36,9 @@ import sys
import tempfile
import time
import packager # pylint: disable=relative-import
sys.path.append(os.getcwd())
import packager
# The MongoDB names for the architectures we support.
ARCH_CHOICES = ["x86_64", "ppc64le", "s390x", "arm64"]
@ -161,7 +163,7 @@ def main():
if prefix is None:
prefix = tempfile.mkdtemp()
print "Working in directory %s" % prefix
print("Working in directory %s" % prefix)
os.chdir(prefix)
try:
@ -222,7 +224,7 @@ def unpack_binaries_into(build_os, arch, spec, where):
os.rename("%s/%s" % (release_dir, releasefile), releasefile)
os.rmdir(release_dir)
except Exception:
exc = sys.exc_value
exc = sys.exc_info()[1]
os.chdir(rootdir)
raise exc
os.chdir(rootdir)
@ -240,7 +242,7 @@ def make_package(distro, build_os, arch, spec, srcdir):
# directory, so the debian directory is needed in all cases (and
# innocuous in the debianoids' sdirs).
for pkgdir in ["debian", "rpm"]:
print "Copying packaging files from %s to %s" % ("%s/%s" % (srcdir, pkgdir), sdir)
print("Copying packaging files from %s to %s" % ("%s/%s" % (srcdir, pkgdir), sdir))
# FIXME: sh-dash-cee is bad. See if tarfile can do this.
packager.sysassert([
"sh", "-c",
@ -277,11 +279,11 @@ def make_deb_repo(repo, distro, build_os):
try:
dirs = set([
os.path.dirname(deb)[2:]
for deb in packager.backtick(["find", ".", "-name", "*.deb"]).split()
for deb in packager.backtick(["find", ".", "-name", "*.deb"]).decode('utf-8').split()
])
for directory in dirs:
st = packager.backtick(["dpkg-scanpackages", directory, "/dev/null"])
with open(directory + "/Packages", "w") as fh:
with open(directory + "/Packages", "wb") as fh:
fh.write(st)
bt = packager.backtick(["gzip", "-9c", directory + "/Packages"])
with open(directory + "/Packages.gz", "wb") as fh:
@ -307,8 +309,8 @@ Description: MongoDB packages
os.chdir(repo + "../../")
s2 = packager.backtick(["apt-ftparchive", "release", "."])
try:
with open("Release", 'w') as fh:
fh.write(s1)
with open("Release", 'wb') as fh:
fh.write(s1.encode('utf-8'))
fh.write(s2)
finally:
os.chdir(oldpwd)
@ -330,7 +332,7 @@ def move_repos_into_place(src, dst): # pylint: disable=too-many-branches
os.mkdir(dname)
break
except OSError:
exc = sys.exc_value
exc = sys.exc_info()[1]
if exc.errno == errno.EEXIST:
pass
else:
@ -350,7 +352,7 @@ def move_repos_into_place(src, dst): # pylint: disable=too-many-branches
os.symlink(dname, tmpnam)
break
except OSError: # as exc: # Python >2.5
exc = sys.exc_value
exc = sys.exc_info()[1]
if exc.errno == errno.EEXIST:
pass
else:
@ -368,7 +370,7 @@ def move_repos_into_place(src, dst): # pylint: disable=too-many-branches
os.symlink(os.readlink(dst), oldnam)
break
except OSError: # as exc: # Python >2.5
exc = sys.exc_value
exc = sys.exc_info()[1]
if exc.errno == errno.EEXIST:
pass
else:

12
buildscripts/promote_silent_failures.py
View File

@ -5,9 +5,6 @@ Any test files with at least 2 executions in the report.json file that have a "s
this script will change the outputted report to have a "fail" status instead.
"""
from __future__ import absolute_import
from __future__ import print_function
import collections
import json
import optparse
@ -31,10 +28,11 @@ def main():
usage = "usage: %prog [options] report.json"
parser = optparse.OptionParser(usage=usage)
parser.add_option("-o", "--output-file", dest="outfile", default="-",
help=("If '-', then the report file is written to stdout."
" Any other value is treated as the output file name. By default,"
" output is written to stdout."))
parser.add_option(
"-o", "--output-file", dest="outfile", default="-",
help=("If '-', then the report file is written to stdout."
" Any other value is treated as the output file name. By default,"
" output is written to stdout."))
(options, args) = parser.parse_args()

5
buildscripts/pylinters.py
View File

@ -1,7 +1,5 @@
#!/usr/bin/env python2
"""Extensible script to run one or more Python Linters across a subset of files in parallel."""
from __future__ import absolute_import
from __future__ import print_function
import argparse
import logging
@ -135,7 +133,8 @@ def _fix_files(linters, config_dict, file_names):
sys.exit(1)
for linter in linter_instances:
run_linter = lambda param1: lint_runner.run(linter.cmd_path + linter.linter.get_fix_cmd_args(param1)) # pylint: disable=cell-var-from-loop
run_linter = lambda param1: lint_runner.run(linter.cmd_path + linter.linter. # pylint: disable=cell-var-from-loop
get_fix_cmd_args(param1)) # pylint: disable=cell-var-from-loop
lint_clean = parallel.parallel_process([os.path.abspath(f) for f in file_names], run_linter)

136
buildscripts/remote_operations.py
View File

@ -1,8 +1,6 @@
#!/usr/bin/env python
"""Remote access utilities, via ssh & scp."""
from __future__ import print_function
import optparse
import os
import posixpath
@ -10,20 +8,7 @@ import re
import shlex
import sys
import time
# The subprocess32 module is untested on Windows and thus isn't recommended for use, even when it's
# installed. See https://github.com/google/python-subprocess32/blob/3.2.7/README.md#usage.
if os.name == "posix" and sys.version_info[0] == 2:
try:
import subprocess32 as subprocess
except ImportError:
import warnings
warnings.warn(("Falling back to using the subprocess module because subprocess32 isn't"
" available. When using the subprocess module, a child process may trigger"
" an invalid free(). See SERVER-22219 for more details."), RuntimeWarning)
import subprocess # type: ignore
else:
import subprocess
import subprocess
# Get relative imports to work when the package is not installed on the PYTHONPATH.
if __name__ == "__main__" and __package__ is None:
@ -231,44 +216,50 @@ def main(): # pylint: disable=too-many-branches,too-many-statements
shell_options = optparse.OptionGroup(parser, "Shell options")
copy_options = optparse.OptionGroup(parser, "Copy options")
parser.add_option("--userHost", dest="user_host", default=None,
help=("User and remote host to execute commands on [REQUIRED]."
" Examples, 'user@1.2.3.4' or 'user@myhost.com'."))
parser.add_option(
"--userHost", dest="user_host", default=None,
help=("User and remote host to execute commands on [REQUIRED]."
" Examples, 'user@1.2.3.4' or 'user@myhost.com'."))
parser.add_option("--operation", dest="operation", default="shell", choices=_OPERATIONS,
help=("Remote operation to perform, choose one of '{}',"
" defaults to '%default'.".format(", ".join(_OPERATIONS))))
parser.add_option(
"--operation", dest="operation", default="shell", choices=_OPERATIONS,
help=("Remote operation to perform, choose one of '{}',"
" defaults to '%default'.".format(", ".join(_OPERATIONS))))
control_options.add_option("--sshConnectionOptions", dest="ssh_connection_options",
default=None, action="append",
help=("SSH connection options which are common to ssh and scp."
" More than one option can be specified either"
" in one quoted string or by specifying"
" this option more than once. Example options:"
" '-i $HOME/.ssh/access.pem -o ConnectTimeout=10"
" -o ConnectionAttempts=10'"))
control_options.add_option(
"--sshConnectionOptions", dest="ssh_connection_options", default=None, action="append",
help=("SSH connection options which are common to ssh and scp."
" More than one option can be specified either"
" in one quoted string or by specifying"
" this option more than once. Example options:"
" '-i $HOME/.ssh/access.pem -o ConnectTimeout=10"
" -o ConnectionAttempts=10'"))
control_options.add_option("--sshOptions", dest="ssh_options", default=None, action="append",
help=("SSH specific options."
" More than one option can be specified either"
" in one quoted string or by specifying"
" this option more than once. Example options:"
" '-t' or '-T'"))
control_options.add_option(
"--sshOptions", dest="ssh_options", default=None, action="append",
help=("SSH specific options."
" More than one option can be specified either"
" in one quoted string or by specifying"
" this option more than once. Example options:"
" '-t' or '-T'"))
control_options.add_option("--scpOptions", dest="scp_options", default=None, action="append",
help=("SCP specific options."
" More than one option can be specified either"
" in one quoted string or by specifying"
" this option more than once. Example options:"
" '-l 5000'"))
control_options.add_option(
"--scpOptions", dest="scp_options", default=None, action="append",
help=("SCP specific options."
" More than one option can be specified either"
" in one quoted string or by specifying"
" this option more than once. Example options:"
" '-l 5000'"))
control_options.add_option("--retries", dest="retries", type=int, default=0,
help=("Number of retries to attempt for operation,"
" defaults to '%default'."))
control_options.add_option(
"--retries", dest="retries", type=int, default=0,
help=("Number of retries to attempt for operation,"
" defaults to '%default'."))
control_options.add_option("--retrySleep", dest="retry_sleep", type=int, default=10,
help=("Number of seconds to wait between retries,"
" defaults to '%default'."))
control_options.add_option(
"--retrySleep", dest="retry_sleep", type=int, default=10,
help=("Number of seconds to wait between retries,"
" defaults to '%default'."))
control_options.add_option("--debug", dest="debug", action="store_true", default=False,
help="Provides debug output.")
@ -276,32 +267,37 @@ def main(): # pylint: disable=too-many-branches,too-many-statements
control_options.add_option("--verbose", dest="verbose", action="store_true", default=False,
help="Print exit status and output at end.")
shell_options.add_option("--commands", dest="remote_commands", default=None, action="append",
help=("Commands to excute on the remote host. The"
" commands must be separated by a ';' and can either"
" be specifed in a quoted string or by specifying"
" this option more than once. A ';' will be added"
" between commands when this option is specifed"
" more than once."))
shell_options.add_option(
"--commands", dest="remote_commands", default=None, action="append",
help=("Commands to excute on the remote host. The"
" commands must be separated by a ';' and can either"
" be specifed in a quoted string or by specifying"
" this option more than once. A ';' will be added"
" between commands when this option is specifed"
" more than once."))
shell_options.add_option("--commandDir", dest="command_dir", default=None,
help=("Working directory on remote to execute commands"
" form. Defaults to remote login directory."))
shell_options.add_option(
"--commandDir", dest="command_dir", default=None,
help=("Working directory on remote to execute commands"
" form. Defaults to remote login directory."))
copy_options.add_option("--file", dest="files", default=None, action="append",
help=("The file to copy to/from remote host. To"
" support spaces in the file, each file must be"
" specified using this option more than once."))
copy_options.add_option(
"--file", dest="files", default=None, action="append",
help=("The file to copy to/from remote host. To"
" support spaces in the file, each file must be"
" specified using this option more than once."))
copy_options.add_option("--remoteDir", dest="remote_dir", default=None,
help=("Remote directory to copy to, only applies when"
" operation is 'copy_to'. Defaults to the login"
" directory on the remote host."))
copy_options.add_option(
"--remoteDir", dest="remote_dir", default=None,
help=("Remote directory to copy to, only applies when"
" operation is 'copy_to'. Defaults to the login"
" directory on the remote host."))
copy_options.add_option("--localDir", dest="local_dir", default=".",
help=("Local directory to copy to, only applies when"
" operation is 'copy_from'. Defaults to the"
" current directory, '%default'."))
copy_options.add_option(
"--localDir", dest="local_dir", default=".",
help=("Local directory to copy to, only applies when"
" operation is 'copy_from'. Defaults to the"
" current directory, '%default'."))
parser.add_option_group(control_options)
parser.add_option_group(shell_options)

5
buildscripts/resmoke.py
View File

@ -1,8 +1,6 @@
#!/usr/bin/env python
"""Command line utility for executing MongoDB tests of all kinds."""
from __future__ import absolute_import
import os.path
import platform
import random
@ -331,7 +329,8 @@ class Resmoke(object): # pylint: disable=too-many-instance-attributes
curator_exists = os.path.isfile(curator_path)
curator_same_version = False
if curator_exists:
curator_version = subprocess.check_output([curator_path, "--version"]).split()
curator_version = subprocess.check_output([curator_path,
"--version"]).decode('utf-8').split()
curator_same_version = git_hash in curator_version
if curator_exists and not curator_same_version:

1
buildscripts/resmokeconfig/__init__.py
View File

@ -1,5 +1,4 @@
"""Resmokeconfig module."""
from __future__ import absolute_import
from .suites import NAMED_SUITES
from .loggers import NAMED_LOGGERS

4
buildscripts/resmokeconfig/loggers/__init__.py
View File

@ -1,7 +1,5 @@
"""Defines a mapping of shortened names for logger configuration files to their full path."""
from __future__ import absolute_import
import os
import os.path
@ -16,7 +14,7 @@ def _get_named_loggers():
named_loggers = {}
try:
(root, _dirs, files) = os.walk(dirname).next()
(root, _dirs, files) = next(os.walk(dirname))
for filename in files:
(short_name, ext) = os.path.splitext(filename)
if ext in (".yml", ".yaml"):

4
buildscripts/resmokeconfig/suites/__init__.py
View File

@ -1,7 +1,5 @@
"""Defines a mapping of shortened names for suite configuration files to their full path."""
from __future__ import absolute_import
import os
import os.path
@ -16,7 +14,7 @@ def _get_named_suites():
named_suites = {}
try:
(root, _dirs, files) = os.walk(dirname).next()
(root, _dirs, files) = next(os.walk(dirname))
for filename in files:
(short_name, ext) = os.path.splitext(filename)
if ext in (".yml", ".yaml"):

2
buildscripts/resmokelib/__init__.py
View File

@ -1,7 +1,5 @@
"""Empty."""
from __future__ import absolute_import
from . import config
from . import errors
from . import logging

31
buildscripts/resmokelib/config.py
View File

@ -1,7 +1,5 @@
"""Configuration options for resmoke.py."""
from __future__ import absolute_import
import collections
import datetime
import itertools
@ -75,7 +73,7 @@ DEFAULTS = {
"repeat_tests_secs": None,
"report_failure_status": "fail",
"report_file": None,
"seed": long(time.time() * 256), # Taken from random.py code in Python 2.7.
"seed": int(time.time() * 256), # Taken from random.py code in Python 2.7.
"service_executor": None,
"shell_conn_string": None,
"shell_port": None,
@ -181,18 +179,19 @@ class SuiteOptions(_SuiteOptions):
description = None
include_tags = None
parent = dict(
zip(SuiteOptions._fields, [
description,
FAIL_FAST,
include_tags,
JOBS,
REPEAT_SUITES,
REPEAT_TESTS,
REPEAT_TESTS_MAX,
REPEAT_TESTS_MIN,
REPEAT_TESTS_SECS,
REPORT_FAILURE_STATUS,
]))
list(
zip(SuiteOptions._fields, [
description,
FAIL_FAST,
include_tags,
JOBS,
REPEAT_SUITES,
REPEAT_TESTS,
REPEAT_TESTS_MAX,
REPEAT_TESTS_MIN,
REPEAT_TESTS_SECS,
REPORT_FAILURE_STATUS,
])))
options = self._asdict()
for field in SuiteOptions._fields:
@ -203,7 +202,7 @@ class SuiteOptions(_SuiteOptions):
SuiteOptions.ALL_INHERITED = SuiteOptions( # type: ignore
**dict(zip(SuiteOptions._fields, itertools.repeat(SuiteOptions.INHERIT))))
**dict(list(zip(SuiteOptions._fields, itertools.repeat(SuiteOptions.INHERIT)))))
##
# Variables that are set by the user at the command line or with --options.

1
buildscripts/resmokelib/core/__init__.py
View File

@ -1,5 +1,4 @@
"""Resmokelib core module."""
from __future__ import absolute_import
from . import process
from . import programs

2
buildscripts/resmokelib/core/jasper_process.py
View File

@ -3,8 +3,6 @@
Serves as an alternative to process.py.
"""
from __future__ import absolute_import
import sys
try:

2
buildscripts/resmokelib/core/network.py
View File

@ -1,7 +1,5 @@
"""Class used to allocate ports for mongod and mongos processes involved in running the tests."""
from __future__ import absolute_import
import collections
import functools
import threading

2
buildscripts/resmokelib/core/pipe.py
View File

@ -5,8 +5,6 @@ Used to avoid deadlocks from the pipe buffer filling up and blocking the subproc
being waited on.
"""
from __future__ import absolute_import
import threading

27
buildscripts/resmokelib/core/process.py
View File

@ -4,34 +4,13 @@ Uses job objects when running on Windows to ensure that all created
processes are terminated.
"""
from __future__ import absolute_import
import atexit
import logging
import os
import os.path
import sys
import threading
# The subprocess32 module resolves the thread-safety issues of the subprocess module in Python 2.x
# when the _posixsubprocess C extension module is also available. Additionally, the _posixsubprocess
# C extension module avoids triggering invalid free() calls on Python's internal data structure for
# thread-local storage by skipping the PyOS_AfterFork() call when the 'preexec_fn' parameter isn't
# specified to subprocess.Popen(). See SERVER-22219 for more details.
#
# The subprocess32 module is untested on Windows and thus isn't recommended for use, even when it's
# installed. See https://github.com/google/python-subprocess32/blob/3.2.7/README.md#usage.
if os.name == "posix" and sys.version_info[0] == 2:
try:
import subprocess32 as subprocess
except ImportError:
import warnings
warnings.warn(("Falling back to using the subprocess module because subprocess32 isn't"
" available. When using the subprocess module, a child process may trigger"
" an invalid free(). See SERVER-22219 for more details."), RuntimeWarning)
import subprocess # type: ignore
else:
import subprocess
import subprocess
from . import pipe # pylint: disable=wrong-import-position
from .. import utils # pylint: disable=wrong-import-position
@ -182,8 +161,8 @@ class Process(object):
finally:
win32api.CloseHandle(mongo_signal_handle)
print "Failed to cleanly exit the program, calling TerminateProcess() on PID: " +\
str(self._process.pid)
print("Failed to cleanly exit the program, calling TerminateProcess() on PID: " +\
str(self._process.pid))
# Adapted from implementation of Popen.terminate() in subprocess.py of Python 2.7
# because earlier versions do not catch exceptions.

2
buildscripts/resmokelib/core/programs.py
View File

@ -3,8 +3,6 @@
Handles all the nitty-gritty parameter conversion.
"""
from __future__ import absolute_import
import json
import os
import os.path

2
buildscripts/resmokelib/logging/__init__.py
View File

@ -1,7 +1,5 @@
"""Extension to the logging package to support buildlogger."""
from __future__ import absolute_import
# Alias the built-in logging.Logger class for type checking arguments. Those interested in
# constructing a new Logger instance should use the loggers.new_logger() function instead.
from logging import Logger

32
buildscripts/resmokelib/logging/buildlogger.py
View File

@ -1,7 +1,5 @@
"""Define handlers for communicating with a buildlogger server."""
from __future__ import absolute_import
import functools
import json
import os
@ -85,7 +83,7 @@ class _LogsSplitter(object):
2 is added to each string size to account for the array representation of the logs,
as each line is preceded by a '[' or a space and followed by a ',' or a ']'.
"""
return len(json.dumps(line, encoding="utf-8")) + 2
return len(json.dumps(line)) + 2
curr_logs = []
curr_logs_size = 0
@ -200,8 +198,8 @@ class _BaseBuildloggerHandler(handlers.BufferedHandler):
# writing the messages to the fallback logkeeper to avoid putting additional pressure on
# the Evergreen database.
BUILDLOGGER_FALLBACK.warning(
"Failed to flush all log output (%d messages) to logkeeper.", len(
self.retry_buffer))
"Failed to flush all log output (%d messages) to logkeeper.",
len(self.retry_buffer))
# We set a flag to indicate that we failed to flush all log output to logkeeper so
# resmoke.py can exit with a special return code.
@ -226,10 +224,11 @@ class BuildloggerTestHandler(_BaseBuildloggerHandler):
@_log_on_error
def _finish_test(self, failed=False):
"""Send a POST request to the APPEND_TEST_LOGS_ENDPOINT with the test status."""
self.post(self.endpoint, headers={
"X-Sendlogs-Test-Done": "true",
"X-Sendlogs-Test-Failed": "true" if failed else "false",
})
self.post(
self.endpoint, headers={
"X-Sendlogs-Test-Done": "true",
"X-Sendlogs-Test-Failed": "true" if failed else "false",
})
def close(self):
"""Close the buildlogger handler."""
@ -262,7 +261,9 @@ class BuildloggerServer(object):
"""Initialize BuildloggerServer."""
tmp_globals = {}
self.config = {}
execfile(_BUILDLOGGER_CONFIG, tmp_globals, self.config)
exec(
compile(open(_BUILDLOGGER_CONFIG, "rb").read(), _BUILDLOGGER_CONFIG, 'exec'),
tmp_globals, self.config)
# Rename "slavename" to "username" if present.
if "slavename" in self.config and "username" not in self.config:
@ -285,11 +286,12 @@ class BuildloggerServer(object):
handler = handlers.HTTPHandler(url_root=_config.BUILDLOGGER_URL, username=username,
password=password, should_retry=True)
response = handler.post(CREATE_BUILD_ENDPOINT, data={
"builder": builder,
"buildnum": build_num,
"task_id": _config.EVERGREEN_TASK_ID,
})
response = handler.post(
CREATE_BUILD_ENDPOINT, data={
"builder": builder,
"buildnum": build_num,
"task_id": _config.EVERGREEN_TASK_ID,
})
return response["id"]

7
buildscripts/resmokelib/logging/flush.py
View File

@ -3,14 +3,11 @@
These instances are used to send logs to buildlogger.
"""
from __future__ import absolute_import
import logging
import sched
import threading
import time
from ..utils import scheduler
_FLUSH_THREAD_LOCK = threading.Lock()
_FLUSH_THREAD = None
@ -96,7 +93,7 @@ class _FlushThread(threading.Thread):
self.__schedule_updated.wait(secs)
self.__schedule_updated.clear()
self.__scheduler = scheduler.Scheduler(time.time, interruptible_sleep)
self.__scheduler = sched.scheduler(time.time, interruptible_sleep)
self.__schedule_updated = threading.Event()
self.__should_stop = threading.Event()
self.__terminated = threading.Event()

2
buildscripts/resmokelib/logging/formatters.py
View File

@ -1,7 +1,5 @@
"""Custom formatters for the logging handlers."""
from __future__ import absolute_import
import logging
import time

12
buildscripts/resmokelib/logging/handlers.py
View File

@ -1,7 +1,5 @@
"""Additional handlers that are used as the base classes of the buildlogger handler."""
from __future__ import absolute_import
import json
import logging
import sys
@ -193,18 +191,15 @@ class HTTPHandler(object):
"""
data = utils.default_if_none(data, [])
data = json.dumps(data, encoding="utf-8")
data = json.dumps(data)
headers = utils.default_if_none(headers, {})
headers["Content-Type"] = "application/json; charset=utf-8"
url = self._make_url(endpoint)
# Versions of Python earlier than 2.7.9 do not support certificate validation. So we
# disable certificate validation for older Python versions.
should_validate_certificates = sys.version_info >= (2, 7, 9)
with warnings.catch_warnings():
if urllib3_exceptions is not None and not should_validate_certificates:
if urllib3_exceptions is not None:
try:
warnings.simplefilter("ignore", urllib3_exceptions.InsecurePlatformWarning)
except AttributeError:
@ -222,8 +217,7 @@ class HTTPHandler(object):
pass
response = self.session.post(url, data=data, headers=headers, timeout=timeout_secs,
auth=self.auth_handler,
verify=should_validate_certificates)
auth=self.auth_handler, verify=True)
response.raise_for_status()

6
buildscripts/resmokelib/logging/loggers.py
View File

@ -1,7 +1,5 @@
"""Module to hold the logger instances themselves."""
from __future__ import absolute_import
import logging
import sys
@ -304,8 +302,8 @@ class FixtureNodeLogger(BaseLogger):
def new_fixture_node_logger(self, node_name):
"""Create a new child FixtureNodeLogger."""
return FixtureNodeLogger(self.fixture_class, self.job_num, "%s:%s" % (self.node_name,
node_name), self)
return FixtureNodeLogger(self.fixture_class, self.job_num,
"%s:%s" % (self.node_name, node_name), self)
class TestsRootLogger(RootLogger):

340
buildscripts/resmokelib/parser.py
View File

@ -1,7 +1,5 @@
"""Parser for command line arguments."""
from __future__ import absolute_import
import collections
import os
import os.path
@ -26,40 +24,45 @@ def _make_parser(): # pylint: disable=too-many-statements
"""Create and return the command line arguments parser."""
parser = optparse.OptionParser()
parser.add_option("--suites", dest="suite_files", metavar="SUITE1,SUITE2",
help=("Comma separated list of YAML files that each specify the configuration"
" of a suite. If the file is located in the resmokeconfig/suites/"
" directory, then the basename without the .yml extension can be"
" specified, e.g. 'core'. If a list of files is passed in as"
" positional arguments, they will be run using the suites'"
" configurations"))
parser.add_option(
"--suites", dest="suite_files", metavar="SUITE1,SUITE2",
help=("Comma separated list of YAML files that each specify the configuration"
" of a suite. If the file is located in the resmokeconfig/suites/"
" directory, then the basename without the .yml extension can be"
" specified, e.g. 'core'. If a list of files is passed in as"
" positional arguments, they will be run using the suites'"
" configurations"))
parser.add_option("--log", dest="logger_file", metavar="LOGGER",
help=("A YAML file that specifies the logging configuration. If the file is"
" located in the resmokeconfig/suites/ directory, then the basename"
" without the .yml extension can be specified, e.g. 'console'."))
parser.add_option(
"--log", dest="logger_file", metavar="LOGGER",
help=("A YAML file that specifies the logging configuration. If the file is"
" located in the resmokeconfig/suites/ directory, then the basename"
" without the .yml extension can be specified, e.g. 'console'."))
parser.add_option("--archiveFile", dest="archive_file", metavar="ARCHIVE_FILE",
help=("Sets the archive file name for the Evergreen task running the tests."
" The archive file is JSON format containing a list of tests that were"
" successfully archived to S3. If unspecified, no data files from tests"
" will be archived in S3. Tests can be designated for archival in the"
" task suite configuration file."))
parser.add_option(
"--archiveFile", dest="archive_file", metavar="ARCHIVE_FILE",
help=("Sets the archive file name for the Evergreen task running the tests."
" The archive file is JSON format containing a list of tests that were"
" successfully archived to S3. If unspecified, no data files from tests"
" will be archived in S3. Tests can be designated for archival in the"
" task suite configuration file."))
parser.add_option("--archiveLimitMb", type="int", dest="archive_limit_mb",
metavar="ARCHIVE_LIMIT_MB",
help=("Sets the limit (in MB) for archived files to S3. A value of 0"
" indicates there is no limit."))
parser.add_option(
"--archiveLimitMb", type="int", dest="archive_limit_mb", metavar="ARCHIVE_LIMIT_MB",
help=("Sets the limit (in MB) for archived files to S3. A value of 0"
" indicates there is no limit."))
parser.add_option("--archiveLimitTests", type="int", dest="archive_limit_tests",
metavar="ARCHIVE_LIMIT_TESTS",
help=("Sets the maximum number of tests to archive to S3. A value"
" of 0 indicates there is no limit."))
parser.add_option(
"--archiveLimitTests", type="int", dest="archive_limit_tests",
metavar="ARCHIVE_LIMIT_TESTS",
help=("Sets the maximum number of tests to archive to S3. A value"
" of 0 indicates there is no limit."))
parser.add_option("--basePort", dest="base_port", metavar="PORT",
help=("The starting port number to use for mongod and mongos processes"
" spawned by resmoke.py or the tests themselves. Each fixture and Job"
" allocates a contiguous range of ports."))
parser.add_option(
"--basePort", dest="base_port", metavar="PORT",
help=("The starting port number to use for mongod and mongos processes"
" spawned by resmoke.py or the tests themselves. Each fixture and Job"
" allocates a contiguous range of ports."))
parser.add_option("--buildloggerUrl", action="store", dest="buildlogger_url", metavar="URL",
help="The root url of the buildlogger server.")
@ -67,19 +70,19 @@ def _make_parser(): # pylint: disable=too-many-statements
parser.add_option("--continueOnFailure", action="store_true", dest="continue_on_failure",
help="Executes all tests in all suites, even if some of them fail.")
parser.add_option("--dbpathPrefix", dest="dbpath_prefix", metavar="PATH",
help=("The directory which will contain the dbpaths of any mongod's started"
" by resmoke.py or the tests themselves."))
parser.add_option(
"--dbpathPrefix", dest="dbpath_prefix", metavar="PATH",
help=("The directory which will contain the dbpaths of any mongod's started"
" by resmoke.py or the tests themselves."))
parser.add_option("--dbtest", dest="dbtest_executable", metavar="PATH",
help="The path to the dbtest executable for resmoke to use.")
parser.add_option("--excludeWithAnyTags", action="append", dest="exclude_with_any_tags",
metavar="TAG1,TAG2",
help=("Comma separated list of tags. Any jstest that contains any of the"
" specified tags will be excluded from any suites that are run."
" The tag '{}' is implicitly part of this list.".format(
_config.EXCLUDED_TAG)))
parser.add_option(
"--excludeWithAnyTags", action="append", dest="exclude_with_any_tags", metavar="TAG1,TAG2",
help=("Comma separated list of tags. Any jstest that contains any of the"
" specified tags will be excluded from any suites that are run."
" The tag '{}' is implicitly part of this list.".format(_config.EXCLUDED_TAG)))
parser.add_option("-f", "--findSuites", action="store_true", dest="find_suites",
help="Lists the names of the suites that will execute the specified tests.")
@ -87,29 +90,30 @@ def _make_parser(): # pylint: disable=too-many-statements
parser.add_option("--genny", dest="genny_executable", metavar="PATH",
help="The path to the genny executable for resmoke to use.")
parser.add_option("--spawnUsing", type="choice", dest="spawn_using", choices=("python",
"jasper"),
help=("Allows you to spawn resmoke processes using python or Jasper."
"Defaults to python. Options are 'python' or 'jasper'."))
parser.add_option(
"--spawnUsing", type="choice", dest="spawn_using", choices=("python", "jasper"),
help=("Allows you to spawn resmoke processes using python or Jasper."
"Defaults to python. Options are 'python' or 'jasper'."))
parser.add_option("--includeWithAnyTags", action="append", dest="include_with_any_tags",
metavar="TAG1,TAG2",
help=("Comma separated list of tags. For the jstest portion of the suite(s),"
" only tests which have at least one of the specified tags will be"
" run."))
parser.add_option(
"--includeWithAnyTags", action="append", dest="include_with_any_tags", metavar="TAG1,TAG2",
help=("Comma separated list of tags. For the jstest portion of the suite(s),"
" only tests which have at least one of the specified tags will be"
" run."))
parser.add_option("-n", action="store_const", const="tests", dest="dry_run",
help="Outputs the tests that would be run.")
# TODO: add support for --dryRun=commands
parser.add_option("--dryRun", type="choice", action="store", dest="dry_run",
choices=("off", "tests"), metavar="MODE",
help=("Instead of running the tests, outputs the tests that would be run"
" (if MODE=tests). Defaults to MODE=%default."))
parser.add_option(
"--dryRun", type="choice", action="store", dest="dry_run", choices=("off", "tests"),
metavar="MODE", help=("Instead of running the tests, outputs the tests that would be run"
" (if MODE=tests). Defaults to MODE=%default."))
parser.add_option("-j", "--jobs", type="int", dest="jobs", metavar="JOBS",
help=("The number of Job instances to use. Each instance will receive its"
" own MongoDB deployment to dispatch tests to."))
parser.add_option(
"-j", "--jobs", type="int", dest="jobs", metavar="JOBS",
help=("The number of Job instances to use. Each instance will receive its"
" own MongoDB deployment to dispatch tests to."))
parser.add_option("-l", "--listSuites", action="store_true", dest="list_suites",
help="Lists the names of the suites available to execute.")
@ -120,24 +124,27 @@ def _make_parser(): # pylint: disable=too-many-statements
parser.add_option("--mongod", dest="mongod_executable", metavar="PATH",
help="The path to the mongod executable for resmoke.py to use.")
parser.add_option("--mongodSetParameters", dest="mongod_set_parameters",
metavar="{key1: value1, key2: value2, ..., keyN: valueN}",
help=("Passes one or more --setParameter options to all mongod processes"
" started by resmoke.py. The argument is specified as bracketed YAML -"
" i.e. JSON with support for single quoted and unquoted keys."))
parser.add_option(
"--mongodSetParameters", dest="mongod_set_parameters",
metavar="{key1: value1, key2: value2, ..., keyN: valueN}",
help=("Passes one or more --setParameter options to all mongod processes"
" started by resmoke.py. The argument is specified as bracketed YAML -"
" i.e. JSON with support for single quoted and unquoted keys."))
parser.add_option("--mongoebench", dest="mongoebench_executable", metavar="PATH",
help=("The path to the mongoebench (benchrun embedded) executable for"
" resmoke.py to use."))
parser.add_option(
"--mongoebench", dest="mongoebench_executable", metavar="PATH",
help=("The path to the mongoebench (benchrun embedded) executable for"
" resmoke.py to use."))
parser.add_option("--mongos", dest="mongos_executable", metavar="PATH",
help="The path to the mongos executable for resmoke.py to use.")
parser.add_option("--mongosSetParameters", dest="mongos_set_parameters",
metavar="{key1: value1, key2: value2, ..., keyN: valueN}",
help=("Passes one or more --setParameter options to all mongos processes"
" started by resmoke.py. The argument is specified as bracketed YAML -"
" i.e. JSON with support for single quoted and unquoted keys."))
parser.add_option(
"--mongosSetParameters", dest="mongos_set_parameters",
metavar="{key1: value1, key2: value2, ..., keyN: valueN}",
help=("Passes one or more --setParameter options to all mongos processes"
" started by resmoke.py. The argument is specified as bracketed YAML -"
" i.e. JSON with support for single quoted and unquoted keys."))
parser.add_option("--nojournal", action="store_true", dest="no_journal",
help="Disables journaling for all mongod's.")
@ -148,55 +155,61 @@ def _make_parser(): # pylint: disable=too-many-statements
parser.add_option("--perfReportFile", dest="perf_report_file", metavar="PERF_REPORT",
help="Writes a JSON file with performance test results.")
parser.add_option("--shellConnString", dest="shell_conn_string", metavar="CONN_STRING",
help="Overrides the default fixture and connects with a mongodb:// connection"
" string to an existing MongoDB cluster instead. This is useful for"
" connecting to a MongoDB deployment started outside of resmoke.py including"
" one running in a debugger.")
parser.add_option(
"--shellConnString", dest="shell_conn_string", metavar="CONN_STRING",
help="Overrides the default fixture and connects with a mongodb:// connection"
" string to an existing MongoDB cluster instead. This is useful for"
" connecting to a MongoDB deployment started outside of resmoke.py including"
" one running in a debugger.")
parser.add_option("--shellPort", dest="shell_port", metavar="PORT",
help="Convenience form of --shellConnString for connecting to an"
" existing MongoDB cluster with the URL mongodb://localhost:[PORT]."
" This is useful for connecting to a server running in a debugger.")
parser.add_option(
"--shellPort", dest="shell_port", metavar="PORT",
help="Convenience form of --shellConnString for connecting to an"
" existing MongoDB cluster with the URL mongodb://localhost:[PORT]."
" This is useful for connecting to a server running in a debugger.")
parser.add_option("--repeat", "--repeatSuites", type="int", dest="repeat_suites", metavar="N",
help="Repeats the given suite(s) N times, or until one fails.")
parser.add_option("--repeatTests", type="int", dest="repeat_tests", metavar="N",
help="Repeats the tests inside each suite N times. This applies to tests"
" defined in the suite configuration as well as tests defined on the command"
" line.")
parser.add_option(
"--repeatTests", type="int", dest="repeat_tests", metavar="N",
help="Repeats the tests inside each suite N times. This applies to tests"
" defined in the suite configuration as well as tests defined on the command"
" line.")
parser.add_option("--repeatTestsMax", type="int", dest="repeat_tests_max", metavar="N",
help="Repeats the tests inside each suite no more than N time when"
" --repeatTestsSecs is specified. This applies to tests defined in the suite"
" configuration as well as tests defined on the command line.")
parser.add_option(
"--repeatTestsMax", type="int", dest="repeat_tests_max", metavar="N",
help="Repeats the tests inside each suite no more than N time when"
" --repeatTestsSecs is specified. This applies to tests defined in the suite"
" configuration as well as tests defined on the command line.")
parser.add_option("--repeatTestsMin", type="int", dest="repeat_tests_min", metavar="N",
help="Repeats the tests inside each suite at least N times when"
" --repeatTestsSecs is specified. This applies to tests defined in the suite"
" configuration as well as tests defined on the command line.")
parser.add_option(
"--repeatTestsMin", type="int", dest="repeat_tests_min", metavar="N",
help="Repeats the tests inside each suite at least N times when"
" --repeatTestsSecs is specified. This applies to tests defined in the suite"
" configuration as well as tests defined on the command line.")
parser.add_option("--repeatTestsSecs", type="float", dest="repeat_tests_secs",
metavar="SECONDS",
help="Repeats the tests inside each suite this amount of time. Note that"
" this option is mutually exclusive with --repeatTests. This applies to"
" tests defined in the suite configuration as well as tests defined on the"
" command line.")
parser.add_option(
"--repeatTestsSecs", type="float", dest="repeat_tests_secs", metavar="SECONDS",
help="Repeats the tests inside each suite this amount of time. Note that"
" this option is mutually exclusive with --repeatTests. This applies to"
" tests defined in the suite configuration as well as tests defined on the"
" command line.")
parser.add_option("--reportFailureStatus", type="choice", action="store",
dest="report_failure_status", choices=("fail",
"silentfail"), metavar="STATUS",
help="Controls if the test failure status should be reported as failed"
" or be silently ignored (STATUS=silentfail). Dynamic test failures will"
" never be silently ignored. Defaults to STATUS=%default.")
parser.add_option(
"--reportFailureStatus", type="choice", action="store", dest="report_failure_status",
choices=("fail", "silentfail"), metavar="STATUS",
help="Controls if the test failure status should be reported as failed"
" or be silently ignored (STATUS=silentfail). Dynamic test failures will"
" never be silently ignored. Defaults to STATUS=%default.")
parser.add_option("--reportFile", dest="report_file", metavar="REPORT",
help="Writes a JSON file with test status and timing information.")
parser.add_option("--seed", type="int", dest="seed", metavar="SEED",
help=("Seed for the random number generator. Useful in combination with the"
" --shuffle option for producing a consistent test execution order."))
parser.add_option(
"--seed", type="int", dest="seed", metavar="SEED",
help=("Seed for the random number generator. Useful in combination with the"
" --shuffle option for producing a consistent test execution order."))
parser.add_option("--serviceExecutor", dest="service_executor", metavar="EXECUTOR",
help="The service executor used by jstests")
@ -212,32 +225,36 @@ def _make_parser(): # pylint: disable=too-many-statements
choices=("commands", "compatibility", "legacy"), metavar="WRITE_MODE",
help="The write mode used by the mongo shell.")
parser.add_option("--shuffle", action="store_const", const="on", dest="shuffle",
help=("Randomizes the order in which tests are executed. This is equivalent"
" to specifying --shuffleMode=on."))
parser.add_option(
"--shuffle", action="store_const", const="on", dest="shuffle",
help=("Randomizes the order in which tests are executed. This is equivalent"
" to specifying --shuffleMode=on."))
parser.add_option("--shuffleMode", type="choice", action="store", dest="shuffle",
choices=("on", "off", "auto"), metavar="ON|OFF|AUTO",
help=("Controls whether to randomize the order in which tests are executed."
" Defaults to auto when not supplied. auto enables randomization in"
" all cases except when the number of jobs requested is 1."))
parser.add_option(
"--shuffleMode", type="choice", action="store", dest="shuffle",
choices=("on", "off", "auto"), metavar="ON|OFF|AUTO",
help=("Controls whether to randomize the order in which tests are executed."
" Defaults to auto when not supplied. auto enables randomization in"
" all cases except when the number of jobs requested is 1."))
parser.add_option("--staggerJobs", type="choice", action="store", dest="stagger_jobs",
choices=("on", "off"), metavar="ON|OFF",
help=("Enables or disables the stagger of launching resmoke jobs."
" Defaults to %default."))
parser.add_option(
"--staggerJobs", type="choice", action="store", dest="stagger_jobs", choices=("on", "off"),
metavar="ON|OFF", help=("Enables or disables the stagger of launching resmoke jobs."
" Defaults to %default."))
parser.add_option("--majorityReadConcern", type="choice", action="store",
dest="majority_read_concern", choices=("on", "off"), metavar="ON|OFF",
help=("Enable or disable majority read concern support."
" Defaults to %default."))
parser.add_option(
"--majorityReadConcern", type="choice", action="store", dest="majority_read_concern",
choices=("on",
"off"), metavar="ON|OFF", help=("Enable or disable majority read concern support."
" Defaults to %default."))
parser.add_option("--storageEngine", dest="storage_engine", metavar="ENGINE",
help="The storage engine used by dbtests and jstests.")
parser.add_option("--storageEngineCacheSizeGB", dest="storage_engine_cache_size_gb",
metavar="CONFIG", help="Sets the storage engine cache size configuration"
" setting for all mongod's.")
parser.add_option(
"--storageEngineCacheSizeGB", dest="storage_engine_cache_size_gb", metavar="CONFIG",
help="Sets the storage engine cache size configuration"
" setting for all mongod's.")
parser.add_option("--tagFile", dest="tag_file", metavar="OPTIONS",
help="A YAML file that associates tests and tags.")
@ -251,9 +268,10 @@ def _make_parser(): # pylint: disable=too-many-statements
parser.add_option("--wiredTigerIndexConfigString", dest="wt_index_config", metavar="CONFIG",
help="Sets the WiredTiger index configuration setting for all mongod's.")
parser.add_option("--executor", dest="executor_file",
help="OBSOLETE: Superceded by --suites; specify --suites=SUITE path/to/test"
" to run a particular test under a particular suite configuration.")
parser.add_option(
"--executor", dest="executor_file",
help="OBSOLETE: Superceded by --suites; specify --suites=SUITE path/to/test"
" to run a particular test under a particular suite configuration.")
evergreen_options = optparse.OptionGroup(
parser, title=_EVERGREEN_OPTIONS_TITLE,
@ -264,29 +282,33 @@ def _make_parser(): # pylint: disable=too-many-statements
evergreen_options.add_option("--buildId", dest="build_id", metavar="BUILD_ID",
help="Sets the build ID of the task.")
evergreen_options.add_option("--distroId", dest="distro_id", metavar="DISTRO_ID",
help=("Sets the identifier for the Evergreen distro running the"
" tests."))
evergreen_options.add_option(
"--distroId", dest="distro_id", metavar="DISTRO_ID",
help=("Sets the identifier for the Evergreen distro running the"
" tests."))
evergreen_options.add_option("--executionNumber", type="int", dest="execution_number",
metavar="EXECUTION_NUMBER",
help=("Sets the number for the Evergreen execution running the"
" tests."))
evergreen_options.add_option(
"--executionNumber", type="int", dest="execution_number", metavar="EXECUTION_NUMBER",
help=("Sets the number for the Evergreen execution running the"
" tests."))
evergreen_options.add_option("--gitRevision", dest="git_revision", metavar="GIT_REVISION",
help=("Sets the git revision for the Evergreen task running the"
" tests."))
evergreen_options.add_option(
"--gitRevision", dest="git_revision", metavar="GIT_REVISION",
help=("Sets the git revision for the Evergreen task running the"
" tests."))
# We intentionally avoid adding a new command line option that starts with --suite so it doesn't
# become ambiguous with the --suites option and break how engineers run resmoke.py locally.
evergreen_options.add_option("--originSuite", dest="origin_suite", metavar="SUITE",
help=("Indicates the name of the test suite prior to the"
" evergreen_generate_resmoke_tasks.py script splitting it"
" up."))
evergreen_options.add_option(
"--originSuite", dest="origin_suite", metavar="SUITE",
help=("Indicates the name of the test suite prior to the"
" evergreen_generate_resmoke_tasks.py script splitting it"
" up."))
evergreen_options.add_option("--patchBuild", action="store_true", dest="patch_build",
help=("Indicates that the Evergreen task running the tests is a"
" patch build."))
evergreen_options.add_option(
"--patchBuild", action="store_true", dest="patch_build",
help=("Indicates that the Evergreen task running the tests is a"
" patch build."))
evergreen_options.add_option("--projectName", dest="project_name", metavar="PROJECT_NAME",
help=("Sets the name of the Evergreen project running the tests."))
@ -301,9 +323,10 @@ def _make_parser(): # pylint: disable=too-many-statements
evergreen_options.add_option("--taskId", dest="task_id", metavar="TASK_ID",
help="Sets the Id of the Evergreen task running the tests.")
evergreen_options.add_option("--variantName", dest="variant_name", metavar="VARIANT_NAME",
help=("Sets the name of the Evergreen build variant running the"
" tests."))
evergreen_options.add_option(
"--variantName", dest="variant_name", metavar="VARIANT_NAME",
help=("Sets the name of the Evergreen build variant running the"
" tests."))
evergreen_options.add_option("--versionId", dest="version_id", metavar="VERSION_ID",
help="Sets the version ID of the task.")
@ -318,10 +341,11 @@ def _make_parser(): # pylint: disable=too-many-statements
metavar="BENCHMARK_FILTER",
help="Regex to filter Google benchmark tests to run.")
benchmark_options.add_option("--benchmarkListTests", dest="benchmark_list_tests",
action="store_true", metavar="BENCHMARK_LIST_TESTS",
help=("Lists all Google benchmark test configurations in each"
" test file."))
benchmark_options.add_option(
"--benchmarkListTests", dest="benchmark_list_tests", action="store_true",
metavar="BENCHMARK_LIST_TESTS",
help=("Lists all Google benchmark test configurations in each"
" test file."))
benchmark_min_time_help = (
"Minimum time to run each benchmark/benchrun test for. Use this option instead of "
@ -339,10 +363,10 @@ def _make_parser(): # pylint: disable=too-many-statements
metavar="BENCHMARK_REPETITIONS", help=benchmark_repetitions_help)
benchrun_devices = ["Android", "Desktop"]
benchmark_options.add_option("--benchrunDevice", dest="benchrun_device", metavar="DEVICE",
type="choice", action="store", choices=benchrun_devices,
help=("The device to run the benchrun test on, choose from {}."
" Defaults to DEVICE='%default'.".format(benchrun_devices)))
benchmark_options.add_option(
"--benchrunDevice", dest="benchrun_device", metavar="DEVICE", type="choice", action="store",
choices=benchrun_devices, help=("The device to run the benchrun test on, choose from {}."
" Defaults to DEVICE='%default'.".format(benchrun_devices)))
benchmark_options.add_option("--benchrunReportRoot", dest="benchrun_report_root",
metavar="PATH", help="The root path for benchrun test report.")
@ -436,8 +460,8 @@ def to_local_args(args=None): # pylint: disable=too-many-locals
else:
other_local_args.append(option_name)
return [arg for arg in (suites_arg, storage_engine_arg)
if arg is not None] + other_local_args + extra_args
return [arg for arg in (suites_arg, storage_engine_arg) if arg is not None
] + other_local_args + extra_args
def parse_command_line():
@ -613,7 +637,7 @@ def _update_config_vars(values): # pylint: disable=too-many-statements
_config.SHELL_CONN_STRING = conn_string
if config:
raise optparse.OptionValueError("Unknown option(s): %s" % (config.keys()))
raise optparse.OptionValueError("Unknown option(s): %s" % (list(config.keys())))
def _get_logging_config(pathname):

2
buildscripts/resmokelib/reportfile.py
View File

@ -1,7 +1,5 @@
"""Manage interactions with the report.json file."""
from __future__ import absolute_import
import json
from . import config

8
buildscripts/resmokelib/selector.py
View File

@ -4,8 +4,6 @@ Defines filtering rules for what tests to include in a suite depending
on whether they apply to C++ unit tests, dbtests, or JS tests.
"""
from __future__ import absolute_import
import collections
import errno
import fnmatch
@ -71,7 +69,7 @@ class TestFileExplorer(object):
A list of paths as a list(str).
"""
tests = []
with open(root_file_path, "rb") as filep:
with open(root_file_path, "r") as filep:
for test_path in filep:
test_path = test_path.strip()
tests.append(test_path)
@ -114,7 +112,7 @@ class TestFileExplorer(object):
program = subprocess.Popen(command, stdout=subprocess.PIPE)
stdout = program.communicate()[0]
return program.returncode, stdout
return program.returncode, stdout.decode("utf-8")
@staticmethod
def parse_tag_file(test_kind):
@ -313,7 +311,7 @@ def make_expression(conf):
elif isinstance(conf, dict):
if len(conf) != 1:
raise ValueError("Tag matching expressions should only contain one key")
key = conf.keys()[0]
key = list(conf.keys())[0]
value = conf[key]
if key == "$allOf":
return _AllOfExpression(_make_expression_list(value))

2
buildscripts/resmokelib/sighandler.py
View File

@ -1,7 +1,5 @@
"""Utility to support asynchronously signaling the current process."""
from __future__ import absolute_import
import atexit
import os
import signal

8
buildscripts/resmokelib/suitesconfig.py
View File

@ -1,7 +1,5 @@
"""Module for retrieving the configuration of resmoke.py test suites."""
from __future__ import absolute_import
import collections
import optparse
import os
@ -33,7 +31,7 @@ def create_test_membership_map(fail_on_missing_selector=False, test_kind=None):
"""
if test_kind is not None:
if isinstance(test_kind, basestring):
if isinstance(test_kind, str):
test_kind = [test_kind]
test_kind = frozenset(test_kind)
@ -117,6 +115,6 @@ def _get_yaml_config(kind, pathname):
pathname = resmokeconfig.NAMED_SUITES[pathname] # Expand 'pathname' to full path.
if not utils.is_yaml_file(pathname) or not os.path.isfile(pathname):
raise optparse.OptionValueError("Expected a %s YAML config, but got '%s'" % (kind,
pathname))
raise optparse.OptionValueError(
"Expected a %s YAML config, but got '%s'" % (kind, pathname))
return utils.load_yaml_file(pathname)

2
buildscripts/resmokelib/testing/__init__.py
View File

@ -1,6 +1,4 @@
"""Extension to the unittest package to support buildlogger and parallel test execution."""
from __future__ import absolute_import
from . import executor
from . import suite

14
buildscripts/resmokelib/testing/executor.py
View File

@ -1,7 +1,5 @@
"""Driver of the test execution framework."""
from __future__ import absolute_import
import threading
import time
@ -68,7 +66,7 @@ class TestSuiteExecutor(object): # pylint: disable=too-many-instance-attributes
jobs_to_start = self.num_tests
# Must be done after getting buildlogger configuration.
self._jobs = [self._make_job(job_num) for job_num in xrange(jobs_to_start)]
self._jobs = [self._make_job(job_num) for job_num in range(jobs_to_start)]
def run(self):
"""Execute the test suite.
@ -130,8 +128,9 @@ class TestSuiteExecutor(object): # pylint: disable=too-many-instance-attributes
test_results_num = len(test_report["results"])
# There should be at least as many tests results as expected number of tests.
if test_results_num < self.num_tests:
raise errors.ResmokeError("{} reported tests is less than {} expected tests"
.format(test_results_num, self.num_tests))
raise errors.ResmokeError(
"{} reported tests is less than {} expected tests".format(
test_results_num, self.num_tests))
# Clear the report so it can be reused for the next execution.
for job in self._jobs:
@ -157,8 +156,9 @@ class TestSuiteExecutor(object): # pylint: disable=too-many-instance-attributes
try:
# Run each Job instance in its own thread.
for job in self._jobs:
thr = threading.Thread(target=job, args=(test_queue, interrupt_flag), kwargs=dict(
setup_flag=setup_flag, teardown_flag=teardown_flag))
thr = threading.Thread(
target=job, args=(test_queue, interrupt_flag), kwargs=dict(
setup_flag=setup_flag, teardown_flag=teardown_flag))
# Do not wait for tests to finish executing if interrupted by the user.
thr.daemon = True
thr.start()

2
buildscripts/resmokelib/testing/fixtures/__init__.py
View File

@ -1,7 +1,5 @@
"""Fixture for executing JSTests against."""
from __future__ import absolute_import
from .external import ExternalFixture as _ExternalFixture
from .interface import NoOpFixture as _NoOpFixture
from .interface import make_fixture

2
buildscripts/resmokelib/testing/fixtures/external.py
View File

@ -1,7 +1,5 @@
"""External fixture for executing JSTests against."""
from __future__ import absolute_import
from . import interface

6
buildscripts/resmokelib/testing/fixtures/interface.py
View File

@ -1,7 +1,5 @@
"""Interface of the different fixtures for executing JSTests against."""
from __future__ import absolute_import
import os.path
import time
@ -25,11 +23,9 @@ def make_fixture(class_name, *args, **kwargs):
return _FIXTURES[class_name](*args, **kwargs)
class Fixture(object):
class Fixture(object, metaclass=registry.make_registry_metaclass(_FIXTURES)):
"""Base class for all fixtures."""
__metaclass__ = registry.make_registry_metaclass(_FIXTURES) # type: ignore
# We explicitly set the 'REGISTERED_NAME' attribute so that PyLint realizes that the attribute
# is defined for all subclasses of Fixture.
REGISTERED_NAME = "Fixture"

28
buildscripts/resmokelib/testing/fixtures/replicaset.py
View File

@ -1,10 +1,9 @@
"""Replica set fixture for executing JSTests against."""
from __future__ import absolute_import
import os.path
import time
import bson.errors
import pymongo
import pymongo.errors
import pymongo.write_concern
@ -77,11 +76,11 @@ class ReplicaSetFixture(interface.ReplFixture): # pylint: disable=too-many-inst
self.replset_name = self.mongod_options.get("replSet", "rs")
if not self.nodes:
for i in xrange(self.num_nodes):
for i in range(self.num_nodes):
node = self._new_mongod(i, self.replset_name)
self.nodes.append(node)
for i in xrange(self.num_nodes):
for i in range(self.num_nodes):
if self.linear_chain and i > 0:
self.nodes[i].mongod_options["set_parameters"][
"failpoint.forceSyncSourceCandidate"] = {
@ -207,10 +206,17 @@ class ReplicaSetFixture(interface.ReplFixture): # pylint: disable=too-many-inst
def check_rcmaj_optime(client, node):
"""Return True if all nodes have caught up with the primary."""
res = client.admin.command({"replSetGetStatus": 1})
# TODO SERVER-40078: The server is reporting invalid
# dates in its response to the replSetGetStatus
# command
try:
res = client.admin.command({"replSetGetStatus": 1})
except bson.errors.InvalidBSON:
return False
read_concern_majority_optime = res["optimes"]["readConcernMajorityOpTime"]
if read_concern_majority_optime >= primary_optime:
if (read_concern_majority_optime["t"] == primary_optime["t"]
and read_concern_majority_optime["ts"] >= primary_optime["ts"]):
up_to_date_nodes.add(node.port)
return len(up_to_date_nodes) == len(self.nodes)
@ -303,7 +309,15 @@ class ReplicaSetFixture(interface.ReplFixture): # pylint: disable=too-many-inst
client_admin = client["admin"]
while True:
status = client_admin.command("replSetGetStatus")
# TODO SERVER-40078: The server is reporting invalid
# dates in its response to the replSetGetStatus
# command
try:
status = client_admin.command("replSetGetStatus")
except bson.errors.InvalidBSON:
time.sleep(0.1)
continue
# The `lastStableRecoveryTimestamp` field contains a stable timestamp guaranteed to
# exist on storage engine recovery to a stable timestamp.
last_stable_recovery_timestamp = status.get("lastStableRecoveryTimestamp", None)

Some files were not shown because too many files have changed in this diff Show More