mongodb/mongo-python-driver
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

removing all support for mongo-qa tests

Browse Source 
allows us to remove SON.from_xml
  - not deprecating this as it was used internally only AFAIK

therefore eliminates driver dependency on elementtree
This commit is contained in:
Mike Dirolf 2010-01-05 14:20:12 -05:00
parent 9529271690
commit 54be5db873
10 changed files with 7 additions and 492 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
README.rst
View File

@ -22,7 +22,6 @@ C extension will not be built. This will negatively affect performance, but ever
Additional dependencies are:
- `ElementTree <http://effbot.org/zone/element-index.htm>`_ (this is included with Python >= 2.5)
- (to generate documentation) sphinx_
- (to auto-discover tests) `nose <http://somethingaboutorange.com/mrl/projects/nose/>`_

6
pymongo/errors.py
View File

@ -61,6 +61,7 @@ class InvalidBSON(ValueError):
"""Raised when trying to create a BSON object from invalid data.
"""
class InvalidStringData(ValueError):
"""Raised when trying to encode a string containing non-UTF8 data.
"""
@ -71,11 +72,6 @@ class InvalidDocument(ValueError):
"""
class UnsupportedTag(ValueError):
"""Raised when trying to parse an unsupported tag in an XML document.
"""
class InvalidId(ValueError):
"""Raised when trying to create an ObjectId from invalid data.
"""

127
pymongo/son.py
View File

@ -18,12 +18,6 @@ Regular dictionaries can be used instead of SON objects, but not when the order
of keys is important. A SON object can be used just like a normal Python
dictionary."""
import datetime
import re
import binascii
import base64
import types
class SON(dict):
"""SON data.
@ -190,7 +184,6 @@ class SON(dict):
def __len__(self):
return len(self.keys())
# Thanks to Jeff Jenkins for the idea and original implementation
def to_dict(self):
"""Convert a SON document to a normal Python dictionary instance.
@ -199,129 +192,13 @@ class SON(dict):
"""
def transform_value(value):
if isinstance(value, types.ListType):
if isinstance(value, list):
return [transform_value(v) for v in value]
if isinstance(value, SON):
value = dict(value)
if isinstance(value, types.DictType):
if isinstance(value, dict):
for k, v in value.iteritems():
value[k] = transform_value(v)
return value
return transform_value(dict(self))
def from_xml(cls, xml):
"""Create an instance of SON from an xml document.
This is really only used for testing, and is probably unnecessary.
"""
try:
import xml.etree.ElementTree as ET
except ImportError:
import elementtree.ElementTree as ET
from code import Code
from binary import Binary
from objectid import ObjectId
from dbref import DBRef
from errors import UnsupportedTag
def pad(list, index):
while index >= len(list):
list.append(None)
def make_array(array):
doc = make_doc(array)
array = []
for (key, value) in doc.items():
index = int(key)
pad(array, index)
array[index] = value
return array
def make_string(string):
return string.text is not None and unicode(string.text) or u""
def make_code(code):
return code.text is not None and Code(code.text) or Code("")
def make_binary(binary):
if binary.text is not None:
return Binary(base64.decodestring(binary.text))
return Binary("")
def make_boolean(bool):
return bool.text == "true"
def make_date(date):
return datetime.datetime.utcfromtimestamp(float(date.text) /
1000.0)
def make_ref(dbref):
return DBRef(make_elem(dbref[0]), make_elem(dbref[1]))
def make_oid(oid):
return ObjectId(binascii.unhexlify(oid.text))
def make_int(data):
return int(data.text)
def make_null(null):
return None
def make_number(number):
return float(number.text)
def make_regex(regex):
return re.compile(make_elem(regex[0]), make_elem(regex[1]))
def make_options(data):
options = 0
if not data.text:
return options
if "i" in data.text:
options |= re.IGNORECASE
if "l" in data.text:
options |= re.LOCALE
if "m" in data.text:
options |= re.MULTILINE
if "s" in data.text:
options |= re.DOTALL
if "u" in data.text:
options |= re.UNICODE
if "x" in data.text:
options |= re.VERBOSE
return options
def make_elem(elem):
try:
return {"array": make_array,
"doc": make_doc,
"string": make_string,
"binary": make_binary,
"boolean": make_boolean,
"code": make_code,
"date": make_date,
"ref": make_ref,
"ns": make_string,
"oid": make_oid,
"int": make_int,
"null": make_null,
"number": make_number,
"pattern": make_string,
"options": make_options,
}[elem.tag](elem)
except KeyError:
raise UnsupportedTag("cannot parse tag: %s" % elem.tag)
def make_doc(doc):
son = SON()
for elem in doc:
son[elem.attrib["name"]] = make_elem(elem)
return son
tree = ET.XML(xml)
doc = tree[1]
return make_doc(doc)
from_xml = classmethod(from_xml)

11
setup.py
View File

@ -21,15 +21,6 @@ from distutils.core import Extension
from pymongo import version
requirements = []
try:
import xml.etree.ElementTree
except ImportError:
try:
import elementtree
except ImportError:
requirements.append("elementtree")
f = open("README.rst")
try:
try:
@ -158,7 +149,7 @@ setup(
url="http://github.com/mongodb/mongo-python-driver",
keywords=["mongo", "mongodb", "pymongo", "gridfs"],
packages=["pymongo", "gridfs"],
install_requires=requirements,
install_requires=[],
features=features,
license="Apache License, Version 2.0",
test_suite="nose.collector",

47
test/test_bson.py
View File

@ -19,20 +19,16 @@
import unittest
import datetime
import re
import glob
import sys
import types
try:
import uuid
should_test_uuid = True
except ImportError:
should_test_uuid = False
sys.path[0:0] = [""]
from nose.plugins.skip import SkipTest
sys.path[0:0] = [""]
import qcheck
from pymongo.binary import Binary
from pymongo.code import Code
@ -40,7 +36,7 @@ from pymongo.objectid import ObjectId
from pymongo.dbref import DBRef
from pymongo.son import SON
from pymongo.bson import BSON, is_valid, _to_dicts
from pymongo.errors import UnsupportedTag, InvalidDocument, InvalidStringData
from pymongo.errors import InvalidDocument, InvalidStringData
class TestBSON(unittest.TestCase):
@ -149,7 +145,7 @@ class TestBSON(unittest.TestCase):
helper({"test": u"hello"})
self.assert_(isinstance(BSON.from_dict({"hello": "world"})
.to_dict()["hello"],
types.UnicodeType))
unicode))
helper({"mike": -10120})
helper({"long": long(10)})
helper({"really big long": 2147483648})
@ -173,43 +169,6 @@ class TestBSON(unittest.TestCase):
qcheck.check_unittest(self, from_then_to_dict,
qcheck.gen_mongo_dict(3))
def test_data_files(self):
# TODO don't hardcode this, actually clone the repo
data_files = "../mongo-qa/modules/bson_tests/tests/*/*.xson"
generate = True
for file_name in glob.glob(data_files):
f = open(file_name, "r")
xml = f.read()
f.close()
try:
doc = SON.from_xml(xml)
bson = BSON.from_dict(doc)
except UnsupportedTag:
print "skipped file %s: %s" % (file_name, sys.exc_info()[1])
continue
try:
f = open(file_name.replace(".xson", ".bson"), "rb")
expected = f.read()
f.close()
self.assertEqual(bson, expected,
"(in %s) %r != %r" % (file_name, bson,
expected))
self.assertEqual(doc, bson.to_dict(),
"(in %s) %r != %r" % (file_name, doc,
bson.to_dict()))
except IOError:
if generate:
print "generating .bson for %s" % file_name
f = open(file_name.replace(".xson", ".bson"), "w")
f.write(bson)
f.close()
def test_bad_encode(self):
self.assertRaises(InvalidStringData, BSON.from_dict,
{"lalala": '\xf4\xe0\xf0\xe1\xc0 Color Touch'})

51
test/test_son.py
View File

@ -15,15 +15,10 @@
"""Tests for the son module."""
import unittest
import datetime
import re
import sys
sys.path[0:0] = [""]
from pymongo.objectid import ObjectId
from pymongo.dbref import DBRef
from pymongo.son import SON
from pymongo.code import Code
class TestSON(unittest.TestCase):
@ -58,52 +53,6 @@ class TestSON(unittest.TestCase):
self.assertEqual(dict, c.to_dict()["blah"][0].__class__)
self.assertEqual(dict, d.to_dict()["blah"]["foo"].__class__)
def test_from_xml(self):
smorgasbord = """
<twonk>
<meta/>
<doc>
<oid name="_id">285a664923b5fcd8ec000000</oid>
<int name="the_answer">42</int>
<string name="b">foo</string>
<boolean name="c">true</boolean>
<number name="pi">3.14159265358979</number>
<array name="an_array">
<string name="0">x</string>
<string name="1">y</string>
<string name="2">z</string>
<doc name="3">
<string name="subobject">yup</string>
</doc>
</array>
<date name="now">123144452057</date>
<ref name="dbref">
<ns>namespace</ns>
<oid>ca5c67496c01d896f7010000</oid>
</ref>
<code name="$where">this is code</code>
<null name="mynull"/>
</doc>
</twonk>
"""
self.assertEqual(SON.from_xml(smorgasbord),
SON([(u"_id", ObjectId("\x28\x5A\x66\x49\x23\xB5\xFC"
"\xD8\xEC\x00\x00\x00")),
(u"the_answer", 42),
(u"b", u"foo"),
(u"c", True),
(u"pi", 3.14159265358979),
(u"an_array", [u"x", u"y", u"z",
SON([(u"subobject", u"yup")])]),
(u"now", datetime.datetime(1973, 11, 26, 6,
47, 32, 57000)),
(u"dbref",
DBRef("namespace",
ObjectId("\xCA\x5C\x67\x49\x6C\x01"
"\xD8\x96\xF7\x01\x00\x00"))),
(u"$where", Code("this is code")),
(u"mynull", None),
]))
if __name__ == "__main__":
unittest.main()

10
tools/README.rst
View File

@ -1,13 +1,3 @@
Tools
=====
This directory contains tools for use with the ``pymongo`` module.
validate and validate.py
========================
These scripts are used by 10gen's driver testing tools to verify the
correctness of ``pymongo`` BSON module (``pymongo.bson``).
driver_tests.py
===============
This script is used by 10gen's
`driver testing framework. <http://mongodb.onconfluence.com/display/DOCS/Mongo+Driver+Test+Framework>`_

203
tools/driver_tests.py
View File

@ -1,203 +0,0 @@
#!/usr/bin/env python
# Copyright 2009 10gen, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Test runner for the driver tests."""
import sys
import os
import datetime
sys.path[0:0] = [os.path.join(os.getcwd(), "..")]
from pymongo.connection import Connection
from pymongo.errors import CollectionInvalid
from pymongo import OFF, SLOW_ONLY, ALL, ASCENDING
import gridfs
def admin(db, out):
tester = db.tester
tester.insert({"test": 1})
try:
db.validate_collection(tester)
print >>out, "true"
except:
print >>out, "false"
try:
db.validate_collection("system.users")
print >>out, "true"
except:
print >>out, "false"
try:
db.validate_collection("$")
print >>out, "true"
except:
print >>out, "false"
levels = {OFF: "off",
SLOW_ONLY: "slowOnly",
ALL: "all"}
print >>out, levels[db.profiling_level()]
db.set_profiling_level(OFF)
print >>out, levels[db.profiling_level()]
def capped(db, out):
collection = db.create_collection("capped1", {"capped": True, "size": 500})
collection.insert({"x": 1})
collection.insert({"x": 2})
# TODO ignoring $nExtents for now
collection2 = db.create_collection("capped2", {"capped": True, "size": 1000})
str = ""
for _ in range(100):
collection2.insert({"dashes": str})
str += "-"
def count1(db, out):
print >>out, db.test1.find().count()
print >>out, db.test2.find().count()
print >>out, db.test3.find({"i": "a"}).count()
print >>out, db.test3.find({"i": 3}).count()
print >>out, db.test3.find({"i": {"$gte": 67}}).count()
def dberror(db, out):
db.reset_error_history()
print >>out, db.error() is None
print >>out, db.previous_error() is None
db._command({"forceerror": 1}, check=False)
print >>out, db.error() is None
print >>out, db.previous_error() is None
db.foo.find_one()
print >>out, db.error() is None
print >>out, db.previous_error() is None
print >>out, db.previous_error()["nPrev"]
db.reset_error_history()
print >>out, db.previous_error() is None
def dbs(db, out):
db.dbs_1.save({"foo": "bar"})
db.dbs_2.save({"psi": "phi"})
print >>out, db.name()
for name in [n for n in sorted(db.collection_names()) if n.startswith("dbs")]:
print >>out, name
db.drop_collection(db.dbs_1)
db.create_collection("dbs_3")
for name in [n for n in sorted(db.collection_names()) if n.startswith("dbs")]:
print >>out, name
def find(db, out):
db.test.insert({"a": 2})
def find1(db, out):
for doc in db.c.find({"x": 1}).sort("y", ASCENDING).skip(20).limit(10):
print >>out, doc["z"]
def indices(db, out):
db.x.drop_index([("field1", ASCENDING)])
for name in db.x.index_information().keys():
print >>out, name
db.y.create_index([("a", ASCENDING),
("b", ASCENDING),
("c", ASCENDING)])
db.y.create_index("d")
for name in sorted(db.y.index_information().keys()):
print >>out, name
def remove(db, out):
db.remove1.remove({})
db.remove2.remove({"a": 3})
def stress1(db, out):
for i in range(50000):
db.stress1.save({"name": "asdf" + str(i),
"date": datetime.datetime.utcnow(),
"id": i,
"blah": "lksjhasoh1298a1shasoidiohaskjasiouashoasasiugoas" +
"lksjhasoh1298a1shasoidiohaskjasiouashoasasiugoas" +
"lksjhasoh1298a1shasoidiohaskjasiouashoasasiugoas" +
"lksjhasoh1298a1shasoidiohaskjasiouashoasasiugoas" +
"lksjhasoh1298a1shasoidiohaskjasiouashoasasiugoas" +
"lksjhasoh1298a1shasoidiohaskjasiouashoasasiugoas",
"subarray": []})
for i in range(10000):
x = db.stress1.find_one({"id": i})
x["subarray"] = "foo" + str(i)
db.stress1.save(x)
db.stress1.create_index("date")
def test1(db, out):
for i in range(100):
db.part1.insert({"x": i})
def update(db, out):
db.foo.update({"x": 1}, {"x": 1, "y": 2})
db.foo.update({"x": 2}, {"x": 1, "y": 7})
db.foo.update({"x": 3}, {"x": 4, "y": 1}, upsert=True)
def gridfs_in(db, time, input):
name = input.name
fs = gridfs.GridFS(db)
f = fs.open(name, "w")
f.write(input.read())
f.close()
def gridfs_out(db, time, input, output):
fs = gridfs.GridFS(db)
f = fs.open(input.name)
output.write(f.read())
f.close()
def main(test, time_file, in_file=None, out_file=None):
db = Connection().driver_test_framework
try:
test_function = globals()[test]
except KeyError:
return
time_file = open(time_file, "w")
args = [db, time_file]
if in_file:
in_file = open(in_file, "r")
args.append(in_file)
if out_file:
out_file = open(out_file, "w")
args.append(out_file)
try:
begin = datetime.datetime.now()
test_function(*args)
end = datetime.datetime.now()
exit_status = 0
except:
begin = None
begin = None
exit_status = 1
raise
if begin and end:
time_file.write("begintime:%s\n" % begin)
time_file.write("endtime:%s\n" % end)
time_file.write("totaltime:%s\n" % (end - begin))
time_file.write("exit_code:%s\n" % exit_status)
time_file.close()
if in_file and hasattr(in_file, "close"):
in_file.close()
if out_file:
out_file.close()
if __name__ == "__main__":
main(*sys.argv[1:])

5
tools/validate
View File

@ -1,5 +0,0 @@
#!/usr/bin/env bash
pushd .. > /dev/null
python "tools/validate.py" $1 $2
popd > /dev/null

38
tools/validate.py
View File

@ -1,38 +0,0 @@
# Copyright 2009 10gen, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
sys.path[0:0] = os.path.join(os.getcwd(), "..")
from pymongo.bson import BSON
from pymongo.son import SON
def main():
xml_file = sys.argv[1]
out_file = sys.argv[2]
f = open(xml_file, "r")
xml = f.read()
f.close()
f = open(out_file, "w")
doc = SON.from_xml(xml)
bson = BSON.from_dict(doc)
f.write(bson)
f.close()
assert doc == bson.to_dict()
if __name__ == "__main__":
main()