From 54be5db873ab61b727f71bf803b1037ce624e8d6 Mon Sep 17 00:00:00 2001 From: Mike Dirolf Date: Tue, 5 Jan 2010 14:20:12 -0500 Subject: [PATCH] removing all support for mongo-qa tests allows us to remove SON.from_xml - not deprecating this as it was used internally only AFAIK therefore eliminates driver dependency on elementtree --- README.rst | 1 - pymongo/errors.py | 6 +- pymongo/son.py | 127 +------------------------- setup.py | 11 +-- test/test_bson.py | 47 +--------- test/test_son.py | 51 ----------- tools/README.rst | 10 --- tools/driver_tests.py | 203 ------------------------------------------ tools/validate | 5 -- tools/validate.py | 38 -------- 10 files changed, 7 insertions(+), 492 deletions(-) delete mode 100755 tools/driver_tests.py delete mode 100755 tools/validate delete mode 100644 tools/validate.py diff --git a/README.rst b/README.rst index 74fcfdd7d..31440c494 100644 --- a/README.rst +++ b/README.rst @@ -22,7 +22,6 @@ C extension will not be built. This will negatively affect performance, but ever Additional dependencies are: -- `ElementTree `_ (this is included with Python >= 2.5) - (to generate documentation) sphinx_ - (to auto-discover tests) `nose `_ diff --git a/pymongo/errors.py b/pymongo/errors.py index a6ace7958..477a39be9 100644 --- a/pymongo/errors.py +++ b/pymongo/errors.py @@ -61,6 +61,7 @@ class InvalidBSON(ValueError): """Raised when trying to create a BSON object from invalid data. """ + class InvalidStringData(ValueError): """Raised when trying to encode a string containing non-UTF8 data. """ @@ -71,11 +72,6 @@ class InvalidDocument(ValueError): """ -class UnsupportedTag(ValueError): - """Raised when trying to parse an unsupported tag in an XML document. - """ - - class InvalidId(ValueError): """Raised when trying to create an ObjectId from invalid data. """ diff --git a/pymongo/son.py b/pymongo/son.py index b11aa7c18..4dc7e640c 100644 --- a/pymongo/son.py +++ b/pymongo/son.py @@ -18,12 +18,6 @@ Regular dictionaries can be used instead of SON objects, but not when the order of keys is important. A SON object can be used just like a normal Python dictionary.""" -import datetime -import re -import binascii -import base64 -import types - class SON(dict): """SON data. @@ -190,7 +184,6 @@ class SON(dict): def __len__(self): return len(self.keys()) - # Thanks to Jeff Jenkins for the idea and original implementation def to_dict(self): """Convert a SON document to a normal Python dictionary instance. @@ -199,129 +192,13 @@ class SON(dict): """ def transform_value(value): - if isinstance(value, types.ListType): + if isinstance(value, list): return [transform_value(v) for v in value] if isinstance(value, SON): value = dict(value) - if isinstance(value, types.DictType): + if isinstance(value, dict): for k, v in value.iteritems(): value[k] = transform_value(v) return value return transform_value(dict(self)) - - def from_xml(cls, xml): - """Create an instance of SON from an xml document. - - This is really only used for testing, and is probably unnecessary. - """ - try: - import xml.etree.ElementTree as ET - except ImportError: - import elementtree.ElementTree as ET - - from code import Code - from binary import Binary - from objectid import ObjectId - from dbref import DBRef - from errors import UnsupportedTag - - def pad(list, index): - while index >= len(list): - list.append(None) - - def make_array(array): - doc = make_doc(array) - array = [] - for (key, value) in doc.items(): - index = int(key) - pad(array, index) - array[index] = value - return array - - def make_string(string): - return string.text is not None and unicode(string.text) or u"" - - def make_code(code): - return code.text is not None and Code(code.text) or Code("") - - def make_binary(binary): - if binary.text is not None: - return Binary(base64.decodestring(binary.text)) - return Binary("") - - def make_boolean(bool): - return bool.text == "true" - - def make_date(date): - return datetime.datetime.utcfromtimestamp(float(date.text) / - 1000.0) - - def make_ref(dbref): - return DBRef(make_elem(dbref[0]), make_elem(dbref[1])) - - def make_oid(oid): - return ObjectId(binascii.unhexlify(oid.text)) - - def make_int(data): - return int(data.text) - - def make_null(null): - return None - - def make_number(number): - return float(number.text) - - def make_regex(regex): - return re.compile(make_elem(regex[0]), make_elem(regex[1])) - - def make_options(data): - options = 0 - if not data.text: - return options - if "i" in data.text: - options |= re.IGNORECASE - if "l" in data.text: - options |= re.LOCALE - if "m" in data.text: - options |= re.MULTILINE - if "s" in data.text: - options |= re.DOTALL - if "u" in data.text: - options |= re.UNICODE - if "x" in data.text: - options |= re.VERBOSE - return options - - def make_elem(elem): - try: - return {"array": make_array, - "doc": make_doc, - "string": make_string, - "binary": make_binary, - "boolean": make_boolean, - "code": make_code, - "date": make_date, - "ref": make_ref, - "ns": make_string, - "oid": make_oid, - "int": make_int, - "null": make_null, - "number": make_number, - "pattern": make_string, - "options": make_options, - }[elem.tag](elem) - except KeyError: - raise UnsupportedTag("cannot parse tag: %s" % elem.tag) - - def make_doc(doc): - son = SON() - for elem in doc: - son[elem.attrib["name"]] = make_elem(elem) - return son - - tree = ET.XML(xml) - doc = tree[1] - - return make_doc(doc) - from_xml = classmethod(from_xml) diff --git a/setup.py b/setup.py index bfec4c2ec..2022a2bd8 100755 --- a/setup.py +++ b/setup.py @@ -21,15 +21,6 @@ from distutils.core import Extension from pymongo import version -requirements = [] -try: - import xml.etree.ElementTree -except ImportError: - try: - import elementtree - except ImportError: - requirements.append("elementtree") - f = open("README.rst") try: try: @@ -158,7 +149,7 @@ setup( url="http://github.com/mongodb/mongo-python-driver", keywords=["mongo", "mongodb", "pymongo", "gridfs"], packages=["pymongo", "gridfs"], - install_requires=requirements, + install_requires=[], features=features, license="Apache License, Version 2.0", test_suite="nose.collector", diff --git a/test/test_bson.py b/test/test_bson.py index b8f3e812b..3d508b680 100644 --- a/test/test_bson.py +++ b/test/test_bson.py @@ -19,20 +19,16 @@ import unittest import datetime import re -import glob import sys -import types - try: import uuid should_test_uuid = True except ImportError: should_test_uuid = False +sys.path[0:0] = [""] from nose.plugins.skip import SkipTest -sys.path[0:0] = [""] - import qcheck from pymongo.binary import Binary from pymongo.code import Code @@ -40,7 +36,7 @@ from pymongo.objectid import ObjectId from pymongo.dbref import DBRef from pymongo.son import SON from pymongo.bson import BSON, is_valid, _to_dicts -from pymongo.errors import UnsupportedTag, InvalidDocument, InvalidStringData +from pymongo.errors import InvalidDocument, InvalidStringData class TestBSON(unittest.TestCase): @@ -149,7 +145,7 @@ class TestBSON(unittest.TestCase): helper({"test": u"hello"}) self.assert_(isinstance(BSON.from_dict({"hello": "world"}) .to_dict()["hello"], - types.UnicodeType)) + unicode)) helper({"mike": -10120}) helper({"long": long(10)}) helper({"really big long": 2147483648}) @@ -173,43 +169,6 @@ class TestBSON(unittest.TestCase): qcheck.check_unittest(self, from_then_to_dict, qcheck.gen_mongo_dict(3)) - def test_data_files(self): - # TODO don't hardcode this, actually clone the repo - data_files = "../mongo-qa/modules/bson_tests/tests/*/*.xson" - generate = True - - for file_name in glob.glob(data_files): - f = open(file_name, "r") - xml = f.read() - f.close() - - try: - doc = SON.from_xml(xml) - bson = BSON.from_dict(doc) - except UnsupportedTag: - print "skipped file %s: %s" % (file_name, sys.exc_info()[1]) - continue - - try: - f = open(file_name.replace(".xson", ".bson"), "rb") - expected = f.read() - f.close() - - self.assertEqual(bson, expected, - "(in %s) %r != %r" % (file_name, bson, - expected)) - self.assertEqual(doc, bson.to_dict(), - "(in %s) %r != %r" % (file_name, doc, - bson.to_dict())) - - except IOError: - if generate: - print "generating .bson for %s" % file_name - - f = open(file_name.replace(".xson", ".bson"), "w") - f.write(bson) - f.close() - def test_bad_encode(self): self.assertRaises(InvalidStringData, BSON.from_dict, {"lalala": '\xf4\xe0\xf0\xe1\xc0 Color Touch'}) diff --git a/test/test_son.py b/test/test_son.py index 94befea7a..e2cb3f3b8 100644 --- a/test/test_son.py +++ b/test/test_son.py @@ -15,15 +15,10 @@ """Tests for the son module.""" import unittest -import datetime -import re import sys sys.path[0:0] = [""] -from pymongo.objectid import ObjectId -from pymongo.dbref import DBRef from pymongo.son import SON -from pymongo.code import Code class TestSON(unittest.TestCase): @@ -58,52 +53,6 @@ class TestSON(unittest.TestCase): self.assertEqual(dict, c.to_dict()["blah"][0].__class__) self.assertEqual(dict, d.to_dict()["blah"]["foo"].__class__) - def test_from_xml(self): - smorgasbord = """ - - - - 285a664923b5fcd8ec000000 - 42 - foo - true - 3.14159265358979 - - x - y - z - - yup - - - 123144452057 - - namespace - ca5c67496c01d896f7010000 - - this is code - - - -""" - self.assertEqual(SON.from_xml(smorgasbord), - SON([(u"_id", ObjectId("\x28\x5A\x66\x49\x23\xB5\xFC" - "\xD8\xEC\x00\x00\x00")), - (u"the_answer", 42), - (u"b", u"foo"), - (u"c", True), - (u"pi", 3.14159265358979), - (u"an_array", [u"x", u"y", u"z", - SON([(u"subobject", u"yup")])]), - (u"now", datetime.datetime(1973, 11, 26, 6, - 47, 32, 57000)), - (u"dbref", - DBRef("namespace", - ObjectId("\xCA\x5C\x67\x49\x6C\x01" - "\xD8\x96\xF7\x01\x00\x00"))), - (u"$where", Code("this is code")), - (u"mynull", None), - ])) if __name__ == "__main__": unittest.main() diff --git a/tools/README.rst b/tools/README.rst index 1378f35ef..71f288ca2 100644 --- a/tools/README.rst +++ b/tools/README.rst @@ -1,13 +1,3 @@ Tools ===== This directory contains tools for use with the ``pymongo`` module. - -validate and validate.py -======================== -These scripts are used by 10gen's driver testing tools to verify the -correctness of ``pymongo`` BSON module (``pymongo.bson``). - -driver_tests.py -=============== -This script is used by 10gen's -`driver testing framework. `_ diff --git a/tools/driver_tests.py b/tools/driver_tests.py deleted file mode 100755 index eb6998ffa..000000000 --- a/tools/driver_tests.py +++ /dev/null @@ -1,203 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2009 10gen, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Test runner for the driver tests.""" - -import sys -import os -import datetime - -sys.path[0:0] = [os.path.join(os.getcwd(), "..")] -from pymongo.connection import Connection -from pymongo.errors import CollectionInvalid -from pymongo import OFF, SLOW_ONLY, ALL, ASCENDING -import gridfs - -def admin(db, out): - tester = db.tester - tester.insert({"test": 1}) - try: - db.validate_collection(tester) - print >>out, "true" - except: - print >>out, "false" - - try: - db.validate_collection("system.users") - print >>out, "true" - except: - print >>out, "false" - - try: - db.validate_collection("$") - print >>out, "true" - except: - print >>out, "false" - - levels = {OFF: "off", - SLOW_ONLY: "slowOnly", - ALL: "all"} - print >>out, levels[db.profiling_level()] - db.set_profiling_level(OFF) - print >>out, levels[db.profiling_level()] - -def capped(db, out): - collection = db.create_collection("capped1", {"capped": True, "size": 500}) - collection.insert({"x": 1}) - collection.insert({"x": 2}) - - # TODO ignoring $nExtents for now - collection2 = db.create_collection("capped2", {"capped": True, "size": 1000}) - str = "" - for _ in range(100): - collection2.insert({"dashes": str}) - str += "-" - -def count1(db, out): - print >>out, db.test1.find().count() - print >>out, db.test2.find().count() - print >>out, db.test3.find({"i": "a"}).count() - print >>out, db.test3.find({"i": 3}).count() - print >>out, db.test3.find({"i": {"$gte": 67}}).count() - -def dberror(db, out): - db.reset_error_history() - print >>out, db.error() is None - print >>out, db.previous_error() is None - db._command({"forceerror": 1}, check=False) - print >>out, db.error() is None - print >>out, db.previous_error() is None - db.foo.find_one() - print >>out, db.error() is None - print >>out, db.previous_error() is None - print >>out, db.previous_error()["nPrev"] - db.reset_error_history() - print >>out, db.previous_error() is None - -def dbs(db, out): - db.dbs_1.save({"foo": "bar"}) - db.dbs_2.save({"psi": "phi"}) - print >>out, db.name() - for name in [n for n in sorted(db.collection_names()) if n.startswith("dbs")]: - print >>out, name - db.drop_collection(db.dbs_1) - db.create_collection("dbs_3") - for name in [n for n in sorted(db.collection_names()) if n.startswith("dbs")]: - print >>out, name - -def find(db, out): - db.test.insert({"a": 2}) - -def find1(db, out): - for doc in db.c.find({"x": 1}).sort("y", ASCENDING).skip(20).limit(10): - print >>out, doc["z"] - -def indices(db, out): - db.x.drop_index([("field1", ASCENDING)]) - for name in db.x.index_information().keys(): - print >>out, name - db.y.create_index([("a", ASCENDING), - ("b", ASCENDING), - ("c", ASCENDING)]) - db.y.create_index("d") - for name in sorted(db.y.index_information().keys()): - print >>out, name - -def remove(db, out): - db.remove1.remove({}) - db.remove2.remove({"a": 3}) - -def stress1(db, out): - for i in range(50000): - db.stress1.save({"name": "asdf" + str(i), - "date": datetime.datetime.utcnow(), - "id": i, - "blah": "lksjhasoh1298a1shasoidiohaskjasiouashoasasiugoas" + - "lksjhasoh1298a1shasoidiohaskjasiouashoasasiugoas" + - "lksjhasoh1298a1shasoidiohaskjasiouashoasasiugoas" + - "lksjhasoh1298a1shasoidiohaskjasiouashoasasiugoas" + - "lksjhasoh1298a1shasoidiohaskjasiouashoasasiugoas" + - "lksjhasoh1298a1shasoidiohaskjasiouashoasasiugoas", - "subarray": []}) - for i in range(10000): - x = db.stress1.find_one({"id": i}) - x["subarray"] = "foo" + str(i) - db.stress1.save(x) - - db.stress1.create_index("date") - -def test1(db, out): - for i in range(100): - db.part1.insert({"x": i}) - -def update(db, out): - db.foo.update({"x": 1}, {"x": 1, "y": 2}) - db.foo.update({"x": 2}, {"x": 1, "y": 7}) - db.foo.update({"x": 3}, {"x": 4, "y": 1}, upsert=True) - -def gridfs_in(db, time, input): - name = input.name - fs = gridfs.GridFS(db) - f = fs.open(name, "w") - f.write(input.read()) - f.close() - -def gridfs_out(db, time, input, output): - fs = gridfs.GridFS(db) - f = fs.open(input.name) - output.write(f.read()) - f.close() - -def main(test, time_file, in_file=None, out_file=None): - db = Connection().driver_test_framework - try: - test_function = globals()[test] - except KeyError: - return - - time_file = open(time_file, "w") - args = [db, time_file] - if in_file: - in_file = open(in_file, "r") - args.append(in_file) - if out_file: - out_file = open(out_file, "w") - args.append(out_file) - try: - begin = datetime.datetime.now() - test_function(*args) - end = datetime.datetime.now() - exit_status = 0 - except: - begin = None - begin = None - exit_status = 1 - raise - - if begin and end: - time_file.write("begintime:%s\n" % begin) - time_file.write("endtime:%s\n" % end) - time_file.write("totaltime:%s\n" % (end - begin)) - time_file.write("exit_code:%s\n" % exit_status) - time_file.close() - if in_file and hasattr(in_file, "close"): - in_file.close() - if out_file: - out_file.close() - -if __name__ == "__main__": - main(*sys.argv[1:]) - diff --git a/tools/validate b/tools/validate deleted file mode 100755 index 33d9912bb..000000000 --- a/tools/validate +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -pushd .. > /dev/null -python "tools/validate.py" $1 $2 -popd > /dev/null diff --git a/tools/validate.py b/tools/validate.py deleted file mode 100644 index dd2d14193..000000000 --- a/tools/validate.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2009 10gen, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys - -sys.path[0:0] = os.path.join(os.getcwd(), "..") -from pymongo.bson import BSON -from pymongo.son import SON - -def main(): - xml_file = sys.argv[1] - out_file = sys.argv[2] - - f = open(xml_file, "r") - xml = f.read() - f.close() - - f = open(out_file, "w") - doc = SON.from_xml(xml) - bson = BSON.from_dict(doc) - f.write(bson) - f.close() - - assert doc == bson.to_dict() - -if __name__ == "__main__": - main()