From 9c182809f3e7ce3b93da99a825d0ad73d07689d9 Mon Sep 17 00:00:00 2001 From: Mike Dirolf Date: Thu, 9 Sep 2010 16:17:36 -0400 Subject: [PATCH] Create a separate bson package PYTHON-60 Many of the pymongo modules have been moved into the bson package. Aliases for those modules have been added to the pymongo package, without deprecation warnings for now. Application developers should begin to use the bson namespace, as deprecation of moved modules will probably begin in the next release. --- pymongo/bson.py => bson/__init__.py | 47 +++---- bson/binary.py | 112 +++++++++++++++ bson/code.py | 78 +++++++++++ bson/dbref.py | 117 ++++++++++++++++ bson/errors.py | 45 ++++++ bson/max_key.py | 32 +++++ bson/min_key.py | 32 +++++ bson/objectid.py | 204 ++++++++++++++++++++++++++++ bson/son.py | 204 ++++++++++++++++++++++++++++ bson/timestamp.py | 96 +++++++++++++ bson/tz_util.py | 45 ++++++ gridfs/grid_file.py | 6 +- pymongo/_cbsonmodule.c | 20 +-- pymongo/binary.py | 99 +------------- pymongo/code.py | 65 +-------- pymongo/collection.py | 26 ++-- pymongo/cursor.py | 6 +- pymongo/database.py | 8 +- pymongo/dbref.py | 104 +------------- pymongo/errors.py | 27 +--- pymongo/helpers.py | 2 +- pymongo/json_util.py | 15 +- pymongo/max_key.py | 21 +-- pymongo/message.py | 2 +- pymongo/min_key.py | 21 +-- pymongo/objectid.py | 191 +------------------------- pymongo/son.py | 191 +------------------------- pymongo/son_manipulator.py | 6 +- pymongo/timestamp.py | 85 +----------- pymongo/tz_util.py | 34 +---- test/qcheck.py | 8 +- test/test_binary.py | 2 +- test/test_bson.py | 25 ++-- test/test_code.py | 2 +- test/test_collection.py | 8 +- test/test_cursor.py | 8 +- test/test_database.py | 13 +- test/test_dbref.py | 4 +- test/test_grid_file.py | 2 +- test/test_json_util.py | 12 +- test/test_objectid.py | 8 +- test/test_son_manipulator.py | 12 +- 42 files changed, 1101 insertions(+), 944 deletions(-) rename pymongo/bson.py => bson/__init__.py (93%) create mode 100644 bson/binary.py create mode 100644 bson/code.py create mode 100644 bson/dbref.py create mode 100644 bson/errors.py create mode 100644 bson/max_key.py create mode 100644 bson/min_key.py create mode 100644 bson/objectid.py create mode 100644 bson/son.py create mode 100644 bson/timestamp.py create mode 100644 bson/tz_util.py diff --git a/pymongo/bson.py b/bson/__init__.py similarity index 93% rename from pymongo/bson.py rename to bson/__init__.py index cd30448b7..398c7704a 100644 --- a/pymongo/bson.py +++ b/bson/__init__.py @@ -12,28 +12,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Tools for dealing with Mongo's BSON data representation. - -Generally not needed to be used by application developers.""" +"""BSON (Binary JSON) encoding and decoding. +""" import struct import re import datetime import calendar -from pymongo.binary import Binary -from pymongo.code import Code -from pymongo.dbref import DBRef -from pymongo.errors import (InvalidBSON, - InvalidDocument, - InvalidName, - InvalidStringData) -from pymongo.max_key import MaxKey -from pymongo.min_key import MinKey -from pymongo.objectid import ObjectId -from pymongo.son import SON -from pymongo.timestamp import Timestamp -from pymongo.tz_util import utc +from bson.binary import Binary +from bson.code import Code +from bson.dbref import DBRef +from bson.errors import (InvalidBSON, + InvalidDocument, + InvalidName, + InvalidStringData) +from bson.max_key import MaxKey +from bson.min_key import MinKey +from bson.objectid import ObjectId +from bson.son import SON +from bson.timestamp import Timestamp +from bson.tz_util import utc try: @@ -300,7 +299,7 @@ def _element_to_bson(key, value, check_keys): if isinstance(value, (int, long)): # TODO this is a really ugly way to check for this... if value > 2 ** 64 / 2 - 1 or value < -2 ** 64 / 2: - raise OverflowError("MongoDB can only handle up to 8-byte ints") + raise OverflowError("BSON can only handle up to 8-byte ints") if value > 2 ** 32 / 2 - 1 or value < -2 ** 32 / 2: return "\x12" + name + struct.pack("`_ + to use + """ + + def __new__(cls, data, subtype=OLD_BINARY_SUBTYPE): + if not isinstance(data, str): + raise TypeError("data must be an instance of str") + if not isinstance(subtype, int): + raise TypeError("subtype must be an instance of int") + if subtype >= 256 or subtype < 0: + raise ValueError("subtype must be contained in [0, 256)") + self = str.__new__(cls, data) + self.__subtype = subtype + return self + + @property + def subtype(self): + """Subtype of this binary data. + """ + return self.__subtype + + def __eq__(self, other): + if isinstance(other, Binary): + return (self.__subtype, str(self)) == (other.subtype, str(other)) + # We don't return NotImplemented here because if we did then + # Binary("foo") == "foo" would return True, since Binary is a + # subclass of str... + return False + + def __ne__(self, other): + return not self == other + + def __repr__(self): + return "Binary(%s, %s)" % (str.__repr__(self), self.__subtype) diff --git a/bson/code.py b/bson/code.py new file mode 100644 index 000000000..740dddcab --- /dev/null +++ b/bson/code.py @@ -0,0 +1,78 @@ +# Copyright 2009-2010 10gen, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tools for representing JavaScript code in BSON. +""" + + +class Code(str): + """BSON's JavaScript code type. + + Raises :class:`TypeError` if `code` is not an instance of + :class:`basestring` or `scope` is not ``None`` or an instance of + :class:`dict`. + + Scope variables can be set by passing a dictionary as the `scope` + argument or by using keyword arguments. If a variable is set as a + keyword argument it will override any setting for that variable in + the `scope` dictionary. + + :Parameters: + - `code`: string containing JavaScript code to be evaluated + - `scope` (optional): dictionary representing the scope in which + `code` should be evaluated - a mapping from identifiers (as + strings) to values + - `**kwargs` (optional): scope variables can also be passed as + keyword arguments + + .. versionadded:: 1.8.1+ + Ability to pass scope values using keyword arguments. + """ + + def __new__(cls, code, scope=None, **kwargs): + if not isinstance(code, basestring): + raise TypeError("code must be an instance of basestring") + + self = str.__new__(cls, code) + + try: + self.__scope = code.scope + except AttributeError: + self.__scope = {} + + if scope is not None: + if not isinstance(scope, dict): + raise TypeError("scope must be an instance of dict") + self.__scope.update(scope) + + self.__scope.update(kwargs) + + return self + + @property + def scope(self): + """Scope dictionary for this instance. + """ + return self.__scope + + def __repr__(self): + return "Code(%s, %r)" % (str.__repr__(self), self.__scope) + + def __eq__(self, other): + if isinstance(other, Code): + return (self.__scope, str(self)) == (other.__scope, str(other)) + return False + + def __ne__(self, other): + return not self == other diff --git a/bson/dbref.py b/bson/dbref.py new file mode 100644 index 000000000..2ce5f45d9 --- /dev/null +++ b/bson/dbref.py @@ -0,0 +1,117 @@ +# Copyright 2009-2010 10gen, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tools for manipulating DBRefs (references to MongoDB documents).""" + +from bson.son import SON + + +class DBRef(object): + """A reference to a document stored in MongoDB. + """ + + def __init__(self, collection, id, database=None, _extra={}, **kwargs): + """Initialize a new :class:`DBRef`. + + Raises :class:`TypeError` if `collection` or `database` is not + an instance of :class:`basestring`. `database` is optional and + allows references to documents to work across databases. Any + additional keyword arguments will create additional fields in + the resultant embedded document. + + :Parameters: + - `collection`: name of the collection the document is stored in + - `id`: the value of the document's ``"_id"`` field + - `database` (optional): name of the database to reference + - `**kwargs` (optional): additional keyword arguments will + create additional, custom fields + + .. versionchanged:: 1.8 + Now takes keyword arguments to specify additional fields. + .. versionadded:: 1.1.1 + The `database` parameter. + + .. mongodoc:: dbrefs + """ + if not isinstance(collection, basestring): + raise TypeError("collection must be an instance of basestring") + if database is not None and not isinstance(database, basestring): + raise TypeError("database must be an instance of basestring") + + self.__collection = collection + self.__id = id + self.__database = database + kwargs.update(_extra) + self.__kwargs = kwargs + + @property + def collection(self): + """Get the name of this DBRef's collection as unicode. + """ + return self.__collection + + @property + def id(self): + """Get this DBRef's _id. + """ + return self.__id + + @property + def database(self): + """Get the name of this DBRef's database. + + Returns None if this DBRef doesn't specify a database. + + .. versionadded:: 1.1.1 + """ + return self.__database + + def __getattr__(self, key): + return self.__kwargs[key] + + def as_doc(self): + """Get the SON document representation of this DBRef. + + Generally not needed by application developers + """ + doc = SON([("$ref", self.collection), + ("$id", self.id)]) + if self.database is not None: + doc["$db"] = self.database + doc.update(self.__kwargs) + return doc + + def __repr__(self): + extra = "".join([", %s=%r" % (k, v) + for k, v in self.__kwargs.iteritems()]) + if self.database is None: + return "DBRef(%r, %r%s)" % (self.collection, self.id, extra) + return "DBRef(%r, %r, %r%s)" % (self.collection, self.id, + self.database, extra) + + def __cmp__(self, other): + if isinstance(other, DBRef): + return cmp([self.__database, self.__collection, + self.__id, self.__kwargs], + [other.__database, other.__collection, + other.__id, other.__kwargs]) + return NotImplemented + + def __hash__(self): + """Get a hash value for this :class:`DBRef`. + + .. versionadded:: 1.1 + """ + return hash((self.__collection, self.__id, + self.__database, self.__kwargs)) diff --git a/bson/errors.py b/bson/errors.py new file mode 100644 index 000000000..28ec2152f --- /dev/null +++ b/bson/errors.py @@ -0,0 +1,45 @@ +# Copyright 2009-2010 10gen, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exceptions raised by the BSON package.""" + + +class BSONError(Exception): + """Base class for all BSON exceptions. + """ + + +class InvalidName(BSONError): + """Raised when an invalid name is used. + """ + + +class InvalidBSON(BSONError): + """Raised when trying to create a BSON object from invalid data. + """ + + +class InvalidStringData(BSONError): + """Raised when trying to encode a string containing non-UTF8 data. + """ + + +class InvalidDocument(BSONError): + """Raised when trying to create a BSON object from an invalid document. + """ + + +class InvalidId(BSONError): + """Raised when trying to create an ObjectId from invalid data. + """ diff --git a/bson/max_key.py b/bson/max_key.py new file mode 100644 index 000000000..8cfefc6f3 --- /dev/null +++ b/bson/max_key.py @@ -0,0 +1,32 @@ +# Copyright 2010 10gen, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Representation for the MongoDB internal MaxKey type. +""" + + +class MaxKey(object): + """MongoDB internal MaxKey type. + """ + + def __eq__(self, other): + if isinstance(other, MaxKey): + return True + return NotImplemented + + def __ne__(self, other): + return not self == other + + def __repr__(self): + return "MaxKey()" diff --git a/bson/min_key.py b/bson/min_key.py new file mode 100644 index 000000000..69162049c --- /dev/null +++ b/bson/min_key.py @@ -0,0 +1,32 @@ +# Copyright 2010 10gen, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Representation for the MongoDB internal MinKey type. +""" + + +class MinKey(object): + """MongoDB internal MinKey type. + """ + + def __eq__(self, other): + if isinstance(other, MinKey): + return True + return NotImplemented + + def __ne__(self, other): + return not self == other + + def __repr__(self): + return "MinKey()" diff --git a/bson/objectid.py b/bson/objectid.py new file mode 100644 index 000000000..6069792e4 --- /dev/null +++ b/bson/objectid.py @@ -0,0 +1,204 @@ +# Copyright 2009-2010 10gen, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tools for working with MongoDB `ObjectIds +`_. +""" + +import calendar +import datetime +try: + import hashlib + _md5func = hashlib.md5 +except ImportError: # for Python < 2.5 + import md5 + _md5func = md5.new +import os +import socket +import struct +import threading +import time + +from bson.errors import InvalidId +from bson.tz_util import utc + + +def _machine_bytes(): + """Get the machine portion of an ObjectId. + """ + machine_hash = _md5func() + machine_hash.update(socket.gethostname()) + return machine_hash.digest()[0:3] + + +class ObjectId(object): + """A MongoDB ObjectId. + """ + + _inc = 0 + _inc_lock = threading.Lock() + + _machine_bytes = _machine_bytes() + + def __init__(self, oid=None): + """Initialize a new ObjectId. + + If `oid` is ``None``, create a new (unique) ObjectId. If `oid` + is an instance of (``basestring``, :class:`ObjectId`) validate + it and use that. Otherwise, a :class:`TypeError` is + raised. If `oid` is invalid, + :class:`~bson.errors.InvalidId` is raised. + + :Parameters: + - `oid` (optional): a valid ObjectId (12 byte binary or 24 character + hex string) + + .. versionadded:: 1.2.1 + The `oid` parameter can be a ``unicode`` instance (that contains + only hexadecimal digits). + + .. mongodoc:: objectids + """ + if oid is None: + self.__generate() + else: + self.__validate(oid) + + @classmethod + def from_datetime(cls, generation_time): + """Create a dummy ObjectId instance with a specific generation time. + + This method is useful for doing range queries on a field + containing :class:`ObjectId` instances. + + .. warning:: + It is not safe to insert a document containing an ObjectId + generated using this method. This method deliberately + eliminates the uniqueness guarantee that ObjectIds + generally provide. ObjectIds generated with this method + should be used exclusively in queries. + + `generation_time` will be converted to UTC. Naive datetime + instances will be treated as though they already contain UTC. + + An example using this helper to get documents where ``"_id"`` + was generated before January 1, 2010 would be: + + >>> gen_time = datetime.datetime(2010, 1, 1) + >>> dummy_id = ObjectId.from_datetime(gen_time) + >>> result = collection.find({"_id": {"$lt": dummy_id}}) + + :Parameters: + - `generation_time`: :class:`~datetime.datetime` to be used + as the generation time for the resulting ObjectId. + + .. versionchanged:: 1.8 + Properly handle timezone aware values for + `generation_time`. + + .. versionadded:: 1.6 + """ + if generation_time.utcoffset() is not None: + generation_time = generation_time - generation_time.utcoffset() + ts = calendar.timegm(generation_time.timetuple()) + oid = struct.pack(">i", int(ts)) + "\x00" * 8 + return cls(oid) + + def __generate(self): + """Generate a new value for this ObjectId. + """ + oid = "" + + # 4 bytes current time + oid += struct.pack(">i", int(time.time())) + + # 3 bytes machine + oid += ObjectId._machine_bytes + + # 2 bytes pid + oid += struct.pack(">H", os.getpid() % 0xFFFF) + + # 3 bytes inc + ObjectId._inc_lock.acquire() + oid += struct.pack(">i", ObjectId._inc)[1:4] + ObjectId._inc = (ObjectId._inc + 1) % 0xFFFFFF + ObjectId._inc_lock.release() + + self.__id = oid + + def __validate(self, oid): + """Validate and use the given id for this ObjectId. + + Raises TypeError if id is not an instance of (str, ObjectId) and + InvalidId if it is not a valid ObjectId. + + :Parameters: + - `oid`: a valid ObjectId + """ + if isinstance(oid, ObjectId): + self.__id = oid.__id + elif isinstance(oid, basestring): + if len(oid) == 12: + self.__id = oid + elif len(oid) == 24: + try: + self.__id = oid.decode("hex") + except TypeError: + raise InvalidId("%s is not a valid ObjectId" % oid) + else: + raise InvalidId("%s is not a valid ObjectId" % oid) + else: + raise TypeError("id must be an instance of (str, ObjectId), " + "not %s" % type(oid)) + + @property + def binary(self): + """12-byte binary representation of this ObjectId. + """ + return self.__id + + @property + def generation_time(self): + """A :class:`datetime.datetime` instance representing the time of + generation for this :class:`ObjectId`. + + The :class:`datetime.datetime` is timezone aware, and + represents the generation time in UTC. It is precise to the + second. + + .. versionchanged:: 1.8 + Now return an aware datetime instead of a naive one. + + .. versionadded:: 1.2 + """ + t = struct.unpack(">i", self.__id[0:4])[0] + return datetime.datetime.fromtimestamp(t, utc) + + def __str__(self): + return self.__id.encode("hex") + + def __repr__(self): + return "ObjectId('%s')" % self.__id.encode("hex") + + def __cmp__(self, other): + if isinstance(other, ObjectId): + return cmp(self.__id, other.__id) + return NotImplemented + + def __hash__(self): + """Get a hash value for this :class:`ObjectId`. + + .. versionadded:: 1.1 + """ + return hash(self.__id) diff --git a/bson/son.py b/bson/son.py new file mode 100644 index 000000000..f2ec9ed87 --- /dev/null +++ b/bson/son.py @@ -0,0 +1,204 @@ +# Copyright 2009-2010 10gen, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tools for creating and manipulating SON, the Serialized Ocument Notation. + +Regular dictionaries can be used instead of SON objects, but not when the order +of keys is important. A SON object can be used just like a normal Python +dictionary.""" + + +class SON(dict): + """SON data. + + A subclass of dict that maintains ordering of keys and provides a + few extra niceties for dealing with SON. SON objects can be + converted to and from BSON. + + The mapping from Python types to BSON types is as follows: + + =================================== ============= =================== + Python Type BSON Type Supported Direction + =================================== ============= =================== + None null both + bool boolean both + int number (int) both + float number (real) both + string string py -> bson + unicode string both + list array both + dict / `SON` object both + datetime.datetime [#dt]_ [#dt2]_ date both + compiled re regex both + `bson.binary.Binary` binary both + `bson.objectid.ObjectId` oid both + `bson.dbref.DBRef` dbref both + None undefined bson -> py + unicode code bson -> py + `bson.code.Code` code py -> bson + unicode symbol bson -> py + =================================== ============= =================== + + Note that to save binary data it must be wrapped as an instance of + `bson.binary.Binary`. Otherwise it will be saved as a BSON string + and retrieved as unicode. + + .. [#dt] datetime.datetime instances will be rounded to the nearest + millisecond when saved + .. [#dt2] all datetime.datetime instances are treated as *naive*. clients + should always use UTC. + """ + + def __init__(self, data=None, **kwargs): + self.__keys = [] + dict.__init__(self) + self.update(data) + self.update(kwargs) + + def __repr__(self): + result = [] + for key in self.__keys: + result.append("(%r, %r)" % (key, self[key])) + return "SON([%s])" % ", ".join(result) + + def __setitem__(self, key, value): + if key not in self: + self.__keys.append(key) + dict.__setitem__(self, key, value) + + def __delitem__(self, key): + self.__keys.remove(key) + dict.__delitem__(self, key) + + def keys(self): + return list(self.__keys) + + def copy(self): + other = SON() + other.update(self) + return other + + # TODO this is all from UserDict.DictMixin. it could probably be made more + # efficient. + # second level definitions support higher levels + def __iter__(self): + for k in self.keys(): + yield k + + def has_key(self, key): + return key in self.keys() + + def __contains__(self, key): + return key in self.keys() + + # third level takes advantage of second level definitions + def iteritems(self): + for k in self: + yield (k, self[k]) + + def iterkeys(self): + return self.__iter__() + + # fourth level uses definitions from lower levels + def itervalues(self): + for _, v in self.iteritems(): + yield v + + def values(self): + return [v for _, v in self.iteritems()] + + def items(self): + return list(self.iteritems()) + + def clear(self): + for key in self.keys(): + del self[key] + + def setdefault(self, key, default=None): + try: + return self[key] + except KeyError: + self[key] = default + return default + + def pop(self, key, *args): + if len(args) > 1: + raise TypeError("pop expected at most 2 arguments, got "\ + + repr(1 + len(args))) + try: + value = self[key] + except KeyError: + if args: + return args[0] + raise + del self[key] + return value + + def popitem(self): + try: + k, v = self.iteritems().next() + except StopIteration: + raise KeyError('container is empty') + del self[k] + return (k, v) + + def update(self, other=None, **kwargs): + # Make progressively weaker assumptions about "other" + if other is None: + pass + elif hasattr(other, 'iteritems'): # iteritems saves memory and lookups + for k, v in other.iteritems(): + self[k] = v + elif hasattr(other, 'keys'): + for k in other.keys(): + self[k] = other[k] + else: + for k, v in other: + self[k] = v + if kwargs: + self.update(kwargs) + + def get(self, key, default=None): + try: + return self[key] + except KeyError: + return default + + def __cmp__(self, other): + if isinstance(other, SON): + return cmp((dict(self.iteritems()), self.keys()), + (dict(other.iteritems()), other.keys())) + return cmp(dict(self.iteritems()), other) + + def __len__(self): + return len(self.keys()) + + def to_dict(self): + """Convert a SON document to a normal Python dictionary instance. + + This is trickier than just *dict(...)* because it needs to be + recursive. + """ + + def transform_value(value): + if isinstance(value, list): + return [transform_value(v) for v in value] + if isinstance(value, SON): + value = dict(value) + if isinstance(value, dict): + for k, v in value.iteritems(): + value[k] = transform_value(v) + return value + + return transform_value(dict(self)) diff --git a/bson/timestamp.py b/bson/timestamp.py new file mode 100644 index 000000000..ac5d3b978 --- /dev/null +++ b/bson/timestamp.py @@ -0,0 +1,96 @@ +# Copyright 2010 10gen, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tools for representing MongoDB internal Timestamps. +""" + +import calendar +import datetime + +from bson.tz_util import utc + + +class Timestamp(object): + """MongoDB internal timestamps used in the opLog. + """ + + def __init__(self, time, inc): + """Create a new :class:`Timestamp`. + + This class is only for use with the MongoDB opLog. If you need + to store a regular timestamp, please use a + :class:`~datetime.datetime`. + + Raises :class:`TypeError` if `time` is not an instance of + :class: `int` or :class:`~datetime.datetime`, or `inc` is not + an instance of :class:`int`. Raises :class:`ValueError` if + `time` or `inc` is not in [0, 2**32). + + :Parameters: + - `time`: time in seconds since epoch UTC, or a naive UTC + :class:`~datetime.datetime`, or an aware + :class:`~datetime.datetime` + - `inc`: the incrementing counter + + .. versionchanged:: 1.7 + `time` can now be a :class:`~datetime.datetime` instance. + """ + if isinstance(time, datetime.datetime): + if time.utcoffset() is not None: + time = time - time.utcoffset() + time = int(calendar.timegm(time.timetuple())) + if not isinstance(time, (int, long)): + raise TypeError("time must be an instance of int") + if not isinstance(inc, (int, long)): + raise TypeError("inc must be an instance of int") + if not 0 <= time < 2 ** 32: + raise ValueError("time must be contained in [0, 2**32)") + if not 0 <= inc < 2 ** 32: + raise ValueError("inc must be contained in [0, 2**32)") + + self.__time = time + self.__inc = inc + + @property + def time(self): + """Get the time portion of this :class:`Timestamp`. + """ + return self.__time + + @property + def inc(self): + """Get the inc portion of this :class:`Timestamp`. + """ + return self.__inc + + def __eq__(self, other): + if isinstance(other, Timestamp): + return (self.__time == other.time and self.__inc == other.inc) + else: + return NotImplemented + + def __ne__(self, other): + return not self == other + + def __repr__(self): + return "Timestamp(%s, %s)" % (self.__time, self.__inc) + + def as_datetime(self): + """Return a :class:`~datetime.datetime` instance corresponding + to the time portion of this :class:`Timestamp`. + + .. versionchanged:: 1.8 + The returned datetime is now timezone aware. + """ + return datetime.datetime.fromtimestamp(self.__time, utc) diff --git a/bson/tz_util.py b/bson/tz_util.py new file mode 100644 index 000000000..8c2abeeb6 --- /dev/null +++ b/bson/tz_util.py @@ -0,0 +1,45 @@ +# Copyright 2010 10gen, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Timezone related utilities for BSON.""" + +from datetime import (timedelta, + tzinfo) + +ZERO = timedelta(0) + + +class FixedOffset(tzinfo): + """Fixed offset timezone, in minutes east from UTC. + + Implementation from the Python `standard library documentation + `_. + """ + + def __init__(self, offset, name): + self.__offset = timedelta(minutes=offset) + self.__name = name + + def utcoffset(self, dt): + return self.__offset + + def tzname(self, dt): + return self.__name + + def dst(self, dt): + return ZERO + + +"""UTC""" +utc = FixedOffset(0, "UTC") diff --git a/gridfs/grid_file.py b/gridfs/grid_file.py index b78dfb9f1..44893b168 100644 --- a/gridfs/grid_file.py +++ b/gridfs/grid_file.py @@ -22,15 +22,15 @@ try: except ImportError: from StringIO import StringIO +from bson.binary import Binary +from bson.objectid import ObjectId from gridfs.errors import (CorruptGridFile, FileExists, NoFile, UnsupportedAPI) from pymongo import ASCENDING -from pymongo.binary import Binary from pymongo.collection import Collection from pymongo.errors import DuplicateKeyError -from pymongo.objectid import ObjectId try: _SEEK_SET = os.SEEK_SET @@ -95,7 +95,7 @@ class GridIn(object): arguments include: - ``"_id"``: unique ID for this file (default: - :class:`~pymongo.objectid.ObjectId`) - this ``"_id"`` must + :class:`~bson.objectid.ObjectId`) - this ``"_id"`` must not have already been used for another file - ``"filename"``: human name for the file diff --git a/pymongo/_cbsonmodule.c b/pymongo/_cbsonmodule.c index 5dad11af1..1c22a0ad7 100644 --- a/pymongo/_cbsonmodule.c +++ b/pymongo/_cbsonmodule.c @@ -233,12 +233,12 @@ static int write_string(bson_buffer* buffer, PyObject* py_string) { return 1; } -/* Get an error class from the pymongo.errors module. +/* Get an error class from the bson.errors module. * * Returns a new ref */ static PyObject* _error(char* name) { PyObject* error; - PyObject* errors = PyImport_ImportModule("pymongo.errors"); + PyObject* errors = PyImport_ImportModule("bson.errors"); if (!errors) { return NULL; } @@ -268,14 +268,14 @@ static int _reload_object(PyObject** object, char* module_name, char* object_nam * * Returns non-zero on failure. */ static int _reload_python_objects(void) { - if (_reload_object(&Binary, "pymongo.binary", "Binary") || - _reload_object(&Code, "pymongo.code", "Code") || - _reload_object(&ObjectId, "pymongo.objectid", "ObjectId") || - _reload_object(&DBRef, "pymongo.dbref", "DBRef") || - _reload_object(&Timestamp, "pymongo.timestamp", "Timestamp") || - _reload_object(&MinKey, "pymongo.min_key", "MinKey") || - _reload_object(&MaxKey, "pymongo.max_key", "MaxKey") || - _reload_object(&UTC, "pymongo.tz_util", "utc") || + if (_reload_object(&Binary, "bson.binary", "Binary") || + _reload_object(&Code, "bson.code", "Code") || + _reload_object(&ObjectId, "bson.objectid", "ObjectId") || + _reload_object(&DBRef, "bson.dbref", "DBRef") || + _reload_object(&Timestamp, "bson.timestamp", "Timestamp") || + _reload_object(&MinKey, "bson.min_key", "MinKey") || + _reload_object(&MaxKey, "bson.max_key", "MaxKey") || + _reload_object(&UTC, "bson.tz_util", "utc") || _reload_object(&RECompile, "re", "compile")) { return 1; } diff --git a/pymongo/binary.py b/pymongo/binary.py index 2807ef0c7..165ecc94e 100644 --- a/pymongo/binary.py +++ b/pymongo/binary.py @@ -12,101 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Tools for representing binary data to be stored in MongoDB. -""" - -BINARY_SUBTYPE = 0 -"""BSON binary subtype for binary data. - -This is becomming the default subtype and should be the most commonly -used. - -.. versionadded:: 1.5 -""" - -FUNCTION_SUBTYPE = 1 -"""BSON binary subtype for functions. - -.. versionadded:: 1.5 -""" - -OLD_BINARY_SUBTYPE = 2 -"""Old BSON binary subtype for binary data. - -This is still the default subtype, but that is changing to -:data:`BINARY_SUBTYPE`. - -.. versionadded:: 1.7 -""" - -UUID_SUBTYPE = 3 -"""BSON binary subtype for a UUID. - -:class:`uuid.UUID` instances will automatically be encoded -by :mod:`~pymongo.bson` using this subtype. - -.. versionadded:: 1.5 -""" - -MD5_SUBTYPE = 5 -"""BSON binary subtype for an MD5 hash. - -.. versionadded:: 1.5 -""" - -USER_DEFINED_SUBTYPE = 128 -"""BSON binary subtype for any user defined structure. - -.. versionadded:: 1.5 -""" - - -class Binary(str): - """Representation of binary data to be stored in or retrieved from MongoDB. - - This is necessary because we want to store Python strings as the - BSON string type. We need to wrap binary data so we can tell the - difference between what should be considered binary data and what - should be considered a string when we encode to BSON. - - Raises TypeError if `data` is not an instance of str or `subtype` - is not an instance of int. Raises ValueError if `subtype` is not - in [0, 256). - - :Parameters: - - `data`: the binary data to represent - - `subtype` (optional): the `binary subtype - `_ - to use - """ - - def __new__(cls, data, subtype=OLD_BINARY_SUBTYPE): - if not isinstance(data, str): - raise TypeError("data must be an instance of str") - if not isinstance(subtype, int): - raise TypeError("subtype must be an instance of int") - if subtype >= 256 or subtype < 0: - raise ValueError("subtype must be contained in [0, 256)") - self = str.__new__(cls, data) - self.__subtype = subtype - return self - - @property - def subtype(self): - """Subtype of this binary data. - """ - return self.__subtype - - def __eq__(self, other): - if isinstance(other, Binary): - return (self.__subtype, str(self)) == (other.subtype, str(other)) - # We don't return NotImplemented here because if we did then - # Binary("foo") == "foo" would return True, since Binary is a - # subclass of str... - return False - - def __ne__(self, other): - return not self == other - - def __repr__(self): - return "Binary(%s, %s)" % (str.__repr__(self), self.__subtype) +from bson.binary import * diff --git a/pymongo/code.py b/pymongo/code.py index 7f570129c..1d158081c 100644 --- a/pymongo/code.py +++ b/pymongo/code.py @@ -12,67 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Tools for representing JavaScript code to be evaluated by MongoDB. -""" - - -class Code(str): - """JavaScript code to be evaluated by MongoDB. - - Raises :class:`TypeError` if `code` is not an instance of - :class:`basestring` or `scope` is not ``None`` or an instance of - :class:`dict`. - - Scope variables can be set by passing a dictionary as the `scope` - argument or by using keyword arguments. If a variable is set as a - keyword argument it will override any setting for that variable in - the `scope` dictionary. - - :Parameters: - - `code`: string containing JavaScript code to be evaluated - - `scope` (optional): dictionary representing the scope in which - `code` should be evaluated - a mapping from identifiers (as - strings) to values - - `**kwargs` (optional): scope variables can also be passed as - keyword arguments - - .. versionadded:: 1.8.1+ - Ability to pass scope values using keyword arguments. - """ - - def __new__(cls, code, scope=None, **kwargs): - if not isinstance(code, basestring): - raise TypeError("code must be an instance of basestring") - - self = str.__new__(cls, code) - - try: - self.__scope = code.scope - except AttributeError: - self.__scope = {} - - if scope is not None: - if not isinstance(scope, dict): - raise TypeError("scope must be an instance of dict") - self.__scope.update(scope) - - self.__scope.update(kwargs) - - return self - - @property - def scope(self): - """Scope dictionary for this instance. - """ - return self.__scope - - def __repr__(self): - return "Code(%s, %r)" % (str.__repr__(self), self.__scope) - - def __eq__(self, other): - if isinstance(other, Code): - return (self.__scope, str(self)) == (other.__scope, str(other)) - return False - - def __ne__(self, other): - return not self == other +from bson.code import * diff --git a/pymongo/collection.py b/pymongo/collection.py index 576fb1718..51bb8e806 100644 --- a/pymongo/collection.py +++ b/pymongo/collection.py @@ -16,12 +16,12 @@ import warnings +from bson.code import Code +from bson.errors import InvalidName +from bson.son import SON from pymongo import (helpers, message) -from pymongo.code import Code from pymongo.cursor import Cursor -from pymongo.errors import InvalidName -from pymongo.son import SON _ZERO = "\x00\x00\x00\x00" @@ -410,10 +410,9 @@ class Collection(object): .. versionadded:: 1.8 Support for passing `getLastError` options as keyword arguments. - .. versionchanged:: 1.7 - Accept any type other than a ``dict`` instance for removal - by ``"_id"``, not just :class:`~pymongo.objectid.ObjectId` - instances. + .. versionchanged:: 1.7 Accept any type other than a ``dict`` + instance for removal by ``"_id"``, not just + :class:`~bson.objectid.ObjectId` instances. .. versionchanged:: 1.4 Return the response to *lastError* if `safe` is ``True``. .. versionchanged:: 1.2 @@ -460,10 +459,9 @@ class Collection(object): Allow passing any of the arguments that are valid for :meth:`find`. - .. versionchanged:: 1.7 - Accept any type other than a ``dict`` instance as an - ``"_id"`` query, not just - :class:`~pymongo.objectid.ObjectId` instances. + .. versionchanged:: 1.7 Accept any type other than a ``dict`` + instance as an ``"_id"`` query, not just + :class:`~bson.objectid.ObjectId` instances. """ if spec_or_id is not None and not isinstance(spec_or_id, dict): spec_or_id = {"_id": spec_or_id} @@ -820,9 +818,9 @@ class Collection(object): - ``None`` to use the entire document as a key. - A :class:`list` of keys (each a :class:`basestring`) to group by. - - A :class:`basestring` or :class:`~pymongo.code.Code` instance - containing a JavaScript function to be applied to each document, - returning the key to group by. + - A :class:`basestring` or :class:`~bson.code.Code` instance + containing a JavaScript function to be applied to each + document, returning the key to group by. :Parameters: - `key`: fields to group by (see above description) diff --git a/pymongo/cursor.py b/pymongo/cursor.py index 0f2006989..678c5e555 100644 --- a/pymongo/cursor.py +++ b/pymongo/cursor.py @@ -14,12 +14,12 @@ """Cursor class to iterate over Mongo query results.""" +from bson.code import Code +from bson.son import SON from pymongo import (helpers, message) -from pymongo.code import Code from pymongo.errors import (InvalidOperation, AutoReconnect) -from pymongo.son import SON _QUERY_OPTIONS = { "tailable_cursor": 2, @@ -485,7 +485,7 @@ class Cursor(object): """Adds a $where clause to this query. The `code` argument must be an instance of :class:`basestring` - or :class:`~pymongo.code.Code` containing a JavaScript + or :class:`~bson.code.Code` containing a JavaScript expression. This expression will be evaluated for each document scanned. Only those documents for which the expression evaluates to *true* will be returned as diff --git a/pymongo/database.py b/pymongo/database.py index 14857cf74..ecfc75404 100644 --- a/pymongo/database.py +++ b/pymongo/database.py @@ -16,14 +16,14 @@ import warnings +from bson.code import Code +from bson.dbref import DBRef +from bson.errors import InvalidName +from bson.son import SON from pymongo import helpers -from pymongo.code import Code from pymongo.collection import Collection -from pymongo.dbref import DBRef from pymongo.errors import (CollectionInvalid, - InvalidName, OperationFailure) -from pymongo.son import SON from pymongo.son_manipulator import ObjectIdInjector diff --git a/pymongo/dbref.py b/pymongo/dbref.py index d6c15c527..8a0cb0283 100644 --- a/pymongo/dbref.py +++ b/pymongo/dbref.py @@ -12,106 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Tools for manipulating DBRefs (references to MongoDB documents).""" - -from pymongo.son import SON - - -class DBRef(object): - """A reference to a document stored in a Mongo database. - """ - - def __init__(self, collection, id, database=None, _extra={}, **kwargs): - """Initialize a new :class:`DBRef`. - - Raises :class:`TypeError` if `collection` or `database` is not - an instance of :class:`basestring`. `database` is optional and - allows references to documents to work across databases. Any - additional keyword arguments will create additional fields in - the resultant embedded document. - - :Parameters: - - `collection`: name of the collection the document is stored in - - `id`: the value of the document's ``"_id"`` field - - `database` (optional): name of the database to reference - - `**kwargs` (optional): additional keyword arguments will - create additional, custom fields - - .. versionchanged:: 1.8 - Now takes keyword arguments to specify additional fields. - .. versionadded:: 1.1.1 - The `database` parameter. - - .. mongodoc:: dbrefs - """ - if not isinstance(collection, basestring): - raise TypeError("collection must be an instance of basestring") - if database is not None and not isinstance(database, basestring): - raise TypeError("database must be an instance of basestring") - - self.__collection = collection - self.__id = id - self.__database = database - kwargs.update(_extra) - self.__kwargs = kwargs - - @property - def collection(self): - """Get the name of this DBRef's collection as unicode. - """ - return self.__collection - - @property - def id(self): - """Get this DBRef's _id. - """ - return self.__id - - @property - def database(self): - """Get the name of this DBRef's database. - - Returns None if this DBRef doesn't specify a database. - - .. versionadded:: 1.1.1 - """ - return self.__database - - def __getattr__(self, key): - return self.__kwargs[key] - - def as_doc(self): - """Get the SON document representation of this DBRef. - - Generally not needed by application developers - """ - doc = SON([("$ref", self.collection), - ("$id", self.id)]) - if self.database is not None: - doc["$db"] = self.database - doc.update(self.__kwargs) - return doc - - def __repr__(self): - extra = "".join([", %s=%r" % (k, v) - for k, v in self.__kwargs.iteritems()]) - if self.database is None: - return "DBRef(%r, %r%s)" % (self.collection, self.id, extra) - return "DBRef(%r, %r, %r%s)" % (self.collection, self.id, - self.database, extra) - - def __cmp__(self, other): - if isinstance(other, DBRef): - return cmp([self.__database, self.__collection, - self.__id, self.__kwargs], - [other.__database, other.__collection, - other.__id, other.__kwargs]) - return NotImplemented - - def __hash__(self): - """Get a hash value for this :class:`DBRef`. - - .. versionadded:: 1.1 - """ - return hash((self.__collection, self.__id, - self.__database, self.__kwargs)) +from bson.dbref import * diff --git a/pymongo/errors.py b/pymongo/errors.py index 05b7c83e0..1bad548e0 100644 --- a/pymongo/errors.py +++ b/pymongo/errors.py @@ -14,6 +14,8 @@ """Exceptions raised by PyMongo.""" +from bson.errors import * + class PyMongoError(Exception): """Base class for all PyMongo exceptions. @@ -82,31 +84,6 @@ class CollectionInvalid(PyMongoError): """ -class InvalidName(PyMongoError): - """Raised when an invalid name is used. - """ - - -class InvalidBSON(PyMongoError): - """Raised when trying to create a BSON object from invalid data. - """ - - -class InvalidStringData(PyMongoError): - """Raised when trying to encode a string containing non-UTF8 data. - """ - - -class InvalidDocument(PyMongoError): - """Raised when trying to create a BSON object from an invalid document. - """ - - -class InvalidId(PyMongoError): - """Raised when trying to create an ObjectId from invalid data. - """ - - class InvalidURI(ConfigurationError): """Raised when trying to parse an invalid mongodb URI. diff --git a/pymongo/helpers.py b/pymongo/helpers.py index 2896f2865..0dc937ee5 100644 --- a/pymongo/helpers.py +++ b/pymongo/helpers.py @@ -22,8 +22,8 @@ except: # for Python < 2.5 _md5func = md5.new import struct +import bson import pymongo -from pymongo import bson from pymongo.errors import (AutoReconnect, OperationFailure, TimeoutError) diff --git a/pymongo/json_util.py b/pymongo/json_util.py index d4dfb25ed..e3f9203d0 100644 --- a/pymongo/json_util.py +++ b/pymongo/json_util.py @@ -31,8 +31,7 @@ Example usage (deserialization):: >>> json.loads(..., object_hook=json_util.object_hook) Currently this does not handle special encoding and decoding for -:class:`~pymongo.binary.Binary` and :class:`~pymongo.code.Code` -instances. +:class:`~bson.binary.Binary` and :class:`~bson.code.Code` instances. .. versionchanged:: 1.8.1+ Handle :class:`uuid.UUID` instances, whenever possible. @@ -60,12 +59,12 @@ try: except ImportError: _use_uuid = False -from pymongo.dbref import DBRef -from pymongo.max_key import MaxKey -from pymongo.min_key import MinKey -from pymongo.objectid import ObjectId -from pymongo.timestamp import Timestamp -from pymongo.tz_util import utc +from bson.dbref import DBRef +from bson.max_key import MaxKey +from bson.min_key import MinKey +from bson.objectid import ObjectId +from bson.timestamp import Timestamp +from bson.tz_util import utc # TODO support Binary and Code # Binary and Code are tricky because they subclass str so json thinks it can diff --git a/pymongo/max_key.py b/pymongo/max_key.py index 8cfefc6f3..b84945fba 100644 --- a/pymongo/max_key.py +++ b/pymongo/max_key.py @@ -1,4 +1,4 @@ -# Copyright 2010 10gen, Inc. +# Copyright 2009-2010 10gen, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,21 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Representation for the MongoDB internal MaxKey type. -""" - - -class MaxKey(object): - """MongoDB internal MaxKey type. - """ - - def __eq__(self, other): - if isinstance(other, MaxKey): - return True - return NotImplemented - - def __ne__(self, other): - return not self == other - - def __repr__(self): - return "MaxKey()" +from bson.max_key import * diff --git a/pymongo/message.py b/pymongo/message.py index b4fa01dc4..12b4eeb55 100644 --- a/pymongo/message.py +++ b/pymongo/message.py @@ -25,7 +25,7 @@ MongoDB. import random import struct -from pymongo import bson +import bson from pymongo.son import SON try: from pymongo import _cbson diff --git a/pymongo/min_key.py b/pymongo/min_key.py index 69162049c..f7b7af697 100644 --- a/pymongo/min_key.py +++ b/pymongo/min_key.py @@ -1,4 +1,4 @@ -# Copyright 2010 10gen, Inc. +# Copyright 2009-2010 10gen, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,21 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Representation for the MongoDB internal MinKey type. -""" - - -class MinKey(object): - """MongoDB internal MinKey type. - """ - - def __eq__(self, other): - if isinstance(other, MinKey): - return True - return NotImplemented - - def __ne__(self, other): - return not self == other - - def __repr__(self): - return "MinKey()" +from bson.min_key import * diff --git a/pymongo/objectid.py b/pymongo/objectid.py index b480288f8..48c5ee810 100644 --- a/pymongo/objectid.py +++ b/pymongo/objectid.py @@ -12,193 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Tools for working with MongoDB `ObjectIds -`_. -""" - -import calendar -import datetime -try: - import hashlib - _md5func = hashlib.md5 -except ImportError: # for Python < 2.5 - import md5 - _md5func = md5.new -import os -import socket -import struct -import threading -import time - -from pymongo.errors import InvalidId -from pymongo.tz_util import utc - - -def _machine_bytes(): - """Get the machine portion of an ObjectId. - """ - machine_hash = _md5func() - machine_hash.update(socket.gethostname()) - return machine_hash.digest()[0:3] - - -class ObjectId(object): - """A Mongo ObjectId. - """ - - _inc = 0 - _inc_lock = threading.Lock() - - _machine_bytes = _machine_bytes() - - def __init__(self, oid=None): - """Initialize a new ObjectId. - - If `oid` is ``None``, create a new (unique) ObjectId. If `oid` - is an instance of (``basestring``, :class:`ObjectId`) validate - it and use that. Otherwise, a :class:`TypeError` is - raised. If `oid` is invalid, - :class:`~pymongo.errors.InvalidId` is raised. - - :Parameters: - - `oid` (optional): a valid ObjectId (12 byte binary or 24 character - hex string) - - .. versionadded:: 1.2.1 - The `oid` parameter can be a ``unicode`` instance (that contains - only hexadecimal digits). - - .. mongodoc:: objectids - """ - if oid is None: - self.__generate() - else: - self.__validate(oid) - - @classmethod - def from_datetime(cls, generation_time): - """Create a dummy ObjectId instance with a specific generation time. - - This method is useful for doing range queries on a field - containing :class:`ObjectId` instances. - - .. warning:: - It is not safe to insert a document containing an ObjectId - generated using this method. This method deliberately - eliminates the uniqueness guarantee that ObjectIds - generally provide. ObjectIds generated with this method - should be used exclusively in queries. - - `generation_time` will be converted to UTC. Naive datetime - instances will be treated as though they already contain UTC. - - An example using this helper to get documents where ``"_id"`` - was generated before January 1, 2010 would be: - - >>> gen_time = datetime.datetime(2010, 1, 1) - >>> dummy_id = ObjectId.from_datetime(gen_time) - >>> result = collection.find({"_id": {"$lt": dummy_id}}) - - :Parameters: - - `generation_time`: :class:`~datetime.datetime` to be used - as the generation time for the resulting ObjectId. - - .. versionchanged:: 1.8 - Properly handle timezone aware values for - `generation_time`. - - .. versionadded:: 1.6 - """ - if generation_time.utcoffset() is not None: - generation_time = generation_time - generation_time.utcoffset() - ts = calendar.timegm(generation_time.timetuple()) - oid = struct.pack(">i", int(ts)) + "\x00" * 8 - return cls(oid) - - def __generate(self): - """Generate a new value for this ObjectId. - """ - oid = "" - - # 4 bytes current time - oid += struct.pack(">i", int(time.time())) - - # 3 bytes machine - oid += ObjectId._machine_bytes - - # 2 bytes pid - oid += struct.pack(">H", os.getpid() % 0xFFFF) - - # 3 bytes inc - ObjectId._inc_lock.acquire() - oid += struct.pack(">i", ObjectId._inc)[1:4] - ObjectId._inc = (ObjectId._inc + 1) % 0xFFFFFF - ObjectId._inc_lock.release() - - self.__id = oid - - def __validate(self, oid): - """Validate and use the given id for this ObjectId. - - Raises TypeError if id is not an instance of (str, ObjectId) and - InvalidId if it is not a valid ObjectId. - - :Parameters: - - `oid`: a valid ObjectId - """ - if isinstance(oid, ObjectId): - self.__id = oid.__id - elif isinstance(oid, basestring): - if len(oid) == 12: - self.__id = oid - elif len(oid) == 24: - try: - self.__id = oid.decode("hex") - except TypeError: - raise InvalidId("%s is not a valid ObjectId" % oid) - else: - raise InvalidId("%s is not a valid ObjectId" % oid) - else: - raise TypeError("id must be an instance of (str, ObjectId), " - "not %s" % type(oid)) - - @property - def binary(self): - """12-byte binary representation of this ObjectId. - """ - return self.__id - - @property - def generation_time(self): - """A :class:`datetime.datetime` instance representing the time of - generation for this :class:`ObjectId`. - - The :class:`datetime.datetime` is timezone aware, and - represents the generation time in UTC. It is precise to the - second. - - .. versionchanged:: 1.8 - Now return an aware datetime instead of a naive one. - - .. versionadded:: 1.2 - """ - t = struct.unpack(">i", self.__id[0:4])[0] - return datetime.datetime.fromtimestamp(t, utc) - - def __str__(self): - return self.__id.encode("hex") - - def __repr__(self): - return "ObjectId('%s')" % self.__id.encode("hex") - - def __cmp__(self, other): - if isinstance(other, ObjectId): - return cmp(self.__id, other.__id) - return NotImplemented - - def __hash__(self): - """Get a hash value for this :class:`ObjectId`. - - .. versionadded:: 1.1 - """ - return hash(self.__id) +from bson.objectid import * diff --git a/pymongo/son.py b/pymongo/son.py index 037d0952b..80b1e8e1f 100644 --- a/pymongo/son.py +++ b/pymongo/son.py @@ -12,193 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Tools for creating and manipulating SON, the Serialized Ocument Notation. - -Regular dictionaries can be used instead of SON objects, but not when the order -of keys is important. A SON object can be used just like a normal Python -dictionary.""" - - -class SON(dict): - """SON data. - - A subclass of dict that maintains ordering of keys and provides a few extra - niceties for dealing with SON. SON objects can be saved and retrieved from - Mongo. - - The mapping from Python types to Mongo types is as follows: - - =================================== ============= =================== - Python Type Mongo Type Supported Direction - =================================== ============= =================== - None null both - bool boolean both - int number (int) both - float number (real) both - string string py -> mongo - unicode string both - list array both - dict / `SON` object both - datetime.datetime [#dt]_ [#dt2]_ date both - compiled re regex both - `pymongo.binary.Binary` binary both - `pymongo.objectid.ObjectId` oid both - `pymongo.dbref.DBRef` dbref both - None undefined mongo -> py - unicode code mongo -> py - `pymongo.code.Code` code py -> mongo - unicode symbol mongo -> py - =================================== ============= =================== - - Note that to save binary data it must be wrapped as an instance of - `pymongo.binary.Binary`. Otherwise it will be saved as a Mongo string and - retrieved as unicode. - - .. [#dt] datetime.datetime instances will be rounded to the nearest - millisecond when saved - .. [#dt2] all datetime.datetime instances are treated as *naive*. clients - should always use UTC. - """ - - def __init__(self, data=None, **kwargs): - self.__keys = [] - dict.__init__(self) - self.update(data) - self.update(kwargs) - - def __repr__(self): - result = [] - for key in self.__keys: - result.append("(%r, %r)" % (key, self[key])) - return "SON([%s])" % ", ".join(result) - - def __setitem__(self, key, value): - if key not in self: - self.__keys.append(key) - dict.__setitem__(self, key, value) - - def __delitem__(self, key): - self.__keys.remove(key) - dict.__delitem__(self, key) - - def keys(self): - return list(self.__keys) - - def copy(self): - other = SON() - other.update(self) - return other - - # TODO this is all from UserDict.DictMixin. it could probably be made more - # efficient. - # second level definitions support higher levels - def __iter__(self): - for k in self.keys(): - yield k - - def has_key(self, key): - return key in self.keys() - - def __contains__(self, key): - return key in self.keys() - - # third level takes advantage of second level definitions - def iteritems(self): - for k in self: - yield (k, self[k]) - - def iterkeys(self): - return self.__iter__() - - # fourth level uses definitions from lower levels - def itervalues(self): - for _, v in self.iteritems(): - yield v - - def values(self): - return [v for _, v in self.iteritems()] - - def items(self): - return list(self.iteritems()) - - def clear(self): - for key in self.keys(): - del self[key] - - def setdefault(self, key, default=None): - try: - return self[key] - except KeyError: - self[key] = default - return default - - def pop(self, key, *args): - if len(args) > 1: - raise TypeError("pop expected at most 2 arguments, got "\ - + repr(1 + len(args))) - try: - value = self[key] - except KeyError: - if args: - return args[0] - raise - del self[key] - return value - - def popitem(self): - try: - k, v = self.iteritems().next() - except StopIteration: - raise KeyError('container is empty') - del self[k] - return (k, v) - - def update(self, other=None, **kwargs): - # Make progressively weaker assumptions about "other" - if other is None: - pass - elif hasattr(other, 'iteritems'): # iteritems saves memory and lookups - for k, v in other.iteritems(): - self[k] = v - elif hasattr(other, 'keys'): - for k in other.keys(): - self[k] = other[k] - else: - for k, v in other: - self[k] = v - if kwargs: - self.update(kwargs) - - def get(self, key, default=None): - try: - return self[key] - except KeyError: - return default - - def __cmp__(self, other): - if isinstance(other, SON): - return cmp((dict(self.iteritems()), self.keys()), - (dict(other.iteritems()), other.keys())) - return cmp(dict(self.iteritems()), other) - - def __len__(self): - return len(self.keys()) - - def to_dict(self): - """Convert a SON document to a normal Python dictionary instance. - - This is trickier than just *dict(...)* because it needs to be - recursive. - """ - - def transform_value(value): - if isinstance(value, list): - return [transform_value(v) for v in value] - if isinstance(value, SON): - value = dict(value) - if isinstance(value, dict): - for k, v in value.iteritems(): - value[k] = transform_value(v) - return value - - return transform_value(dict(self)) +from bson.son import * diff --git a/pymongo/son_manipulator.py b/pymongo/son_manipulator.py index 1a07e4349..de90d9ac1 100644 --- a/pymongo/son_manipulator.py +++ b/pymongo/son_manipulator.py @@ -18,9 +18,9 @@ New manipulators should be defined as subclasses of SONManipulator and can be installed on a database by calling `pymongo.database.Database.add_son_manipulator`.""" -from pymongo.dbref import DBRef -from pymongo.objectid import ObjectId -from pymongo.son import SON +from bson.dbref import DBRef +from bson.objectid import ObjectId +from bson.son import SON class SONManipulator(object): diff --git a/pymongo/timestamp.py b/pymongo/timestamp.py index 7c77c0b06..1c72fa246 100644 --- a/pymongo/timestamp.py +++ b/pymongo/timestamp.py @@ -1,4 +1,4 @@ -# Copyright 2010 10gen, Inc. +# Copyright 2009-2010 10gen, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,85 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Tools for representing MongoDB internal Timestamps. -""" - -import calendar -import datetime - -from pymongo.tz_util import utc - - -class Timestamp(object): - """MongoDB internal timestamps used in the opLog. - """ - - def __init__(self, time, inc): - """Create a new :class:`Timestamp`. - - This class is only for use with the MongoDB opLog. If you need - to store a regular timestamp, please use a - :class:`~datetime.datetime`. - - Raises :class:`TypeError` if `time` is not an instance of - :class: `int` or :class:`~datetime.datetime`, or `inc` is not - an instance of :class:`int`. Raises :class:`ValueError` if - `time` or `inc` is not in [0, 2**32). - - :Parameters: - - `time`: time in seconds since epoch UTC, or a naive UTC - :class:`~datetime.datetime`, or an aware - :class:`~datetime.datetime` - - `inc`: the incrementing counter - - .. versionchanged:: 1.7 - `time` can now be a :class:`~datetime.datetime` instance. - """ - if isinstance(time, datetime.datetime): - if time.utcoffset() is not None: - time = time - time.utcoffset() - time = int(calendar.timegm(time.timetuple())) - if not isinstance(time, (int, long)): - raise TypeError("time must be an instance of int") - if not isinstance(inc, (int, long)): - raise TypeError("inc must be an instance of int") - if not 0 <= time < 2 ** 32: - raise ValueError("time must be contained in [0, 2**32)") - if not 0 <= inc < 2 ** 32: - raise ValueError("inc must be contained in [0, 2**32)") - - self.__time = time - self.__inc = inc - - @property - def time(self): - """Get the time portion of this :class:`Timestamp`. - """ - return self.__time - - @property - def inc(self): - """Get the inc portion of this :class:`Timestamp`. - """ - return self.__inc - - def __eq__(self, other): - if isinstance(other, Timestamp): - return (self.__time == other.time and self.__inc == other.inc) - else: - return NotImplemented - - def __ne__(self, other): - return not self == other - - def __repr__(self): - return "Timestamp(%s, %s)" % (self.__time, self.__inc) - - def as_datetime(self): - """Return a :class:`~datetime.datetime` instance corresponding - to the time portion of this :class:`Timestamp`. - - .. versionchanged:: 1.8 - The returned datetime is now timezone aware. - """ - return datetime.datetime.fromtimestamp(self.__time, utc) +from bson.timestamp import * diff --git a/pymongo/tz_util.py b/pymongo/tz_util.py index 71176499c..ab4723bf3 100644 --- a/pymongo/tz_util.py +++ b/pymongo/tz_util.py @@ -1,4 +1,4 @@ -# Copyright 2010 10gen, Inc. +# Copyright 2009-2010 10gen, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,34 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Timezone related utilities for PyMongo.""" - -from datetime import (timedelta, - tzinfo) - -ZERO = timedelta(0) - - -class FixedOffset(tzinfo): - """Fixed offset timezone, in minutes east from UTC. - - Implementation from the Python `standard library documentation - `_. - """ - - def __init__(self, offset, name): - self.__offset = timedelta(minutes=offset) - self.__name = name - - def utcoffset(self, dt): - return self.__offset - - def tzname(self, dt): - return self.__name - - def dst(self, dt): - return ZERO - - -"""UTC""" -utc = FixedOffset(0, "UTC") +from bson.tz_util import * diff --git a/test/qcheck.py b/test/qcheck.py index 6d45940e3..6eb4f4df7 100644 --- a/test/qcheck.py +++ b/test/qcheck.py @@ -19,10 +19,10 @@ import re import sys sys.path[0:0] = [""] -from pymongo.binary import Binary -from pymongo.objectid import ObjectId -from pymongo.dbref import DBRef -from pymongo.son import SON +from bson.binary import Binary +from bson.dbref import DBRef +from bson.objectid import ObjectId +from bson.son import SON gen_target = 100 reduction_attempts = 10 diff --git a/test/test_binary.py b/test/test_binary.py index 7184c288f..fc9d704b5 100644 --- a/test/test_binary.py +++ b/test/test_binary.py @@ -18,7 +18,7 @@ import unittest import sys sys.path[0:0] = [""] -from pymongo.binary import Binary +from bson.binary import Binary class TestBinary(unittest.TestCase): diff --git a/test/test_bson.py b/test/test_bson.py index fb90a9344..c4be51411 100644 --- a/test/test_bson.py +++ b/test/test_bson.py @@ -29,19 +29,20 @@ sys.path[0:0] = [""] from nose.plugins.skip import SkipTest +from bson import BSON, is_valid, _to_dicts +from bson.binary import Binary +from bson.code import Code +from bson.objectid import ObjectId +from bson.dbref import DBRef +from bson.son import SON +from bson.timestamp import Timestamp +from bson.errors import (InvalidDocument, + InvalidStringData) +from bson.max_key import MaxKey +from bson.min_key import MinKey +from bson.tz_util import (FixedOffset, + utc) import pymongo -from pymongo.binary import Binary -from pymongo.code import Code -from pymongo.objectid import ObjectId -from pymongo.dbref import DBRef -from pymongo.son import SON -from pymongo.timestamp import Timestamp -from pymongo.bson import BSON, is_valid, _to_dicts -from pymongo.errors import InvalidDocument, InvalidStringData -from pymongo.max_key import MaxKey -from pymongo.min_key import MinKey -from pymongo.tz_util import (FixedOffset, - utc) import qcheck class TestBSON(unittest.TestCase): diff --git a/test/test_code.py b/test/test_code.py index 6a1a9d722..84fa199fa 100644 --- a/test/test_code.py +++ b/test/test_code.py @@ -18,7 +18,7 @@ import unittest import sys sys.path[0:0] = [""] -from pymongo.code import Code +from bson.code import Code class TestCode(unittest.TestCase): diff --git a/test/test_collection.py b/test/test_collection.py index c8e2ce48c..cd7c3f264 100644 --- a/test/test_collection.py +++ b/test/test_collection.py @@ -27,9 +27,11 @@ from nose.plugins.skip import SkipTest sys.path[0:0] = [""] +from bson.binary import Binary +from bson.code import Code +from bson.objectid import ObjectId +from bson.son import SON from pymongo import ASCENDING, DESCENDING -from pymongo.binary import Binary -from pymongo.code import Code from pymongo.collection import Collection from pymongo.errors import (DuplicateKeyError, InvalidDocument, @@ -37,8 +39,6 @@ from pymongo.errors import (DuplicateKeyError, InvalidOperation, OperationFailure, TimeoutError) -from pymongo.objectid import ObjectId -from pymongo.son import SON from test.test_connection import get_connection from test import (qcheck, version) diff --git a/test/test_cursor.py b/test/test_cursor.py index da5b7b096..fa6b92de1 100644 --- a/test/test_cursor.py +++ b/test/test_cursor.py @@ -22,11 +22,13 @@ sys.path[0:0] = [""] from nose.plugins.skip import SkipTest -from pymongo.errors import InvalidOperation, OperationFailure +from bson.code import Code +from pymongo import (ASCENDING, + DESCENDING) from pymongo.cursor import Cursor from pymongo.database import Database -from pymongo.code import Code -from pymongo import ASCENDING, DESCENDING +from pymongo.errors import (InvalidOperation, + OperationFailure) from test_connection import get_connection import version diff --git a/test/test_database.py b/test/test_database.py index ddb6a0c6f..1d59b147c 100644 --- a/test/test_database.py +++ b/test/test_database.py @@ -20,24 +20,25 @@ import sys sys.path[0:0] = [""] import unittest +from bson.code import Code +from bson.dbref import DBRef +from bson.errors import InvalidName +from bson.objectid import ObjectId +from bson.son import SON from pymongo import (ALL, ASCENDING, DESCENDING, helpers, OFF, SLOW_ONLY) -from pymongo.code import Code from pymongo.collection import Collection from pymongo.connection import Connection from pymongo.database import Database -from pymongo.dbref import DBRef from pymongo.errors import (CollectionInvalid, - InvalidName, InvalidOperation, OperationFailure) -from pymongo.objectid import ObjectId -from pymongo.son import SON -from pymongo.son_manipulator import AutoReference, NamespaceInjector +from pymongo.son_manipulator import (AutoReference, + NamespaceInjector) from test import version from test.test_connection import get_connection diff --git a/test/test_dbref.py b/test/test_dbref.py index 7b6723e51..f7e523d27 100644 --- a/test/test_dbref.py +++ b/test/test_dbref.py @@ -18,8 +18,8 @@ import unittest import sys sys.path[0:0] = [""] -from pymongo.objectid import ObjectId -from pymongo.dbref import DBRef +from bson.objectid import ObjectId +from bson.dbref import DBRef class TestDBRef(unittest.TestCase): diff --git a/test/test_grid_file.py b/test/test_grid_file.py index 55390ec4a..fba9a98bf 100644 --- a/test/test_grid_file.py +++ b/test/test_grid_file.py @@ -27,6 +27,7 @@ import sys import unittest sys.path[0:0] = [""] +from bson.objectid import ObjectId from gridfs.grid_file import (_SEEK_CUR, _SEEK_END, GridIn, @@ -34,7 +35,6 @@ from gridfs.grid_file import (_SEEK_CUR, GridOut) from gridfs.errors import (NoFile, UnsupportedAPI) -from pymongo.objectid import ObjectId from test_connection import get_connection import qcheck diff --git a/test/test_json_util.py b/test/test_json_util.py index a59a5a9bc..2aeaf8b1a 100644 --- a/test/test_json_util.py +++ b/test/test_json_util.py @@ -36,13 +36,13 @@ from nose.plugins.skip import SkipTest sys.path[0:0] = [""] +from bson.objectid import ObjectId +from bson.dbref import DBRef +from bson.min_key import MinKey +from bson.max_key import MaxKey +from bson.timestamp import Timestamp +from bson.tz_util import utc from pymongo.json_util import default, object_hook -from pymongo.objectid import ObjectId -from pymongo.dbref import DBRef -from pymongo.min_key import MinKey -from pymongo.max_key import MaxKey -from pymongo.timestamp import Timestamp -from pymongo.tz_util import utc class TestJsonUtil(unittest.TestCase): diff --git a/test/test_objectid.py b/test/test_objectid.py index b746711f9..ffca43f53 100644 --- a/test/test_objectid.py +++ b/test/test_objectid.py @@ -23,10 +23,10 @@ sys.path[0:0] = [""] from nose.plugins.skip import SkipTest -from pymongo.errors import InvalidId -from pymongo.objectid import ObjectId -from pymongo.tz_util import (FixedOffset, - utc) +from bson.errors import InvalidId +from bson.objectid import ObjectId +from bson.tz_util import (FixedOffset, + utc) def oid(x): return ObjectId() diff --git a/test/test_son_manipulator.py b/test/test_son_manipulator.py index ddbf3b46f..889e2515b 100644 --- a/test/test_son_manipulator.py +++ b/test/test_son_manipulator.py @@ -19,13 +19,15 @@ import unittest import sys sys.path[0:0] = [""] -import qcheck -from pymongo.objectid import ObjectId -from pymongo.son import SON -from pymongo.son_manipulator import SONManipulator, ObjectIdInjector -from pymongo.son_manipulator import NamespaceInjector, ObjectIdShuffler +from bson.objectid import ObjectId +from bson.son import SON from pymongo.database import Database +from pymongo.son_manipulator import (NamespaceInjector, + ObjectIdInjector, + ObjectIdShuffler, + SONManipulator) from test_connection import get_connection +import qcheck class TestSONManipulator(unittest.TestCase):