diff --git a/gridfs/__init__.py b/gridfs/__init__.py index f8a66ea2f..550f93bcf 100644 --- a/gridfs/__init__.py +++ b/gridfs/__init__.py @@ -25,10 +25,11 @@ from collections import Mapping from gridfs.errors import NoFile from gridfs.grid_file import (GridIn, GridOut, - GridOutCursor) + GridOutCursor, + DEFAULT_CHUNK_SIZE) from pymongo import (ASCENDING, DESCENDING) -from pymongo.common import UNAUTHORIZED_CODES +from pymongo.common import UNAUTHORIZED_CODES, validate_string from pymongo.database import Database from pymongo.errors import ConfigurationError, OperationFailure @@ -46,6 +47,9 @@ class GridFS(object): - `database`: database to use - `collection` (optional): root collection to use + .. versionchanged:: 3.1 + Indexes are only ensured on the first write to the DB. + .. versionchanged:: 3.0 `database` must use an acknowledged :attr:`~pymongo.database.Database.write_concern` @@ -64,29 +68,6 @@ class GridFS(object): self.__files = self.__collection.files self.__chunks = self.__collection.chunks - def __is_secondary(self): - return not self.__database.client._is_writable() - - def __create_index(self, coll, key, **kwargs): - try: - if not self.__is_secondary(): - coll.create_index(key, **kwargs) - except OperationFailure as exc: - if not (exc.code in UNAUTHORIZED_CODES - or "authorized" in str(exc)): - raise exc - - - def __ensure_index_files_id(self): - self.__create_index(self.__chunks, - [("files_id", ASCENDING), ("n", ASCENDING)], - unique=True) - - def __ensure_index_filename(self): - self.__create_index(self.__files, - [("filename", ASCENDING), - ("uploadDate", DESCENDING)]) - def new_file(self, **kwargs): """Create a new file in GridFS. @@ -180,17 +161,15 @@ class GridFS(object): Raises :class:`~gridfs.errors.NoFile` if no such version of that file exists. - An index on ``{filename: 1, uploadDate: -1}`` will - automatically be created when this method is called the first - time. - :Parameters: - `filename`: ``"filename"`` of the file to get, or `None` - `version` (optional): version of the file to get (defaults to -1, the most recent version uploaded) - `**kwargs` (optional): find files by custom metadata. + + .. versionchanged:: 3.1 + ``get_version`` no longer ensures indexes. """ - self.__ensure_index_filename() query = kwargs if filename is not None: query["filename"] = filename @@ -237,8 +216,10 @@ class GridFS(object): :Parameters: - `file_id`: ``"_id"`` of the file to delete + + .. versionchanged:: 3.1 + ``delete`` no longer ensures indexes. """ - self.__ensure_index_files_id() self.__files.delete_one({"_id": file_id}) self.__chunks.delete_many({"files_id": file_id}) @@ -246,15 +227,9 @@ class GridFS(object): """List the names of all files stored in this instance of :class:`GridFS`. - An index on ``{filename: 1, uploadDate: -1}`` will - automatically be created when this method is called the first - time. - - .. versionchanged:: 2.7 - ``list`` ensures an index, the same as ``get_version``. + .. versionchanged:: 3.1 + ``list`` no longer ensures indexes. """ - self.__ensure_index_filename() - # With an index, distinct includes documents with no filename # as None. return [ @@ -381,3 +356,364 @@ class GridFS(object): if kwargs: return self.__files.find_one(kwargs, ["_id"]) is not None return self.__files.find_one(document_or_id, ["_id"]) is not None + + +class GridFSBucket(object): + """An instance of GridFS on top of a single Database.""" + + def __init__(self, db, bucket_name="fs", + chunk_size_bytes=DEFAULT_CHUNK_SIZE, write_concern=None, + read_preference=None): + """Create a new instance of :class:`GridFSBucket`. + + Raises :exc:`TypeError` if `database` is not an instance of + :class:`~pymongo.database.Database`. + + Raises :exc:`~pymongo.errors.ConfigurationError` if `write_concern` + is not acknowledged. + + :Parameters: + - `database`: database to use. + - `bucket_name` (optional): The name of the bucket. Defaults to 'fs'. + - `chunk_size_bytes` (optional): The chunk size in bytes. Defaults + to 255KB. + - `write_concern` (optional): The + :class:`~pymongo.write_concern.WriteConcern` to use. If ``None`` + (the default) db.write_concern is used. + - `read_preference` (optional): The read preference to use. If + ``None`` (the default) db.read_preference is used. + + .. versionadded:: 3.1 + + .. mongodoc:: gridfs + """ + if not isinstance(db, Database): + raise TypeError("database must be an instance of Database") + + wtc = write_concern if write_concern is not None else db.write_concern + if not wtc.acknowledged: + raise ConfigurationError('write concern must be acknowledged') + + self._db = db + self._bucket_name = bucket_name + self._collection = db[bucket_name] + + self._chunks = self._collection.chunks.with_options( + write_concern=write_concern, + read_preference=read_preference) + + self._files = self._collection.files.with_options( + write_concern=write_concern, + read_preference=read_preference) + + self._chunk_size_bytes = chunk_size_bytes + + def open_upload_stream(self, filename, chunk_size_bytes=None, + metadata=None): + """Opens a Stream that the application can write the contents of the + file to. + + The user must specify the filename, and can choose to add any + additional information in the metadata field of the file document or + modify the chunk size. + For example:: + + my_db = MongoClient().test + fs = GridFSBucket(my_db) + grid_in, file_id = fs.open_upload_stream( + "test_file", chunk_size_bytes=4, + metadata={"contentType": "text/plain"}) + grid_in.write("data I want to store!") + grid_in.close() # uploaded on close + + Returns an instance of :class:`~gridfs.grid_file.GridIn` and the _id + of the file to upload. + + Raises :exc:`~gridfs.errors.NoFile` if no such version of + that file exists. + Raises :exc:`~ValueError` if `filename` is not a string. + + :Parameters: + - `filename`: The name of the file to upload. + - `chunk_size_bytes` (options): The number of bytes per chunk of this + file. Defaults to the chunk_size_bytes in :class:`GridFSBucket`. + - `metadata` (optional): User data for the 'metadata' field of the + files collection document. If not provided the metadata field will + be omitted from the files collection document. + """ + validate_string("filename", filename) + + opts = {"filename": filename, + "chunk_size": (chunk_size_bytes if chunk_size_bytes + is not None else self._chunk_size_bytes)} + if metadata is not None: + opts["metadata"] = metadata + + gin = GridIn(self._collection, **opts) + return gin, gin._id + + def upload_from_stream(self, filename, source, chunk_size_bytes=None, + metadata=None): + """Uploads a user file to a GridFS bucket. + + Reads the contents of the user file from `source` and uploads + it to the file `filename`. Source can be a string or file-like object. + For example:: + + my_db = MongoClient().test + fs = GridFSBucket(my_db) + file_id = fs.upload_from_stream( + "test_file", + "data I want to store!", + chunk_size_bytes=4, + metadata={"contentType": "text/plain"}) + + Returns the _id of the uploaded file. + + Raises :exc:`~gridfs.errors.NoFile` if no such version of + that file exists. + Raises :exc:`~ValueError` if `filename` is not a string. + + :Parameters: + - `filename`: The name of the file to upload. + - `source`: The source stream of the content to be uploaded. Must be + a file-like object that implements :meth:`read` or a string. + - `chunk_size_bytes` (options): The number of bytes per chunk of this + file. Defaults to the chunk_size_bytes of :class:`GridFSBucket`. + - `metadata` (optional): User data for the 'metadata' field of the + files collection document. If not provided the metadata field will + be omitted from the files collection document. + """ + gin, _id = self.open_upload_stream(filename, chunk_size_bytes, + metadata) + try: + gin.write(source) + finally: + gin.close() + + return _id + + def open_download_stream(self, file_id): + """Opens a Stream from which the application can read the contents of + the stored file specified by file_id. + + For example:: + + my_db = MongoClient().test + fs = GridFSBucket(my_db) + # get _id of file to read. + file_id = fs.upload_from_stream("test_file", "data I want to store!") + grid_out = fs.open_download_stream(file_id) + contents = grid_out.read() + + Returns an instance of :class:`~gridfs.grid_file.GridOut`. + + Raises :exc:`~gridfs.errors.NoFile` if no file with file_id exists. + + :Parameters: + -`file_id`: The _id of the file to be downloaded. + """ + gout = GridOut(self._collection, file_id) + + # Raise NoFile now, instead of on first attribute access. + gout._ensure_file() + return gout + + def download_to_stream(self, file_id, destination): + """Downloads the contents of the stored file specified by file_id and + writes the contents to `destination`. + + For example:: + + my_db = MongoClient().test + fs = GridFSBucket(my_db) + # Get _id of file to read + file_id = fs.upload_from_stream("test_file", "data I want to store!") + # Get file to write to + file = open('myfile','rw') + fs.download_to_stream(file_id, file) + contents = file.read() + + Raises :exc:`~gridfs.errors.NoFile` if no file with file_id exists. + + :Parameters: + -`file_id`: The _id of the file to be downloaded. + -`destination`: a file-like object implementing :meth:`write`. + """ + gout = self.open_download_stream(file_id) + destination.write(gout) + + def delete(self, file_id): + """Given an file_id, delete this stored file's files collection document + and associated chunks from a GridFS bucket. + + For example:: + + my_db = MongoClient().test + fs = GridFSBucket(my_db) + # Get _id of file to delete + file_id = fs.upload_from_stream("test_file", "data I want to store!") + fs.delete(file_id) + + Raises :exc:`~gridfs.errors.NoFile` if no file with file_id exists. + + :Parameters: + -`file_id`: The _id of the file to be deleted. + """ + res = self._files.delete_one({"_id": file_id}) + self._chunks.delete_many({"files_id": file_id}) + if not res.deleted_count: + raise NoFile( + "no file could be deleted because none matched %s" % file_id) + + def find(self, *args, **kwargs): + """Find and return the files collection documents that match ``filter`` + + Returns a cursor that iterates across files matching + arbitrary queries on the files collection. Can be combined + with other modifiers for additional control. + + For example:: + + for grid_data in fs.find({"filename": "lisa.txt"}, + no_cursor_timeout=True): + data = grid_data.read() + + would iterate through all versions of "lisa.txt" stored in GridFS. + Note that setting no_cursor_timeout to True may be important to + prevent the cursor from timing out during long multi-file processing + work. + + As another example, the call:: + + most_recent_three = fs.find().sort("uploadDate", -1).limit(3) + + would return a cursor to the three most recently uploaded files + in GridFS. + + Follows a similar interface to + :meth:`~pymongo.collection.Collection.find` + in :class:`~pymongo.collection.Collection`. + + :Parameters: + - `filter`: Search query. + - `batch_size` (optional): The number of documents to return per + batch. + - `limit` (optional): The maximum number of documents to return. + - `no_cursor_timeout` (optional): The server normally times out idle + cursors after an inactivity period (10 minutes) to prevent excess + memory use. Set this option to True prevent that. + - `skip` (optional): The number of documents to skip before + returning. + - `sort` (optional): The order by which to sort results. Defaults to + None. + """ + return GridOutCursor(self._collection, *args, **kwargs) + + def open_download_stream_by_name(self, filename, revision=-1): + """Opens a Stream from which the application can read the contents of + `filename` and optional `revision`. + + For example:: + + my_db = MongoClient().test + fs = GridFSBucket(my_db) + grid_out = fs.open_download_stream_by_name("test_file") + contents = grid_out.read() + + Returns an instance of :class:`~gridfs.grid_file.GridOut`. + + Raises :exc:`~gridfs.errors.NoFile` if no such version of + that file exists. + + Raises :exc:`~ValueError` filename is not a string. + + :Parameters: + - `filename`: The name of the file to read from. + - `revision` (optional): Which revision (documents with the same + filename and different uploadDate) of the file to retrieve. + Defaults to -1 (the most recent revision). + + :Note: Revision numbers are defined as follows: + 0 = the original stored file + 1 = the first revision + 2 = the second revision + etc... + -2 = the second most recent revision + -1 = the most recent revision + """ + validate_string("filename", filename) + + query = {"filename": filename} + + cursor = self._files.find(query) + if revision < 0: + skip = abs(revision) - 1 + cursor.limit(-1).skip(skip).sort("uploadDate", DESCENDING) + else: + cursor.limit(-1).skip(revision).sort("uploadDate", ASCENDING) + try: + grid_file = next(cursor) + return GridOut(self._collection, file_document=grid_file) + except StopIteration: + raise NoFile( + "no version %d for filename %r" % (revision, filename)) + + def download_to_stream_by_name(self, filename, destination, revision=-1): + """Write the contents of `filename` (with optional `revision`) to + `destination`. + + For example:: + + my_db = MongoClient().test + fs = GridFSBucket(my_db) + # Get file to write to + file = open('myfile','w') + fs.download_to_stream_by_name("test_file", file) + + Raises :exc:`~gridfs.errors.NoFile` if no such version of + that file exists. + + Raises :exc:`~ValueError` if `filename` is not a string. + + :Parameters: + - `filename`: The name of the file to read from. + - `destination`: A file-like object that implements :meth:`write`. + - `revision` (optional): Which revision (documents with the same + filename and different uploadDate) of the file to retrieve. + Defaults to -1 (the most recent revision). + + :Note: Revision numbers are defined as follows: + 0 = the original stored file + 1 = the first revision + 2 = the second revision + etc... + -2 = the second most recent revision + -1 = the most recent revision + """ + gout = self.open_download_stream_by_name(filename, revision) + + destination.write(gout) + + def rename(self, file_id, new_filename): + """Renames the stored file with the specified file_id. + + For example:: + + my_db = MongoClient().test + fs = GridFSBucket(my_db) + # Get _id of file to rename + file_id = fs.upload_from_stream("test_file", "data I want to store!") + fs.rename(file_id, "new_test_name") + + Raises :exc:`~gridfs.errors.NoFile` if no file with file_id exists. + + :Parameters: + - `file_id`: The _id of the file to be renamed. + - `new_filename`: The new name of the file. + """ + result = self._files.update_one({"_id": file_id}, + {"$set": {"filename": new_filename}}) + if not result.matched_count: + raise NoFile("no files could be renamed %r because none " + "matched file_id %i" % (new_filename, file_id)) diff --git a/gridfs/grid_file.py b/gridfs/grid_file.py index d43b259cd..c7e6c39d9 100644 --- a/gridfs/grid_file.py +++ b/gridfs/grid_file.py @@ -13,25 +13,22 @@ # limitations under the License. """Tools for representing files stored in GridFS.""" - import datetime import math import os +from hashlib import md5 + from bson.binary import Binary from bson.objectid import ObjectId from bson.py3compat import text_type, StringIO -from gridfs.errors import (CorruptGridFile, - FileExists, - NoFile) +from gridfs.errors import CorruptGridFile, FileExists, NoFile from pymongo import ASCENDING from pymongo.collection import Collection -from pymongo.common import UNAUTHORIZED_CODES from pymongo.cursor import Cursor from pymongo.errors import (ConfigurationError, DuplicateKeyError, OperationFailure) -from pymongo.read_preferences import ReadPreference try: _SEEK_SET = os.SEEK_SET @@ -50,6 +47,9 @@ NEWLN = b"\n" # Slightly under a power of 2, to work well with server's record allocations. DEFAULT_CHUNK_SIZE = 255 * 1024 +_C_INDEX = [("files_id", ASCENDING), ("n", ASCENDING)] +_F_INDEX = [("filename", ASCENDING), ("uploadDate", ASCENDING)] + def _grid_in_property(field_name, docstring, read_only=False, closed_only=False): @@ -155,6 +155,7 @@ class GridIn(object): if "chunk_size" in kwargs: kwargs["chunkSize"] = kwargs.pop("chunk_size") + kwargs['md5'] = md5() # Defaults kwargs["_id"] = kwargs.get("_id", ObjectId()) kwargs["chunkSize"] = kwargs.get("chunkSize", DEFAULT_CHUNK_SIZE) @@ -167,18 +168,30 @@ class GridIn(object): object.__setattr__(self, "_closed", False) object.__setattr__(self, "_ensured_index", False) - def _ensure_index(self): - if not object.__getattribute__(self, "_ensured_index"): + def __create_index(self, collection, index, unique): + doc = collection.find_one(projection={"_id": 1}) + if doc is None: try: - self._coll.chunks.create_index( - [("files_id", ASCENDING), ("n", ASCENDING)], - unique=True) - except OperationFailure as exc: - if not (exc.code in UNAUTHORIZED_CODES - or "authorized" in str(exc)): - raise exc + indexes = list(collection.list_indexes()) + except OperationFailure: + indexes = [] + if index not in indexes: + collection.create_index(index, unique=unique) + + def __ensure_indexes(self): + if not object.__getattribute__(self, "_ensured_index"): + self.__create_index(self._coll.files, _F_INDEX, False) + self.__create_index(self._coll.chunks, _C_INDEX, True) object.__setattr__(self, "_ensured_index", True) + def abort(self): + """Remove all chunks/files that may have been uploaded and close. + """ + self._coll.chunks.delete_many({"files_id": self._file['_id']}) + self._coll.files.delete_one({"_id": self._file['_id']}) + object.__setattr__(self, "_closed", True) + + @property def closed(self): """Is this file closed? @@ -225,7 +238,8 @@ class GridIn(object): """ # Ensure the index, even if there's nothing to write, so # the filemd5 command always succeeds. - self._ensure_index() + self.__ensure_indexes() + self._file['md5'].update(data) if not data: return @@ -255,12 +269,7 @@ class GridIn(object): try: self.__flush_buffer() - db = self._coll.database - md5 = db.command( - "filemd5", self._id, root=self._coll.name, - read_preference=ReadPreference.PRIMARY)["md5"] - - self._file["md5"] = md5 + self._file['md5'] = self._file["md5"].hexdigest() self._file["length"] = self._position self._file["uploadDate"] = datetime.datetime.utcnow() @@ -326,10 +335,14 @@ class GridIn(object): # Make sure to flush only when _buffer is complete space = self.chunk_size - self._buffer.tell() if space: - to_write = read(space) + try: + to_write = read(space) + except: + self.abort() + raise self._buffer.write(to_write) if len(to_write) < space: - return # EOF or incomplete + return # EOF or incomplete self.__flush_buffer() to_write = read(self.chunk_size) while to_write and len(to_write) == self.chunk_size: @@ -475,6 +488,16 @@ class GridOut(object): received += len(chunk_data) data.write(chunk_data) + # Detect extra chunks. + max_chunk_n = math.ceil(self.length / float(self.chunk_size)) + chunk = self.__chunks.find_one({"files_id": self._id, + "n": {"$gte": max_chunk_n}}) + # According to spec, ignore extra chunks if they are empty. + if chunk is not None and len(chunk['data']): + raise CorruptGridFile( + "Extra chunk found: expected %i chunks but found " + "chunk with n=%i" % (max_chunk_n, chunk['n'])) + self.__position -= received - size # Return 'size' bytes and store the rest. @@ -605,7 +628,7 @@ class GridOutCursor(Cursor): of an arbitrary query against the GridFS files collection. """ def __init__(self, collection, filter=None, skip=0, limit=0, - no_cursor_timeout=False, sort=None): + no_cursor_timeout=False, sort=None, batch_size=0): """Create a new cursor, similar to the normal :class:`~pymongo.cursor.Cursor`. @@ -621,7 +644,8 @@ class GridOutCursor(Cursor): super(GridOutCursor, self).__init__( collection.files, filter, skip=skip, limit=limit, - no_cursor_timeout=no_cursor_timeout, sort=sort) + no_cursor_timeout=no_cursor_timeout, sort=sort, + batch_size=batch_size) def next(self): """Get next GridOut object from cursor. diff --git a/test/gridfs/delete.json b/test/gridfs/delete.json new file mode 100644 index 000000000..cb92bc3d5 --- /dev/null +++ b/test/gridfs/delete.json @@ -0,0 +1,412 @@ +{ + "data": { + "files": [ + { + "_id": { + "$oid": "000000000000000000000001" + }, + "length": 0, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "d41d8cd98f00b204e9800998ecf8427e", + "filename": "length-0", + "contentType": "application/octet-stream", + "aliases": [ + + ], + "metadata": { + } + }, + { + "_id": { + "$oid": "000000000000000000000002" + }, + "length": 0, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "d41d8cd98f00b204e9800998ecf8427e", + "filename": "length-0-with-empty-chunk", + "contentType": "application/octet-stream", + "aliases": [ + + ], + "metadata": { + } + }, + { + "_id": { + "$oid": "000000000000000000000003" + }, + "length": 2, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "c700ed4fdb1d27055aa3faa2c2432283", + "filename": "length-2", + "contentType": "application/octet-stream", + "aliases": [ + + ], + "metadata": { + } + }, + { + "_id": { + "$oid": "000000000000000000000004" + }, + "length": 8, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "dd254cdc958e53abaa67da9f797125f5", + "filename": "length-8", + "contentType": "application/octet-stream", + "aliases": [ + + ], + "metadata": { + } + }, + { + "_id": { + "$oid": "000000000000000000000005" + }, + "length": 8, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "dd254cdc958e53abaa67da9f797125f5", + "filename": "length-8-with-empty-chunk", + "contentType": "application/octet-stream", + "aliases": [ + + ], + "metadata": { + } + } + ], + "chunks": [ + { + "_id": { + "$oid": "000000000000000000000001" + }, + "files_id": { + "$oid": "000000000000000000000002" + }, + "n": 0, + "data": { + "$hex": "" + } + }, + { + "_id": { + "$oid": "000000000000000000000002" + }, + "files_id": { + "$oid": "000000000000000000000003" + }, + "n": 0, + "data": { + "$hex": "1122" + } + }, + { + "_id": { + "$oid": "000000000000000000000003" + }, + "files_id": { + "$oid": "000000000000000000000004" + }, + "n": 0, + "data": { + "$hex": "11223344" + } + }, + { + "_id": { + "$oid": "000000000000000000000004" + }, + "files_id": { + "$oid": "000000000000000000000004" + }, + "n": 1, + "data": { + "$hex": "55667788" + } + }, + { + "_id": { + "$oid": "000000000000000000000005" + }, + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 0, + "data": { + "$hex": "11223344" + } + }, + { + "_id": { + "$oid": "000000000000000000000006" + }, + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 1, + "data": { + "$hex": "55667788" + } + }, + { + "_id": { + "$oid": "000000000000000000000007" + }, + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 2, + "data": { + "$hex": "" + } + } + ] + }, + "tests": [ + { + "description": "Delete when length is 0", + "act": { + "operation": "delete", + "arguments": { + "id": { + "$oid": "000000000000000000000001" + } + } + }, + "assert": { + "result": "void", + "data": [ + { + "delete": "expected.files", + "deletes": [ + { + "q": { + "_id": { + "$oid": "000000000000000000000001" + } + }, + "limit": 1 + } + ] + } + ] + } + }, + { + "description": "Delete when length is 0 and there is one extra empty chunk", + "act": { + "operation": "delete", + "arguments": { + "id": { + "$oid": "000000000000000000000002" + } + } + }, + "assert": { + "result": "void", + "data": [ + { + "delete": "expected.files", + "deletes": [ + { + "q": { + "_id": { + "$oid": "000000000000000000000002" + } + }, + "limit": 1 + } + ] + }, + { + "delete": "expected.chunks", + "deletes": [ + { + "q": { + "files_id": { + "$oid": "000000000000000000000002" + } + }, + "limit": 0 + } + ] + } + ] + } + }, + { + "description": "Delete when length is 8", + "act": { + "operation": "delete", + "arguments": { + "id": { + "$oid": "000000000000000000000004" + } + } + }, + "assert": { + "result": "void", + "data": [ + { + "delete": "expected.files", + "deletes": [ + { + "q": { + "_id": { + "$oid": "000000000000000000000004" + } + }, + "limit": 1 + } + ] + }, + { + "delete": "expected.chunks", + "deletes": [ + { + "q": { + "files_id": { + "$oid": "000000000000000000000004" + } + }, + "limit": 0 + } + ] + } + ] + } + }, + { + "description": "Delete when length is 8 and there is one extra empty chunk", + "act": { + "operation": "delete", + "arguments": { + "id": { + "$oid": "000000000000000000000005" + } + } + }, + "assert": { + "result": "void", + "data": [ + { + "delete": "expected.files", + "deletes": [ + { + "q": { + "_id": { + "$oid": "000000000000000000000005" + } + }, + "limit": 1 + } + ] + }, + { + "delete": "expected.chunks", + "deletes": [ + { + "q": { + "files_id": { + "$oid": "000000000000000000000005" + } + }, + "limit": 0 + } + ] + } + ] + } + }, + { + "description": "Delete when files entry does not exist", + "act": { + "operation": "delete", + "arguments": { + "id": { + "$oid": "000000000000000000000000" + } + } + }, + "assert": { + "error": "FileNotFound" + } + }, + { + "description": "Delete when files entry does not exist and there are orphaned chunks", + "arrange": { + "data": [ + { + "delete": "fs.files", + "deletes": [ + { + "q": { + "_id": { + "$oid": "000000000000000000000005" + } + }, + "limit": 1 + } + ] + } + ] + }, + "act": { + "operation": "delete", + "arguments": { + "id": { + "$oid": "000000000000000000000005" + } + } + }, + "assert": { + "error": "FileNotFound", + "data": [ + { + "delete": "expected.files", + "deletes": [ + { + "q": { + "_id": { + "$oid": "000000000000000000000005" + } + }, + "limit": 1 + } + ] + }, + { + "delete": "expected.chunks", + "deletes": [ + { + "q": { + "files_id": { + "$oid": "000000000000000000000005" + } + }, + "limit": 0 + } + ] + } + ] + } + } + ] +} \ No newline at end of file diff --git a/test/gridfs/download.json b/test/gridfs/download.json new file mode 100644 index 000000000..3e50912a9 --- /dev/null +++ b/test/gridfs/download.json @@ -0,0 +1,564 @@ +{ + "data": { + "files": [ + { + "_id": { + "$oid": "000000000000000000000001" + }, + "length": 0, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "d41d8cd98f00b204e9800998ecf8427e", + "filename": "length-0", + "contentType": "application/octet-stream", + "aliases": [ + + ], + "metadata": { + } + }, + { + "_id": { + "$oid": "000000000000000000000002" + }, + "length": 0, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "d41d8cd98f00b204e9800998ecf8427e", + "filename": "length-0-with-empty-chunk", + "contentType": "application/octet-stream", + "aliases": [ + + ], + "metadata": { + } + }, + { + "_id": { + "$oid": "000000000000000000000003" + }, + "length": 2, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "c700ed4fdb1d27055aa3faa2c2432283", + "filename": "length-2", + "contentType": "application/octet-stream", + "aliases": [ + + ], + "metadata": { + } + }, + { + "_id": { + "$oid": "000000000000000000000004" + }, + "length": 8, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "dd254cdc958e53abaa67da9f797125f5", + "filename": "length-8", + "contentType": "application/octet-stream", + "aliases": [ + + ], + "metadata": { + } + }, + { + "_id": { + "$oid": "000000000000000000000005" + }, + "length": 10, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "57d83cd477bfb1ccd975ab33d827a92b", + "filename": "length-10", + "contentType": "application/octet-stream", + "aliases": [ + + ], + "metadata": { + } + }, + { + "_id": { + "$oid": "000000000000000000000006" + }, + "length": 12, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "6289ac1db331d1c7677a4b7e123178f9", + "filename": "length-12-with-empty-chunk", + "contentType": "application/octet-stream", + "aliases": [ + + ], + "metadata": { + } + } + ], + "chunks": [ + { + "_id": { + "$oid": "000000000000000000000001" + }, + "files_id": { + "$oid": "000000000000000000000002" + }, + "n": 0, + "data": { + "$hex": "" + } + }, + { + "_id": { + "$oid": "000000000000000000000002" + }, + "files_id": { + "$oid": "000000000000000000000003" + }, + "n": 0, + "data": { + "$hex": "1122" + } + }, + { + "_id": { + "$oid": "000000000000000000000003" + }, + "files_id": { + "$oid": "000000000000000000000004" + }, + "n": 0, + "data": { + "$hex": "11223344" + } + }, + { + "_id": { + "$oid": "000000000000000000000004" + }, + "files_id": { + "$oid": "000000000000000000000004" + }, + "n": 1, + "data": { + "$hex": "55667788" + } + }, + { + "_id": { + "$oid": "000000000000000000000005" + }, + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 0, + "data": { + "$hex": "11223344" + } + }, + { + "_id": { + "$oid": "000000000000000000000006" + }, + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 1, + "data": { + "$hex": "55667788" + } + }, + { + "_id": { + "$oid": "000000000000000000000007" + }, + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 2, + "data": { + "$hex": "99aa" + } + }, + { + "_id": { + "$oid": "000000000000000000000008" + }, + "files_id": { + "$oid": "000000000000000000000006" + }, + "n": 0, + "data": { + "$hex": "11223344" + } + }, + { + "_id": { + "$oid": "000000000000000000000009" + }, + "files_id": { + "$oid": "000000000000000000000006" + }, + "n": 1, + "data": { + "$hex": "55667788" + } + }, + { + "_id": { + "$oid": "000000000000000000000010" + }, + "files_id": { + "$oid": "000000000000000000000006" + }, + "n": 2, + "data": { + "$hex": "99aabbcc" + } + }, + { + "_id": { + "$oid": "000000000000000000000011" + }, + "files_id": { + "$oid": "000000000000000000000006" + }, + "n": 3, + "data": { + "$hex": "" + } + } + ] + }, + "tests": [ + { + "description": "Download when length is zero", + "act": { + "operation": "download", + "arguments": { + "id": { + "$oid": "000000000000000000000001" + }, + "options": { + } + } + }, + "assert": { + "result": { + "$hex": "" + } + } + }, + { + "description": "Download when length is zero and there is one empty chunk", + "act": { + "operation": "download", + "arguments": { + "id": { + "$oid": "000000000000000000000002" + }, + "options": { + } + } + }, + "assert": { + "result": { + "$hex": "" + } + } + }, + { + "description": "Download when there is one chunk", + "act": { + "operation": "download", + "arguments": { + "id": { + "$oid": "000000000000000000000003" + }, + "options": { + } + } + }, + "assert": { + "result": { + "$hex": "1122" + } + } + }, + { + "description": "Download when there are two chunks", + "act": { + "operation": "download", + "arguments": { + "id": { + "$oid": "000000000000000000000004" + }, + "options": { + } + } + }, + "assert": { + "result": { + "$hex": "1122334455667788" + } + } + }, + { + "description": "Download when there are three chunks", + "act": { + "operation": "download", + "arguments": { + "id": { + "$oid": "000000000000000000000005" + }, + "options": { + } + } + }, + "assert": { + "result": { + "$hex": "112233445566778899aa" + } + } + }, + { + "description": "Download when there are three chunks and one extra empty chunk at the end", + "act": { + "operation": "download", + "arguments": { + "id": { + "$oid": "000000000000000000000006" + }, + "options": { + } + } + }, + "assert": { + "result": { + "$hex": "112233445566778899aabbcc" + } + } + }, + { + "description": "Download when files entry does not exist", + "act": { + "operation": "download", + "arguments": { + "id": { + "$oid": "000000000000000000000000" + }, + "options": { + } + } + }, + "assert": { + "error": "FileNotFound" + } + }, + { + "description": "Download when an intermediate chunk is missing", + "arrange": { + "data": [ + { + "delete": "fs.chunks", + "deletes": [ + { + "q": { + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 1 + }, + "limit": 1 + } + ] + } + ] + }, + "act": { + "operation": "download", + "arguments": { + "id": { + "$oid": "000000000000000000000005" + } + } + }, + "assert": { + "error": "ChunkIsMissing" + } + }, + { + "description": "Download when final chunk is missing", + "arrange": { + "data": [ + { + "delete": "fs.chunks", + "deletes": [ + { + "q": { + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 1 + }, + "limit": 1 + } + ] + } + ] + }, + "act": { + "operation": "download", + "arguments": { + "id": { + "$oid": "000000000000000000000005" + } + } + }, + "assert": { + "error": "ChunkIsMissing" + } + }, + { + "description": "Download when there is an extra chunk", + "arrange": { + "data": [ + { + "insert": "fs.chunks", + "documents": [ + { + "_id": { + "$oid": "000000000000000000000012" + }, + "files_id": { + "$oid": "000000000000000000000004" + }, + "n": 2, + "data": { + "$hex": "99" + } + } + ] + } + ] + }, + "act": { + "operation": "download", + "arguments": { + "id": { + "$oid": "000000000000000000000004" + } + } + }, + "assert": { + "error": "ExtraChunk" + } + }, + { + "description": "Download when an intermediate chunk is the wrong size", + "arrange": { + "data": [ + { + "update": "fs.chunks", + "updates": [ + { + "q": { + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 1 + }, + "u": { + "$set": { + "data": { + "$hex": "556677" + } + } + } + }, + { + "q": { + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 2 + }, + "u": { + "$set": { + "data": { + "$hex": "8899aa" + } + } + } + } + ] + } + ] + }, + "act": { + "operation": "download", + "arguments": { + "id": { + "$oid": "000000000000000000000005" + } + } + }, + "assert": { + "error": "ChunkIsWrongSize" + } + }, + { + "description": "Download when final chunk is the wrong size", + "arrange": { + "data": [ + { + "update": "fs.chunks", + "updates": [ + { + "q": { + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 2 + }, + "u": { + "$set": { + "data": { + "$hex": "99" + } + } + } + } + ] + } + ] + }, + "act": { + "operation": "download", + "arguments": { + "id": { + "$oid": "000000000000000000000005" + } + } + }, + "assert": { + "error": "ChunkIsWrongSize" + } + } + ] +} \ No newline at end of file diff --git a/test/gridfs/download_by_name.json b/test/gridfs/download_by_name.json new file mode 100644 index 000000000..7f4d46bb3 --- /dev/null +++ b/test/gridfs/download_by_name.json @@ -0,0 +1,255 @@ +{ + "data": { + "files": [ + { + "_id": { + "$oid": "000000000000000000000001" + }, + "length": 1, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "47ed733b8d10be225eceba344d533586", + "filename": "abc", + "contentType": "application/octet-stream", + "aliases": [ + + ], + "metadata": { + } + }, + { + "_id": { + "$oid": "000000000000000000000002" + }, + "length": 1, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-02T00:00:00.000Z" + }, + "md5": "b15835f133ff2e27c7cb28117bfae8f4", + "filename": "abc", + "contentType": "application/octet-stream", + "aliases": [ + + ], + "metadata": { + } + }, + { + "_id": { + "$oid": "000000000000000000000003" + }, + "length": 1, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-03T00:00:00.000Z" + }, + "md5": "eccbc87e4b5ce2fe28308fd9f2a7baf3", + "filename": "abc", + "contentType": "application/octet-stream", + "aliases": [ + + ], + "metadata": { + } + }, + { + "_id": { + "$oid": "000000000000000000000004" + }, + "length": 1, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-04T00:00:00.000Z" + }, + "md5": "f623e75af30e62bbd73d6df5b50bb7b5", + "filename": "abc", + "contentType": "application/octet-stream", + "aliases": [ + + ], + "metadata": { + } + }, + { + "_id": { + "$oid": "000000000000000000000005" + }, + "length": 1, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-05T00:00:00.000Z" + }, + "md5": "4c614360da93c0a041b22e537de151eb", + "filename": "abc", + "contentType": "application/octet-stream", + "aliases": [ + + ], + "metadata": { + } + } + ], + "chunks": [ + { + "_id": { + "$oid": "000000000000000000000001" + }, + "files_id": { + "$oid": "000000000000000000000001" + }, + "n": 0, + "data": { + "$hex": "11" + } + }, + { + "_id": { + "$oid": "000000000000000000000002" + }, + "files_id": { + "$oid": "000000000000000000000002" + }, + "n": 0, + "data": { + "$hex": "22" + } + }, + { + "_id": { + "$oid": "000000000000000000000003" + }, + "files_id": { + "$oid": "000000000000000000000003" + }, + "n": 0, + "data": { + "$hex": "33" + } + }, + { + "_id": { + "$oid": "000000000000000000000004" + }, + "files_id": { + "$oid": "000000000000000000000004" + }, + "n": 0, + "data": { + "$hex": "44" + } + }, + { + "_id": { + "$oid": "000000000000000000000005" + }, + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 0, + "data": { + "$hex": "55" + } + } + ] + }, + "tests": [ + { + "description": "Download_by_name when revision is 0", + "act": { + "operation": "download_by_name", + "arguments": { + "filename": "abc", + "options": { + "revision": 0 + } + } + }, + "assert": { + "result": { + "$hex": "11" + } + } + }, + { + "description": "Download_by_name when revision is 1", + "act": { + "operation": "download_by_name", + "arguments": { + "filename": "abc", + "options": { + "revision": 1 + } + } + }, + "assert": { + "result": { + "$hex": "22" + } + } + }, + { + "description": "Download_by_name when revision is -2", + "act": { + "operation": "download_by_name", + "arguments": { + "filename": "abc", + "options": { + "revision": -2 + } + } + }, + "assert": { + "result": { + "$hex": "44" + } + } + }, + { + "description": "Download_by_name when revision is -1", + "act": { + "operation": "download_by_name", + "arguments": { + "filename": "abc", + "options": { + "revision": -1 + } + } + }, + "assert": { + "result": { + "$hex": "55" + } + } + }, + { + "description": "Download_by_name when files entry does not exist", + "act": { + "operation": "download_by_name", + "arguments": { + "filename": "xyz" + } + }, + "assert": { + "error": "FileNotFound" + } + }, + { + "description": "Download_by_name when revision does not exist", + "act": { + "operation": "download_by_name", + "arguments": { + "filename": "abc", + "options": { + "revision": 999 + } + } + }, + "assert": { + "error": "RevisionNotFound" + } + } + ] +} \ No newline at end of file diff --git a/test/gridfs/upload.json b/test/gridfs/upload.json new file mode 100644 index 000000000..95dfd79e3 --- /dev/null +++ b/test/gridfs/upload.json @@ -0,0 +1,391 @@ +{ + "data": { + "files": [ + + ], + "chunks": [ + + ] + }, + "tests": [ + { + "description": "Upload when length is 0", + "act": { + "operation": "upload", + "arguments": { + "filename": "filename", + "source": { + "$hex": "" + }, + "options": { + "chunkSizeBytes": 4 + } + } + }, + "assert": { + "result": "&result", + "data": [ + { + "insert": "expected.files", + "documents": [ + { + "_id": "*result", + "length": 0, + "chunkSize": 4, + "uploadDate": "*actual", + "md5": "d41d8cd98f00b204e9800998ecf8427e", + "filename": "filename" + } + ] + } + ] + } + }, + { + "description": "Upload when length is 1", + "act": { + "operation": "upload", + "arguments": { + "filename": "filename", + "source": { + "$hex": "11" + }, + "options": { + "chunkSizeBytes": 4 + } + } + }, + "assert": { + "result": "&result", + "data": [ + { + "insert": "expected.files", + "documents": [ + { + "_id": "*result", + "length": 1, + "chunkSize": 4, + "uploadDate": "*actual", + "md5": "47ed733b8d10be225eceba344d533586", + "filename": "filename" + } + ] + }, + { + "insert": "expected.chunks", + "documents": [ + { + "_id": "*actual", + "files_id": "*result", + "n": 0, + "data": { + "$hex": "11" + } + } + ] + } + ] + } + }, + { + "description": "Upload when length is 3", + "act": { + "operation": "upload", + "arguments": { + "filename": "filename", + "source": { + "$hex": "112233" + }, + "options": { + "chunkSizeBytes": 4 + } + } + }, + "assert": { + "result": "&result", + "data": [ + { + "insert": "expected.files", + "documents": [ + { + "_id": "*result", + "length": 3, + "chunkSize": 4, + "uploadDate": "*actual", + "md5": "bafae3a174ab91fc70db7a6aa50f4f52", + "filename": "filename" + } + ] + }, + { + "insert": "expected.chunks", + "documents": [ + { + "_id": "*actual", + "files_id": "*result", + "n": 0, + "data": { + "$hex": "112233" + } + } + ] + } + ] + } + }, + { + "description": "Upload when length is 4", + "act": { + "operation": "upload", + "arguments": { + "filename": "filename", + "source": { + "$hex": "11223344" + }, + "options": { + "chunkSizeBytes": 4 + } + } + }, + "assert": { + "result": "&result", + "data": [ + { + "insert": "expected.files", + "documents": [ + { + "_id": "*result", + "length": 4, + "chunkSize": 4, + "uploadDate": "*actual", + "md5": "7e7c77cff5705d1f7574a25ef6662117", + "filename": "filename" + } + ] + }, + { + "insert": "expected.chunks", + "documents": [ + { + "_id": "*actual", + "files_id": "*result", + "n": 0, + "data": { + "$hex": "11223344" + } + } + ] + } + ] + } + }, + { + "description": "Upload when length is 5", + "act": { + "operation": "upload", + "arguments": { + "filename": "filename", + "source": { + "$hex": "1122334455" + }, + "options": { + "chunkSizeBytes": 4 + } + } + }, + "assert": { + "result": "&result", + "data": [ + { + "insert": "expected.files", + "documents": [ + { + "_id": "*result", + "length": 5, + "chunkSize": 4, + "uploadDate": "*actual", + "md5": "283d4fea5dded59cf837d3047328f5af", + "filename": "filename" + } + ] + }, + { + "insert": "expected.chunks", + "documents": [ + { + "_id": "*actual", + "files_id": "*result", + "n": 0, + "data": { + "$hex": "11223344" + } + }, + { + "_id": "*actual", + "files_id": "*result", + "n": 1, + "data": { + "$hex": "55" + } + } + ] + } + ] + } + }, + { + "description": "Upload when length is 8", + "act": { + "operation": "upload", + "arguments": { + "filename": "filename", + "source": { + "$hex": "1122334455667788" + }, + "options": { + "chunkSizeBytes": 4 + } + } + }, + "assert": { + "result": "&result", + "data": [ + { + "insert": "expected.files", + "documents": [ + { + "_id": "*result", + "length": 8, + "chunkSize": 4, + "uploadDate": "*actual", + "md5": "dd254cdc958e53abaa67da9f797125f5", + "filename": "filename" + } + ] + }, + { + "insert": "expected.chunks", + "documents": [ + { + "_id": "*actual", + "files_id": "*result", + "n": 0, + "data": { + "$hex": "11223344" + } + }, + { + "_id": "*actual", + "files_id": "*result", + "n": 1, + "data": { + "$hex": "55667788" + } + } + ] + } + ] + } + }, + { + "description": "Upload when contentType is provided", + "act": { + "operation": "upload", + "arguments": { + "filename": "filename", + "source": { + "$hex": "11" + }, + "options": { + "chunkSizeBytes": 4, + "contentType": "image/jpeg" + } + } + }, + "assert": { + "result": "&result", + "data": [ + { + "insert": "expected.files", + "documents": [ + { + "_id": "*result", + "length": 1, + "chunkSize": 4, + "uploadDate": "*actual", + "md5": "47ed733b8d10be225eceba344d533586", + "filename": "filename", + "contentType": "image/jpeg" + } + ] + }, + { + "insert": "expected.chunks", + "documents": [ + { + "_id": "*actual", + "files_id": "*result", + "n": 0, + "data": { + "$hex": "11" + } + } + ] + } + ] + } + }, + { + "description": "Upload when metadata is provided", + "act": { + "operation": "upload", + "arguments": { + "filename": "filename", + "source": { + "$hex": "11" + }, + "options": { + "chunkSizeBytes": 4, + "metadata": { + "x": 1 + } + } + } + }, + "assert": { + "result": "&result", + "data": [ + { + "insert": "expected.files", + "documents": [ + { + "_id": "*result", + "length": 1, + "chunkSize": 4, + "uploadDate": "*actual", + "md5": "47ed733b8d10be225eceba344d533586", + "filename": "filename", + "metadata": { + "x": 1 + } + } + ] + }, + { + "insert": "expected.chunks", + "documents": [ + { + "_id": "*actual", + "files_id": "*result", + "n": 0, + "data": { + "$hex": "11" + } + } + ] + } + ] + } + } + ] +} \ No newline at end of file diff --git a/test/test_grid_file.py b/test/test_grid_file.py index 637225449..5fc4a2694 100644 --- a/test/test_grid_file.py +++ b/test/test_grid_file.py @@ -32,7 +32,7 @@ from gridfs.grid_file import (DEFAULT_CHUNK_SIZE, GridOutCursor) from gridfs.errors import NoFile from pymongo import MongoClient -from pymongo.errors import ConfigurationError, ConnectionFailure +from pymongo.errors import ConfigurationError, ServerSelectionTimeoutError from test import (IntegrationTest, host, port, @@ -173,7 +173,6 @@ class TestGridFile(IntegrationTest): a.metadata = {"foo": 1} self.assertEqual({"foo": 1}, a.metadata) - self.assertRaises(AttributeError, getattr, a, "md5") self.assertRaises(AttributeError, setattr, a, "md5", 5) a.close() @@ -241,8 +240,8 @@ class TestGridFile(IntegrationTest): def test_grid_out_custom_opts(self): one = GridIn(self.db.fs, _id=5, filename="my_file", - contentType="text/html", chunkSize=1000, aliases=["foo"], - metadata={"foo": 1, "bar": 2}, bar=3, baz="hello") + contentType="text/html", chunkSize=1000, aliases=["foo"], + metadata={"foo": 1, "bar": 2}, bar=3, baz="hello") one.write(b"hello world") one.close() @@ -272,7 +271,8 @@ class TestGridFile(IntegrationTest): two = GridOut(self.db.fs, file_document=self.db.fs.files.find_one()) self.assertEqual(b"foo bar", two.read()) - three = GridOut(self.db.fs, 5, file_document=self.db.fs.files.find_one()) + three = GridOut(self.db.fs, 5, + file_document=self.db.fs.files.find_one()) self.assertEqual(b"foo bar", three.read()) four = GridOut(self.db.fs, file_document={}) @@ -604,8 +604,9 @@ Bye""")) serverSelectionTimeoutMS=10) fs = client.db.fs infile = GridIn(fs, file_id=-1, chunk_size=1) - self.assertRaises(ConnectionFailure, infile.write, b'data') - self.assertRaises(ConnectionFailure, infile.close) + self.assertRaises(ServerSelectionTimeoutError, infile.write, b'data') + self.assertRaises(ServerSelectionTimeoutError, infile.close) + def test_unacknowledged(self): # w=0 is prohibited. diff --git a/test/test_gridfs.py b/test/test_gridfs.py index 1e314e753..6a54f1337 100644 --- a/test/test_gridfs.py +++ b/test/test_gridfs.py @@ -27,7 +27,9 @@ import gridfs from bson.binary import Binary from bson.py3compat import u, StringIO, string_type from pymongo.mongo_client import MongoClient -from pymongo.errors import ConfigurationError, ConnectionFailure +from pymongo.errors import (ConfigurationError, + ConnectionFailure, + ServerSelectionTimeoutError) from pymongo.read_preferences import ReadPreference from gridfs.errors import CorruptGridFile, FileExists, NoFile from test.test_replica_set_client import TestReplicaSetClientBase @@ -40,9 +42,7 @@ from test.utils import (joinall, single_client, one, rs_client, - rs_or_single_client, - rs_or_single_client_noauth, - remove_all_users) + rs_or_single_client) class JustWrite(threading.Thread): @@ -174,24 +174,21 @@ class TestGridfs(IntegrationTest): finally: self.fs.delete(files_id) - def test_delete_ensures_index(self): + def test_put_ensures_index(self): # setUp has dropped collections. names = self.db.collection_names() self.assertFalse([name for name in names if name.startswith('fs')]) chunks = self.db.fs.chunks + files = self.db.fs.files + self.fs.put(b"junk") - self.fs.delete(file_id=1) - - # delete() has ensured an index on (files_id, n). - # index_information() is like: - # { - # '_id_': {'key': [('_id', 1)]}, - # 'files_id_1_n_1': {'key': [('files_id', 1), ('n', 1)]} - # } self.assertTrue(any( info.get('key') == [('files_id', 1), ('n', 1)] for info in chunks.index_information().values())) + self.assertTrue(any( + info.get('key') == [('filename', 1), ('uploadDate', 1)] + for info in files.index_information().values())) def test_alt_collection(self): oid = self.alt.put(b"hello world") @@ -246,7 +243,7 @@ class TestGridfs(IntegrationTest): # Should have created 100 versions of 'test' file self.assertEqual( 100, - self.db.fs.files.find({'filename':'test'}).count() + self.db.fs.files.find({'filename': 'test'}).count() ) def test_get_last_version(self): @@ -404,11 +401,11 @@ class TestGridfs(IntegrationTest): serverSelectionTimeoutMS=10) db = client.db gfs = gridfs.GridFS(db) - self.assertRaises(ConnectionFailure, gfs.list) + self.assertRaises(ServerSelectionTimeoutError, gfs.list) fs = gridfs.GridFS(db) - f = fs.new_file() # Still no connection. - self.assertRaises(ConnectionFailure, f.close) + f = fs.new_file() + self.assertRaises(ServerSelectionTimeoutError, f.close) def test_gridfs_find(self): self.fs.put(b"test2", filename="two") @@ -418,7 +415,7 @@ class TestGridfs(IntegrationTest): self.fs.put(b"test1", filename="one") time.sleep(0.01) self.fs.put(b"test2++", filename="two") - self.assertEqual(3, self.fs.find({"filename":"two"}).count()) + self.assertEqual(3, self.fs.find({"filename": "two"}).count()) self.assertEqual(4, self.fs.find().count()) cursor = self.fs.find( no_cursor_timeout=False).sort("uploadDate", -1).skip(1).limit(2) @@ -516,23 +513,6 @@ class TestGridfsReplicaSet(TestReplicaSetClientBase): rsc.pymongo_test.drop_collection('fs.files') rsc.pymongo_test.drop_collection('fs.chunks') -class TestGridfsAuth(IntegrationTest): - - @client_context.require_auth - def test_gridfs_readonly(self): - # "self.client" is logged in as root. Make a read-only user. - auth_db = self.client.test_gridfs_readonly - auth_db.add_user('readonly', 'pw', readOnly=True) - self.addCleanup(remove_all_users, auth_db) - - db = rs_or_single_client_noauth().test_gridfs_readonly - db.authenticate('readonly', 'pw') - - fs = gridfs.GridFS(db) - file = fs.new_file() - file._ensure_index() - fs.list() - if __name__ == "__main__": unittest.main() diff --git a/test/test_gridfs_bucket.py b/test/test_gridfs_bucket.py new file mode 100644 index 000000000..a06adc034 --- /dev/null +++ b/test/test_gridfs_bucket.py @@ -0,0 +1,430 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2015 MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the gridfs package. +""" +import datetime +import threading +import time + +import gridfs + +from bson.binary import Binary +from bson.py3compat import StringIO, string_type +from gridfs.errors import NoFile, CorruptGridFile +from pymongo.errors import (ConfigurationError, + ConnectionFailure, + ServerSelectionTimeoutError, + OperationFailure) +from pymongo.mongo_client import MongoClient +from pymongo.read_preferences import ReadPreference +from test import (client_context, + IntegrationTest) +from test.test_replica_set_client import TestReplicaSetClientBase +from test.utils import (joinall, + single_client, + one, + rs_client, + rs_or_single_client, + rs_or_single_client_noauth, + remove_all_users) + + +class JustWrite(threading.Thread): + + def __init__(self, gfs, num): + threading.Thread.__init__(self) + self.gfs = gfs + self.num = num + self.setDaemon(True) + + def run(self): + for _ in range(self.num): + file, _ = self.gfs.open_upload_stream("test") + file.write(b"hello") + file.close() + + +class JustRead(threading.Thread): + + def __init__(self, gfs, num, results): + threading.Thread.__init__(self) + self.gfs = gfs + self.num = num + self.results = results + self.setDaemon(True) + + def run(self): + for _ in range(self.num): + file = self.gfs.open_download_stream_by_name("test") + data = file.read() + self.results.append(data) + assert data == b"hello" + + +class TestGridfs(IntegrationTest): + + @classmethod + def setUpClass(cls): + super(TestGridfs, cls).setUpClass() + cls.fs = gridfs.GridFSBucket(cls.db) + cls.alt = gridfs.GridFSBucket( + cls.db, bucket_name="alt") + + def setUp(self): + self.db.drop_collection("fs.files") + self.db.drop_collection("fs.chunks") + self.db.drop_collection("alt.files") + self.db.drop_collection("alt.chunks") + + def test_basic(self): + oid = self.fs.upload_from_stream("test_filename", + b"hello world") + self.assertEqual(b"hello world", + self.fs.open_download_stream(oid).read()) + self.assertEqual(1, self.db.fs.files.count()) + self.assertEqual(1, self.db.fs.chunks.count()) + + self.fs.delete(oid) + self.assertRaises(NoFile, self.fs.open_download_stream, oid) + self.assertEqual(0, self.db.fs.files.count()) + self.assertEqual(0, self.db.fs.chunks.count()) + + def test_multi_chunk_delete(self): + self.db.fs.drop() + self.assertEqual(0, self.db.fs.files.count()) + self.assertEqual(0, self.db.fs.chunks.count()) + gfs = gridfs.GridFSBucket(self.db) + oid = gfs.upload_from_stream("test_filename", + b"hello", + chunk_size_bytes=1) + self.assertEqual(1, self.db.fs.files.count()) + self.assertEqual(5, self.db.fs.chunks.count()) + gfs.delete(oid) + self.assertEqual(0, self.db.fs.files.count()) + self.assertEqual(0, self.db.fs.chunks.count()) + + def test_empty_file(self): + oid = self.fs.upload_from_stream("test_filename", + b"") + self.assertEqual(b"", self.fs.open_download_stream(oid).read()) + self.assertEqual(1, self.db.fs.files.count()) + self.assertEqual(0, self.db.fs.chunks.count()) + + raw = self.db.fs.files.find_one() + self.assertEqual(0, raw["length"]) + self.assertEqual(oid, raw["_id"]) + self.assertTrue(isinstance(raw["uploadDate"], datetime.datetime)) + self.assertEqual(255 * 1024, raw["chunkSize"]) + self.assertTrue(isinstance(raw["md5"], string_type)) + + def test_corrupt_chunk(self): + files_id = self.fs.upload_from_stream("test_filename", + b'foobar') + self.db.fs.chunks.update_one({'files_id': files_id}, + {'$set': {'data': Binary(b'foo', 0)}}) + try: + out = self.fs.open_download_stream(files_id) + self.assertRaises(CorruptGridFile, out.read) + + out = self.fs.open_download_stream(files_id) + self.assertRaises(CorruptGridFile, out.readline) + finally: + self.fs.delete(files_id) + + def test_upload_ensures_index(self): + # setUp has dropped collections. + names = self.db.collection_names() + self.assertFalse([name for name in names if name.startswith('fs')]) + + chunks = self.db.fs.chunks + files = self.db.fs.files + self.fs.upload_from_stream("filename", b"junk") + + self.assertTrue(any( + info.get('key') == [('files_id', 1), ('n', 1)] + for info in chunks.index_information().values())) + self.assertTrue(any( + info.get('key') == [('filename', 1), ('uploadDate', 1)] + for info in files.index_information().values())) + + def test_alt_collection(self): + oid = self.alt.upload_from_stream("test_filename", + b"hello world") + self.assertEqual(b"hello world", + self.alt.open_download_stream(oid).read()) + self.assertEqual(1, self.db.alt.files.count()) + self.assertEqual(1, self.db.alt.chunks.count()) + + self.alt.delete(oid) + self.assertRaises(NoFile, self.alt.open_download_stream, oid) + self.assertEqual(0, self.db.alt.files.count()) + self.assertEqual(0, self.db.alt.chunks.count()) + + self.assertRaises(NoFile, self.alt.open_download_stream, "foo") + self.alt.upload_from_stream("foo", + b"hello world") + self.assertEqual(b"hello world", + self.alt.open_download_stream_by_name("foo").read()) + + self.alt.upload_from_stream("mike", b"") + self.alt.upload_from_stream("test", b"foo") + self.alt.upload_from_stream("hello world", b"") + + self.assertEqual(set(["mike", "test", "hello world", "foo"]), + set(k["filename"] for k in list( + self.db.alt.files.find()))) + + def test_threaded_reads(self): + self.fs.upload_from_stream("test", b"hello") + + threads = [] + results = [] + for i in range(10): + threads.append(JustRead(self.fs, 10, results)) + threads[i].start() + + joinall(threads) + + self.assertEqual( + 100 * [b'hello'], + results + ) + + def test_threaded_writes(self): + threads = [] + for i in range(10): + threads.append(JustWrite(self.fs, 10)) + threads[i].start() + + joinall(threads) + + fstr = self.fs.open_download_stream_by_name("test") + self.assertEqual(fstr.read(), b"hello") + + # Should have created 100 versions of 'test' file + self.assertEqual( + 100, + self.db.fs.files.find({'filename': 'test'}).count() + ) + + def test_get_last_version(self): + one = self.fs.upload_from_stream("test", b"foo") + time.sleep(0.01) + two, _ = self.fs.open_upload_stream("test") + two.write(b"bar") + two.close() + time.sleep(0.01) + two = two._id + three = self.fs.upload_from_stream("test", b"baz") + + self.assertEqual(b"baz", + self.fs.open_download_stream_by_name("test").read()) + self.fs.delete(three) + self.assertEqual(b"bar", + self.fs.open_download_stream_by_name("test").read()) + self.fs.delete(two) + self.assertEqual(b"foo", + self.fs.open_download_stream_by_name("test").read()) + self.fs.delete(one) + self.assertRaises(NoFile, + self.fs.open_download_stream_by_name, "test") + + def test_get_version(self): + self.fs.upload_from_stream("test", b"foo") + time.sleep(0.01) + self.fs.upload_from_stream("test", b"bar") + time.sleep(0.01) + self.fs.upload_from_stream("test", b"baz") + time.sleep(0.01) + + self.assertEqual(b"foo", self.fs.open_download_stream_by_name( + "test", revision=0).read()) + self.assertEqual(b"bar", self.fs.open_download_stream_by_name( + "test", revision=1).read()) + self.assertEqual(b"baz", self.fs.open_download_stream_by_name( + "test", revision=2).read()) + + self.assertEqual(b"baz", self.fs.open_download_stream_by_name( + "test", revision=-1).read()) + self.assertEqual(b"bar", self.fs.open_download_stream_by_name( + "test", revision=-2).read()) + self.assertEqual(b"foo", self.fs.open_download_stream_by_name( + "test", revision=-3).read()) + + self.assertRaises(NoFile, self.fs.open_download_stream_by_name, + "test", revision=3) + self.assertRaises(NoFile, self.fs.open_download_stream_by_name, + "test", revision=-4) + + def test_upload_from_stream_filelike(self): + oid = self.fs.upload_from_stream("test_file", + StringIO(b"hello world"), + chunk_size_bytes=1) + self.assertEqual(11, self.db.fs.chunks.count()) + self.assertEqual(b"hello world", + self.fs.open_download_stream(oid).read()) + + def test_missing_length_iter(self): + # Test fix that guards against PHP-237 + self.fs.upload_from_stream("empty", b"") + doc = self.db.fs.files.find_one({"filename": "empty"}) + doc.pop("length") + self.db.fs.files.replace_one({"_id": doc["_id"]}, doc) + fstr = self.fs.open_download_stream_by_name("empty") + + def iterate_file(grid_file): + for _ in grid_file: + pass + return True + + self.assertTrue(iterate_file(fstr)) + + def test_gridfs_lazy_connect(self): + client = MongoClient('badhost', connect=False, + serverSelectionTimeoutMS=0) + cdb = client.db + gfs = gridfs.GridFSBucket(cdb) + self.assertRaises(ServerSelectionTimeoutError, gfs.delete, 0) + + gfs = gridfs.GridFSBucket(cdb) + self.assertRaises( + ServerSelectionTimeoutError, + gfs.upload_from_stream, "test", b"") # Still no connection. + + def test_gridfs_find(self): + self.fs.upload_from_stream("two", b"test2") + time.sleep(0.01) + self.fs.upload_from_stream("two", b"test2+") + time.sleep(0.01) + self.fs.upload_from_stream("one", b"test1") + time.sleep(0.01) + self.fs.upload_from_stream("two", b"test2++") + self.assertEqual(3, self.fs.find({"filename": "two"}).count()) + self.assertEqual(4, self.fs.find({}).count()) + cursor = self.fs.find( + {}, no_cursor_timeout=False, sort=[("uploadDate", -1)], + skip=1, limit=2) + gout = next(cursor) + self.assertEqual(b"test1", gout.read()) + cursor.rewind() + gout = next(cursor) + self.assertEqual(b"test1", gout.read()) + gout = next(cursor) + self.assertEqual(b"test2+", gout.read()) + self.assertRaises(StopIteration, cursor.__next__) + cursor.close() + self.assertRaises(TypeError, self.fs.find, {}, {"_id": True}) + + def test_grid_in_non_int_chunksize(self): + # Lua, and perhaps other buggy GridFS clients, store size as a float. + data = b'data' + self.fs.upload_from_stream('f', data) + self.db.fs.files.update_one({'filename': 'f'}, + {'$set': {'chunkSize': 100.0}}) + + self.assertEqual(data, + self.fs.open_download_stream_by_name('f').read()) + + def test_unacknowledged(self): + # w=0 is prohibited. + with self.assertRaises(ConfigurationError): + gridfs.GridFSBucket(rs_or_single_client(w=0).pymongo_test) + + def test_rename(self): + _id = self.fs.upload_from_stream("first_name", b'testing') + self.assertEqual(b'testing', self.fs.open_download_stream_by_name( + "first_name").read()) + + self.fs.rename(_id, "second_name") + self.assertRaises(NoFile, self.fs.open_download_stream_by_name, + "first_name") + self.assertEqual(b"testing", self.fs.open_download_stream_by_name( + "second_name").read()) + + def test_abort(self): + gin, file_id = self.fs.open_upload_stream("test_filename", + chunk_size_bytes=5) + gin.write(b"test1") + gin.write(b"test2") + gin.write(b"test3") + self.assertEqual(3, self.db.fs.chunks.count( + {"files_id": file_id})) + gin.abort() + self.assertTrue(gin.closed) + self.assertRaises(ValueError, gin.write, b"test4") + self.assertEqual(0, self.db.fs.chunks.count( + {"files_id": file_id})) + +class TestGridfsBucketReplicaSet(TestReplicaSetClientBase): + + def test_gridfs_replica_set(self): + rsc = rs_client( + w=self.w, wtimeout=5000, + read_preference=ReadPreference.SECONDARY) + + gfs = gridfs.GridFSBucket(rsc.pymongo_test) + oid = gfs.upload_from_stream("test_filename", b'foo') + content = gfs.open_download_stream(oid).read() + self.assertEqual(b'foo', content) + + def test_gridfs_secondary(self): + primary_host, primary_port = self.primary + primary_connection = single_client(primary_host, primary_port) + + secondary_host, secondary_port = one(self.secondaries) + secondary_connection = single_client( + secondary_host, secondary_port, + read_preference=ReadPreference.SECONDARY) + + primary_connection.pymongo_test.drop_collection("fs.files") + primary_connection.pymongo_test.drop_collection("fs.chunks") + + # Should detect it's connected to secondary and not attempt to + # create index + gfs = gridfs.GridFSBucket(secondary_connection.pymongo_test) + + # This won't detect secondary, raises error + self.assertRaises(ConnectionFailure, gfs.upload_from_stream, + "test_filename", b'foo') + + def test_gridfs_secondary_lazy(self): + # Should detect it's connected to secondary and not attempt to + # create index. + secondary_host, secondary_port = one(self.secondaries) + client = single_client( + secondary_host, + secondary_port, + read_preference=ReadPreference.SECONDARY, + connect=False) + + # Still no connection. + gfs = gridfs.GridFSBucket(client.test_gridfs_secondary_lazy) + + # Connects, doesn't create index. + self.assertRaises(NoFile, gfs.open_download_stream_by_name, + "test_filename") + self.assertRaises(ConnectionFailure, gfs.upload_from_stream, + "test_filename", b'data') + + def tearDown(self): + rsc = client_context.rs_client + rsc.pymongo_test.drop_collection('fs.files') + rsc.pymongo_test.drop_collection('fs.chunks') + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_gridfs_spec.py b/test/test_gridfs_spec.py new file mode 100644 index 000000000..736a6f9e3 --- /dev/null +++ b/test/test_gridfs_spec.py @@ -0,0 +1,221 @@ +# Copyright 2015 MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test GridFSBucket class.""" +import copy +import datetime +import os +import sys +import re + +import gridfs + +sys.path[0:0] = [""] + +from bson import Binary +from bson.json_util import loads +from bson.py3compat import bytes_from_hex +from gridfs.errors import NoFile, CorruptGridFile +from test import (unittest, + IntegrationTest) + +# Commands. +_COMMANDS = {"delete": lambda coll, doc: [coll.delete_many(d["q"]) + for d in doc['deletes']], + "insert": lambda coll, doc: coll.insert_many(doc['documents']), + "update": lambda coll, doc: [coll.update_many(u["q"], u["u"]) + for u in doc['updates']] + } + +# Location of JSON test specifications. +_TEST_PATH = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + 'gridfs') + + +def camel_to_snake(camel): + # Regex to convert CamelCase to snake_case. Special case for _id. + if camel == "id": + return "file_id" + snake = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', camel) + return re.sub('([a-z0-9])([A-Z])', r'\1_\2', snake).lower() + + +class TestAllScenarios(IntegrationTest): + @classmethod + def setUpClass(cls): + super(TestAllScenarios, cls).setUpClass() + cls.fs = gridfs.GridFSBucket(cls.db) + cls.str_to_cmd = { + "upload": cls.fs.upload_from_stream, + "download": cls.fs.open_download_stream, + "delete": cls.fs.delete, + "download_by_name": cls.fs.open_download_stream_by_name} + + def init_db(self, data, test): + self.db.drop_collection("fs.files") + self.db.drop_collection("fs.chunks") + self.db.drop_collection("expected.files") + self.db.drop_collection("expected.chunks") + + # Read in data. + if data['files']: + self.db.fs.files.insert_many(data['files']) + self.db.expected.files.insert_many(data['files']) + if data['chunks']: + self.db.fs.chunks.insert_many(data['chunks']) + self.db.expected.chunks.insert_many(data['chunks']) + + # Make initial modifications. + if "arrange" in test: + for cmd in test['arrange'].get('data', []): + for key in cmd.keys(): + if key in _COMMANDS: + coll = self.db.get_collection(cmd[key]) + _COMMANDS[key](coll, cmd) + + def init_expected_db(self, test, result): + # Modify outcome DB. + for cmd in test['assert'].get('data', []): + for key in cmd.keys(): + if key in _COMMANDS: + # Replace wildcards in inserts. + for doc in cmd.get('documents', []): + keylist = doc.keys() + for dockey in copy.deepcopy(list(keylist)): + if "result" in str(doc[dockey]): + doc[dockey] = result + if "actual" in str(doc[dockey]): # Avoid duplicate + doc.pop(dockey) + # Move contentType to metadata. + if dockey == "contentType": + doc["metadata"] = {dockey: doc.pop(dockey)} + coll = self.db.get_collection(cmd[key]) + _COMMANDS[key](coll, cmd) + + if test['assert'].get('result') == "&result": + test['assert']['result'] = result + + def sorted_list(self, coll, ignore_id): + to_sort = [] + for doc in coll.find(): + docstr = "{" + if ignore_id: # Cannot compare _id in chunks collection. + doc.pop("_id") + for k in sorted(doc.keys()): + if k == "uploadDate": # Can't compare datetime. + self.assertTrue(isinstance(doc[k], datetime.datetime)) + else: + docstr += "%s:%s " % (k, repr(doc[k])) + to_sort.append(docstr + "}") + return to_sort + + +def create_test(scenario_def): + def run_scenario(self): + + # Run tests. + self.assertTrue(scenario_def['tests'], "tests cannot be empty") + for test in scenario_def['tests']: + self.init_db(scenario_def['data'], test) + + # Run GridFs Operation. + operation = self.str_to_cmd[test['act']['operation']] + args = test['act']['arguments'] + extra_opts = args.pop("options", {}) + if "contentType" in extra_opts: + extra_opts["metadata"] = { + "contentType": extra_opts.pop("contentType")} + + args.update(extra_opts) + + converted_args = dict((camel_to_snake(c), v) + for c, v in args.items()) + + error = None + try: + result = operation(**converted_args) + + if 'download' in test['act']['operation']: + result = Binary(result.read()) + except Exception as exc: + error = exc + + self.init_expected_db(test, result) + + # Asserts. + errors = {"FileNotFound": NoFile, + "ChunkIsMissing": CorruptGridFile, + "ExtraChunk": CorruptGridFile, + "ChunkIsWrongSize": CorruptGridFile, + "RevisionNotFound": NoFile} + + if test['assert'].get("error", False): + self.assertIsNotNone(error) + self.assertTrue(isinstance(error, + errors[test['assert']['error']])) + else: + self.assertIsNone(error) + + if 'result' in test['assert']: + if test['assert']['result'] == 'void': + test['assert']['result'] = None + self.assertEqual(result, test['assert'].get('result')) + + if 'data' in test['assert']: + # Create alphabetized list + self.assertEqual( + set(self.sorted_list(self.db.fs.chunks, True)), + set(self.sorted_list(self.db.expected.chunks, True))) + + self.assertEqual( + set(self.sorted_list(self.db.fs.files, False)), + set(self.sorted_list(self.db.expected.files, False))) + + return run_scenario + + +def create_tests(): + for dirpath, _, filenames in os.walk(_TEST_PATH): + for filename in filenames: + with open(os.path.join(dirpath, filename)) as scenario_stream: + scenario_def = loads(scenario_stream.read()) + + # Because object_hook is already defined by bson.json_util, + # and everything is named 'data' + def str2hex(jsn): + for key, val in jsn.items(): + if key in ("data", "source", "result"): + if "$hex" in val: + jsn[key] = Binary(bytes_from_hex(val['$hex'])) + if isinstance(jsn[key], dict): + str2hex(jsn[key]) + if isinstance(jsn[key], list): + for k in jsn[key]: + str2hex(k) + + str2hex(scenario_def) + + # Construct test from scenario. + new_test = create_test(scenario_def) + test_name = 'test_%s' % ( + os.path.splitext(filename)[0]) + new_test.__name__ = test_name + setattr(TestAllScenarios, new_test.__name__, new_test) + + +create_tests() + +if __name__ == "__main__": + unittest.main()