Hello community, here is the log from the commit of package python-pymongo for openSUSE:Factory checked in at 2014-08-27 07:46:44 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-pymongo (Old) and /work/SRC/openSUSE:Factory/.python-pymongo.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-pymongo" Changes: -------- --- /work/SRC/openSUSE:Factory/python-pymongo/python-pymongo.changes 2014-07-26 09:42:25.000000000 +0200 +++ /work/SRC/openSUSE:Factory/.python-pymongo.new/python-pymongo.changes 2014-08-27 07:47:10.000000000 +0200 @@ -1,0 +2,16 @@ +Tue Aug 26 10:18:16 UTC 2014 - [email protected] + +- Update to version 2.7.2 + * Insert _id in document after applying non-copying SONManipulators + (https://jira.mongodb.org/browse/PYTHON-709) + * Fix exhaust cursor error-handling (https://jira.mongodb.org/browse/PYTHON-736) + * Handle network errors when adding existing credentials to sockets + (https://jira.mongodb.org/browse/PYTHON-732) + * ObjectId.is_valid(None) should be False (https://jira.mongodb.org/browse/PYTHON-712) + * Clarify versionchanged line for bulk insert (https://jira.mongodb.org/browse/PYTHON-738) + * Work around localhost exception issues in add_user when connected to + MongoDB >= 2.7.1 (https://jira.mongodb.org/browse/PYTHON-714) + * Fix Bulk API legacy upsert _id compatibility (https://jira.mongodb.org/browse/PYTHON-705) + * SON.to_dict shouldn't change original data (https://jira.mongodb.org/browse/PYTHON-710) + +------------------------------------------------------------------- Old: ---- pymongo-2.7.1.tar.gz New: ---- pymongo-2.7.2.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-pymongo.spec ++++++ --- /var/tmp/diff_new_pack.1OZ0g9/_old 2014-08-27 07:47:11.000000000 +0200 +++ /var/tmp/diff_new_pack.1OZ0g9/_new 2014-08-27 07:47:11.000000000 +0200 @@ -17,7 +17,7 @@ Name: python-pymongo -Version: 2.7.1 +Version: 2.7.2 Release: 0 Url: http://github.com/mongodb/mongo-python-driver Summary: Python driver for MongoDB ++++++ pymongo-2.7.1.tar.gz -> pymongo-2.7.2.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pymongo-2.7.1/PKG-INFO new/pymongo-2.7.2/PKG-INFO --- old/pymongo-2.7.1/PKG-INFO 2014-05-23 23:49:14.000000000 +0200 +++ new/pymongo-2.7.2/PKG-INFO 2014-07-29 23:46:02.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: pymongo -Version: 2.7.1 +Version: 2.7.2 Summary: Python driver for MongoDB <http://www.mongodb.org> Home-page: http://github.com/mongodb/mongo-python-driver Author: Bernie Hackett diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pymongo-2.7.1/bson/objectid.py new/pymongo-2.7.2/bson/objectid.py --- old/pymongo-2.7.1/bson/objectid.py 2014-05-23 22:28:24.000000000 +0200 +++ new/pymongo-2.7.2/bson/objectid.py 2014-07-29 23:29:27.000000000 +0200 @@ -140,6 +140,9 @@ .. versionadded:: 2.3 """ + if not oid: + return False + try: ObjectId(oid) return True diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pymongo-2.7.1/bson/son.py new/pymongo-2.7.2/bson/son.py --- old/pymongo-2.7.1/bson/son.py 2014-05-23 22:28:24.000000000 +0200 +++ new/pymongo-2.7.2/bson/son.py 2014-07-29 23:29:27.000000000 +0200 @@ -226,12 +226,12 @@ def transform_value(value): if isinstance(value, list): return [transform_value(v) for v in value] - if isinstance(value, SON): - value = dict(value) - if isinstance(value, dict): - for k, v in value.iteritems(): - value[k] = transform_value(v) - return value + elif isinstance(value, dict): + return dict([ + (k, transform_value(v)) + for k, v in value.iteritems()]) + else: + return value return transform_value(dict(self)) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pymongo-2.7.1/doc/changelog.rst new/pymongo-2.7.2/doc/changelog.rst --- old/pymongo-2.7.1/doc/changelog.rst 2014-05-23 22:37:25.000000000 +0200 +++ new/pymongo-2.7.2/doc/changelog.rst 2014-07-29 23:29:27.000000000 +0200 @@ -1,6 +1,23 @@ Changelog ========= +Changes in Version 2.7.2 +------------------------ + +Version 2.7.2 includes fixes for upsert reporting in the bulk API for MongoDB +versions previous to 2.6, a regression in how son manipulators are applied in +:meth:`~pymongo.collection.Collection.insert`, a few obscure connection pool +semaphore leaks, and a few other minor issues. See the list of issues resolved +for full details. + +Issues Resolved +............... + +See the `PyMongo 2.7.2 release notes in JIRA`_ for the list of resolved issues +in this release. + +.. _PyMongo 2.7.2 release notes in JIRA: https://jira.mongodb.org/browse/PYTHON/fixforversion/14005 + Changes in Version 2.7.1 ------------------------ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pymongo-2.7.1/pymongo/__init__.py new/pymongo-2.7.2/pymongo/__init__.py --- old/pymongo-2.7.1/pymongo/__init__.py 2014-05-23 22:40:53.000000000 +0200 +++ new/pymongo-2.7.2/pymongo/__init__.py 2014-07-29 23:31:05.000000000 +0200 @@ -77,7 +77,7 @@ ALL = 2 """Profile all operations.""" -version_tuple = (2, 7, 1) +version_tuple = (2, 7, 2) def get_version_string(): if isinstance(version_tuple[-1], basestring): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pymongo-2.7.1/pymongo/bulk.py new/pymongo-2.7.2/pymongo/bulk.py --- old/pymongo-2.7.1/pymongo/bulk.py 2014-05-23 22:28:24.000000000 +0200 +++ new/pymongo-2.7.2/pymongo/bulk.py 2014-07-29 23:29:27.000000000 +0200 @@ -101,13 +101,25 @@ if run.op_type == _INSERT: full_result['nInserted'] += 1 + elif run.op_type == _UPDATE: if "upserted" in result: doc = {u"index": run.index(index), u"_id": result["upserted"]} full_result["upserted"].append(doc) full_result['nUpserted'] += affected + # Versions of MongoDB before 2.6 don't return the _id for an + # upsert if _id is not an ObjectId. + elif result.get("updatedExisting") == False and affected == 1: + op = run.ops[index] + # If _id is in both the update document *and* the query spec + # the update document _id takes precedence. + _id = op['u'].get('_id', op['q'].get('_id')) + doc = {u"index": run.index(index), u"_id": _id} + full_result["upserted"].append(doc) + full_result['nUpserted'] += affected else: full_result['nMatched'] += affected + elif run.op_type == _DELETE: full_result['nRemoved'] += affected diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pymongo-2.7.1/pymongo/collection.py new/pymongo-2.7.2/pymongo/collection.py --- old/pymongo-2.7.1/pymongo/collection.py 2014-05-23 22:28:24.000000000 +0200 +++ new/pymongo-2.7.2/pymongo/collection.py 2014-07-29 23:29:27.000000000 +0200 @@ -356,7 +356,7 @@ Support for passing `getLastError` options as keyword arguments. .. versionchanged:: 1.1 - Bulk insert works with any iterable + Bulk insert works with an iterable sequence of documents. .. mongodoc:: insert """ @@ -378,11 +378,14 @@ def gen(): db = self.__database for doc in docs: + # Apply user-configured SON manipulators. This order of + # operations is required for backwards compatibility, + # see PYTHON-709. + doc = db._apply_incoming_manipulators(doc, self) if '_id' not in doc: doc['_id'] = ObjectId() - # Apply user-configured SON manipulators. - doc = db._fix_incoming(doc, self) + doc = db._apply_incoming_copying_manipulators(doc, self) ids.append(doc['_id']) yield doc else: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pymongo-2.7.1/pymongo/cursor.py new/pymongo-2.7.2/pymongo/cursor.py --- old/pymongo-2.7.1/pymongo/cursor.py 2014-05-23 22:28:24.000000000 +0200 +++ new/pymongo-2.7.2/pymongo/cursor.py 2014-07-29 23:29:27.000000000 +0200 @@ -22,8 +22,8 @@ from pymongo import helpers, message, read_preferences from pymongo.read_preferences import ReadPreference, secondary_ok_commands from pymongo.errors import (AutoReconnect, - CursorNotFound, - InvalidOperation) + InvalidOperation, + OperationFailure) _QUERY_OPTIONS = { "tailable_cursor": 2, @@ -56,6 +56,15 @@ self.pool.maybe_return_socket(self.sock) self.sock, self.pool = None, None + def error(self): + """Clean up after an error on the managed socket. + """ + if self.sock: + self.sock.close() + + # Return the closed socket to avoid a semaphore leak in the pool. + self.close() + # TODO might be cool to be able to do find().include("foo") or # find().exclude(["bar", "baz"]) or find().slice("a", 1, 2) as an @@ -914,8 +923,14 @@ # due to a socket timeout. self.__killed = True raise - else: # exhaust cursor - no getMore message - response = client._exhaust_next(self.__exhaust_mgr.sock) + else: + # Exhaust cursor - no getMore message. + try: + response = client._exhaust_next(self.__exhaust_mgr.sock) + except AutoReconnect: + self.__killed = True + self.__exhaust_mgr.error() + raise try: response = helpers._unpack_response(response, self.__id, @@ -923,8 +938,10 @@ self.__tz_aware, self.__uuid_subtype, self.__compile_re) - except CursorNotFound: + except OperationFailure: self.__killed = True + # Make sure exhaust socket is returned immediately, if necessary. + self.__die() # If this is a tailable cursor the error is likely # due to capped collection roll over. Setting # self.__killed to True ensures Cursor.alive will be @@ -936,8 +953,11 @@ # Don't send kill cursors to another server after a "not master" # error. It's completely pointless. self.__killed = True + # Make sure exhaust socket is returned immediately, if necessary. + self.__die() client.disconnect() raise + self.__id = response["cursor_id"] # starting from doesn't get set on getmore's for tailable cursors diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pymongo-2.7.1/pymongo/database.py new/pymongo-2.7.2/pymongo/database.py --- old/pymongo-2.7.1/pymongo/database.py 2014-05-23 22:28:24.000000000 +0200 +++ new/pymongo-2.7.2/pymongo/database.py 2014-07-29 23:29:27.000000000 +0200 @@ -247,6 +247,16 @@ return Collection(self, name, **opts) + def _apply_incoming_manipulators(self, son, collection): + for manipulator in self.__incoming_manipulators: + son = manipulator.transform_incoming(son, collection) + return son + + def _apply_incoming_copying_manipulators(self, son, collection): + for manipulator in self.__incoming_copying_manipulators: + son = manipulator.transform_incoming(son, collection) + return son + def _fix_incoming(self, son, collection): """Apply manipulators to an incoming SON object before it gets stored. @@ -254,10 +264,8 @@ - `son`: the son object going into the database - `collection`: the collection the son object is being saved in """ - for manipulator in self.__incoming_manipulators: - son = manipulator.transform_incoming(son, collection) - for manipulator in self.__incoming_copying_manipulators: - son = manipulator.transform_incoming(son, collection) + son = self._apply_incoming_manipulators(son, collection) + son = self._apply_incoming_copying_manipulators(son, collection) return son def _fix_outgoing(self, son, collection): @@ -782,17 +790,20 @@ try: uinfo = self.command("usersInfo", name, read_preference=ReadPreference.PRIMARY) + self._create_or_update_user( + (not uinfo["users"]), name, password, read_only, **kwargs) except OperationFailure, exc: # MongoDB >= 2.5.3 requires the use of commands to manage # users. if exc.code in common.COMMAND_NOT_FOUND_CODES: self._legacy_add_user(name, password, read_only, **kwargs) - return - raise - - # Create the user if not found in uinfo, otherwise update one. - self._create_or_update_user( - (not uinfo["users"]), name, password, read_only, **kwargs) + # Unauthorized. MongoDB >= 2.7.1 has a narrow localhost exception, + # and we must add a user before sending commands. + elif exc.code == 13: + self._create_or_update_user( + True, name, password, read_only, **kwargs) + else: + raise def remove_user(self, name): """Remove user `name` from this :class:`Database`. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pymongo-2.7.1/pymongo/mongo_client.py new/pymongo-2.7.2/pymongo/mongo_client.py --- old/pymongo-2.7.1/pymongo/mongo_client.py 2014-05-23 22:28:24.000000000 +0200 +++ new/pymongo-2.7.2/pymongo/mongo_client.py 2014-07-29 23:29:27.000000000 +0200 @@ -673,13 +673,16 @@ 'max_write_batch_size', common.MAX_WRITE_BATCH_SIZE) def __simple_command(self, sock_info, dbname, spec): - """Send a command to the server. + """Send a command to the server. May raise AutoReconnect. """ rqst_id, msg, _ = message.query(0, dbname + '.$cmd', 0, -1, spec) start = time.time() try: sock_info.sock.sendall(msg) response = self.__receive_message_on_socket(1, rqst_id, sock_info) + except socket.error, e: + sock_info.close() + raise AutoReconnect(e) except: sock_info.close() raise @@ -916,7 +919,7 @@ "%s %s" % (host_details, str(why))) try: self.__check_auth(sock_info) - except OperationFailure: + except: connection_pool.maybe_return_socket(sock_info) raise return sock_info @@ -1189,27 +1192,35 @@ sock_info = self.__socket(member) exhaust = kwargs.get('exhaust') try: - try: - if not exhaust and "network_timeout" in kwargs: - sock_info.sock.settimeout(kwargs["network_timeout"]) - response = self.__send_and_receive(message, sock_info) - - if not exhaust: - if "network_timeout" in kwargs: - sock_info.sock.settimeout(self.__net_timeout) + if not exhaust and "network_timeout" in kwargs: + sock_info.sock.settimeout(kwargs["network_timeout"]) + + response = self.__send_and_receive(message, sock_info) - return (None, (response, sock_info, member.pool)) - except (ConnectionFailure, socket.error), e: - self.disconnect() - raise AutoReconnect(str(e)) - finally: if not exhaust: + if "network_timeout" in kwargs: + sock_info.sock.settimeout(self.__net_timeout) + member.pool.maybe_return_socket(sock_info) + return (None, (response, sock_info, member.pool)) + except (ConnectionFailure, socket.error), e: + self.disconnect() + member.pool.maybe_return_socket(sock_info) + raise AutoReconnect(str(e)) + except: + member.pool.maybe_return_socket(sock_info) + raise + def _exhaust_next(self, sock_info): """Used with exhaust cursors to get the next batch off the socket. + + Can raise AutoReconnect. """ - return self.__receive_message_on_socket(1, None, sock_info) + try: + return self.__receive_message_on_socket(1, None, sock_info) + except socket.error, e: + raise AutoReconnect(str(e)) def start_request(self): """Ensure the current thread or greenlet always uses the same socket diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pymongo-2.7.1/pymongo/mongo_replica_set_client.py new/pymongo-2.7.2/pymongo/mongo_replica_set_client.py --- old/pymongo-2.7.1/pymongo/mongo_replica_set_client.py 2014-05-23 22:28:24.000000000 +0200 +++ new/pymongo-2.7.2/pymongo/mongo_replica_set_client.py 2014-07-29 23:29:27.000000000 +0200 @@ -1711,8 +1711,13 @@ def _exhaust_next(self, sock_info): """Used with exhaust cursors to get the next batch off the socket. + + Can raise AutoReconnect. """ - return self.__recv_msg(1, None, sock_info) + try: + return self.__recv_msg(1, None, sock_info) + except socket.error, e: + raise AutoReconnect(str(e)) def start_request(self): """Ensure the current thread or greenlet always uses the same socket diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pymongo-2.7.1/pymongo.egg-info/PKG-INFO new/pymongo-2.7.2/pymongo.egg-info/PKG-INFO --- old/pymongo-2.7.1/pymongo.egg-info/PKG-INFO 2014-05-23 23:49:14.000000000 +0200 +++ new/pymongo-2.7.2/pymongo.egg-info/PKG-INFO 2014-07-29 23:46:02.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: pymongo -Version: 2.7.1 +Version: 2.7.2 Summary: Python driver for MongoDB <http://www.mongodb.org> Home-page: http://github.com/mongodb/mongo-python-driver Author: Bernie Hackett diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pymongo-2.7.1/setup.py new/pymongo-2.7.2/setup.py --- old/pymongo-2.7.1/setup.py 2014-05-23 22:40:36.000000000 +0200 +++ new/pymongo-2.7.2/setup.py 2014-07-29 23:31:13.000000000 +0200 @@ -33,7 +33,7 @@ from distutils.errors import DistutilsPlatformError, DistutilsExecError from distutils.core import Extension -version = "2.7.1" +version = "2.7.2" f = open("README.rst") try: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pymongo-2.7.1/test/test_bulk.py new/pymongo-2.7.2/test/test_bulk.py --- old/pymongo-2.7.1/test/test_bulk.py 2014-05-23 22:28:24.000000000 +0200 +++ new/pymongo-2.7.2/test/test_bulk.py 2014-07-29 23:29:27.000000000 +0200 @@ -600,6 +600,28 @@ self.assertEqual(1, self.coll.find({'x': 1}).count()) + def test_client_generated_upsert_id(self): + batch = self.coll.initialize_ordered_bulk_op() + batch.find({'_id': 0}).upsert().update_one({'$set': {'a': 0}}) + batch.find({'a': 1}).upsert().replace_one({'_id': 1}) + if not version.at_least(self.coll.database.connection, (2, 6, 0)): + # This case is only possible in MongoDB versions before 2.6. + batch.find({'_id': 3}).upsert().replace_one({'_id': 2}) + else: + # This is just here to make the counts right in all cases. + batch.find({'_id': 2}).upsert().replace_one({'_id': 2}) + result = batch.execute() + self.assertEqualResponse( + {'nMatched': 0, + 'nModified': 0, + 'nUpserted': 3, + 'nInserted': 0, + 'nRemoved': 0, + 'upserted': [{'index': 0, '_id': 0}, + {'index': 1, '_id': 1}, + {'index': 2, '_id': 2}]}, + result) + def test_single_ordered_batch(self): batch = self.coll.initialize_ordered_bulk_op() batch.insert({'a': 1}) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pymongo-2.7.1/test/test_client.py new/pymongo-2.7.2/test/test_client.py --- old/pymongo-2.7.1/test/test_client.py 2014-05-23 22:28:24.000000000 +0200 +++ new/pymongo-2.7.2/test/test_client.py 2014-07-29 23:29:27.000000000 +0200 @@ -34,7 +34,7 @@ from pymongo.mongo_client import MongoClient from pymongo.database import Database from pymongo.pool import SocketInfo -from pymongo import thread_util, common +from pymongo import auth, thread_util, common from pymongo.errors import (AutoReconnect, ConfigurationError, ConnectionFailure, @@ -52,9 +52,11 @@ server_started_with_auth, TestRequestMixin, _TestLazyConnectMixin, + _TestExhaustCursorMixin, lazy_client_trial, NTHREADS, - get_pool) + get_pool, + one) def get_client(*args, **kwargs): @@ -999,6 +1001,42 @@ client = get_client(_connect=False) client.pymongo_test.test.remove(w=0) + def test_auth_network_error(self): + # Make sure there's no semaphore leak if we get a network error + # when authenticating a new socket with cached credentials. + auth_client = get_client() + if not server_started_with_auth(auth_client): + raise SkipTest('Authentication is not enabled on server') + + auth_client.admin.add_user('admin', 'password') + auth_client.admin.authenticate('admin', 'password') + try: + # Get a client with one socket so we detect if it's leaked. + c = get_client(max_pool_size=1, waitQueueTimeoutMS=1) + + # Simulate an authenticate() call on a different socket. + credentials = auth._build_credentials_tuple( + 'MONGODB-CR', 'admin', + unicode('admin'), unicode('password'), + {}) + + c._cache_credentials('test', credentials, connect=False) + + # Cause a network error on the actual socket. + pool = get_pool(c) + socket_info = one(pool.sockets) + socket_info.sock.close() + + # In __check_auth, the client authenticates its socket with the + # new credential, but gets a socket.error. Should be reraised as + # AutoReconnect. + self.assertRaises(AutoReconnect, c.test.collection.find_one) + + # No semaphore leak, the pool is allowed to make a new socket. + c.test.collection.find_one() + finally: + remove_all_users(auth_client.admin) + class TestClientLazyConnect(unittest.TestCase, _TestLazyConnectMixin): def _get_client(self, **kwargs): @@ -1110,5 +1148,10 @@ c.db.collection.find_one() +class TestExhaustCursor(_TestExhaustCursorMixin, unittest.TestCase): + def _get_client(self, **kwargs): + return get_client(**kwargs) + + if __name__ == "__main__": unittest.main() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pymongo-2.7.1/test/test_database.py new/pymongo-2.7.2/test/test_database.py --- old/pymongo-2.7.1/test/test_database.py 2014-05-23 22:28:24.000000000 +0200 +++ new/pymongo-2.7.2/test/test_database.py 2014-07-29 23:29:27.000000000 +0200 @@ -45,6 +45,7 @@ OperationFailure) from pymongo.son_manipulator import (AutoReference, NamespaceInjector, + SONManipulator, ObjectIdShuffler) from test import version from test.utils import (catch_warnings, get_command_line, @@ -996,6 +997,29 @@ "maxTimeAlwaysTimeOut", mode="off") + def test_object_to_dict_transformer(self): + # PYTHON-709: Some users rely on their custom SONManipulators to run + # before any other checks, so they can insert non-dict objects and + # have them dictified before the _id is inserted or any other + # processing. + class Thing(object): + def __init__(self, value): + self.value = value + + class ThingTransformer(SONManipulator): + def transform_incoming(self, thing, collection): + return {'value': thing.value} + + db = self.client.foo + db.add_son_manipulator(ThingTransformer()) + t = Thing('value') + + db.test.remove() + db.test.insert([t]) + out = db.test.find_one() + self.assertEqual('value', out.get('value')) + + if __name__ == "__main__": unittest.main() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pymongo-2.7.1/test/test_objectid.py new/pymongo-2.7.2/test/test_objectid.py --- old/pymongo-2.7.1/test/test_objectid.py 2014-05-23 22:28:24.000000000 +0200 +++ new/pymongo-2.7.2/test/test_objectid.py 2014-07-29 23:29:27.000000000 +0200 @@ -181,6 +181,7 @@ self.assertEqual(oid_1_9, oid_1_10) def test_is_valid(self): + self.assertFalse(ObjectId.is_valid(None)) self.assertFalse(ObjectId.is_valid(4)) self.assertFalse(ObjectId.is_valid(175.0)) self.assertFalse(ObjectId.is_valid({"test": 4})) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pymongo-2.7.1/test/test_read_preferences.py new/pymongo-2.7.2/test/test_read_preferences.py --- old/pymongo-2.7.1/test/test_read_preferences.py 2014-05-23 22:28:24.000000000 +0200 +++ new/pymongo-2.7.2/test/test_read_preferences.py 2014-07-29 23:29:27.000000000 +0200 @@ -309,11 +309,11 @@ # Distinct self._test_fn(True, lambda: self.c.pymongo_test.command( - 'distinct', 'test', key={'a': 1})) + 'distinct', 'test', key='a')) self._test_fn(True, lambda: self.c.pymongo_test.command( - 'distinct', 'test', key={'a': 1}, query={'a': 1})) + 'distinct', 'test', key='a', query={'a': 1})) self._test_fn(True, lambda: self.c.pymongo_test.command(SON([ - ('distinct', 'test'), ('key', {'a': 1}), ('query', {'a': 1})]))) + ('distinct', 'test'), ('key', 'a'), ('query', {'a': 1})]))) # Geo stuff. Make sure a 2d index is created and replicated self.c.pymongo_test.system.indexes.insert({ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pymongo-2.7.1/test/test_replica_set_client.py new/pymongo-2.7.2/test/test_replica_set_client.py --- old/pymongo-2.7.1/test/test_replica_set_client.py 2014-05-23 22:28:24.000000000 +0200 +++ new/pymongo-2.7.2/test/test_replica_set_client.py 2014-07-29 23:29:27.000000000 +0200 @@ -45,13 +45,14 @@ ConnectionFailure, InvalidName, OperationFailure, InvalidOperation) +from pymongo import auth from test import version, port, pair from test.pymongo_mocks import MockReplicaSetClient from test.utils import ( delay, assertReadFrom, assertReadFromAll, read_from_which_host, remove_all_users, assertRaisesExactly, TestRequestMixin, one, server_started_with_auth, pools_from_rs_client, get_pool, - _TestLazyConnectMixin) + _TestLazyConnectMixin, _TestExhaustCursorMixin) class TestReplicaSetClientAgainstStandalone(unittest.TestCase): @@ -1126,6 +1127,42 @@ self.assertFalse(client.alive()) + def test_auth_network_error(self): + # Make sure there's no semaphore leak if we get a network error + # when authenticating a new socket with cached credentials. + auth_client = self._get_client() + if not server_started_with_auth(auth_client): + raise SkipTest('Authentication is not enabled on server') + + auth_client.admin.add_user('admin', 'password') + auth_client.admin.authenticate('admin', 'password') + try: + # Get a client with one socket so we detect if it's leaked. + c = self._get_client(max_pool_size=1, waitQueueTimeoutMS=1) + + # Simulate an authenticate() call on a different socket. + credentials = auth._build_credentials_tuple( + 'MONGODB-CR', 'admin', + unicode('admin'), unicode('password'), + {}) + + c._cache_credentials('test', credentials, connect=False) + + # Cause a network error on the actual socket. + pool = get_pool(c) + socket_info = one(pool.sockets) + socket_info.sock.close() + + # In __check_auth, the client authenticates its socket with the + # new credential, but gets a socket.error. Should be reraised as + # AutoReconnect. + self.assertRaises(AutoReconnect, c.test.collection.find_one) + + # No semaphore leak, the pool is allowed to make a new socket. + c.test.collection.find_one() + finally: + remove_all_users(auth_client.admin) + class TestReplicaSetWireVersion(unittest.TestCase): def test_wire_version(self): @@ -1237,5 +1274,12 @@ self.assertEqual(c.max_write_batch_size, 2) +class TestReplicaSetClientExhaustCursor( + _TestExhaustCursorMixin, + TestReplicaSetClientBase): + + # Base class implements _get_client already. + pass + if __name__ == "__main__": unittest.main() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pymongo-2.7.1/test/test_son.py new/pymongo-2.7.2/test/test_son.py --- old/pymongo-2.7.1/test/test_son.py 2014-05-23 22:28:24.000000000 +0200 +++ new/pymongo-2.7.2/test/test_son.py 2014-07-29 23:29:27.000000000 +0200 @@ -67,6 +67,15 @@ ('mike', 'awesome'), ('hello', 'world')))) + # Embedded SON. + d4 = SON([('blah', {'foo': SON()})]) + self.assertEqual(d4, {'blah': {'foo': {}}}) + self.assertEqual(d4, {'blah': {'foo': SON()}}) + self.assertNotEqual(d4, {'blah': {'foo': []}}) + + # Original data unaffected. + self.assertEqual(SON, d4['blah']['foo'].__class__) + def test_to_dict(self): a1 = SON() b2 = SON([("blah", SON())]) @@ -81,6 +90,9 @@ self.assertEqual(dict, c3.to_dict()["blah"][0].__class__) self.assertEqual(dict, d4.to_dict()["blah"]["foo"].__class__) + # Original data unaffected. + self.assertEqual(SON, d4['blah']['foo'].__class__) + def test_pickle(self): simple_son = SON([]) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/pymongo-2.7.1/test/utils.py new/pymongo-2.7.2/test/utils.py --- old/pymongo-2.7.1/test/utils.py 2014-05-23 22:28:24.000000000 +0200 +++ new/pymongo-2.7.2/test/utils.py 2014-07-29 23:29:27.000000000 +0200 @@ -15,14 +15,18 @@ """Utilities for testing pymongo """ +import gc import os import struct import sys import threading +import time from nose.plugins.skip import SkipTest + +from bson.son import SON from pymongo import MongoClient, MongoReplicaSetClient -from pymongo.errors import AutoReconnect +from pymongo.errors import AutoReconnect, ConnectionFailure, OperationFailure from pymongo.pool import NO_REQUEST, NO_SOCKET_YET, SocketInfo from test import host, port, version @@ -453,7 +457,7 @@ # Make concurrency bugs more likely to manifest. interval = None if not sys.platform.startswith('java'): - if sys.version_info >= (3, 2): + if hasattr(sys, 'getswitchinterval'): interval = sys.getswitchinterval() sys.setswitchinterval(1e-6) else: @@ -472,7 +476,7 @@ finally: if not sys.platform.startswith('java'): - if sys.version_info >= (3, 2): + if hasattr(sys, 'setswitchinterval'): sys.setswitchinterval(interval) else: sys.setcheckinterval(interval) @@ -586,6 +590,121 @@ c.max_message_size) +class _TestExhaustCursorMixin(object): + """Test that clients properly handle errors from exhaust cursors. + + Inherit from this class and from unittest.TestCase, and override + _get_client(self, **kwargs). + """ + def test_exhaust_query_server_error(self): + # When doing an exhaust query, the socket stays checked out on success + # but must be checked in on error to avoid semaphore leaks. + client = self._get_client(max_pool_size=1) + if is_mongos(client): + raise SkipTest("Can't use exhaust cursors with mongos") + if not version.at_least(client, (2, 2, 0)): + raise SkipTest("mongod < 2.2.0 closes exhaust socket on error") + + collection = client.pymongo_test.test + pool = get_pool(client) + + sock_info = one(pool.sockets) + # This will cause OperationFailure in all mongo versions since + # the value for $orderby must be a document. + cursor = collection.find( + SON([('$query', {}), ('$orderby', True)]), exhaust=True) + self.assertRaises(OperationFailure, cursor.next) + self.assertFalse(sock_info.closed) + + # The semaphore was decremented despite the error. + self.assertTrue(pool._socket_semaphore.acquire(blocking=False)) + + def test_exhaust_getmore_server_error(self): + # When doing a getmore on an exhaust cursor, the socket stays checked + # out on success but must be checked in on error to avoid semaphore + # leaks. + client = self._get_client(max_pool_size=1) + if is_mongos(client): + raise SkipTest("Can't use exhaust cursors with mongos") + + # A separate client that doesn't affect the test client's pool. + client2 = self._get_client() + + collection = client.pymongo_test.test + collection.remove() + + # Enough data to ensure it streams down for a few milliseconds. + long_str = 'a' * (256 * 1024) + collection.insert([{'a': long_str} for _ in range(200)]) + + pool = get_pool(client) + pool._check_interval_seconds = None # Never check. + sock_info = one(pool.sockets) + + cursor = collection.find(exhaust=True) + + # Initial query succeeds. + cursor.next() + + # Cause a server error on getmore. + client2.pymongo_test.test.drop() + self.assertRaises(OperationFailure, list, cursor) + + # Make sure the socket is still valid + self.assertEqual(0, collection.count()) + + def test_exhaust_query_network_error(self): + # When doing an exhaust query, the socket stays checked out on success + # but must be checked in on error to avoid semaphore leaks. + client = self._get_client(max_pool_size=1) + if is_mongos(client): + raise SkipTest("Can't use exhaust cursors with mongos") + + collection = client.pymongo_test.test + pool = get_pool(client) + pool._check_interval_seconds = None # Never check. + + # Cause a network error. + sock_info = one(pool.sockets) + sock_info.sock.close() + cursor = collection.find(exhaust=True) + self.assertRaises(ConnectionFailure, cursor.next) + self.assertTrue(sock_info.closed) + + # The semaphore was decremented despite the error. + self.assertTrue(pool._socket_semaphore.acquire(blocking=False)) + + def test_exhaust_getmore_network_error(self): + # When doing a getmore on an exhaust cursor, the socket stays checked + # out on success but must be checked in on error to avoid semaphore + # leaks. + client = self._get_client(max_pool_size=1) + if is_mongos(client): + raise SkipTest("Can't use exhaust cursors with mongos") + + collection = client.pymongo_test.test + collection.remove() + collection.insert([{} for _ in range(200)]) # More than one batch. + pool = get_pool(client) + pool._check_interval_seconds = None # Never check. + + cursor = collection.find(exhaust=True) + + # Initial query succeeds. + cursor.next() + + # Cause a network error. + sock_info = cursor._Cursor__exhaust_mgr.sock + sock_info.sock.close() + + # A getmore fails. + self.assertRaises(ConnectionFailure, list, cursor) + self.assertTrue(sock_info.closed) + + # The semaphore was decremented despite the error. + self.assertTrue(pool._socket_semaphore.acquire(blocking=False)) + + # Backport of WarningMessage from python 2.6, with fixed syntax for python 2.4. class WarningMessage(object): -- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
