Hello community, here is the log from the commit of package python-mongodict for openSUSE:Factory checked in at 2015-05-15 07:43:46 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-mongodict (Old) and /work/SRC/openSUSE:Factory/.python-mongodict.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-mongodict" Changes: -------- --- /work/SRC/openSUSE:Factory/python-mongodict/python-mongodict.changes 2013-10-17 20:44:51.000000000 +0200 +++ /work/SRC/openSUSE:Factory/.python-mongodict.new/python-mongodict.changes 2015-05-15 07:44:16.000000000 +0200 @@ -1,0 +2,16 @@ +Thu May 7 16:47:28 UTC 2015 - [email protected] + +- update to version 0.3.1: + This version BREAKS DATA COMPATIBILITY with 0.3.0: + * Option to specify index type: key-only or key-and-value. If you + specify "key and value" you can't have documents (key + value) + greater than 16MB. Default option moved to key-only (fixes + issue #14) + * Replace value with v as the key inside Mongo document to save + space. Note that you will need to migrate your data! A little + script is provided +- add python-pymongo as a build dependency to ensure its + availability +- add CHANGELOG.rst and migrate_data.py to the documentation + +------------------------------------------------------------------- Old: ---- mongodict-0.3.0.tar.gz New: ---- mongodict-0.3.1.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-mongodict.spec ++++++ --- /var/tmp/diff_new_pack.iWmtuV/_old 2015-05-15 07:44:17.000000000 +0200 +++ /var/tmp/diff_new_pack.iWmtuV/_new 2015-05-15 07:44:17.000000000 +0200 @@ -1,7 +1,7 @@ # # spec file for package python-mongodict # -# Copyright (c) 2013 SUSE LINUX Products GmbH, Nuernberg, Germany. +# Copyright (c) 2015 SUSE LINUX GmbH, Nuernberg, Germany. # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -13,17 +13,19 @@ # published by the Open Source Initiative. # Please submit bugfixes or comments via http://bugs.opensuse.org/ +# Name: python-mongodict -Version: 0.3.0 +Version: 0.3.1 Release: 0 -License: GPL-3.0 Summary: MongoDB-backed Python dict-like interface -Url: https://github.com/turicas/mongodict/ +License: GPL-3.0 Group: Development/Languages/Python +Url: https://github.com/turicas/mongodict/ Source: https://pypi.python.org/packages/source/m/mongodict/mongodict-%{version}.tar.gz BuildRequires: python-devel +BuildRequires: python-pymongo Requires: python-pymongo BuildRoot: %{_tmppath}/%{name}-%{version}-build %if 0%{?suse_version} && 0%{?suse_version} <= 1110 @@ -43,6 +45,8 @@ %prep %setup -q -n mongodict-%{version} +# remove unwanted executable bit +chmod -x migrate_data.py %build python setup.py build @@ -52,7 +56,7 @@ %files %defattr(-,root,root,-) -%doc README.rst LICENSE +%doc README.rst LICENSE CHANGELOG.rst migrate_data.py %{python_sitelib}/* %changelog ++++++ mongodict-0.3.0.tar.gz -> mongodict-0.3.1.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mongodict-0.3.0/CHANGELOG.rst new/mongodict-0.3.1/CHANGELOG.rst --- old/mongodict-0.3.0/CHANGELOG.rst 1970-01-01 01:00:00.000000000 +0100 +++ new/mongodict-0.3.1/CHANGELOG.rst 2014-02-17 04:01:39.000000000 +0100 @@ -0,0 +1,72 @@ +Change Log +========== + +Version 0.3.1 +------------- + +This version BREAKS DATA COMPATIBILITY with 0.3.0. Please see below: + +- Option to specify index type: key-only or key-and-value. If you specify "key + and value" you can't have documents (key + value) greater than 16MB. Default + option moved to key-only (fixes `issue #14 + <https://github.com/turicas/mongodict/issues/14>`_); +- Replace ``value`` with ``v`` as the key inside Mongo document to save space. + Note that you will need to migrate your data! A little script is provided. + + +Version 0.3.0 +------------- + +- Added a "codec" concept, so user can customized how value data is saved. + `Pickle is the default codec <https://github.com/turicas/mongodict/issues/12>`_; +- Added a + `migration script <https://github.com/turicas/mongodict/blob/master/migrate_data.py>`_ + to migrate data from mongodict <= 0.2.1; +- `Using basic mapping (dict) interface tests from python source + <https://github.com/turicas/mongodict/issues/13>`_; +- Updated `tox <https://pypi.python.org/pypi/tox>`_ config to use Python 2.7.5 + and Python 3.3.2 to run tests. + + +Version 0.2.1 +------------- + +- [Bug] Fix utf8 encoding problem when using ``bson.Binary`` (#11) +- [Bug] Fix 16MB-limit problem when calling ``MongoDict.keys`` (#10) +- [Bug] ``MongoDict.clear()`` do not remove collection indexes anymore +- Raise *instance* of ``KeyError`` instead of class +- Add ``auth`` parameter to ``MongoDict`` (#8) +- Add information about tox in README +- Fixes setup.py to work with python 2.6 +- Use ``collection.count()`` instead of ``cursor.count()`` + + +Version 0.2.0 +------------- + +- Add Python 3.2 support (thanks, @jucacrispim) +- Optimize all queries to hit only the index +- Add docstrings to entire module +- Add ``mongodict.__all__`` +- Add ``MongoDict.__del__`` +- Use tox to run tests + + +Version 0.1.1 +------------- + +- Set ``safe=True`` by default (slower) +- Did some optimizations on ``__getitem__`` and ``__delitem__`` +- Added method ``clear``, an optimized way to delete all keys +- An ``UnicodeDecodeError`` is raised if one of the strings + (key or value) is not ``unicode`` or ``str`` encoded with ``utf-8`` codec +- Replaced ``insert`` with ``update`` - it was generating an exception when + updating (already existent) keys +- Add benchmark (Python dict vs Redis vs mongodict) + + +Version 0.1.0 +------------- + +- First version released +- Basic ``dict``-like behaviour diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mongodict-0.3.0/PKG-INFO new/mongodict-0.3.1/PKG-INFO --- old/mongodict-0.3.0/PKG-INFO 2013-05-22 16:05:13.000000000 +0200 +++ new/mongodict-0.3.1/PKG-INFO 2014-02-17 04:53:27.000000000 +0100 @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: mongodict -Version: 0.3.0 +Version: 0.3.1 Summary: MongoDB-backed Python dict-like interface Home-page: https://github.com/turicas/mongodict/ Author: Álvaro Justen diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mongodict-0.3.0/migrate_data.py new/mongodict-0.3.1/migrate_data.py --- old/mongodict-0.3.0/migrate_data.py 1970-01-01 01:00:00.000000000 +0100 +++ new/mongodict-0.3.1/migrate_data.py 2014-02-17 04:18:17.000000000 +0100 @@ -0,0 +1,130 @@ +#!/usr/bin/env python +# coding: utf-8 + +''' +This script does data migration of data from mongodict <= 0.2.1 to 0.3.0 and +from 0.3.0 to 0.3.1. +The first migration is needed since the new version uses another codec by +default (pickle instead of JSON/BSON) and the later one because the key name +changes on document (`value` to `v`). +''' + +import datetime +import sys +import time + +import mongodict +import pymongo + + +encode_nothing = lambda x: x +decode_nothing = lambda x: x +report_template = ('\r{:010d} keys / {:010d} ({:6.2f}%) migrated. ' + 'Duration: {}, ETA: {}') +REPORT_INTERVAL = 10 # in number of keys + + +def print_report(counter, total, start_time): + percentual = 100 * (counter / float(total)) + duration = datetime.timedelta((time.time() - start_time) / (24 * 3600)) + duration = str(duration).split('.')[0] + rate = counter / (time.time() - start_time) + eta = datetime.timedelta(((total - counter) / rate) / (24 * 3600)) + eta = str(eta).split('.')[0] + report = report_template.format(counter, total, percentual, + duration, eta) + sys.stdout.write(report) + sys.stdout.flush() + + +def migrate_codec(config_old, config_new): + '''Migrate data from mongodict <= 0.2.1 to 0.3.0 + `config_old` and `config_new` should be dictionaries with the keys + regarding to MongoDB server: + - `host` + - `port` + - `database` + - `collection` + ''' + assert mongodict.__version__ in [(0, 3, 0), (0, 3, 1)] + + connection = pymongo.Connection(host=config_old['host'], + port=config_old['port']) + database = connection[config_old['database']] + collection = database[config_old['collection']] + new_dict = mongodict.MongoDict(**config_new) # uses pickle codec by default + total_pairs = collection.count() + start_time = time.time() + for counter, pair in enumerate(collection.find(), start=1): + key, value = pair['_id'], pair['value'] + new_dict[key] = value + if counter % REPORT_INTERVAL == 0: + print_report(counter, total_pairs, start_time) + print_report(counter, total_pairs, start_time) + print('') + +def migrate_key(config_old, config_new): + '''Migrate data from mongodict == 0.3.0 to 0.3.1 + `config_old` and `config_new` should be dictionaries with the keys + regarding to MongoDB server: + - `host` + - `port` + - `database` + - `collection` + ''' + assert mongodict.__version__ == (0, 3, 1) + + connection = pymongo.Connection(host=config_old['host'], + port=config_old['port']) + database = connection[config_old['database']] + collection = database[config_old['collection']] + new_dict = mongodict.MongoDict(**config_new) # uses `v` as default key + total_pairs = collection.count() + start_time = time.time() + for counter, pair in enumerate(collection.find(), start=1): + key, value = pair['_id'], new_dict.decode_value(pair['value']) + new_dict[key] = value + if counter % REPORT_INTERVAL == 0: + print_report(counter, total_pairs, start_time) + print_report(counter, total_pairs, start_time) + print('') + + +def parse_mongo_data(data_as_string): + info = data_as_string.split(':') + port, database, collection = info[1].split('/') + return {'host': info[0], 'port': int(port), 'database': database, + 'collection': collection} + + +def main(): + if mongodict.__version__[1] <= 2: + sys.stderr.write('You need mongodict >= 0.3.0 to run this script.\n') + exit(1) + + if len(sys.argv) < 4: + error_message = ('ERROR: usage: {} <old host:port/db/coll> ' + '<new host:port/db/coll> <codec|key>\n').format(sys.argv[0]) + sys.stderr.write(error_message) + exit(2) + + try: + old_config = parse_mongo_data(sys.argv[1]) + new_config = parse_mongo_data(sys.argv[2]) + except IndexError: + sys.stderr.write('Error parsing MongoDB server data. Please check.\n') + exit(3) + migration_type = sys.argv[3] + if migration_type not in ('codec', 'key'): + sys.stderr.write('Invalid migration type: {}'.format(migration_type)) + exit(4) + + print('Starting migration...') + if migration_type == 'codec': + migrate_codec(old_config, new_config) + elif migration_type == 'key': + migrate_key(old_config, new_config) + + +if __name__ == '__main__': + main() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mongodict-0.3.0/mongodict.py new/mongodict-0.3.1/mongodict.py --- old/mongodict-0.3.0/mongodict.py 2013-05-22 15:15:10.000000000 +0200 +++ new/mongodict-0.3.1/mongodict.py 2014-02-17 03:56:03.000000000 +0100 @@ -28,27 +28,26 @@ from bson import Binary -__version__ = (0, 3, 0) +__version__ = (0, 3, 1) __all__ = ['MongoDict'] +INDEX_KEY = [('_id', 1)] +INDEX_KEY_VALUE = [('_id', 1), ('v', 1)] if sys.version_info[0] == 2: binary_type = str else: binary_type = bytes - def pickle_dumps(value): return pickle.dumps(value, protocol=pickle.HIGHEST_PROTOCOL) - class MongoDict(MutableMapping): ''' ``dict``-like interface for storing data in MongoDB ''' - _index = [('_id', 1), ('value', 1)] def __init__(self, host='localhost', port=27017, database='mongodict', collection='main', codec=(pickle_dumps, pickle.loads), - safe=True, auth=None, default=None): + safe=True, auth=None, default=None, index_type='key'): ''' MongoDB-backed Python ``dict``-like interface Create a new MongoDB connection. @@ -61,6 +60,12 @@ if not self._db.authenticate(*auth): raise ValueError('Cannot authenticate to MongoDB server.') self._collection = self._db[collection] + if index_type == 'key': + self._index = INDEX_KEY + elif index_type == 'key-value': + self._index = INDEX_KEY_VALUE + else: + raise ValueError(u'Error: unknown `index_type`') self._collection.ensure_index(self._index) self.encode_value = lambda value: Binary(codec[0](value)) self.decode_value = lambda value: codec[1](binary_type(value)) @@ -74,7 +79,7 @@ ''' value = self.encode_value(value) return self._collection.update({'_id': key}, - {'_id': key, 'value': value}, + {'_id': key, 'v': value}, upsert=True) def __getitem__(self, key): @@ -84,11 +89,11 @@ If not found, raises ``KeyError``. ''' result = self._collection.find({'_id': key}, - {'value': 1, '_id': 0})\ + {'v': 1, '_id': 0})\ .hint(self._index) if result.count() == 0: raise KeyError(key) - return self.decode_value(result[0]['value']) + return self.decode_value(result[0]['v']) def __delitem__(self, key): ''' Delete the key/value for key ``key`` diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mongodict-0.3.0/setup.py new/mongodict-0.3.1/setup.py --- old/mongodict-0.3.0/setup.py 2013-05-22 15:16:32.000000000 +0200 +++ new/mongodict-0.3.1/setup.py 2014-02-17 03:55:54.000000000 +0100 @@ -10,7 +10,7 @@ author_name = 'Álvaro Justen'.decode('utf-8') setup(name='mongodict', - version='0.3.0', + version='0.3.1', author=author_name, author_email='[email protected]', url='https://github.com/turicas/mongodict/',
