Hi, I am trying to upgrade from calendar server version 4.2 to version 7.0 by starting version 7.0 of the server connected to a database originating from version 4.2. The upgrade process seems to start correctly, but after a while the process is aborted. Some properties in the resource_property table contains UTF-8 characters and that seems to abort the upgrade process and make the server shut down. The following lines are an example from the error log when the problem happens:
2015-12-14 13:14:03+0000 [-] [calendarserver.tap.util#error] Step failure Traceback (most recent call last): File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 442, in errback self._startRunCallbacks(fail) File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 509, in _startRunCallbacks self._runCallbacks() File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 596, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1192, in gotResult _inlineCallbacks(r, g, deferred) --- <exception caught here> --- File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1134, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/share/python/calendarserver/lib/python2.7/site-packages/CalendarServer-7.0.a1_unknown<http://-py2.7.egg/txdav/common/datastore/upgrade/sql/upgrade.py>", line 147, in databaseUpgra de yield self.upgradeVersion(actual_version, required_version, dialect) File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1134, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/share/python/calendarserver/lib/python2.7/site-packages/CalendarServer-7.0.a1_unknown<http://-py2.7.egg/txdav/common/datastore/upgrade/sql/upgrade.py>", line 212, in upgradeVersio n yield self.applyUpgrade(fp) File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1134, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/share/python/calendarserver/lib/python2.7/site-packages/CalendarServer-7.0.a1_unknown<http://-py2.7.egg/txdav/common/datastore/upgrade/sql/upgrade.py>", line 361, in applyUpgrade yield doUpgrade(self.sqlStore) File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1134, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/share/python/calendarserver/lib/python2.7/site-packages/CalendarServer-7.0.a1_unknown<http://-py2.7.egg/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_4_to_5.py>", line 43, in doUpgrade yield updateCalendarHomes(sqlStore, config.UpgradeHomePrefix) File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1134, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/share/python/calendarserver/lib/python2.7/site-packages/CalendarServer-7.0.a1_unknown<http://-py2.7.egg/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_4_to_5.py>", line 60, in updateCalendarHomes yield doToEachHomeNotAtVersion(sqlStore, schema.CALENDAR_HOME, UPGRADE_TO_VERSION, updateCalendarHome, "Update Calendar Home", filterOwnerUID=prefix) File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1134, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/share/python/calendarserver/lib/python2.7/site-packages/CalendarServer-7.0.a1_unknown<http://-py2.7.egg/txdav/common/datastore/upgrade/sql/upgrades/util.py>", line 179, in doToEachHomeNotAtVersion yield doIt(txn, homeResourceID) File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1134, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/python/failure.py", line 389, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/share/python/calendarserver/lib/python2.7/site-packages/CalendarServer-7.0.a1_unknown<http://-py2.7.egg/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_4_to_5.py>", line 71, in updateCalendarHome yield moveCalendarTimezoneProperties(home) File "/usr/share/python/calendarserver/lib/python2.7/site-packages/twisted/internet/defer.py", line 1136, in _inlineCallbacks result = g.send(result) File "/usr/share/python/calendarserver/lib/python2.7/site-packages/CalendarServer-7.0.a1_unknown<http://-py2.7.egg/txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_4_to_5.py>", line 90, in moveCalendarTimezoneProperties prop = calendar.properties().get(PropertyName.fromElement(caldavxml.CalendarTimeZone)) File "/usr/share/python/calendarserver/lib/python2.7/UserDict.py", line 168, in get return self[key] File "/usr/share/python/calendarserver/lib/python2.7/site-packages/CalendarServer-7.0.a1_unknown<http://-py2.7.egg/txdav/base/propertystore/base.py>", line 227, in __getitem__ return self._getitem_uid(key, self._perUser) File "/usr/share/python/calendarserver/lib/python2.7/site-packages/CalendarServer-7.0.a1_unknown<http://-py2.7.egg/txdav/base/propertystore/sql.py>", line 275, in _getitem_uid return WebDAVDocument.fromString(value).root_element File "/usr/share/python/calendarserver/lib/python2.7/site-packages/CalendarServer-7.0.a1_unknown<http://-py2.7.egg/txdav/xml/parser_base.py>", line 43, in fromString source = StringIO(source) exceptions.UnicodeEncodeError: 'ascii' codec can't encode character u'\u2212' in position 348: ordinal not in range(128) The properties that I have found to be related to this problem are the following: CALDAV:calendar-timezone CS:calendar-availability They are both processed in txdav/common/datastore/upgrade/sql/upgrades/calendar_upgrade_from_4_to_5.py. The values of these properties seems to be valid xml data with the encoding attribute set to 'UTF-8', i.e.: <?xml version='1.0' encoding='UTF-8'?> <calendar-timezone xmlns='urn:ietf:params:xml:ns:caldav'><