URL: https://github.com/freeipa/freeipa/pull/460 Author: MartinBasti Title: #460: [Py3] ipa-server-install, ipa-server-upgrade fixes Action: opened
PR body: """ ipa-server-install --setup-dns now work without BytesWarnings under python3, ipa-server-upgrade should work on IPA side but there are issues on pyldap side. """ To pull the PR as Git branch: git remote add ghfreeipa https://github.com/freeipa/freeipa git fetch ghfreeipa pull/460/head:pr460 git checkout pr460
From 28e19fd55154ad588dffe09a208fa03e394e1dca Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Fri, 10 Feb 2017 17:05:02 +0100 Subject: [PATCH 1/8] py3: use ConfigParser instead of SafeConfigParser DeprecationWarning: The SafeConfigParser class has been renamed to ConfigParser in Python 3.2. This alias will be removed in future versions. Use ConfigParser directly instead. https://fedorahosted.org/freeipa/ticket/4985 --- ipalib/install/sysrestore.py | 6 +++++- ipaserver/install/installutils.py | 7 ++++++- ipaserver/install/ipa_backup.py | 7 ++++++- ipaserver/install/ipa_replica_prepare.py | 7 ++++++- ipaserver/install/ipa_restore.py | 7 ++++++- ipaserver/install/server/upgrade.py | 6 +++++- 6 files changed, 34 insertions(+), 6 deletions(-) diff --git a/ipalib/install/sysrestore.py b/ipalib/install/sysrestore.py index b1bf4b9..5c21956 100644 --- a/ipalib/install/sysrestore.py +++ b/ipalib/install/sysrestore.py @@ -31,7 +31,11 @@ import six # pylint: disable=import-error -from six.moves.configparser import SafeConfigParser +if six.PY3: + # The SafeConfigParser class has been renamed to ConfigParser in Py3 + from configparser import ConfigParser as SafeConfigParser +else: + from ConfigParser import SafeConfigParser # pylint: enable=import-error from ipaplatform.tasks import tasks diff --git a/ipaserver/install/installutils.py b/ipaserver/install/installutils.py index ab2596c..a774200 100644 --- a/ipaserver/install/installutils.py +++ b/ipaserver/install/installutils.py @@ -41,7 +41,12 @@ import ldapurl import six # pylint: disable=import-error -from six.moves.configparser import SafeConfigParser, NoOptionError +if six.PY3: + # The SafeConfigParser class has been renamed to ConfigParser in Py3 + from configparser import ConfigParser as SafeConfigParser +else: + from ConfigParser import SafeConfigParser +from six.moves.configparser import NoOptionError # pylint: enable=import-error from ipalib.install import sysrestore diff --git a/ipaserver/install/ipa_backup.py b/ipaserver/install/ipa_backup.py index c11120b..1dd8044 100644 --- a/ipaserver/install/ipa_backup.py +++ b/ipaserver/install/ipa_backup.py @@ -23,8 +23,13 @@ import time import pwd +import six # pylint: disable=import-error -from six.moves.configparser import SafeConfigParser +if six.PY3: + # The SafeConfigParser class has been renamed to ConfigParser in Py3 + from configparser import ConfigParser as SafeConfigParser +else: + from ConfigParser import SafeConfigParser # pylint: enable=import-error from ipaplatform.paths import paths diff --git a/ipaserver/install/ipa_replica_prepare.py b/ipaserver/install/ipa_replica_prepare.py index e7070b6..8b24c39 100644 --- a/ipaserver/install/ipa_replica_prepare.py +++ b/ipaserver/install/ipa_replica_prepare.py @@ -30,8 +30,13 @@ # pylint: enable=deprecated-module import dns.resolver +import six # pylint: disable=import-error -from six.moves.configparser import SafeConfigParser +if six.PY3: + # The SafeConfigParser class has been renamed to ConfigParser in Py3 + from configparser import ConfigParser as SafeConfigParser +else: + from ConfigParser import SafeConfigParser # pylint: enable=import-error from ipaserver.install import certs, installutils, bindinstance, dsinstance diff --git a/ipaserver/install/ipa_restore.py b/ipaserver/install/ipa_restore.py index 89cf9e6..42dd03e 100644 --- a/ipaserver/install/ipa_restore.py +++ b/ipaserver/install/ipa_restore.py @@ -25,8 +25,13 @@ import ldif import itertools +import six # pylint: disable=import-error -from six.moves.configparser import SafeConfigParser +if six.PY3: + # The SafeConfigParser class has been renamed to ConfigParser in Py3 + from configparser import ConfigParser as SafeConfigParser +else: + from ConfigParser import SafeConfigParser # pylint: enable=import-error from ipaclient.install.client import update_ipa_nssdb diff --git a/ipaserver/install/server/upgrade.py b/ipaserver/install/server/upgrade.py index 0e034ef..5413b48 100644 --- a/ipaserver/install/server/upgrade.py +++ b/ipaserver/install/server/upgrade.py @@ -15,7 +15,11 @@ import six # pylint: disable=import-error -from six.moves.configparser import SafeConfigParser +if six.PY3: + # The SafeConfigParser class has been renamed to ConfigParser in Py3 + from configparser import ConfigParser as SafeConfigParser +else: + from ConfigParser import SafeConfigParser # pylint: enable=import-error from ipalib import api From a6b9b5aa2687160e113a37e369a20c1899032f5c Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Fri, 10 Feb 2017 17:13:15 +0100 Subject: [PATCH 2/8] py3: ipaldap: encode Boolean as bytes Python LDAP requires bytes https://fedorahosted.org/freeipa/ticket/4985 --- ipapython/ipaldap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ipapython/ipaldap.py b/ipapython/ipaldap.py index 82d45b9..a579197 100644 --- a/ipapython/ipaldap.py +++ b/ipapython/ipaldap.py @@ -844,9 +844,9 @@ def encode(self, val): # entered for a boolean value instead of the boolean clause. if isinstance(val, bool): if val: - return 'TRUE' + return b'TRUE' else: - return 'FALSE' + return b'FALSE' elif isinstance(val, (unicode, six.integer_types, Decimal, DN, Principal)): return six.text_type(val).encode('utf-8') From bd6696416e73a55db9f987d8b7202af499990a92 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Fri, 10 Feb 2017 17:36:19 +0100 Subject: [PATCH 3/8] py3: softhsm key_id must be bytes softhsm works with bytes, so key_id must be byte otherwise we get errors from bytes and string comparison https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/install/dnskeysyncinstance.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ipaserver/install/dnskeysyncinstance.py b/ipaserver/install/dnskeysyncinstance.py index 861a170..603df43 100644 --- a/ipaserver/install/dnskeysyncinstance.py +++ b/ipaserver/install/dnskeysyncinstance.py @@ -283,8 +283,7 @@ def __setup_replica_keys(self): while True: # check if key with this ID exist in softHSM # id is 16 Bytes long - key_id = "".join(chr(random.randint(0, 255)) - for _ in range(0, 16)) + key_id = bytes(random.randint(0, 255) for _ in range(0, 16)) replica_pubkey_dn = DN(('ipk11UniqueId', 'autogenerate'), dn_base) From c998e2b0e5b4ccd63b02ed68f25822872d83905c Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Fri, 10 Feb 2017 19:01:12 +0100 Subject: [PATCH 4/8] [WIP] py3: LDAP updates: use only bytes/raw values Functions mix unicode and bytes, use only bytes. https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/install/ldapupdate.py | 29 ++++++++++++++++++++--------- ipaserver/install/schemaupdate.py | 3 ++- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/ipaserver/install/ldapupdate.py b/ipaserver/install/ldapupdate.py index c6ab3e2..f52328f 100644 --- a/ipaserver/install/ldapupdate.py +++ b/ipaserver/install/ldapupdate.py @@ -609,14 +609,15 @@ def _create_default_entry(self, dn, default): attr = item['attr'] value = item['value'] - e = entry.get(attr) + e = entry.raw.get(attr) if e: # multi-valued attribute e = list(e) e.append(value) else: e = [value] - entry[attr] = e + + entry.raw[attr] = e entry.reset_modlist() return entry @@ -650,6 +651,16 @@ def _apply_update_disposition(self, updates, entry): attr = update['attr'] update_value = update['value'] + # do not mix comparison of bytes and unicode, everything in this + # function should be compared as bytes + if isinstance(update_value, (list, tuple)): + update_value = [ + v.encode('utf-8') if isinstance(v, unicode) else v + for v in update_value + ] + elif isinstance(update_value, unicode): + update_value = update_value.encode('utf-8') + entry_values = entry.raw.get(attr, []) if action == 'remove': self.debug("remove: '%s' from %s, current value %s", safe_output(attr, update_value), attr, safe_output(attr,entry_values)) @@ -660,7 +671,7 @@ def _apply_update_disposition(self, updates, entry): "remove: '%s' not in %s", safe_output(attr, update_value), attr) else: - entry[attr] = entry_values + entry.raw[attr] = entry_values self.debug('remove: updated value %s', safe_output( attr, entry_values)) elif action == 'add': @@ -672,7 +683,7 @@ def _apply_update_disposition(self, updates, entry): pass entry_values.append(update_value) self.debug('add: updated value %s', safe_output(attr, entry_values)) - entry[attr] = entry_values + entry.raw[attr] = entry_values elif action == 'addifnew': self.debug("addifnew: '%s' to %s, current value %s", safe_output(attr, update_value), attr, safe_output(attr, entry_values)) # Only add the attribute if it doesn't exist. Only works @@ -680,7 +691,7 @@ def _apply_update_disposition(self, updates, entry): if entry.get('objectclass') and len(entry_values) == 0: entry_values.append(update_value) self.debug('addifnew: set %s to %s', attr, safe_output(attr, entry_values)) - entry[attr] = entry_values + entry.raw[attr] = entry_values elif action == 'addifexist': self.debug("addifexist: '%s' to %s, current value %s", safe_output(attr, update_value), attr, safe_output(attr, entry_values)) # Only add the attribute if the entry doesn't exist. We @@ -688,7 +699,7 @@ def _apply_update_disposition(self, updates, entry): if entry.get('objectclass'): entry_values.append(update_value) self.debug('addifexist: set %s to %s', attr, safe_output(attr, entry_values)) - entry[attr] = entry_values + entry.raw[attr] = entry_values elif action == 'only': self.debug("only: set %s to '%s', current value %s", attr, safe_output(attr, update_value), safe_output(attr, entry_values)) if only.get(attr): @@ -696,7 +707,7 @@ def _apply_update_disposition(self, updates, entry): else: entry_values = [update_value] only[attr] = True - entry[attr] = entry_values + entry.raw[attr] = entry_values self.debug('only: updated value %s', safe_output(attr, entry_values)) elif action == 'onlyifexist': self.debug("onlyifexist: '%s' to %s, current value %s", safe_output(attr, update_value), attr, safe_output(attr, entry_values)) @@ -709,7 +720,7 @@ def _apply_update_disposition(self, updates, entry): entry_values = [update_value] only[attr] = True self.debug('onlyifexist: set %s to %s', attr, safe_output(attr, entry_values)) - entry[attr] = entry_values + entry.raw[attr] = entry_values elif action == 'deleteentry': # skip this update type, it occurs in __delete_entries() return None @@ -724,7 +735,7 @@ def _apply_update_disposition(self, updates, entry): else: entry_values.append(new) self.debug('replace: updated value %s', safe_output(attr, entry_values)) - entry[attr] = entry_values + entry.raw[attr] = entry_values return entry diff --git a/ipaserver/install/schemaupdate.py b/ipaserver/install/schemaupdate.py index 468a834..59b65f5 100644 --- a/ipaserver/install/schemaupdate.py +++ b/ipaserver/install/schemaupdate.py @@ -143,7 +143,6 @@ def update_schema(schema_files, ldapi=False, dm_password=None,): # Note: An add will automatically replace any existing # schema with the same OID. So, we only add. value = add_x_origin(new_obj) - new_elements.append(value) if old_obj: old_attr = old_entries_by_oid.get(oid) @@ -152,6 +151,8 @@ def update_schema(schema_files, ldapi=False, dm_password=None,): else: log.debug('Add: %s', value) + new_elements.append(value.encode('utf-8')) + modified = modified or new_elements schema_entry[attrname].extend(new_elements) # we need to iterate schema updates, due to dependencies (SUP) From 232bcfc658f132f622f30b64bcc098c47044cda2 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Fri, 10 Feb 2017 19:02:35 +0100 Subject: [PATCH 5/8] py3: schemaupdate: fix BytesWarning str() was called on bytes https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/install/schemaupdate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipaserver/install/schemaupdate.py b/ipaserver/install/schemaupdate.py index 59b65f5..cecde30 100644 --- a/ipaserver/install/schemaupdate.py +++ b/ipaserver/install/schemaupdate.py @@ -119,7 +119,7 @@ def update_schema(schema_files, ldapi=False, dm_password=None,): # The exact representation the DS gives us for each OID # (for debug logging) - old_entries_by_oid = {cls(str(attr)).oid: str(attr) + old_entries_by_oid = {cls(attr.decode('utf-8')).oid: attr.decode('utf-8') for (attrname, cls) in SCHEMA_ELEMENT_CLASSES for attr in schema_entry[attrname]} From 6cad1911c8ac0ac39babdaa6b25ddf34e5ae6289 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Mon, 13 Feb 2017 17:14:51 +0100 Subject: [PATCH 6/8] py3: cainstance: fix BytesWarning https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/install/cainstance.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/ipaserver/install/cainstance.py b/ipaserver/install/cainstance.py index d869641..b170034 100644 --- a/ipaserver/install/cainstance.py +++ b/ipaserver/install/cainstance.py @@ -1623,11 +1623,16 @@ def repair_profile_caIPAserviceCert(): return indicators = [ - "policyset.serverCertSet.1.default.params.name=" - "CN=$request.req_subject_name.cn$, OU=pki-ipa, O=IPA ", - "policyset.serverCertSet.9.default.params.crlDistPointsPointName_0=" - "https://ipa.example.com/ipa/crl/MasterCRL.bin";, - ] + ( + b"policyset.serverCertSet.1.default.params.name=" + b"CN=$request.req_subject_name.cn$, OU=pki-ipa, O=IPA " + ), + ( + b"policyset.serverCertSet.9.default.params." + b"crlDistPointsPointName_0=" + b"https://ipa.example.com/ipa/crl/MasterCRL.bin"; + ), + ] need_repair = all(l in cur_config for l in indicators) if need_repair: From a08f9ddff879755b7e284509d3f06e47f8225c11 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Mon, 13 Feb 2017 17:33:21 +0100 Subject: [PATCH 7/8] py3: urlfetch: use "file://" prefix with filenames with py3 urlopen used internally with pyldap doesn't work with raw filepaths without specifying "file://" prefix. This works on both py2/py3 https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/install/schemaupdate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ipaserver/install/schemaupdate.py b/ipaserver/install/schemaupdate.py index cecde30..935e576 100644 --- a/ipaserver/install/schemaupdate.py +++ b/ipaserver/install/schemaupdate.py @@ -125,7 +125,8 @@ def update_schema(schema_files, ldapi=False, dm_password=None,): for filename in schema_files: log.debug('Processing schema LDIF file %s', filename) - _dn, new_schema = ldap.schema.subentry.urlfetch(filename) + url = "file://{}".format(filename) + _dn, new_schema = ldap.schema.subentry.urlfetch(url) for attrname, cls in SCHEMA_ELEMENT_CLASSES: for oids_set in _get_oid_dependency_order(new_schema, cls): From 40ac4dd2e59551b9215997d00fb5ff8f566404d1 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Mon, 13 Feb 2017 17:34:19 +0100 Subject: [PATCH 8/8] py3: update_mod_nss_cipher_suite: ordering doesn't work with None Py3 doesn't support ordering with None value https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/install/server/upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipaserver/install/server/upgrade.py b/ipaserver/install/server/upgrade.py index 5413b48..4cc93ff 100644 --- a/ipaserver/install/server/upgrade.py +++ b/ipaserver/install/server/upgrade.py @@ -1405,7 +1405,7 @@ def update_mod_nss_cipher_suite(http): root_logger.info('[Updating mod_nss cipher suite]') revision = sysupgrade.get_upgrade_state('nss.conf', 'cipher_suite_updated') - if revision >= httpinstance.NSS_CIPHER_REVISION: + if revision and revision >= httpinstance.NSS_CIPHER_REVISION: root_logger.debug("Cipher suite already updated") return
-- Manage your subscription for the Freeipa-devel mailing list: https://www.redhat.com/mailman/listinfo/freeipa-devel Contribute to FreeIPA: http://www.freeipa.org/page/Contribute/Code
