URL: https://github.com/freeipa/freeipa/pull/393
Author: MartinBasti
 Title: #393: [Py3] allow to run wsgi - part1
Action: synchronized

To pull the PR as Git branch:
git remote add ghfreeipa https://github.com/freeipa/freeipa
git fetch ghfreeipa pull/393/head:pr393
git checkout pr393
From 3fe24269ceebe72c97b2a181e1c522e4968e4d5b Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Mon, 9 Jan 2017 11:53:59 +0100
Subject: [PATCH 01/31] py3: create_cert_db: write to file in a compatible way

Py3 expect bytes to be writed using os.write. Instead of that using
io module is more pythonic.

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaserver/install/httpinstance.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ipaserver/install/httpinstance.py b/ipaserver/install/httpinstance.py
index bacd5fc..ded0553 100644
--- a/ipaserver/install/httpinstance.py
+++ b/ipaserver/install/httpinstance.py
@@ -19,6 +19,7 @@
 
 from __future__ import print_function
 
+import io
 import os
 import os.path
 import pwd
@@ -314,9 +315,8 @@ def create_cert_db(self):
 
         # Create the password file for this db
         password = ipautil.ipa_generate_password()
-        f = os.open(pwd_file, os.O_CREAT | os.O_RDWR)
-        os.write(f, password)
-        os.close(f)
+        with io.open(pwd_file, 'w') as f:
+            f.write(password)
 
         ipautil.run([paths.CERTUTIL, "-d", database, "-f", pwd_file, "-N"])
 

From bce5b4bd3de2c2d98c53695294a661d2c6b2a95f Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Tue, 10 Jan 2017 13:45:11 +0100
Subject: [PATCH 02/31] py3: service.py: replace mkstemp by NamedTemporaryFile

NamedTemporaryfile can be used in more pythonic way and file can be
opened in textual mode that is required with PY3

https://fedorahosted.org/freeipa/ticket/4985
---
 ipapython/ipautil.py         | 2 +-
 ipaserver/install/service.py | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/ipapython/ipautil.py b/ipapython/ipautil.py
index e3e4611..34d10ef 100644
--- a/ipapython/ipautil.py
+++ b/ipapython/ipautil.py
@@ -852,7 +852,7 @@ def ipa_generate_password(entropy_bits=256, uppercase=1, lowercase=1, digits=1,
     rnd = random.SystemRandom()
 
     todo_entropy = entropy_bits
-    password = ''
+    password = u''
     # Generate required character classes:
     # The order of generated characters is fixed to comply with check in
     # NSS function sftk_newPinCheck() in nss/lib/softoken/fipstokn.c.
diff --git a/ipaserver/install/service.py b/ipaserver/install/service.py
index 6451f92..fbe3f23 100644
--- a/ipaserver/install/service.py
+++ b/ipaserver/install/service.py
@@ -208,9 +208,10 @@ def _ldap_mod(self, ldif, sub_dict=None, raise_on_err=True,
         args += ["-H", ldap_uri]
 
         if dm_password:
-            [pw_fd, pw_name] = tempfile.mkstemp()
-            os.write(pw_fd, dm_password)
-            os.close(pw_fd)
+            with tempfile.NamedTemporaryFile(
+                    mode='w', delete=False) as pw_file:
+                pw_file.write(dm_password)
+                pw_name = pw_file.name
             auth_parms = ["-x", "-D", "cn=Directory Manager", "-y", pw_name]
         # Use GSSAPI auth when not using DM password or not being root
         elif os.getegid() != 0:

From bd2df6b979d88d407e455047076f78e932a43f1e Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Mon, 9 Jan 2017 12:42:23 +0100
Subject: [PATCH 03/31] py3: open temporary ldif file in text mode

ldif parser uses file in text mode, so we have to open it in text mode
in py3

Also values passed to parser should be bytes

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaserver/install/dsinstance.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/ipaserver/install/dsinstance.py b/ipaserver/install/dsinstance.py
index 5a28026..ceb7bf3 100644
--- a/ipaserver/install/dsinstance.py
+++ b/ipaserver/install/dsinstance.py
@@ -587,14 +587,15 @@ def __update_dse_ldif(self):
             'dse.ldif'
         )
 
-        with tempfile.NamedTemporaryFile(delete=False) as new_dse_ldif:
+        with tempfile.NamedTemporaryFile(
+                mode='w', delete=False) as new_dse_ldif:
             temp_filename = new_dse_ldif.name
             with open(dse_filename, "r") as input_file:
                 parser = installutils.ModifyLDIF(input_file, new_dse_ldif)
                 parser.replace_value(
                         'cn=config,cn=ldbm database,cn=plugins,cn=config',
                         'nsslapd-db-locks',
-                        ['50000']
+                        [b'50000']
                         )
                 if self.config_ldif:
                     # parse modifications from ldif file supplied by the admin

From d34f4dd4a9ec63ee2129e62bcb1d0b26d6db3aca Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Mon, 9 Jan 2017 19:01:29 +0100
Subject: [PATCH 04/31] py3: ldap modlist must have keys as string, not bytes

https://fedorahosted.org/freeipa/ticket/4985
---
 ipapython/ipaldap.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ipapython/ipaldap.py b/ipapython/ipaldap.py
index daee068..11448f0 100644
--- a/ipapython/ipaldap.py
+++ b/ipapython/ipaldap.py
@@ -886,7 +886,8 @@ def encode(self, val):
         elif isinstance(val, tuple):
             return tuple(self.encode(m) for m in val)
         elif isinstance(val, dict):
-            dct = dict((self.encode(k), self.encode(v)) for k, v in val.items())
+            # key in dict must be str not bytes
+            dct = dict((k, self.encode(v)) for k, v in val.items())
             return dct
         elif isinstance(val, datetime.datetime):
             return val.strftime(LDAP_GENERALIZED_TIME_FORMAT)

From aaf8cf2d7ae62c3936ae5bf954a69b749031c4f6 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Mon, 9 Jan 2017 19:26:04 +0100
Subject: [PATCH 05/31] py3: ipautil: open tempfiles in text mode

Code in ipautlis works with text, so tempfiles should be open in
textmode otherwise TypeErrors are raised

https://fedorahosted.org/freeipa/ticket/4985
---
 ipapython/ipautil.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ipapython/ipautil.py b/ipapython/ipautil.py
index 34d10ef..f2b3d74 100644
--- a/ipapython/ipautil.py
+++ b/ipapython/ipautil.py
@@ -1020,7 +1020,7 @@ def config_replace_variables(filepath, replacevars=dict(), appendvars=dict()):
     orig_stat = os.stat(filepath)
     old_values = dict()
     temp_filename = None
-    with tempfile.NamedTemporaryFile(delete=False) as new_config:
+    with tempfile.NamedTemporaryFile(mode="w", delete=False) as new_config:
         temp_filename = new_config.name
         with open(filepath, 'r') as f:
             for line in f:
@@ -1106,7 +1106,7 @@ def add_options(config, replacevars, appendvars, oldvars):
     orig_stat = os.stat(filepath)
     old_values = dict()
     temp_filename = None
-    with tempfile.NamedTemporaryFile(delete=False) as new_config:
+    with tempfile.NamedTemporaryFile(mode='w', delete=False) as new_config:
         temp_filename = new_config.name
         with open(filepath, 'r') as f:
             in_section = False

From a34e296c38cd3d6d1145fee7dc0e5a14c6520e3b Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Mon, 9 Jan 2017 19:28:57 +0100
Subject: [PATCH 06/31] py3: CA/KRA: config parser requires string

basedn is DN object it has to be converted to string before it can be
used with config parser

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaserver/install/cainstance.py  | 3 ++-
 ipaserver/install/krainstance.py | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/ipaserver/install/cainstance.py b/ipaserver/install/cainstance.py
index 423aa3f..2722194 100644
--- a/ipaserver/install/cainstance.py
+++ b/ipaserver/install/cainstance.py
@@ -33,6 +33,7 @@
 import time
 import tempfile
 
+import six
 # pylint: disable=import-error
 from six.moves.configparser import ConfigParser, RawConfigParser
 # pylint: enable=import-error
@@ -500,7 +501,7 @@ def __spawn_instance(self):
         # Directory server
         config.set("CA", "pki_ds_ldap_port", "389")
         config.set("CA", "pki_ds_password", self.dm_password)
-        config.set("CA", "pki_ds_base_dn", self.basedn)
+        config.set("CA", "pki_ds_base_dn", six.text_type(self.basedn))
         config.set("CA", "pki_ds_database", "ipaca")
 
         if self.use_ldaps:
diff --git a/ipaserver/install/krainstance.py b/ipaserver/install/krainstance.py
index 1f38c86..4f897cb 100644
--- a/ipaserver/install/krainstance.py
+++ b/ipaserver/install/krainstance.py
@@ -22,6 +22,7 @@
 import shutil
 import tempfile
 
+import six
 # pylint: disable=import-error
 from six.moves.configparser import ConfigParser
 # pylint: enable=import-error
@@ -190,7 +191,7 @@ def __spawn_instance(self):
         # Directory server
         config.set("KRA", "pki_ds_ldap_port", "389")
         config.set("KRA", "pki_ds_password", self.dm_password)
-        config.set("KRA", "pki_ds_base_dn", self.basedn)
+        config.set("KRA", "pki_ds_base_dn", six.text_type(self.basedn))
         config.set("KRA", "pki_ds_database", "ipaca")
         config.set("KRA", "pki_ds_create_new_db", "False")
 

From 9a12b369ae99a3a3d3bd9c4dd22ea62c49ffddbd Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Tue, 10 Jan 2017 13:33:41 +0100
Subject: [PATCH 07/31] py3: write CA/KRA config into file opened in text mode

config parser writes data as text so CA/KRA should be opened in textual
mode otherwise type errors are raised from installer

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaserver/install/cainstance.py  | 2 +-
 ipaserver/install/krainstance.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ipaserver/install/cainstance.py b/ipaserver/install/cainstance.py
index 2722194..4c9f967 100644
--- a/ipaserver/install/cainstance.py
+++ b/ipaserver/install/cainstance.py
@@ -599,7 +599,7 @@ def __spawn_instance(self):
         config.set("Tomcat", "pki_ajp_host", "::1")
 
         # Generate configuration file
-        with open(cfg_file, "wb") as f:
+        with open(cfg_file, "w") as f:
             config.write(f)
 
         self.backup_state('installed', True)
diff --git a/ipaserver/install/krainstance.py b/ipaserver/install/krainstance.py
index 4f897cb..95672ee 100644
--- a/ipaserver/install/krainstance.py
+++ b/ipaserver/install/krainstance.py
@@ -263,7 +263,7 @@ def __spawn_instance(self):
                 admin_path.write(cert)
 
         # Generate configuration file
-        with open(cfg_file, "wb") as f:
+        with open(cfg_file, "w") as f:
             config.write(f)
 
         try:

From 08ce2f5e2b293be257f338abdf13d7bc5f54a13a Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Tue, 10 Jan 2017 16:44:46 +0100
Subject: [PATCH 08/31] py3: cainstance: replace mkstemp with
 NamedTemporaryFile

With Python3 files must be opened in textual mode to write text, and
best practise is to use fileobject instead fo os.write() and manual
encodig

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaserver/install/cainstance.py | 49 +++++++++++++++++++++--------------------
 1 file changed, 25 insertions(+), 24 deletions(-)

diff --git a/ipaserver/install/cainstance.py b/ipaserver/install/cainstance.py
index 4c9f967..20677cf 100644
--- a/ipaserver/install/cainstance.py
+++ b/ipaserver/install/cainstance.py
@@ -657,13 +657,12 @@ def import_ra_cert(self, rafile):
         Used when setting up replication
         """
         # Add the new RA cert to the database in /etc/httpd/alias
-        (agent_fd, agent_name) = tempfile.mkstemp()
-        os.write(agent_fd, self.dm_password)
-        os.close(agent_fd)
-        try:
-            import_pkcs12(rafile, agent_name, self.ra_agent_db, self.ra_agent_pwd)
-        finally:
-            os.remove(agent_name)
+        with tempfile.NamedTemporaryFile(mode="w") as agent_file:
+            agent_file.write(self.dm_password)
+            agent_file.flush()
+
+            import_pkcs12(
+                rafile, agent_file.name, self.ra_agent_db, self.ra_agent_pwd)
 
         self.configure_agent_renewal()
 
@@ -759,10 +758,9 @@ def __import_ca_chain(self):
 
         ca_dn = DN(self.ca_subject)
         for cert in certlist:
-            try:
-                chain_fd, chain_name = tempfile.mkstemp()
-                os.write(chain_fd, cert)
-                os.close(chain_fd)
+            with tempfile.NamedTemporaryFile(mode="w") as chain_file:
+                chain_file.write(cert)
+                chain_file.flush()
                 (_rdn, subject_dn) = certs.get_cert_nickname(cert)
                 if subject_dn == ca_dn:
                     nick = get_ca_nickname(self.realm)
@@ -772,10 +770,8 @@ def __import_ca_chain(self):
                     trust_flags = ',,'
                 self.__run_certutil(
                     ['-A', '-t', trust_flags, '-n', nick, '-a',
-                     '-i', chain_name]
+                     '-i', chain_file.name]
                 )
-            finally:
-                os.remove(chain_name)
 
         # Restore NSS trust flags of all previously existing certificates
         for nick, trust_flags in cert_backup_list:
@@ -783,13 +779,15 @@ def __import_ca_chain(self):
 
     def __request_ra_certificate(self):
         # create a temp file storing the pwd
-        (agent_fd, agent_pwdfile) = tempfile.mkstemp(dir=paths.VAR_LIB_IPA)
-        os.write(agent_fd, self.admin_password)
-        os.close(agent_fd)
+        agent_file = tempfile.NamedTemporaryFile(
+            mode="w", dir=paths.VAR_LIB_IPA, delete=False)
+        agent_file.write(self.admin_password)
+        agent_file.close()
 
         # create a temp pem file storing the CA chain
-        (chain_fd, chain_file) = tempfile.mkstemp(dir=paths.VAR_LIB_IPA)
-        os.close(chain_fd)
+        chain_file = tempfile.NamedTemporaryFile(
+            mode="w", dir=paths.VAR_LIB_IPA, delete=False)
+        chain_file.close()
 
         chain = self.__get_ca_chain()
         data = base64.b64decode(chain)
@@ -799,17 +797,17 @@ def __request_ra_certificate(self):
              "-inform",
              "DER",
              "-print_certs",
-             "-out", chain_file,
+             "-out", chain_file.name,
              ], stdin=data, capture_output=False)
 
         agent_args = [paths.DOGTAG_IPA_CA_RENEW_AGENT_SUBMIT,
                       "--dbdir", self.agent_db,
                       "--nickname", "ipa-ca-agent",
-                      "--cafile", chain_file,
+                      "--cafile", chain_file.name,
                       "--ee-url", 'http://%s:8080/ca/ee/ca/' % self.fqdn,
                       "--agent-url",
                       'https://%s:8443/ca/agent/ca/' % self.fqdn,
-                      "--sslpinfile", agent_pwdfile]
+                      "--sslpinfile", agent_file.name]
         helper = " ".join(agent_args)
 
         # configure certmonger renew agent to use temporary agent cert
@@ -842,8 +840,11 @@ def __request_ra_certificate(self):
             certmonger.modify_ca_helper(
                 ipalib.constants.RENEWAL_CA_NAME, old_helper)
             # remove the pwdfile
-            os.remove(agent_pwdfile)
-            os.remove(chain_file)
+            for f in (agent_file, chain_file):
+                try:
+                    os.remove(f.name)
+                except OSError:
+                    pass
 
     def __setup_sign_profile(self):
         # Tell the profile to automatically issue certs for RAs

From b7bc75317c504729427fdffbbf485fe7a33a5200 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Tue, 10 Jan 2017 18:21:13 +0100
Subject: [PATCH 09/31] py3: _httplib_request: don't convert string to bytes

There is no need to encode hostname to bytes. UTF-8 characters must be
encoded in different format in URL anyway and it causes only error in
Py3. String must be unicode to support Py2.

https://fedorahosted.org/freeipa/ticket/4985
---
 ipapython/dogtag.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/ipapython/dogtag.py b/ipapython/dogtag.py
index eb1f73e..37e7a58 100644
--- a/ipapython/dogtag.py
+++ b/ipapython/dogtag.py
@@ -188,9 +188,7 @@ def _httplib_request(
 
     Perform a HTTP(s) request.
     """
-    if isinstance(host, unicode):
-        host = host.encode('utf-8')
-    uri = '%s://%s%s' % (protocol, ipautil.format_netloc(host, port), path)
+    uri = u'%s://%s%s' % (protocol, ipautil.format_netloc(host, port), path)
     root_logger.debug('request %s %s', method, uri)
     root_logger.debug('request body %r', request_body)
 

From 2d4668a5ca9c4921381bc9a95a2056a7fcfd0a5c Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Tue, 10 Jan 2017 18:24:16 +0100
Subject: [PATCH 10/31] py3: HTTPResponse has no 'dict' attribute in 'msg'

There is no 'dict' attribute in 'msg', but 'msg' attribute is dict-like object
in both py2/3, so it can be used instead.

https://fedorahosted.org/freeipa/ticket/4985
---
 ipapython/dogtag.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ipapython/dogtag.py b/ipapython/dogtag.py
index 37e7a58..c6a8346 100644
--- a/ipapython/dogtag.py
+++ b/ipapython/dogtag.py
@@ -205,7 +205,7 @@ def _httplib_request(
         res = conn.getresponse()
 
         http_status = res.status
-        http_headers = res.msg.dict
+        http_headers = res.msg
         http_body = res.read()
         conn.close()
     except Exception as e:

From f438a8a4560f801298af15b43abd0d1821549306 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Wed, 11 Jan 2017 10:23:04 +0100
Subject: [PATCH 11/31] py3: add_entry_to_group: attribute name must be string
 not bytes

With bytes as attribute name pyldap raises type error

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaserver/plugins/ldap2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ipaserver/plugins/ldap2.py b/ipaserver/plugins/ldap2.py
index a04be38..25fbfb8 100644
--- a/ipaserver/plugins/ldap2.py
+++ b/ipaserver/plugins/ldap2.py
@@ -417,7 +417,7 @@ def add_entry_to_group(self, dn, group_dn, member_attr='member', allow_same=Fals
         # update group entry
         try:
             with self.error_handler():
-                modlist = [(a, self.encode(b), self.encode(c))
+                modlist = [(a, b, self.encode(c))
                            for a, b, c in modlist]
                 self.conn.modify_s(str(group_dn), modlist)
         except errors.DatabaseError:

From ac255084f90f731b36bff8a4181ae68f63dbf122 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Wed, 11 Jan 2017 12:35:08 +0100
Subject: [PATCH 12/31] py3: __add_acl: use standard ipaldap methods

Using raw pyldap interface we have to keep vaules as bytes. Is easier to
migrate to ipaldap and use strings without decoding and encoding.

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaserver/install/cainstance.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/ipaserver/install/cainstance.py b/ipaserver/install/cainstance.py
index 20677cf..a73a9c4 100644
--- a/ipaserver/install/cainstance.py
+++ b/ipaserver/install/cainstance.py
@@ -1528,22 +1528,19 @@ def __add_acls(new_rules):
     Return ``True`` if any ACLs were added otherwise ``False``.
 
     """
-    server_id = installutils.realm_to_serverid(api.env.realm)
-    dogtag_uri = 'ldapi://%%2fvar%%2frun%%2fslapd-%s.socket' % server_id
     updated = False
 
     dn = DN(('cn', 'aclResources'), ('o', 'ipaca'))
 
-    conn = ldap2.ldap2(api, ldap_uri=dogtag_uri)
-    if not conn.isconnected():
-        conn.connect(autobind=True)
-    cur_rules = conn.get_entry(dn).get('resourceACLS', [])
+    conn = api.Backend.ldap2
+    entry = conn.get_entry(dn)
+    cur_rules = entry.get('resourceACLS', [])
     add_rules = [rule for rule in new_rules if rule not in cur_rules]
     if add_rules:
-        conn.conn.modify_s(str(dn), [(ldap.MOD_ADD, 'resourceACLS', add_rules)])
+        cur_rules.extend(add_rules)
+        conn.update_entry(entry)
         updated = True
 
-    conn.disconnect()
     return updated
 
 

From 2c514587c6a2096d84e9d237d96ec544a67c2512 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Wed, 11 Jan 2017 13:03:53 +0100
Subject: [PATCH 13/31] py3: make_filter_from_attr: use string instead of bytes

Method escape_filter_chars() requires string as parameter instead of
bytes. 'value_to_utf8' returns bytes thus this code has to be removed.

https://fedorahosted.org/freeipa/ticket/4985
---
 ipapython/ipaldap.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ipapython/ipaldap.py b/ipapython/ipaldap.py
index 11448f0..1dabb71 100644
--- a/ipapython/ipaldap.py
+++ b/ipapython/ipaldap.py
@@ -1246,10 +1246,10 @@ def make_filter_from_attr(
             return cls.combine_filters(flts, rules)
         elif value is not None:
             if isinstance(value, bytes):
-                if six.PY3:
-                    value = value.decode('raw_unicode_escape')
+                value = value.decode('raw_unicode_escape')
             else:
-                value = value_to_utf8(value)
+                value = six.text_type(value)
+            # escape_filter_chars requires string as parameter
             value = ldap.filter.escape_filter_chars(value)
             if not exact:
                 template = '%s'

From 5306810ec5287f7da773ad4fc1cf499510037a52 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Wed, 11 Jan 2017 13:39:04 +0100
Subject: [PATCH 14/31] py3: convert_attribute_members: don't use bytes as
 parameter for DN

due perfomance improvement in e4930b3235e5d61d227a7e43d30a8feb7f35664d
we have to decode value before it can be used in DN() constructor.

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaserver/plugins/baseldap.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ipaserver/plugins/baseldap.py b/ipaserver/plugins/baseldap.py
index 9d6bfc7..e7bf43c 100644
--- a/ipaserver/plugins/baseldap.py
+++ b/ipaserver/plugins/baseldap.py
@@ -654,7 +654,7 @@ def convert_attribute_members(self, entry_attrs, *keys, **options):
             del entry_attrs[attr]
 
             for member in value:
-                memberdn = DN(member)
+                memberdn = DN(member.decode('utf-8'))
                 for ldap_obj_name in self.attribute_members[attr]:
                     ldap_obj = self.api.Object[ldap_obj_name]
                     try:

From 875c3807cd1bacdcda260ff355d056ba0d4b2205 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Wed, 11 Jan 2017 14:38:25 +0100
Subject: [PATCH 15/31] dogtag.py: fix exception logging of JSON data

'read_ca' and 'create_ca' have no logging when exception happened and it
masks real reason why it failed.
---
 ipaserver/plugins/dogtag.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/ipaserver/plugins/dogtag.py b/ipaserver/plugins/dogtag.py
index 73c14ed..142f838 100644
--- a/ipaserver/plugins/dogtag.py
+++ b/ipaserver/plugins/dogtag.py
@@ -2116,16 +2116,20 @@ def create_ca(self, dn):
         )
         try:
             return json.loads(resp_body)
-        except:
-            raise errors.RemoteRetrieveError(reason=_("Response from CA was not valid JSON"))
+        except Exception as e:
+            self.log.debug(e, exc_info=True)
+            raise errors.RemoteRetrieveError(
+                reason=_("Response from CA was not valid JSON"))
 
     def read_ca(self, ca_id):
         _status, _resp_headers, resp_body = self._ssldo(
             'GET', ca_id, headers={'Accept': 'application/json'})
         try:
             return json.loads(resp_body)
-        except:
-            raise errors.RemoteRetrieveError(reason=_("Response from CA was not valid JSON"))
+        except Exception as e:
+            self.log.debug(e, exc_info=True)
+            raise errors.RemoteRetrieveError(
+                reason=_("Response from CA was not valid JSON"))
 
     def read_ca_cert(self, ca_id):
         _status, _resp_headers, resp_body = self._ssldo(

From 4d65321618b6b135ae2129346052bd5956f0a7df Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Thu, 12 Jan 2017 16:20:43 +0100
Subject: [PATCH 16/31] py3: decode bytes for json.loads()

In py 3.5 json.loads requires to have string as input, all bytes must be
decoded.

Note: python 3.6 supports bytes for json.loads()

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaclient/plugins/vault.py  |  2 +-
 ipalib/rpc.py               |  3 ++-
 ipapython/dogtag.py         |  1 +
 ipapython/ipautil.py        | 50 +++++++++++++++++++++++++++++++++++++++++++++
 ipaserver/plugins/dogtag.py |  9 ++++----
 5 files changed, 59 insertions(+), 6 deletions(-)

diff --git a/ipaclient/plugins/vault.py b/ipaclient/plugins/vault.py
index 29157c7..9efb1f1 100644
--- a/ipaclient/plugins/vault.py
+++ b/ipaclient/plugins/vault.py
@@ -969,7 +969,7 @@ def forward(self, *args, **options):
         json_vault_data = decoding_ctx.cipher_op(wrapped_vault_data)\
             + decoding_ctx.digest_final()
 
-        vault_data = json.loads(json_vault_data)
+        vault_data = json.loads(json_vault_data.decode('utf-8'))
         data = base64.b64decode(vault_data[u'data'].encode('utf-8'))
 
         encrypted_key = None
diff --git a/ipalib/rpc.py b/ipalib/rpc.py
index 921f5cb..fb739f8 100644
--- a/ipalib/rpc.py
+++ b/ipalib/rpc.py
@@ -1101,7 +1101,8 @@ def __request(self, name, args):
         )
 
         try:
-            response = json_decode_binary(json.loads(response.decode('ascii')))
+            response = json_decode_binary(
+                json.loads(response.decode('utf-8')))
         except ValueError as e:
             raise JSONError(error=str(e))
 
diff --git a/ipapython/dogtag.py b/ipapython/dogtag.py
index c6a8346..01fc5cb 100644
--- a/ipapython/dogtag.py
+++ b/ipapython/dogtag.py
@@ -209,6 +209,7 @@ def _httplib_request(
         http_body = res.read()
         conn.close()
     except Exception as e:
+        root_logger.exception("httplib request failed:")
         raise NetworkError(uri=uri, error=str(e))
 
     root_logger.debug('response status %d',    http_status)
diff --git a/ipapython/ipautil.py b/ipapython/ipautil.py
index f2b3d74..c8f87ef 100644
--- a/ipapython/ipautil.py
+++ b/ipapython/ipautil.py
@@ -19,6 +19,7 @@
 
 from __future__ import print_function
 
+import codecs
 import string
 import tempfile
 import subprocess
@@ -1361,6 +1362,55 @@ def escape_seq(seq, *args):
     return tuple(a.replace(seq, u'\\{}'.format(seq)) for a in args)
 
 
+def decode_json(data):
+    """Decode JSON bytes to string with proper encoding
+
+    Only for supporting Py 3.5
+
+    Py 3.6 supports bytes as parameter for json.load, we can drop this when
+    there is no need for python 3.5 anymore
+
+    Code from:
+        https://bugs.python.org/file43513/json_detect_encoding_3.patch
+
+    :param data: JSON bytes
+    :return: return JSON string
+    """
+
+    def detect_encoding(b):
+        bstartswith = b.startswith
+        if bstartswith((codecs.BOM_UTF32_BE, codecs.BOM_UTF32_LE)):
+            return 'utf-32'
+        if bstartswith((codecs.BOM_UTF16_BE, codecs.BOM_UTF16_LE)):
+            return 'utf-16'
+        if bstartswith(codecs.BOM_UTF8):
+            return 'utf-8-sig'
+
+        if len(b) >= 4:
+            if not b[0]:
+                # 00 00 -- -- - utf-32-be
+                # 00 XX -- -- - utf-16-be
+                return 'utf-16-be' if b[1] else 'utf-32-be'
+            if not b[1]:
+                # XX 00 00 00 - utf-32-le
+                # XX 00 XX XX - utf-16-le
+                return 'utf-16-le' if b[2] or b[3] else 'utf-32-le'
+        elif len(b) == 2:
+            if not b[0]:
+                # 00 XX - utf-16-be
+                return 'utf-16-be'
+            if not b[1]:
+                # XX 00 - utf-16-le
+                return 'utf-16-le'
+        # default
+        return 'utf-8'
+
+    if isinstance(data, six.text_type):
+        return data
+
+    return data.decode(detect_encoding(data), 'surrogatepass')
+
+
 class APIVersion(tuple):
     """API version parser and handler
 
diff --git a/ipaserver/plugins/dogtag.py b/ipaserver/plugins/dogtag.py
index 142f838..fbfe608 100644
--- a/ipaserver/plugins/dogtag.py
+++ b/ipaserver/plugins/dogtag.py
@@ -1232,7 +1232,8 @@ class ra_certprofile(RestClient):
     @staticmethod
     def _parse_dogtag_error(body):
         try:
-            return pki.PKIException.from_json(json.loads(body))
+            return pki.PKIException.from_json(
+                json.loads(ipautil.decode_json(body)))
         except Exception:
             return None
 
@@ -1667,7 +1668,7 @@ def request_certificate(
         )
 
         try:
-            resp_obj = json.loads(http_body)
+            resp_obj = json.loads(ipautil.decode_json(http_body))
         except ValueError:
             raise errors.RemoteRetrieveError(reason=_("Response from CA was not valid JSON"))
 
@@ -2115,7 +2116,7 @@ def create_ca(self, dn):
             body=json.dumps({"parentID": "host-authority", "dn": unicode(dn)}),
         )
         try:
-            return json.loads(resp_body)
+            return json.loads(ipautil.decode_json(resp_body))
         except Exception as e:
             self.log.debug(e, exc_info=True)
             raise errors.RemoteRetrieveError(
@@ -2125,7 +2126,7 @@ def read_ca(self, ca_id):
         _status, _resp_headers, resp_body = self._ssldo(
             'GET', ca_id, headers={'Accept': 'application/json'})
         try:
-            return json.loads(resp_body)
+            return json.loads(ipautil.decode_json(resp_body))
         except Exception as e:
             self.log.debug(e, exc_info=True)
             raise errors.RemoteRetrieveError(

From 104b23a281d4be63f1c66af0003fa07138a6e820 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Wed, 11 Jan 2017 16:54:25 +0100
Subject: [PATCH 17/31] py3: session.py decode server name to str

This fix is temporal because Memcache will be removed soon, so it is
more workaround than fix

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaserver/session.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ipaserver/session.py b/ipaserver/session.py
index 85deb15..020dcc1 100644
--- a/ipaserver/session.py
+++ b/ipaserver/session.py
@@ -828,7 +828,7 @@ def get_server_statistics(self):
         result = {}
         stats = self.mc.get_stats()
         for server in stats:
-            match = self.mc_server_stat_name_re.search(server[0])
+            match = self.mc_server_stat_name_re.search(server[0].decode())
             if match:
                 name = match.group(1)
                 result[name] = server[1]

From c8d6fe44e635a7485bdf1d9472bbf8bb994bbde0 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Wed, 11 Jan 2017 17:13:52 +0100
Subject: [PATCH 18/31] py3: rpcserver: decode input because json requires
 string

json library parses string so input must be decoded

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaserver/rpcserver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ipaserver/rpcserver.py b/ipaserver/rpcserver.py
index 1da4ec4..7f800ac 100644
--- a/ipaserver/rpcserver.py
+++ b/ipaserver/rpcserver.py
@@ -195,7 +195,7 @@ def read_input(environ):
         length = int(environ.get('CONTENT_LENGTH'))
     except (ValueError, TypeError):
         return
-    return environ['wsgi.input'].read(length)
+    return environ['wsgi.input'].read(length).decode('utf-8')
 
 
 def params_2_args_options(params):

From 4cb606e6aa750be1a7008755e14b1a111e1454e6 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Wed, 11 Jan 2017 17:15:49 +0100
Subject: [PATCH 19/31] Py3: Fix undefined variable

Variable 'e' has only local scope in except block in Py3

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaserver/rpcserver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ipaserver/rpcserver.py b/ipaserver/rpcserver.py
index 7f800ac..306d085 100644
--- a/ipaserver/rpcserver.py
+++ b/ipaserver/rpcserver.py
@@ -404,7 +404,7 @@ def wsgi_execute(self, environ):
                       type(self).__name__,
                       principal,
                       name,
-                      type(e).__name__)
+                      type(error).__name__)
 
         version = options.get('version', VERSION_WITHOUT_CAPABILITIES)
         return self.marshal(result, error, _id, version)

From e5462a3c50e3be467bedab95e0dac964af47d38a Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Wed, 11 Jan 2017 17:24:16 +0100
Subject: [PATCH 20/31] py3: session: fix r/w ccache data

ccache contains binary data, so it should be read and write in binary
mode

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaserver/session.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/ipaserver/session.py b/ipaserver/session.py
index 020dcc1..0f3a9ad 100644
--- a/ipaserver/session.py
+++ b/ipaserver/session.py
@@ -21,6 +21,7 @@
 import os
 import re
 import time
+import io
 
 # pylint: disable=import-error
 from six.moves.urllib.parse import urlparse
@@ -1228,9 +1229,8 @@ def load_ccache_data(ccache_name):
     scheme, name = krb5_parse_ccache(ccache_name)
     if scheme == 'FILE':
         root_logger.debug('reading ccache data from file "%s"', name)
-        src = open(name)
-        ccache_data = src.read()
-        src.close()
+        with io.open(name, "rb") as src:
+            ccache_data = src.read()
         return ccache_data
     else:
         raise ValueError('ccache scheme "%s" unsupported (%s)', scheme, ccache_name)
@@ -1239,9 +1239,8 @@ def bind_ipa_ccache(ccache_data, scheme='FILE'):
     if scheme == 'FILE':
         name = _get_krbccache_pathname()
         root_logger.debug('storing ccache data into file "%s"', name)
-        dst = open(name, 'w')
-        dst.write(ccache_data)
-        dst.close()
+        with io.open(name, 'wb') as dst:
+            dst.write(ccache_data)
     else:
         raise ValueError('ccache scheme "%s" unsupported', scheme)
 

From 4eca41be27c21d021c7df66099d88740289e3a20 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Thu, 12 Jan 2017 18:50:56 +0100
Subject: [PATCH 21/31] py3: WSGI executioners must return bytes in list

WSGI prints TypeError into error log when IPA doesn't return bytes in
list as result

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaserver/rpcserver.py | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/ipaserver/rpcserver.py b/ipaserver/rpcserver.py
index 306d085..8a18f94 100644
--- a/ipaserver/rpcserver.py
+++ b/ipaserver/rpcserver.py
@@ -144,7 +144,7 @@ def not_found(self, environ, start_response, url, message):
         self.info('%s: URL="%s", %s', status, url, message)
         start_response(status, response_headers)
         output = _not_found_template % dict(url=escape(url))
-        return [output]
+        return [output.encode('utf-8')]
 
     def bad_request(self, environ, start_response, message):
         """
@@ -157,7 +157,7 @@ def bad_request(self, environ, start_response, message):
 
         start_response(status, response_headers)
         output = _bad_request_template % dict(message=escape(message))
-        return [output]
+        return [output.encode('utf-8')]
 
     def internal_error(self, environ, start_response, message):
         """
@@ -170,7 +170,7 @@ def internal_error(self, environ, start_response, message):
 
         start_response(status, response_headers)
         output = _internal_error_template % dict(message=escape(message))
-        return [output]
+        return [output.encode('utf-8')]
 
     def unauthorized(self, environ, start_response, message, reason):
         """
@@ -185,7 +185,7 @@ def unauthorized(self, environ, start_response, message, reason):
 
         start_response(status, response_headers)
         output = _unauthorized_template % dict(message=escape(message))
-        return [output]
+        return [output.encode('utf-8')]
 
 def read_input(environ):
     """
@@ -427,7 +427,7 @@ def __call__(self, environ, start_response):
         except Exception:
             self.exception('WSGI %s.__call__():', self.name)
             status = HTTP_STATUS_SERVER_ERROR
-            response = status
+            response = status.encode('utf-8')
             headers = [('Content-Type', 'text/plain; charset=utf-8')]
 
         session_data = getattr(context, 'session_data', None)
@@ -489,7 +489,8 @@ def marshal(self, result, error, _id=None,
             version=unicode(VERSION),
         )
         response = json_encode_binary(response, version)
-        return json.dumps(response, sort_keys=True, indent=4)
+        dump = json.dumps(response, sort_keys=True, indent=4)
+        return dump.encode('utf-8')
 
     def unmarshal(self, data):
         try:
@@ -672,7 +673,7 @@ def __call__(self, environ, start_response):
                     'xmlserver', user_ccache, environ, start_response, headers)
         except PublicError as e:
             status = HTTP_STATUS_SUCCESS
-            response = status
+            response = status.encode('utf-8')
             start_response(status, headers)
             return self.marshal(None, e)
         finally:
@@ -758,7 +759,8 @@ def marshal(self, result, error, _id=None,
             if isinstance(result, dict):
                 self.debug('response: entries returned %d', result.get('count', 1))
             response = (result,)
-        return xml_dumps(response, version, methodresponse=True)
+        dump = xml_dumps(response, version, methodresponse=True)
+        return dump.encode('utf-8')
 
 
 class jsonserver_session(jsonserver, KerberosSession):
@@ -782,7 +784,7 @@ def _on_finalize(self):
     def need_login(self, start_response):
         status = '401 Unauthorized'
         headers = []
-        response = ''
+        response = b''
 
         self.debug('jsonserver_session: %s need login', status)
 
@@ -1252,7 +1254,7 @@ def _on_finalize(self):
     def need_login(self, start_response):
         status = '401 Unauthorized'
         headers = []
-        response = ''
+        response = b''
 
         self.debug('xmlserver_session: %s need login', status)
 

From 11ca1126d6e77ee786a1c049c132e2a42017c892 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Fri, 13 Jan 2017 12:11:19 +0100
Subject: [PATCH 22/31] py3: rpcserver fix undefined variable

variable 'e' is valid only in except block in py3, so it must be
assigned to different variable for further usage

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaserver/rpcserver.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ipaserver/rpcserver.py b/ipaserver/rpcserver.py
index 8a18f94..45550fb 100644
--- a/ipaserver/rpcserver.py
+++ b/ipaserver/rpcserver.py
@@ -389,8 +389,9 @@ def wsgi_execute(self, environ):
                 )
                 # get at least some context of what is going on
                 params = options
+                error = e
             if error:
-                result_string = type(e).__name__
+                result_string = type(error).__name__
             else:
                 result_string = 'SUCCESS'
             self.info('[%s] %s: %s(%s): %s',

From 4d2d9e5adc06508c7979e3c21cac3d2a5db4bbf1 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Fri, 13 Jan 2017 12:16:28 +0100
Subject: [PATCH 23/31] py3: ipaldap: update encode/decode methods

Update encoding/decoding accordingly to work under Py3

Removing functions that were used only once in code and give no real
improvements

https://fedorahosted.org/freeipa/ticket/4985
---
 ipapython/ipaldap.py | 41 +++++++----------------------------------
 1 file changed, 7 insertions(+), 34 deletions(-)

diff --git a/ipapython/ipaldap.py b/ipapython/ipaldap.py
index 1dabb71..cc0c1c7 100644
--- a/ipapython/ipaldap.py
+++ b/ipapython/ipaldap.py
@@ -68,36 +68,6 @@
 DIRMAN_DN = DN(('cn', 'directory manager'))
 
 
-def unicode_from_utf8(val):
-    '''
-    val is a UTF-8 encoded string, return a unicode object.
-    '''
-    return val.decode('utf-8')
-
-
-def value_to_utf8(val):
-    '''
-    Coerce the val parameter to a UTF-8 encoded string representation
-    of the val.
-    '''
-
-    # If val is not a string we need to convert it to a string
-    # (specifically a unicode string). Naively we might think we need to
-    # call str(val) to convert to a string. This is incorrect because if
-    # val is already a unicode object then str() will call
-    # encode(default_encoding) returning a str object encoded with
-    # default_encoding. But we don't want to apply the default_encoding!
-    # Rather we want to guarantee the val object has been converted to a
-    # unicode string because from a unicode string we want to explicitly
-    # encode to a str using our desired encoding (utf-8 in this case).
-    #
-    # Note: calling unicode on a unicode object simply returns the exact
-    # same object (with it's ref count incremented). This means calling
-    # unicode on a unicode object is effectively a no-op, thus it's not
-    # inefficient.
-
-    return unicode(val).encode('utf-8')
-
 class _ServerSchema(object):
     '''
     Properties of a schema retrieved from an LDAP server.
@@ -876,7 +846,7 @@ def encode(self, val):
                 return 'FALSE'
         elif isinstance(val, (unicode, six.integer_types, Decimal, DN,
                               Principal)):
-            return value_to_utf8(val)
+            return six.text_type(val).encode('utf-8')
         elif isinstance(val, DNSName):
             return val.to_text()
         elif isinstance(val, bytes):
@@ -908,9 +878,12 @@ def decode(self, val, attr):
                 elif target_type is unicode:
                     return val.decode('utf-8')
                 elif target_type is datetime.datetime:
-                    return datetime.datetime.strptime(val, LDAP_GENERALIZED_TIME_FORMAT)
+                    return datetime.datetime.strptime(
+                        val.decode('utf-8'), LDAP_GENERALIZED_TIME_FORMAT)
                 elif target_type is DNSName:
-                    return DNSName.from_text(val)
+                    return DNSName.from_text(val.decode('utf-8'))
+                elif target_type in (DN, Principal):
+                    return target_type(val.decode('utf-8'))
                 else:
                     return target_type(val)
             except Exception:
@@ -922,7 +895,7 @@ def decode(self, val, attr):
         elif isinstance(val, tuple):
             return tuple(self.decode(m, attr) for m in val)
         elif isinstance(val, dict):
-            dct = dict((unicode_from_utf8(k), self.decode(v, k)) for k, v in val.items())
+            dct = dict((k.decode('utf-8'), self.decode(v, k)) for k, v in val.items())
             return dct
         elif val is None:
             return None

From 6a0324583c89c8b886791933198a4ccd75e5e0ef Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Fri, 13 Jan 2017 12:37:47 +0100
Subject: [PATCH 24/31] py3: get_effective_rights: values passed to ldap must
 be bytes

Values passed to LDAP must be bytes

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaserver/plugins/ldap2.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/ipaserver/plugins/ldap2.py b/ipaserver/plugins/ldap2.py
index 25fbfb8..c4b7580 100644
--- a/ipaserver/plugins/ldap2.py
+++ b/ipaserver/plugins/ldap2.py
@@ -289,7 +289,10 @@ def get_effective_rights(self, dn, attrs_list):
         principal = getattr(context, 'principal')
         entry = self.find_entry_by_attr("krbprincipalname", principal,
             "krbPrincipalAux", base_dn=self.api.env.basedn)
-        sctrl = [GetEffectiveRightsControl(True, "dn: " + str(entry.dn))]
+        sctrl = [
+            GetEffectiveRightsControl(
+                True, "dn: {0}".format(entry.dn).encode('utf-8'))
+        ]
         self.conn.set_option(_ldap.OPT_SERVER_CONTROLS, sctrl)
         try:
             entry = self.get_entry(dn, attrs_list)

From 73ca10c0c7fb99a7d15c8319e7bc38008bcad2f5 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Fri, 13 Jan 2017 12:40:20 +0100
Subject: [PATCH 25/31] py3: can_read: attributelevelrights is already string

Remove decode() as it causes error in py3 because the attribute is
already string not bytes

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaserver/plugins/ldap2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ipaserver/plugins/ldap2.py b/ipaserver/plugins/ldap2.py
index c4b7580..71c095d 100644
--- a/ipaserver/plugins/ldap2.py
+++ b/ipaserver/plugins/ldap2.py
@@ -310,7 +310,7 @@ def can_write(self, dn, attr):
 
         attrs = self.get_effective_rights(dn, [attr])
         if 'attributelevelrights' in attrs:
-            attr_rights = attrs.get('attributelevelrights')[0].decode('UTF-8')
+            attr_rights = attrs.get('attributelevelrights')[0]
             (attr, rights) = attr_rights.split(':')
             if 'w' in rights:
                 return True

From 6b5026605cf051c2a8f10829cf63ca6f0034bd03 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Fri, 13 Jan 2017 13:04:07 +0100
Subject: [PATCH 26/31] Use dict comprehension

---
 ipapython/ipaldap.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ipapython/ipaldap.py b/ipapython/ipaldap.py
index cc0c1c7..26fb69c 100644
--- a/ipapython/ipaldap.py
+++ b/ipapython/ipaldap.py
@@ -895,7 +895,9 @@ def decode(self, val, attr):
         elif isinstance(val, tuple):
             return tuple(self.decode(m, attr) for m in val)
         elif isinstance(val, dict):
-            dct = dict((k.decode('utf-8'), self.decode(v, k)) for k, v in val.items())
+            dct = {
+                k.decode('utf-8'): self.decode(v, k) for k, v in val.items()
+            }
             return dct
         elif val is None:
             return None

From 0280fa281cac7d7f6044309120ed5b0c485b68c5 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Fri, 13 Jan 2017 14:50:11 +0100
Subject: [PATCH 27/31] Principal: validate type of input parameter

Bytes are unsupported and we should raise a TypeError from Principal
__init__ method otherwise we get hard to debug result
---
 ipapython/kerberos.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/ipapython/kerberos.py b/ipapython/kerberos.py
index 3d3530c..9b02790 100644
--- a/ipapython/kerberos.py
+++ b/ipapython/kerberos.py
@@ -66,7 +66,12 @@ class Principal(object):
     Container for the principal name and realm according to RFC 1510
     """
     def __init__(self, components, realm=None):
-        if isinstance(components, six.string_types):
+        if isinstance(components, six.binary_type):
+            raise TypeError(
+                "Cannot create a principal object from bytes: {!r}".format(
+                    components)
+            )
+        elif isinstance(components, six.string_types):
             # parse principal components from realm
             self.components, self.realm = self._parse_from_text(
                 components, realm)

From 6823124b0f5e6fc1e5a6df506bd47192a3d59e8c Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Fri, 13 Jan 2017 18:54:34 +0100
Subject: [PATCH 28/31] py3: fix CSR encoding inside framework

csr must be in string because framework excpects only strings, so we
have to decode it back

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaserver/plugins/cert.py   | 4 +++-
 ipaserver/plugins/dogtag.py | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/ipaserver/plugins/cert.py b/ipaserver/plugins/cert.py
index d8bfc1c..4d3f1cf 100644
--- a/ipaserver/plugins/cert.py
+++ b/ipaserver/plugins/cert.py
@@ -804,7 +804,9 @@ def execute(self, csr, all=False, raw=False, **kw):
         try:
             # re-serialise to PEM, in case the user-supplied data has
             # extraneous material that will cause Dogtag to freak out
-            csr_pem = csr_obj.public_bytes(serialization.Encoding.PEM)
+            # keep it as string not bytes, it is required later
+            csr_pem = csr_obj.public_bytes(
+                serialization.Encoding.PEM).decode('utf-8')
             result = self.Backend.ra.request_certificate(
                 csr_pem, profile_id, ca_id, request_type=request_type)
         except errors.HTTPRequestError as e:
diff --git a/ipaserver/plugins/dogtag.py b/ipaserver/plugins/dogtag.py
index fbfe608..7ea1b56 100644
--- a/ipaserver/plugins/dogtag.py
+++ b/ipaserver/plugins/dogtag.py
@@ -1634,7 +1634,7 @@ def request_certificate(
         self.debug('%s.request_certificate()', type(self).__name__)
 
         # Call CMS
-        template = '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+        template = u'''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
             <CertEnrollmentRequest>
                 <ProfileID>{profile}</ProfileID>
                 <Input id="i1">

From 27b774006a4acdef6bb1952215fbe5f52476d987 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Thu, 19 Jan 2017 14:30:23 +0100
Subject: [PATCH 29/31] py3: fingerprint_hex_sha256: fix encoding/decoding

https://fedorahosted.org/freeipa/ticket/4985
---
 ipapython/ssh.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/ipapython/ssh.py b/ipapython/ssh.py
index 57752ae..2edfa8a 100644
--- a/ipapython/ssh.py
+++ b/ipapython/ssh.py
@@ -192,9 +192,8 @@ def openssh(self):
 
     def fingerprint_hex_sha256(self):
         # OpenSSH trims the trailing '=' of base64 sha256 FP representation
-        # Using unicode argument converts the result to unicode object
-        fp = base64.b64encode(sha256(self._key).digest()).rstrip(u'=')
-        return 'SHA256:{fp}'.format(fp=fp)
+        fp = base64.b64encode(sha256(self._key).digest()).rstrip(b'=')
+        return u'SHA256:{fp}'.format(fp=fp.decode('utf-8'))
 
     def _fingerprint_dns(self, fpfunc, fptype):
         if self._keytype == 'ssh-rsa':

From 79055cf864f9cd8bcb94c94373f90ec4531f8c51 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Thu, 19 Jan 2017 15:28:15 +0100
Subject: [PATCH 30/31] py3: strip_header: support both bytes and unicode

Various method passed various bytes or unicode as parameter

https://fedorahosted.org/freeipa/ticket/4985
---
 ipalib/x509.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/ipalib/x509.py b/ipalib/x509.py
index 13327c1..bc25390 100644
--- a/ipalib/x509.py
+++ b/ipalib/x509.py
@@ -85,9 +85,16 @@ def strip_header(pem):
     """
     Remove the header and footer from a certificate.
     """
-    s = pem.find("-----BEGIN CERTIFICATE-----")
+    if isinstance(pem, bytes):
+        re_begin = b"-----BEGIN CERTIFICATE-----"
+        re_end = b"-----END CERTIFICATE-----"
+    else:
+        re_begin = u"-----BEGIN CERTIFICATE-----"
+        re_end = u"-----END CERTIFICATE-----"
+
+    s = pem.find(re_begin)
     if s >= 0:
-        e = pem.find("-----END CERTIFICATE-----")
+        e = pem.find(re_end)
         pem = pem[s+27:e]
 
     return pem

From cba4eb63687fdc5480994dce5de7625d1e0ff9e8 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Thu, 19 Jan 2017 16:11:08 +0100
Subject: [PATCH 31/31] py3: normalize_certificate: support both bytes and
 unicode

https://fedorahosted.org/freeipa/ticket/4985
---
 ipalib/x509.py | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/ipalib/x509.py b/ipalib/x509.py
index bc25390..83b9330 100644
--- a/ipalib/x509.py
+++ b/ipalib/x509.py
@@ -254,13 +254,22 @@ def normalize_certificate(rawcert):
 
     rawcert = strip_header(rawcert)
 
-    if util.isvalid_base64(rawcert):
-        try:
-            dercert = base64.b64decode(rawcert)
-        except Exception as e:
-            raise errors.Base64DecodeError(reason=str(e))
-    else:
+    try:
+        if isinstance(rawcert, bytes):
+            # base64 must work with utf-8, otherwise it is raw bin certificate
+            decoded_cert = rawcert.decode('utf-8')
+        else:
+            decoded_cert = rawcert
+    except UnicodeDecodeError:
         dercert = rawcert
+    else:
+        if util.isvalid_base64(decoded_cert):
+            try:
+                dercert = base64.b64decode(decoded_cert)
+            except Exception as e:
+                raise errors.Base64DecodeError(reason=str(e))
+        else:
+            dercert = rawcert
 
     # At this point we should have a DER certificate.
     # Attempt to decode it.
-- 
Manage your subscription for the Freeipa-devel mailing list:
https://www.redhat.com/mailman/listinfo/freeipa-devel
Contribute to FreeIPA: http://www.freeipa.org/page/Contribute/Code

Reply via email to