URL: https://github.com/freeipa/freeipa/pull/382
Author: MartinBasti
 Title: #382: [Py3] ipa-server-install fixes (working NTP, DS, CA install steps)
Action: synchronized

To pull the PR as Git branch:
git remote add ghfreeipa https://github.com/freeipa/freeipa
git fetch ghfreeipa pull/382/head:pr382
git checkout pr382
From 79df0fa37b859ff78e38221efc81b7863b00329a Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Mon, 9 Jan 2017 11:53:59 +0100
Subject: [PATCH 01/16] py3: create_cert_db: write to file in a compatible way

Py3 expect bytes to be writed using os.write. Instead of that using
io module is more pythonic.

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaserver/install/httpinstance.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ipaserver/install/httpinstance.py b/ipaserver/install/httpinstance.py
index bacd5fc..ded0553 100644
--- a/ipaserver/install/httpinstance.py
+++ b/ipaserver/install/httpinstance.py
@@ -19,6 +19,7 @@
 
 from __future__ import print_function
 
+import io
 import os
 import os.path
 import pwd
@@ -314,9 +315,8 @@ def create_cert_db(self):
 
         # Create the password file for this db
         password = ipautil.ipa_generate_password()
-        f = os.open(pwd_file, os.O_CREAT | os.O_RDWR)
-        os.write(f, password)
-        os.close(f)
+        with io.open(pwd_file, 'w') as f:
+            f.write(password)
 
         ipautil.run([paths.CERTUTIL, "-d", database, "-f", pwd_file, "-N"])
 

From 1ec0ddc6803235a96603cda8b7982856b243bdad Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Tue, 10 Jan 2017 13:45:11 +0100
Subject: [PATCH 02/16] py3: service.py: replace mkstemp by NamedTemporaryFile

NamedTemporaryfile can be used in more pythonic way and file can be
opened in textual mode that is required with PY3

https://fedorahosted.org/freeipa/ticket/4985
---
 ipapython/ipautil.py         | 2 +-
 ipaserver/install/service.py | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/ipapython/ipautil.py b/ipapython/ipautil.py
index e3e4611..34d10ef 100644
--- a/ipapython/ipautil.py
+++ b/ipapython/ipautil.py
@@ -852,7 +852,7 @@ def ipa_generate_password(entropy_bits=256, uppercase=1, lowercase=1, digits=1,
     rnd = random.SystemRandom()
 
     todo_entropy = entropy_bits
-    password = ''
+    password = u''
     # Generate required character classes:
     # The order of generated characters is fixed to comply with check in
     # NSS function sftk_newPinCheck() in nss/lib/softoken/fipstokn.c.
diff --git a/ipaserver/install/service.py b/ipaserver/install/service.py
index 6451f92..fbe3f23 100644
--- a/ipaserver/install/service.py
+++ b/ipaserver/install/service.py
@@ -208,9 +208,10 @@ def _ldap_mod(self, ldif, sub_dict=None, raise_on_err=True,
         args += ["-H", ldap_uri]
 
         if dm_password:
-            [pw_fd, pw_name] = tempfile.mkstemp()
-            os.write(pw_fd, dm_password)
-            os.close(pw_fd)
+            with tempfile.NamedTemporaryFile(
+                    mode='w', delete=False) as pw_file:
+                pw_file.write(dm_password)
+                pw_name = pw_file.name
             auth_parms = ["-x", "-D", "cn=Directory Manager", "-y", pw_name]
         # Use GSSAPI auth when not using DM password or not being root
         elif os.getegid() != 0:

From 1a171af5e114a7cb20dac3eb83ccd54e9dbe264c Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Mon, 9 Jan 2017 12:42:23 +0100
Subject: [PATCH 03/16] py3: open temporary ldif file in text mode

ldif parser uses file in text mode, so we have to open it in text mode
in py3

Also values passed to parser should be bytes

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaserver/install/dsinstance.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/ipaserver/install/dsinstance.py b/ipaserver/install/dsinstance.py
index 5a28026..ceb7bf3 100644
--- a/ipaserver/install/dsinstance.py
+++ b/ipaserver/install/dsinstance.py
@@ -587,14 +587,15 @@ def __update_dse_ldif(self):
             'dse.ldif'
         )
 
-        with tempfile.NamedTemporaryFile(delete=False) as new_dse_ldif:
+        with tempfile.NamedTemporaryFile(
+                mode='w', delete=False) as new_dse_ldif:
             temp_filename = new_dse_ldif.name
             with open(dse_filename, "r") as input_file:
                 parser = installutils.ModifyLDIF(input_file, new_dse_ldif)
                 parser.replace_value(
                         'cn=config,cn=ldbm database,cn=plugins,cn=config',
                         'nsslapd-db-locks',
-                        ['50000']
+                        [b'50000']
                         )
                 if self.config_ldif:
                     # parse modifications from ldif file supplied by the admin

From 131da939e38bded1ffb991062b7581c7a3b498e7 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Mon, 9 Jan 2017 19:01:29 +0100
Subject: [PATCH 04/16] py3: ldap modlist must have keys as string, not bytes

https://fedorahosted.org/freeipa/ticket/4985
---
 ipapython/ipaldap.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ipapython/ipaldap.py b/ipapython/ipaldap.py
index 3ee40bf..57f7dc1 100644
--- a/ipapython/ipaldap.py
+++ b/ipapython/ipaldap.py
@@ -887,7 +887,8 @@ def encode(self, val):
         elif isinstance(val, tuple):
             return tuple(self.encode(m) for m in val)
         elif isinstance(val, dict):
-            dct = dict((self.encode(k), self.encode(v)) for k, v in val.items())
+            # key in dict must be str not bytes
+            dct = dict((k, self.encode(v)) for k, v in val.items())
             return dct
         elif isinstance(val, datetime.datetime):
             return val.strftime(LDAP_GENERALIZED_TIME_FORMAT)

From b36403d1a3a226566d8edba8c4b390b500b81d50 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Mon, 9 Jan 2017 19:26:04 +0100
Subject: [PATCH 05/16] py3: ipautil: open tempfiles in text mode

Code in ipautlis works with text, so tempfiles should be open in
textmode otherwise TypeErrors are raised

https://fedorahosted.org/freeipa/ticket/4985
---
 ipapython/ipautil.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ipapython/ipautil.py b/ipapython/ipautil.py
index 34d10ef..f2b3d74 100644
--- a/ipapython/ipautil.py
+++ b/ipapython/ipautil.py
@@ -1020,7 +1020,7 @@ def config_replace_variables(filepath, replacevars=dict(), appendvars=dict()):
     orig_stat = os.stat(filepath)
     old_values = dict()
     temp_filename = None
-    with tempfile.NamedTemporaryFile(delete=False) as new_config:
+    with tempfile.NamedTemporaryFile(mode="w", delete=False) as new_config:
         temp_filename = new_config.name
         with open(filepath, 'r') as f:
             for line in f:
@@ -1106,7 +1106,7 @@ def add_options(config, replacevars, appendvars, oldvars):
     orig_stat = os.stat(filepath)
     old_values = dict()
     temp_filename = None
-    with tempfile.NamedTemporaryFile(delete=False) as new_config:
+    with tempfile.NamedTemporaryFile(mode='w', delete=False) as new_config:
         temp_filename = new_config.name
         with open(filepath, 'r') as f:
             in_section = False

From 6a57801aec224e6da8fbe179924f090bfc7bd663 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Mon, 9 Jan 2017 19:28:57 +0100
Subject: [PATCH 06/16] py3: CA/KRA: config parser requires string

basedn is DN object it has to be converted to string before it can be
used with config parser

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaserver/install/cainstance.py  | 3 ++-
 ipaserver/install/krainstance.py | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/ipaserver/install/cainstance.py b/ipaserver/install/cainstance.py
index 423aa3f..2722194 100644
--- a/ipaserver/install/cainstance.py
+++ b/ipaserver/install/cainstance.py
@@ -33,6 +33,7 @@
 import time
 import tempfile
 
+import six
 # pylint: disable=import-error
 from six.moves.configparser import ConfigParser, RawConfigParser
 # pylint: enable=import-error
@@ -500,7 +501,7 @@ def __spawn_instance(self):
         # Directory server
         config.set("CA", "pki_ds_ldap_port", "389")
         config.set("CA", "pki_ds_password", self.dm_password)
-        config.set("CA", "pki_ds_base_dn", self.basedn)
+        config.set("CA", "pki_ds_base_dn", six.text_type(self.basedn))
         config.set("CA", "pki_ds_database", "ipaca")
 
         if self.use_ldaps:
diff --git a/ipaserver/install/krainstance.py b/ipaserver/install/krainstance.py
index 1f38c86..4f897cb 100644
--- a/ipaserver/install/krainstance.py
+++ b/ipaserver/install/krainstance.py
@@ -22,6 +22,7 @@
 import shutil
 import tempfile
 
+import six
 # pylint: disable=import-error
 from six.moves.configparser import ConfigParser
 # pylint: enable=import-error
@@ -190,7 +191,7 @@ def __spawn_instance(self):
         # Directory server
         config.set("KRA", "pki_ds_ldap_port", "389")
         config.set("KRA", "pki_ds_password", self.dm_password)
-        config.set("KRA", "pki_ds_base_dn", self.basedn)
+        config.set("KRA", "pki_ds_base_dn", six.text_type(self.basedn))
         config.set("KRA", "pki_ds_database", "ipaca")
         config.set("KRA", "pki_ds_create_new_db", "False")
 

From 7da4a39c7db063288679f1d49b6d121b53d264e0 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Tue, 10 Jan 2017 13:33:41 +0100
Subject: [PATCH 07/16] py3: write CA/KRA config into file opened in text mode

config parser writes data as text so CA/KRA should be opened in textual
mode otherwise type errors are raised from installer

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaserver/install/cainstance.py  | 2 +-
 ipaserver/install/krainstance.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ipaserver/install/cainstance.py b/ipaserver/install/cainstance.py
index 2722194..4c9f967 100644
--- a/ipaserver/install/cainstance.py
+++ b/ipaserver/install/cainstance.py
@@ -599,7 +599,7 @@ def __spawn_instance(self):
         config.set("Tomcat", "pki_ajp_host", "::1")
 
         # Generate configuration file
-        with open(cfg_file, "wb") as f:
+        with open(cfg_file, "w") as f:
             config.write(f)
 
         self.backup_state('installed', True)
diff --git a/ipaserver/install/krainstance.py b/ipaserver/install/krainstance.py
index 4f897cb..95672ee 100644
--- a/ipaserver/install/krainstance.py
+++ b/ipaserver/install/krainstance.py
@@ -263,7 +263,7 @@ def __spawn_instance(self):
                 admin_path.write(cert)
 
         # Generate configuration file
-        with open(cfg_file, "wb") as f:
+        with open(cfg_file, "w") as f:
             config.write(f)
 
         try:

From 48dd833f2a086c340e572ebb6ced3e2100be34bf Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Tue, 10 Jan 2017 16:44:46 +0100
Subject: [PATCH 08/16] py3: cainstance: replace mkstemp with
 NamedTemporaryFile

With Python3 files must be opened in textual mode to write text, and
best practise is to use fileobject instead fo os.write() and manual
encodig

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaserver/install/cainstance.py | 49 +++++++++++++++++++++--------------------
 1 file changed, 25 insertions(+), 24 deletions(-)

diff --git a/ipaserver/install/cainstance.py b/ipaserver/install/cainstance.py
index 4c9f967..20677cf 100644
--- a/ipaserver/install/cainstance.py
+++ b/ipaserver/install/cainstance.py
@@ -657,13 +657,12 @@ def import_ra_cert(self, rafile):
         Used when setting up replication
         """
         # Add the new RA cert to the database in /etc/httpd/alias
-        (agent_fd, agent_name) = tempfile.mkstemp()
-        os.write(agent_fd, self.dm_password)
-        os.close(agent_fd)
-        try:
-            import_pkcs12(rafile, agent_name, self.ra_agent_db, self.ra_agent_pwd)
-        finally:
-            os.remove(agent_name)
+        with tempfile.NamedTemporaryFile(mode="w") as agent_file:
+            agent_file.write(self.dm_password)
+            agent_file.flush()
+
+            import_pkcs12(
+                rafile, agent_file.name, self.ra_agent_db, self.ra_agent_pwd)
 
         self.configure_agent_renewal()
 
@@ -759,10 +758,9 @@ def __import_ca_chain(self):
 
         ca_dn = DN(self.ca_subject)
         for cert in certlist:
-            try:
-                chain_fd, chain_name = tempfile.mkstemp()
-                os.write(chain_fd, cert)
-                os.close(chain_fd)
+            with tempfile.NamedTemporaryFile(mode="w") as chain_file:
+                chain_file.write(cert)
+                chain_file.flush()
                 (_rdn, subject_dn) = certs.get_cert_nickname(cert)
                 if subject_dn == ca_dn:
                     nick = get_ca_nickname(self.realm)
@@ -772,10 +770,8 @@ def __import_ca_chain(self):
                     trust_flags = ',,'
                 self.__run_certutil(
                     ['-A', '-t', trust_flags, '-n', nick, '-a',
-                     '-i', chain_name]
+                     '-i', chain_file.name]
                 )
-            finally:
-                os.remove(chain_name)
 
         # Restore NSS trust flags of all previously existing certificates
         for nick, trust_flags in cert_backup_list:
@@ -783,13 +779,15 @@ def __import_ca_chain(self):
 
     def __request_ra_certificate(self):
         # create a temp file storing the pwd
-        (agent_fd, agent_pwdfile) = tempfile.mkstemp(dir=paths.VAR_LIB_IPA)
-        os.write(agent_fd, self.admin_password)
-        os.close(agent_fd)
+        agent_file = tempfile.NamedTemporaryFile(
+            mode="w", dir=paths.VAR_LIB_IPA, delete=False)
+        agent_file.write(self.admin_password)
+        agent_file.close()
 
         # create a temp pem file storing the CA chain
-        (chain_fd, chain_file) = tempfile.mkstemp(dir=paths.VAR_LIB_IPA)
-        os.close(chain_fd)
+        chain_file = tempfile.NamedTemporaryFile(
+            mode="w", dir=paths.VAR_LIB_IPA, delete=False)
+        chain_file.close()
 
         chain = self.__get_ca_chain()
         data = base64.b64decode(chain)
@@ -799,17 +797,17 @@ def __request_ra_certificate(self):
              "-inform",
              "DER",
              "-print_certs",
-             "-out", chain_file,
+             "-out", chain_file.name,
              ], stdin=data, capture_output=False)
 
         agent_args = [paths.DOGTAG_IPA_CA_RENEW_AGENT_SUBMIT,
                       "--dbdir", self.agent_db,
                       "--nickname", "ipa-ca-agent",
-                      "--cafile", chain_file,
+                      "--cafile", chain_file.name,
                       "--ee-url", 'http://%s:8080/ca/ee/ca/' % self.fqdn,
                       "--agent-url",
                       'https://%s:8443/ca/agent/ca/' % self.fqdn,
-                      "--sslpinfile", agent_pwdfile]
+                      "--sslpinfile", agent_file.name]
         helper = " ".join(agent_args)
 
         # configure certmonger renew agent to use temporary agent cert
@@ -842,8 +840,11 @@ def __request_ra_certificate(self):
             certmonger.modify_ca_helper(
                 ipalib.constants.RENEWAL_CA_NAME, old_helper)
             # remove the pwdfile
-            os.remove(agent_pwdfile)
-            os.remove(chain_file)
+            for f in (agent_file, chain_file):
+                try:
+                    os.remove(f.name)
+                except OSError:
+                    pass
 
     def __setup_sign_profile(self):
         # Tell the profile to automatically issue certs for RAs

From fdc9efb6e923f5db4fbb730c14ce62be8b778576 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Tue, 10 Jan 2017 18:21:13 +0100
Subject: [PATCH 09/16] py3: _httplib_request: don't convert string to bytes

There is no need to encode hostname to bytes. UTF-8 characters must be
encoded in different format in URL anyway and it causes only error in
Py3. String must be unicode to support Py2.

https://fedorahosted.org/freeipa/ticket/4985
---
 ipapython/dogtag.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/ipapython/dogtag.py b/ipapython/dogtag.py
index eb1f73e..37e7a58 100644
--- a/ipapython/dogtag.py
+++ b/ipapython/dogtag.py
@@ -188,9 +188,7 @@ def _httplib_request(
 
     Perform a HTTP(s) request.
     """
-    if isinstance(host, unicode):
-        host = host.encode('utf-8')
-    uri = '%s://%s%s' % (protocol, ipautil.format_netloc(host, port), path)
+    uri = u'%s://%s%s' % (protocol, ipautil.format_netloc(host, port), path)
     root_logger.debug('request %s %s', method, uri)
     root_logger.debug('request body %r', request_body)
 

From a9887a8e965340419bcaa45651b43a18cf7f81a4 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Tue, 10 Jan 2017 18:24:16 +0100
Subject: [PATCH 10/16] py3: HTTPResponse has no 'dict' attribute in 'msg'

There is no 'dict' attribute in 'msg', but 'msg' attribute is dict-like object
in both py2/3, so it can be used instead.

https://fedorahosted.org/freeipa/ticket/4985
---
 ipapython/dogtag.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ipapython/dogtag.py b/ipapython/dogtag.py
index 37e7a58..c6a8346 100644
--- a/ipapython/dogtag.py
+++ b/ipapython/dogtag.py
@@ -205,7 +205,7 @@ def _httplib_request(
         res = conn.getresponse()
 
         http_status = res.status
-        http_headers = res.msg.dict
+        http_headers = res.msg
         http_body = res.read()
         conn.close()
     except Exception as e:

From 19463bbf9f11d22c7a9f75e74ce79bf5eb37c665 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Wed, 11 Jan 2017 10:23:04 +0100
Subject: [PATCH 11/16] py3: add_entry_to_group: attribute name must be string
 not bytes

With bytes as attribute name pyldap raises type error

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaserver/plugins/ldap2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ipaserver/plugins/ldap2.py b/ipaserver/plugins/ldap2.py
index a04be38..25fbfb8 100644
--- a/ipaserver/plugins/ldap2.py
+++ b/ipaserver/plugins/ldap2.py
@@ -417,7 +417,7 @@ def add_entry_to_group(self, dn, group_dn, member_attr='member', allow_same=Fals
         # update group entry
         try:
             with self.error_handler():
-                modlist = [(a, self.encode(b), self.encode(c))
+                modlist = [(a, b, self.encode(c))
                            for a, b, c in modlist]
                 self.conn.modify_s(str(group_dn), modlist)
         except errors.DatabaseError:

From 07b3618de080f7fbb7649fc4fc7754d29a4b7e28 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Wed, 11 Jan 2017 12:35:08 +0100
Subject: [PATCH 12/16] py3: __add_acl: use standard ipaldap methods

Using raw pyldap interface we have to keep vaules as bytes. Is easier to
migrate to ipaldap and use strings without decoding and encoding.

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaserver/install/cainstance.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/ipaserver/install/cainstance.py b/ipaserver/install/cainstance.py
index 20677cf..a73a9c4 100644
--- a/ipaserver/install/cainstance.py
+++ b/ipaserver/install/cainstance.py
@@ -1528,22 +1528,19 @@ def __add_acls(new_rules):
     Return ``True`` if any ACLs were added otherwise ``False``.
 
     """
-    server_id = installutils.realm_to_serverid(api.env.realm)
-    dogtag_uri = 'ldapi://%%2fvar%%2frun%%2fslapd-%s.socket' % server_id
     updated = False
 
     dn = DN(('cn', 'aclResources'), ('o', 'ipaca'))
 
-    conn = ldap2.ldap2(api, ldap_uri=dogtag_uri)
-    if not conn.isconnected():
-        conn.connect(autobind=True)
-    cur_rules = conn.get_entry(dn).get('resourceACLS', [])
+    conn = api.Backend.ldap2
+    entry = conn.get_entry(dn)
+    cur_rules = entry.get('resourceACLS', [])
     add_rules = [rule for rule in new_rules if rule not in cur_rules]
     if add_rules:
-        conn.conn.modify_s(str(dn), [(ldap.MOD_ADD, 'resourceACLS', add_rules)])
+        cur_rules.extend(add_rules)
+        conn.update_entry(entry)
         updated = True
 
-    conn.disconnect()
     return updated
 
 

From cbbd4b3556b9f8ac6f2aa1662e318084508e3d4e Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Wed, 11 Jan 2017 13:03:53 +0100
Subject: [PATCH 13/16] py3: make_filter_from_attr: use string instead of bytes

Method escape_filter_chars() requires string as parameter instead of
bytes. 'value_to_utf8' returns bytes thus this code has to be removed.

https://fedorahosted.org/freeipa/ticket/4985
---
 ipapython/ipaldap.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ipapython/ipaldap.py b/ipapython/ipaldap.py
index 57f7dc1..81d8c8d 100644
--- a/ipapython/ipaldap.py
+++ b/ipapython/ipaldap.py
@@ -1252,8 +1252,9 @@ def make_filter_from_attr(
                 value = u'\\'.join(
                     value[i:i+2] for i in six.moves.range(-2, len(value), 2))
             else:
-                value = value_to_utf8(value)
+                value = six.text_type(value)
                 value = ldap.filter.escape_filter_chars(value)
+
             if not exact:
                 template = '%s'
                 if leading_wildcard:

From 50f2c8d5083fdf2eb3768db7fdd22e511f11b4a2 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Wed, 11 Jan 2017 13:39:04 +0100
Subject: [PATCH 14/16] py3: convert_attribute_members: don't use bytes as
 parameter for DN

due perfomance improvement in e4930b3235e5d61d227a7e43d30a8feb7f35664d
we have to decode value before it can be used in DN() constructor.

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaserver/plugins/baseldap.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ipaserver/plugins/baseldap.py b/ipaserver/plugins/baseldap.py
index 9d6bfc7..e7bf43c 100644
--- a/ipaserver/plugins/baseldap.py
+++ b/ipaserver/plugins/baseldap.py
@@ -654,7 +654,7 @@ def convert_attribute_members(self, entry_attrs, *keys, **options):
             del entry_attrs[attr]
 
             for member in value:
-                memberdn = DN(member)
+                memberdn = DN(member.decode('utf-8'))
                 for ldap_obj_name in self.attribute_members[attr]:
                     ldap_obj = self.api.Object[ldap_obj_name]
                     try:

From 91a98d0a6005f22eb840fbfabe7b470bb3f41d6e Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Wed, 11 Jan 2017 14:38:25 +0100
Subject: [PATCH 15/16] dogtag.py: fix exception logging of JSON data

'read_ca' and 'create_ca' have no logging when exception happened and it
masks real reason why it failed.
---
 ipaserver/plugins/dogtag.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/ipaserver/plugins/dogtag.py b/ipaserver/plugins/dogtag.py
index 73c14ed..142f838 100644
--- a/ipaserver/plugins/dogtag.py
+++ b/ipaserver/plugins/dogtag.py
@@ -2116,16 +2116,20 @@ def create_ca(self, dn):
         )
         try:
             return json.loads(resp_body)
-        except:
-            raise errors.RemoteRetrieveError(reason=_("Response from CA was not valid JSON"))
+        except Exception as e:
+            self.log.debug(e, exc_info=True)
+            raise errors.RemoteRetrieveError(
+                reason=_("Response from CA was not valid JSON"))
 
     def read_ca(self, ca_id):
         _status, _resp_headers, resp_body = self._ssldo(
             'GET', ca_id, headers={'Accept': 'application/json'})
         try:
             return json.loads(resp_body)
-        except:
-            raise errors.RemoteRetrieveError(reason=_("Response from CA was not valid JSON"))
+        except Exception as e:
+            self.log.debug(e, exc_info=True)
+            raise errors.RemoteRetrieveError(
+                reason=_("Response from CA was not valid JSON"))
 
     def read_ca_cert(self, ca_id):
         _status, _resp_headers, resp_body = self._ssldo(

From 7cc60209c063231ba16f1d2ba0157f1d7613736f Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Thu, 12 Jan 2017 16:20:43 +0100
Subject: [PATCH 16/16] py3: decode bytes for json.loads()

In py 3.5 json.loads requires to have string as input, all bytes must be
decoded.

Note: python 3.6 supports bytes for json.loads()

https://fedorahosted.org/freeipa/ticket/4985
---
 ipaclient/plugins/vault.py  |  2 +-
 ipalib/rpc.py               |  3 ++-
 ipapython/dogtag.py         |  1 +
 ipapython/ipautil.py        | 50 +++++++++++++++++++++++++++++++++++++++++++++
 ipaserver/plugins/dogtag.py |  9 ++++----
 5 files changed, 59 insertions(+), 6 deletions(-)

diff --git a/ipaclient/plugins/vault.py b/ipaclient/plugins/vault.py
index 29157c7..9efb1f1 100644
--- a/ipaclient/plugins/vault.py
+++ b/ipaclient/plugins/vault.py
@@ -969,7 +969,7 @@ def forward(self, *args, **options):
         json_vault_data = decoding_ctx.cipher_op(wrapped_vault_data)\
             + decoding_ctx.digest_final()
 
-        vault_data = json.loads(json_vault_data)
+        vault_data = json.loads(json_vault_data.decode('utf-8'))
         data = base64.b64decode(vault_data[u'data'].encode('utf-8'))
 
         encrypted_key = None
diff --git a/ipalib/rpc.py b/ipalib/rpc.py
index 921f5cb..fb739f8 100644
--- a/ipalib/rpc.py
+++ b/ipalib/rpc.py
@@ -1101,7 +1101,8 @@ def __request(self, name, args):
         )
 
         try:
-            response = json_decode_binary(json.loads(response.decode('ascii')))
+            response = json_decode_binary(
+                json.loads(response.decode('utf-8')))
         except ValueError as e:
             raise JSONError(error=str(e))
 
diff --git a/ipapython/dogtag.py b/ipapython/dogtag.py
index c6a8346..01fc5cb 100644
--- a/ipapython/dogtag.py
+++ b/ipapython/dogtag.py
@@ -209,6 +209,7 @@ def _httplib_request(
         http_body = res.read()
         conn.close()
     except Exception as e:
+        root_logger.exception("httplib request failed:")
         raise NetworkError(uri=uri, error=str(e))
 
     root_logger.debug('response status %d',    http_status)
diff --git a/ipapython/ipautil.py b/ipapython/ipautil.py
index f2b3d74..c8f87ef 100644
--- a/ipapython/ipautil.py
+++ b/ipapython/ipautil.py
@@ -19,6 +19,7 @@
 
 from __future__ import print_function
 
+import codecs
 import string
 import tempfile
 import subprocess
@@ -1361,6 +1362,55 @@ def escape_seq(seq, *args):
     return tuple(a.replace(seq, u'\\{}'.format(seq)) for a in args)
 
 
+def decode_json(data):
+    """Decode JSON bytes to string with proper encoding
+
+    Only for supporting Py 3.5
+
+    Py 3.6 supports bytes as parameter for json.load, we can drop this when
+    there is no need for python 3.5 anymore
+
+    Code from:
+        https://bugs.python.org/file43513/json_detect_encoding_3.patch
+
+    :param data: JSON bytes
+    :return: return JSON string
+    """
+
+    def detect_encoding(b):
+        bstartswith = b.startswith
+        if bstartswith((codecs.BOM_UTF32_BE, codecs.BOM_UTF32_LE)):
+            return 'utf-32'
+        if bstartswith((codecs.BOM_UTF16_BE, codecs.BOM_UTF16_LE)):
+            return 'utf-16'
+        if bstartswith(codecs.BOM_UTF8):
+            return 'utf-8-sig'
+
+        if len(b) >= 4:
+            if not b[0]:
+                # 00 00 -- -- - utf-32-be
+                # 00 XX -- -- - utf-16-be
+                return 'utf-16-be' if b[1] else 'utf-32-be'
+            if not b[1]:
+                # XX 00 00 00 - utf-32-le
+                # XX 00 XX XX - utf-16-le
+                return 'utf-16-le' if b[2] or b[3] else 'utf-32-le'
+        elif len(b) == 2:
+            if not b[0]:
+                # 00 XX - utf-16-be
+                return 'utf-16-be'
+            if not b[1]:
+                # XX 00 - utf-16-le
+                return 'utf-16-le'
+        # default
+        return 'utf-8'
+
+    if isinstance(data, six.text_type):
+        return data
+
+    return data.decode(detect_encoding(data), 'surrogatepass')
+
+
 class APIVersion(tuple):
     """API version parser and handler
 
diff --git a/ipaserver/plugins/dogtag.py b/ipaserver/plugins/dogtag.py
index 142f838..fbfe608 100644
--- a/ipaserver/plugins/dogtag.py
+++ b/ipaserver/plugins/dogtag.py
@@ -1232,7 +1232,8 @@ class ra_certprofile(RestClient):
     @staticmethod
     def _parse_dogtag_error(body):
         try:
-            return pki.PKIException.from_json(json.loads(body))
+            return pki.PKIException.from_json(
+                json.loads(ipautil.decode_json(body)))
         except Exception:
             return None
 
@@ -1667,7 +1668,7 @@ def request_certificate(
         )
 
         try:
-            resp_obj = json.loads(http_body)
+            resp_obj = json.loads(ipautil.decode_json(http_body))
         except ValueError:
             raise errors.RemoteRetrieveError(reason=_("Response from CA was not valid JSON"))
 
@@ -2115,7 +2116,7 @@ def create_ca(self, dn):
             body=json.dumps({"parentID": "host-authority", "dn": unicode(dn)}),
         )
         try:
-            return json.loads(resp_body)
+            return json.loads(ipautil.decode_json(resp_body))
         except Exception as e:
             self.log.debug(e, exc_info=True)
             raise errors.RemoteRetrieveError(
@@ -2125,7 +2126,7 @@ def read_ca(self, ca_id):
         _status, _resp_headers, resp_body = self._ssldo(
             'GET', ca_id, headers={'Accept': 'application/json'})
         try:
-            return json.loads(resp_body)
+            return json.loads(ipautil.decode_json(resp_body))
         except Exception as e:
             self.log.debug(e, exc_info=True)
             raise errors.RemoteRetrieveError(
-- 
Manage your subscription for the Freeipa-devel mailing list:
https://www.redhat.com/mailman/listinfo/freeipa-devel
Contribute to FreeIPA: http://www.freeipa.org/page/Contribute/Code

Reply via email to