URL: https://github.com/freeipa/freeipa/pull/393 Author: mbasti-rh Title: #393: [WIP] Py3 allow to run wsgi Action: synchronized
To pull the PR as Git branch: git remote add ghfreeipa https://github.com/freeipa/freeipa git fetch ghfreeipa pull/393/head:pr393 git checkout pr393
From 928a2db32a36e7326d127ba711a2630ff7d74cc1 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Mon, 9 Jan 2017 11:53:59 +0100 Subject: [PATCH 01/21] py3: create_cert_db: write to file in a compatible way Py3 expect bytes to be writed using os.write. Instead of that using io module is more pythonic. https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/install/httpinstance.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ipaserver/install/httpinstance.py b/ipaserver/install/httpinstance.py index bacd5fc..ded0553 100644 --- a/ipaserver/install/httpinstance.py +++ b/ipaserver/install/httpinstance.py @@ -19,6 +19,7 @@ from __future__ import print_function +import io import os import os.path import pwd @@ -314,9 +315,8 @@ def create_cert_db(self): # Create the password file for this db password = ipautil.ipa_generate_password() - f = os.open(pwd_file, os.O_CREAT | os.O_RDWR) - os.write(f, password) - os.close(f) + with io.open(pwd_file, 'w') as f: + f.write(password) ipautil.run([paths.CERTUTIL, "-d", database, "-f", pwd_file, "-N"]) From 8c157e9643019bf6ffadab68efdb272d80038871 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Tue, 10 Jan 2017 13:45:11 +0100 Subject: [PATCH 02/21] py3: service.py: replace mkstemp by NamedTemporaryFile NamedTemporaryfile can be used in more pythonic way and file can be opened in textual mode that is required with PY3 https://fedorahosted.org/freeipa/ticket/4985 --- ipapython/ipautil.py | 2 +- ipaserver/install/service.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ipapython/ipautil.py b/ipapython/ipautil.py index e3e4611..34d10ef 100644 --- a/ipapython/ipautil.py +++ b/ipapython/ipautil.py @@ -852,7 +852,7 @@ def ipa_generate_password(entropy_bits=256, uppercase=1, lowercase=1, digits=1, rnd = random.SystemRandom() todo_entropy = entropy_bits - password = '' + password = u'' # Generate required character classes: # The order of generated characters is fixed to comply with check in # NSS function sftk_newPinCheck() in nss/lib/softoken/fipstokn.c. diff --git a/ipaserver/install/service.py b/ipaserver/install/service.py index 6451f92..fbe3f23 100644 --- a/ipaserver/install/service.py +++ b/ipaserver/install/service.py @@ -208,9 +208,10 @@ def _ldap_mod(self, ldif, sub_dict=None, raise_on_err=True, args += ["-H", ldap_uri] if dm_password: - [pw_fd, pw_name] = tempfile.mkstemp() - os.write(pw_fd, dm_password) - os.close(pw_fd) + with tempfile.NamedTemporaryFile( + mode='w', delete=False) as pw_file: + pw_file.write(dm_password) + pw_name = pw_file.name auth_parms = ["-x", "-D", "cn=Directory Manager", "-y", pw_name] # Use GSSAPI auth when not using DM password or not being root elif os.getegid() != 0: From 35af241c73eb9a3a38a7f613ff24751bd5a10fae Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Mon, 9 Jan 2017 12:42:23 +0100 Subject: [PATCH 03/21] py3: open temporary ldif file in text mode ldif parser uses file in text mode, so we have to open it in text mode in py3 Also values passed to parser should be bytes https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/install/dsinstance.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ipaserver/install/dsinstance.py b/ipaserver/install/dsinstance.py index 6e7019a..a38d4f7 100644 --- a/ipaserver/install/dsinstance.py +++ b/ipaserver/install/dsinstance.py @@ -589,14 +589,15 @@ def __update_dse_ldif(self): 'dse.ldif' ) - with tempfile.NamedTemporaryFile(delete=False) as new_dse_ldif: + with tempfile.NamedTemporaryFile( + mode='w', delete=False) as new_dse_ldif: temp_filename = new_dse_ldif.name with open(dse_filename, "r") as input_file: parser = installutils.ModifyLDIF(input_file, new_dse_ldif) parser.replace_value( 'cn=config,cn=ldbm database,cn=plugins,cn=config', 'nsslapd-db-locks', - ['50000'] + [b'50000'] ) if self.config_ldif: # parse modifications from ldif file supplied by the admin From 747a29ef0c09276c5b6fb71990d9a9271bce2c25 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Mon, 9 Jan 2017 19:01:29 +0100 Subject: [PATCH 04/21] py3: ldap modlist must have keys as string, not bytes https://fedorahosted.org/freeipa/ticket/4985 --- ipapython/ipaldap.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ipapython/ipaldap.py b/ipapython/ipaldap.py index daee068..11448f0 100644 --- a/ipapython/ipaldap.py +++ b/ipapython/ipaldap.py @@ -886,7 +886,8 @@ def encode(self, val): elif isinstance(val, tuple): return tuple(self.encode(m) for m in val) elif isinstance(val, dict): - dct = dict((self.encode(k), self.encode(v)) for k, v in val.items()) + # key in dict must be str not bytes + dct = dict((k, self.encode(v)) for k, v in val.items()) return dct elif isinstance(val, datetime.datetime): return val.strftime(LDAP_GENERALIZED_TIME_FORMAT) From c49eaad48f76c0af7cdd3be62d4203532e1bc846 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Mon, 9 Jan 2017 19:26:04 +0100 Subject: [PATCH 05/21] py3: ipautil: open tempfiles in text mode Code in ipautlis works with text, so tempfiles should be open in textmode otherwise TypeErrors are raised https://fedorahosted.org/freeipa/ticket/4985 --- ipapython/ipautil.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ipapython/ipautil.py b/ipapython/ipautil.py index 34d10ef..f2b3d74 100644 --- a/ipapython/ipautil.py +++ b/ipapython/ipautil.py @@ -1020,7 +1020,7 @@ def config_replace_variables(filepath, replacevars=dict(), appendvars=dict()): orig_stat = os.stat(filepath) old_values = dict() temp_filename = None - with tempfile.NamedTemporaryFile(delete=False) as new_config: + with tempfile.NamedTemporaryFile(mode="w", delete=False) as new_config: temp_filename = new_config.name with open(filepath, 'r') as f: for line in f: @@ -1106,7 +1106,7 @@ def add_options(config, replacevars, appendvars, oldvars): orig_stat = os.stat(filepath) old_values = dict() temp_filename = None - with tempfile.NamedTemporaryFile(delete=False) as new_config: + with tempfile.NamedTemporaryFile(mode='w', delete=False) as new_config: temp_filename = new_config.name with open(filepath, 'r') as f: in_section = False From 5c8f333c8e3bc3204ff79a4cd70edea2f4d49250 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Mon, 9 Jan 2017 19:28:57 +0100 Subject: [PATCH 06/21] py3: CA/KRA: config parser requires string basedn is DN object it has to be converted to string before it can be used with config parser https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/install/cainstance.py | 3 ++- ipaserver/install/krainstance.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ipaserver/install/cainstance.py b/ipaserver/install/cainstance.py index f933479..5dd91b2 100644 --- a/ipaserver/install/cainstance.py +++ b/ipaserver/install/cainstance.py @@ -35,6 +35,7 @@ import shlex import pipes +import six # pylint: disable=import-error from six.moves.configparser import ConfigParser, RawConfigParser # pylint: enable=import-error @@ -502,7 +503,7 @@ def __spawn_instance(self): # Directory server config.set("CA", "pki_ds_ldap_port", "389") config.set("CA", "pki_ds_password", self.dm_password) - config.set("CA", "pki_ds_base_dn", self.basedn) + config.set("CA", "pki_ds_base_dn", six.text_type(self.basedn)) config.set("CA", "pki_ds_database", "ipaca") if self.use_ldaps: diff --git a/ipaserver/install/krainstance.py b/ipaserver/install/krainstance.py index 1f38c86..4f897cb 100644 --- a/ipaserver/install/krainstance.py +++ b/ipaserver/install/krainstance.py @@ -22,6 +22,7 @@ import shutil import tempfile +import six # pylint: disable=import-error from six.moves.configparser import ConfigParser # pylint: enable=import-error @@ -190,7 +191,7 @@ def __spawn_instance(self): # Directory server config.set("KRA", "pki_ds_ldap_port", "389") config.set("KRA", "pki_ds_password", self.dm_password) - config.set("KRA", "pki_ds_base_dn", self.basedn) + config.set("KRA", "pki_ds_base_dn", six.text_type(self.basedn)) config.set("KRA", "pki_ds_database", "ipaca") config.set("KRA", "pki_ds_create_new_db", "False") From 9209c5040227f39b8c6a98875be3f9390374d256 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Tue, 10 Jan 2017 13:33:41 +0100 Subject: [PATCH 07/21] py3: write CA/KRA config into file opened in text mode config parser writes data as text so CA/KRA should be opened in textual mode otherwise type errors are raised from installer https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/install/cainstance.py | 2 +- ipaserver/install/krainstance.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ipaserver/install/cainstance.py b/ipaserver/install/cainstance.py index 5dd91b2..f015ae3 100644 --- a/ipaserver/install/cainstance.py +++ b/ipaserver/install/cainstance.py @@ -601,7 +601,7 @@ def __spawn_instance(self): config.set("Tomcat", "pki_ajp_host", "::1") # Generate configuration file - with open(cfg_file, "wb") as f: + with open(cfg_file, "w") as f: config.write(f) self.backup_state('installed', True) diff --git a/ipaserver/install/krainstance.py b/ipaserver/install/krainstance.py index 4f897cb..95672ee 100644 --- a/ipaserver/install/krainstance.py +++ b/ipaserver/install/krainstance.py @@ -263,7 +263,7 @@ def __spawn_instance(self): admin_path.write(cert) # Generate configuration file - with open(cfg_file, "wb") as f: + with open(cfg_file, "w") as f: config.write(f) try: From 6614eb2b012b68ad0fc5785abdc7f301a184d344 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Tue, 10 Jan 2017 16:44:46 +0100 Subject: [PATCH 08/21] py3: cainstance: replace mkstemp with NamedTemporaryFile With Python3 files must be opened in textual mode to write text, and best practise is to use fileobject instead fo os.write() and manual encodig https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/install/cainstance.py | 49 +++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/ipaserver/install/cainstance.py b/ipaserver/install/cainstance.py index f015ae3..b6b8eb4 100644 --- a/ipaserver/install/cainstance.py +++ b/ipaserver/install/cainstance.py @@ -659,13 +659,12 @@ def import_ra_cert(self, rafile): Used when setting up replication """ # Add the new RA cert to the database in /etc/httpd/alias - (agent_fd, agent_name) = tempfile.mkstemp() - os.write(agent_fd, self.dm_password) - os.close(agent_fd) - try: - import_pkcs12(rafile, agent_name, self.ra_agent_db, self.ra_agent_pwd) - finally: - os.remove(agent_name) + with tempfile.NamedTemporaryFile(mode="w") as agent_file: + agent_file.write(self.dm_password) + agent_file.flush() + + import_pkcs12( + rafile, agent_file.name, self.ra_agent_db, self.ra_agent_pwd) self.configure_agent_renewal() @@ -761,10 +760,9 @@ def __import_ca_chain(self): ca_dn = DN(self.ca_subject) for cert in certlist: - try: - chain_fd, chain_name = tempfile.mkstemp() - os.write(chain_fd, cert) - os.close(chain_fd) + with tempfile.NamedTemporaryFile(mode="w") as chain_file: + chain_file.write(cert) + chain_file.flush() (_rdn, subject_dn) = certs.get_cert_nickname(cert) if subject_dn == ca_dn: nick = get_ca_nickname(self.realm) @@ -774,10 +772,8 @@ def __import_ca_chain(self): trust_flags = ',,' self.__run_certutil( ['-A', '-t', trust_flags, '-n', nick, '-a', - '-i', chain_name] + '-i', chain_file.name] ) - finally: - os.remove(chain_name) # Restore NSS trust flags of all previously existing certificates for nick, trust_flags in cert_backup_list: @@ -785,13 +781,15 @@ def __import_ca_chain(self): def __request_ra_certificate(self): # create a temp file storing the pwd - (agent_fd, agent_pwdfile) = tempfile.mkstemp(dir=paths.VAR_LIB_IPA) - os.write(agent_fd, self.admin_password) - os.close(agent_fd) + agent_file = tempfile.NamedTemporaryFile( + mode="w", dir=paths.VAR_LIB_IPA, delete=False) + agent_file.write(self.admin_password) + agent_file.close() # create a temp pem file storing the CA chain - (chain_fd, chain_file) = tempfile.mkstemp(dir=paths.VAR_LIB_IPA) - os.close(chain_fd) + chain_file = tempfile.NamedTemporaryFile( + mode="w", dir=paths.VAR_LIB_IPA, delete=False) + chain_file.close() chain = self.__get_ca_chain() data = base64.b64decode(chain) @@ -801,17 +799,17 @@ def __request_ra_certificate(self): "-inform", "DER", "-print_certs", - "-out", chain_file, + "-out", chain_file.name, ], stdin=data, capture_output=False) agent_args = [paths.DOGTAG_IPA_CA_RENEW_AGENT_SUBMIT, "--dbdir", self.agent_db, "--nickname", "ipa-ca-agent", - "--cafile", chain_file, + "--cafile", chain_file.name, "--ee-url", 'http://%s:8080/ca/ee/ca/' % self.fqdn, "--agent-url", 'https://%s:8443/ca/agent/ca/' % self.fqdn, - "--sslpinfile", agent_pwdfile] + "--sslpinfile", agent_file.name] helper = " ".join(agent_args) # configure certmonger renew agent to use temporary agent cert @@ -844,8 +842,11 @@ def __request_ra_certificate(self): certmonger.modify_ca_helper( ipalib.constants.RENEWAL_CA_NAME, old_helper) # remove the pwdfile - os.remove(agent_pwdfile) - os.remove(chain_file) + for f in (agent_file, chain_file): + try: + os.remove(f.name) + except OSError: + pass def __setup_sign_profile(self): # Tell the profile to automatically issue certs for RAs From 06ba6c003862c252782822b50b4167042f4ffd05 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Tue, 10 Jan 2017 18:21:13 +0100 Subject: [PATCH 09/21] py3: _httplib_request: don't convert string to bytes There is no need to encode hostname to bytes. UTF-8 characters must be encoded in different format in URL anyway and it causes only error in Py3. String must be unicode to support Py2. https://fedorahosted.org/freeipa/ticket/4985 --- ipapython/dogtag.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ipapython/dogtag.py b/ipapython/dogtag.py index eb1f73e..37e7a58 100644 --- a/ipapython/dogtag.py +++ b/ipapython/dogtag.py @@ -188,9 +188,7 @@ def _httplib_request( Perform a HTTP(s) request. """ - if isinstance(host, unicode): - host = host.encode('utf-8') - uri = '%s://%s%s' % (protocol, ipautil.format_netloc(host, port), path) + uri = u'%s://%s%s' % (protocol, ipautil.format_netloc(host, port), path) root_logger.debug('request %s %s', method, uri) root_logger.debug('request body %r', request_body) From 806e97e8c722175f66ef3671370b9a18ec38dc98 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Tue, 10 Jan 2017 18:24:16 +0100 Subject: [PATCH 10/21] py3: HTTPResponse has no 'dict' attribute in 'msg' There is no 'dict' attribute in 'msg', but 'msg' attribute is dict-like object in both py2/3, so it can be used instead. https://fedorahosted.org/freeipa/ticket/4985 --- ipapython/dogtag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipapython/dogtag.py b/ipapython/dogtag.py index 37e7a58..c6a8346 100644 --- a/ipapython/dogtag.py +++ b/ipapython/dogtag.py @@ -205,7 +205,7 @@ def _httplib_request( res = conn.getresponse() http_status = res.status - http_headers = res.msg.dict + http_headers = res.msg http_body = res.read() conn.close() except Exception as e: From 916e59470e488408986f6faa12f6d98c16bddcb0 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Wed, 11 Jan 2017 10:23:04 +0100 Subject: [PATCH 11/21] py3: add_entry_to_group: attribute name must be string not bytes With bytes as attribute name pyldap raises type error https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/plugins/ldap2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipaserver/plugins/ldap2.py b/ipaserver/plugins/ldap2.py index a04be38..25fbfb8 100644 --- a/ipaserver/plugins/ldap2.py +++ b/ipaserver/plugins/ldap2.py @@ -417,7 +417,7 @@ def add_entry_to_group(self, dn, group_dn, member_attr='member', allow_same=Fals # update group entry try: with self.error_handler(): - modlist = [(a, self.encode(b), self.encode(c)) + modlist = [(a, b, self.encode(c)) for a, b, c in modlist] self.conn.modify_s(str(group_dn), modlist) except errors.DatabaseError: From eb0fab758a10f08a7250ae186d401b219ae22e01 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Wed, 11 Jan 2017 12:35:08 +0100 Subject: [PATCH 12/21] py3: __add_acl: use standard ipaldap methods Using raw pyldap interface we have to keep vaules as bytes. Is easier to migrate to ipaldap and use strings without decoding and encoding. https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/install/cainstance.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/ipaserver/install/cainstance.py b/ipaserver/install/cainstance.py index b6b8eb4..6ce717d 100644 --- a/ipaserver/install/cainstance.py +++ b/ipaserver/install/cainstance.py @@ -1552,22 +1552,19 @@ def __add_acls(new_rules): Return ``True`` if any ACLs were added otherwise ``False``. """ - server_id = installutils.realm_to_serverid(api.env.realm) - dogtag_uri = 'ldapi://%%2fvar%%2frun%%2fslapd-%s.socket' % server_id updated = False dn = DN(('cn', 'aclResources'), ('o', 'ipaca')) - conn = ldap2.ldap2(api, ldap_uri=dogtag_uri) - if not conn.isconnected(): - conn.connect(autobind=True) - cur_rules = conn.get_entry(dn).get('resourceACLS', []) + conn = api.Backend.ldap2 + entry = conn.get_entry(dn) + cur_rules = entry.get('resourceACLS', []) add_rules = [rule for rule in new_rules if rule not in cur_rules] if add_rules: - conn.conn.modify_s(str(dn), [(ldap.MOD_ADD, 'resourceACLS', add_rules)]) + cur_rules.extend(add_rules) + conn.update_entry(entry) updated = True - conn.disconnect() return updated From 7db2f9a743e3f62192bf30c27ac1ebc43ea85846 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Wed, 11 Jan 2017 13:03:53 +0100 Subject: [PATCH 13/21] py3: make_filter_from_attr: use string instead of bytes Method escape_filter_chars() requires string as parameter instead of bytes. 'value_to_utf8' returns bytes thus this code has to be removed. https://fedorahosted.org/freeipa/ticket/4985 --- ipapython/ipaldap.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ipapython/ipaldap.py b/ipapython/ipaldap.py index 11448f0..88ec8a7 100644 --- a/ipapython/ipaldap.py +++ b/ipapython/ipaldap.py @@ -1245,11 +1245,11 @@ def make_filter_from_attr( ] return cls.combine_filters(flts, rules) elif value is not None: - if isinstance(value, bytes): - if six.PY3: - value = value.decode('raw_unicode_escape') + if isinstance(value, six.binary_type): + value = value.decode('raw_unicode_escape') else: - value = value_to_utf8(value) + value = six.text_type(value) + # escape_filter_chars requires string as parameter value = ldap.filter.escape_filter_chars(value) if not exact: template = '%s' From 3679cb755db164a3954bfe74f836e4aaaf083f16 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Wed, 11 Jan 2017 13:39:04 +0100 Subject: [PATCH 14/21] py3: convert_attribute_members: don't use bytes as parameter for DN due perfomance improvement in e4930b3235e5d61d227a7e43d30a8feb7f35664d we have to decode value before it can be used in DN() constructor. https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/plugins/baseldap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipaserver/plugins/baseldap.py b/ipaserver/plugins/baseldap.py index 9d6bfc7..e7bf43c 100644 --- a/ipaserver/plugins/baseldap.py +++ b/ipaserver/plugins/baseldap.py @@ -654,7 +654,7 @@ def convert_attribute_members(self, entry_attrs, *keys, **options): del entry_attrs[attr] for member in value: - memberdn = DN(member) + memberdn = DN(member.decode('utf-8')) for ldap_obj_name in self.attribute_members[attr]: ldap_obj = self.api.Object[ldap_obj_name] try: From bb170f436ca11ecd8df1406d0c032f1b66ff0d46 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Wed, 11 Jan 2017 14:38:25 +0100 Subject: [PATCH 15/21] dogtag.py: fix exception logging of JSON data 'read_ca' and 'create_ca' have no logging when exception happened and it masks real reason why it failed. --- ipaserver/plugins/dogtag.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ipaserver/plugins/dogtag.py b/ipaserver/plugins/dogtag.py index 73c14ed..142f838 100644 --- a/ipaserver/plugins/dogtag.py +++ b/ipaserver/plugins/dogtag.py @@ -2116,16 +2116,20 @@ def create_ca(self, dn): ) try: return json.loads(resp_body) - except: - raise errors.RemoteRetrieveError(reason=_("Response from CA was not valid JSON")) + except Exception as e: + self.log.debug(e, exc_info=True) + raise errors.RemoteRetrieveError( + reason=_("Response from CA was not valid JSON")) def read_ca(self, ca_id): _status, _resp_headers, resp_body = self._ssldo( 'GET', ca_id, headers={'Accept': 'application/json'}) try: return json.loads(resp_body) - except: - raise errors.RemoteRetrieveError(reason=_("Response from CA was not valid JSON")) + except Exception as e: + self.log.debug(e, exc_info=True) + raise errors.RemoteRetrieveError( + reason=_("Response from CA was not valid JSON")) def read_ca_cert(self, ca_id): _status, _resp_headers, resp_body = self._ssldo( From 19c4cd0d9992f2795190694fc617577b80aeba89 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Thu, 12 Jan 2017 16:20:43 +0100 Subject: [PATCH 16/21] py3: decode bytes for json.loads() In py 3.5 json.loads requires to have string as input, all bytes must be decoded. Note: python 3.6 supports bytes for json.loads() https://fedorahosted.org/freeipa/ticket/4985 --- .../certmonger/dogtag-ipa-ca-renew-agent-submit | 2 +- ipaclient/plugins/vault.py | 3 +- ipaclient/remote_plugins/schema.py | 4 +- ipalib/rpc.py | 3 +- ipapython/dogtag.py | 1 + ipapython/ipautil.py | 50 ++++++++++++++++++++++ ipaserver/plugins/dogtag.py | 9 ++-- ipaserver/rpcserver.py | 2 +- 8 files changed, 64 insertions(+), 10 deletions(-) diff --git a/install/certmonger/dogtag-ipa-ca-renew-agent-submit b/install/certmonger/dogtag-ipa-ca-renew-agent-submit index 2e137ad..a5c8d86 100755 --- a/install/certmonger/dogtag-ipa-ca-renew-agent-submit +++ b/install/certmonger/dogtag-ipa-ca-renew-agent-submit @@ -135,7 +135,7 @@ def call_handler(_handler, *args, **kwargs): cookie = os.environ.pop('CERTMONGER_CA_COOKIE', None) if cookie is not None: try: - context = json.loads(cookie) + context = json.loads(ipautil.decode_json(cookie)) if not isinstance(context, dict): raise TypeError except (TypeError, ValueError): diff --git a/ipaclient/plugins/vault.py b/ipaclient/plugins/vault.py index 29157c7..5f9ff0c 100644 --- a/ipaclient/plugins/vault.py +++ b/ipaclient/plugins/vault.py @@ -43,6 +43,7 @@ from ipalib import Bytes, Flag, Str from ipalib.plugable import Registry from ipalib import _ +from ipapython.ipautil import decode_json def validated_read(argname, filename, mode='r', encoding=None): @@ -969,7 +970,7 @@ def forward(self, *args, **options): json_vault_data = decoding_ctx.cipher_op(wrapped_vault_data)\ + decoding_ctx.digest_final() - vault_data = json.loads(json_vault_data) + vault_data = json.loads(decode_json(json_vault_data)) data = base64.b64decode(vault_data[u'data'].encode('utf-8')) encrypted_key = None diff --git a/ipaclient/remote_plugins/schema.py b/ipaclient/remote_plugins/schema.py index 15c03f4..626a3c5 100644 --- a/ipaclient/remote_plugins/schema.py +++ b/ipaclient/remote_plugins/schema.py @@ -21,7 +21,7 @@ from ipalib.frontend import Object from ipalib.output import Output from ipalib.parameters import DefaultFrom, Flag, Password, Str -from ipapython.ipautil import fsdecode +from ipapython.ipautil import fsdecode, decode_json from ipapython.dn import DN from ipapython.dnsutil import DNSName from ipapython.ipa_log_manager import log_mgr @@ -522,7 +522,7 @@ def _write_schema(self, fingerprint): def _read(self, path): with zipfile.ZipFile(self._file, 'r') as zf: - return json.loads(zf.read(path).decode('utf-8')) + return json.loads(decode_json(zf.read(path))) def read_namespace_member(self, namespace, member): value = self._dict[namespace][member] diff --git a/ipalib/rpc.py b/ipalib/rpc.py index 921f5cb..da710dc 100644 --- a/ipalib/rpc.py +++ b/ipalib/rpc.py @@ -1101,7 +1101,8 @@ def __request(self, name, args): ) try: - response = json_decode_binary(json.loads(response.decode('ascii'))) + response = json_decode_binary( + json.loads(ipautil.decode_json(response))) except ValueError as e: raise JSONError(error=str(e)) diff --git a/ipapython/dogtag.py b/ipapython/dogtag.py index c6a8346..01fc5cb 100644 --- a/ipapython/dogtag.py +++ b/ipapython/dogtag.py @@ -209,6 +209,7 @@ def _httplib_request( http_body = res.read() conn.close() except Exception as e: + root_logger.exception("httplib request failed:") raise NetworkError(uri=uri, error=str(e)) root_logger.debug('response status %d', http_status) diff --git a/ipapython/ipautil.py b/ipapython/ipautil.py index f2b3d74..c8f87ef 100644 --- a/ipapython/ipautil.py +++ b/ipapython/ipautil.py @@ -19,6 +19,7 @@ from __future__ import print_function +import codecs import string import tempfile import subprocess @@ -1361,6 +1362,55 @@ def escape_seq(seq, *args): return tuple(a.replace(seq, u'\\{}'.format(seq)) for a in args) +def decode_json(data): + """Decode JSON bytes to string with proper encoding + + Only for supporting Py 3.5 + + Py 3.6 supports bytes as parameter for json.load, we can drop this when + there is no need for python 3.5 anymore + + Code from: + https://bugs.python.org/file43513/json_detect_encoding_3.patch + + :param data: JSON bytes + :return: return JSON string + """ + + def detect_encoding(b): + bstartswith = b.startswith + if bstartswith((codecs.BOM_UTF32_BE, codecs.BOM_UTF32_LE)): + return 'utf-32' + if bstartswith((codecs.BOM_UTF16_BE, codecs.BOM_UTF16_LE)): + return 'utf-16' + if bstartswith(codecs.BOM_UTF8): + return 'utf-8-sig' + + if len(b) >= 4: + if not b[0]: + # 00 00 -- -- - utf-32-be + # 00 XX -- -- - utf-16-be + return 'utf-16-be' if b[1] else 'utf-32-be' + if not b[1]: + # XX 00 00 00 - utf-32-le + # XX 00 XX XX - utf-16-le + return 'utf-16-le' if b[2] or b[3] else 'utf-32-le' + elif len(b) == 2: + if not b[0]: + # 00 XX - utf-16-be + return 'utf-16-be' + if not b[1]: + # XX 00 - utf-16-le + return 'utf-16-le' + # default + return 'utf-8' + + if isinstance(data, six.text_type): + return data + + return data.decode(detect_encoding(data), 'surrogatepass') + + class APIVersion(tuple): """API version parser and handler diff --git a/ipaserver/plugins/dogtag.py b/ipaserver/plugins/dogtag.py index 142f838..fbfe608 100644 --- a/ipaserver/plugins/dogtag.py +++ b/ipaserver/plugins/dogtag.py @@ -1232,7 +1232,8 @@ class ra_certprofile(RestClient): @staticmethod def _parse_dogtag_error(body): try: - return pki.PKIException.from_json(json.loads(body)) + return pki.PKIException.from_json( + json.loads(ipautil.decode_json(body))) except Exception: return None @@ -1667,7 +1668,7 @@ def request_certificate( ) try: - resp_obj = json.loads(http_body) + resp_obj = json.loads(ipautil.decode_json(http_body)) except ValueError: raise errors.RemoteRetrieveError(reason=_("Response from CA was not valid JSON")) @@ -2115,7 +2116,7 @@ def create_ca(self, dn): body=json.dumps({"parentID": "host-authority", "dn": unicode(dn)}), ) try: - return json.loads(resp_body) + return json.loads(ipautil.decode_json(resp_body)) except Exception as e: self.log.debug(e, exc_info=True) raise errors.RemoteRetrieveError( @@ -2125,7 +2126,7 @@ def read_ca(self, ca_id): _status, _resp_headers, resp_body = self._ssldo( 'GET', ca_id, headers={'Accept': 'application/json'}) try: - return json.loads(resp_body) + return json.loads(ipautil.decode_json(resp_body)) except Exception as e: self.log.debug(e, exc_info=True) raise errors.RemoteRetrieveError( diff --git a/ipaserver/rpcserver.py b/ipaserver/rpcserver.py index 1da4ec4..c52f335 100644 --- a/ipaserver/rpcserver.py +++ b/ipaserver/rpcserver.py @@ -493,7 +493,7 @@ def marshal(self, result, error, _id=None, def unmarshal(self, data): try: - d = json.loads(data) + d = json.loads(ipautil.decode_json(data)) except ValueError as e: raise JSONError(error=e) if not isinstance(d, dict): From 7ff4b83a954b3fa1ee5e0dbd960b6a7a3d24f76f Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Wed, 11 Jan 2017 16:54:25 +0100 Subject: [PATCH 17/21] py3: session.py decode server name to str This fix is temporal because Memcache will be removed soon, so it is more workaround than fix https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipaserver/session.py b/ipaserver/session.py index 85deb15..020dcc1 100644 --- a/ipaserver/session.py +++ b/ipaserver/session.py @@ -828,7 +828,7 @@ def get_server_statistics(self): result = {} stats = self.mc.get_stats() for server in stats: - match = self.mc_server_stat_name_re.search(server[0]) + match = self.mc_server_stat_name_re.search(server[0].decode()) if match: name = match.group(1) result[name] = server[1] From a754021ff7f9dbcddb3fcfb4c48865d836e13992 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Wed, 11 Jan 2017 17:13:52 +0100 Subject: [PATCH 18/21] py3: rpcserver: decode input because json requires string json library parses string so input must be decoded https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/rpcserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipaserver/rpcserver.py b/ipaserver/rpcserver.py index c52f335..6659f7d 100644 --- a/ipaserver/rpcserver.py +++ b/ipaserver/rpcserver.py @@ -195,7 +195,7 @@ def read_input(environ): length = int(environ.get('CONTENT_LENGTH')) except (ValueError, TypeError): return - return environ['wsgi.input'].read(length) + return environ['wsgi.input'].read(length).decode('utf-8') def params_2_args_options(params): From 5b4c9d844784dc5704f43ade3bf8327f225a4bc0 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Wed, 11 Jan 2017 17:15:49 +0100 Subject: [PATCH 19/21] Py3: Fix undefined variable Variable 'e' has only local scope in except block in Py3 https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/rpcserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipaserver/rpcserver.py b/ipaserver/rpcserver.py index 6659f7d..4951d74 100644 --- a/ipaserver/rpcserver.py +++ b/ipaserver/rpcserver.py @@ -404,7 +404,7 @@ def wsgi_execute(self, environ): type(self).__name__, principal, name, - type(e).__name__) + type(error).__name__) version = options.get('version', VERSION_WITHOUT_CAPABILITIES) return self.marshal(result, error, _id, version) From 985deaf185f7e086c1a345e7d16daa15a4d07dd2 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Wed, 11 Jan 2017 17:24:16 +0100 Subject: [PATCH 20/21] py3: session: fix r/w ccache data ccache contains binary data, so it should be read and write in binary mode https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/session.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/ipaserver/session.py b/ipaserver/session.py index 020dcc1..0f3a9ad 100644 --- a/ipaserver/session.py +++ b/ipaserver/session.py @@ -21,6 +21,7 @@ import os import re import time +import io # pylint: disable=import-error from six.moves.urllib.parse import urlparse @@ -1228,9 +1229,8 @@ def load_ccache_data(ccache_name): scheme, name = krb5_parse_ccache(ccache_name) if scheme == 'FILE': root_logger.debug('reading ccache data from file "%s"', name) - src = open(name) - ccache_data = src.read() - src.close() + with io.open(name, "rb") as src: + ccache_data = src.read() return ccache_data else: raise ValueError('ccache scheme "%s" unsupported (%s)', scheme, ccache_name) @@ -1239,9 +1239,8 @@ def bind_ipa_ccache(ccache_data, scheme='FILE'): if scheme == 'FILE': name = _get_krbccache_pathname() root_logger.debug('storing ccache data into file "%s"', name) - dst = open(name, 'w') - dst.write(ccache_data) - dst.close() + with io.open(name, 'wb') as dst: + dst.write(ccache_data) else: raise ValueError('ccache scheme "%s" unsupported', scheme) From de44fbc914e66ced4a06fa7d99c45875100e62b2 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Thu, 12 Jan 2017 18:50:56 +0100 Subject: [PATCH 21/21] py3: WSGI executioners must return bytes in list WSGI prints TypeError into error log when IPA doesn't return bytes in list as result https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/rpcserver.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/ipaserver/rpcserver.py b/ipaserver/rpcserver.py index 4951d74..d48c75c 100644 --- a/ipaserver/rpcserver.py +++ b/ipaserver/rpcserver.py @@ -144,7 +144,7 @@ def not_found(self, environ, start_response, url, message): self.info('%s: URL="%s", %s', status, url, message) start_response(status, response_headers) output = _not_found_template % dict(url=escape(url)) - return [output] + return [output.encode('utf-8')] def bad_request(self, environ, start_response, message): """ @@ -157,7 +157,7 @@ def bad_request(self, environ, start_response, message): start_response(status, response_headers) output = _bad_request_template % dict(message=escape(message)) - return [output] + return [output.encode('utf-8')] def internal_error(self, environ, start_response, message): """ @@ -170,7 +170,7 @@ def internal_error(self, environ, start_response, message): start_response(status, response_headers) output = _internal_error_template % dict(message=escape(message)) - return [output] + return [output.encode('utf-8')] def unauthorized(self, environ, start_response, message, reason): """ @@ -185,7 +185,7 @@ def unauthorized(self, environ, start_response, message, reason): start_response(status, response_headers) output = _unauthorized_template % dict(message=escape(message)) - return [output] + return [output.encode('utf-8')] def read_input(environ): """ @@ -427,7 +427,7 @@ def __call__(self, environ, start_response): except Exception: self.exception('WSGI %s.__call__():', self.name) status = HTTP_STATUS_SERVER_ERROR - response = status + response = status.encode('utf-8') headers = [('Content-Type', 'text/plain; charset=utf-8')] session_data = getattr(context, 'session_data', None) @@ -489,7 +489,8 @@ def marshal(self, result, error, _id=None, version=unicode(VERSION), ) response = json_encode_binary(response, version) - return json.dumps(response, sort_keys=True, indent=4) + dump = json.dumps(response, sort_keys=True, indent=4) + return dump.encode('utf-8') def unmarshal(self, data): try: @@ -672,7 +673,7 @@ def __call__(self, environ, start_response): 'xmlserver', user_ccache, environ, start_response, headers) except PublicError as e: status = HTTP_STATUS_SUCCESS - response = status + response = status.encode('utf-8') start_response(status, headers) return self.marshal(None, e) finally: @@ -758,7 +759,8 @@ def marshal(self, result, error, _id=None, if isinstance(result, dict): self.debug('response: entries returned %d', result.get('count', 1)) response = (result,) - return xml_dumps(response, version, methodresponse=True) + dump = xml_dumps(response, version, methodresponse=True) + return dump.encode('utf-8') class jsonserver_session(jsonserver, KerberosSession): @@ -782,7 +784,7 @@ def _on_finalize(self): def need_login(self, start_response): status = '401 Unauthorized' headers = [] - response = '' + response = b'' self.debug('jsonserver_session: %s need login', status) @@ -1252,7 +1254,7 @@ def _on_finalize(self): def need_login(self, start_response): status = '401 Unauthorized' headers = [] - response = '' + response = b'' self.debug('xmlserver_session: %s need login', status)
-- Manage your subscription for the Freeipa-devel mailing list: https://www.redhat.com/mailman/listinfo/freeipa-devel Contribute to FreeIPA: http://www.freeipa.org/page/Contribute/Code
