URL: https://github.com/SSSD/sssd/pull/122 Author: lslebodn Title: #122: Port integration tests to python3 Action: opened
PR body: """ Attached patches fixed python 2/3 compatibility of integration tests. You will need to install following packages in fedora: python3-ldb, python3-pyldap, python3-requests And for testing purposes you can use ``` make intgcheck-run PYTHON2=/usr/bin/python2 make intgcheck-run PYTHON2=/usr/bin/python3 ``` I will need to figure out how to incorporate it into CI script. """ To pull the PR as Git branch: git remote add ghsssd https://github.com/SSSD/sssd git fetch ghsssd pull/122/head:pr122 git checkout pr122
From 686cd90b43904eb9b047a18b308471540ec12ce0 Mon Sep 17 00:00:00 2001 From: Lukas Slebodnik <[email protected]> Date: Wed, 21 Dec 2016 19:08:07 +0100 Subject: [PATCH 01/11] sssd_ldb.py: Remove a leftover debug message "print" is a function in python3. We can remove it because it is not important. Portable solution would be: from __future__ import print_function print (basedn) --- src/tests/intg/sssd_ldb.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tests/intg/sssd_ldb.py b/src/tests/intg/sssd_ldb.py index 35b389d..399ec8a 100644 --- a/src/tests/intg/sssd_ldb.py +++ b/src/tests/intg/sssd_ldb.py @@ -77,7 +77,6 @@ def _basedn(self, name, domain, entry_type): def get_entry_attr(self, cache_type, entry_type, name, domain, attr): dbconn = self._get_dbconn(cache_type) basedn = self._basedn(name, domain, entry_type) - print basedn res = dbconn.search(base=basedn, scope=ldb.SCOPE_BASE, attrs=[attr]) if res.count != 1: From eea9b156b95ff454274c6ca8bbafb80037f98d21 Mon Sep 17 00:00:00 2001 From: Lukas Slebodnik <[email protected]> Date: Wed, 21 Dec 2016 19:21:14 +0100 Subject: [PATCH 02/11] intg: Fix python2,3 urllib The three modules urllib, urllib2 and urlparse has been reorganized into three new modules, urllib.request, urllib.parse and urllib.error. And urllib.quote was moved into urllib.parse. --- src/tests/intg/ds_openldap.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/tests/intg/ds_openldap.py b/src/tests/intg/ds_openldap.py index fb230a0..c6bb7f2 100644 --- a/src/tests/intg/ds_openldap.py +++ b/src/tests/intg/ds_openldap.py @@ -19,7 +19,6 @@ import hashlib import base64 -import urllib import time import ldap import os @@ -30,6 +29,11 @@ from util import * from ds import DS +try: + from urllib import quote as url_quote +except ImportError: + from urllib.parse import quote as url_quote + def hash_password(password): """Generate userPassword value for a password.""" @@ -183,7 +187,7 @@ def _setup_config(self): def setup(self): """Setup the instance.""" ldapi_socket = self.run_dir + "/ldapi" - ldapi_url = "ldapi://" + urllib.quote(ldapi_socket, "") + ldapi_url = "ldapi://" + url_quote(ldapi_socket, "") url_list = ldapi_url + " " + self.ldap_url os.makedirs(self.conf_slapd_d_dir) From 47b296366469aae49a487b805a52d1501eef64f9 Mon Sep 17 00:00:00 2001 From: Lukas Slebodnik <[email protected]> Date: Thu, 12 Jan 2017 14:29:17 +0100 Subject: [PATCH 03/11] intg: Avoid using xrange in tests In Python 2 range() returns a list, and xrange() returns an object that will only generate the items in the range when needed, saving memory. In Python 3, the range() function is gone, and xrange() has been renamed range(). We do not strictly require list or iterator in code therefore we can use range which is in poth version. And we do not use big ranges there so the memory overhead with list on python2 is not big. --- src/tests/intg/ent.py | 6 +++--- src/tests/intg/test_secrets.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/tests/intg/ent.py b/src/tests/intg/ent.py index 2d3d02a..63c7e89 100644 --- a/src/tests/intg/ent.py +++ b/src/tests/intg/ent.py @@ -123,7 +123,7 @@ def _diff(ent, pattern, desc_map={}): if not d: pattern_matches[pi] += 1 - unmatched_pattern = [pattern[pi] for pi in xrange(0, len(pattern)) + unmatched_pattern = [pattern[pi] for pi in range(0, len(pattern)) if pattern_matches[pi] == 0] items = _get_desc(desc_map, None)[0] + "s" @@ -144,9 +144,9 @@ def _diff(ent, pattern, desc_map={}): pattern_matches[pi] += 1 ent_matches[ei] += 1 - unmatched_pattern = [pattern[pi] for pi in xrange(0, len(pattern)) + unmatched_pattern = [pattern[pi] for pi in range(0, len(pattern)) if pattern_matches[pi] == 0] - unmatched_ent = [ent[pi] for pi in xrange(0, len(ent)) + unmatched_ent = [ent[pi] for pi in range(0, len(ent)) if ent_matches[pi] == 0] items = _get_desc(desc_map, None)[0] + "s" diff --git a/src/tests/intg/test_secrets.py b/src/tests/intg/test_secrets.py index 7a9de1a..062dcb6 100644 --- a/src/tests/intg/test_secrets.py +++ b/src/tests/intg/test_secrets.py @@ -145,7 +145,7 @@ def test_crd_ops(setup_for_secrets, secrets_cli): MAX_SECRETS = 10 sec_value = "value" - for x in xrange(MAX_SECRETS): + for x in range(MAX_SECRETS): cli.set_secret(str(x), sec_value) with pytest.raises(HTTPError) as err507: @@ -153,7 +153,7 @@ def test_crd_ops(setup_for_secrets, secrets_cli): assert str(err507.value).startswith("507") # Delete all stored secrets used for max secrets tests - for x in xrange(MAX_SECRETS): + for x in range(MAX_SECRETS): cli.del_secret(str(x)) # Don't allow storing a secrets which has a payload larger @@ -198,7 +198,7 @@ def test_containers(setup_for_secrets, secrets_cli): # Don't allow creating a container after reaching the max nested level DEFAULT_CONTAINERS_NEST_LEVEL = 4 container = "mycontainer" - for x in xrange(DEFAULT_CONTAINERS_NEST_LEVEL): + for x in range(DEFAULT_CONTAINERS_NEST_LEVEL): container += "%s/" % str(x) cli.create_container(container) From a1a52067d43718f33630d9eed3ea4cb92cd684dc Mon Sep 17 00:00:00 2001 From: Lukas Slebodnik <[email protected]> Date: Thu, 12 Jan 2017 14:09:40 +0100 Subject: [PATCH 04/11] intg: Avoid using iteritems for dictionary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Originally, Python items() built a real list of tuples and returned that. That could potentially take a lot of extra memory. Python iteritems() returned an iterator-generator. The original remains for backwards compatibility. One of Python 3’s changes is that items() now return iterators, and a list is never fully built. The iteritems() method is also gone, since items() in Python 3 works like viewitems() in Python2. But we do not have a lot of values in dictionary; so it does not worth to optimize returned list from "items()" on Python2 --- src/tests/intg/ent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/intg/ent.py b/src/tests/intg/ent.py index 63c7e89..bb9870d 100644 --- a/src/tests/intg/ent.py +++ b/src/tests/intg/ent.py @@ -106,7 +106,7 @@ def _diff(ent, pattern, desc_map={}): if not isinstance(ent, dict): return "not a dict, " + str(type(ent)) - for key, value in pattern.iteritems(): + for key, value in pattern.items(): item_name, item_map = _get_desc(desc_map, key) d = _diff(ent[key], value, item_map) if d: From 76154828ba4581067067ca1c9fb19db470023780 Mon Sep 17 00:00:00 2001 From: Lukas Slebodnik <[email protected]> Date: Wed, 21 Dec 2016 19:29:20 +0100 Subject: [PATCH 05/11] intg: Use bytes with hash function Python3 expects bytes as an input for hash function. We need to convert string to bytes before hashing Traceback (most recent call last): File "src/tests/intg/test_ldap.py", line 51, in ds_inst ds_inst.setup() File "src/tests/intg/ds_openldap.py", line 200, in setup self._setup_config() File "src/tests/intg/ds_openldap.py", line 76, in _setup_config admin_pw_hash = hash_password(self.admin_pw) File "src/tests/intg/ds_openldap.py", line 41, in hash_password hash = hashlib.sha1(password) TypeError: Unicode-objects must be encoded before hashing --- src/tests/intg/ds_openldap.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/tests/intg/ds_openldap.py b/src/tests/intg/ds_openldap.py index c6bb7f2..bbc101a 100644 --- a/src/tests/intg/ds_openldap.py +++ b/src/tests/intg/ds_openldap.py @@ -38,9 +38,10 @@ def hash_password(password): """Generate userPassword value for a password.""" salt = os.urandom(4) - hash = hashlib.sha1(password) + hash = hashlib.sha1(password.encode('utf-8')) hash.update(salt) - return "{SSHA}" + base64.standard_b64encode(hash.digest() + salt) + hash_base64 = base64.standard_b64encode(hash.digest() + salt) + return "{SSHA}" + hash_base64.decode('utf-8') class DSOpenLDAP(DS): From 17005e2c7fe1128e35af9565169b82051a0cda7e Mon Sep 17 00:00:00 2001 From: Lukas Slebodnik <[email protected]> Date: Wed, 21 Dec 2016 19:30:57 +0100 Subject: [PATCH 06/11] intg: Fix creating of slapd configuration The python module subprocess expect bytes as an input. Traceback (most recent call last): File "src/tests/intg/test_ldap.py", line 51, in ds_inst ds_inst.setup() File "src/tests/intg/ds_openldap.py", line 201, in setup self._setup_config() File "src/tests/intg/ds_openldap.py", line 169, in _setup_config slapadd.communicate(config) File "/usr/lib64/python3.6/subprocess.py", line 821, in communicate self._stdin_write(input) File "/usr/lib64/python3.6/subprocess.py", line 776, in _stdin_write self.stdin.write(input) TypeError: a bytes-like object is required, not 'str' --- src/tests/intg/ds_openldap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/intg/ds_openldap.py b/src/tests/intg/ds_openldap.py index bbc101a..0f58e51 100644 --- a/src/tests/intg/ds_openldap.py +++ b/src/tests/intg/ds_openldap.py @@ -166,7 +166,7 @@ def _setup_config(self): ["slapadd", "-F", self.conf_slapd_d_dir, "-b", "cn=config"], stdin=subprocess.PIPE, close_fds=True ) - slapadd.communicate(config) + slapadd.communicate(config.encode('utf-8')) if slapadd.returncode != 0: raise Exception("Failed to add configuration with slapadd") From c40b26c631852fa9678960ebc1474f26ed190745 Mon Sep 17 00:00:00 2001 From: Lukas Slebodnik <[email protected]> Date: Thu, 12 Jan 2017 14:01:52 +0100 Subject: [PATCH 07/11] intg: Use bytes for value of attributes in ldif Python3 version of ldap module require string for name of attribute but bytes for value of attribute. It was not a problem in python2 due to unicode changes in python3 --- src/tests/intg/ds_openldap.py | 24 ++++++++++---------- src/tests/intg/ldap_ent.py | 46 +++++++++++++++++++++----------------- src/tests/intg/test_enumeration.py | 8 ++++--- src/tests/intg/test_ldap.py | 24 ++++++++++---------- src/tests/intg/test_netgroup.py | 6 ++--- src/tests/intg/test_ts_cache.py | 15 +++++++------ 6 files changed, 65 insertions(+), 58 deletions(-) diff --git a/src/tests/intg/ds_openldap.py b/src/tests/intg/ds_openldap.py index 0f58e51..6a074c9 100644 --- a/src/tests/intg/ds_openldap.py +++ b/src/tests/intg/ds_openldap.py @@ -233,15 +233,15 @@ def setup(self): # modlist = [ (ldap.MOD_DELETE, "olcObjectClasses", - "{7}( 2.5.6.9 NAME 'groupOfNames' " - "DESC 'RFC2256: a group of names (DNs)' SUP top " - "STRUCTURAL MUST ( member $ cn ) MAY ( businessCategory $ " - "seeAlso $ owner $ ou $ o $ description ) )"), + b"{7}( 2.5.6.9 NAME 'groupOfNames' " + b"DESC 'RFC2256: a group of names (DNs)' SUP top " + b"STRUCTURAL MUST ( member $ cn ) MAY ( businessCategory $ " + b"seeAlso $ owner $ ou $ o $ description ) )"), (ldap.MOD_ADD, "olcObjectClasses", - "{7}( 2.5.6.9 NAME 'groupOfNames' " - "DESC 'RFC2256: a group of names (DNs)' SUP top " - "STRUCTURAL MUST ( cn ) MAY ( member $ businessCategory $ " - "seeAlso $ owner $ ou $ o $ description ) )"), + b"{7}( 2.5.6.9 NAME 'groupOfNames' " + b"DESC 'RFC2256: a group of names (DNs)' SUP top " + b"STRUCTURAL MUST ( cn ) MAY ( member $ businessCategory $ " + b"seeAlso $ owner $ ou $ o $ description ) )"), ] ldap_conn = ldap.initialize(ldapi_url) ldap_conn.simple_bind_s(self.admin_rdn + ",cn=config", self.admin_pw) @@ -254,15 +254,15 @@ def setup(self): ldap_conn = ldap.initialize(self.ldap_url) ldap_conn.simple_bind_s(self.admin_dn, self.admin_pw) ldap_conn.add_s(self.base_dn, [ - ("objectClass", ["dcObject", "organization"]), - ("o", "Example Company"), + ("objectClass", [b"dcObject", b"organization"]), + ("o", b"Example Company"), ]) ldap_conn.add_s("cn=Manager," + self.base_dn, [ - ("objectClass", "organizationalRole"), + ("objectClass", b"organizationalRole"), ]) for ou in ("Users", "Groups", "Netgroups", "Services", "Policies"): ldap_conn.add_s("ou=" + ou + "," + self.base_dn, [ - ("objectClass", ["top", "organizationalUnit"]), + ("objectClass", [b"top", b"organizationalUnit"]), ]) ldap_conn.unbind_s() diff --git a/src/tests/intg/ldap_ent.py b/src/tests/intg/ldap_ent.py index c912844..6b6d8f9 100644 --- a/src/tests/intg/ldap_ent.py +++ b/src/tests/intg/ldap_ent.py @@ -28,53 +28,54 @@ def user(base_dn, uid, uidNumber, gidNumber, """ Generate an RFC2307(bis) user add-modlist for passing to ldap.add* """ - uidNumber = str(uidNumber) - gidNumber = str(gidNumber) + uidNumber = str(uidNumber).encode('utf-8') + gidNumber = str(gidNumber).encode('utf-8') user = ( "uid=" + uid + ",ou=Users," + base_dn, [ - ('objectClass', ['top', 'inetOrgPerson', 'posixAccount']), - ('cn', [uidNumber if cn is None else cn]), - ('sn', ['User' if sn is None else sn]), + ('objectClass', [b'top', b'inetOrgPerson', b'posixAccount']), + ('cn', [uidNumber if cn is None else cn.encode('utf-8')]), + ('sn', [b'User' if sn is None else sn.encode('utf-8')]), ('uidNumber', [uidNumber]), ('gidNumber', [gidNumber]), - ('userPassword', ['Password' + uidNumber + ('userPassword', [b'Password' + uidNumber if userPassword is None - else userPassword]), - ('homeDirectory', ['/home/' + uid + else userPassword.encode('utf-8')]), + ('homeDirectory', [b'/home/' + uid.encode('utf-8') if homeDirectory is None - else homeDirectory]), - ('loginShell', ['/bin/bash' + else homeDirectory.encode('utf-8')]), + ('loginShell', [b'/bin/bash' if loginShell is None - else loginShell]), + else loginShell.encode('utf-8')]), ] ) if gecos is not None: - user[1].append(('gecos', [gecos])) + user[1].append(('gecos', [gecos.encode('utf-8')])) return user -def group(base_dn, cn, gidNumber, member_uids=[]): +def group(base_dn, cn, gidNumber, member_uids=()): """ Generate an RFC2307 group add-modlist for passing to ldap.add*. """ - gidNumber = str(gidNumber) + gidNumber = str(gidNumber).encode('utf-8') attr_list = [ - ('objectClass', ['top', 'posixGroup']), + ('objectClass', [b'top', b'posixGroup']), ('gidNumber', [gidNumber]) ] if len(member_uids) > 0: - attr_list.append(('memberUid', member_uids)) + mem_uids = [member.encode('utf-8') for member in member_uids] + attr_list.append(('memberUid', mem_uids)) return ("cn=" + cn + ",ou=Groups," + base_dn, attr_list) -def group_bis(base_dn, cn, gidNumber, member_uids=[], member_gids=[]): +def group_bis(base_dn, cn, gidNumber, member_uids=(), member_gids=()): """ Generate an RFC2307bis group add-modlist for passing to ldap.add*. """ - gidNumber = str(gidNumber) + gidNumber = str(gidNumber).encode('utf-8') attr_list = [ - ('objectClass', ['top', 'extensibleObject', 'groupOfNames']), + ('objectClass', [b'top', b'extensibleObject', b'groupOfNames']), ('gidNumber', [gidNumber]) ] member_list = [] @@ -83,7 +84,8 @@ def group_bis(base_dn, cn, gidNumber, member_uids=[], member_gids=[]): for gid in member_gids: member_list.append("cn=" + gid + ",ou=Groups," + base_dn) if len(member_list) > 0: - attr_list.append(('member', member_list)) + mem_list = [member.encode('utf-8') for member in member_list] + attr_list.append(('member', mem_list)) return ("cn=" + cn + ",ou=Groups," + base_dn, attr_list) @@ -92,11 +94,13 @@ def netgroup(base_dn, cn, triples=(), members=()): Generate an RFC2307bis netgroup add-modlist for passing to ldap.add*. """ attr_list = [ - ('objectClass', ['top', 'nisNetgroup']) + ('objectClass', [b'top', b'nisNetgroup']) ] if triples: + triples = [triple.encode('utf-8') for triple in triples] attr_list.append(('nisNetgroupTriple', triples)) if members: + members = [member.encode('utf-8') for member in members] attr_list.append(('memberNisNetgroup', members)) return ("cn=" + cn + ",ou=Netgroups," + base_dn, attr_list) diff --git a/src/tests/intg/test_enumeration.py b/src/tests/intg/test_enumeration.py index ece09f7..5cb6c3e 100644 --- a/src/tests/intg/test_enumeration.py +++ b/src/tests/intg/test_enumeration.py @@ -446,7 +446,7 @@ def test_add_remove_membership_rfc2307(ldap_conn, user_and_group_rfc2307): # Add user to group ent.assert_group_by_name("group", dict(mem=ent.contains_only())) ldap_conn.modify_s("cn=group,ou=Groups," + ldap_conn.ds_inst.base_dn, - [(ldap.MOD_REPLACE, "memberUid", "user")]) + [(ldap.MOD_REPLACE, "memberUid", b"user")]) time.sleep(INTERACTIVE_TIMEOUT) ent.assert_group_by_name("group", dict(mem=ent.contains_only("user"))) # Remove user from group @@ -462,19 +462,21 @@ def test_add_remove_membership_rfc2307_bis(ldap_conn, Test user and group membership addition and removal are reflected by SSSD, with RFC2307bis schema """ + base_dn_bytes = ldap_conn.ds_inst.base_dn.encode('utf-8') + time.sleep(INTERACTIVE_TIMEOUT/2) # Add user to group1 ent.assert_group_by_name("group1", dict(mem=ent.contains_only())) ldap_conn.modify_s("cn=group1,ou=Groups," + ldap_conn.ds_inst.base_dn, [(ldap.MOD_REPLACE, "member", - "uid=user,ou=Users," + ldap_conn.ds_inst.base_dn)]) + b"uid=user,ou=Users," + base_dn_bytes)]) time.sleep(INTERACTIVE_TIMEOUT) ent.assert_group_by_name("group1", dict(mem=ent.contains_only("user"))) # Add group1 to group2 ldap_conn.modify_s("cn=group2,ou=Groups," + ldap_conn.ds_inst.base_dn, [(ldap.MOD_REPLACE, "member", - "cn=group1,ou=Groups," + ldap_conn.ds_inst.base_dn)]) + b"cn=group1,ou=Groups," + base_dn_bytes)]) time.sleep(INTERACTIVE_TIMEOUT) ent.assert_group_by_name("group2", dict(mem=ent.contains_only("user"))) diff --git a/src/tests/intg/test_ldap.py b/src/tests/intg/test_ldap.py index bf25d95..848cb41 100644 --- a/src/tests/intg/test_ldap.py +++ b/src/tests/intg/test_ldap.py @@ -770,13 +770,13 @@ def test_extra_attribute_already_exists(ldap_conn, extra_attributes): user = 'user' extra_attribute = 'givenName' - given_name = 'unix_user' + given_name = b'unix_user' user_dn = "uid=" + user + ",ou=Users," + ldap_conn.ds_inst.base_dn - old = {'objectClass': ['top', 'inetOrgPerson', 'posixAccount']} - new = {'objectClass': ['top', 'inetOrgPerson', 'posixAccount', - 'extensibleObject']} + old = {'objectClass': [b'top', b'inetOrgPerson', b'posixAccount']} + new = {'objectClass': [b'top', b'inetOrgPerson', b'posixAccount', + b'extensibleObject']} ldif = ldap.modlist.modifyModlist(old, new) ldap_conn.modify_s(user_dn, ldif) @@ -848,9 +848,9 @@ def test_remove_user_from_group(ldap_conn, remove_user_from_group): dict(mem=ent.contains_only("user1", "user2"))) # removing of user2 from group1 - old = {'member': ["uid=user1,ou=Users,dc=example,dc=com", - "uid=user2,ou=Users,dc=example,dc=com"]} - new = {'member': ["uid=user1,ou=Users,dc=example,dc=com"]} + old = {'member': [b"uid=user1,ou=Users,dc=example,dc=com", + b"uid=user2,ou=Users,dc=example,dc=com"]} + new = {'member': [b"uid=user1,ou=Users,dc=example,dc=com"]} ldif = ldap.modlist.modifyModlist(old, new) ldap_conn.modify_s(group1_dn, ldif) @@ -863,7 +863,7 @@ def test_remove_user_from_group(ldap_conn, remove_user_from_group): ent.assert_group_by_name("group1", dict(mem=ent.contains_only("user1"))) # removing of user1 from group1 - old = {'member': ["uid=user1,ou=Users,dc=example,dc=com"]} + old = {'member': [b"uid=user1,ou=Users,dc=example,dc=com"]} new = {'member': []} ldif = ldap.modlist.modifyModlist(old, new) @@ -912,9 +912,9 @@ def test_remove_user_from_nested_group(ldap_conn, "user2"))) # removing of group2 from group3 - old = {'member': ["cn=group1,ou=Groups,dc=example,dc=com", - "cn=group2,ou=Groups,dc=example,dc=com"]} - new = {'member': ["cn=group1,ou=Groups,dc=example,dc=com"]} + old = {'member': [b"cn=group1,ou=Groups,dc=example,dc=com", + b"cn=group2,ou=Groups,dc=example,dc=com"]} + new = {'member': [b"cn=group1,ou=Groups,dc=example,dc=com"]} ldif = ldap.modlist.modifyModlist(old, new) ldap_conn.modify_s(group3_dn, ldif) @@ -933,7 +933,7 @@ def test_remove_user_from_nested_group(ldap_conn, dict(mem=ent.contains_only("user1"))) # removing of group1 from group3 - old = {'member': ["cn=group1,ou=Groups,dc=example,dc=com"]} + old = {'member': [b"cn=group1,ou=Groups,dc=example,dc=com"]} new = {'member': []} ldif = ldap.modlist.modifyModlist(old, new) diff --git a/src/tests/intg/test_netgroup.py b/src/tests/intg/test_netgroup.py index f1d801f..3cf5dac 100644 --- a/src/tests/intg/test_netgroup.py +++ b/src/tests/intg/test_netgroup.py @@ -426,8 +426,8 @@ def test_removing_nested_netgroups(removing_nested_netgroups, ldap_conn): ('host2', 'user2', 'domain2')]) # removing of t2841_netgroup1 from t2841_netgroup3 - old = {'memberNisNetgroup': ["t2841_netgroup1", "t2841_netgroup2"]} - new = {'memberNisNetgroup': ["t2841_netgroup2"]} + old = {'memberNisNetgroup': [b"t2841_netgroup1", b"t2841_netgroup2"]} + new = {'memberNisNetgroup': [b"t2841_netgroup2"]} ldif = ldap.modlist.modifyModlist(old, new) ldap_conn.modify_s(netgrp_dn, ldif) @@ -448,7 +448,7 @@ def test_removing_nested_netgroups(removing_nested_netgroups, ldap_conn): assert netgroups == [('host2', 'user2', 'domain2')] # removing of t2841_netgroup2 from t2841_netgroup3 - old = {'memberNisNetgroup': ["t2841_netgroup2"]} + old = {'memberNisNetgroup': [b"t2841_netgroup2"]} new = {'memberNisNetgroup': []} ldif = ldap.modlist.modifyModlist(old, new) diff --git a/src/tests/intg/test_ts_cache.py b/src/tests/intg/test_ts_cache.py index c49a5c9..c327f3a 100644 --- a/src/tests/intg/test_ts_cache.py +++ b/src/tests/intg/test_ts_cache.py @@ -319,7 +319,7 @@ def test_group_2307bis_update_same_attrs(ldap_conn, # modifyTimestamp attribute, but the attributes themselves will be the same # from sssd's point of view ldap_conn.modify_s("cn=group1,ou=Groups," + ldap_conn.ds_inst.base_dn, - [(ldap.MOD_ADD, "description", "group one")]) + [(ldap.MOD_ADD, "description", b"group one")]) # wait for slapd to change its database time.sleep(1) @@ -348,9 +348,10 @@ def test_group_2307bis_update_diff_attrs(ldap_conn, ldb_conn, "group1", ("user1", "user11", "user21")) + user_dn = b"uid=user1,ou=Users," + \ + ldap_conn.ds_inst.base_dn.encode('utf-8') ldap_conn.modify_s("cn=group1,ou=Groups," + ldap_conn.ds_inst.base_dn, - [(ldap.MOD_DELETE, "member", - "uid=user1,ou=Users," + ldap_conn.ds_inst.base_dn)]) + [(ldap.MOD_DELETE, "member", user_dn)]) # wait for slapd to change its database time.sleep(1) @@ -437,7 +438,7 @@ def test_group_2307_update_same_attrs(ldap_conn, # modifyTimestamp attribute, but the attributes themselves will be the same # from sssd's point of view ldap_conn.modify_s("cn=group1,ou=Groups," + ldap_conn.ds_inst.base_dn, - [(ldap.MOD_ADD, "description", "group one")]) + [(ldap.MOD_ADD, "description", b"group one")]) # wait for slapd to change its database time.sleep(1) @@ -467,7 +468,7 @@ def test_group_2307_update_diff_attrs(ldap_conn, ("user1", "user11", "user21")) ldap_conn.modify_s("cn=group1,ou=Groups," + ldap_conn.ds_inst.base_dn, - [(ldap.MOD_DELETE, "memberUid", "user1")]) + [(ldap.MOD_DELETE, "memberUid", b"user1")]) # wait for slapd to change its database time.sleep(1) @@ -548,7 +549,7 @@ def test_user_update_same_attrs(ldap_conn, # modifyTimestamp attribute, but the attributes themselves will be the same # from sssd's point of view ldap_conn.modify_s("uid=user1,ou=Users," + ldap_conn.ds_inst.base_dn, - [(ldap.MOD_ADD, "description", "user one")]) + [(ldap.MOD_ADD, "description", b"user one")]) # wait for slapd to change its database time.sleep(1) @@ -578,7 +579,7 @@ def test_user_update_diff_attrs(ldap_conn, # modifyTimestamp attribute, but the attributes themselves will be the same # from sssd's point of view ldap_conn.modify_s("uid=user1,ou=Users," + ldap_conn.ds_inst.base_dn, - [(ldap.MOD_REPLACE, "loginShell", "/bin/zsh")]) + [(ldap.MOD_REPLACE, "loginShell", b"/bin/zsh")]) # wait for slapd to change its database time.sleep(1) From 9e7f8dd19bfae110ffa0827ab6044ab1ab9e07e2 Mon Sep 17 00:00:00 2001 From: Lukas Slebodnik <[email protected]> Date: Thu, 12 Jan 2017 16:00:46 +0100 Subject: [PATCH 08/11] intg: Use bytes as input in ctypes Python module ctypes directly uses C functions and C functions expect "char *" as a string and not pointer to wide characters "wchar *". --- src/tests/intg/sssd_id.py | 4 ++-- src/tests/intg/sssd_netgroup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tests/intg/sssd_id.py b/src/tests/intg/sssd_id.py index 95c9ab7..4f020c3 100644 --- a/src/tests/intg/sssd_id.py +++ b/src/tests/intg/sssd_id.py @@ -61,8 +61,8 @@ def call_sssd_initgroups(user, gid): limit = c_long(-1) errno = POINTER(c_int)(c_int(0)) - res = func(c_char_p(user), c_uint32(gid), start, size, p_groups, limit, - errno) + res = func(c_char_p(user.encode('utf-8)')), c_uint32(gid), start, size, + p_groups, limit, errno) gids = [] if res == NssReturnCode.SUCCESS: diff --git a/src/tests/intg/sssd_netgroup.py b/src/tests/intg/sssd_netgroup.py index 2c7f76f..5b22469 100644 --- a/src/tests/intg/sssd_netgroup.py +++ b/src/tests/intg/sssd_netgroup.py @@ -73,7 +73,7 @@ class Netgrent(Structure): class NetgroupRetriever(object): def __init__(self, name): - self.name = name + self.name = name.encode('utf-8') self.needed_groups = [] self.known_groups = [] self.netgroups = [] From f586294b8e24babaf543cdc4414518316bcbf7a4 Mon Sep 17 00:00:00 2001 From: Lukas Slebodnik <[email protected]> Date: Thu, 12 Jan 2017 16:04:22 +0100 Subject: [PATCH 09/11] intg: Return strings from ctypes wrappers Python module ctypes directly uses C functions from libraries. C functions usually returns "char *" when string is expected. But python3 uses unicode for string. Encoding returned bytes ("char *") to unicode strings simplify tests in python3 --- src/tests/intg/sssd_netgroup.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/tests/intg/sssd_netgroup.py b/src/tests/intg/sssd_netgroup.py index 5b22469..0ea3db3 100644 --- a/src/tests/intg/sssd_netgroup.py +++ b/src/tests/intg/sssd_netgroup.py @@ -225,9 +225,10 @@ def _flat_fetch_netgroups(self, name): self.known_groups.append(nested_netgroup) if result_p[0].type == NetgroupType.TRIPLE_VAL: - result.append((result_p[0].val.triple.host, - result_p[0].val.triple.user, - result_p[0].val.triple.domain)) + triple = result_p[0].val.triple + result.append((triple.host.decode('utf-8'), + triple.user.decode('utf-8'), + triple.domain.decode('utf-8'))) res, errno, result_p = self._getnetgrent_r(result_p, buff, buff_len) From 220e3d1549ae6823e7c5f9ab48dbf64908d2dc30 Mon Sep 17 00:00:00 2001 From: Lukas Slebodnik <[email protected]> Date: Thu, 12 Jan 2017 16:21:20 +0100 Subject: [PATCH 10/11] intg: Convert output of executed commands to strings We know that all our tested utilities (sssctl, sss_cache, sss_override) returns strings and not binary data. We can safely encode output to string (unicode) in python3 --- src/tests/intg/ldap_local_override_test.py | 9 +++++---- src/tests/intg/test_sssctl.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/tests/intg/ldap_local_override_test.py b/src/tests/intg/ldap_local_override_test.py index 714268f..d78f3cc 100644 --- a/src/tests/intg/ldap_local_override_test.py +++ b/src/tests/intg/ldap_local_override_test.py @@ -586,11 +586,12 @@ def env_show_user_override(request, ldap_conn, def test_show_user_override(ldap_conn, env_show_user_override): - out = check_output(['sss_override', 'user-show', 'user1']) + out = check_output(['sss_override', 'user-show', 'user1']).decode('utf-8') assert out == "user1@LDAP:ov_user1:10010:20010:Overriden User 1:"\ "/home/ov/user1:/bin/ov_user1_shell:\n" - out = check_output(['sss_override', 'user-show', 'user2@LDAP']) + out = check_output(['sss_override', 'user-show', + 'user2@LDAP']).decode('utf-8') assert out == "user2@LDAP:ov_user2:10020:20020:Overriden User 2:"\ "/home/ov/user2:/bin/ov_user2_shell:\n" @@ -612,7 +613,7 @@ def env_find_user_override(request, ldap_conn, def test_find_user_override(ldap_conn, env_find_user_override): - out = check_output(['sss_override', 'user-find']) + out = check_output(['sss_override', 'user-find']).decode('utf-8') # Expected override of users exp_usr_ovrd = ['user1@LDAP:ov_user1:10010:20010:Overriden User 1:' @@ -624,7 +625,7 @@ def test_find_user_override(ldap_conn, env_find_user_override): out = check_output(['sss_override', 'user-find', '--domain=LDAP']) - assert set(out.splitlines()) == set(exp_usr_ovrd) + assert set(out.decode('utf-8').splitlines()) == set(exp_usr_ovrd) # Unexpected parameter is reported ret = subprocess.call(['sss_override', 'user-find', 'PARAM']) diff --git a/src/tests/intg/test_sssctl.py b/src/tests/intg/test_sssctl.py index c485996..0df5d0b 100644 --- a/src/tests/intg/test_sssctl.py +++ b/src/tests/intg/test_sssctl.py @@ -207,7 +207,7 @@ def get_call_output(cmd): process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, ret = process.communicate() - return output + return output.decode('utf-8') def test_user_show_basic_sanity(ldap_conn, sanity_rfc2307, portable_LC_ALL): From eb46788f91706b44f1038002871849cdc01ab6af Mon Sep 17 00:00:00 2001 From: Lukas Slebodnik <[email protected]> Date: Thu, 12 Jan 2017 17:06:40 +0100 Subject: [PATCH 11/11] intg: Return list for enumeration functions The documentation of get_passwd_list/get_group_list says that they return group/user database entry list. However, ther return class 'map' with python3 due to changes in high level function "map". Traceback (most recent call last): File "/dev/shm/sssd/src/tests/intg/ent_test.py", line 141, in test_assert_passwd_list ent.assert_passwd_list(ent.contains()) File "/dev/shm/sssd/src/tests/intg/ent.py", line 243, in assert_passwd_list assert not d, d AssertionError: not a list, <class 'map'> --- src/tests/intg/ent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/intg/ent.py b/src/tests/intg/ent.py index bb9870d..23236ee 100644 --- a/src/tests/intg/ent.py +++ b/src/tests/intg/ent.py @@ -233,7 +233,7 @@ def get_passwd_list(): for i, v in enumerate(passwd_list): if v.pw_name == "root" and v.pw_uid == 0 and v.pw_gid == 0: del passwd_list[i] - return map(_convert_passwd, passwd_list) + return list(map(_convert_passwd, passwd_list)) raise Exception("no root user found") @@ -393,7 +393,7 @@ def get_group_list(): for i, v in enumerate(group_list): if v.gr_name == "root" and v.gr_gid == 0: del group_list[i] - return map(_convert_group, group_list) + return list(map(_convert_group, group_list)) raise Exception("no root group found")
_______________________________________________ sssd-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
