URL: https://github.com/freeipa/freeipa/pull/2111 Author: tiran Title: #2111: Tune DS replication settings Action: opened
PR body: """ Tune 389-DS replication settings to improve performance and avoid timeouts. During installation of a replica, the value of nsDS5ReplicaBindDnGroupCheckInterval is reduced to 2 seconds. At the end of the installation, the value is increased sensible production settings. This avoids long delays during replication. See: https://pagure.io/freeipa/issue/7617 Signed-off-by: Christian Heimes <chei...@redhat.com> """ To pull the PR as Git branch: git remote add ghfreeipa https://github.com/freeipa/freeipa git fetch ghfreeipa pull/2111/head:pr2111 git checkout pr2111
From 5de55b1014647dca09b6e111771af4241a69a722 Mon Sep 17 00:00:00 2001 From: Christian Heimes <chei...@redhat.com> Date: Tue, 3 Jul 2018 19:40:05 +0200 Subject: [PATCH] Tune DS replication settings Tune 389-DS replication settings to improve performance and avoid timeouts. During installation of a replica, the value of nsDS5ReplicaBindDnGroupCheckInterval is reduced to 2 seconds. At the end of the installation, the value is increased sensible production settings. This avoids long delays during replication. See: https://pagure.io/freeipa/issue/7617 Signed-off-by: Christian Heimes <chei...@redhat.com> --- ipaserver/install/cainstance.py | 7 ++- ipaserver/install/dsinstance.py | 48 ++++++++++------- ipaserver/install/replication.py | 74 +++++++++++++++++++-------- ipaserver/install/server/replicainstall.py | 2 + ipaserver/install/server/upgrade.py | 14 +++-- ipatests/test_integration/test_external_ca.py | 5 +- 6 files changed, 106 insertions(+), 44 deletions(-) diff --git a/ipaserver/install/cainstance.py b/ipaserver/install/cainstance.py index 0fcd579d1c..313809df14 100644 --- a/ipaserver/install/cainstance.py +++ b/ipaserver/install/cainstance.py @@ -419,6 +419,8 @@ def configure_instance(self, host_name, dm_password, admin_password, self.teardown_admin) self.step("starting certificate server instance", self.start_instance) + self.step("Finalize replication settings", + self.finalize_replica_config) # Step 1 of external is getting a CSR so we don't need to do these # steps until we get a cert back from the external CA. if self.external != 1: @@ -1253,13 +1255,16 @@ def __create_ds_db(self): api.Backend.ldap2.add_entry(entry) def __setup_replication(self): - repl = replication.CAReplicationManager(self.realm, self.fqdn) repl.setup_cs_replication(self.master_host) # Activate Topology for o=ipaca segments self.__update_topology() + def finalize_replica_config(self): + repl = replication.CAReplicationManager(self.realm, self.fqdn) + repl.finalize_replica_config(self.master_host) + def __enable_instance(self): basedn = ipautil.realm_to_suffix(self.realm) if not self.clone: diff --git a/ipaserver/install/dsinstance.py b/ipaserver/install/dsinstance.py index d00799a47d..492bc67474 100644 --- a/ipaserver/install/dsinstance.py +++ b/ipaserver/install/dsinstance.py @@ -418,6 +418,20 @@ def create_replica(self, realm_name, master_fqdn, fqdn, self.start_creation(runtime=30) + def _get_replication_manager(self): + # Always connect to self over ldapi + ldap_uri = ipaldap.get_ldap_uri(protocol='ldapi', realm=self.realm) + conn = ipaldap.LDAPClient(ldap_uri) + conn.external_bind() + repl = replication.ReplicationManager( + self.realm, self.fqdn, self.dm_password, conn=conn + ) + if self.dm_password is not None and not self.promote: + bind_dn = DN(('cn', 'Directory Manager')) + bind_pw = self.dm_password + else: + bind_dn = bind_pw = None + return repl, bind_dn, bind_pw def __setup_replica(self): """ @@ -434,26 +448,24 @@ def __setup_replica(self): self.realm, self.dm_password) - # Always connect to self over ldapi - ldap_uri = ipaldap.get_ldap_uri(protocol='ldapi', realm=self.realm) - conn = ipaldap.LDAPClient(ldap_uri) - conn.external_bind() - repl = replication.ReplicationManager(self.realm, - self.fqdn, - self.dm_password, conn=conn) - - if self.dm_password is not None and not self.promote: - bind_dn = DN(('cn', 'Directory Manager')) - bind_pw = self.dm_password - else: - bind_dn = bind_pw = None - - repl.setup_promote_replication(self.master_fqdn, - r_binddn=bind_dn, - r_bindpw=bind_pw, - cacert=self.ca_file) + repl, bind_dn, bind_pw = self._get_replication_manager() + repl.setup_promote_replication( + self.master_fqdn, + r_binddn=bind_dn, + r_bindpw=bind_pw, + cacert=self.ca_file + ) self.run_init_memberof = repl.needs_memberof_fixup() + def finalize_replica_config(self): + repl, bind_dn, bind_pw = self._get_replication_manager() + repl.finalize_replica_config( + self.master_fqdn, + r_binddn=bind_dn, + r_bindpw=bind_pw, + cacert=self.ca_file + ) + def __configure_sasl_mappings(self): # we need to remove any existing SASL mappings in the directory as otherwise they # they may conflict. diff --git a/ipaserver/install/replication.py b/ipaserver/install/replication.py index 5ce8fa689c..435786de42 100644 --- a/ipaserver/install/replication.py +++ b/ipaserver/install/replication.py @@ -75,6 +75,17 @@ 'internalModifiersName', 'internalModifyTimestamp') +# settings for cn=replica,cn=$DB,cn=mapping tree,cn=config +# during replica installation +REPLICA_CREATION_SETTINGS = { + "nsds5ReplicaReleaseTimeout": ["20"], + "nsds5ReplicaBackoffMax": ["3"], + "nsDS5ReplicaBindDnGroupCheckInterval": ["2"] +} +# after replica installation +REPLICA_FINAL_SETTINGS = REPLICA_CREATION_SETTINGS.copy() +REPLICA_FINAL_SETTINGS["nsDS5ReplicaBindDnGroupCheckInterval"] = ["60"] + def replica_conn_check(master_host, host_name, realm, check_ca, dogtag_master_ds_port, admin_password=None, @@ -201,9 +212,13 @@ def wait_for_entry(connection, dn, timeout, attr=None, attrvalue='*', class ReplicationManager(object): - """Manage replication agreements between DS servers, and sync - agreements with Windows servers""" - def __init__(self, realm, hostname, dirman_passwd, port=PORT, starttls=False, conn=None): + """Manage replication agreements + + between DS servers, and sync agreements with Windows servers + """ + + def __init__(self, realm, hostname, dirman_passwd=None, port=PORT, + starttls=False, conn=None): self.hostname = hostname self.port = port self.dirman_passwd = dirman_passwd @@ -481,22 +496,16 @@ def replica_config(self, conn, replica_id, replica_binddn): except errors.NotFound: pass else: - managers = {DN(m) for m in entry.get('nsDS5ReplicaBindDN', [])} - - mods = [] - if replica_binddn not in managers: + binddns = entry.setdefault('nsDS5ReplicaBindDN', []) + if replica_binddn not in {DN(m) for m in binddns}: # Add the new replication manager - mods.append( - (ldap.MOD_ADD, 'nsDS5ReplicaBindDN', replica_binddn) - ) - if 'nsds5replicareleasetimeout' not in entry: - # See https://pagure.io/freeipa/issue/7488 - mods.append( - (ldap.MOD_ADD, 'nsds5replicareleasetimeout', ['60']) - ) - - if mods: - conn.modify_s(dn, mods) + binddns.append(replica_binddn) + for key, value in REPLICA_CREATION_SETTINGS.items(): + entry[key] = value + try: + conn.update_entry(entry) + except errors.EmptyModlist: + pass self.set_replica_binddngroup(conn, entry) @@ -515,9 +524,8 @@ def replica_config(self, conn, replica_id, replica_binddn): nsds5flags=["1"], nsds5replicabinddn=[replica_binddn], nsds5replicabinddngroup=[self.repl_man_group_dn], - nsds5replicabinddngroupcheckinterval=["60"], - nsds5replicareleasetimeout=["60"], nsds5replicalegacyconsumer=["off"], + **REPLICA_CREATION_SETTINGS, ) conn.add_entry(entry) @@ -543,6 +551,32 @@ def setup_changelog(self, conn): except errors.DuplicateEntry: return + def _finalize_replica_config(self, conn): + """Change replica settings to final values + + During replica installation, some settings are configured for faster + replication. + """ + dn = self.replica_dn() + entry = conn.get_entry(dn) + for key, value in REPLICA_FINAL_SETTINGS.items(): + entry[key] = value + try: + conn.update_entry(entry) + except errors.EmptyModlist: + pass + + def finalize_replica_config(self, r_hostname, r_binddn=None, + r_bindpw=None, cacert=paths.IPA_CA_CRT): + ldap_uri = ipaldap.get_ldap_uri(r_hostname) + r_conn = ipaldap.LDAPClient(ldap_uri, cacert=cacert) + if r_bindpw: + r_conn.simple_bind(r_binddn, r_bindpw) + else: + r_conn.gssapi_bind() + self._finalize_replica_config(self.conn) + self.finalize_replica_config(r_conn) + def setup_chaining_backend(self, conn): chaindn = DN(('cn', 'chaining database'), ('cn', 'plugins'), ('cn', 'config')) benamebase = "chaindb" diff --git a/ipaserver/install/server/replicainstall.py b/ipaserver/install/server/replicainstall.py index 3ec2d6526a..b29d64e815 100644 --- a/ipaserver/install/server/replicainstall.py +++ b/ipaserver/install/server/replicainstall.py @@ -1505,6 +1505,8 @@ def install(installer): # Apply any LDAP updates. Needs to be done after the replica is synced-up service.print_msg("Applying LDAP updates") ds.apply_updates() + service.print_msg("Finalize replication settings") + ds.finalize_replica_config() if kra_enabled: kra.install(api, config, options, custodia=custodia) diff --git a/ipaserver/install/server/upgrade.py b/ipaserver/install/server/upgrade.py index 0fe3dd10a7..4b59d78cb9 100644 --- a/ipaserver/install/server/upgrade.py +++ b/ipaserver/install/server/upgrade.py @@ -48,6 +48,7 @@ from ipaserver.install import dogtaginstance from ipaserver.install import krbinstance from ipaserver.install import adtrustinstance +from ipaserver.install import replication from ipaserver.install.upgradeinstance import IPAUpgrade from ipaserver.install.ldapupdate import BadSyntax @@ -1680,11 +1681,16 @@ def update_replica_config(db_suffix): except ipalib.errors.NotFound: return # entry does not exist until a replica is installed - if 'nsds5replicareleasetimeout' not in entry: - # See https://pagure.io/freeipa/issue/7488 - logger.info("Adding nsds5replicaReleaseTimeout=60 to %s", dn) - entry['nsds5replicareleasetimeout'] = '60' + for key, value in replication.REPLICA_FINAL_SETTINGS.items(): + if entry.get(key) != value: + entry[key] = value + + try: api.Backend.ldap2.update_entry(entry) + except ipalib.errors.EmptyModlist: + pass + else: + logger.info("Updated entry %s", dn) def migrate_to_authselect(): diff --git a/ipatests/test_integration/test_external_ca.py b/ipatests/test_integration/test_external_ca.py index 5a7aa16123..60cd643545 100644 --- a/ipatests/test_integration/test_external_ca.py +++ b/ipatests/test_integration/test_external_ca.py @@ -148,7 +148,10 @@ def test_external_ca(self): '-LLL', '-o', 'ldif-wrap=no']) - assert 'nsds5ReplicaReleaseTimeout: 60' in result.stdout_text + # case insensitive match + text = result.stdout_text.lower() + assert 'nsds5ReplicaReleaseTimeout: 20'.lower() in text + assert 'nsDS5ReplicaBindDnGroupCheckInterval: 60'.lower() in text def test_client_installation_with_otp(self): # Test for issue 7526: client installation fails with one-time
_______________________________________________ FreeIPA-devel mailing list -- freeipa-devel@lists.fedorahosted.org To unsubscribe send an email to freeipa-devel-le...@lists.fedorahosted.org Fedora Code of Conduct: https://getfedora.org/code-of-conduct.html List Guidelines: https://fedoraproject.org/wiki/Mailing_list_guidelines List Archives: https://lists.fedoraproject.org/archives/list/freeipa-devel@lists.fedorahosted.org/message/FR6SFTKXFMUSVWPSWIYRHNDOPX6YNHXG/
