URL: https://github.com/freeipa/freeipa/pull/2649
Author: tiran
Title: #2649: Add more LDAP indices
Action: opened
PR body:
"""
## Add more LDAP indices
An index is used to optimize an LDAP operation. Without an index, 389-DS
has to perform a partial or even full table scan. A full database scan can
easily take 10 seconds or more in a large installation.
* automountMapKey: eq, pres (was: eq)
* autoMountMapName: eq
* ipaConfigString: eq
* ipaEnabledFlag: eq
* ipaKrbAuthzData: eq, sub
* ipServicePort: eq
* ipServiceProtocol: eq
* accessRuleType: eq
* hostCategory: eq
automountMapKey and autoMountMapName filters are used for automount.
Installation and service discovery (CA, KRA) use ipaConfigString to find
active services and CA renewal master.
SSSD filters with ipaEnabledFlag, accessRuleType, and hostCategory to
find and cache HBAC rules for each host.
The SSSD plugin for nssswitch service uses ipServicePort and
ipServiceProtocol.
ipaKrbAuthzData is used by ipa host-del. The framework performs a
'*arg*' query, therefore a sub index is required, too.
Fixes: pagure.io/freeipa/issue/7786
Fixes: pagure.io/freeipa/issue/7787
Fixes: pagure.io/freeipa/issue/7790
Fixes: pagure.io/freeipa/issue/7792
## LDAPUpdate: Batch index tasks
The LDAPUpdate framework now keeps record of all changed/added indices
and batches all changed attribute in a single index task. It makes
updates much faster when multiple indices are added or modified.
"""
To pull the PR as Git branch:
git remote add ghfreeipa https://github.com/freeipa/freeipa
git fetch ghfreeipa pull/2649/head:pr2649
git checkout pr2649
From 4a6c6c2b8a6fb9b9b141dd437dc10843f68980eb Mon Sep 17 00:00:00 2001
From: Christian Heimes <chei...@redhat.com>
Date: Wed, 5 Dec 2018 12:57:56 +0100
Subject: [PATCH 1/2] LDAPUpdate: Batch index tasks
The LDAPUpdate framework now keeps record of all changed/added indices
and batches all changed attribute in a single index task. It makes
updates much faster when multiple indices are added or modified.
Signed-off-by: Christian Heimes <chei...@redhat.com>
---
ipaserver/install/ldapupdate.py | 47 ++++++++++++++++++++-------------
1 file changed, 29 insertions(+), 18 deletions(-)
diff --git a/ipaserver/install/ldapupdate.py b/ipaserver/install/ldapupdate.py
index aff45bbede..0e69908b01 100644
--- a/ipaserver/install/ldapupdate.py
+++ b/ipaserver/install/ldapupdate.py
@@ -134,7 +134,14 @@ def safe_output(attr, values):
class LDAPUpdate:
- action_keywords = ["default", "add", "remove", "only", "onlyifexist", "deleteentry", "replace", "addifnew", "addifexist"]
+ action_keywords = [
+ "default", "add", "remove", "only", "onlyifexist", "deleteentry",
+ "replace", "addifnew", "addifexist"
+ ]
+ index_suffix = DN(
+ ('cn', 'index'), ('cn', 'userRoot'), ('cn', 'ldbm database'),
+ ('cn', 'plugins'), ('cn', 'config')
+ )
def __init__(self, dm_password=None, sub_dict={},
online=True, ldapi=False):
@@ -515,8 +522,8 @@ def emit_plugin_update(update):
return all_updates
- def create_index_task(self, attribute):
- """Create a task to update an index for an attribute"""
+ def create_index_task(self, *attributes):
+ """Create a task to update an index for attributes"""
# Sleep a bit to ensure previous operations are complete
time.sleep(5)
@@ -525,7 +532,7 @@ def create_index_task(self, attribute):
# cn_uuid.time is in nanoseconds, but other users of LDAPUpdate expect
# seconds in 'TIME' so scale the value down
self.sub_dict['TIME'] = int(cn_uuid.time/1e9)
- cn = "indextask_%s_%s_%s" % (attribute, cn_uuid.time, cn_uuid.clock_seq)
+ cn = "indextask_%s_%s" % (cn_uuid.time, cn_uuid.clock_seq)
dn = DN(('cn', cn), ('cn', 'index'), ('cn', 'tasks'), ('cn', 'config'))
e = self.conn.make_entry(
@@ -533,11 +540,13 @@ def create_index_task(self, attribute):
objectClass=['top', 'extensibleObject'],
cn=[cn],
nsInstance=['userRoot'],
- nsIndexAttribute=[attribute],
+ nsIndexAttribute=list(attributes),
)
- logger.debug("Creating task to index attribute: %s", attribute)
- logger.debug("Task id: %s", dn)
+ logger.info(
+ "Creating task %s to index attributes: %s",
+ dn, ', '.join(attributes)
+ )
self.conn.add_entry(e)
@@ -571,8 +580,8 @@ def monitor_index_task(self, dn):
time.sleep(1)
continue
- if status.lower().find("finished") > -1:
- logger.debug("Indexing finished")
+ if "finished" in status.lower():
+ logger.info("Indexing finished")
break
logger.debug("Indexing in progress")
@@ -792,7 +801,7 @@ def _update_record(self, update):
entry = self._apply_update_disposition(update.get('updates'), entry)
if entry is None:
# It might be None if it is just deleting an entry
- return
+ return None, False
self.print_entity(entry, "Final value after applying updates")
@@ -811,7 +820,7 @@ def _update_record(self, update):
# this may not be an error (e.g. entries in NIS container)
logger.error("Parent DN of %s may not exist, cannot "
"create the entry", entry.dn)
- return
+ return entry, False
added = True
self.modified = True
except Exception as e:
@@ -846,12 +855,7 @@ def _update_record(self, update):
if updated:
self.modified = True
- if entry.dn.endswith(DN(('cn', 'index'), ('cn', 'userRoot'),
- ('cn', 'ldbm database'), ('cn', 'plugins'),
- ('cn', 'config'))) and (added or updated):
- taskid = self.create_index_task(entry.single_value['cn'])
- self.monitor_index_task(taskid)
- return
+ return entry, added or updated
def _delete_record(self, updates):
"""
@@ -903,13 +907,20 @@ def create_connection(self):
raise RuntimeError("Offline updates are not supported.")
def _run_updates(self, all_updates):
+ reindex_attributes = set()
for update in all_updates:
if 'deleteentry' in update:
self._delete_record(update)
elif 'plugin' in update:
self._run_update_plugin(update['plugin'])
else:
- self._update_record(update)
+ entry, modified = self._update_record(update)
+ if modified and entry.dn.endswith(self.index_suffix):
+ reindex_attributes.add(entry.single_value['cn'])
+
+ if reindex_attributes:
+ task_dn = self.create_index_task(*sorted(reindex_attributes))
+ self.monitor_index_task(task_dn)
def update(self, files, ordered=True):
"""Execute the update. files is a list of the update files to use.
From 760db6af9082c8835a527bbde4c6861852c637d6 Mon Sep 17 00:00:00 2001
From: Christian Heimes <chei...@redhat.com>
Date: Wed, 5 Dec 2018 17:55:58 +0100
Subject: [PATCH 2/2] Add more LDAP indices
An index is used to optimize an LDAP operation. Without an index, 389-DS
has to perform a partial or even full table scan. A full database scan can
easily take 10 seconds or more in a large installation.
* automountMapKey: eq, pres (was: eq)
* autoMountMapName: eq
* ipaConfigString: eq
* ipaEnabledFlag: eq
* ipaKrbAuthzData: eq, sub
* ipServicePort: eq
* ipServiceProtocol: eq
* accessRuleType: eq
* hostCategory: eq
automountMapKey and autoMountMapName filters are used for automount.
Installation and service discovery (CA, KRA) use ipaConfigString to find
active services and CA renewal master.
SSSD filters with ipaEnabledFlag, accessRuleType, and hostCategory to
find and cache HBAC rules for each host.
The SSSD plugin for nssswitch service uses ipServicePort and
ipServiceProtocol.
ipaKrbAuthzData is used by ipa host-del. The framework performs a
'*arg*' query, therefore a sub index is required, too.
Fixes: https://pagure.io/freeipa/issue/7786
Fixes: https://pagure.io/freeipa/issue/7787
Fixes: https://pagure.io/freeipa/issue/7790
Fixes: https://pagure.io/freeipa/issue/7792
Signed-off-by: Christian Heimes <chei...@redhat.com>
---
install/share/indices.ldif | 66 +++++++++++++++++++++++++++++++
install/updates/20-indices.update | 58 +++++++++++++++++++++++++++
2 files changed, 124 insertions(+)
diff --git a/install/share/indices.ldif b/install/share/indices.ldif
index e91ef01ed7..0a1945e680 100644
--- a/install/share/indices.ldif
+++ b/install/share/indices.ldif
@@ -216,6 +216,40 @@ ObjectClass: top
ObjectClass: nsIndex
nsSystemIndex: false
nsIndexType: eq
+nsIndexType: pres
+
+dn: cn=automountMapName,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+changetype: add
+cn: automountMapNam
+ObjectClass: top
+ObjectClass: nsIndex
+nsSystemIndex: false
+nsIndexType: eq
+
+dn: cn=ipaConfigString,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+changetype: add
+cn: ipaConfigString
+objectClass:top
+objectClass:nsIndex
+nsSystemIndex: false
+nsIndexType: eq
+
+dn: cn=ipaEnabledFlag,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+changetype: add
+cn: ipaEnabledFlag
+objectClass:top
+objectClass:nsIndex
+nsSystemIndex: false
+nsIndexType: eq
+
+dn: cn=ipaKrbAuthzData,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+changetype: add
+cn: ipaKrbAuthzData
+objectClass: top
+objectClass: nsIndex
+nsSystemIndex: false
+nsIndexType: eq
+nsIndexType: sub
dn: cn=ipakrbprincipalalias,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
changetype: add
@@ -271,6 +305,22 @@ nsSystemIndex: false
nsIndexType: eq
nsIndexType: pres
+dn: cn=ipServicePort,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+changetype: add
+cn: ipServicePort
+objectClass:top
+objectClass:nsIndex
+nsSystemIndex: false
+nsIndexType: eq
+
+dn: cn=ipServiceProtocol,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+changetype: add
+cn: ipServiceProtocol
+objectClass:top
+objectClass:nsIndex
+nsSystemIndex: false
+nsIndexType: eq
+
dn: cn=krbCanonicalName,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
changetype: add
cn: krbCanonicalName
@@ -333,3 +383,19 @@ objectClass: nsindex
nssystemindex: false
nsindextype: eq
nsindextype: sub
+
+dn: cn=accessRuleType,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+changetype: add
+cn: accessRuleType
+objectClass:top
+objectClass:nsIndex
+nsSystemIndex: false
+nsIndexType: eq
+
+dn: cn=hostCategory,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+changetype: add
+cn: hostCategory
+objectClass:top
+objectClass:nsIndex
+nsSystemIndex: false
+nsIndexType: eq
diff --git a/install/updates/20-indices.update b/install/updates/20-indices.update
index d1704adfc2..5cc4b7302a 100644
--- a/install/updates/20-indices.update
+++ b/install/updates/20-indices.update
@@ -151,6 +151,36 @@ default:ObjectClass: top
default:ObjectClass: nsIndex
default:nsSystemIndex: false
default:nsIndexType: eq
+default:nsIndexType: pres
+
+dn: cn=automountMapName,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+default:cn: automountMapName
+default:ObjectClass: top
+default:ObjectClass: nsIndex
+default:nsSystemIndex: false
+default:nsIndexType: eq
+
+dn: cn=ipaConfigString,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+default: cn: ipaConfigString
+default: objectClass:top
+default: objectClass:nsIndex
+default: nsSystemIndex: false
+default: nsIndexType: eq
+
+dn: cn=ipaEnabledFlag,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+default: cn: ipaEnabledFlag
+default: objectClass:top
+default: objectClass:nsIndex
+default: nsSystemIndex: false
+default: nsIndexType: eq
+
+dn: cn=ipaKrbAuthzData,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+default: cn: ipaKrbAuthzData
+default: objectClass: top
+default: objectClass: nsIndex
+default: nsSystemIndex: false
+default: nsIndexType: eq
+default: nsIndexType: sub
dn: cn=ipakrbprincipalalias,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
default:cn: ipakrbprincipalalias
@@ -243,6 +273,20 @@ default:nsSystemIndex: false
only:nsIndexType: eq
only:nsIndexType: pres
+dn: cn=ipServicePort,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+default: cn: ipServicePort
+default: objectClass:top
+default: objectClass:nsIndex
+default: nsSystemIndex: false
+default: nsIndexType: eq
+
+dn: cn=ipServiceProtocol,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+default: cn: ipServiceProtocol
+default: objectClass:top
+default: objectClass:nsIndex
+default: nsSystemIndex: false
+default: nsIndexType: eq
+
dn: cn=krbPrincipalName,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
default:cn: krbPrincipalName
default:ObjectClass: top
@@ -308,3 +352,17 @@ default: objectclass: nsindex
default: nssystemindex: false
default: nsindextype: eq
default: nsindextype: sub
+
+dn: cn=accessRuleType,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+default: cn: accessRuleType
+default: objectClass:top
+default: objectClass:nsIndex
+default: nsSystemIndex: false
+default: nsIndexType: eq
+
+dn: cn=hostCategory,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+default: cn: hostCategory
+default: objectClass:top
+default: objectClass:nsIndex
+default: nsSystemIndex: false
+default: nsIndexType: eq
_______________________________________________
FreeIPA-devel mailing list -- freeipa-devel@lists.fedorahosted.org
To unsubscribe send an email to freeipa-devel-le...@lists.fedorahosted.org
Fedora Code of Conduct: https://getfedora.org/code-of-conduct.html
List Guidelines: https://fedoraproject.org/wiki/Mailing_list_guidelines
List Archives:
https://lists.fedorahosted.org/archives/list/freeipa-devel@lists.fedorahosted.org
