I did some analysis on migration and found several areas impacting performance:

1. We were calling user_mod to reset the magic value in description to not create a UPG. This caused a lot of unnecessary queries to display the user.

2. We check the remote LDAP server to make sure that the GID is valid and added a cache. We lacked a negative cache.

3. The biggest drag on performance was managing the default users group. After about 1000 users it would take about half a second to calculate the modlist and another half second for 389-ds to apply the change.

This patch addresses all of these.

For the last what I do is only do the group addition every 100 records. A query is run to find all users who aren't in the default users group and those are added.

I also added a bit of logging so one can better track the progress of migration.

I migrated 12.5k users with compat enabled in 3 1/2 hours.

I migrated the same 12.5k users and 2k groups with compat disabled in 30 minutes.

By contrast when I started, with compat enabled, I migrated:

1000 users in 7 minutes
2000 users in 27 minutes
3000 users in 1 hour

rob
>From f0b8bf2473d63af81b5dc5bef17499fa5e04aa85 Mon Sep 17 00:00:00 2001
From: Rob Crittenden <rcrit...@redhat.com>
Date: Tue, 29 Jan 2013 11:19:30 -0500
Subject: [PATCH] Improve migration performance

Add new users to the default users group in batches of 100. The
biggest overhead of migration is in calculating the modlist when
managing the default user's group and applying the changes. A
significant amount of time can be saved by not doing this on every
add operation.

Some other minor improvements include:

Add a negative cache for groups not found in the remote LDAP server.
Replace call to user_mod with a direct LDAP update.
Catch some occurances of LimitError and handle more gracefully.

I also added some debug logging to report on migration status and
performance.

https://fedorahosted.org/freeipa/ticket/3386
---
 ipalib/plugins/migration.py | 70 +++++++++++++++++++++++++++++++++++++++------
 1 file changed, 62 insertions(+), 8 deletions(-)

diff --git a/ipalib/plugins/migration.py b/ipalib/plugins/migration.py
index 81df59a23ff46b770076cb56a7d78fed8db26029..a254d60e742192d60f19c00a918a83df0326d49e 100644
--- a/ipalib/plugins/migration.py
+++ b/ipalib/plugins/migration.py
@@ -31,6 +31,7 @@ if api.env.in_server and api.env.context in ['lite', 'server']:
         raise e
 from ipalib import _
 from ipapython.dn import DN
+import datetime
 
 __doc__ = _("""
 Migration to IPA
@@ -107,6 +108,7 @@ EXAMPLES:
 # USER MIGRATION CALLBACKS AND VARS
 
 _krb_err_msg = _('Kerberos principal %s already exists. Use \'ipa user-mod\' to set it manually.')
+_krb_failed_msg = _('Unable to determine Kerberos principal %s already exists. Use \'ipa user-mod\' to set it manually.')
 _grp_err_msg = _('Failed to add user to the default group. Use \'ipa group-add-member\' to add manually.')
 _ref_err_msg = _('Migration of LDAP search reference is not supported.')
 _dn_err_msg = _('Malformed DN')
@@ -123,13 +125,17 @@ def _pre_migrate_user(ldap, pkey, dn, entry_attrs, failed, config, ctx, **kwargs
     has_upg = ctx['has_upg']
     search_bases = kwargs.get('search_bases', None)
     valid_gids = kwargs['valid_gids']
+    invalid_gids = kwargs['invalid_gids']
 
     if 'gidnumber' not in entry_attrs:
         raise errors.NotFound(reason=_('%(user)s is not a POSIX user') % dict(user=pkey))
     else:
         # See if the gidNumber at least points to a valid group on the remote
         # server.
-        if entry_attrs['gidnumber'][0] not in valid_gids:
+        if entry_attrs['gidnumber'][0] in invalid_gids:
+            api.log.warn('GID number %s of migrated user %s does not point to a known group.' \
+                         % (entry_attrs['gidnumber'][0], pkey))
+        elif entry_attrs['gidnumber'][0] not in valid_gids:
             try:
                 (remote_dn, remote_entry) = ds_ldap.find_entry_by_attr(
                     'gidnumber', entry_attrs['gidnumber'][0], 'posixgroup',
@@ -139,10 +145,13 @@ def _pre_migrate_user(ldap, pkey, dn, entry_attrs, failed, config, ctx, **kwargs
             except errors.NotFound:
                 api.log.warn('GID number %s of migrated user %s does not point to a known group.' \
                              % (entry_attrs['gidnumber'][0], pkey))
+                invalid_gids.append(entry_attrs['gidnumber'][0])
             except errors.SingleMatchExpected, e:
                 # GID number matched more groups, this should not happen
                 api.log.warn('GID number %s of migrated user %s should match 1 group, but it matched %d groups' \
                              % (entry_attrs['gidnumber'][0], pkey, e.found))
+            except errors.LimitsExceeded, e:
+                api.log.warn('Search limit exceeded searching for GID %s' % entry_attrs['gidnumber'][0])
 
     # We don't want to create a UPG so set the magic value in description
     # to let the DS plugin know.
@@ -182,6 +191,8 @@ def _pre_migrate_user(ldap, pkey, dn, entry_attrs, failed, config, ctx, **kwargs
         )
     except errors.NotFound:
         entry_attrs['krbprincipalname'] = principal
+    except errors.LimitsExceeded:
+        failed[pkey] = unicode(_krb_failed_msg % principal)
     else:
         failed[pkey] = unicode(_krb_err_msg % principal)
 
@@ -232,19 +243,47 @@ def _pre_migrate_user(ldap, pkey, dn, entry_attrs, failed, config, ctx, **kwargs
 
 def _post_migrate_user(ldap, pkey, dn, entry_attrs, failed, config, ctx):
     assert isinstance(dn, DN)
-    # add user to the default group
-    try:
-        ldap.add_entry_to_group(dn, ctx['def_group_dn'])
-    except errors.ExecutionError, e:
-        failed[pkey] = unicode(_grp_err_msg)
+
+    _update_default_group(ldap, pkey, config, ctx, False)
+
     if 'description' in entry_attrs and NO_UPG_MAGIC in entry_attrs['description']:
         entry_attrs['description'].remove(NO_UPG_MAGIC)
-        kw = {'setattr': unicode('description=%s' % ','.join(entry_attrs['description']))}
+        update_attrs = dict(description = entry_attrs['description'])
         try:
-            api.Command['user_mod'](pkey, **kw)
+            ldap.update_entry(dn, update_attrs)
         except (errors.EmptyModlist, errors.NotFound):
             pass
 
+def _update_default_group(ldap, pkey, config, ctx, force):
+    migrate_cnt = ctx['migrate_cnt']
+    group_dn = ctx['def_group_dn']
+
+    # Purposely let this fire when migrate_cnt == 0 so on re-running migration
+    # it can catch any users migrated but not added to the default group.
+    if force or migrate_cnt % 100 == 0:
+        s = datetime.datetime.now()
+        searchfilter = "(&(objectclass=posixAccount)(!(memberof=%s)))" % group_dn
+        (result, truncated) = ldap.find_entries(searchfilter,
+            ['member'], api.env.container_user, scope=_ldap.SCOPE_SUBTREE,
+            time_limit = -1)
+        new_members = []
+        (group_dn, group_entry_attrs) = ldap.get_entry(group_dn, ['member'])
+        for m in result:
+            if m[0] not in group_entry_attrs.get('member', []):
+                new_members.append(m[0])
+        if len(new_members) > 0:
+            members = group_entry_attrs.get('member', [])
+            members.extend(new_members)
+            group_entry_attrs['member'] = members
+        try:
+            ldap.update_entry(group_dn, group_entry_attrs)
+        except errors.EmptyModlist:
+            pass
+
+        e = datetime.datetime.now()
+        d = e - s
+        api.log.debug('Adding users to group duration %s' % d)
+
 # GROUP MIGRATION CALLBACKS AND VARS
 
 def _pre_migrate_group(ldap, pkey, dn, entry_attrs, failed, config, ctx, **kwargs):
@@ -619,6 +658,7 @@ can use their Kerberos accounts.''')
         migrated = {} # {'OBJ': ['PKEY1', 'PKEY2', ...], ...}
         failed = {} # {'OBJ': {'PKEY1': 'Failed 'cos blabla', ...}, ...}
         search_bases = self._get_search_bases(options, ds_base_dn, self.migrate_order)
+        migration_start = datetime.datetime.now()
         for ldap_obj_name in self.migrate_order:
             ldap_obj = self.api.Object[ldap_obj_name]
 
@@ -682,7 +722,11 @@ can use their Kerberos accounts.''')
             context['has_upg'] = ldap.has_upg()
 
             valid_gids = []
+            invalid_gids = []
+            migrate_cnt = 0
             for (dn, entry_attrs) in entries:
+                context['migrate_cnt'] = migrate_cnt
+                s = datetime.datetime.now()
                 if dn is None:  # LDAP search reference
                     failed[ldap_obj_name][entry_attrs[0]] = unicode(_ref_err_msg)
                     continue
@@ -724,6 +768,7 @@ can use their Kerberos accounts.''')
                             config, context, schema = options['schema'],
                             search_bases = search_bases,
                             valid_gids = valid_gids,
+                            invalid_gids = invalid_gids,
                             **blacklists
                         )
                         assert isinstance(dn, DN)
@@ -755,6 +800,15 @@ can use their Kerberos accounts.''')
                         ldap, pkey, dn, entry_attrs, failed[ldap_obj_name],
                         config, context,
                     )
+                e = datetime.datetime.now()
+                d = e - s
+                total_dur = e - migration_start
+                migrate_cnt += 1
+                if migrate_cnt > 0 and migrate_cnt % 100:
+                    api.log.info("%d %ss migrated. %s elapsed." % (migrate_cnt, ldap_obj_name, total_dur))
+                api.log.debug("%d %ss migrated, duration: %s (total %s)" % (migrate_cnt, ldap_obj_name, d, total_dur))
+
+        _update_default_group(ldap, pkey, config, context, True)
 
         return (migrated, failed)
 
-- 
1.8.1

_______________________________________________
Freeipa-devel mailing list
Freeipa-devel@redhat.com
https://www.redhat.com/mailman/listinfo/freeipa-devel

Reply via email to