Hello,

A performance bottleneck during provisioning was described http://www.freeipa.org/page/V4/Performance_Improvements#typical_provisioning:_ldapadd_entries.2C_migrate-ds...


I wrote the attached script that is following http://www.freeipa.org/page/V4/Performance_Improvements#Algorithm

This is a preliminary script that needs to be improved but just to check it matches basic requirements I am sending it to the alias to get some feedback.

The output of a provisioning session is below. The steps are:

 * install freeipa-server
 * create a provisioning file using
   https://github.com/freeipa/freeipa-tools/blob/master/create-test-data.py
 * ipa-provision.py prepare
 * ipa-provision.py import
 * ipa-provision.py finish

Note: you will likely need to define the DM password with the option '-w <password>'

regards
thierry


/tmp/ipa-provision.py prepare -f /tmp/1K_users_800_hosts_groups_hostgroups_sudorules_hbac_rules.ldif -d 1
entrycache = 10485760
dncache    = 10485760
dbcache    = 209715200
dblock     = 10000
Preparation of the bulk import is now completed
If you want to continue:
- run /tmp/ipa-provision.py import -f /tmp/1K_users_800_hosts_groups_hostgroups_sudorules_hbac_rules.ldif
  - run /tmp/ipa-provision.py finish -b <suffix_dn>

If you want to not continue:
  - run /tmp/ipa-provision.py abort



/tmp/ipa-provision import -f /tmp/1K_users_800_hosts_groups_hostgroups_sudorules_hbac_rules.ldif -debug 2 adding new entry "uid=user0,cn=users,cn=accounts,dc=abc,dc=idm,dc=lab,dc=eng,dc=brq,dc=redhat,dc=com"

adding new entry "uid=user1,cn=users,cn=accounts,dc=abc,dc=idm,dc=lab,dc=eng,dc=brq,dc=redhat,dc=com"

adding new entry "uid=user2,cn=users,cn=accounts,dc=abc,dc=idm,dc=lab,dc=eng,dc=brq,dc=redhat,dc=com"
...
adding new entry "ipaUniqueID=autogenerate,cn=hbac,dc=abc,dc=idm,dc=lab,dc=eng,dc=brq,dc=redhat,dc=com"

adding new entry "ipaUniqueID=autogenerate,cn=hbac,dc=abc,dc=idm,dc=lab,dc=eng,dc=brq,dc=redhat,dc=com"

Bulk import is now completed
You need to run:
  - run /tmp/ipa-provision.py finish -b <suffix_dn>



/tmp/ipa-provision.py finish

Waiting for (objectClass=inetorgperson) fixup to complete...
Completed filter (objectClass=inetorgperson)

Waiting for (objectClass=ipausergroup) fixup to complete...
Completed filter (objectClass=ipausergroup)

Waiting for (objectClass=ipahost) fixup to complete...
Completed filter (objectClass=ipahost)

Waiting for (objectClass=ipahostgroup) fixup to complete...
Completed filter (objectClass=ipahostgroup)

Waiting for (objectClass=ipasudorule) fixup to complete...
Completed filter (objectClass=ipasudorule)

Waiting for (objectClass=ipahbacrule) fixup to complete...
Completed filter (objectClass=ipahbacrule)

Bulk import and fixup tasks are now completed

#! /usr/bin/python2

import os
import sys
try:
    import psutil
except ImportError:
    raise ("Missing module: psutil (try 'dnf install python-psutil')")
import time
import argparse
import ldap
import socket
import re
import shutil
import pdb
import pwd
import grp
from subprocess import Popen, PIPE
from socket import getfqdn


DEFAULT_PORT_ROOT = str(389)
DEFAULT_ROOT_DN   = 'cn=Directory Manager'
DEFAULT_PASSWORD  = 'Secret123'
DEFAULT_DEBUG_LVL = 0
ENVIRON_PORT      = 'BULK_PORT'
ENVIRON_ROOTDN    = 'BULK_ROOTDN'
ENVIRON_PASSWORD  = 'BULK_PASSWORD'
ENVIRON_DEBUG_LVL = 'BULK_DEBUG_LVL'

#path = os.pathsep.join(("/usr/lib64/python2.7","/usr/lib64/python2.7/plat-linux2","/usr/lib64/python2.7/lib-dynload","/usr/lib64/python2.7/site-packages","/usr/lib64/python2.7/site-packages/psutil","/usr/lib/python2.7/site-packages","/usr/lib/python2.7/site-packages/setuptools-0.6c11-py2.7.egg-info"))
#os.environ['PYTHONPATH'] = path

# DS attributes & entries
ds_config           = "cn=config"
ds_schema_dir       = "nsslapd-schemadir"
ds_dbconfig_dn      = "cn=config,cn=ldbm database,cn=plugins,cn=config"
ds_dbcache_attr     = 'nsslapd-dbcachesize'
ds_dblock_attr      = 'nsslapd-db-locks'
ds_backend_dn       = "cn=ldbm database,cn=plugins,cn=config"
ds_domainBackend_dn = "cn=userRoot,%s" % ds_backend_dn
ds_backend_ecache   = "nsslapd-cachememsize"
ds_backend_dncache  = "nsslapd-dncachememsize"
ds_plugin           = "cn=plugins,cn=config"
ds_membeof_plugin   = "cn=MemberOf Plugin,%s" % ds_plugin
ds_schemaCompat_plugin = "cn=Schema Compatibility,%s" % ds_plugin
ds_uid              = pwd.getpwnam("dirsrv").pw_uid
ds_gid              = grp.getgrnam("dirsrv").gr_gid
ds_server_id        = None

# Fixup data
fixup_filters = ["(objectClass=inetorgperson)", "(objectClass=ipausergroup)", "(objectClass=ipahost)", "(objectClass=ipahostgroup)", "(objectClass=ipasudorule)", "(objectClass=ipahbacrule)"]
taskStatus = "nstaskstatus"

# ipactl command
ipactl  = "/usr/sbin/ipactl"

kilo = 1024
mega = kilo * kilo
systemMemory = None
entryCache = None
dbCache = None

# workaround for https://fedorahosted.org/389/ticket/48868
ticket48868_fixed = False

# return the total memory size available on the system
def getSystemMemory():
    global systemMemory
    if not systemMemory:
        from psutil import virtual_memory

        mem = virtual_memory()
        systemMemory = mem.total
        
    return systemMemory
        
def getTuning(ldifentries, avgEntryDNSz):
    # First estimate the DN cache
    # Let's say it is for each DN
    #  - sizeof SLAPI_DN = 32bytes
    #  - size of DN * 2 (normalized and case normalized)
    # But make sure it is in the range:
    #  10mb (default)
    #  50mb (rule of thumb)
    szPerEntry = (2 * avgEntryDNSz)
    szPerEntry += 32
    dncache = max(ldifentries * szPerEntry, 10 * mega)
    dncache = min(dncache, 50 * mega)
    
    # Second compute the entryCache and DB cache according 
    # to the amount of memory on the system
    memory = getSystemMemory()/mega
    if memory > (8*kilo):
        # the system has 8Gb => entry cache = 400Mb , db cache = 500Mb
        if ticket48868_fixed:
            entryCache = 400 * mega
        else:
            entryCache = 10 * mega
        dbCache    = 500 * mega
    elif memory > (4*kilo):
        # the system has 4Gb => entry cache = 400Mb , db cache = 200Mb
        if ticket48868_fixed:
            entryCache = 400 * mega
        else:
            entryCache = 10 * mega
        dbCache    = 200 * mega
    elif memory > (2*kilo):
        # the system has 2Gb => entry cache = 200Mb , db cache = 100Mb
        if ticket48868_fixed:
            entryCache = 200 * mega
        else:
            entryCache = 10 * mega
        dbCache    = 100 * mega
    else:
        # the system has less than 2Gb => entry cache = 100Mb , db cache = 100Mb
        if ticket48868_fixed:
            entryCache = 100 * mega
        else:
            entryCache = 10 * mega
        dbCache    = 100 * mega
        
    # The minimum number of locks is  10K
    # The maximum number of locks is 200K
    # It can range proportionaly to the number of entries
    dblock = max(min(200000, ldifentries/2), 10000)
        
    return (entryCache, dbCache, dncache, dblock)


def entriesInfo(ldiffile):
    # returns Information about the ldif entries
    #  - number of entries
    #  - average size of entries
    #  - average size of entries DN
    ldifentries = 0
    entriesDNLen = 0
    entriesSize = 0
    with open(ldiffile) as file_obj:
        for line in file_obj:
            if line.startswith('dn: '):
                ldifentries += 1
                entriesDNLen += (len(line) - 4)
            entriesSize += len(line)
            
    return (ldifentries, entriesSize/ldifentries, entriesDNLen/ldifentries)

def ds_action(action='start'):
    global debug_level
    if not action:
        raise ("ds_action: invalid action")
    if not action.lower() in ['start', 'restart', 'stop']:
        raise ("ds_action: unknown action (%s)" % str(action))
    
    process = Popen(['/usr/bin/systemctl', action, 'dirsrv@%s.service' % getServerID()], stdout=PIPE, stderr=PIPE)
    for line in iter(process.stdout.readline,''):
        if debug_level > 1:
            print line
        
def ipa_action(action='start'):
    global debug_level
    if not action:
        raise ("ipa_action: invalid action")
    if not action.lower() in ['start', 'restart', 'stop']:
        raise ("ipa_action: unknown action (%s)" % str(action))
    
    process = Popen([ipactl, action], stdout=PIPE, stderr=PIPE)
    for line in iter(process.stdout.readline,''):
        if debug_level > 1:
            print line
        
def getServerID():
    global ds_server_id
    
    if ds_server_id:
        return ds_server_id
    
    # This is not define, so compute it from systemctl
    process = Popen(['systemctl', 'list-dependencies', 'dirsrv.target', '--plain'], stdout=PIPE, stderr=PIPE)
    for line in iter(process.stdout.readline,''):
        m = re.search('dirsrv@(.+?).service', line)
        if m:
                ds_server_id = m.group(1)
                return ds_server_id
    
    raise("getServerID: unable to retrieve serverID\nEdit %s and set ds_server_id\n(for example ds_server_id=%s)" % 
            (os.path.basename(__file__), 
            socket.getfqdn().partition('.')[2].upper().replace('.','-')))



def get_default_password():
    password = os.environ.get(ENVIRON_PASSWORD, None)
    if not password:
        password = DEFAULT_PASSWORD
    return password

def get_default_port():
    port = os.environ.get(ENVIRON_PORT, None)
    if not port:
        port = DEFAULT_PORT_ROOT
    return port

def get_default_rootdn():
    rootdn = os.environ.get(ENVIRON_ROOTDN, None)
    if not rootdn:
        rootdn = DEFAULT_ROOT_DN
    return rootdn

def get_default_debuglvl():
    rootdn = os.environ.get(ENVIRON_DEBUG_LVL, None)
    if not rootdn:
        rootdn = DEFAULT_DEBUG_LVL
    return rootdn

def updateDBLock(oldDSE, newDSE, dblock):
    dbentry = False
    for old_line in oldDSE:
	if old_line.startswith('dn: %s' % ds_dbconfig_dn):
            # This is the database entry config
            dbentry = True
            newDSE.write("%s" % old_line)
            continue

	if dbentry:
            if old_line.startswith(ds_dblock_attr):
                # The old config contained dblock
                # just REPLACE it with the correct value
                newDSE.write("%s: %d\n" % (ds_dblock_attr, int(dblock)))
                dbentry = False
                continue
            else:
                if old_line == '\n':
                    # The old config did not contain dblock
                    # ADD the correct value
                    newDSE.write("%s: %d\n" % (ds_dblock_attr, int(dblock)))
                    dbentry = False
                newDSE.write("%s" % old_line)
                continue
	else:
		newDSE.write("%s" % old_line)


def setNewTuning(conn=None, entrycache=None, dncache=None, dbcache=None, dblock=None):
    global debug_level
    from subprocess import Popen, PIPE, call
    
    if debug_level > 0:
        print "entrycache = %s" % str(entrycache)
        print "dncache    = %s" % str(dncache)
        print "dbcache    = %s" % str(dbcache)
        print "dblock     = %s" % str(dblock)
        
    if not conn:
        raise ("Unable to set the appropriate tuning: unknown connection")
    
    # First save dse.ldif into a backup
    ds_config_dse = '/etc/dirsrv/slapd-%s/dse.ldif' % getServerID()
    ds_config_dse_bak = '%s.bulk_import_bak.%s' % (ds_config_dse, time.strftime("%Y%m%d-%H%M%S"))
    shutil.copy2(ds_config_dse, ds_config_dse_bak)
    os.chown(ds_config_dse_bak, ds_uid, ds_gid)

    #
    # set the Database tuning
    #
    mods = []
    if dbcache:
        mods.append((ldap.MOD_REPLACE, ds_dbcache_attr, str(dbcache)))

    if len(mods) > 0:
        conn.modify_s(ds_dbconfig_dn, mods)
        
    # set the backend tuning
    mods = []
    if entrycache:
        mods.append((ldap.MOD_REPLACE, ds_backend_ecache, str(entrycache)))
    if dncache:
        mods.append((ldap.MOD_REPLACE, ds_backend_dncache, str(dncache)))
    if len(mods) > 0:
        conn.modify_s(ds_domainBackend_dn, mods)
        
    # Disable 
    #   memberof plugin: major perf impact
    #   schema compat: not usefull during provisioning
    mods = [(ldap.MOD_REPLACE, "nsslapd-pluginEnabled", 'off')]
    conn.modify_s(ds_membeof_plugin, mods)
    conn.modify_s(ds_schemaCompat_plugin, mods)
        
    # Stop IPA and be paranoid also DS
    ipa_action('stop')
    ds_action('stop')

    
    # dblock can not be updated online, so we need to update dse.ldif
    ds_config_dse_new = '%s.bulk_import_new.%s' % (ds_config_dse, time.strftime("%Y%m%d-%H%M%S"))
    newDSE = open(ds_config_dse_new , 'w')
    oldDSE = open(ds_config_dse, 'r')
    updateDBLock(oldDSE, newDSE, dblock)
    shutil.move(ds_config_dse_new, ds_config_dse)
    os.chown(ds_config_dse, ds_uid, ds_gid)
    
    # Start DS with the new settings
    ds_action('start')
    



def backupCurrentTuning(conn, currentSettingFile):
        results = conn.search_s(ds_dbconfig_dn, ldap.SCOPE_BASE, "(objectclass=*)",[ds_dbcache_attr, ds_dblock_attr])
        dn, entry = results[0]
        currentSettingFile.write("dn: %s\n" % ds_dbconfig_dn)
        currentSettingFile.write("changetype: modify\n")
        currentSettingFile.write("replace: %s\n" % ds_dbcache_attr)
        currentSettingFile.write("%s: %s\n" % (ds_dbcache_attr, entry[ds_dbcache_attr][0]))
        currentSettingFile.write("-\n")
        currentSettingFile.write("replace: %s\n" % ds_dblock_attr)
        currentSettingFile.write("%s: %s\n\n" % (ds_dblock_attr,  entry[ds_dblock_attr][0]))

        results = conn.search_s(ds_domainBackend_dn, ldap.SCOPE_BASE, "(objectclass=nsBackendInstance)",[ds_backend_ecache, ds_backend_dncache])
        dn, entry = results[0]
        currentSettingFile.write("dn: %s\n" % ds_domainBackend_dn)
        currentSettingFile.write("changetype: modify\n")
        currentSettingFile.write("replace: %s\n" % ds_backend_ecache)
        currentSettingFile.write("%s: %s\n" % (ds_backend_ecache, entry[ds_backend_ecache][0]))
        currentSettingFile.write("-\n")
        currentSettingFile.write("replace: %s\n" % ds_backend_dncache)
        currentSettingFile.write("%s: %s\n\n" % (ds_backend_dncache,  entry[ds_backend_dncache][0]))
    
class bulkImport(object):
    def __init__(self):
        self.version = '0.1'

    # Prepare:
    #   - Tune DS to adapt it to the system/entries
    #   - Stop IPA to give more room to DS
    def _prepare_subparser(self, subparsers):
        prepare_parser = subparsers.add_parser(
                'prepare',
                help='Prepare Bulk Import')
        prepare_parser.add_argument('-f', '--ldiffile', dest='ldiffile', type=str, nargs='?', required=True,
                metavar='LDIF-FILE',
                            help='File containing the entries in LDIF format')
        prepare_parser.add_argument('-w', '--password', dest='password', type=str, nargs='?',
                metavar='PASSWORD',
                            help='Server Identifier (Default: %s) ' % get_default_password())
        prepare_parser.add_argument('-p', '--port', dest='port', type=int, nargs='?',
            metavar='NON-SECURE-PORT',
                        help='Normal Port to listen (Default: %s) ' % (get_default_port()))
        prepare_parser.add_argument('-D', '--rootDN', dest='root_dn', type=str, nargs='?',
            metavar='ROOT-DN',
                        help='Uses DN as Directory Manager DN (Default: \'%s\')' % (get_default_rootdn()))
        prepare_parser.add_argument('-debug', '--debug', dest='debug_level', type=int, nargs='?',
                metavar='DEBUG_LEVEL',
                            help='Debug level (Default: 0)')
                            
        prepare_parser.set_defaults(func=self.prepare_action)
        
    def _import_subparser(self, subparsers):
        import_parser = subparsers.add_parser(
                'import',
                help='Bulk Import')
        import_parser.add_argument('-f', '--ldiffile', dest='ldiffile', type=str, nargs='?', required=True,
                metavar='LDIF-FILE',
                            help='File containing the entries in LDIF format')
        import_parser.add_argument('-w', '--password', dest='password', type=str, nargs='?',
                metavar='PASSWORD',
                            help='Server Identifier (Default: %s) ' % get_default_password())
        import_parser.add_argument('-p', '--port', dest='port', type=int, nargs='?',
            metavar='NON-SECURE-PORT',
                        help='Normal Port to listen (Default: %s) ' % (get_default_port()))
        import_parser.add_argument('-D', '--rootDN', dest='root_dn', type=str, nargs='?',
            metavar='ROOT-DN',
                        help='Uses DN as Directory Manager DN (Default: \'%s\')' % (get_default_rootdn()))
        import_parser.add_argument('-debug', '--debug', dest='debug_level', type=int, nargs='?',
                metavar='DEBUG_LEVEL',
                            help='Debug level (Default: 0)')
                            
        import_parser.set_defaults(func=self.import_action)
    
    def _finish_subparser(self, subparsers):
        finish_parser = subparsers.add_parser(
                'finish',
                help='Finish Bulk Import')
        finish_parser.add_argument('-b', '--base', dest='basedn', type=str, nargs='?',
                metavar='BASE-DN',
                            help='Base DN where to apply the fixup')
        finish_parser.add_argument('-w', '--password', dest='password', type=str, nargs='?',
                metavar='PASSWORD',
                            help='Server Identifier (Default: %s) ' % get_default_password())
        finish_parser.add_argument('-p', '--port', dest='port', type=int, nargs='?',
            metavar='NON-SECURE-PORT',
                        help='Normal Port to listen (Default: %s) ' % (get_default_port()))
        finish_parser.add_argument('-D', '--rootDN', dest='root_dn', type=str, nargs='?',
            metavar='ROOT-DN',
                        help='Uses DN as Directory Manager DN (Default: \'%s\')' % (get_default_rootdn()))
        finish_parser.add_argument('-debug', '--debug', dest='debug_level', type=int, nargs='?',
                metavar='DEBUG_LEVEL',
                            help='Debug level (Default: 0)')
                            
        finish_parser.set_defaults(func=self.finish_action)
    #
    # parser of the main command. It contains subcommands
    #
    def get_parser(self, argv):
        global debug_level
        
        parser = argparse.ArgumentParser(
        description='Bulk import of IPA entries')
    
        subparsers = parser.add_subparsers(
                metavar='SUBCOMMAND',
                help='The action to perform')

        #pdb.set_trace()
        # subcommands
        self._prepare_subparser(subparsers)
        self._import_subparser(subparsers)
        self._finish_subparser(subparsers)


        # Sanity check that the debug level is valid
        args = vars(parser.parse_args(argv))
        debug_level = args.get('debug_level', None)
        if debug_level and (int(debug_level) < 1 or int(debug_level > 5)):
            raise Exception("invalid debug level: range 1..5")
        
        return parser
        
    def prepare_action(self, args):
        global ds_dbcache_attr 
        global ds_dblock_attr
        
        args = vars(args)
        
        # retrieve the arguments
        password = args.get('password', None)
        if not password:
            password = get_default_password()
            
        port = args.get('port', None)
        if not port:
            port = get_default_port()
        port = int(port)
            
        rootdn = args.get('root_dn', None)
        if not rootdn:
            rootdn = get_default_rootdn()
        
        # Check the ldif file and stats about the entries
        ldiffile = args.get('ldiffile', None)
        if not ldiffile:
            raise Exception("File is mandatory argument")
        if not os.path.isfile(ldiffile):
            raise Exception("File %s does not exist" % ldiffile)
        ldiffile = os.path.abspath(ldiffile)
        if not os.access(ldiffile, os.R_OK):
            raise Exception("File %s is not readable" % ldiffile)
        ldifentries, avgEntrySz, avgEntryDNSz = entriesInfo(ldiffile)
        

        
        # Now we are going to set the appropriate tuning
        # DS needs to be started
        ds_action('start')

        # Establish the connection to the directory
        ldap_uri = 'ldap:///%s:%d' % (socket.gethostname(), port)
        conn = ldap.initialize(ldap_uri)
        conn.simple_bind_s(rootdn, password)
        
        # backup the current setting to be able to restore it later
        extCurrentSetting = time.strftime("%Y%m%d-%H%M%S")
        currentSettingFile = open('bulk_import_bak.%s' % extCurrentSetting, 'w')
        backupCurrentTuning(conn, currentSettingFile)
        currentSettingFile.close()
        
        # time to set DS tuning
        (entryCache, dbCache, dnCache, dblock) = getTuning(ldifentries, avgEntryDNSz)
        setNewTuning(conn=conn, entrycache=entryCache, dbcache=dbCache, dncache=dnCache, dblock=dblock)
        
        print "Preparation of the bulk import is now completed"
        print "If you want to continue:"
        print "  - run %s import -f %s" % (__file__, ldiffile)
        print "  - run %s finish -b <suffix_dn>" % (__file__)
        print ""
        print "If you want to not continue:"
        print "  - run %s abort" % __file__
        print ""


    def import_action(self, args):

        args = vars(args)

        # retrieve the arguments
        password = args.get('password', None)
        if not password:
            password = get_default_password()

        port = args.get('port', None)
        if not port:
            port = get_default_port()
        port = int(port)

        rootdn = args.get('root_dn', None)
        if not rootdn:
            rootdn = get_default_rootdn()

        # Check the ldif file and stats about the entries
        ldiffile = args.get('ldiffile', None)
        if not ldiffile:
            raise Exception("File is mandatory argument")
        if not os.path.isfile(ldiffile):
            raise Exception("File %s does not exist" % ldiffile)
        ldiffile = os.path.abspath(ldiffile)
        if not os.access(ldiffile, os.R_OK):
            raise Exception("File %s is not readable" % ldiffile)

        debug_level = args.get('debug_level', None)
        
        # Now we are going to set the appropriate tuning
        # DS needs to be started
        ds_action('start')

        cmd = ["/usr/bin/ldapadd", "-h", socket.gethostname(), "-p", str(port), "-D", "%s" % rootdn, "-w", password, "-f", ldiffile]
        #pdb.set_trace()
        process = Popen(cmd, stdout=PIPE, stderr=PIPE)
        for line in iter(process.stdout.readline,''):
            if debug_level > 1:
                print line
                
        print "Bulk import is now completed"
        print "You need to run:"
        print "  - run %s finish -b <suffix_dn>" % (__file__)
                
    def finish_action(self, args):

        args = vars(args)

        # retrieve the arguments
        password = args.get('password', None)
        if not password:
            password = get_default_password()

        port = args.get('port', None)
        if not port:
            port = get_default_port()
        port = int(port)

        rootdn = args.get('root_dn', None)
        if not rootdn:
            rootdn = get_default_rootdn()

        basedn = args.get('basedn', None)

        debug_level = args.get('debug_level', None)
        
        # Reenable memberof
        #  - start DS
        #  - establish a connection
        #  - enable the plugin
        #  - restart DS
        #  - establish a connection
        ds_action('start')
        
        # Establish the connection to the directory
        ldap_uri = 'ldap:///%s:%d' % (socket.gethostname(), port)
        conn = ldap.initialize(ldap_uri)
        conn.simple_bind_s(rootdn, password)
        
        # enable the plugin 
        mods = [(ldap.MOD_REPLACE, "nsslapd-pluginEnabled", 'on')]
        conn.modify_s(ds_membeof_plugin, mods)
        ds_action('restart')
        
        # reopen the connection
        conn = ldap.initialize(ldap_uri)
        conn.simple_bind_s(rootdn, password)
        
        if not basedn:
            results = conn.search_s("", ldap.SCOPE_BASE, "(objectclass=*)",["defaultnamingcontext"])
            dn, entry = results[0]
            basedn = entry["defaultnamingcontext"][0]
        
        # run the fixup
        #pdb.set_trace()
        for fixup_filter in fixup_filters:
            cmd = ["/usr/sbin/fixup-memberof.pl", "-D", "%s" % rootdn, "-w", password, "-Z", getServerID(), "-b", basedn, "-f", fixup_filter, "-P", "LDAP"]
            process = Popen(cmd, stdout=PIPE, stderr=PIPE)
            taskdn = None
            for line in iter(process.stdout.readline,''):
                m = re.search('Successfully added task entry "(.+?)"', line)
                if m:
                    taskdn = m.group(1)
                    break
            print "Waiting for %s fixup to complete..." % fixup_filter

            delay = 1
            while True:
                results = conn.search_s(taskdn, ldap.SCOPE_BASE, "(objectclass=*)",[taskStatus])
                dn, entry = results[0]
                if taskStatus in entry:
                    m = re.search('Memberof task finished', entry[taskStatus][0])
                    if not m:
                        delay = min(delay*2, 10)
                        if delay > 10:
                            print "Wait 10s more"
                        time.sleep(delay)
                    else:
                        print "Completed filter %s" % fixup_filter
                        print ""
                        break
                else:
                    delay = min(delay*2, 10)
                    if delay > 10:
                        print "Wait 10s more"
                    time.sleep(1)
        
        
        # Fixup is not completed - reenable Schema Compat
        mods = [(ldap.MOD_REPLACE, "nsslapd-pluginEnabled", 'off')]
        conn.modify_s(ds_schemaCompat_plugin, mods)
        ds_action('restart')
        
        print "Bulk import and fixup tasks are now completed"


        
    def main(self, argv):
        parser = self.get_parser(argv)
        args = parser.parse_args(argv)
        args.func(args)
        return

if __name__ == "__main__":
    bulkImport().main(sys.argv[1:])

-- 
Manage your subscription for the Freeipa-devel mailing list:
https://www.redhat.com/mailman/listinfo/freeipa-devel
Contribute to FreeIPA: http://www.freeipa.org/page/Contribute/Code

Reply via email to