A set of pki-server commands has been added to simplify upgrading TPS VLV indexes.
https://fedorahosted.org/pki/ticket/2354 https://fedorahosted.org/pki/ticket/2263 https://fedorahosted.org/pki/ticket/2269 Upgrade instruction: http://pki.fedoraproject.org/wiki/Database_Upgrade_for_PKI_10.3.x#Fixing_VLV_filters_and_sort_orders -- Endi S. Dewata
>From 6fe6e89f049395f92b85b31cb1285587836e9d53 Mon Sep 17 00:00:00 2001 From: "Endi S. Dewata" <[email protected]> Date: Wed, 8 Jun 2016 06:12:22 +0200 Subject: [PATCH] Added TPS VLV management CLI. A set of pki-server commands has been added to simplify upgrading TPS VLV indexes. https://fedorahosted.org/pki/ticket/2354 https://fedorahosted.org/pki/ticket/2263 https://fedorahosted.org/pki/ticket/2269 --- base/common/python/pki/util.py | 12 + base/server/python/pki/server/__init__.py | 10 + base/server/python/pki/server/cli/kra.py | 35 +-- base/server/python/pki/server/cli/tps.py | 456 ++++++++++++++++++++++++++++++ 4 files changed, 489 insertions(+), 24 deletions(-) diff --git a/base/common/python/pki/util.py b/base/common/python/pki/util.py index 7220a7d498741e4a2c6b405a75e34ea4eaa8934f..2cac1d8160bf749286ffff4d7175ae580fb029ff 100644 --- a/base/common/python/pki/util.py +++ b/base/common/python/pki/util.py @@ -123,3 +123,15 @@ def chown(path, uid, gid): os.chown(itempath, uid, gid) elif os.path.isdir(itempath): chown(itempath, uid, gid) + + +def customize_file(input_file, output_file, params): + """ + Customize a file with specified parameters. + """ + + with open(input_file) as infile, open(output_file, 'w') as outfile: + for line in infile: + for src, target in params.items(): + line = line.replace(src, target) + outfile.write(line) diff --git a/base/server/python/pki/server/__init__.py b/base/server/python/pki/server/__init__.py index 8347311cfe1692a500f21d86b0ac0c8261e1d752..bf705fd358b00fb36fe9df25d7c6d74cff0d4154 100644 --- a/base/server/python/pki/server/__init__.py +++ b/base/server/python/pki/server/__init__.py @@ -35,6 +35,7 @@ import tempfile import pki import pki.nssdb +import pki.util INSTANCE_BASE_DIR = '/var/lib/pki' REGISTRY_DIR = '/etc/sysconfig/pki' @@ -370,6 +371,15 @@ class PKISubsystem(object): return connection + def customize_file(self, input_file, output_file): + params = { + '{instanceId}': self.instance.name, + '{database}': self.config['internaldb.database'], + '{rootSuffix}': self.config['internaldb.basedn'] + } + + pki.util.customize_file(input_file, output_file, params) + def __repr__(self): return str(self.instance) + '/' + self.name diff --git a/base/server/python/pki/server/cli/kra.py b/base/server/python/pki/server/cli/kra.py index 29d9e0168b95d1b82e5770679d680b7324dba545..95ef710fa7def92d02e894a281c4977093aed6b6 100644 --- a/base/server/python/pki/server/cli/kra.py +++ b/base/server/python/pki/server/cli/kra.py @@ -31,6 +31,7 @@ import tempfile import pki.cli + KRA_VLVS = ['allKeys', 'kraAll', 'kraArchival', 'kraRecovery', 'kraCanceled', 'kraCanceledEnrollment', 'kraCanceledRecovery', @@ -40,18 +41,6 @@ KRA_VLV_PATH = '/usr/share/pki/kra/conf/vlv.ldif' KRA_VLV_TASKS_PATH = '/usr/share/pki/kra/conf/vlvtasks.ldif' -def create_ldif(instance, subsystem, ldif_path, out_file): - subs = {'{instanceId}': instance.name, - '{database}': subsystem.config['internaldb.database'], - '{rootSuffix}': subsystem.config['internaldb.basedn']} - - with open(ldif_path) as infile, open(out_file, 'w') as outfile: - for line in infile: - for src, target in subs.items(): - line = line.replace(src, target) - outfile.write(line) - - class KRACLI(pki.cli.CLI): def __init__(self): @@ -170,7 +159,7 @@ class KRADBCLI(pki.cli.CLI): def __init__(self): super(KRADBCLI, self).__init__( - 'db', 'KRA DB management commands') + 'db', 'KRA database management commands') self.add_module(KRADBVLVCLI()) @@ -188,8 +177,6 @@ class KRADBVLVCLI(pki.cli.CLI): class KRADBVLVAddCLI(pki.cli.CLI): - KRA_VLV_PATH = '/usr/share/pki/kra/conf/vlv.ldif' - def __init__(self): super(KRADBVLVAddCLI, self).__init__( 'add', 'Add KRA VLVs') @@ -200,7 +187,7 @@ class KRADBVLVAddCLI(pki.cli.CLI): print() print(' -i, --instance <instance ID> Instance ID (default: pki-tomcat).') print(' -D, --bind-dn <Bind DN> Connect DN (default: cn=Directory Manager).') - print(' -w, --bind-password <password> Password to connect to DB.') + print(' -w, --bind-password <password> Password to connect to database.') print(' -v, --verbose Run in verbose mode.') print(' -g, --generate-ldif <outfile> Generate LDIF of required changes.') print(' --help Show help message.') @@ -260,12 +247,12 @@ class KRADBVLVAddCLI(pki.cli.CLI): return if self.out_file: - create_ldif(instance, subsystem, self.KRA_VLV_PATH, self.out_file) + subsystem.customize_file(KRA_VLV_PATH, self.out_file) print('KRA VLVs written to ' + self.out_file) return ldif_file = tempfile.NamedTemporaryFile(delete=False) - create_ldif(instance, subsystem, self.KRA_VLV_PATH, ldif_file.name) + subsystem.customize_file(KRA_VLV_PATH, ldif_file.name) conn = subsystem.open_database(bind_dn=bind_dn, bind_password=bind_password) @@ -280,7 +267,7 @@ class KRADBVLVAddCLI(pki.cli.CLI): os.unlink(ldif_file.name) conn.close() - print('KRA VLVs added to the DB for ' + instance.name) + print('KRA VLVs added to the database for ' + instance.name) class KRADBVLVDeleteCLI(pki.cli.CLI): @@ -295,7 +282,7 @@ class KRADBVLVDeleteCLI(pki.cli.CLI): print() print(' -i, --instance <instance ID> Instance ID (default: pki-tomcat).') print(' -D, --bind-dn <Bind DN> Connect DN (default: cn=Directory Manager).') - print(' -w, --bind-password <password> Password to connect to DB.') + print(' -w, --bind-password <password> Password to connect to database.') print(' -g, --generate-ldif <outfile> Generate LDIF of required changes.') print(' -v, --verbose Run in verbose mode.') print(' --help Show help message.') @@ -395,7 +382,7 @@ class KRADBVLVDeleteCLI(pki.cli.CLI): finally: conn.close() - print('KRA VLVs deleted from the DB for ' + instance.name) + print('KRA VLVs deleted from the database for ' + instance.name) class KRADBVLVReindexCLI(pki.cli.CLI): @@ -410,7 +397,7 @@ class KRADBVLVReindexCLI(pki.cli.CLI): print() print(' -i, --instance <instance ID> Instance ID (default: pki-tomcat).') print(' -D, --bind-dn <Bind DN> Connect DN (default: cn=Directory Manager).') - print(' -w, --bind-password <password> Password to connect to DB.') + print(' -w, --bind-password <password> Password to connect to database.') print(' -g, --generate-ldif <outfile> Generate LDIF of required changes.') print(' -v, --verbose Run in verbose mode.') print(' --help Show help message.') @@ -472,12 +459,12 @@ class KRADBVLVReindexCLI(pki.cli.CLI): return if self.out_file: - create_ldif(instance, subsystem, KRA_VLV_TASKS_PATH, self.out_file) + subsystem.customize_file(KRA_VLV_TASKS_PATH, self.out_file) print('KRA VLV reindex task written to ' + self.out_file) return ldif_file = tempfile.NamedTemporaryFile(delete=False) - create_ldif(instance, subsystem, KRA_VLV_TASKS_PATH, ldif_file.name) + subsystem.customize_file(KRA_VLV_TASKS_PATH, ldif_file.name) conn = subsystem.open_database(bind_dn=bind_dn, bind_password=bind_password) diff --git a/base/server/python/pki/server/cli/tps.py b/base/server/python/pki/server/cli/tps.py index f40223ddb70f97e5ee7a35005a9c0e9b6da1a268..0b9a28df463307fd1636a8b8c15cb2f56321f651 100644 --- a/base/server/python/pki/server/cli/tps.py +++ b/base/server/python/pki/server/cli/tps.py @@ -22,14 +22,22 @@ from __future__ import absolute_import from __future__ import print_function import getopt import io +import ldap +import ldap.modlist +import ldif import os import shutil import sys import tempfile +import time import pki.cli +TPS_VLV_PATH = '/usr/share/pki/tps/conf/vlv.ldif' +TPS_VLV_TASKS_PATH = '/usr/share/pki/tps/conf/vlvtasks.ldif' + + class TPSCLI(pki.cli.CLI): def __init__(self): @@ -37,6 +45,7 @@ class TPSCLI(pki.cli.CLI): 'tps', 'TPS management commands') self.add_module(TPSCloneCLI()) + self.add_module(TPSDBCLI()) class TPSCloneCLI(pki.cli.CLI): @@ -139,3 +148,450 @@ class TPSClonePrepareCLI(pki.cli.CLI): finally: shutil.rmtree(tmpdir) + + +class TPSDBCLI(pki.cli.CLI): + + def __init__(self): + super(TPSDBCLI, self).__init__( + 'db', 'TPS database management commands') + + self.add_module(TPSDBVLVCLI()) + + +class TPSDBVLVCLI(pki.cli.CLI): + + def __init__(self): + super(TPSDBVLVCLI, self).__init__( + 'vlv', 'TPS VLV management commands') + + self.add_module(TPSDBVLVFindCLI()) + self.add_module(TPSDBVLVAddCLI()) + self.add_module(TPSDBVLVDeleteCLI()) + self.add_module(TPSDBVLVReindexCLI()) + + +class TPSDBVLVFindCLI(pki.cli.CLI): + + def __init__(self): + super(TPSDBVLVFindCLI, self).__init__( + 'find', 'Find TPS VLVs') + + def print_help(self): + print('Usage: pki-server tps-db-vlv-find [OPTIONS]') + print() + print(' -i, --instance <instance ID> Instance ID (default: pki-tomcat).') + print(' -D, --bind-dn <Bind DN> Connect DN (default: cn=Directory Manager).') + print(' -w, --bind-password <password> Password to connect to database.') + print(' -v, --verbose Run in verbose mode.') + print(' --help Show help message.') + print() + + def execute(self, args): + try: + opts, _ = getopt.gnu_getopt( + args, + 'i:D:w:x:g:v', + ['instance=', 'bind-dn=', 'bind-password=', 'generate-ldif=', + 'verbose', 'help'] + ) + + except getopt.GetoptError as e: + print('ERROR: ' + str(e)) + self.print_help() + sys.exit(1) + + instance_name = 'pki-tomcat' + bind_dn = None + bind_password = None + + for o, a in opts: + if o in ('-i', '--instance'): + instance_name = a + + elif o in ('-D', '--bind-dn'): + bind_dn = a + + elif o in ('-w', '--bind-password'): + bind_password = a + + elif o in ('-v', '--verbose'): + self.set_verbose(True) + + elif o == '--help': + self.print_help() + sys.exit() + + else: + print('ERROR: unknown option ' + o) + self.print_help() + sys.exit(1) + + instance = pki.server.PKIInstance(instance_name) + instance.load() + + subsystem = instance.get_subsystem('tps') + + if not subsystem: + raise Exception('Subsystem not found') + + self.find_vlv(subsystem, bind_dn, bind_password) + + def find_vlv(self, subsystem, bind_dn, bind_password): + + conn = subsystem.open_database(bind_dn=bind_dn, + bind_password=bind_password) + + try: + database = subsystem.config['internaldb.database'] + base_dn = 'cn=' + database + ',cn=ldbm database, cn=plugins, cn=config' + + if self.verbose: + print('Searching %s' % base_dn) + + entries = conn.ldap.search_s( + base_dn, + ldap.SCOPE_SUBTREE, + '(|(objectClass=vlvSearch)(objectClass=vlvIndex))') + + self.print_message('%d entries found' % len(entries)) + + if not entries: + return + + first = True + for entry in entries: + dn = entry[0] + attrs = entry[1] + + if first: + first = False + else: + print() + + print(' dn: %s' % dn) + for key, values in attrs.items(): + for value in values: + print(' %s: %s' % (key, value)) + + finally: + conn.close() + + +class TPSDBVLVAddCLI(pki.cli.CLI): + + def __init__(self): + super(TPSDBVLVAddCLI, self).__init__( + 'add', 'Add TPS VLVs') + + def print_help(self): + print('Usage: pki-server tps-db-vlv-add [OPTIONS]') + print() + print(' -i, --instance <instance ID> Instance ID (default: pki-tomcat).') + print(' -D, --bind-dn <Bind DN> Connect DN (default: cn=Directory Manager).') + print(' -w, --bind-password <password> Password to connect to database.') + print(' -v, --verbose Run in verbose mode.') + print(' -g, --generate-ldif <outfile> Generate LDIF of required changes.') + print(' --help Show help message.') + print() + + def execute(self, args): + try: + opts, _ = getopt.gnu_getopt( + args, + 'i:D:w:x:g:v', + ['instance=', 'bind-dn=', 'bind-password=', 'generate-ldif=', + 'verbose', 'help'] + ) + + except getopt.GetoptError as e: + print('ERROR: ' + str(e)) + self.print_help() + sys.exit(1) + + instance_name = 'pki-tomcat' + bind_dn = 'cn=Directory Manager' + bind_password = None + out_file = None + + for o, a in opts: + if o in ('-i', '--instance'): + instance_name = a + + elif o in ('-D', '--bind-dn'): + bind_dn = a + + elif o in ('-w', '--bind-password'): + bind_password = a + + elif o in ('-g', '--generate-ldif'): + out_file = a + + elif o in ('-v', '--verbose'): + self.set_verbose(True) + + elif o == '--help': + self.print_help() + sys.exit() + + else: + print('ERROR: unknown option ' + o) + self.print_help() + sys.exit(1) + + instance = pki.server.PKIInstance(instance_name) + instance.load() + + subsystem = instance.get_subsystem('tps') + + if not subsystem: + raise Exception('Subsystem not found') + + if out_file: + subsystem.customize_file(TPS_VLV_PATH, out_file) + self.print_message('VLVs: %s' % out_file) + return + + self.add_vlv(subsystem, bind_dn, bind_password) + + def add_vlv(self, subsystem, bind_dn, bind_password): + + input_file = tempfile.NamedTemporaryFile(delete=False) + subsystem.customize_file(TPS_VLV_PATH, input_file.name) + + conn = subsystem.open_database(bind_dn=bind_dn, + bind_password=bind_password) + + try: + parser = ldif.LDIFRecordList(open(input_file.name, 'rb')) + + parser.parse() + + for dn, entry in parser.all_records: + + if self.verbose: + print('Adding %s' % dn) + + add_modlist = ldap.modlist.addModlist(entry) + conn.ldap.add_s(dn, add_modlist) + + finally: + os.unlink(input_file.name) + conn.close() + + self.print_message('VLVs added') + + +class TPSDBVLVDeleteCLI(pki.cli.CLI): + + def __init__(self): + super(TPSDBVLVDeleteCLI, self).__init__( + 'del', 'Delete TPS VLVs') + + def print_help(self): + print('Usage: pki-server tps-db-vlv-del [OPTIONS]') + print() + print(' -i, --instance <instance ID> Instance ID (default: pki-tomcat).') + print(' -D, --bind-dn <Bind DN> Connect DN (default: cn=Directory Manager).') + print(' -w, --bind-password <password> Password to connect to DB.') + print(' -v, --verbose Run in verbose mode.') + print(' --help Show help message.') + print() + + def execute(self, args): + try: + opts, _ = getopt.gnu_getopt( + args, + 'i:D:w:x:g:v', + ['instance=', 'bind-dn=', 'bind-password=', + 'verbose', 'help'] + ) + + except getopt.GetoptError as e: + print('ERROR: ' + str(e)) + self.print_help() + sys.exit(1) + + instance_name = 'pki-tomcat' + bind_dn = None + bind_password = None + + for o, a in opts: + if o in ('-i', '--instance'): + instance_name = a + + elif o in ('-D', '--bind-dn'): + bind_dn = a + + elif o in ('-w', '--bind-password'): + bind_password = a + + elif o in ('-v', '--verbose'): + self.set_verbose(True) + + elif o == '--help': + self.print_help() + sys.exit() + + else: + print('ERROR: unknown option ' + o) + self.print_help() + sys.exit(1) + + instance = pki.server.PKIInstance(instance_name) + instance.load() + + subsystem = instance.get_subsystem('tps') + + if not subsystem: + raise Exception('Subsystem not found') + + self.delete_vlv(subsystem, bind_dn, bind_password) + + def delete_vlv(self, subsystem, bind_dn, bind_password): + + conn = subsystem.open_database(bind_dn=bind_dn, + bind_password=bind_password) + try: + database = subsystem.config['internaldb.database'] + base_dn = 'cn=' + database + ',cn=ldbm database, cn=plugins, cn=config' + + if self.verbose: + print('Searching %s' % base_dn) + + entries = conn.ldap.search_s( + base_dn, + ldap.SCOPE_SUBTREE, + '(|(objectClass=vlvSearch)(objectClass=vlvIndex))') + + if not entries: + self.print_message('VLVs not found') + return + + for entry in reversed(entries): + dn = entry[0] + + if self.verbose: + print('Deleting %s' % dn) + + conn.ldap.delete_s(dn) + + finally: + conn.close() + + self.print_message('VLVs deleted') + + +class TPSDBVLVReindexCLI(pki.cli.CLI): + + def __init__(self): + super(TPSDBVLVReindexCLI, self).__init__( + 'reindex', 'Re-index TPS VLVs') + + def print_help(self): + print('Usage: pki-server tps-db-vlv-reindex [OPTIONS]') + print() + print(' -i, --instance <instance ID> Instance ID (default: pki-tomcat).') + print(' -D, --bind-dn <Bind DN> Connect DN (default: cn=Directory Manager).') + print(' -w, --bind-password <password> Password to connect to database.') + print(' -g, --generate-ldif <outfile> Generate LDIF of required changes.') + print(' -v, --verbose Run in verbose mode.') + print(' --help Show help message.') + print() + + def execute(self, args): + try: + opts, _ = getopt.gnu_getopt( + args, + 'i:D:w:x:g:v', + ['instance=', 'bind-dn=', 'bind-password=', 'generate-ldif=', + 'verbose', 'help'] + ) + + except getopt.GetoptError as e: + print('ERROR: ' + str(e)) + self.print_help() + sys.exit(1) + + instance_name = 'pki-tomcat' + bind_dn = 'cn=Directory Manager' + bind_password = None + out_file = None + + for o, a in opts: + if o in ('-i', '--instance'): + instance_name = a + + elif o in ('-D', '--bind-dn'): + bind_dn = a + + elif o in ('-w', '--bind-password'): + bind_password = a + + elif o in ('-g', '--generate-ldif'): + out_file = a + + elif o in ('-v', '--verbose'): + self.set_verbose(True) + + elif o == '--help': + self.print_help() + sys.exit() + + else: + print('ERROR: unknown option ' + o) + self.print_help() + sys.exit(1) + + instance = pki.server.PKIInstance(instance_name) + instance.load() + + subsystem = instance.get_subsystem('tps') + + if not subsystem: + raise Exception('Subsystem not found') + + if out_file: + subsystem.customize_file(TPS_VLV_TASKS_PATH, out_file) + self.print_message('Reindex task: %s' % out_file) + return + + self.reindex_vlv(subsystem, bind_dn, bind_password) + + def reindex_vlv(self, subsystem, bind_dn, bind_password): + + input_file = tempfile.NamedTemporaryFile(delete=False) + subsystem.customize_file(TPS_VLV_TASKS_PATH, input_file.name) + + conn = subsystem.open_database(bind_dn=bind_dn, + bind_password=bind_password) + + try: + parser = ldif.LDIFRecordList(open(input_file.name, 'rb')) + + parser.parse() + + for dn, entry in parser.all_records: + + if self.verbose: + print('Adding %s' % dn) + + add_modlist = ldap.modlist.addModlist(entry) + conn.ldap.add_s(dn, add_modlist) + + while True: + time.sleep(1) + + try: + if self.verbose: + print('Checking %s' % dn) + + conn.ldap.search_s(dn, ldap.SCOPE_BASE) + except ldap.NO_SUCH_OBJECT: + break + + finally: + os.unlink(input_file.name) + conn.close() + + self.print_message('Reindex complete') -- 2.5.5
_______________________________________________ Pki-devel mailing list [email protected] https://www.redhat.com/mailman/listinfo/pki-devel
