The patchset ensure, the upgrade order will respect ordering of entries
in *.update files.
Required for: https://fedorahosted.org/freeipa/ticket/4904
Patch 205 also fixes https://fedorahosted.org/freeipa/ticket/3560
Required patch mbasti-0203
Patches attached.
--
Martin Basti
From 9dd32e80f37b852feb980fd4ef2ec7c082ffc1a5 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Thu, 26 Feb 2015 12:01:19 +0100
Subject: [PATCH 2/5] Server Upgrade: do not sort updates by DN
Ticket: https://fedorahosted.org/freeipa/ticket/4904
---
ipaserver/install/ldapupdate.py | 15 ++-------------
1 file changed, 2 insertions(+), 13 deletions(-)
diff --git a/ipaserver/install/ldapupdate.py b/ipaserver/install/ldapupdate.py
index 53d5407d5e8a15abe13f2f6d8b3df74ca100ea5a..e8516ff86a951f828c4213f8e70db613b99ed8c4 100644
--- a/ipaserver/install/ldapupdate.py
+++ b/ipaserver/install/ldapupdate.py
@@ -784,22 +784,11 @@ class LDAPUpdate:
raise RuntimeError("Offline updates are not supported.")
def _run_updates(self, all_updates):
- # For adds and updates we want to apply updates from shortest
- # to greatest length of the DN.
- # For deletes we want the reverse
- def update_sort_key(dn_update):
- dn, update = dn_update
- assert isinstance(dn, DN)
- return len(dn)
- sorted_updates = sorted(all_updates.iteritems(), key=update_sort_key)
-
- for dn, update in sorted_updates:
+ for dn, update in all_updates.iteritems():
self._update_record(update)
- # Now run the deletes in reversed order
- sorted_updates.reverse()
- for dn, update in sorted_updates:
+ for dn, update in all_updates.iteritems():
self._delete_record(update)
def update(self, files, ordered=False):
--
2.1.0
From 689caca4d322a18108756b530522be625f5b0964 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Thu, 5 Mar 2015 16:56:02 +0100
Subject: [PATCH 3/5] Server Upgrade: Upgrade one file per time
* Files are sorted alphabetically, no numbering required anymore
* One file updated per time
Ticket: https://fedorahosted.org/freeipa/ticket/3560
---
ipaserver/install/ldapupdate.py | 54 ++++++++++++++---------------------------
1 file changed, 18 insertions(+), 36 deletions(-)
diff --git a/ipaserver/install/ldapupdate.py b/ipaserver/install/ldapupdate.py
index e8516ff86a951f828c4213f8e70db613b99ed8c4..3b4aa58d9e84006510c23e4aa7d52a84c205f79c 100644
--- a/ipaserver/install/ldapupdate.py
+++ b/ipaserver/install/ldapupdate.py
@@ -793,44 +793,27 @@ class LDAPUpdate:
def update(self, files, ordered=False):
"""Execute the update. files is a list of the update files to use.
+ :param ordered: Update files are executed in alphabetical order
- If ordered is True then the updates the file must be of the form
- ##-name.update where ## is an integer between 10 and 89. The
- changes are applied to LDAP at the end of each value divisible
- by 10, so after 20, 30, etc.
-
- returns True if anything was changed, otherwise False
+ returns True if anything was changed, otherwise False
"""
- pat = re.compile(r'(\d+)-.*\.update')
all_updates = {}
- r = 20
- if self.plugins:
- self.info('PRE_UPDATE')
- updates = api.Backend.updateclient.update(PRE_UPDATE, self.dm_password, self.ldapi, self.live_run)
- self.merge_updates(all_updates, updates)
try:
self.create_connection()
- if ordered and all_updates:
+ if self.plugins:
+ self.info('PRE_UPDATE')
+ updates = api.Backend.updateclient.update(PRE_UPDATE, self.dm_password, self.ldapi, self.live_run)
+ self.merge_updates(all_updates, updates)
# flush out PRE_UPDATE plugin updates before we begin
self._run_updates(all_updates)
all_updates = {}
- for f in files:
- name = os.path.basename(f)
- if ordered:
- m = pat.match(name)
- if not m:
- raise RuntimeError("Filename does not match format #-name.update: %s" % f)
- index = int(m.group(1))
- if index < 10 or index > 90:
- raise RuntimeError("Index not legal range: %d" % index)
-
- if index >= r:
- self._run_updates(all_updates)
- all_updates = {}
- r += 10
+ upgrade_files = files
+ if ordered:
+ upgrade_files = sorted(files)
+ for f in upgrade_files:
try:
self.info("Parsing update file '%s'" % f)
data = self.read_file(f)
@@ -839,17 +822,16 @@ class LDAPUpdate:
sys.exit(e)
self.parse_update_file(f, data, all_updates)
+ self._run_updates(all_updates)
+ all_updates = {}
- self._run_updates(all_updates)
+ if self.plugins:
+ self.info('POST_UPDATE')
+ updates = api.Backend.updateclient.update(POST_UPDATE, self.dm_password, self.ldapi, self.live_run)
+ self.merge_updates(all_updates, updates)
+ self._run_updates(all_updates)
finally:
- if self.conn: self.conn.unbind()
-
- if self.plugins:
- self.info('POST_UPDATE')
- all_updates = {}
- updates = api.Backend.updateclient.update(POST_UPDATE, self.dm_password, self.ldapi, self.live_run)
- self.merge_updates(all_updates, updates)
- self._run_updates(all_updates)
+ self.close_connection()
return self.modified
--
2.1.0
From 4545a8ebdbe2b5e36682328097ff47c12c311b1a Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Thu, 5 Mar 2015 18:42:03 +0100
Subject: [PATCH 4/5] Server Upgrade: Set modified to false, before each update
Variable self.modified should be set to false before each run of update
Ticket: https://fedorahosted.org/freeipa/ticket/3560
---
ipaserver/install/ldapupdate.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/ipaserver/install/ldapupdate.py b/ipaserver/install/ldapupdate.py
index 3b4aa58d9e84006510c23e4aa7d52a84c205f79c..92b6d56ae51537b8020c3a162be5593364d31aca 100644
--- a/ipaserver/install/ldapupdate.py
+++ b/ipaserver/install/ldapupdate.py
@@ -797,7 +797,7 @@ class LDAPUpdate:
returns True if anything was changed, otherwise False
"""
-
+ self.modified = False
all_updates = {}
try:
self.create_connection()
@@ -841,6 +841,7 @@ class LDAPUpdate:
Apply updates internally as opposed to from a file.
updates is a dictionary containing the updates
"""
+ self.modified = False
if not self.conn:
self.create_connection()
--
2.1.0
From 4966763205c5990dc34188efb1fe59261fa7e851 Mon Sep 17 00:00:00 2001
From: Martin Basti <mba...@redhat.com>
Date: Fri, 6 Mar 2015 15:14:17 +0100
Subject: [PATCH 5/5] Server Upgrade: Update entries in order specified in file
Dictionary replaced with list. Particular upgrades are
executed in the same order as they are specified in update
a file.
Different updates for the smae cn, are not merged into one upgrade
https://fedorahosted.org/freeipa/ticket/4904
---
ipaserver/install/ldapupdate.py | 75 +++++---------------------
ipaserver/install/plugins/adtrust.py | 10 ++--
ipaserver/install/plugins/ca_renewal_master.py | 2 -
ipaserver/install/plugins/dns.py | 6 +--
ipaserver/install/plugins/rename_managed.py | 7 +--
ipaserver/install/plugins/update_passsync.py | 3 +-
ipaserver/install/plugins/update_uniqueness.py | 2 +-
ipaserver/install/plugins/updateclient.py | 5 +-
ipaserver/install/plugins/upload_cacrt.py | 8 +--
9 files changed, 30 insertions(+), 88 deletions(-)
diff --git a/ipaserver/install/ldapupdate.py b/ipaserver/install/ldapupdate.py
index 92b6d56ae51537b8020c3a162be5593364d31aca..3e4fc3f7a1de52f7019e674bb04000a082e53ea7 100644
--- a/ipaserver/install/ldapupdate.py
+++ b/ipaserver/install/ldapupdate.py
@@ -137,22 +137,20 @@ class LDAPUpdate:
4: 'cn=bob,ou=people,dc=example,dc=com',
}
- all_updates = {
- 'dn': 'cn=config,dc=example,dc=com':
+ all_updates = [
{
'dn': 'cn=config,dc=example,dc=com',
'default': ['attr1':default1'],
'updates': ['action:attr1:value1',
'action:attr2:value2]
},
- 'dn': 'cn=bob,ou=people,dc=example,dc=com':
{
'dn': 'cn=bob,ou=people,dc=example,dc=com',
'default': ['attr3':default3'],
'updates': ['action:attr3:value3',
'action:attr4:value4],
}
- }
+ ]
The default and update lists are "dispositions"
@@ -279,49 +277,6 @@ class LDAPUpdate:
if fd != sys.stdin: fd.close()
return text
- def _combine_updates(self, all_updates, update):
- 'Combine a new update with the list of total updates'
- dn = update.get('dn')
- assert isinstance(dn, DN)
-
- if not all_updates.get(dn):
- all_updates[dn] = update
- return
-
- existing_update = all_updates[dn]
- if 'default' in update:
- disposition_list = existing_update.setdefault('default', [])
- disposition_list.extend(update['default'])
- elif 'updates' in update:
- disposition_list = existing_update.setdefault('updates', [])
- disposition_list.extend(update['updates'])
- else:
- self.debug("Unknown key in updates %s" % update.keys())
-
- def merge_updates(self, all_updates, updates):
- '''
- Add the new_update dict to the all_updates dict. If an entry
- in the new_update already has an entry in all_updates merge
- the two entries sensibly assuming the new entries take
- precedence. Otherwise just add the new entry.
- '''
-
- for new_update in updates:
- for new_dn, new_entry in new_update.iteritems():
- existing_entry = all_updates.get(new_dn)
- if existing_entry:
- # If the existing entry is marked for deletion but the
- # new entry is not also a delete then clear the delete
- # flag otherwise the newer update will be lost.
- if existing_entry.has_key('deleteentry') and not new_entry.has_key('deleteentry'):
- self.warning("ldapupdate: entry '%s' previously marked for deletion but" +
- " this subsequent update reestablishes it: %s", new_dn, new_entry)
- del existing_entry['deleteentry']
- existing_entry.update(new_entry)
- else:
- all_updates[new_dn] = new_entry
-
-
def parse_update_file(self, data_source_name, source_data, all_updates):
"""Parse the update file into a dictonary of lists and apply the update
for each DN in the file."""
@@ -380,11 +335,12 @@ class LDAPUpdate:
def emit_update(update):
'''
- When processing a dn is completed emit the update by merging it into
- the set of all updates.
+ When processing a dn is completed emit the update by appending it
+ into list of all updates
'''
-
- self._combine_updates(all_updates, update)
+ dn = update.get('dn')
+ assert isinstance(dn, DN)
+ all_updates.append(update)
# Iterate over source input lines
for source_line in source_data:
@@ -421,7 +377,6 @@ class LDAPUpdate:
continue
else:
emit_item(logical_line)
- logical_line = ''
logical_line = source_line
if dn is not None:
@@ -784,11 +739,10 @@ class LDAPUpdate:
raise RuntimeError("Offline updates are not supported.")
def _run_updates(self, all_updates):
-
- for dn, update in all_updates.iteritems():
+ for update in all_updates:
self._update_record(update)
- for dn, update in all_updates.iteritems():
+ for update in all_updates:
self._delete_record(update)
def update(self, files, ordered=False):
@@ -798,16 +752,14 @@ class LDAPUpdate:
returns True if anything was changed, otherwise False
"""
self.modified = False
- all_updates = {}
+ all_updates = []
try:
self.create_connection()
if self.plugins:
self.info('PRE_UPDATE')
updates = api.Backend.updateclient.update(PRE_UPDATE, self.dm_password, self.ldapi, self.live_run)
- self.merge_updates(all_updates, updates)
# flush out PRE_UPDATE plugin updates before we begin
- self._run_updates(all_updates)
- all_updates = {}
+ self._run_updates(updates)
upgrade_files = files
if ordered:
@@ -823,13 +775,12 @@ class LDAPUpdate:
self.parse_update_file(f, data, all_updates)
self._run_updates(all_updates)
- all_updates = {}
+ all_updates = []
if self.plugins:
self.info('POST_UPDATE')
updates = api.Backend.updateclient.update(POST_UPDATE, self.dm_password, self.ldapi, self.live_run)
- self.merge_updates(all_updates, updates)
- self._run_updates(all_updates)
+ self._run_updates(updates)
finally:
self.close_connection()
diff --git a/ipaserver/install/plugins/adtrust.py b/ipaserver/install/plugins/adtrust.py
index 1290278cf5e5a8d25638e20c95690360d5837eac..dbec429aa4890d0a3eeef1ea3ed4524ecbcb39e8 100644
--- a/ipaserver/install/plugins/adtrust.py
+++ b/ipaserver/install/plugins/adtrust.py
@@ -65,11 +65,10 @@ class update_default_range(PostUpdate):
'iparangetype:ipa-local',
]
- updates = {}
dn = DN(('cn', '%s_id_range' % api.env.realm),
api.env.container_ranges, api.env.basedn)
- updates[dn] = {'dn': dn, 'default': range_entry}
+ update = {'dn': dn, 'default': range_entry}
# Default range entry has a hard-coded range size to 200000 which is
# a default range size in ipa-server-install. This could cause issues
@@ -115,7 +114,7 @@ class update_default_range(PostUpdate):
root_logger.error("default_range: %s", "\n".join(msg))
- return (False, True, [updates])
+ return (False, True, [update])
class update_default_trust_view(PostUpdate):
@@ -156,13 +155,12 @@ class update_default_trust_view(PostUpdate):
# We have a server with AD trust support without Default Trust View.
# Create the Default Trust View entry.
- updates = {}
- updates[default_trust_view_dn] = {
+ update = {
'dn': default_trust_view_dn,
'default': default_trust_view_entry
}
- return (False, True, [updates])
+ return (False, True, [update])
api.register(update_default_range)
api.register(update_default_trust_view)
diff --git a/ipaserver/install/plugins/ca_renewal_master.py b/ipaserver/install/plugins/ca_renewal_master.py
index e246639103ed55997af5d283d996579102d2d265..b0fb527a3e02364bb3593ff3068ae510c4ff051e 100644
--- a/ipaserver/install/plugins/ca_renewal_master.py
+++ b/ipaserver/install/plugins/ca_renewal_master.py
@@ -98,10 +98,8 @@ class update_ca_renewal_master(PostUpdate):
dn = DN(('cn', 'CA'), ('cn', self.api.env.host), base_dn)
update = {
- dn: {
'dn': dn,
'updates': ['add:ipaConfigString: caRenewalMaster'],
- },
}
return (False, True, [update])
diff --git a/ipaserver/install/plugins/dns.py b/ipaserver/install/plugins/dns.py
index 5f40be4926a7bcaf2853ff764761e18e54e51c7c..f562978bcbcc02321c0e9a668af88b4f596f8556 100644
--- a/ipaserver/install/plugins/dns.py
+++ b/ipaserver/install/plugins/dns.py
@@ -133,13 +133,11 @@ class update_dns_limits(PostUpdate):
for limit in self.limit_attributes:
limit_updates.append('only:%s:%s' % (limit, self.limit_value))
- dnsupdates = {}
- dnsupdates[dns_service_dn] = {'dn': dns_service_dn,
- 'updates': limit_updates}
+ dnsupdate = {'dn': dns_service_dn, 'updates': limit_updates}
root_logger.debug("DNS: limits for service %s will be updated" % dns_service_dn)
- return (False, True, [dnsupdates])
+ return (False, True, [dnsupdate])
api.register(update_dns_limits)
diff --git a/ipaserver/install/plugins/rename_managed.py b/ipaserver/install/plugins/rename_managed.py
index 13e6dae5d226426bb0867e5943b05722fc04aa98..adb814c1799ebbdb57118acf1ba6a52550f2f818 100644
--- a/ipaserver/install/plugins/rename_managed.py
+++ b/ipaserver/install/plugins/rename_managed.py
@@ -98,7 +98,8 @@ class GenerateUpdateMixin(object):
old_update = {'dn': entry.dn, 'deleteentry': None}
# Add the delete and replacement updates to the list of all updates
- update_list.append({entry.dn: old_update, new_dn: new_update})
+ update_list.append(old_update)
+ update_list.append(new_update)
else:
# Update the template dn by replacing the old containter with the new container
@@ -125,11 +126,11 @@ class GenerateUpdateMixin(object):
'default': entry_to_update(entry)}
# Add the replacement update to the collection of all updates
- update_list.append({new_dn: new_update})
+ update_list.append(new_update)
if len(update_list) > 0:
restart = True
- update_list.sort(reverse=True)
+ update_list.sort(reverse=True, key=lambda x: x['dn'])
return (restart, update_list)
diff --git a/ipaserver/install/plugins/update_passsync.py b/ipaserver/install/plugins/update_passsync.py
index d6595a06f4deb62b853d716012a8c594c6a76451..e0d2fc01cbc66af24cb908b80d3a031903fdc463 100644
--- a/ipaserver/install/plugins/update_passsync.py
+++ b/ipaserver/install/plugins/update_passsync.py
@@ -70,9 +70,8 @@ class update_passync_privilege_update(PostUpdate):
update = {'dn': passsync_privilege_dn,
'updates': ["add:member:'%s'" % passsync_dn]}
- updates = {passsync_privilege_dn: update}
sysupgrade.set_upgrade_state('winsync', 'passsync_privilege_updated', True)
- return (False, True, [updates])
+ return (False, True, [update])
api.register(update_passync_privilege_update)
diff --git a/ipaserver/install/plugins/update_uniqueness.py b/ipaserver/install/plugins/update_uniqueness.py
index 3017d5ac13b223a80ad1171d5adcde8fb4343562..e0ee150a7337a052731a0ed26eb64a4a8c01fb90 100644
--- a/ipaserver/install/plugins/update_uniqueness.py
+++ b/ipaserver/install/plugins/update_uniqueness.py
@@ -218,7 +218,7 @@ class update_uniqueness_plugins_to_new_syntax(PreUpdate):
"plugin %s (%s)",
entry.dn, e)
- update_list.append({entry.dn: update})
+ update_list.append(update)
return False, True, update_list
diff --git a/ipaserver/install/plugins/updateclient.py b/ipaserver/install/plugins/updateclient.py
index 8f5c5b5fdbc2b7bfec8be342ee267425c93b47cf..85ee3f8a0d17710f99a4d18c95fa3c4f93cba672 100644
--- a/ipaserver/install/plugins/updateclient.py
+++ b/ipaserver/install/plugins/updateclient.py
@@ -128,10 +128,7 @@ class updateclient(backend.Executioner):
self.restart(dm_password, live_run)
if apply_now:
- updates = {}
- for entry in res:
- updates.update(entry)
- ld.update_from_dict(updates)
+ ld.update_from_dict(res)
elif res:
result.extend(res)
diff --git a/ipaserver/install/plugins/upload_cacrt.py b/ipaserver/install/plugins/upload_cacrt.py
index 66270ae7613e935fc8df4bc90aa5001296e1c06d..dcdefee0536eb2e80fbc34823526646aee1bd61c 100644
--- a/ipaserver/install/plugins/upload_cacrt.py
+++ b/ipaserver/install/plugins/upload_cacrt.py
@@ -45,7 +45,7 @@ class update_upload_cacrt(PostUpdate):
if ca_chain:
ca_nickname = ca_chain[-1]
- updates = {}
+ updates = []
for nickname, trust_flags in db.list_certs():
if 'u' in trust_flags:
@@ -64,7 +64,7 @@ class update_upload_cacrt(PostUpdate):
if ca_enabled:
entry.append('ipaConfigString:ipaCA')
entry.append('ipaConfigString:compatCA')
- updates[dn] = {'dn': dn, 'default': entry}
+ updates.append({'dn': dn, 'default': entry})
if ca_cert:
dn = DN(('cn', 'CACert'), ('cn', 'ipa'), ('cn','etc'),
@@ -74,9 +74,9 @@ class update_upload_cacrt(PostUpdate):
'cn:CAcert',
'cACertificate;binary:%s' % ca_cert,
]
- updates[dn] = {'dn': dn, 'default': entry}
+ updates.append({'dn': dn, 'default': entry})
- return (False, True, [updates])
+ return (False, True, updates)
def _make_entry(self, cert, nickname, trust_flags):
dn = DN(('cn', nickname), ('cn', 'certificates'), ('cn', 'ipa'),
--
2.1.0
--
Manage your subscription for the Freeipa-devel mailing list:
https://www.redhat.com/mailman/listinfo/freeipa-devel
Contribute to FreeIPA: http://www.freeipa.org/page/Contribute/Code
