Yedidyah Bar David has uploaded a new change for review. Change subject: packaging: setup: allow tuning dump/restore ......................................................................
packaging: setup: allow tuning dump/restore Allow setting: - dump format (custom or plain) - filter (gzip, bzip2, xz etc) - number of restore jobs (pg_restore -j N) The default is same as today - custom format, -j 1, no filter. Currently no interaction to set these. Values used are written to the answer file, which can be edited manually. Currently only backup/restore done during engine-setup are affected. Note that the bug 1188132 mentioned below talks about engine-backup. This will be possible while working on bug 1064503, which talks about rewriting engine-backup using the engine-setup framework. Change-Id: I99de636d4da15a253f2186084902f0f133603f4d Related-To: https://bugzilla.redhat.com/1188132 Signed-off-by: Yedidyah Bar David <[email protected]> --- M packaging/setup/ovirt_engine_setup/engine/constants.py M packaging/setup/ovirt_engine_setup/engine_common/constants.py M packaging/setup/ovirt_engine_setup/engine_common/database.py M packaging/setup/plugins/ovirt-engine-common/ovirt-engine/db/connection.py 4 files changed, 129 insertions(+), 18 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/90/39190/1 diff --git a/packaging/setup/ovirt_engine_setup/engine/constants.py b/packaging/setup/ovirt_engine_setup/engine/constants.py index ab6ef6b..7002b62 100644 --- a/packaging/setup/ovirt_engine_setup/engine/constants.py +++ b/packaging/setup/ovirt_engine_setup/engine/constants.py @@ -304,6 +304,11 @@ DEFAULT_DB_PASSWORD = '' DEFAULT_DB_SECURED = False DEFAULT_DB_SECURED_HOST_VALIDATION = False + DEFAULT_DB_DUMP_FORMAT = 'c' + DEFAULT_DB_DUMP_FILTER = None + DEFAULT_DB_RESTORE_COMMAND = 'pg_restore' + DEFAULT_DB_RESTORE_FILTER = None + DEFAULT_DB_RESTORE_JOBS = 1 @util.export @@ -376,6 +381,11 @@ DEK.CONNECTION: EngineDBEnv.CONNECTION, DEK.PGPASSFILE: EngineDBEnv.PGPASS_FILE, DEK.NEW_DATABASE: EngineDBEnv.NEW_DATABASE, + DEK.DUMP_FORMAT: EngineDBEnv.DUMP_FORMAT, + DEK.DUMP_FILTER: EngineDBEnv.DUMP_FILTER, + DEK.RESTORE_COMMAND: EngineDBEnv.RESTORE_COMMAND, + DEK.RESTORE_FILTER: EngineDBEnv.RESTORE_FILTER, + DEK.RESTORE_JOBS: EngineDBEnv.RESTORE_JOBS, } @classproperty @@ -388,6 +398,11 @@ DEK.USER: Defaults.DEFAULT_DB_USER, DEK.PASSWORD: Defaults.DEFAULT_DB_PASSWORD, DEK.DATABASE: Defaults.DEFAULT_DB_DATABASE, + DEK.DUMP_FORMAT: Defaults.DEFAULT_DB_DUMP_FORMAT, + DEK.DUMP_FILTER: Defaults.DEFAULT_DB_DUMP_FILTER, + DEK.RESTORE_COMMAND: Defaults.DEFAULT_DB_RESTORE_COMMAND, + DEK.RESTORE_FILTER: Defaults.DEFAULT_DB_RESTORE_FILTER, + DEK.RESTORE_JOBS: Defaults.DEFAULT_DB_RESTORE_JOBS, } @@ -461,6 +476,36 @@ @osetupattrs( answerfile=True, ) + def DUMP_FORMAT(self): + return 'OVESETUP_DB/dumpFormat' + + @osetupattrs( + answerfile=True, + ) + def DUMP_FILTER(self): + return 'OVESETUP_DB/dumpFilter' + + @osetupattrs( + answerfile=True, + ) + def RESTORE_COMMAND(self): + return 'OVESETUP_DB/restoreCommand' + + @osetupattrs( + answerfile=True, + ) + def RESTORE_FILTER(self): + return 'OVESETUP_DB/restoreFilter' + + @osetupattrs( + answerfile=True, + ) + def RESTORE_JOBS(self): + return 'OVESETUP_DB/restoreJobs' + + @osetupattrs( + answerfile=True, + ) def FIX_DB_VIOLATIONS(self): return 'OVESETUP_DB/fixDbViolations' diff --git a/packaging/setup/ovirt_engine_setup/engine_common/constants.py b/packaging/setup/ovirt_engine_setup/engine_common/constants.py index 89097f8..4c89f2c 100644 --- a/packaging/setup/ovirt_engine_setup/engine_common/constants.py +++ b/packaging/setup/ovirt_engine_setup/engine_common/constants.py @@ -204,6 +204,11 @@ CONNECTION = 'connection' PGPASSFILE = 'pgpassfile' NEW_DATABASE = 'newDatabase' + DUMP_FORMAT = 'dumpFormat' + DUMP_FILTER = 'dumpFilter' + RESTORE_COMMAND = 'restoreCommand' + RESTORE_FILTER = 'restoreFilter' + RESTORE_JOBS = 'restoreJobs' REQUIRED_KEYS = ( HOST, @@ -216,6 +221,11 @@ CONNECTION, PGPASSFILE, NEW_DATABASE, + DUMP_FORMAT, + DUMP_FILTER, + RESTORE_COMMAND, + RESTORE_FILTER, + RESTORE_JOBS, ) DEFAULTS_KEYS = ( diff --git a/packaging/setup/ovirt_engine_setup/engine_common/database.py b/packaging/setup/ovirt_engine_setup/engine_common/database.py index e3bca58..fb3e472 100644 --- a/packaging/setup/ovirt_engine_setup/engine_common/database.py +++ b/packaging/setup/ovirt_engine_setup/engine_common/database.py @@ -21,6 +21,7 @@ import gettext import os import re +import shlex import socket import tempfile @@ -556,26 +557,61 @@ file=backupFile, ) ) - self._plugin.execute( - ( - self.command.get('pg_dump'), - '-E', 'UTF8', - '--disable-dollar-quoting', - '--disable-triggers', - '--format=c', - '-U', self.environment[self._dbenvkeys[DEK.USER]], - '-h', self.environment[self._dbenvkeys[DEK.HOST]], - '-p', str(self.environment[self._dbenvkeys[DEK.PORT]]), - '-f', backupFile, - self.environment[self._dbenvkeys[DEK.DATABASE]], + dump_args = [ + self.command.get('pg_dump'), + '-E', 'UTF8', + '--disable-dollar-quoting', + '--disable-triggers', + '--format={f}'.format( + f=self.environment[self._dbenvkeys[DEK.DUMP_FORMAT]], ), - envAppend={ - 'PGPASSWORD': '', - 'PGPASSFILE': self.environment[ - self._dbenvkeys[DEK.PGPASSFILE] - ], - }, + '-U', self.environment[self._dbenvkeys[DEK.USER]], + '-h', self.environment[self._dbenvkeys[DEK.HOST]], + '-p', str(self.environment[self._dbenvkeys[DEK.PORT]]), + ] + if self.environment[self._dbenvkeys[DEK.DUMP_FILTER]] is None: + dump_args.extend( + ( + '-f', backupFile, + ) + ) + dump_args.extend( + ( + self.environment[self._dbenvkeys[DEK.DATABASE]], + ) ) + pipe = [ + { + 'args': dump_args, + } + ] + stdout = None + if self.environment[self._dbenvkeys[DEK.DUMP_FILTER]] is not None: + # For dump, we could in principle not allow None and always pipe, + # e.g. to 'cat' if we do not want a filter. For restore (below) + # we cannot, because pg_restore -j requires a regular file. + pipe.append( + { + 'args': shlex.split( + self.environment[self._dbenvkeys[DEK.DUMP_FILTER]] + ), + } + ) + stdout = open(backupFile, 'w') + try: + self._plugin.executePipeRaw( + pipe, + envAppend={ + 'PGPASSWORD': '', + 'PGPASSFILE': self.environment[ + self._dbenvkeys[DEK.PGPASSFILE] + ], + }, + stdout=stdout, + ) + finally: + if stdout is not None: + stdout.close() return backupFile diff --git a/packaging/setup/plugins/ovirt-engine-common/ovirt-engine/db/connection.py b/packaging/setup/plugins/ovirt-engine-common/ovirt-engine/db/connection.py index 8961408..386da41 100644 --- a/packaging/setup/plugins/ovirt-engine-common/ovirt-engine/db/connection.py +++ b/packaging/setup/plugins/ovirt-engine-common/ovirt-engine/db/connection.py @@ -84,6 +84,26 @@ oenginecons.EngineDBEnv.DATABASE, None ) + self.environment.setdefault( + oenginecons.EngineDBEnv.DUMP_FORMAT, + oenginecons.Defaults.DEFAULT_DB_DUMP_FORMAT + ) + self.environment.setdefault( + oenginecons.EngineDBEnv.DUMP_FILTER, + oenginecons.Defaults.DEFAULT_DB_DUMP_FILTER + ) + self.environment.setdefault( + oenginecons.EngineDBEnv.RESTORE_COMMAND, + oenginecons.Defaults.DEFAULT_DB_RESTORE_COMMAND + ) + self.environment.setdefault( + oenginecons.EngineDBEnv.RESTORE_FILTER, + oenginecons.Defaults.DEFAULT_DB_RESTORE_FILTER + ) + self.environment.setdefault( + oenginecons.EngineDBEnv.RESTORE_JOBS, + oenginecons.Defaults.DEFAULT_DB_RESTORE_JOBS + ) self.environment[oenginecons.EngineDBEnv.CONNECTION] = None self.environment[oenginecons.EngineDBEnv.STATEMENT] = None -- To view, visit https://gerrit.ovirt.org/39190 To unsubscribe, visit https://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I99de636d4da15a253f2186084902f0f133603f4d Gerrit-PatchSet: 1 Gerrit-Project: ovirt-engine Gerrit-Branch: master Gerrit-Owner: Yedidyah Bar David <[email protected]> _______________________________________________ Engine-patches mailing list [email protected] http://lists.ovirt.org/mailman/listinfo/engine-patches
