Yedidyah Bar David has uploaded a new change for review. Change subject: packaging: on upgrade first backup ......................................................................
packaging: on upgrade first backup Bug-Url: https://bugzilla.redhat.com/1046679 Change-Id: I1b85a6aee99e94856b233f1c2285fd69e31e053f Signed-off-by: Yedidyah Bar David <[email protected]> --- M packaging/legacy-setup/common_utils.py M packaging/legacy-setup/decorators.py M packaging/legacy-setup/ovirt-engine-dwh-setup.py 3 files changed, 125 insertions(+), 74 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-dwh refs/changes/97/22797/1 diff --git a/packaging/legacy-setup/common_utils.py b/packaging/legacy-setup/common_utils.py index c3e3995..89d0236 100755 --- a/packaging/legacy-setup/common_utils.py +++ b/packaging/legacy-setup/common_utils.py @@ -444,19 +444,31 @@ db_dict['dbname'], db_dict["username"], sql_query)) - cmd = [ - EXEC_PSQL, - "--pset=tuples_only=on", - "--set", - "ON_ERROR_STOP=1", - "--dbname", db_dict['dbname'], - "--host", db_dict["host"], - "--port", db_dict["port"], - "--username", db_dict["username"], - "-w", - "-c", sql_query, - ] - return execCmd(cmdList=cmd, failOnError=fail_on_error, msg=err_msg, envDict=envDict) + if ( + db_dict['username'] == 'admin' and + db_dict['password'] == 'dummy' and + localHost(db_dict['host']) + ): + output, rc = runPostgresSuQuery( + query=sql_query, + database=db_dict['dbname'], + failOnError=fail_on_error, + ) + else: + cmd = [ + EXEC_PSQL, + "--pset=tuples_only=on", + "--set", + "ON_ERROR_STOP=1", + "--dbname", db_dict['dbname'], + "--host", db_dict["host"], + "--port", db_dict["port"], + "--username", db_dict["username"], + "-w", + "-c", sql_query, + ] + (output, rc) = execCmd(cmdList=cmd, failOnError=fail_on_error, msg=err_msg, envDict=envDict) + return output, rc def isEngineUp(): ''' @@ -517,7 +529,7 @@ def isPostgresServerUp(): output, rc = runPostgresSuQuery( - query='"select 1;"', + query='select 1;', database='template1', failOnError=False, ) @@ -638,7 +650,7 @@ localHost(db_dict['host']) ): output, rc = runPostgresSuQuery( - query='"select 1;"', + query='select 1;', database=db_dict['dbname'], failOnError=False, ) @@ -662,7 +674,7 @@ localHost(db_dict['host']) ): output, rc = runPostgresSuQuery( - query='"select 1 from history_configuration;"', + query='select 1 from history_configuration;', database=db_dict['dbname'], failOnError=False, ) @@ -878,7 +890,9 @@ maskList=[], useShell=False, usePipeFiles=False, - envDict=None + envDict=None, + stdIn=None, + stdOut=None, ): """ Run external shell command with 'shell=false' @@ -896,10 +910,23 @@ stdOutFD = subprocess.PIPE stdInFD = subprocess.PIPE + stdInFile = None + if usePipeFiles: (stdErrFD, stdErrFile) = tempfile.mkstemp(dir="/tmp") (stdOutFD, stdOutFile) = tempfile.mkstemp(dir="/tmp") (stdInFD, stdInFile) = tempfile.mkstemp(dir="/tmp") + + if stdIn is not None: + logging.debug("input = %s"%(stdIn)) + if stdInFile is None: + (stdInFD, stdInFile) = tempfile.mkstemp(dir="/tmp") + os.write(stdInFD, stdIn) + os.lseek(stdInFD, os.SEEK_SET, 0) + + if stdOut is not None: + f = open(stdOut, 'w') + stdOutFD = f.fileno() # Copy os.environ and update with envDict if provided env = os.environ.copy() @@ -935,7 +962,7 @@ logging.debug("output = %s"%(out)) logging.debug("stderr = %s"%(err)) logging.debug("retcode = %s"%(proc.returncode)) - output = out + err + output = str(out) + str(err) if failOnError and proc.returncode != 0: raise Exception(msg) return ("".join(output.splitlines(True)), proc.returncode) @@ -952,6 +979,7 @@ def getDbSize(db_dict, PGPASS_FILE): + logging.debug("%s getDbSize started", db_dict['dbname']) # Returns db size in MB sql = "SELECT pg_database_size(\'%s\')" % db_dict['dbname'] @@ -981,6 +1009,7 @@ def performBackup(db_dict, backupPath, PGPASS_FILE): + logging.debug("%s performBackup started", db_dict['dbname']) # Check abvailable space dbSize = getDbSize(db_dict, PGPASS_FILE) backupPathFree = getAvailableSpace(backupPath) @@ -1053,19 +1082,33 @@ '--disable-dollar-quoting', '--disable-triggers', '--format=p', - '-U', db_dict['username'], - '-h', db_dict['host'], - '-p', db_dict['port'], '-Fc', - '-f', backup_file, - db_dict['dbname'], ] - execCmd( - cmdList=cmd, - failOnError=True, - msg='Error during DB backup.', - envDict={'ENGINE_PGPASS': PGPASS_FILE} - ) + if ( + db_dict['username'] == 'admin' and + db_dict['password'] == 'dummy' and + localHost(db_dict['host']) + ): + cmd.append(db_dict['dbname']) + output, rc = runPostgresSuCommand( + command=cmd, + failOnError=True, + output=backup_file, + ) + else: + cmd.extend( + '-U', db_dict['username'], + '-h', db_dict['host'], + '-p', db_dict['port'], + '-f', backup_file, + db_dict['dbname'], + ) + execCmd( + cmdList=cmd, + failOnError=True, + msg='Error during DB backup.', + envDict={'ENGINE_PGPASS': PGPASS_FILE} + ) logging.debug("%s DB Backup completed successfully", db_dict['dbname']) @@ -1117,7 +1160,7 @@ def createDB(database, owner): sql_query_set = [ ( - '"CREATE DATABASE {database} encoding \'UTF8\' LC_COLLATE \'en_US.UTF-8\' LC_CTYPE \'en_US.UTF-8\' template template0 owner {owner};"'.format( + 'CREATE DATABASE {database} encoding \'UTF8\' LC_COLLATE \'en_US.UTF-8\' LC_CTYPE \'en_US.UTF-8\' template template0 owner {owner};'.format( database=database, owner=owner, ) @@ -1128,6 +1171,7 @@ def createUser(user, password, option='', database='', validate=True): + logging.debug('Starting createUser user: {user}'.format(user=user)) if validate and not userValid(user): return @@ -1272,10 +1316,10 @@ def testLocalDb(): sql_query_set = [ ( - '"create database engine_test;"' + 'create database engine_test;' ), ( - '"drop database engine_test;"' + 'drop database engine_test;' ), ] for sql_query in sql_query_set: @@ -1288,15 +1332,23 @@ logging.debug('Updating DB owner') _RE_OWNER = re.compile(r'^([\w,.\(\)]+\s+)+OWNER TO (?P<owner>\w+).*$') out, rc = runPostgresSuCommand( - command=EXEC_PGDUMP, - database=db_dict['dbname'], + command=( + EXEC_PGDUMP, + db_dict['dbname'], + ), ) sql_query_set = [] + sql_query_set.append( + 'ALTER DATABASE {database} OWNER TO {owner};'.format( + database=db_dict['dbname'], + owner=db_dict['username'], + ) + ) for line in out.splitlines(): matcher = _RE_OWNER.match(line) if matcher is not None: sql_query_set.append( - '"{query}"'.format( + '{query}'.format( query=line.replace( 'OWNER TO {orig}'.format(orig=matcher.group('owner')), 'OWNER TO {new}'.format(new=db_dict['username']) @@ -1304,11 +1356,10 @@ ) ) - for sql_query in sql_query_set: - runPostgresSuQuery( - query=sql_query, - database=db_dict['dbname'], - ) + runPostgresSuQuery( + query=sql_query_set, + database=db_dict['dbname'], + ) def updatePgHba(database, user): content = [] @@ -1377,57 +1428,58 @@ def restorePgHba(): return configHbaIdent('ident', 'md5') -def runPostgresSuCommand(command, database=None, failOnError=True): - sql_command = [ - command, - ] - if database is not None: - sql_command.append(database) - +def runPostgresSuCommand(command, failOnError=True, output=None): + logging.debug("starting runPostgresSuCommand command: %s" % str(command)) cmd = [ EXEC_SU, '-l', 'postgres', '-c', '{command}'.format( - command=' '.join(sql_command), + command=' '.join(command), ) ] return execCmd( cmdList=cmd, - failOnError=failOnError + failOnError=failOnError, + stdOut=output, ) def runPostgresSuQuery(query, database=None, failOnError=True): - sql_command = [ + logging.debug("starting runPostgresSuQuery database: %s query: %s" % + (database, + query)) + command = [ EXEC_PSQL, '-U', 'postgres', ] if database is not None: - sql_command.extend( + command.extend( ( - '-d', database + '-d', database, + '-tA', ) ) - sql_command.extend( - ( - '-tAc', query, - ) - ) + stdIn = None + if isinstance(query, list) or isinstance(query, tuple): + stdIn = '\n'.join(query) + else: + stdIn = query cmd = [ EXEC_SU, '-l', 'postgres', '-c', '{command}'.format( - command=' '.join(sql_command), + command=' '.join(command), ) ] return execCmd( cmdList=cmd, - failOnError=failOnError + failOnError=failOnError, + stdIn=stdIn, ) @@ -1473,7 +1525,7 @@ print 'Error: user name can only include alphanumeric characters' return False - sql_query = '"select 1 from pg_roles where rolname=\'{user}\';"'.format( + sql_query = 'select 1 from pg_roles where rolname=\'{user}\';'.format( user=user ) diff --git a/packaging/legacy-setup/decorators.py b/packaging/legacy-setup/decorators.py old mode 100644 new mode 100755 diff --git a/packaging/legacy-setup/ovirt-engine-dwh-setup.py b/packaging/legacy-setup/ovirt-engine-dwh-setup.py index 9c8850d..1abd157 100755 --- a/packaging/legacy-setup/ovirt-engine-dwh-setup.py +++ b/packaging/legacy-setup/ovirt-engine-dwh-setup.py @@ -359,7 +359,7 @@ file_handler.close() def userExists(user): - sql_query = '"select 1 from pg_roles where rolname=\'{user}\';"'.format( + sql_query = 'select 1 from pg_roles where rolname=\'{user}\';'.format( user=user ) @@ -501,7 +501,6 @@ if not dbExists: print 'Could not connect to remote database - please try again.\n' else: - # Why do we allow host/port options? If they are different from engine db, we won't be able to connect to it db_dict['username'] = options['REMOTE_DB_USER'] db_dict['password'] = options['REMOTE_DB_PASSWORD'] @@ -526,19 +525,9 @@ if dbExists: try: - if utils.localHost(db_dict['host']) and not owned: - if not userExists(db_dict['username']): - utils.createUser( - user=db_dict['username'], - password=db_dict['password'], - option='createdb', - validate=False, - ) - utils.updateDbOwner(db_dict) - if options['BACKUP_DB'] is None: doBackup = utils.performBackup( - db_dict, + working_db_dict, DB_BACKUPS_DIR, PGPASS_TEMP ) @@ -554,7 +543,7 @@ if doBackup: utils.backupDB( backupFile, - db_dict, + working_db_dict, PGPASS_TEMP, ) print DB_FILE.format( @@ -580,6 +569,16 @@ utils.startEtl() sys.exit(0) + if utils.localHost(db_dict['host']) and not owned: + if not userExists(db_dict['username']): + utils.createUser( + user=db_dict['username'], + password=db_dict['password'], + option='createdb', + validate=False, + ) + utils.updateDbOwner(db_dict) + if hasData: # Backup went ok, so upgrade upgradeDB(db_dict) -- To view, visit http://gerrit.ovirt.org/22797 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I1b85a6aee99e94856b233f1c2285fd69e31e053f Gerrit-PatchSet: 1 Gerrit-Project: ovirt-dwh Gerrit-Branch: master Gerrit-Owner: Yedidyah Bar David <[email protected]> _______________________________________________ Engine-patches mailing list [email protected] http://lists.ovirt.org/mailman/listinfo/engine-patches
