Yedidyah Bar David has uploaded a new change for review. Change subject: packaging: on upgrade first backup ......................................................................
packaging: on upgrade first backup Backup before doing any changes to the database Dump only schema for changing ownership Run all ownership changes in a single psql command Bug-Url: https://bugzilla.redhat.com/1046679 Bug-Url: https://bugzilla.redhat.com/1046676 Change-Id: I1b85a6aee99e94856b233f1c2285fd69e31e053f Signed-off-by: Yedidyah Bar David <[email protected]> --- M packaging/common_utils.py M packaging/ovirt-engine-dwh-setup.py 2 files changed, 155 insertions(+), 81 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-dwh refs/changes/06/22906/1 diff --git a/packaging/common_utils.py b/packaging/common_utils.py index 2dd0bc1..db4c201 100755 --- a/packaging/common_utils.py +++ b/packaging/common_utils.py @@ -444,19 +444,31 @@ db_dict['dbname'], db_dict["username"], sql_query)) - cmd = [ - EXEC_PSQL, - "--pset=tuples_only=on", - "--set", - "ON_ERROR_STOP=1", - "--dbname", db_dict['dbname'], - "--host", db_dict["host"], - "--port", db_dict["port"], - "--username", db_dict["username"], - "-w", - "-c", sql_query, - ] - return execCmd(cmdList=cmd, failOnError=fail_on_error, msg=err_msg, envDict=envDict) + if ( + db_dict['username'] == 'admin' and + db_dict['password'] == 'dummy' and + localHost(db_dict['host']) + ): + output, rc = runPostgresSuQuery( + query=[sql_query], + database=db_dict['dbname'], + failOnError=fail_on_error, + ) + else: + cmd = [ + EXEC_PSQL, + "--pset=tuples_only=on", + "--set", + "ON_ERROR_STOP=1", + "--dbname", db_dict['dbname'], + "--host", db_dict["host"], + "--port", db_dict["port"], + "--username", db_dict["username"], + "-w", + "-c", sql_query, + ] + output, rc = execCmd(cmdList=cmd, failOnError=fail_on_error, msg=err_msg, envDict=envDict) + return output, rc def isEngineUp(): ''' @@ -878,7 +890,9 @@ maskList=[], useShell=False, usePipeFiles=False, - envDict=None + envDict=None, + stdIn=None, + stdOut=None, ): """ Run external shell command with 'shell=false' @@ -896,10 +910,23 @@ stdOutFD = subprocess.PIPE stdInFD = subprocess.PIPE + stdInFile = None + if usePipeFiles: (stdErrFD, stdErrFile) = tempfile.mkstemp(dir="/tmp") (stdOutFD, stdOutFile) = tempfile.mkstemp(dir="/tmp") (stdInFD, stdInFile) = tempfile.mkstemp(dir="/tmp") + + if stdIn is not None: + logging.debug("input = %s"%(stdIn)) + if stdInFile is None: + (stdInFD, stdInFile) = tempfile.mkstemp(dir="/tmp") + os.write(stdInFD, stdIn) + os.lseek(stdInFD, os.SEEK_SET, 0) + + if stdOut is not None: + f = open(stdOut, 'w') + stdOutFD = f.fileno() # Copy os.environ and update with envDict if provided env = os.environ.copy() @@ -935,7 +962,7 @@ logging.debug("output = %s"%(out)) logging.debug("stderr = %s"%(err)) logging.debug("retcode = %s"%(proc.returncode)) - output = out + err + output = str(out) + str(err) if failOnError and proc.returncode != 0: raise Exception(msg) return ("".join(output.splitlines(True)), proc.returncode) @@ -1041,28 +1068,51 @@ logging.debug("%s DB Backup started", db_dict['dbname']) # Run backup - cmd = [ - EXEC_PGDUMP, - '-C', - '-E', - 'UTF8', - '-w', - '--disable-dollar-quoting', - '--disable-triggers', - '--format=p', - '-U', db_dict['username'], - '-h', db_dict['host'], - '-p', db_dict['port'], - '-Fc', - '-f', backup_file, - db_dict['dbname'], - ] - execCmd( - cmdList=cmd, - failOnError=True, - msg='Error during DB backup.', - envDict={'ENGINE_PGPASS': PGPASS_FILE} - ) + if ( + db_dict['username'] == 'admin' and + db_dict['password'] == 'dummy' and + localHost(db_dict['host']) + ): + cmd = [ + EXEC_PGDUMP, + '-C', + '-E', + 'UTF8', + '-w', + '--disable-dollar-quoting', + '--disable-triggers', + '--format=p', + '-Fc', + db_dict['dbname'], + ] + output, rc = runPostgresSuCommand( + command=cmd, + failOnError=True, + output=backup_file, + ) + else: + cmd = [ + EXEC_PGDUMP, + '-C', + '-E', + 'UTF8', + '-w', + '--disable-dollar-quoting', + '--disable-triggers', + '--format=p', + '-U', db_dict['username'], + '-h', db_dict['host'], + '-p', db_dict['port'], + '-Fc', + '-f', backup_file, + db_dict['dbname'], + ] + execCmd( + cmdList=cmd, + failOnError=True, + msg='Error during DB backup.', + envDict={'ENGINE_PGPASS': PGPASS_FILE} + ) logging.debug("%s DB Backup completed successfully", db_dict['dbname']) @@ -1285,15 +1335,24 @@ logging.debug('Updating DB owner') _RE_OWNER = re.compile(r'^([\w,.\(\)]+\s+)+OWNER TO (?P<owner>\w+).*$') out, rc = runPostgresSuCommand( - command=EXEC_PGDUMP, - database=db_dict['dbname'], + command=( + EXEC_PGDUMP, + '-s', + db_dict['dbname'], + ), ) sql_query_set = [] + sql_query_set.append( + 'ALTER DATABASE {database} OWNER TO {owner};'.format( + database=db_dict['dbname'], + owner=db_dict['username'], + ) + ) for line in out.splitlines(): matcher = _RE_OWNER.match(line) if matcher is not None: sql_query_set.append( - '"{query}"'.format( + '{query}'.format( query=line.replace( 'OWNER TO {orig}'.format(orig=matcher.group('owner')), 'OWNER TO {new}'.format(new=db_dict['username']) @@ -1301,11 +1360,10 @@ ) ) - for sql_query in sql_query_set: - runPostgresSuQuery( - query=sql_query, - database=db_dict['dbname'], - ) + runPostgresSuQuery( + query=sql_query_set, + database=db_dict['dbname'], + ) def updatePgHba(database, user): content = [] @@ -1374,61 +1432,69 @@ def restorePgHba(): return configHbaIdent('ident', 'md5') -def runPostgresSuCommand(command, database=None, failOnError=True): - sql_command = [ - command, - ] - if database is not None: - sql_command.append(database) - +def runPostgresSuCommand(command, failOnError=True, output=None): cmd = [ EXEC_SU, '-l', 'postgres', '-c', '{command}'.format( - command=' '.join(sql_command), + command=' '.join(command), ) ] return execCmd( cmdList=cmd, - failOnError=failOnError + failOnError=failOnError, + stdOut=output, ) def runPostgresSuQuery(query, database=None, failOnError=True): - sql_command = [ + logging.debug("starting runPostgresSuQuery database: %s query: %s" % + (database, + query)) + command = [ EXEC_PSQL, '-U', 'postgres', ] if database is not None: - sql_command.extend( + command.extend( ( '-d', database ) ) - sql_command.extend( + command.extend( ( - '-tAc', query, + '-tA', ) ) + stdIn = None + if isinstance(query, list) or isinstance(query, tuple): + stdIn = '\n'.join(query) + else: + command.extend( + ( + '-c', query, + ) + ) cmd = [ EXEC_SU, '-l', 'postgres', '-c', '{command}'.format( - command=' '.join(sql_command), + command=' '.join(command), ) ] return execCmd( cmdList=cmd, - failOnError=failOnError + failOnError=failOnError, + stdIn=stdIn, ) -def saveConfig(configFile, username, password, dbname, readonly): +def saveConfig(configFile, username, password, dbname, readonly, uid, gid, perms): with open(configFile, 'w') as fdwh: content = ( 'DWH_DB_USER={user}\n' @@ -1446,6 +1512,8 @@ readonly=readonly, ) fdwh.write(content) + os.chown(configFile, uid, gid) + os.chmod(configFile, perms) def userValid(user): if user in ( diff --git a/packaging/ovirt-engine-dwh-setup.py b/packaging/ovirt-engine-dwh-setup.py index 8567b95..6a76c4a 100755 --- a/packaging/ovirt-engine-dwh-setup.py +++ b/packaging/ovirt-engine-dwh-setup.py @@ -315,6 +315,7 @@ if working_db_dict is not None: logging.debug("getDBStatus working username %s" % working_db_dict['username']) + logging.debug("getDBStatus returns: exists %s owned %s hasData %s" % (exists, owned, hasData)) return exists, owned, hasData, working_db_dict @@ -341,7 +342,8 @@ file_handler.editParam("ovirtEngineHistoryDbJdbcConnection", "jdbc\:postgresql\://%s\:%s/%s?stringtype\=unspecified" % (db_dict["host"], db_dict["port"], db_dict['dbname'])) file_handler.close() - os.chown(FILE_DB_CONN, OVIRT_UID, OVIRT_GID) + os.chown(FILE_DB_CONN, 0, OVIRT_GID) + os.chmod(FILE_DB_CONN, 0o640) def isVersionSupported(rawMinimalVersion, rawCurrentVersion): """ @@ -370,7 +372,8 @@ file_handler.open() file_handler.editParam("etlVersion", "%s.%s" % (currentVersion, currentMinorVersion)) file_handler.close() - os.chown(FILE_DB_CONN, OVIRT_UID, OVIRT_GID) + os.chown(FILE_DB_CONN, 0, OVIRT_GID) + os.chmod(FILE_DB_CONN, 0o640) def userExists(user): sql_query = '"select 1 from pg_roles where rolname=\'{user}\';"'.format( @@ -504,12 +507,16 @@ password=db_dict['password'], dbname=db_dict['dbname'], readonly=db_dict['readonly'], + uid=0, + gid=OVIRT_GID, + perms=0o640, ) dbExists, owned, hasData, working_db_dict = getDBStatus(db_dict, PGPASS_TEMP) - if not utils.localHost(db_dict["host"]) and not dbExists: + if not utils.localHost(db_dict["host"]) and not hasData: print 'Remote installation is selected.\n' + dbExists, tmpowned, tmphasData = utils.dbExists(db_dict, PGPASS_TEMP) if options['REMOTE_DB_USER'] is None: while not dbExists: ( @@ -521,7 +528,7 @@ if os.path.exists(PGPASS_TEMP): os.remove(PGPASS_TEMP) PGPASS_TEMP = utils.createTempPgpass(db_dict) - dbExists, owned, hasData, working_db_dict = getDBStatus(db_dict, PGPASS_TEMP) + dbExists, tmpowned, tmphasData = utils.dbExists(db_dict, PGPASS_TEMP) if not dbExists: print 'Could not connect to remote database - please try again.\n' else: @@ -549,23 +556,9 @@ if dbExists: try: - if utils.localHost(db_dict['host']): - if not owned: - if not userExists(db_dict['username']): - utils.createUser( - user=db_dict['username'], - password=db_dict['password'], - option='createdb', - validate=False, - ) - utils.updateDbOwner(db_dict) - elif hasData: - db_dict['username'] = working_db_dict['username'] - db_dict['password'] = working_db_dict['password'] - if options['BACKUP_DB'] is None: doBackup = utils.performBackup( - db_dict, + working_db_dict, DB_BACKUPS_DIR, PGPASS_TEMP ) @@ -581,7 +574,7 @@ if doBackup: utils.backupDB( backupFile, - db_dict, + working_db_dict, PGPASS_TEMP, ) print DB_FILE.format( @@ -606,6 +599,16 @@ time.sleep(20) utils.startEtl() sys.exit(0) + + if utils.localHost(db_dict['host']) and not owned: + if not userExists(db_dict['username']): + utils.createUser( + user=db_dict['username'], + password=db_dict['password'], + option='createdb', + validate=False, + ) + utils.updateDbOwner(db_dict) if hasData: # Backup went ok, so upgrade @@ -668,6 +671,9 @@ password=db_dict['password'], dbname=db_dict['dbname'], readonly=db_dict['readonly'], + uid=0, + gid=OVIRT_GID, + perms=0o640, ) else: -- To view, visit http://gerrit.ovirt.org/22906 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I1b85a6aee99e94856b233f1c2285fd69e31e053f Gerrit-PatchSet: 1 Gerrit-Project: ovirt-dwh Gerrit-Branch: ovirt-3.3 Gerrit-Owner: Yedidyah Bar David <[email protected]> _______________________________________________ Engine-patches mailing list [email protected] http://lists.ovirt.org/mailman/listinfo/engine-patches
