Alex Lourie has uploaded a new change for review. Change subject: packaging: working with new DB creation methods ......................................................................
packaging: working with new DB creation methods Current implementation works with legacy DB creation. The current implementation adds working with DB created with otopi-based engine setup utility. Change-Id: I390d53b3768f08b2c62f31aeb32c1992f11b6999 Signed-off-by: Alex Lourie <[email protected]> --- M data-warehouse/historydbscripts_postgres/ovirt-engine-history-db-install.sh M packaging/common_utils.py M packaging/ovirt-engine-dwh-setup.py M packaging/ovirt-engine-dwh.spec.in 4 files changed, 219 insertions(+), 59 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-dwh refs/changes/40/16740/1 diff --git a/data-warehouse/historydbscripts_postgres/ovirt-engine-history-db-install.sh b/data-warehouse/historydbscripts_postgres/ovirt-engine-history-db-install.sh index ef322df..cc1b3dc 100755 --- a/data-warehouse/historydbscripts_postgres/ovirt-engine-history-db-install.sh +++ b/data-warehouse/historydbscripts_postgres/ovirt-engine-history-db-install.sh @@ -18,11 +18,8 @@ SCRIPT_NAME="ovirt-history-db-install" CUR_DATE=`date +"%Y_%m_%d_%H_%M_%S"` -#list of mandatory rpms - please add here any additional rpms required before configuring the postgresql server -REQUIRED_RPMS=(postgresql-server postgresql postgresql-libs postgresql-contrib uuid) - #postgresql data dir -ENGINE_PGPASS=/etc/ovirt-engine/.pgpass +ENGINE_PGPASS=${ENGINE_PGPASS:-/etc/ovirt-engine/.pgpass} PGDATA=/var/lib/pgsql/data #location of ovirt db scripts @@ -48,7 +45,7 @@ USER=`/usr/bin/whoami` usage() { - printf "Usage: ${ME} [-h] [-s SERVERNAME [-p PORT]] [-d DATABASE] [-u USERNAME] [-r 'remote'] -l LOGFILE\n" + printf "Usage: ${ME} [-h] [-s SERVERNAME [-p PORT]] [-d DATABASE] [-u USERNAME] [-r 'remote'] -l LOGFILE [-L LOGDIR]\n" printf "\n" printf "\t-s SERVERNAME - The database servername for the database (def. ${DB_HOST})\n" printf "\t-p PORT - The database port for the database (def. ${DB_PORT})\n" @@ -63,13 +60,14 @@ } #EXTERNAL ARGS -while getopts :s:p:d:u:w:l:r:h option; do +while getopts :s:p:d:u:w:l:L:r:h option; do case $option in s) DB_HOST=$OPTARG;; p) DB_PORT=$OPTARG;; d) DB_NAME=$OPTARG;; u) DB_ADMIN=$OPTARG;; l) LOG_FILE=$OPTARG;; + L) LOG_PATH=$OPTARG;; r) REMOTE_INSTALL=$OPTARG;; h) usage;; esac @@ -133,33 +131,6 @@ echo "#ovirt engine history db installer log file on $HOST" > $LOG_FILE } - -#TODO: check if postgresql patch is installed -verifyPostgresPkgAreInstalled() -{ - echo "[$SCRIPT_NAME] verifying required rpms are installed." >> $LOG_FILE - for rpm in "${REQUIRED_RPMS[@]}"; do - verifyPkgIsInstalled ${rpm} - done -} - -verifyPkgIsInstalled() -{ - RPM="$1" - rpm -q $RPM >> $LOG_FILE 2>&1 - _verifyRC $? "error, rpm $RPM is not installed" -} - -verifyPostgresService() -{ - echo "[$SCRIPT_NAME] verifying postgres binary exists." >> $LOG_FILE - - if [ ! -x $PSQL_BIN ] - then - echo "[$SCRIPT_NAME] postgres psql command cannot be executed from $PSQL_BIN" - exit 1 - fi -} initPgsqlDB() { @@ -316,8 +287,6 @@ # The following should only run during local installation if [[ $LOCAL_DB_SET -eq 1 ]] then - verifyPostgresPkgAreInstalled - verifyPostgresService initPgsqlDB turnPgsqlOnStartup startPgsqlService postgres postgres diff --git a/packaging/common_utils.py b/packaging/common_utils.py index f425547..c17d7c5 100755 --- a/packaging/common_utils.py +++ b/packaging/common_utils.py @@ -6,6 +6,8 @@ import csv import logging import os +import pwd +import grp import traceback import datetime import re @@ -14,6 +16,8 @@ import shutil from decorators import transactionDisplay import tempfile +import random +import string #text colors RED = "\033[0;31m" @@ -31,6 +35,8 @@ PGPASS_FILE_USER_LINE = "DB USER credentials" PGPASS_FILE_ADMIN_LINE = "DB ADMIN credentials" FILE_ENGINE_CONFIG_BIN="/usr/bin/engine-config" +FILE_DATABASE_CONFIG = "/etc/ovirt-engine/engine.conf.d/10-setup-database.conf" +FILE_PGHBA = '/var/lib/pgsql/data/pg_hba.conf' # ERRORS # TODO: Move all errors here and make them consistent @@ -261,7 +267,7 @@ return ret, rc -def execSqlCmd(db_dict, sql_query, fail_on_error=False, err_msg="Failed running sql query"): +def execSqlCmd(db_dict, sql_query, fail_on_error=False, err_msg="Failed running sql query", env={}): logging.debug("running sql query on host: %s, port: %s, db: %s, user: %s, query: \'%s\'." % (db_dict["host"], db_dict["port"], @@ -279,7 +285,7 @@ "--username", db_dict["username"], "-c", sql_query, ] - return execCmd(cmdList=cmd, failOnError=fail_on_error, msg=err_msg) + return execCmd(cmdList=cmd, failOnError=fail_on_error, msg=err_msg, envDict=env) def isEngineUp(): ''' @@ -349,11 +355,23 @@ if not isPostgresUp(): startPostgresService() +def stopPostgres(): + ''' + stops the postgresql service + ''' + if isPostgresUp(): + stopPostgresService() + @transactionDisplay("Starting PostgresSql") def startPostgresService(): logging.debug("starting postgresql") cmd = ["service", "postgresql", "start"] execCmd(cmdList=cmd, failOnError=True, msg="Failed while trying to start the postgresql service") + +def stopPostgresService(): + logging.debug("stopping postgresql") + cmd = ["service", "postgresql", "stop"] + execCmd(cmdList=cmd, failOnError=True, msg="Failed while trying to stop the postgresql service") def stopEtl(): """ @@ -600,9 +618,9 @@ # Copy os.environ and update with envDict if provided env = os.environ.copy() - if not "PGPASSFILE" in env.keys(): - env["PGPASSFILE"] = FILE_PG_PASS env.update(envDict or {}) + if not "ENGINE_PGPASS" in env.keys(): + env["ENGINE_PGPASS"] = FILE_PG_PASS # We use close_fds to close any file descriptors we have so it won't be copied to forked childs proc = subprocess.Popen( @@ -743,3 +761,102 @@ msg='Error during DB backup.', ) logging.debug("%s DB Backup completed successfully", db_dict['name']) + + +def getUsernameId(username): + return pwd.getpwnam(username)[2] + + +def getGroupId(groupName): + return grp.getgrnam(groupName)[2] + + +def createTempPgpass(db_dict): + + fd, pgpass = tempfile.mkstemp( + prefix='pgpass', + suffix='.tmp', + ) + os.close(fd) + os.chmod(pgpass, 0o600) + with open(pgpass, 'w') as f: + f.write( + ( + '# DB USER credentials.\n' + '{host}:{port}:{database}:{user}:{password}\n' + ).format( + host=db_dict['host'], + port=db_dict['port'], + database=db_dict['name'], + username=db_dict['username'], + password=db_dict['password'], + ), + ) + + return pgpass + + +def generatePassword(): + return '%s%s' % ( + ''.join([random.choice(string.digits) for i in xrange(4)]), + ''.join([random.choice(string.letters) for i in xrange(4)]), + ) + + +def createUser(user, password, option='createdb'): + sql_query = ( + '"CREATE ROLE {user} with ' + '{option} login encrypted password \"{password}\";"' + ) + sql_command = [ + EXEC_PSQL, + '-U', 'postgres', + '-c', + sql_query.format( + user=user, + option=option, + password=password, + ), + ] + cmd = [ + '/usr/bin/su', + '-l', + 'postgres', + '-c', + '{command}'.format( + command=' '.join(sql_command), + ) + ] + + execCmd( + cmdList=cmd, + failOnError=True + ) + + +def updatePgHba(database, user): + content = [] + with open(FILE_PGHBA, 'r') as pghba: + for line in pghba.readlines().split(): + if user in line: + return + + if 'engine engine 0.0.0.0/0' in line: + for address in ('0.0.0.0/0', '::0/0'): + content.append( + ( + '{host:7} ' + '{user:15} ' + '{database:15} ' + '{address:23} ' + '{auth}' + ).format( + host='host', + user=user, + database=database, + address=address, + auth='md5', + ) + ) + + content.append(line) diff --git a/packaging/ovirt-engine-dwh-setup.py b/packaging/ovirt-engine-dwh-setup.py index 518d69a..6f7aeb4 100755 --- a/packaging/ovirt-engine-dwh-setup.py +++ b/packaging/ovirt-engine-dwh-setup.py @@ -17,6 +17,7 @@ import time import traceback import glob +import shutil import argparse import common_utils as utils from decorators import transactionDisplay @@ -30,11 +31,14 @@ FILE_DB_CONN = "/etc/ovirt-engine/ovirt-engine-dwh/Default.properties" FILE_ENGINE_CONF_DEFAULTS = "/usr/share/ovirt-engine/conf/engine.conf.defaults" FILE_ENGINE_CONF = "/etc/ovirt-engine/engine.conf" +FILE_DATABASE_CONFIG = "/etc/ovirt-engine/engine.conf.d/10-setup-database.conf" +FILE_DATABASE_DWH_CONFIG = "/etc/ovirt-engine/engine.conf.d/10-setup-database-dwh.conf" DB_BACKUPS_DIR = "/var/lib/ovirt-engine/backups" DB_NAME = "ovirt_engine_history" -DB_USER_NAME = "postgres" +DB_USER = 'ovirt-engine-history' DB_PORT = "5432" DB_HOST = "localhost" +PGPASS_TEMP = '' # DB messages DB_FILE = ( @@ -54,7 +58,11 @@ def dbExists(db_dict): logging.debug("checking if %s db already exists" % db_dict["name"]) - (output, rc) = utils.execSqlCmd(db_dict, "select 1") + (output, rc) = utils.execSqlCmd( + db_dict=db_dict, + sql_query="select 1", + envDict={'ENGINE_PGPASS': PGPASS_TEMP}, + ) if (rc != 0): return False else: @@ -72,20 +80,31 @@ # Set ovirt-history-db-install.sh args - logfile if utils.localHost(db_dict["host"]): - install_type = "local" + utils.createUser(db_dict['username'], db_dict['password']) + utils.updatePgHba(db_dict['name'], db_dict['username']) + utils.stopPostgres() + utils.startPostgres() + install = 'local' else: - install_type = "remote" - cmd = [EXEC_CREATE_DB, - "-l", dbLogFilename, - "-u", db_dict["username"], - "-s", db_dict["host"], - "-p", db_dict["port"], - "-r", install_type, - ] + install = 'remote' + + cmd = [ + EXEC_CREATE_DB, + '-l', dbLogFilename, + '-u', db_dict['username'], + '-s', db_dict['host'], + '-p', db_dict['port'], + '-r', install, + ] # Create db using shell command - output, rc = utils.execCmd(cmd, None, True, ERR_DB_CREATE_FAILED) + output, rc = utils.execCmd( + cmdList=cmd, + failOnError=True, + msg=ERR_DB_CREATE_FAILED, + ) logging.debug('Successfully installed %s DB' % db_dict["name"]) + @transactionDisplay("Upgrade DB") def upgradeDB(db_dict): @@ -113,12 +132,42 @@ os.chdir(currDir) raise + def getDbDictFromOptions(): - db_dict = {"name" : DB_NAME, - "host" : utils.getDbHostName(), - "port" : utils.getDbPort(), - "username" : utils.getDbAdminUser(), - "password" : utils.getPassFromFile(utils.getDbAdminUser())} + if os.path.exists(FILE_DATABASE_CONFIG): + handler = utils.TextConfigFileHandler(FILE_DATABASE_CONFIG) + dhandler = utils.TextConfigFileHandler(FILE_DATABASE_DWH_CONFIG) + handler.open() + dhandler.open() + db_dict = { + 'name': ( + dhandler.getParam('DWH_DATABASE') or + DB_NAME + ), + 'host': handler.getParam('ENGINE_DB_HOST'), + 'port': handler.getParam('ENGINE_DB_PORT'), + 'username': ( + dhandler.getParam('DWH_USER') or + DB_USER + ), + 'password': ( + dhandler.getParam('DWH_PASSWORD') or + utils.generatePassword() + ), + 'engine_user': handler.getParam('ENGINE_DB_USER'), + 'engine_pass': handler.getParam('ENGINE_DB_PASSWORD'), + } + handler.close() + dhandler.close() + else: + db_dict = { + 'name': DB_NAME, + 'host': utils.getDbHostName(), + 'port': utils.getDbPort(), + 'username': utils.getDbAdminUser(), + 'password': utils.getPassFromFile(utils.getDbAdminUser()), + } + return db_dict @@ -133,9 +182,9 @@ file_handler = utils.TextConfigFileHandler(FILE_DB_CONN) file_handler.open() file_handler.editParam("ovirtEngineHistoryDbPassword", db_dict["password"]) - file_handler.editParam("ovirtEngineHistoryDbUser", db_dict["username"]) - file_handler.editParam("ovirtEngineDbPassword", db_dict["password"]) - file_handler.editParam("ovirtEngineDbUser", db_dict["username"]) + file_handler.editParam("ovirtEngineHistoryDbUser", DB_USER) + file_handler.editParam("ovirtEngineDbPassword", db_dict["engine_pass"]) + file_handler.editParam("ovirtEngineDbUser", db_dict["engine_user"]) file_handler.editParam("ovirtEngineDbJdbcConnection", "jdbc\:postgresql\://%s\:%s/engine?stringtype\=unspecified" % (db_dict["host"], db_dict["port"])) file_handler.editParam("ovirtEngineHistoryDbJdbcConnection", @@ -178,6 +227,8 @@ doBackup = None backupFile = None + global PGPASS_TEMP + parser = argparse.ArgumentParser(description='Installs or upgrades your oVirt Engine DWH') # Catch when calling ovirt-engine-dwh-setup --help args = parser.parse_args() @@ -206,6 +257,7 @@ setVersion() # Create/Upgrade DB + PGPASS_TEMP = utils.createTempPgpass(db_dict) if dbExists(db_dict): try: doBackup = utils.performBackup(db_dict, DB_BACKUPS_DIR) @@ -234,6 +286,25 @@ else: createDB(db_dict) + if os.path.exists(PGPASS_TEMP): + shutil.remove(PGPASS_TEMP) + + if not os.path.exists( + FILE_DATABASE_DWH_CONFIG + ): + with open(FILE_DATABASE_DWH_CONFIG, 'w') as fdwh: + fdwh.write( + ( + 'DWH_USER={user}\n' + 'DWH_PASSWORD={password}\n' + 'DWH_DATABASE={database}' + ).format( + user=db_dict['username'], + password=db_dict['password'], + database=db_dict['database'], + ) + ) + # Start Services utils.startEngine() # Sleep for 20 secs to allow health applet to start diff --git a/packaging/ovirt-engine-dwh.spec.in b/packaging/ovirt-engine-dwh.spec.in index 543425d..a4d8715 100644 --- a/packaging/ovirt-engine-dwh.spec.in +++ b/packaging/ovirt-engine-dwh.spec.in @@ -50,9 +50,12 @@ Requires: log4j Requires: dom4j >= 1.6.1 Requires: postgresql-jdbc +Requires: postgresql >= 8.4.7 +Requires: postgresql-libs >= 8.4.7 Requires: postgresql-server >= 8.4.7 Requires: postgresql-contrib >= 8.4.7 Requires: logrotate +Requires: uuid Requires: ovirt-engine %description -- To view, visit http://gerrit.ovirt.org/16740 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I390d53b3768f08b2c62f31aeb32c1992f11b6999 Gerrit-PatchSet: 1 Gerrit-Project: ovirt-dwh Gerrit-Branch: master Gerrit-Owner: Alex Lourie <[email protected]> _______________________________________________ Engine-patches mailing list [email protected] http://lists.ovirt.org/mailman/listinfo/engine-patches
