Alon Bar-Lev has uploaded a new change for review. Change subject: db: cleanup: cleanup file handling ......................................................................
db: cleanup: cleanup file handling 1. use proper finds to locate files. 2. drop chdir during script execution Change-Id: I54337aa6760ff488a1a72fe18bf2563c132e3fb6 Signed-off-by: Alon Bar-Lev <[email protected]> --- M packaging/dbscripts/cleandb.sh M packaging/dbscripts/create_schema.sh M packaging/dbscripts/dbfunc-common.sh M packaging/dbscripts/dbfunc-custom.sh M packaging/dbscripts/upgrade.sh 5 files changed, 72 insertions(+), 80 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/22/24922/1 diff --git a/packaging/dbscripts/cleandb.sh b/packaging/dbscripts/cleandb.sh index 670e59c..7b0833b 100755 --- a/packaging/dbscripts/cleandb.sh +++ b/packaging/dbscripts/cleandb.sh @@ -4,9 +4,7 @@ # Cleans DB by dropping all DB objects ################################################################################ -#include db general functions -cd "$(dirname "$0")" -. ./dbfunc-custom.sh +. "$(dirname "$0")/dbfunc-custom.sh" cleanup() { dbfunc_cleanup diff --git a/packaging/dbscripts/create_schema.sh b/packaging/dbscripts/create_schema.sh index 8d3082b..0e1096d 100755 --- a/packaging/dbscripts/create_schema.sh +++ b/packaging/dbscripts/create_schema.sh @@ -1,7 +1,6 @@ #!/bin/sh -cd "$(dirname "$0")" -. ./dbfunc-custom.sh +. "$(dirname "$0")/dbfunc-custom.sh" cleanup() { dbfunc_cleanup @@ -47,13 +46,13 @@ dbfunc_psql_statement_die "ALTER DATABASE \"${DBFUNC_DB_DATABASE}\" SET client_min_messages=ERROR;" > /dev/null echo "Creating tables..." -dbfunc_psql_die --file="create_tables.sql" > /dev/null +dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/create_tables.sql" > /dev/null echo "Creating functions..." -dbfunc_psql_die --file="create_functions.sql" > /dev/null +dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/create_functions.sql" > /dev/null echo "Creating common functions..." -dbfunc_psql_die --file="common_sp.sql" > /dev/null +dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/common_sp.sql" > /dev/null #inserting initial data dbfunc_common_hook_init_insert_data diff --git a/packaging/dbscripts/dbfunc-common.sh b/packaging/dbscripts/dbfunc-common.sh index decf85f..ecb7de4 100644 --- a/packaging/dbscripts/dbfunc-common.sh +++ b/packaging/dbscripts/dbfunc-common.sh @@ -1,5 +1,7 @@ -. ./dbfunc-base.sh +. "$(dirname "$0")/dbfunc-base.sh" + +DBFUNC_COMMON_DBSCRIPTS_DIR="$(dirname "$0")" dbfunc_common_hook_init_insert_data() { return 0 @@ -34,7 +36,7 @@ # common stored procedures are executed first (for new added functions to be valid) local file="${DBFUNC_TMPDIR}/cleandb.sql" - dbfunc_psql_die --file="common_sp.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/common_sp.sql" > /dev/null { dbfunc_psql_statement_die "select * from generate_drop_all_seq_syntax();" dbfunc_psql_statement_die "select * from generate_drop_all_tables_syntax();" @@ -50,9 +52,9 @@ } dbfunc_common_upgrade() { - local res="$(find upgrade/ -name '*.sql' -or -name '*.sh' | wc -l)" + local files="$(_dbfunc_common_get_files "upgrade" 1)" local CMD - if [ "${res}" -gt 0 ]; then + if [ -n "${files}" ]; then local state="FAILED" local comment="" local updated=0 @@ -67,13 +69,11 @@ local current="$(_dbfunc_common_get_current_version)" # we should remove leading blank (from select result) and zero in order not to treat number as octal local last="$(expr substr "${current}" 3 7)" - local files="$(_dbfunc_common_get_files "upgrade" 1)" local file - for file in $(ls ${files} | sort); do - local before="$(_dbfunc_common_get_db_time)" - local checksum="$(md5sum "${file}" | cut -d " " -f1)" - # upgrade/dd_dd_dddd* => dddddddd - local ver="$(echo "${file}" | sed -e 's#upgrade/\(..........\).*#\1#' -e 's/_//g')" + echo "${files}" | while read file; do + before="$(_dbfunc_common_get_db_time)" + checksum="$(md5sum "${file}" | cut -d " " -f1)" + ver="$(_dbfunc_common_get_file_version "${file}")" if [ "${ver}" -gt "${current}" ] ; then # we should remove leading zero in order not to treat number as octal local xver="$(expr substr "${ver}" 2 7)" @@ -86,9 +86,8 @@ if [ "${xverMajor}" -eq "${lastMajor}" ]; then if [ $((${xver} - ${last})) -gt 10 ]; then _dbfunc_common_set_last_version - echo "Illegal script version number ${ver},version should be in max 10 gap from last installed version: 0${last}" - echo "Please fix numbering to interval 0$(( ${last} + 1)) to 0$(( ${last} + 10)) and run the upgrade script." - exit 1 + die "Illegal script version number ${ver},version should be in max 10 gap from last installed version: 0${last} +Please fix numbering to interval 0$(( ${last} + 1)) to 0$(( ${last} + 10)) and run the upgrade script." fi fi # check if script was already installed with other version name. @@ -134,7 +133,7 @@ ) values ( trim('${ver}'), - '${file}', + '$(echo "${file}" | sed "s#^${DBFUNC_COMMON_DBSCRIPTS_DIR}/##")', '${checksum}', '${DBFUNC_DB_USER}', cast(trim('${before}') as timestamp), @@ -145,7 +144,7 @@ ); " > /dev/null fi - done + done || exit $? _dbfunc_common_set_last_version # restore views & SPs if dropped @@ -179,7 +178,7 @@ local file="${DBFUNC_TMPDIR}/drop_views.sql" # common stored procedures are executed first (for new added functions to be valid) - dbfunc_psql_die --file="common_sp.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/common_sp.sql" > /dev/null dbfunc_psql_statement_die "select * from generate_drop_all_views_syntax();" > "${file}" dbfunc_psql_die --file="${file}" > /dev/null } @@ -189,23 +188,23 @@ # common stored procedures are executed first (for new added functions to be valid) local file="${DBFUNC_TMPDIR}/drop_sps.sql" - dbfunc_psql_die --file="common_sp.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/common_sp.sql" > /dev/null dbfunc_psql_statement_die "select * from generate_drop_all_functions_syntax();" > "${file}" dbfunc_psql_die --file="${file}" > /dev/null # recreate generic functions - dbfunc_psql_die --file="create_functions.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/create_functions.sql" > /dev/null } #refreshes sps dbfunc_common_sps_refresh() { echo "Creating stored procedures..." - local sql - for sql in $(ls *sp.sql | sort); do - echo "Creating stored procedures from ${sql}..." - dbfunc_psql_die --file="${sql}" > /dev/null - done - dbfunc_psql_die --file="common_sp.sql" > /dev/null + local file + find "${DBFUNC_COMMON_DBSCRIPTS_DIR}" -name '*sp.sql' | sort | while read file; do + echo "Creating stored procedures from ${file}..." + dbfunc_psql_die --file="${file}" > /dev/null + done || exit $? + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/common_sp.sql" > /dev/null } #unlocks the given VM/Template and its disks or a given disk @@ -345,7 +344,7 @@ _dbfunc_common_install_common_func() { # common stored procedures are executed first (for new added functions to be valid) - dbfunc_psql_die --file="common_sp.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/common_sp.sql" > /dev/null } _dbfunc_common_run_pre_upgrade() { @@ -356,7 +355,7 @@ #update sequence numers dbfunc_common_hook_sequence_numbers_update #run pre upgrade scripts - _dbfunc_common_psql_statements_in_dir 'pre_upgrade' 'pre-upgrade' + _dbfunc_common_psql_statements_in_dir 'pre_upgrade' dbfunc_common_hook_materialized_views_install #drop materialized views to support views changesin upgrade #Materialized views are restored in the post_upgrade step @@ -369,9 +368,9 @@ dbfunc_common_hook_views_refresh dbfunc_common_sps_refresh #Running post-upgrade scripts - _dbfunc_common_psql_statements_in_dir 'post_upgrade' 'post-upgrade' + _dbfunc_common_psql_statements_in_dir 'post_upgrade' #run custom materialized views if exists - custom_materialized_views_file="upgrade/post_upgrade/custom/create_materialized_views.sql" + custom_materialized_views_file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/upgrade/post_upgrade/custom/create_materialized_views.sql" if [ -f "${custom_materialized_views_file}" ]; then echo "running custom materialized views from '${custom_materialized_views_file}'..." if ! dbfunc_psql --file="${custom_materialized_views_file}"; then @@ -383,16 +382,18 @@ dbfunc_common_hook_materialized_viewsrefresh_ } +_dbfunc_common_get_file_version() { + local file="$1" + basename "${file}" | sed -e 's#\(..........\).*#\1#' -e 's/_//g' +} + # Runs all the SQL scripts in directory upgrade/$1/ -# The second argument is the label to use while notifying -# the user about the running of the script _dbfunc_common_psql_statements_in_dir() { - if [ -d "upgrade/$1" ]; then - files="$(_dbfunc_common_get_files "upgrade/${1}" 1)" - for execFile in $(ls ${files} | sort); do - _dbfunc_common_run_file "${execFile}" - done - fi + local dir="$1" + local file + _dbfunc_common_get_files "upgrade/${dir}" 1 | while read file; do + _dbfunc_common_run_file "${file}" + done || exit $? } _dbfunc_common_run_required_scripts() { @@ -403,29 +404,26 @@ while read line; do expr="$(echo "${line}" | cut -d " " -f1 | grep "\-\-#source")" [ -z "${expr}" ] && break - sql=$(echo $line | cut -d " " -f2) - valid=$(echo $sql | grep "_sp.sql") - if [ -z "${valid}" ]; then - echo "invalid source file $sql in $file , source files must end with '_sp.sql'" - exit 1 - fi + sql="$(echo $line | cut -d " " -f2)" + echo "${sql}" | grep -q "_sp.sql" || \ + die "invalid source file $sql in $file , source files must end with '_sp.sql'" echo "Running helper functions from '${sql}' for '${file}'" - dbfunc_psql_die --file="${sql}" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/${sql}" > /dev/null done < "${script}" } _dbfunc_common_run_file() { - local execFile="${1}" - if [ -x "${execFile}" ]; then - echo "Running upgrade shell script '${execFile}'..." + local file="${1}" + if [ -x "${file}" ]; then + echo "Running upgrade shell script '${file}'..." DBFUNC_DB_HOST="${DBFUNC_DB_HOST}" \ DBFUNC_DB_PORT="${DBFUNC_DB_PORT}" \ DBFUNC_DB_USER="${DBFUNC_DB_USER}" \ DBFUNC_DB_DATABASE="${DBFUNC_DB_DATABASE}" \ - "./${execFile}" + "${file}" else - echo "Running upgrade sql script '${execFile}'..." - dbfunc_psql_die --file="${execFile}" > /dev/null + echo "Running upgrade sql script '${file}'..." + dbfunc_psql_die --file="${file}" > /dev/null fi } @@ -472,8 +470,12 @@ # gets a directory and required depth and return all sql & sh files _dbfunc_common_get_files() { - find ${1} -maxdepth ${2} -name '*.sql' -print - find ${1} -maxdepth ${2} -name '*.sh' -print + local dir="$1" + local maxdepth="$2" + find "${DBFUNC_COMMON_DBSCRIPTS_DIR}/${dir}" \ + -maxdepth "${maxdepth}" \ + -name '*.sql' -or -name '*.sh' | \ + sort } _dbfunc_common_is_view_or_sp_changed() { @@ -495,17 +497,12 @@ } _dbfunc_common_validate_version_uniqueness() { - local prev="" - local files="$(_dbfunc_common_get_files "upgrade" 1)" local file - for file in $(ls ${files} | sort) ; do - local ver="$(echo "${file}" | sed -e 's#upgrade/\(..........\).*#\1#' -e 's/_//g')" - if [ "${ver}" = "${prev}" ]; then - echo "Operation aborted, found duplicate version: ${ver}" - exit 1 - fi + _dbfunc_common_get_files "upgrade" 1 | while read file; do + ver="$(_dbfunc_common_get_file_version "${file}")" + [ "${ver}" != "${prev}" ] || die "Operation aborted, found duplicate version: ${ver}" prev="${ver}" - done + done || exit $? } #adds a record to audit_log in case of calling unlock_entity diff --git a/packaging/dbscripts/dbfunc-custom.sh b/packaging/dbscripts/dbfunc-custom.sh index b7cb061..e8018f6 100644 --- a/packaging/dbscripts/dbfunc-custom.sh +++ b/packaging/dbscripts/dbfunc-custom.sh @@ -1,5 +1,5 @@ -. ./dbfunc-common.sh +. "$(dirname "$0")/dbfunc-common.sh" DBFUNC_DB_USER="${DBFUNC_DB_USER:-engine}" DBFUNC_DB_DATABASE="${DBFUNC_DB_DATABASE:-engine}" @@ -8,29 +8,29 @@ dbfunc_common_hook_init_insert_data() { echo "Inserting data..." - dbfunc_psql_die --file="insert_data.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/insert_data.sql" > /dev/null echo "Inserting pre-defined roles..." - dbfunc_psql_die --file="insert_predefined_roles.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/insert_predefined_roles.sql" > /dev/null } dbfunc_common_hook_pre_upgrade() { if [ -n "${DBFUNC_CUSTOM_CLEAN_TASKS}" ]; then echo "Cleaning tasks metadata..." - dbfunc_psql_die --file="delete_async_tasks_and_compensation_data.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/delete_async_tasks_and_compensation_data.sql" > /dev/null fi } #refreshes views dbfunc_common_hook_views_refresh() { echo "Creating views..." - dbfunc_psql_die --file="create_views.sql" > /dev/null - dbfunc_psql_die --file="create_dwh_views.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/create_views.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/create_dwh_views.sql" > /dev/null } # Materilized views functions, override with empty implementation on DBs that not supporting that dbfunc_common_hook_materialized_views_install() { - dbfunc_psql_die --file="materialized_views_sp.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/materialized_views_sp.sql" > /dev/null } dbfunc_common_hook_materialized_views_drop() { @@ -44,5 +44,5 @@ } dbfunc_common_hook_sequence_numbers_update() { - dbfunc_psql_die --file="update_sequence_numbers.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/update_sequence_numbers.sql" > /dev/null } diff --git a/packaging/dbscripts/upgrade.sh b/packaging/dbscripts/upgrade.sh index 4b4ed56..a242f04 100755 --- a/packaging/dbscripts/upgrade.sh +++ b/packaging/dbscripts/upgrade.sh @@ -30,9 +30,7 @@ # problems) ################################################################################ -#include db general functions -cd "$(dirname "$0")" -. ./dbfunc-custom.sh +. "$(dirname "$0")/dbfunc-custom.sh" cleanup() { dbfunc_cleanup @@ -76,7 +74,7 @@ esac done -dbfunc_psql_die --file="upgrade/03_02_0000_set_version.sql" > /dev/null +dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/upgrade/03_02_0000_set_version.sql" > /dev/null if [ -n "${VERSION}" ]; then dbfunc_psql_statement_die " update schema_version -- To view, visit http://gerrit.ovirt.org/24922 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I54337aa6760ff488a1a72fe18bf2563c132e3fb6 Gerrit-PatchSet: 1 Gerrit-Project: ovirt-engine Gerrit-Branch: master Gerrit-Owner: Alon Bar-Lev <[email protected]> _______________________________________________ Engine-patches mailing list [email protected] http://lists.ovirt.org/mailman/listinfo/engine-patches
