Alon Bar-Lev has uploaded a new change for review. Change subject: db: cleanup: cleanup file handling ......................................................................
db: cleanup: cleanup file handling 1. use proper finds to locate files. 2. drop chdir during script execution Change-Id: I54337aa6760ff488a1a72fe18bf2563c132e3fb6 Signed-off-by: Alon Bar-Lev <[email protected]> --- M build/dbscripts-duplicate_upgrade_scripts.sh M packaging/dbscripts/cleandb.sh M packaging/dbscripts/create_schema.sh M packaging/dbscripts/dbfunc-common.sh M packaging/dbscripts/dbfunc-custom.sh M packaging/dbscripts/refreshStoredProcedures.sh M packaging/dbscripts/unlock_entity.sh M packaging/dbscripts/upgrade.sh M packaging/setup/dbutils/changedbowner.sh M packaging/setup/dbutils/encodingvalidator.sh M packaging/setup/dbutils/fkvalidator.sh M packaging/setup/dbutils/taskcleaner.sh 12 files changed, 67 insertions(+), 77 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/22/25222/1 diff --git a/build/dbscripts-duplicate_upgrade_scripts.sh b/build/dbscripts-duplicate_upgrade_scripts.sh index 908843d..f9f2b12 100755 --- a/build/dbscripts-duplicate_upgrade_scripts.sh +++ b/build/dbscripts-duplicate_upgrade_scripts.sh @@ -1,8 +1,6 @@ #!/bin/sh -cd "$(dirname "$0")/../packaging/dbscripts" - -OUT="$(ls upgrade | grep -P '\d{2}_\d{2}.\d{2,8}' -o | uniq -d)" +OUT="$(find "$(dirname "$0")/../packaging/dbscripts" | grep -P '\d{2}_\d{2}_\d{2,8}' -o | uniq -d)" if [ -n "${OUT}" ]; then echo "Found duplicate upgrade scripts with version $(echo ${OUT}), please resolve and retry" >&2 diff --git a/packaging/dbscripts/cleandb.sh b/packaging/dbscripts/cleandb.sh index e99f5d5..a361e7b 100755 --- a/packaging/dbscripts/cleandb.sh +++ b/packaging/dbscripts/cleandb.sh @@ -4,9 +4,8 @@ # Cleans DB by dropping all DB objects ################################################################################ -#include db general functions -cd "$(dirname "$0")" -. ./dbfunc-custom.sh +DBFUNC_COMMON_DBSCRIPTS_DIR="$(dirname "$0")" +. "${DBFUNC_COMMON_DBSCRIPTS_DIR}/dbfunc-custom.sh" cleanup() { dbfunc_cleanup diff --git a/packaging/dbscripts/create_schema.sh b/packaging/dbscripts/create_schema.sh index faabf5e..31c60d1 100755 --- a/packaging/dbscripts/create_schema.sh +++ b/packaging/dbscripts/create_schema.sh @@ -1,7 +1,7 @@ #!/bin/sh -cd "$(dirname "$0")" -. ./dbfunc-custom.sh +DBFUNC_COMMON_DBSCRIPTS_DIR="$(dirname "$0")" +. "${DBFUNC_COMMON_DBSCRIPTS_DIR}/dbfunc-custom.sh" cleanup() { dbfunc_cleanup @@ -47,13 +47,13 @@ dbfunc_psql_die --command="ALTER DATABASE \"${DBFUNC_DB_DATABASE}\" SET client_min_messages=ERROR;" > /dev/null echo "Creating tables..." -dbfunc_psql_die --file="create_tables.sql" > /dev/null +dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/create_tables.sql" > /dev/null echo "Creating functions..." -dbfunc_psql_die --file="create_functions.sql" > /dev/null +dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/create_functions.sql" > /dev/null echo "Creating common functions..." -dbfunc_psql_die --file="common_sp.sql" > /dev/null +dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/common_sp.sql" > /dev/null #inserting initial data dbfunc_common_init_insert_data diff --git a/packaging/dbscripts/dbfunc-common.sh b/packaging/dbscripts/dbfunc-common.sh index 76745fe..837e1ff 100644 --- a/packaging/dbscripts/dbfunc-common.sh +++ b/packaging/dbscripts/dbfunc-common.sh @@ -1,5 +1,7 @@ -. ./dbfunc-base.sh +DBFUNC_COMMON_DBSCRIPTS_DIR="${DBFUNC_COMMON_DBSCRIPTS_DIR:-$(dirname "$0")}" + +. "${DBFUNC_COMMON_DBSCRIPTS_DIR}/dbfunc-base.sh" #DBFUNC_COMMON_MD5DIR= @@ -33,7 +35,7 @@ #cleans db by dropping all objects dbfunc_common_schema_drop() { - dbfunc_psql_die --file="common_sp.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/common_sp.sql" > /dev/null local statement statement="$( dbfunc_psql_die --command="select * from generate_drop_all_seq_syntax();" @@ -51,11 +53,11 @@ dbfunc_common_upgrade() { - dbfunc_psql_die --file=upgrade/03_02_0000_set_version.sql > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/upgrade/03_02_0000_set_version.sql" > /dev/null - local res="$(find upgrade/ -name '*.sql' -or -name '*.sh' | wc -l)" + local files="$(_dbfunc_common_get_files "upgrade" 1)" local CMD - if [ "${res}" -gt 0 ]; then + if [ -n "${files}" ]; then local state="FAILED" local comment="" local updated=0 @@ -70,13 +72,11 @@ local current="$(_dbfunc_common_get_current_version)" # we should remove leading blank (from select result) and zero in order not to treat number as octal local last="$(expr substr "${current}" 3 7)" - local files="$(_dbfunc_common_get_files "upgrade" 1)" local file - for file in $(ls ${files} | sort); do - local before="$(_dbfunc_common_get_db_time)" - local checksum="$(md5sum "${file}" | cut -d " " -f1)" - # upgrade/dd_dd_dddd* => dddddddd - local ver="$(_dbfunc_common_get_file_version "${file}")" + echo "${files}" | while read file; do + before="$(_dbfunc_common_get_db_time)" + checksum="$(md5sum "${file}" | cut -d " " -f1)" + ver="$(_dbfunc_common_get_file_version "${file}")" if [ "${ver}" -gt "${current}" ] ; then # we should remove leading zero in order not to treat number as octal local xver="$(expr substr "${ver}" 2 7)" @@ -135,7 +135,7 @@ ) values ( trim('${ver}'), - '${file}', + '$(echo "${file}" | sed "s#^${DBFUNC_COMMON_DBSCRIPTS_DIR}/##")', '${checksum}', '${DBFUNC_DB_USER}', cast(trim('${before}') as timestamp), @@ -146,7 +146,7 @@ ); " > /dev/null fi - done + done || exit $? _dbfunc_common_set_last_version # restore views & SPs if dropped @@ -178,14 +178,14 @@ #drops views before upgrade or refresh operations dbfunc_common_views_drop() { # common stored procedures are executed first (for new added functions to be valid) - dbfunc_psql_die --file="common_sp.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/common_sp.sql" > /dev/null dbfunc_psql_die --command="select * from generate_drop_all_views_syntax();" | \ dbfunc_psql_die > /dev/null } #drops sps before upgrade or refresh operations dbfunc_common_sps_drop() { - dbfunc_psql_die --file="common_sp.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/common_sp.sql" > /dev/null local statement statement="$( dbfunc_psql_die --command="select * from generate_drop_all_functions_syntax();" @@ -193,18 +193,18 @@ dbfunc_psql_die --command="${statement}" > /dev/null # recreate generic functions - dbfunc_psql_die --file="create_functions.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/create_functions.sql" > /dev/null } #refreshes sps dbfunc_common_sps_refresh() { echo "Creating stored procedures..." - local sql - for sql in $(ls *sp.sql | sort); do - echo "Creating stored procedures from ${sql}..." - dbfunc_psql_die --file="${sql}" > /dev/null - done - dbfunc_psql_die --file="common_sp.sql" > /dev/null + local file + find "${DBFUNC_COMMON_DBSCRIPTS_DIR}" -name '*sp.sql' | sort | while read file; do + echo "Creating stored procedures from ${file}..." + dbfunc_psql_die --file="${file}" > /dev/null + done || exit $? + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/common_sp.sql" > /dev/null } #unlocks the given VM/Template and its disks or a given disk @@ -341,7 +341,7 @@ dbfunc_common_views_drop dbfunc_common_sps_drop # common stored procedures are executed first (for new added functions to be valid) - dbfunc_psql_die --file="common_sp.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/common_sp.sql" > /dev/null #update sequence numers dbfunc_common_hook_sequence_numbers_update #run pre upgrade scripts @@ -360,7 +360,7 @@ #Running post-upgrade scripts _dbfunc_common_psql_statements_in_dir 'post_upgrade' #run custom materialized views if exists - custom_materialized_views_file="upgrade/post_upgrade/custom/create_materialized_views.sql" + custom_materialized_views_file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/upgrade/post_upgrade/custom/create_materialized_views.sql" if [ -f "${custom_materialized_views_file}" ]; then echo "running custom materialized views from '${custom_materialized_views_file}'..." if ! dbfunc_psql --file="${custom_materialized_views_file}"; then @@ -375,12 +375,9 @@ # Runs all the SQL scripts in directory upgrade/$1/ _dbfunc_common_psql_statements_in_dir() { local dir="$1" - if [ -d "upgrade/${dir}" ]; then - files="$(_dbfunc_common_get_files "upgrade/${dir}" 1)" - for file in $(ls ${files} | sort); do - _dbfunc_common_run_file "${file}" - done - fi + _dbfunc_common_get_files "upgrade/${dir}" 1 | while read file; do + _dbfunc_common_run_file "${file}" + done || exit $? } _dbfunc_common_run_required_scripts() { @@ -395,7 +392,7 @@ echo "${sql}" | grep -q "_sp.sql" || \ die "invalid source file ${sql} in ${file}, source files must end with '_sp.sql'" echo "Running helper functions from '${sql}' for '${file}'" - dbfunc_psql_die --file="${sql}" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/${sql}" > /dev/null done < "${script}" } @@ -406,7 +403,7 @@ echo "Running upgrade shell script '${file}'..." ( eval "$(set | grep '^DBFUNC_' | sed 's/^\([^=]*\)=.*/export \1/')" - "./${file}" + "${file}" ) else echo "Running upgrade sql script '${file}'..." @@ -457,7 +454,7 @@ _dbfunc_common_get_files() { local dir="$1" local maxdepth="$2" - find "${dir}" \ + find "${DBFUNC_COMMON_DBSCRIPTS_DIR}/${dir}" \ -maxdepth "${maxdepth}" \ -name '*.sql' -or -name '*.sh' | \ sort @@ -467,7 +464,12 @@ local files="$(_dbfunc_common_get_files "upgrade" 3)" local md5sum_file="${DBFUNC_COMMON_MD5DIR}/.${DBFUNC_DB_DATABASE}.scripts.md5" local md5sum_tmp_file="${md5sum_file}.tmp" - md5sum ${files} create_*views.sql *_sp.sql > "${md5sum_tmp_file}" + + { + _dbfunc_common_get_files "upgrade" 3 + find "${DBFUNC_COMMON_DBSCRIPTS_DIR}" -name 'create_*views.sql' -or -name '*_sp.sql' + } | sort | uniq | xargs -d '\n' md5sum > "${md5sum_tmp_file}" + diff -s -q "${md5sum_file}" "${md5sum_tmp_file}" > /dev/null 2>&1 result=$? @@ -487,14 +489,12 @@ } _dbfunc_common_validate_version_uniqueness() { - local prev="" - local files="$(_dbfunc_common_get_files "upgrade" 1)" local file - for file in $(ls ${files} | sort) ; do - local ver="$(_dbfunc_common_get_file_version "${file}")" + _dbfunc_common_get_files "upgrade" 1 | while read file; do + ver="$(_dbfunc_common_get_file_version "${file}")" [ "${ver}" != "${prev}" ] || die "Operation aborted, found duplicate version: ${ver}" prev="${ver}" - done + done || exit $? } #adds a record to audit_log in case of calling unlock_entity diff --git a/packaging/dbscripts/dbfunc-custom.sh b/packaging/dbscripts/dbfunc-custom.sh index ddf50d5..3865d86 100644 --- a/packaging/dbscripts/dbfunc-custom.sh +++ b/packaging/dbscripts/dbfunc-custom.sh @@ -1,5 +1,5 @@ -. ./dbfunc-common.sh +. "${DBFUNC_COMMON_DBSCRIPTS_DIR}/dbfunc-common.sh" DBFUNC_DB_USER="${DBFUNC_DB_USER:-engine}" DBFUNC_DB_DATABASE="${DBFUNC_DB_DATABASE:-engine}" @@ -8,29 +8,29 @@ dbfunc_common_hook_init_insert_data() { echo "Inserting data..." - dbfunc_psql_die --file="insert_data.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/insert_data.sql" > /dev/null echo "Inserting pre-defined roles..." - dbfunc_psql_die --file="insert_predefined_roles.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/insert_predefined_roles.sql" > /dev/null } dbfunc_common_hook_pre_upgrade() { if [ -n "${DBFUNC_CUSTOM_CLEAN_TASKS}" ]; then echo "Cleaning tasks metadata..." - dbfunc_psql_die --file="delete_async_tasks_and_compensation_data.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/delete_async_tasks_and_compensation_data.sql" > /dev/null fi } #refreshes views dbfunc_common_hook_views_refresh() { echo "Creating views..." - dbfunc_psql_die --file="create_views.sql" > /dev/null - dbfunc_psql_die --file="create_dwh_views.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/create_views.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/create_dwh_views.sql" > /dev/null } # Materilized views functions, override with empty implementation on DBs that not supporting that dbfunc_common_hook_materialized_views_install() { - dbfunc_psql_die --file="materialized_views_sp.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/materialized_views_sp.sql" > /dev/null } dbfunc_common_hook_materialized_views_drop() { @@ -44,5 +44,5 @@ } dbfunc_common_hook_sequence_numbers_update() { - dbfunc_psql_die --file="update_sequence_numbers.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/update_sequence_numbers.sql" > /dev/null } diff --git a/packaging/dbscripts/refreshStoredProcedures.sh b/packaging/dbscripts/refreshStoredProcedures.sh index 0d0d8df..e0a8506 100755 --- a/packaging/dbscripts/refreshStoredProcedures.sh +++ b/packaging/dbscripts/refreshStoredProcedures.sh @@ -1,7 +1,7 @@ #!/bin/sh -cd "$(dirname "$0")" -. ./dbfunc-custom.sh +DBFUNC_COMMON_DBSCRIPTS_DIR="$(dirname "$0")" +. "${DBFUNC_COMMON_DBSCRIPTS_DIR}/dbfunc-custom.sh" cleanup() { dbfunc_cleanup diff --git a/packaging/dbscripts/unlock_entity.sh b/packaging/dbscripts/unlock_entity.sh index 39d5f07..61180f7 100755 --- a/packaging/dbscripts/unlock_entity.sh +++ b/packaging/dbscripts/unlock_entity.sh @@ -1,7 +1,7 @@ #!/bin/sh -cd "$(dirname "$0")" -. ./dbfunc-custom.sh +DBFUNC_COMMON_DBSCRIPTS_DIR="$(dirname "$0")" +. "${DBFUNC_COMMON_DBSCRIPTS_DIR}/dbfunc-custom.sh" cleanup() { dbfunc_cleanup diff --git a/packaging/dbscripts/upgrade.sh b/packaging/dbscripts/upgrade.sh index 5458b6a..9a768b9 100755 --- a/packaging/dbscripts/upgrade.sh +++ b/packaging/dbscripts/upgrade.sh @@ -30,9 +30,8 @@ # problems) ################################################################################ -#include db general functions -cd "$(dirname "$0")" -. ./dbfunc-custom.sh +DBFUNC_COMMON_DBSCRIPTS_DIR="$(dirname "$0")" +. "${DBFUNC_COMMON_DBSCRIPTS_DIR}/dbfunc-custom.sh" cleanup() { dbfunc_cleanup diff --git a/packaging/setup/dbutils/changedbowner.sh b/packaging/setup/dbutils/changedbowner.sh index 0ea6dee..987ceec 100755 --- a/packaging/setup/dbutils/changedbowner.sh +++ b/packaging/setup/dbutils/changedbowner.sh @@ -4,9 +4,7 @@ # database ################################################################ -#include db general functions -cd "$(dirname "$0")" -. ./dbfunc-base.sh +. "$(dirname "$0")/dbfunc-base.sh" cleanup() { dbfunc_cleanup diff --git a/packaging/setup/dbutils/encodingvalidator.sh b/packaging/setup/dbutils/encodingvalidator.sh index 54fe9f5..c2a2e7d 100755 --- a/packaging/setup/dbutils/encodingvalidator.sh +++ b/packaging/setup/dbutils/encodingvalidator.sh @@ -7,9 +7,7 @@ # It is highly recomended to backup the database before using this utility. ############################################################################################################### -#include db general functions -cd "$(dirname "${0}")" -. ./dbfunc-base.sh +. "$(dirname "${0}")/dbfunc-base.sh" cleanup() { dbfunc_cleanup diff --git a/packaging/setup/dbutils/fkvalidator.sh b/packaging/setup/dbutils/fkvalidator.sh index bd29dd5..e24c2e4 100755 --- a/packaging/setup/dbutils/fkvalidator.sh +++ b/packaging/setup/dbutils/fkvalidator.sh @@ -20,8 +20,7 @@ ############################################################################################################### #include db general functions -cd "$(dirname "$0")" -. ./dbfunc-base.sh +. "$(dirname "$0")/dbfunc-base.sh" cleanup() { dbfunc_cleanup @@ -113,6 +112,6 @@ fi # Install fkvalidator procedures -dbfunc_psql_die --file=./fkvalidator_sp.sql > /dev/null +dbfunc_psql_die --file="$(dirname "$0")/fkvalidator_sp.sql" > /dev/null # Execute validate_db_fks "${FIXIT}" "${DBFUNC_VERBOSE}" diff --git a/packaging/setup/dbutils/taskcleaner.sh b/packaging/setup/dbutils/taskcleaner.sh index 5a85e0c..2cf22c3 100755 --- a/packaging/setup/dbutils/taskcleaner.sh +++ b/packaging/setup/dbutils/taskcleaner.sh @@ -17,8 +17,7 @@ # should be cleaned as well. ############################################################################################################### -cd "$(dirname "$0")" -. ./dbfunc-base.sh +. "$(dirname "$0")/dbfunc-base.sh" cleanup() { dbfunc_cleanup @@ -235,6 +234,6 @@ fi # Install taskcleaner procedures -dbfunc_psql_die --file=./taskcleaner_sp.sql > /dev/null +dbfunc_psql_die --file="$(dirname "$0")/taskcleaner_sp.sql" > /dev/null # Execute dbfunc_psql_die --command="${CMD1}${CMD2}" -- To view, visit http://gerrit.ovirt.org/25222 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I54337aa6760ff488a1a72fe18bf2563c132e3fb6 Gerrit-PatchSet: 1 Gerrit-Project: ovirt-engine Gerrit-Branch: master Gerrit-Owner: Alon Bar-Lev <[email protected]> _______________________________________________ Engine-patches mailing list [email protected] http://lists.ovirt.org/mailman/listinfo/engine-patches
