Alon Bar-Lev has uploaded a new change for review. Change subject: db: cleanup: cleanup file handling ......................................................................
db: cleanup: cleanup file handling 1. use proper finds to locate files. 2. drop chdir during script execution Change-Id: I54337aa6760ff488a1a72fe18bf2563c132e3fb6 Signed-off-by: Alon Bar-Lev <[email protected]> --- M build/dbscripts-duplicate_upgrade_scripts.sh M packaging/dbscripts/cleandb.sh M packaging/dbscripts/create_schema.sh M packaging/dbscripts/dbfunc-common.sh M packaging/dbscripts/dbfunc-custom.sh M packaging/dbscripts/refreshStoredProcedures.sh M packaging/dbscripts/unlock_entity.sh M packaging/dbscripts/upgrade.sh M packaging/setup/dbutils/changedbowner.sh M packaging/setup/dbutils/encodingvalidator.sh M packaging/setup/dbutils/fkvalidator.sh M packaging/setup/dbutils/taskcleaner.sh 12 files changed, 61 insertions(+), 76 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/33/25133/1 diff --git a/build/dbscripts-duplicate_upgrade_scripts.sh b/build/dbscripts-duplicate_upgrade_scripts.sh index 908843d..f9f2b12 100755 --- a/build/dbscripts-duplicate_upgrade_scripts.sh +++ b/build/dbscripts-duplicate_upgrade_scripts.sh @@ -1,8 +1,6 @@ #!/bin/sh -cd "$(dirname "$0")/../packaging/dbscripts" - -OUT="$(ls upgrade | grep -P '\d{2}_\d{2}.\d{2,8}' -o | uniq -d)" +OUT="$(find "$(dirname "$0")/../packaging/dbscripts" | grep -P '\d{2}_\d{2}_\d{2,8}' -o | uniq -d)" if [ -n "${OUT}" ]; then echo "Found duplicate upgrade scripts with version $(echo ${OUT}), please resolve and retry" >&2 diff --git a/packaging/dbscripts/cleandb.sh b/packaging/dbscripts/cleandb.sh index 670e59c..c91b9b5 100755 --- a/packaging/dbscripts/cleandb.sh +++ b/packaging/dbscripts/cleandb.sh @@ -4,9 +4,8 @@ # Cleans DB by dropping all DB objects ################################################################################ -#include db general functions -cd "$(dirname "$0")" -. ./dbfunc-custom.sh +DBFUNC_COMMON_DBSCRIPTS_DIR="$(dirname "$0")" +. "${DBFUNC_COMMON_DBSCRIPTS_DIR}/dbfunc-custom.sh" cleanup() { dbfunc_cleanup diff --git a/packaging/dbscripts/create_schema.sh b/packaging/dbscripts/create_schema.sh index 1c9e9c8..2d84b81 100755 --- a/packaging/dbscripts/create_schema.sh +++ b/packaging/dbscripts/create_schema.sh @@ -1,7 +1,7 @@ #!/bin/sh -cd "$(dirname "$0")" -. ./dbfunc-custom.sh +DBFUNC_COMMON_DBSCRIPTS_DIR="$(dirname "$0")" +. "${DBFUNC_COMMON_DBSCRIPTS_DIR}/dbfunc-custom.sh" cleanup() { dbfunc_cleanup @@ -47,13 +47,13 @@ dbfunc_psql_die --command="ALTER DATABASE \"${DBFUNC_DB_DATABASE}\" SET client_min_messages=ERROR;" > /dev/null echo "Creating tables..." -dbfunc_psql_die --file="create_tables.sql" > /dev/null +dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/create_tables.sql" > /dev/null echo "Creating functions..." -dbfunc_psql_die --file="create_functions.sql" > /dev/null +dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/create_functions.sql" > /dev/null echo "Creating common functions..." -dbfunc_psql_die --file="common_sp.sql" > /dev/null +dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/common_sp.sql" > /dev/null #inserting initial data dbfunc_common_hook_init_insert_data diff --git a/packaging/dbscripts/dbfunc-common.sh b/packaging/dbscripts/dbfunc-common.sh index a70681e..b567638 100644 --- a/packaging/dbscripts/dbfunc-common.sh +++ b/packaging/dbscripts/dbfunc-common.sh @@ -1,5 +1,7 @@ -. ./dbfunc-base.sh +DBFUNC_COMMON_DBSCRIPTS_DIR="${DBFUNC_COMMON_DBSCRIPTS_DIR:-$(dirname "$0")}" + +. "${DBFUNC_COMMON_DBSCRIPTS_DIR}/dbfunc-base.sh" #DBFUNC_COMMON_MD5DIR= @@ -33,7 +35,7 @@ #cleans db by dropping all objects dbfunc_database_clean() { - dbfunc_psql_die --file="common_sp.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/common_sp.sql" > /dev/null local statement statement="$( dbfunc_psql_die --command="select * from generate_drop_all_seq_syntax();" @@ -50,9 +52,9 @@ } dbfunc_common_upgrade() { - local res="$(find upgrade/ -name '*.sql' -or -name '*.sh' | wc -l)" + local files="$(_dbfunc_common_get_files "upgrade" 1)" local CMD - if [ "${res}" -gt 0 ]; then + if [ -n "${files}" ]; then local state="FAILED" local comment="" local updated=0 @@ -67,13 +69,11 @@ local current="$(_dbfunc_common_get_current_version)" # we should remove leading blank (from select result) and zero in order not to treat number as octal local last="$(expr substr "${current}" 3 7)" - local files="$(_dbfunc_common_get_files "upgrade" 1)" local file - for file in $(ls ${files} | sort); do - local before="$(_dbfunc_common_get_db_time)" - local checksum="$(md5sum "${file}" | cut -d " " -f1)" - # upgrade/dd_dd_dddd* => dddddddd - local ver="$(_dbfunc_common_get_file_version "${file}")" + echo "${files}" | while read file; do + before="$(_dbfunc_common_get_db_time)" + checksum="$(md5sum "${file}" | cut -d " " -f1)" + ver="$(_dbfunc_common_get_file_version "${file}")" if [ "${ver}" -gt "${current}" ] ; then # we should remove leading zero in order not to treat number as octal local xver="$(expr substr "${ver}" 2 7)" @@ -132,7 +132,7 @@ ) values ( trim('${ver}'), - '${file}', + '$(echo "${file}" | sed "s#^${DBFUNC_COMMON_DBSCRIPTS_DIR}/##")', '${checksum}', '${DBFUNC_DB_USER}', cast(trim('${before}') as timestamp), @@ -143,7 +143,7 @@ ); " > /dev/null fi - done + done || exit $? _dbfunc_common_set_last_version # restore views & SPs if dropped @@ -175,14 +175,14 @@ #drops views before upgrade or refresh operations dbfunc_common_views_drop() { # common stored procedures are executed first (for new added functions to be valid) - dbfunc_psql_die --file="common_sp.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/common_sp.sql" > /dev/null dbfunc_psql_die --command="select * from generate_drop_all_views_syntax();" | \ dbfunc_psql_die > /dev/null } #drops sps before upgrade or refresh operations dbfunc_common_sps_drop() { - dbfunc_psql_die --file="common_sp.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/common_sp.sql" > /dev/null local statement statement="$( dbfunc_psql_die --command="select * from generate_drop_all_functions_syntax();" @@ -190,18 +190,18 @@ dbfunc_psql_die --command="${statement}" > /dev/null # recreate generic functions - dbfunc_psql_die --file="create_functions.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/create_functions.sql" > /dev/null } #refreshes sps dbfunc_common_sps_refresh() { echo "Creating stored procedures..." - local sql - for sql in $(ls *sp.sql | sort); do - echo "Creating stored procedures from ${sql}..." - dbfunc_psql_die --file="${sql}" > /dev/null - done - dbfunc_psql_die --file="common_sp.sql" > /dev/null + local file + find "${DBFUNC_COMMON_DBSCRIPTS_DIR}" -name '*sp.sql' | sort | while read file; do + echo "Creating stored procedures from ${file}..." + dbfunc_psql_die --file="${file}" > /dev/null + done || exit $? + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/common_sp.sql" > /dev/null } #unlocks the given VM/Template and its disks or a given disk @@ -338,7 +338,7 @@ dbfunc_common_views_drop dbfunc_common_sps_drop # common stored procedures are executed first (for new added functions to be valid) - dbfunc_psql_die --file="common_sp.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/common_sp.sql" > /dev/null #update sequence numers dbfunc_common_hook_sequence_numbers_update #run pre upgrade scripts @@ -357,7 +357,7 @@ #Running post-upgrade scripts _dbfunc_common_psql_statements_in_dir 'post_upgrade' #run custom materialized views if exists - custom_materialized_views_file="upgrade/post_upgrade/custom/create_materialized_views.sql" + custom_materialized_views_file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/upgrade/post_upgrade/custom/create_materialized_views.sql" if [ -f "${custom_materialized_views_file}" ]; then echo "running custom materialized views from '${custom_materialized_views_file}'..." if ! dbfunc_psql --file="${custom_materialized_views_file}"; then @@ -372,12 +372,9 @@ # Runs all the SQL scripts in directory upgrade/$1/ _dbfunc_common_psql_statements_in_dir() { local dir="$1" - if [ -d "upgrade/${dir}" ]; then - files="$(_dbfunc_common_get_files "upgrade/${dir}" 1)" - for file in $(ls ${files} | sort); do - _dbfunc_common_run_file "${file}" - done - fi + _dbfunc_common_get_files "upgrade/${dir}" 1 | while read file; do + _dbfunc_common_run_file "${file}" + done || exit $? } _dbfunc_common_run_required_scripts() { @@ -392,7 +389,7 @@ echo "${sql}" | grep -q "_sp.sql" || \ die "invalid source file ${sql} in ${file}, source files must end with '_sp.sql'" echo "Running helper functions from '${sql}' for '${file}'" - dbfunc_psql_die --file="${sql}" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/${sql}" > /dev/null done < "${script}" } @@ -403,7 +400,7 @@ echo "Running upgrade shell script '${file}'..." ( eval "$(set | grep '^DBFUNC_' | sed 's/^\([^=]*\)=.*/export \1/')" - "./${file}" + "${file}" ) else echo "Running upgrade sql script '${file}'..." @@ -454,7 +451,7 @@ _dbfunc_common_get_files() { local dir="$1" local maxdepth="$2" - find "${dir}" \ + find "${DBFUNC_COMMON_DBSCRIPTS_DIR}/${dir}" \ -maxdepth "${maxdepth}" \ -name '*.sql' -or -name '*.sh' | \ sort @@ -484,14 +481,12 @@ } _dbfunc_common_validate_version_uniqueness() { - local prev="" - local files="$(_dbfunc_common_get_files "upgrade" 1)" local file - for file in $(ls ${files} | sort) ; do - local ver="$(_dbfunc_common_get_file_version "${file}")" + _dbfunc_common_get_files "upgrade" 1 | while read file; do + ver="$(_dbfunc_common_get_file_version "${file}")" [ "${ver}" != "${prev}" ] || die "Operation aborted, found duplicate version: ${ver}" prev="${ver}" - done + done || exit $? } #adds a record to audit_log in case of calling unlock_entity diff --git a/packaging/dbscripts/dbfunc-custom.sh b/packaging/dbscripts/dbfunc-custom.sh index ddf50d5..3865d86 100644 --- a/packaging/dbscripts/dbfunc-custom.sh +++ b/packaging/dbscripts/dbfunc-custom.sh @@ -1,5 +1,5 @@ -. ./dbfunc-common.sh +. "${DBFUNC_COMMON_DBSCRIPTS_DIR}/dbfunc-common.sh" DBFUNC_DB_USER="${DBFUNC_DB_USER:-engine}" DBFUNC_DB_DATABASE="${DBFUNC_DB_DATABASE:-engine}" @@ -8,29 +8,29 @@ dbfunc_common_hook_init_insert_data() { echo "Inserting data..." - dbfunc_psql_die --file="insert_data.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/insert_data.sql" > /dev/null echo "Inserting pre-defined roles..." - dbfunc_psql_die --file="insert_predefined_roles.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/insert_predefined_roles.sql" > /dev/null } dbfunc_common_hook_pre_upgrade() { if [ -n "${DBFUNC_CUSTOM_CLEAN_TASKS}" ]; then echo "Cleaning tasks metadata..." - dbfunc_psql_die --file="delete_async_tasks_and_compensation_data.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/delete_async_tasks_and_compensation_data.sql" > /dev/null fi } #refreshes views dbfunc_common_hook_views_refresh() { echo "Creating views..." - dbfunc_psql_die --file="create_views.sql" > /dev/null - dbfunc_psql_die --file="create_dwh_views.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/create_views.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/create_dwh_views.sql" > /dev/null } # Materilized views functions, override with empty implementation on DBs that not supporting that dbfunc_common_hook_materialized_views_install() { - dbfunc_psql_die --file="materialized_views_sp.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/materialized_views_sp.sql" > /dev/null } dbfunc_common_hook_materialized_views_drop() { @@ -44,5 +44,5 @@ } dbfunc_common_hook_sequence_numbers_update() { - dbfunc_psql_die --file="update_sequence_numbers.sql" > /dev/null + dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/update_sequence_numbers.sql" > /dev/null } diff --git a/packaging/dbscripts/refreshStoredProcedures.sh b/packaging/dbscripts/refreshStoredProcedures.sh index 239d885..eadb043 100755 --- a/packaging/dbscripts/refreshStoredProcedures.sh +++ b/packaging/dbscripts/refreshStoredProcedures.sh @@ -1,7 +1,7 @@ #!/bin/sh -cd "$(dirname "$0")" -. ./dbfunc-custom.sh +DBFUNC_COMMON_DBSCRIPTS_DIR="$(dirname "$0")" +. "${DBFUNC_COMMON_DBSCRIPTS_DIR}/dbfunc-custom.sh" cleanup() { dbfunc_cleanup diff --git a/packaging/dbscripts/unlock_entity.sh b/packaging/dbscripts/unlock_entity.sh index 4154fc7..1d2f8c9 100755 --- a/packaging/dbscripts/unlock_entity.sh +++ b/packaging/dbscripts/unlock_entity.sh @@ -1,7 +1,7 @@ #!/bin/sh -cd "$(dirname "$0")" -. ./dbfunc-custom.sh +DBFUNC_COMMON_DBSCRIPTS_DIR="$(dirname "$0")" +. "${DBFUNC_COMMON_DBSCRIPTS_DIR}/dbfunc-custom.sh" cleanup() { dbfunc_cleanup diff --git a/packaging/dbscripts/upgrade.sh b/packaging/dbscripts/upgrade.sh index 8e72ec7..78550a7 100755 --- a/packaging/dbscripts/upgrade.sh +++ b/packaging/dbscripts/upgrade.sh @@ -30,9 +30,8 @@ # problems) ################################################################################ -#include db general functions -cd "$(dirname "$0")" -. ./dbfunc-custom.sh +DBFUNC_COMMON_DBSCRIPTS_DIR="$(dirname "$0")" +. "${DBFUNC_COMMON_DBSCRIPTS_DIR}/dbfunc-custom.sh" cleanup() { dbfunc_cleanup @@ -76,7 +75,7 @@ esac done -dbfunc_psql_die --file="upgrade/03_02_0000_set_version.sql" > /dev/null +dbfunc_psql_die --file="${DBFUNC_COMMON_DBSCRIPTS_DIR}/upgrade/03_02_0000_set_version.sql" > /dev/null if [ -n "${VERSION}" ]; then dbfunc_psql_die --command=" update schema_version diff --git a/packaging/setup/dbutils/changedbowner.sh b/packaging/setup/dbutils/changedbowner.sh index 0bad47b..34fc40b 100755 --- a/packaging/setup/dbutils/changedbowner.sh +++ b/packaging/setup/dbutils/changedbowner.sh @@ -4,9 +4,7 @@ # database ################################################################ -#include db general functions -cd "$(dirname "$0")" -. ./dbfunc-base.sh +. "$(dirname "$0")/dbfunc-base.sh" cleanup() { dbfunc_cleanup diff --git a/packaging/setup/dbutils/encodingvalidator.sh b/packaging/setup/dbutils/encodingvalidator.sh index c21cf39..49f79ca 100755 --- a/packaging/setup/dbutils/encodingvalidator.sh +++ b/packaging/setup/dbutils/encodingvalidator.sh @@ -7,9 +7,7 @@ # It is highly recomended to backup the database before using this utility. ############################################################################################################### -#include db general functions -cd "$(dirname "${0}")" -. ./dbfunc-base.sh +. "$(dirname "${0}")/dbfunc-base.sh" cleanup() { dbfunc_cleanup diff --git a/packaging/setup/dbutils/fkvalidator.sh b/packaging/setup/dbutils/fkvalidator.sh index a745784..ff17346 100755 --- a/packaging/setup/dbutils/fkvalidator.sh +++ b/packaging/setup/dbutils/fkvalidator.sh @@ -20,8 +20,7 @@ ############################################################################################################### #include db general functions -cd "$(dirname "$0")" -. ./dbfunc-base.sh +. "$(dirname "$0")/dbfunc-base.sh" cleanup() { dbfunc_cleanup @@ -115,6 +114,6 @@ fi # Install fkvalidator procedures -dbfunc_psql_die --file=./fkvalidator_sp.sql > /dev/null +dbfunc_psql_die --file="$(dirname "$0")/fkvalidator_sp.sql" > /dev/null # Execute validate_db_fks "${FIXIT}" "${DBFUNC_VERBOSE}" diff --git a/packaging/setup/dbutils/taskcleaner.sh b/packaging/setup/dbutils/taskcleaner.sh index eb2c6e5..e481b6f 100755 --- a/packaging/setup/dbutils/taskcleaner.sh +++ b/packaging/setup/dbutils/taskcleaner.sh @@ -17,8 +17,7 @@ # should be cleaned as well. ############################################################################################################### -cd "$(dirname "$0")" -. ./dbfunc-base.sh +. "$(dirname "$0")/dbfunc-base.sh" cleanup() { dbfunc_cleanup @@ -238,6 +237,6 @@ fi # Install taskcleaner procedures -dbfunc_psql_die --file=./taskcleaner_sp.sql > /dev/null +dbfunc_psql_die --file="$(dirname "$0")/taskcleaner_sp.sql" > /dev/null # Execute dbfunc_psql_die --command="${CMD1}${CMD2}" -- To view, visit http://gerrit.ovirt.org/25133 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I54337aa6760ff488a1a72fe18bf2563c132e3fb6 Gerrit-PatchSet: 1 Gerrit-Project: ovirt-engine Gerrit-Branch: master Gerrit-Owner: Alon Bar-Lev <[email protected]> _______________________________________________ Engine-patches mailing list [email protected] http://lists.ovirt.org/mailman/listinfo/engine-patches
