Set user hdfs cmd for putting files on hdfs In the case where we're building ODM tables to be stored as avro, we need to be able to store the schemas on the hdfs. This requires us to specify and run an hdfs command with the user that owns the odm setup files locally. The user_hdfs_cmd allows us to reference the user which is running this script, but also likely owns the avro schema files locally.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/2ae343d9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/2ae343d9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/2ae343d9 Branch: refs/heads/SPOT-181_ODM Commit: 2ae343d921f3b0b64cfe70951dd14dcdecd3c2f2 Parents: 8fa024b Author: Tadd Wood <t...@arcadiadata.com> Authored: Wed Feb 14 14:00:57 2018 -0800 Committer: Tadd Wood <t...@arcadiadata.com> Committed: Wed Feb 14 14:00:57 2018 -0800 ---------------------------------------------------------------------- spot-setup/odm/odm_setup.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/2ae343d9/spot-setup/odm/odm_setup.sh ---------------------------------------------------------------------- diff --git a/spot-setup/odm/odm_setup.sh b/spot-setup/odm/odm_setup.sh index 38731de..1927f0e 100755 --- a/spot-setup/odm/odm_setup.sh +++ b/spot-setup/odm/odm_setup.sh @@ -100,7 +100,9 @@ source $SPOTCONF # Check no-sudo argument and set the proper hdfs command to run our create table statements later if [[ ${no_sudo} == "true" ]]; then hdfs_cmd="hdfs" + user_hdfs_cmd="hdfs" + # If HADOOP_USER_NAME already set, don't attempt to set as hdfs if [[ ! -z "${HADOOP_USER_NAME}" ]]; then log "HADOOP_USER_NAME: ${HADOOP_USER_NAME}" else @@ -109,6 +111,7 @@ if [[ ${no_sudo} == "true" ]]; then fi else hdfs_cmd="sudo -u hdfs hdfs" + user_hdfs_cmd="sudo -u ${USER} hdfs" fi # Creating HDFS user's folder @@ -178,9 +181,9 @@ do if [ "$format" == "avro" ] ; then log "Adding ${f} Avro schema to ${HUSER}/$d/schema ..." - log "${hdfs_cmd} dfs -put -f $f.avsc ${HUSER}/$d/schema/$f.avsc" + log "${user_hdfs_cmd} dfs -put -f $f.avsc ${HUSER}/$d/schema/$f.avsc" - ${hdfs_cmd} dfs -put -f $f.avsc ${HUSER}/$d/schema/$f.avsc + ${user_hdfs_cmd} dfs -put -f $f.avsc ${HUSER}/$d/schema/$f.avsc log "Creating ODM Impala Avro table ${f}..." log "${impala_db_shell} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} --var=ODM_AVRO_URL=hdfs://${HUSER}/${d}/schema/${f}.avsc -c -f create_${f}_avro.sql"
