Many bug fixes. All JIRAs listed below. TRAFODION-1545 Adding Management nodes TRAFODION-1691 Smaller subscripts Trafodion-1681 Need full HADOOP path for hdfs Trafodion-1689 DCS HA Trafodion-1756 kernel.pid_max persistent on reboot Trafodion-1694 kernel.pid_max full path Trafodion-1693 CLUSTERNAME being set Trafodion-1574 Increasing maintainability of scripts Trafodion-1711 curl commands should accept http/https Trafodion-1478 Create /hbase/archive/data/default
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/ee4c2539 Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/ee4c2539 Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/ee4c2539 Branch: refs/heads/master Commit: ee4c25397f647805c112ffdec55bd9c24a30a432 Parents: bd86a46 Author: Amanda Moran <[email protected]> Authored: Wed Jan 20 23:54:05 2016 +0000 Committer: Amanda Moran <[email protected]> Committed: Wed Jan 20 23:54:05 2016 +0000 ---------------------------------------------------------------------- install/installer/cloud_cli_setup | 16 +- install/installer/dcs_installer | 32 ++- install/installer/rest_installer | 10 +- .../tools/traf_cloudera_uninstall_suse | 5 + install/installer/traf_add_sudoAccess | 8 +- install/installer/traf_add_user | 37 ++-- install/installer/traf_cloudera_mods98 | 94 ++++++--- install/installer/traf_config | 94 +-------- install/installer/traf_config_check | 166 +++++++++++++-- install/installer/traf_config_setup | 60 ++++-- install/installer/traf_create_systemdefaults | 203 ------------------- install/installer/traf_getHadoopNodes | 13 +- install/installer/traf_hortonworks_mods98 | 91 ++++++--- install/installer/traf_package_setup | 22 +- install/installer/traf_setup | 36 +--- install/installer/traf_sqgen | 96 +++++++++ install/installer/traf_start | 17 +- install/installer/trafodion_config_default | 7 +- install/installer/trafodion_install | 107 +++++++--- install/installer/trafodion_uninstaller | 6 +- 20 files changed, 598 insertions(+), 522 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ee4c2539/install/installer/cloud_cli_setup ---------------------------------------------------------------------- diff --git a/install/installer/cloud_cli_setup b/install/installer/cloud_cli_setup index ce653b6..980c61e 100755 --- a/install/installer/cloud_cli_setup +++ b/install/installer/cloud_cli_setup @@ -27,7 +27,8 @@ TRAF_CONFIG=/etc/trafodion/trafodion_config source $TRAF_CONFIG if [ $node_count -eq 1 ]; then - exit 0 + TRAF_PDSH="" + TRAF_PDCP="/bin/cp" else TRAF_PDSH="pdsh $MY_NODES" TRAF_PDCP="pdcp -r $MY_NODES" @@ -36,20 +37,25 @@ fi ###### Setting up CLI for AWS as sudo id if [[ ! -z $AWS_CLOUD ]]; then - ###### Check for existence of aws binary +###### Check for existence of aws if [[ ! -d /usr/local/aws ]]; then cd $HOME curl "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip"; -o "awscli-bundle.zip" unzip $HOME/awscli-bundle.zip - $TRAF_PDCP $HOME/awscli-bundle $HOME - $TRAF_PDSH "sudo $HOME/awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws" | tee -a $INSTALL_LOG + if [[ $node_count -gt "1" ]]; then + $TRAF_PDCP $HOME/awscli-bundle $HOME + $TRAF_PDSH "sudo $HOME/awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws" | tee -a $INSTALL_LOG + else + sudo $HOME/awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws | tee -a $INSTALL_LOG + fi $TRAF_PDSH "sudo rm -rf $HOME/awscli-bundle" rm -f $HOME/awscli-bundle.zip fi - ###### Check if .aws configuration file exist for Trafodion Id +###### Check for existence of .aws for trafodion id + ###### Check if aws configuration file exist for Trafodion Id AWSCONFIG_DIR=`sudo su $TRAF_USER --login --command "ls -altr | grep -w ".aws" | wc -l" ` if [[ $AWSCONFIG_DIR == 0 ]]; then ###### Execute 'aws configure' command as a Trafodion Id http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ee4c2539/install/installer/dcs_installer ---------------------------------------------------------------------- diff --git a/install/installer/dcs_installer b/install/installer/dcs_installer index f1b190d..1f873c8 100755 --- a/install/installer/dcs_installer +++ b/install/installer/dcs_installer @@ -27,12 +27,12 @@ export TRAF_CONFIG="/etc/trafodion/trafodion_config" -if [ $node_count -eq 1 ]; then +if [ $all_node_count -eq 1 ]; then TRAF_PDSH="" TRAF_PDCP="" else - TRAF_PDSH="pdsh -R exec $MY_NODES $PDSH_SSH_CMD" - TRAF_PDCP="pdcp -R ssh $MY_NODES" + TRAF_PDSH="pdsh -R exec $ALL_NODES $PDSH_SSH_CMD" + TRAF_PDCP="pdcp -R ssh $ALL_NODES" fi function print_usage { @@ -131,8 +131,6 @@ if [[ -z "$NODE_LIST" ]]; then exit -1 fi -# Set server count default if no user supplied value -node_count=$(echo $NODE_LIST | wc -w) if [ -z $DCS_SERVER_COUNT ]; then DCS_SERVER_COUNT=$node_count fi @@ -196,8 +194,6 @@ echo $DCS_PRIMARY_MASTER_NODE > $DCS_DIR/conf/master if [[ "$ENABLE_HA" == "true" ]]; then - - ######Configure dcs-site.xml file rm dcs-site.temp 2>/dev/null cat dcs-site.xml | sed -e "s@</configuration>@ <property>\n <name>dcs.master.floating.ip</name>\n <value>true</value>\n </property>\n</configuration>@" > dcs-site.temp cp dcs-site.temp dcs-site.xml @@ -213,31 +209,25 @@ if [[ "$ENABLE_HA" == "true" ]]; then cp dcs-site.temp dcs-site.xml rm dcs-site.temp 2>/dev/null - ######Configure the backup masters file echo "***INFO: modifying $DCS_DIR/conf/backup-masters file" + for node in $BACKUP_DCS_NODES do echo $node >> $DCS_DIR/conf/backup-masters done - + ######Configure trafci wrapper script to use Floating IP address newHName="HNAME=$FLOATING_IP:23400" sed -i -e "s/HNAME=localhost:23400/$newHName/g" $SQ_ROOT/trafci/bin/trafci - - ######Copy HA configuration to all nodes in the cluster - #Following lines are being copied by the caller script. Will remove these lines later - #if [[ $node_count -gt 1 ]]; then - # $TRAF_PDCP $DCS_DIR/conf/dcs-site.xml $DCS_DIR/conf/dcs-site.xml - # $TRAF_PDCP $DCS_DIR/conf/backup-masters $DCS_DIR/conf/backup-masters - # $TRAF_PDCP $SQ_ROOT/trafci/bin/trafci $SQ_ROOT/trafci/bin/trafci - #fi + else - ######Configure trafci wrapper script - newHName="HNAME=$DCS_PRIMARY_MASTER_NODE:23400" - sed -i -e "s/HNAME=localhost:23400/$newHName/g" $SQ_ROOT/trafci/bin/trafci + + ######Configure trafci wrapper script + sed -i -e "s/HNAME=localhost:23400/HNAME=$DCS_PRIMARY_MASTER_NODE:23400/g" $SQ_ROOT/trafci/bin/trafci fi + echo "***INFO: creating $DCS_DIR/conf/servers file" rm servers 2>/dev/null let per_node=$DCS_SERVER_COUNT/$node_count @@ -254,5 +244,7 @@ do (( count++ )) done + + echo "***INFO: End of DCS install." http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ee4c2539/install/installer/rest_installer ---------------------------------------------------------------------- diff --git a/install/installer/rest_installer b/install/installer/rest_installer index f02da87..68888c2 100755 --- a/install/installer/rest_installer +++ b/install/installer/rest_installer @@ -110,14 +110,10 @@ cd $REST_DIR/conf echo "***INFO: modifying $REST_DIR/conf/rest-site.xml" # get zookeeper quorum -if [ "$node_count" -ne "1" ]; then - lineNumber=$(grep -n "zookeeper\.quorum" /etc/hbase/conf/hbase-site.xml | sed 's/\:.*//') - lineNumber=$((lineNumber+1)) +lineNumber=$(grep -n "zookeeper\.quorum" /etc/hbase/conf/hbase-site.xml | sed 's/\:.*//') +lineNumber=$((lineNumber+1)) - ZOOKEEPER_NODES=`sed "$lineNumber!d" /etc/hbase/conf/hbase-site.xml | sed 's/\/value.*//' | sed 's/.*>//' | sed 's/.$//'` -else - ZOOKEEPER_NODES=$(hostname -s) -fi +ZOOKEEPER_NODES=`sed "$lineNumber!d" /etc/hbase/conf/hbase-site.xml | sed 's/\/value.*//' | sed 's/.*>//' | sed 's/.$//'` # add zookeeper quorum property to end of configuration rm rest-site.temp 2>/dev/null http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ee4c2539/install/installer/tools/traf_cloudera_uninstall_suse ---------------------------------------------------------------------- diff --git a/install/installer/tools/traf_cloudera_uninstall_suse b/install/installer/tools/traf_cloudera_uninstall_suse index e476434..c556106 100755 --- a/install/installer/tools/traf_cloudera_uninstall_suse +++ b/install/installer/tools/traf_cloudera_uninstall_suse @@ -119,6 +119,11 @@ done echo "***INFO: Removing any Cloudera repos" sudo rm -rf /etc/yum.repos.d/cloudera* +sudo rm -rf /etc/zypp/repos.d/cloudera-manager.repo* +sudo rm -rf /var/cache/zypp/packages/cloudera* +sudo rm -rf /var/cache/zypp/raw/cloudera* +sudo rm -rf /var/cache/zypp/solv/cloudera* + #============================================= echo "***INFO: Removing Cloudera symbolic directory links" http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ee4c2539/install/installer/traf_add_sudoAccess ---------------------------------------------------------------------- diff --git a/install/installer/traf_add_sudoAccess b/install/installer/traf_add_sudoAccess index eda398b..93b4dc8 100755 --- a/install/installer/traf_add_sudoAccess +++ b/install/installer/traf_add_sudoAccess @@ -23,12 +23,12 @@ TRAF_CONFIG=/etc/trafodion/trafodion_config source $TRAF_CONFIG -if [ $node_count -eq 1 ]; then +if [ $all_node_count -eq 1 ]; then TRAF_PDSH="" TRAF_PDCP="" else - TRAF_PDSH="pdsh -R exec $MY_NODES $PDSH_SSH_CMD" - TRAF_PDCP="pdcp -R ssh $MY_NODES" + TRAF_PDSH="pdsh -R exec $ALL_NODES $PDSH_SSH_CMD" + TRAF_PDCP="pdcp -R ssh $ALL_NODES" fi @@ -44,7 +44,7 @@ echo "## Allow Trafodion id to run commands needed to configure floating IP" >> echo "%trafodion ALL = NOPASSWD: IP, ARP" >> $sudoFile -if [[ $node_count -gt "1" ]]; then +if [[ $all_node_count -gt "1" ]]; then $TRAF_PDCP $sudoFile $HOME $TRAF_PDSH sudo cp $HOME/$sudoFile /etc/sudoers.d/ $TRAF_PDSH sudo rm -f $HOME/$sudoFile http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ee4c2539/install/installer/traf_add_user ---------------------------------------------------------------------- diff --git a/install/installer/traf_add_user b/install/installer/traf_add_user index a3ca012..8afbc78 100755 --- a/install/installer/traf_add_user +++ b/install/installer/traf_add_user @@ -55,16 +55,15 @@ if [ $? -ne 0 ]; then exit -1 fi -# create MY_NODES (used for pdsh/pdcp) from NODE_LIST -if [ $node_count -eq 1 ]; then +if [ $all_node_count -eq 1 ]; then TRAF_PDSH="" TRAF_PDCP="" else # use the -S option to cause pdsh to return largest of # the remote command return values so we can tell if one # or more of the remote commands failed - TRAF_PDSH="pdsh -R exec $MY_NODES ssh -q -n %h" - TRAF_PDCP="pdcp -R ssh $MY_NODES" + TRAF_PDSH="pdsh -R exec $ALL_NODES ssh -q -n %h" + TRAF_PDCP="pdcp -R ssh $ALL_NODES" fi #============================== # Find an unused uid and gid on all nodes for the Trafodion userid. @@ -87,7 +86,7 @@ if [ -z "$traf_gid" ]; then while [ $found -ne 0 ] do ((traf_gid++)) - if [ $node_count -eq 1 ]; then + if [ $all_node_count -eq 1 ]; then found=$(cat /etc/group | sed -e 's@^.*:x:\([0-9]*\):.*@\1@' | grep ^$traf_gid$ 2>/dev/null | wc -l) else found=$($TRAF_PDSH "cat /etc/group | sed -e 's@^.*:x:\([0-9]*\):.*@\1@' | grep ^$traf_gid$ " 2>/dev/null | wc -l) @@ -119,7 +118,7 @@ traf_uid=500 # start at uid 500 (just a randomly chosen start) while [ $found -ne 0 ] do ((traf_uid++)) - if [ $node_count -eq 1 ]; then + if [ $all_node_count -eq 1 ]; then found=$(cat /etc/passwd | sed -e 's@^.*:x:\([0-9]*\):\([0-9]*\):.*:.*$@\1@' | grep ^$traf_uid$ 2>/dev/null | wc -l) else found=$($TRAF_PDSH "cat /etc/passwd | sed -e 's@^.*:x:\([0-9]*\):\([0-9]*\):.*:.*\$@\1@' | grep ^$traf_uid$ " 2>/dev/null | wc -l) @@ -148,6 +147,11 @@ if [ $return_code -ne 0 ]; then exit -1 fi fi + +$TRAF_PDSH sudo chmod 777 $TRAF_CONFIG +$TRAF_PDSH sudo sed -i '/TRAF_USER_PASSWORD\=/d' $TRAF_CONFIG +$TRAF_PDSH sudo chmod 777 $TRAF_CONFIG + #================================================= # Setup passwordless ssh for this Trafodion userid @@ -191,7 +195,7 @@ rm $TRAF_WORKDIR/bashrc_${TRAF_USER}_temp3 #================================================= # copy TRAF_USER_DIR to all other nodes if more than one -if [ "$node_count" -ne 1 ]; then +if [ "$all_node_count" -ne 1 ]; then # Have to copy in a round-about way since passwordless # ssh is not setup yet for the Trafodion userid and pdcp # may not work with sudo. @@ -200,37 +204,38 @@ if [ "$node_count" -ne 1 ]; then sudo cp -r $TRAF_USER_DIR $TRAF_WORKDIR sudo rm -rf $TRAF_WORKDIR/$TRAF_USER/.pulse 2>/dev/null sudo chown -R $USER.$(id -ng) $TRAF_WORKDIR/$TRAF_USER - $PDCP -r $MY_NODES -x $HOSTNAME $TRAF_WORKDIR/$TRAF_USER $TRAF_WORKDIR - $PDSH $MY_NODES -x $HOSTNAME $PDSH_SSH_CMD sudo cp -r $TRAF_WORKDIR/$TRAF_USER $default_home - $PDSH $MY_NODES -x $HOSTNAME $PDSH_SSH_CMD sudo chown -R $TRAF_USER.$TRAF_GROUP $TRAF_USER_DIR + $PDCP -r $ALL_NODES -x $HOSTNAME $TRAF_WORKDIR/$TRAF_USER $TRAF_WORKDIR + $PDSH $ALL_NODES -x $HOSTNAME $PDSH_SSH_CMD sudo cp -r $TRAF_WORKDIR/$TRAF_USER $default_home + $PDSH $ALL_NODES -x $HOSTNAME $PDSH_SSH_CMD sudo chown -R $TRAF_USER.$TRAF_GROUP $TRAF_USER_DIR fi #================================================= # Cycle through all nodes to create known_hosts file, even if only one node echo "***INFO: Creating known_hosts file for all nodes" NODES_SHORT="" -for node in $NODE_LIST +for node in $ALL_NODE_LIST do + echo $node + echo $ALL_NODE_LIST newNode=$(ssh -q -n $node hostname -s) NODES_SHORT="$NODES_SHORT $newNode" done NODES_LONG="" -for node in $NODE_LIST +for node in $ALL_NODE_LIST do newNode=$(ssh -q -n $node hostname -f) NODES_LONG="$NODES_LONG $newNode" done - -./traf_createPasswordLessSSH --nodes "$NODE_LIST" +./traf_createPasswordLessSSH --nodes "$ALL_NODE_LIST" ./traf_createPasswordLessSSH --nodes "$NODES_SHORT" ./traf_createPasswordLessSSH --nodes "$NODES_LONG" -if [ "$node_count" -ne 1 ]; then +if [ "$all_node_count" -ne 1 ]; then # Copy known_hosts file to all other nodes - sudo su $TRAF_USER --command "$PDCP $MY_NODES -x $HOSTNAME $TRAF_USER_DIR/.ssh/known_hosts $TRAF_USER_DIR/.ssh" + sudo su $TRAF_USER --command "$PDCP $ALL_NODES -x $HOSTNAME $TRAF_USER_DIR/.ssh/known_hosts $TRAF_USER_DIR/.ssh" fi #================================================= http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ee4c2539/install/installer/traf_cloudera_mods98 ---------------------------------------------------------------------- diff --git a/install/installer/traf_cloudera_mods98 b/install/installer/traf_cloudera_mods98 index 9e7b63e..b112413 100755 --- a/install/installer/traf_cloudera_mods98 +++ b/install/installer/traf_cloudera_mods98 @@ -58,6 +58,7 @@ fi traf_util_jar="trafodion-utility-${TRAF_VERSION}.jar" + # The permissions the Trafodion build process creates on the hbase-trx jar # files does not work well with the installation process so we change them sudo chmod -R 777 $UNTAR_DIR/export/lib @@ -118,42 +119,42 @@ sudo chmod 777 $HOME/hbase-site.xml sudo cp $HOME/hbase-site.xml $TRAF_WORKDIR sudo chown trafodion.trafodion $TRAF_WORKDIR/hbase-site.xml -sudo su $HDFS_USER --command "hadoop fs -mkdir /hbase-staging" 2> $HOME/traf_temp_output +sudo su $HDFS_USER --command "$HADOOP_BIN_PATH/hadoop fs -mkdir /hbase-staging" 2> $HOME/traf_temp_output if [ $? != 0 ]; then # ok if directory already exists dir_exists=$(grep "File exists" $HOME/traf_temp_output | wc -l) if [ $dir_exists -eq 0 ]; then - echo "***ERROR: 'hadoop fs -mkdir /hbase-staging' command failed" + echo "***ERROR: '$HADOOP_BIN_PATH/hadoop fs -mkdir /hbase-staging' command failed" echo "***ERROR: $(cat $HOME/traf_temp_output)" exit -1 fi fi -sudo su $HDFS_USER --command "hadoop fs -chown -R $HBASE_USER:$HBASE_GROUP /hbase-staging" -sudo su $HDFS_USER --command "hadoop fs -mkdir /bulkload" 2> $HOME/traf_temp_output +sudo su $HDFS_USER --command "$HADOOP_BIN_PATH/hadoop fs -chown -R $HBASE_USER:$HBASE_GROUP /hbase-staging" +sudo su $HDFS_USER --command "$HADOOP_BIN_PATH/hadoop fs -mkdir /bulkload" 2> $HOME/traf_temp_output if [ $? != 0 ]; then # ok if directory already exists dir_exists=$(grep "File exists" $HOME/traf_temp_output | wc -l) if [ $dir_exists -eq 0 ]; then - echo "***ERROR: 'hadoop fs -mkdir /bulkload' command failed" + echo "***ERROR: '$HADOOP_BIN_PATH/hadoop fs -mkdir /bulkload' command failed" echo "***ERROR: $(cat $HOME/traf_temp_output)" exit -1 fi fi -sudo su $HDFS_USER --command "hadoop fs -chown -R $TRAF_USER:trafodion /bulkload" +sudo su $HDFS_USER --command "$HADOOP_BIN_PATH/hadoop fs -chown -R $TRAF_USER:trafodion /bulkload" # Create lobs directory -sudo su $HDFS_USER --command "hadoop fs -mkdir /lobs" 2> $HOME/traf_temp_output +sudo su $HDFS_USER --command "$HADOOP_BIN_PATH/hadoop fs -mkdir /lobs" 2> $HOME/traf_temp_output if [ $? != 0 ]; then # ok if directory already exists dir_exists=$(grep "File exists" $HOME/traf_temp_output | wc -l) if [ $dir_exists -eq 0 ]; then - echo "***ERROR: 'hadoop fs -mkdir /lobs' command failed" + echo "***ERROR: '$HADOOP_BIN_PATH/hadoop fs -mkdir /lobs' command failed" echo "***ERROR: $(cat $LOCAL_WORKDIR/traf_temp_output)" exit -1 fi fi -sudo su $HDFS_USER --command "hadoop fs -chown -R $TRAF_USER:trafodion /lobs" +sudo su $HDFS_USER --command "$HADOOP_BIN_PATH/hadoop fs -chown -R $TRAF_USER:trafodion /lobs" #===================================== # Modify hadoop settings as needed by Trafodion @@ -162,7 +163,7 @@ rm $HOME/traf_hdfs1_config_temp 2> /dev/null rm $HOME/traf_hbase_config_temp 2> /dev/null # change the hdfs configuration using Cloudera's REST API -curl -X PUT -H 'Content-Type:application/json' -u $ADMIN:$PASSWORD --data \ +curl -k -X PUT -H 'Content-Type:application/json' -u $ADMIN:$PASSWORD --data \ '{ "roleTypeConfigs" : [ { "roleType" : "NAMENODE", "items": [ { @@ -181,7 +182,7 @@ curl -X PUT -H 'Content-Type:application/json' -u $ADMIN:$PASSWORD --data \ "value":"true" } ] }' \ -http://$URL/api/v1/clusters/$CLUSTER_NAME/services/hdfs/config > $LOCAL_WORKDIR/traf_hdfs_config_temp +$URL/api/v1/clusters/$CLUSTER_NAME/services/hdfs/config > $LOCAL_WORKDIR/traf_hdfs_config_temp if [ $? != 0 ]; then echo "***ERROR: Unable to modify HDFS configuration through Cloudera's REST API." @@ -211,7 +212,7 @@ rm $LOCAL_WORKDIR/traf_hdfs_config_temp 2> /dev/null # NOTE: hbase.regionserver.lease.period is used as it is equivalent to # hbase.client.scanner.timeout.period and Cloudera only allows # hbase.regionserver.lease.period to be set through the REST API. -curl -X PUT -H 'Content-Type:application/json' -u $ADMIN:$PASSWORD --data \ +curl -k -X PUT -H 'Content-Type:application/json' -u $ADMIN:$PASSWORD --data \ '{ "roleTypeConfigs" : [ { "roleType" : "MASTER", "items" : [ { @@ -232,7 +233,7 @@ curl -X PUT -H 'Content-Type:application/json' -u $ADMIN:$PASSWORD --data \ } ] } ] }' \ -http://$URL/api/v1/clusters/$CLUSTER_NAME/services/$HBASE/config > $LOCAL_WORKDIR/traf_hbase_config_temp +$URL/api/v1/clusters/$CLUSTER_NAME/services/$HBASE/config > $LOCAL_WORKDIR/traf_hbase_config_temp if [ $? != 0 ]; then echo "***ERROR: Unable to modify HBase configuration through Cloudera's REST API." @@ -260,7 +261,7 @@ rm $LOCAL_WORKDIR/traf_hbase_config_temp 2> /dev/null # Change zookeeper config using Cloudera REST API -curl -X PUT -H 'Content-Type:application/json' -u $ADMIN:$PASSWORD --data \ +curl -k -X PUT -H 'Content-Type:application/json' -u $ADMIN:$PASSWORD --data \ '{ "roleTypeConfigs" : [ { "roleType" : "SERVER", "items": [ { @@ -270,7 +271,7 @@ curl -X PUT -H 'Content-Type:application/json' -u $ADMIN:$PASSWORD --data \ } ] }' \ -http://$URL/api/v1/clusters/$CLUSTER_NAME/services/zookeeper/config > $LOCAL_WORKDIR/traf_zookeeper_config_temp +$URL/api/v1/clusters/$CLUSTER_NAME/services/zookeeper/config > $LOCAL_WORKDIR/traf_zookeeper_config_temp # in most cases curl does not return an error # so curl's actual output needs to be checked, too @@ -296,8 +297,8 @@ rm $LOCAL_WORKDIR/traf_zookeeper_config_temp 2> /dev/null poll_time=30 echo "***INFO: restarting Hadoop to pickup Trafodion transaction jar" echo "***INFO: ...polling every $poll_time seconds until restart is completed." -restart_info=$(curl -X POST -u $ADMIN:$PASSWORD \ - http://$URL/api/v1/clusters/$CLUSTER_NAME/commands/restart) +restart_info=$(curl -k -X POST -u $ADMIN:$PASSWORD \ + $URL/api/v1/clusters/$CLUSTER_NAME/commands/restart) echo $restart_info command_id=$(echo $restart_info | grep id | awk '{print $4}' | sed -e 's@,@@' ) echo "***DEBUG: Cloudera command_id=$command_id" @@ -306,8 +307,8 @@ echo "***DEBUG: Cloudera command_id=$command_id" active=1 while [ $active -ne 0 ]; do sleep $poll_time - curl -u $ADMIN:$PASSWORD \ - http://$URL/api/v1/commands/$command_id \ + curl -k -u $ADMIN:$PASSWORD \ + $URL/api/v1/commands/$command_id \ > $LOCAL_WORKDIR/hbase_restart_status_temp cat $LOCAL_WORKDIR/hbase_restart_status_temp echo "***INFO: ...polling every $poll_time seconds until restart is completed." @@ -326,38 +327,60 @@ echo "***INFO: Hadoop restart completed successfully" # wait to make sure HDFS is fully restarted and out of safemode echo "***INFO: waiting for HDFS to exit safemode" -sudo su hdfs --command "hdfs dfsadmin -safemode wait" +sudo su hdfs --command "$HADOOP_BIN_PATH/hdfs dfsadmin -safemode wait" #==================================================== # NOTE: These command must be done AFTER acls are # enabled and HDFS has been restarted echo "***INFO: Setting HDFS ACLs for snapshot scan support" -sudo su hdfs --command "hdfs dfs -mkdir -p /hbase/archive" +sudo su hdfs --command "$HADOOP_BIN_PATH/hdfs dfs -mkdir -p /hbase/archive" +if [ $? != 0 ]; then + echo "***ERROR: ($HADOOP_BIN_PATH/hdfs dfs -mkdir -p /hbase/archive) command failed" + exit -1 +fi +sudo su hdfs --command "$HADOOP_BIN_PATH/hdfs dfs -chown hbase:hbase /hbase/archive" +if [ $? != 0 ]; then + echo "***ERROR: ($HADOOP_BIN_PATH/hdfs dfs -chown hbase:hbase /hbase/archive) command failed" + exit -1 +fi +sudo su hdfs --command "$HADOOP_BIN_PATH/hdfs dfs -setfacl -R -m user:$TRAF_USER:rwx /hbase/archive" if [ $? != 0 ]; then - echo "***ERROR: (hdfs dfs -mkdir -p /hbase/archive) command failed" + echo "***ERROR: ($HADOOP_BIN_PATH/hdfs dfs -setfacl -R -m user:$TRAF_USER:rwx /hbase/archive) command failed" exit -1 fi -sudo su hdfs --command "hdfs dfs -chown hbase:hbase /hbase/archive" +sudo su hdfs --command "$HADOOP_BIN_PATH/hdfs dfs -setfacl -R -m default:user:$TRAF_USER:rwx /hbase/archive" if [ $? != 0 ]; then - echo "***ERROR: (hdfs dfs -chown hbase:hbase /hbase/archive) command failed" + echo "***ERROR: ($HADOOP_BIN_PATH/hdfs dfs -setfacl -R -m default:user:$TRAF_USER:rwx /hbase/archive) command failed" exit -1 fi -sudo su hdfs --command "hdfs dfs -setfacl -R -m user:$TRAF_USER:rwx /hbase/archive" +sudo su hdfs --command "$HADOOP_BIN_PATH/hdfs dfs -setfacl -R -m mask::rwx /hbase/archive" if [ $? != 0 ]; then - echo "***ERROR: (hdfs dfs -setfacl -R -m user:$TRAF_USER:rwx /hbase/archive) command failed" + echo "***ERROR: ($HADOOP_BIN_PATH/hdfs dfs -setfacl -R -m mask::rwx /hbase/archive) command failed" exit -1 fi -sudo su hdfs --command "hdfs dfs -setfacl -R -m default:user:$TRAF_USER:rwx /hbase/archive" +sudo su hdfs --command "$HADOOP_BIN_PATH/hdfs dfs -mkdir -p /hbase/archive/data" if [ $? != 0 ]; then - echo "***ERROR: (hdfs dfs -setfacl -R -m default:user:$TRAF_USER:rwx /hbase/archive) command failed" + echo "***ERROR: ($HADOOP_BIN_PATH/hdfs dfs -mkdir -p /hbase/archive/data) command failed" exit -1 fi -sudo su hdfs --command "hdfs dfs -setfacl -R -m mask::rwx /hbase/archive" +sudo su hdfs --command "$HADOOP_BIN_PATH/hdfs dfs -chown hbase:hbase /hbase/archive/data" if [ $? != 0 ]; then - echo "***ERROR: (hdfs dfs -setfacl -R -m mask::rwx /hbase/archive) command failed" + echo "***ERROR: ($HADOOP_BIN_PATH/hdfs dfs -chown hbase:hbase /hbase/archive/data) command failed" exit -1 fi +sudo su hdfs --command "$HADOOP_BIN_PATH/hdfs dfs -mkdir -p /hbase/archive/data/default" +if [ $? != 0 ]; then + echo "***ERROR: ($HADOOP_BIN_PATH/hdfs dfs -mkdir -p /hbase/archive/data) command failed" + exit -1 +fi +sudo su hdfs --command "$HADOOP_BIN_PATH/hdfs dfs -chown hbase:hbase /hbase/archive/data/default" +if [ $? != 0 ]; then + echo "***ERROR: ($HADOOP_BIN_PATH/hdfs dfs -chown hbase:hbase /hbase/archive/data) command failed" + exit -1 +fi + + MODS_COMPLETE="Y" sudo chmod 777 $TRAF_CONFIG sed -i '/MODS_COMPLETE\=/d' $TRAF_CONFIG @@ -365,3 +388,14 @@ echo "export MODS_COMPLETE=\"$MODS_COMPLETE\"" >> $TRAF_CONFIG sudo chmod 777 $TRAF_CONFIG source $TRAF_CONFIG +TRAF_CONFIG_FILE="trafodion_config" +TRAF_CONFIG_DIR="/etc/trafodion" + +if [ $node_count -ne 1 ]; then + cp $TRAF_CONFIG $LOCAL_WORKDIR + $TRAF_PDCP $LOCAL_WORKDIR/$TRAF_CONFIG_FILE $HOME + $TRAF_PDSH sudo mkdir -p $TRAF_CONFIG_DIR + $TRAF_PDSH sudo cp $HOME/$TRAF_CONFIG_FILE $TRAF_CONFIG_DIR + $TRAF_PDSH sudo chmod 777 $TRAF_CONFIG +fi + http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ee4c2539/install/installer/traf_config ---------------------------------------------------------------------- diff --git a/install/installer/traf_config b/install/installer/traf_config index 57feb2d..be4cc1f 100755 --- a/install/installer/traf_config +++ b/install/installer/traf_config @@ -146,11 +146,7 @@ sed -e "s@export SQ_HOME=.*@export SQ_HOME=$SQ_ROOT@" <bashrc_temp >$HOME/.bashr cp .bashrc bashrc_temp sed -e "s@export TOOLSDIR=.*@export TOOLSDIR=$TRAF_TOOLSDIR@" <bashrc_temp >$HOME/.bashrc -# Delete any existing MY_NODES and NODE_LIST from .bashrc -#cp .bashrc bashrc_temp -#grep -vE "MY_NODES|NODE_LIST" bashrc_temp > $HOME/.bashrc - -if [[ $node_count == "1" ]]; then +if [[ $all_node_count == "1" ]]; then # since sqgen uses the existence of /usr/bin/pdsh to determine multi-node cluster # and sqgen uses the existence of /usr/bin/pdcp to determine if it needs to copy @@ -172,15 +168,15 @@ sed -e "s@export MY_NODES=.*@export MY_NODES=\"$MY_NODES\"@" <bashrc_temp >$HOME # re-source .bashrc to pick up these changes source ~/.bashrc -if [ "$node_count" -ne "1" ]; then +if [ "$all_node_count" -ne "1" ]; then echo "***INFO: copying .bashrc file to all nodes" | tee -a $INSTALL_LOG - $PDCP $MY_NODES -x $HOSTNAME $HOME/.bashrc $HOME + $PDCP $ALL_NODES -x $HOSTNAME $HOME/.bashrc $HOME fi #========================================== -if [ "$node_count" -ne "1" ]; then +if [ "$all_node_count" -ne "1" ]; then echo "***INFO: copying sqconfig file ($SQCONFIG_MASTER) to $SQ_ROOT/sql/scripts/sqconfig" | tee -a $INSTALL_LOG cp $SQCONFIG_MASTER $SQ_ROOT/sql/scripts/sqconfig fi @@ -189,12 +185,12 @@ cd $SQ_ROOT #================================================= -if [ "$node_count" -ne "1" ]; then +if [ "$all_node_count" -ne "1" ]; then echo "***INFO: Creating $SQ_ROOT directory on all nodes" | tee -a $INSTALL_LOG - $PDSH $MY_NODES -x $HOSTNAME $PDSH_SSH_CMD mkdir -p $SQ_ROOT + $PDSH $ALL_NODES -x $HOSTNAME $PDSH_SSH_CMD mkdir -p $SQ_ROOT if [[ -f $HOME/sqenvcom.sh ]]; then echo "****INFO: Copying over sqenvcom.sh" - $PDCP $MY_NODES $HOME/sqenvcom.sh $SQ_ROOT + $PDCP $ALL_NODES $HOME/sqenvcom.sh $SQ_ROOT fi fi @@ -219,79 +215,3 @@ if [ ! -z $REST_BUILD ]; then fi #========================================== -echo "***INFO: starting sqgen" | tee -a $INSTALL_LOG -cd $MY_SQROOT/sql/scripts -sqgen -SQGEN_RC=$? -if [ "$SQGEN_RC" != "0" ]; then - echo "***ERROR: sqgen failed with RC=$SQGEN_RC. Check install log file for details." | tee -a $INSTALL_LOG - exit -1; -fi -#========================================== - -if [ "$node_count" -ne "1" ]; then - echo "***INFO: copying $HOME/sqcert directory to all nodes" | tee -a $INSTALL_LOG - $PDCP $MY_NODES -x $HOSTNAME -r $HOME/sqcert $HOME -fi - -#========================================== - -if [ "$node_count" -ne "1" ]; then - echo "***INFO: copying install to all nodes" | tee -a $INSTALL_LOG - $PDCP $MY_NODES -x $HOSTNAME -r $SQ_ROOT $SQ_ROOT/.. - - if [ "$?" != "0" ]; then - echo "***ERROR: Unable to copy Trafodion install to all machines in this cluster. Check install log files for details." | tee -a $INSTALL_LOG - exit -1; - fi -fi - -#========================================= - # get first node listed in MY_NODES - - if [[ "$node_count" -eq "1" ]]; then - STARTING_NODE=$NODE_LIST - else - STARTING_NODE=$(echo $MY_NODES | awk '{print $2}') - fi - -#========================================== -#Copying traf_authentication_conf to scripts directory -if [[ "$LDAP_SECURITY" == "Y" ]]; then - if [ "$node_count" -ne "1" ]; then - echo "***INFO: Copying $LDAP_AUTH_FILE to all nodes" | tee -a $INSTALL_LOG - $PDCP $MY_NODES -r $HOME/$LDAP_AUTH_FILE $SQ_ROOT/sql/scripts/traf_authentication_config - if [ "$?" != "0" ]; then - echo "***ERROR: Unable to copy $LDAP_AUTH_FILE to all machines in this cluster. Check install log files for details." | tee -a $INSTALL_LOG - exit -1; - fi - else - echo "***INFO: Copying $LDAP_AUTH_FILE to scripts directory." | tee -a $INSTALL_LOG - cp -rf $HOME_DIR/$TRAF_USER/$LDAP_AUTH_FILE $SQ_ROOT/sql/scripts/traf_authentication_config - fi - - #Check traf_authentication_config for errors - echo "***INFO: Checking LDAP Configuration file for errors." - ldapconfigcheck -file $SQ_ROOT/sql/scripts/traf_authentication_config - if [ "$?" != "0" ]; then - echo "***ERROR: traf_authentication_config not configured correctly." - echo "***ERROR: Install will continue WITHOUT simple security turned on." - echo "***ERROR: Please review wiki for manual steps to setup simple security." - else - if [[ $LDAP_LEVEL == "1" ]]; then - echo "TLS_CACERT $HOME/$LDAP_CERT_BASE" > $HOME_DIR/$TRAF_USER/.ldaprc - echo "TLS_REQCERT demand" >> $HOME_DIR/$TRAF_USER/.ldaprc - fi - echo "***INFO: Enabling security. Running traf_authentication_setup" - ssh $STARTING_NODE "cd $MY_SQROOT/sql/scripts; traf_authentication_setup --setup --file traf_authentication_config" - - if [[ $? != "0" ]]; then - echo "***WARNING: Error during script traf_authentication_setup" - echo "***WARNING: Install will continue WITHOUT simple security turned on." - echo "***WARNING: Please review wiki for manual steps to setup simple security." - fi - fi -fi - -#========================================== -echo "***INFO: Installation setup completed successfully." | tee -a $INSTALL_LOG http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ee4c2539/install/installer/traf_config_check ---------------------------------------------------------------------- diff --git a/install/installer/traf_config_check b/install/installer/traf_config_check index 5b9eb49..d7bbee4 100755 --- a/install/installer/traf_config_check +++ b/install/installer/traf_config_check @@ -222,8 +222,135 @@ fi source $TRAF_CONFIG } -function checkJavaVersion { +function checkManageNodeList { + +if [[ -z $MANAGE_NODES ]]; then + errorFound=1 + echo "MANAGE NODE LIST" >> $ERROR_LOG + echo "***ERROR: MANAGE_NODES variable not set in config file." >> $ERROR_LOG +else + + ERROR_NODES="" + NODES="" + + for node in $MANAGE_NODES + do + newNode=$(ssh -q -n $node hostname) + + if [[ $? -ne "0" ]]; then + errorFound=1 + echo "MANAGE NODE LIST" >> $ERROR_LOG + echo "***ERROR: Could not ssh to $node." >> $ERROR_LOG + echo "***ERROR: Please check node names for typos" >> $ERROR_LOG + majorErrorFound + fi + + nodeName=$(echo $newNode | sed 's/[^a-zA-Z0-9\ \.\-]//g') + + if [[ "$nodeName" != "$newNode" ]]; then + errorFound=1 + echo "MANAGE NODE LIST" >> $ERROR_LOG + echo "***ERROR: Nodes are not named properly." >> $ERROR_LOG + echo "***ERROR: Nodes can have no special characters other than '-' and '.'" >> $ERROR_LOG + fi + + if [[ "$newNode" =~ ^([0-9]{1,3})[.]([0-9]{1,3})[.]([0-9]{1,3})[.]([0-9]{1,3})$ ]]; then + errorFound=1 + echo "MANAGE NODE LIST" >> $ERROR_LOG + echo "***ERROR: Nodes are not named properly." >> $ERROR_LOG + echo "***ERROR: IP addresses are not supported. Please use node names." >> $ERROR_LOG + fi + + #Testing sudo access on all nodes + ssh -q -n $newNode sudo echo "***INFO: Testing sudo access on node $newNode" + if [ $? -ne "0" ]; then + error=1 + ERROR_NODES="$ERROR_NODES $newNode" + fi + NODES="$NODES $newNode" + done + + NODE_LIST=$NODES + if [[ $error == "1" ]]; then + errorFound=1 + echo "MANAGE NODE LIST" >> $ERROR_LOG + echo "***ERROR: $ERROR_NODES does not have sudo access." >> $ERROR_LOG + echo "***ERROR: Must have sudo access on all nodes." >> $ERROR_LOG + fi + + + for node in $MANAGE_NODES + do + ssh -q -n $node echo "***INFO: Testing ssh on $node" + if [[ $? -ne "0" ]]; then + error=1 + ERROR_NODES="$ERROR_NODES $node" + fi + done + + if [[ $error == "1" ]]; then + errorFound=1 + echo "TRAFODION NODE LIST" >> $ERROR_LOG + echo "***ERROR: Could not ssh to $ERROR_NODES." >> $ERROR_LOG + echo "***ERROR: Check permissions and known hosts files." >> $ERROR_LOG + fi + + + sudo chmod 777 $TRAF_CONFIG + sed -i '/MANAGE_NODES\=/d' $TRAF_CONFIG + echo "export MANAGE_NODES=\"$MANAGE_NODES\"" >> $TRAF_CONFIG + sudo chmod 777 $TRAF_CONFIG + + manage_node_count=$(echo $MANAGE_NODES | wc -w) + + sudo chmod 777 $TRAF_CONFIG + sed -i '/manage_node_count\=/d' $TRAF_CONFIG + echo "export manage_node_count=\"$manage_node_count\"" >> $TRAF_CONFIG + sudo chmod 777 $TRAF_CONFIG + + MY_MANAGE_NODES="" + + for node in $MANAGE_NODES + do + MY_MANAGE_NODES="$MY_MANAGE_NODES -w $node" + done + + sudo chmod 777 $TRAF_CONFIG + sed -i '/MY_MANAGE_NODES\=/d' $TRAF_CONFIG + echo "export MY_MANAGE_NODES=\"$MY_MANAGE_NODES\"" >> $TRAF_CONFIG + sudo chmod 777 $TRAF_CONFIG + +fi +source $TRAF_CONFIG +} + +function createAllNodes { + +ALL_NODE_LIST="$NODE_LIST $MANAGE_NODES" + +sudo chmod 777 $TRAF_CONFIG +sed -i '/ALL_NODE_LIST\=/d' $TRAF_CONFIG +echo "export ALL_NODE_LIST=\"$ALL_NODE_LIST\"" >> $TRAF_CONFIG +sudo chmod 777 $TRAF_CONFIG + +all_node_count=$(echo $ALL_NODE_LIST | wc -w) + +sudo chmod 777 $TRAF_CONFIG +sed -i '/all_node_count\=/d' $TRAF_CONFIG +echo "export all_node_count=\"$all_node_count\"" >> $TRAF_CONFIG +sudo chmod 777 $TRAF_CONFIG + +ALL_NODES="$MY_MANAGE_NODES $MY_NODES" + +sudo chmod 777 $TRAF_CONFIG +sed -i '/ALL_NODES\=/d' $TRAF_CONFIG +echo "export ALL_NODES=\"$ALL_NODES\"" >> $TRAF_CONFIG +sudo chmod 777 $TRAF_CONFIG + +} + +function checkJavaVersion { if [[ -z "$JAVA_HOME" ]]; then errorFound=1 echo "JAVA HOME" >> $ERROR_LOG @@ -363,21 +490,20 @@ if [[ -z "$URL" ]]; then echo "***ERROR: Hadoop URL variable not set in config file." majorErrorFound else - validURL=$(curl -s --head $URL | head -n 1 | grep "OK" | wc -l) + validURL=$(curl -k -s --head $URL | head -n 1 | grep "OK" | wc -l) if [[ $validURL -ne "1" ]]; then errorFound=1 echo "HADOOP URL" >> $ERROR_LOG - echo "***ERROR: Could not access http://$URL"; >> $ERROR_LOG - echo "***ERROR: Do not include 'http or https'" >> $ERROR_LOG + echo "***ERROR: Could not access $URL" >> $ERROR_LOG echo "***ERROR: Check that URL and port are correct or if $HADOOP_TYPE is up" >> $ERROR_LOG majorErrorFound fi - hadoopVersion=$(curl -su $ADMIN:$PASSWORD http://$URL/api/v1/clusters | grep version | grep -c CDH) + hadoopVersion=$(curl -k -su $ADMIN:$PASSWORD $URL/api/v1/clusters | grep version | grep -c CDH) if [[ $hadoopVersion -ne "1" ]]; then - hadoopVersion=$(curl -su $ADMIN:$PASSWORD http://$URL/api/v1/clusters | grep version | grep -c HDP) + hadoopVersion=$(curl -k -su $ADMIN:$PASSWORD $URL/api/v1/clusters | grep version | grep -c HDP) if [[ $hadoopVersion -ne "1" ]]; then errorFound=1 echo "HADOOP URL" >> $ERROR_LOG @@ -396,14 +522,14 @@ else sudo chmod 777 $TRAF_CONFIG source $TRAF_CONFIG - temp=$(curl -su $ADMIN:$PASSWORD http://$URL/api/v1/clusters |grep name | sed -e 's@[,]@@'g | sed 's/^[^\:]* ://') + temp=$(curl -k -su $ADMIN:$PASSWORD $URL/api/v1/clusters |grep name | sed -e 's@[,]@@'g | sed 's/^[^\:]* ://') CLUSTER_NAME=$(echo $temp | sed -e 's/^"//' -e 's/"$//') CLUSTER_NAME=${CLUSTER_NAME// /%20} if [ -z $CLUSTER_NAME ]; then errorFound=1 echo "HADOOP URL" >> $ERROR_LOG - echo "***ERROR: Could not access http://$URL"; >> $ERROR_LOG + echo "***ERROR: Could not access $URL" >> $ERROR_LOG echo "***ERROR: Check URL and port are correct or if $hadoop_type is up." >> $ERROR_LOG majorErrorFound fi @@ -415,9 +541,10 @@ else source $TRAF_CONFIG if [[ $node_count -ne 1 ]]; then + nameOfInstallNode=$(hostname -s) sudo chmod 777 $TRAF_CONFIG sed -i '/CLUSTERNAME\=/d' $TRAF_CONFIG - echo "export CLUSTERNAME=\"$CLUSTER_NAME\"" >> $TRAF_CONFIG + echo "export CLUSTERNAME=\"$nameOfInstallNode\"" >> $TRAF_CONFIG sudo chmod 777 $TRAF_CONFIG source $TRAF_CONFIG fi @@ -471,7 +598,7 @@ done function checkRoleGroups { - regionGroup1=$(curl -su $ADMIN:$PASSWORD http://$URL/api/v10/clusters/$CLUSTER_NAME/services/hbase/roleConfigGroups | grep displayName | grep "RegionServer Group 1" | wc -l) + regionGroup1=$(curl -k -su $ADMIN:$PASSWORD $URL/api/v10/clusters/$CLUSTER_NAME/services/hbase/roleConfigGroups | grep displayName | grep "RegionServer Group 1" | wc -l) if [[ $regionGroup1 -ge "1" ]]; then errorFound=1 @@ -500,8 +627,8 @@ if [[ $CDH_5_3_HDP_2_2_SUPPORT == "N" ]]; then fi fi else - versionInstalled=$(ssh -q -n $node grep "Version" hbaseVersion.txt | sed 's/,.*//' | sed 's/.*\-//' | grep cdh5.[0-1].* | wc -l) - nameOfVersion=$(ssh -q -n $node grep "Version" hbaseVersion.txt | sed 's/,.*//' | sed 's/.*\-//' | grep cdh5.[0-1].*) + versionInstalled=$(ssh -q -n $node grep "Version" $HOME/hbaseVersion.txt | sed 's/,.*//' | sed 's/.*\-//' | grep cdh5.[0-1].* | wc -l) + nameOfVersion=$(ssh -q -n $node grep "Version" $HOME/hbaseVersion.txt | sed 's/,.*//' | sed 's/.*\-//' | grep cdh5.[0-1].*) if [[ $versionInstalled -eq "0" ]]; then errorFound=1 echo "HADOOP VERSION" >> $ERROR_LOG @@ -511,10 +638,10 @@ if [[ $CDH_5_3_HDP_2_2_SUPPORT == "N" ]]; then fi fi else - nameOfVersion=$(ssh -q -n $node grep "Version" hbaseVersion.txt | sed 's/,.*//' | sed 's/.*\-//' | grep cdh5.[2-3].*) + nameOfVersion=$(ssh -q -n $node grep "Version" $HOME/hbaseVersion.txt | sed 's/,.*//' | sed 's/.*\-//' | grep cdh5.[2-3].*) #Check that Cloudera 5.[n>3].* is not installed. if [[ -z $nameOfVersion ]]; then - versionInstalled=$(ssh -q -n $node grep "Version" hbaseVersion.txt | sed 's/,.*//' | sed 's/.*\-//' | grep cdh[4-6].[0-9].* | wc -l) + versionInstalled=$(ssh -q -n $node grep "Version" $HOME/hbaseVersion.txt | sed 's/,.*//' | sed 's/.*\-//' | grep cdh[4-6].[0-9].* | wc -l) if [[ $versionInstalled -gt "0" ]]; then errorFound=1 echo "HADOOP VERSION" >> $ERROR_LOG @@ -700,8 +827,8 @@ fi function checkHadoopSupport { -if [[ -z $TRAF_BUILD ]]; then - TRAF_BUILD=$(tar -tf $TRAF_PACKAGE | grep "trafodion_server") +if [[ -z $TRAF_BUILD ]] || [[ ! -e $TRAF_BUILD ]] ; then + TRAF_BUILD=$(tar -tf $TRAF_PACKAGE | grep "trafodion_.*server") tar -xzf $TRAF_PACKAGE --directory $LOCAL_WORKDIR TRAF_BUILD_PATH=$LOCAL_WORKDIR/$TRAF_BUILD else @@ -769,6 +896,12 @@ checkTrafPassword checkNodeList +if [[ $MANAGE_ENABLED == "Y" ]]; then + checkManageNodeList +fi + +createAllNodes + checkJavaVersion checkHomeDir @@ -794,3 +927,4 @@ checkDCS checkLDAP majorErrorFound + http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ee4c2539/install/installer/traf_config_setup ---------------------------------------------------------------------- diff --git a/install/installer/traf_config_setup b/install/installer/traf_config_setup index 209e6bb..e67958c 100755 --- a/install/installer/traf_config_setup +++ b/install/installer/traf_config_setup @@ -149,6 +149,10 @@ if [ -f $LOCAL_TRAF_CONFIG ]; then source $LOCAL_TRAF_CONFIG fi +if [[ -f $TRAF_CONFIG ]]; then + source $TRAF_CONFIG +fi + if [[ "$suseLinux" -ge "1" ]]; then SUSE_LINUX="true" else @@ -186,7 +190,7 @@ if [[ "$CLOUD_CONFIG" == "Y" ]]; then CLOUD_TYPE=$answer1 fi echo "export CLOUD_TYPE=\"$CLOUD_TYPE\"" >> $LOCAL_TRAF_CONFIG - case "$CLOUD_TYPE" in + case "$CLOUD_TYPE" in 1) AWS_CLOUD=true echo "export AWS_CLOUD=\"$AWS_CLOUD\"" >> $LOCAL_TRAF_CONFIG ;; @@ -211,7 +215,7 @@ fi #============================================== # List of Nodes -echo -n "Enter list of nodes (blank separated), default [$NODE_LIST]: " +echo -n "Enter list of Trafodion nodes (blank separated), default [$NODE_LIST]: " read answer if [[ -z "$answer" ]]; then if [ -z "$NODE_LIST" ]; then @@ -226,6 +230,33 @@ echo "export NODE_LIST=\"$NODE_LIST\"" >> $LOCAL_TRAF_CONFIG node_count=$(echo $NODE_LIST | wc -w) #============================================== +echo -n "Are management nodes enabled (Y/N), default is N: " +read answer + +if [ -z $answer ]; then + echo "export MANAGE_ENABLED=\"N\"" >> $LOCAL_TRAF_CONFIG +else + if [[ "${answer}" =~ ^[Yy]$ ]]; then + echo "export MANAGE_ENABLED=\"Y\"" >> $LOCAL_TRAF_CONFIG + #List of Management Nodes + echo -n "Enter list management nodes (blank separated), default [$MANAGE_NODES]: " + read answer + if [[ -z "$answer" ]]; then + if [ -z "$MANAGE_NODES" ]; then + echo "***ERROR: Must enter list of management nodes." + exit -1 + fi + else + MANAGE_NODES="$answer" + fi + + echo "export MANAGE_NODES=\"$MANAGE_NODES\"" >> $LOCAL_TRAF_CONFIG + else + echo "export MANAGE_ENABLED=\"N\"" >> $LOCAL_TRAF_CONFIG + fi +fi + +#============================================== # Home directory if [[ "$userForTrafodion" == "0" ]]; then echo -n "Enter Trafodion userid's home directory prefix, default is [$HOME_DIR]: " @@ -328,7 +359,6 @@ if [ $package -eq 0 ]; then echo "***ERROR: $DCS_BUILD does not exist" exit -1 fi - #TODO: do a quick sanity check on the file to make sure it is a DCS build tar file # since it is not a package tar file, we'll need to prompt for the rest build file @@ -396,7 +426,7 @@ fi #============================================== #Hadoop URL -echo -n "Enter Hadoop external network URL:port (no 'http://' needed), default is [$URL]: " +echo -n "Enter full Hadoop external network URL:port (include 'http://' or 'https://), default is [$URL]: " read answer if [ -z $answer ]; then @@ -408,13 +438,12 @@ else URL=$answer fi -validURL=$(curl -s --head $URL | head -n 1 | grep "OK" | wc -l) +validURL=$(curl -k -s --head $URL | head -n 1 | grep "OK" | wc -l) if [[ $validURL -eq "1" ]]; then echo "export URL=\"$URL\"" >> $LOCAL_TRAF_CONFIG else - echo "***ERROR: Could not access http://$URL"; - echo "***ERROR: Do not include 'http or https'" + echo "***ERROR: Could not access $URL" echo "***ERROR: Check that URL and port are correct or if $HADOOP_TYPE is up" exit -1 fi @@ -514,6 +543,7 @@ fi echo "export DCS_SERVERS_PARM=\"$DCS_SERVERS_PARM\"" >> $LOCAL_TRAF_CONFIG #============================================== + #Get the node name where the customer would like to start DcsMaster process echo -n "Enter the node of primary DcsMaster, default [$DCS_PRIMARY_MASTER_NODE]: " @@ -528,7 +558,7 @@ echo "export DCS_PRIMARY_MASTER_NODE=\"$DCS_PRIMARY_MASTER_NODE\"" >> $LOCAL_TRA #============================================== #Enable HA -echo -n "Enable High Availability (Y/N), default is N: " +echo -n "Enable High Avalability (Y/N), default is N: " read answer if [[ ! -z $answer ]]; then @@ -544,9 +574,9 @@ fi echo "export ENABLE_HA=\"$ENABLE_HA\"" >> $LOCAL_TRAF_CONFIG #============================================== -#Get the floating IP address, interface name and the list of backup nodes to be used +#Add Floating IP if HA Enabled if [[ "$ENABLE_HA" == "true" ]]; then - ######Get the floating IP address + ######Get the floating IP address echo -n "Enter floating IP address for setting up HA: " read answer1 @@ -572,16 +602,16 @@ if [[ "$ENABLE_HA" == "true" ]]; then ######Get the list of backup nodes to be used echo -n "Enter backup nodes for HA (blank seperated): " - read answer3 - - if [[ -z "$answer3" ]]; then + read answer3 + + if [[ -z "$answer3" ]]; then echo "Backup nodes must be specified." exit 1 else BACKUP_DCS_NODES="$answer3" fi echo "export BACKUP_DCS_NODES=\"$BACKUP_DCS_NODES\"" >> $LOCAL_TRAF_CONFIG -fi +fi #============================================== #Simple security @@ -678,7 +708,7 @@ if [[ $SCANNER_MODE == "N" ]]; then read answer4 if [[ -z "$answer4" ]]; then if [ -z "$LDAP_LEVEL" ]; then - echo "***ERROR: Must enter LDAP Encryption level." + echo "***ERROR: Much enter LDAP Encryption level." exit -1 fi else http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ee4c2539/install/installer/traf_create_systemdefaults ---------------------------------------------------------------------- diff --git a/install/installer/traf_create_systemdefaults b/install/installer/traf_create_systemdefaults deleted file mode 100755 index 8901207..0000000 --- a/install/installer/traf_create_systemdefaults +++ /dev/null @@ -1,203 +0,0 @@ -#!/bin/bash -# @@@ START COPYRIGHT @@@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# @@@ END COPYRIGHT @@@ -# -# This script will create the SQSystemdefaults.conf file -# by extracting info from Cloudera or Hortonworks REST API. -# -# NOTE: Currently only works for Cloudera installations -# - -USER_NAME="admin" -PASSWORD="admin" - - -CLOUDERA=$(rpm -qa | grep cloudera | wc -l) -AMBARI=$(rpm -qa | grep ambari | wc -l) - -if [[ $CLOUDERA -ne 0 ]]; then - typeset URL_PORT=$(hostname --long):7180 -fi - -if [[ $AMBARI -ne 0 ]]; then - typeset URL_PORT=$(hostname --long):8080 -fi - -mkdir -p $MY_SQROOT/etc -SYS_DEFAULTS=$MY_SQROOT/etc/SQSystemDefaults.conf - -function displayHelp -{ - echo - echo "This script will create the SQSystemdefaults.conf file." - echo - echo "Usage: $(basename $0) [options]" - echo - echo "Options:" - echo " --help Print this message and exit." - echo " --cloudera_userid Cloudera username if not chosen will default to admin" - echo " --cloudera_password Cloudera password if not chosen will default to admin" - echo " --cloudera_url Cloudera url:port if not chosen will default to hostname:7180" - echo " --ambari_userid Ambari username if not chosen will default to admin" - echo " --ambari_password Ambari password if not chosen will default to admin" - echo " --ambari_url Ambari url:port if not chosen will default to hostname:8080" - echo - -} - -while [ $# -gt 0 ]; do - - case $1 in - --cloudera_userid) - shift - USER_NAME=$1 - ;; - - --cloudera_password) - shift - PASSWORD=$1 - ;; - - --cloudera_url) - shift - URL_PORT=$1 - ;; - - --cluster_name) - shift - CLUSTER_NAME=$1 - ;; - - --ambari_userid) - shift - USER_NAME=$1 - ;; - - --ambari_password) - shift - PASSWORD=$1; - ;; - - --ambari_url) - shift - URL_PORT=$1 - ;; - - --help) - shift - displayHelp - exit - ;; - - **) - echo "Error! Incorrect Command." - exit - ;; - - esac - shift -done - -#================================= -# These functions extract either the value or default-value -# for a given Hive or Hbase configuration property -function get_hive_config() -{ - cat $MY_SQROOT/traf_hive_conf | \ - grep -A2 $1 | \ - grep --max-count=1 -E "default|value" | \ - sed -e 's@[",:]@@g' | \ - awk '{print $2}' -} - -function get_hbase_config() -{ - cat $MY_SQROOT/traf_hbase_conf | \ - grep -A2 $1 | \ - grep --max-count=1 -E "default|value" | \ - sed -e 's@[",:]@@g' | \ - awk '{print $2}' -} -#================================= - -# Get the hive & hbase configurations from Cloudera using REST API -rm $MY_SQROOT/traf_hive_config_temp 2> /dev/null -rm $MY_SQROOT/traf_hbase_config_temp 2> /dev/null -curl -u "$USER_NAME:$PASSWORD" \ - http://$URL_PORT/api/v1/clusters/$CLUSTER_NAME/services/hive1/config?view=full \ - > $MY_SQROOT/traf_hive_config_temp -if [ $? != "0" ]; then - echo "***ERROR: unable to get hive configuration info from Cloudera. Check that Cloudera is up." - exit -1 -fi - -# In most cases curl does not return an error -# so curl's actual output needs to be checked, too -curl_error=$(grep TITLE $MY_SQROOT/traf_hive_config_temp | grep Error | wc -l) -if [ $curl_error -ne 0 ]; then - echo "***ERROR: unable to get hive configuration info from Cloudera. Check that Cloudera is up." - exit -1 -fi - -curl -u "$USER_NAME:$PASSWORD" \ - http://$URL_PORT/api/v1/clusters/$CLUSTER_NAME/services/hbase1/config?view=full \ - > $MY_SQROOT/traf_hbase_config_temp -if [ $? != "0" ]; then - echo "***ERROR: unable to get hive configuration info from Cloudera. Check that Cloudera is up." - exit -1 -fi - -# In most cases curl does not return an error -# so curl's actual output needs to be checked, too. -curl_error=$(grep TITLE $MY_SQROOT/traf_hbase_config_temp | grep Error | wc -l) -if [ $curl_error -ne 0 ]; then - echo "***ERROR: unable to get hbase configuration info from Cloudera. Check that Cloudera is up." - exit -1 -fi - - -# Pull out just the names, values and defaults to make parsing the -# JSON output a little easier. NOTE: there isn't always a 'value' line -grep -E "\"name\"|\"value\"|\"default\"" $MY_SQROOT/traf_hive_config_temp > $MY_SQROOT/traf_hive_conf -grep -E "\"name\"|\"value\"|\"default\"" $MY_SQROOT/traf_hbase_config_temp > $MY_SQROOT/traf_hbase_conf -rm $MY_SQROOT/traf_hive_config_temp -rm $MY_SQROOT/traf_hbase_config_temp - -# Get all the settings we need -HBASE_THRIFT_PORT=$(get_hbase_config "hbase_thriftserver_port") -HIVE_HOST=$(get_hive_config "hive_metastore_database_host") -HIVE_PORT=$(get_hive_config "hive_metastore_database_port") -HIVE_PASSWORD=$(get_hive_config "hive_metastore_database_password") -HIVE_USER=$(get_hive_config "hive_metastore_database_user") -HIVE_METASTORE_NAME=$(get_hive_config "hive_metastore_database_name") -HIVE_TYPE=$(get_hive_config "hive_metastore_database_type") - -# Write them out to the conf file -echo "HBASE_THRIFT_PORT, $HBASE_THRIFT_PORT" > $SYS_DEFAULTS -echo "HIVE_METADATA_CPCC_URL, tcp://$HIVE_HOST:$HIVE_PORT" >> $SYS_DEFAULTS -echo "HIVE_METADATA_USER, $HIVE_USER" >> $SYS_DEFAULTS -echo "HIVE_METADATA_PASSWORD, $HIVE_PASSWORD" >> $SYS_DEFAULTS -echo "HIVE_METADATA_SCHEMA, $HIVE_METASTORE_NAME" >> $SYS_DEFAULTS - -echo "***INFO: generated $SYS_DEFAULTS file" -cat $SYS_DEFAULTS - -echo "***INFO: $SYS_DEFAULTS file created" http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ee4c2539/install/installer/traf_getHadoopNodes ---------------------------------------------------------------------- diff --git a/install/installer/traf_getHadoopNodes b/install/installer/traf_getHadoopNodes index d09f925..127d0c2 100755 --- a/install/installer/traf_getHadoopNodes +++ b/install/installer/traf_getHadoopNodes @@ -1,4 +1,4 @@ -# +#!/bin/bash # @@@ START COPYRIGHT @@@ # # Licensed to the Apache Software Foundation (ASF) under one @@ -21,8 +21,6 @@ # @@@ END COPYRIGHT @@@ # -#!/bin/bash - source /etc/trafodion/trafodion_config TRAF_CONFIG=/etc/trafodion/trafodion_config @@ -31,20 +29,22 @@ echo "***INFO: Getting list of all $HADOOP_TYPE nodes" curlRC=0 if [[ $HADOOP_TYPE == "cloudera" ]]; then - curl -su $ADMIN:$PASSWORD http://$URL/api/v6/cm/deployment > tempFile + curl -k -su $ADMIN:$PASSWORD $URL/api/v10/hosts > tempFile curlRC=$? numberHadoopNodes=$(grep -r "hostname" tempFile | wc -l) grep -r "hostname" tempFile > tempFile2 if [[ -d /opt/cloudera/parcels/CDH ]]; then HADOOP_PATH="/opt/cloudera/parcels/CDH/lib/hbase/lib" + HADOOP_BIN_PATH="/opt/cloudera/parcels/CDH/bin" else HADOOP_PATH="/usr/lib/hbase/lib" + HADOOP_BIN_PATH="/usr/bin" fi fi if [[ $HADOOP_TYPE == "hortonworks" ]]; then - curl -su $ADMIN:$PASSWORD http://$URL/api/v1/clusters/$CLUSTER_NAME/hosts > tempFile + curl -k -su $ADMIN:$PASSWORD $URL/api/v1/clusters/$CLUSTER_NAME/hosts > tempFile curlRC=$? numberHadoopNodes=$(grep -r "host_name" tempFile | wc -l) grep -r "host_name" tempFile > tempFile2 @@ -104,6 +104,9 @@ sudo chmod 777 $TRAF_CONFIG sed -i '/HADOOP_PATH\=/d' $TRAF_CONFIG echo "export HADOOP_PATH=\"$HADOOP_PATH\"" >> $TRAF_CONFIG sudo chmod 777 $TRAF_CONFIG +sed -i '/HADOOP_BIN_PATH\=/d' $TRAF_CONFIG +echo "export HADOOP_BIN_PATH=\"$HADOOP_BIN_PATH\"" >> $TRAF_CONFIG +sudo chmod 777 $TRAF_CONFIG sed -i '/hadoop_node_count\=/d' $TRAF_CONFIG echo "export hadoop_node_count=\"$hadoop_node_count\"" >> $TRAF_CONFIG http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ee4c2539/install/installer/traf_hortonworks_mods98 ---------------------------------------------------------------------- diff --git a/install/installer/traf_hortonworks_mods98 b/install/installer/traf_hortonworks_mods98 index c4b66a2..a849df7 100755 --- a/install/installer/traf_hortonworks_mods98 +++ b/install/installer/traf_hortonworks_mods98 @@ -66,8 +66,10 @@ else echo "***ERROR: unable to determine Hadoop's java version" exit -1 fi + traf_util_jar="trafodion-utility-${TRAF_VERSION}.jar" + # The permissions the Trafodion build process creates on the hbase-trx jar # files does not work well with the installation process so we change them sudo chmod -R 777 $UNTAR_DIR/export/lib @@ -93,9 +95,8 @@ if [ $node_count -ne 1 ]; then cp $UNTAR_DIR/export/lib/$traf_util_jar $LOCAL_WORKDIR $PDCP $MY_HADOOP_NODES $LOCAL_WORKDIR/$hbase_trx_jar $LOCAL_WORKDIR $PDCP $MY_HADOOP_NODES $LOCAL_WORKDIR/$traf_util_jar $LOCAL_WORKDIR - - $PDSH $MY_HADOOP_NODES $PDSH_SSH_CMD sudo cp $LOCAL_WORKDIR/$hbase_trx_jar $HADOOP_PATH $PDSH $MY_HADOOP_NODES $PDSH_SSH_CMD sudo cp $LOCAL_WORKDIR/$traf_util_jar $HADOOP_PATH + $PDSH $MY_HADOOP_NODES $PDSH_SSH_CMD sudo cp $LOCAL_WORKDIR/$hbase_trx_jar $HADOOP_PATH $PDSH $MY_HADOOP_NODES $PDSH_SSH_CMD sudo chmod 644 $HADOOP_PATH/$hbase_trx_jar $PDSH $MY_HADOOP_NODES $PDSH_SSH_CMD sudo chmod 644 $HADOOP_PATH/$traf_util_jar @@ -113,7 +114,6 @@ else ssh -q -n $node sudo chmod 777 $TRAF_WORKDIR scp -q $UNTAR_DIR/export/lib/$hbase_trx_jar $(whoami)@$node:$TRAF_WORKDIR scp -q $UNTAR_DIR/export/lib/$traf_util_jar $(whoami)@$node:$TRAF_WORKDIR - ssh -q -n $node sudo cp $TRAF_WORKDIR/$hbase_trx_jar $HADOOP_PATH ssh -q -n $node sudo cp $TRAF_WORKDIR/$traf_util_jar $HADOOP_PATH ssh -q -n $node sudo chmod 644 $HADOOP_PATH/$hbase_trx_jar @@ -271,10 +271,10 @@ echo poll_time=30 echo "***INFO: Restarting HBase to pick up config changes for Trafodion" echo "***INFO: Stopping HBase..." -curl -u $ADMIN:$PASSWORD \ +curl -k -u $ADMIN:$PASSWORD \ -H "X-Requested-By: Trafodion" \ -X PUT -d '{"ServiceInfo": { "state" : "INSTALLED" }}' \ - http://$URL/api/v1/clusters/$CLUSTER_NAME/services/HBASE > $TRAF_WORKDIR/traf_hbase_restart_temp + $URL/api/v1/clusters/$CLUSTER_NAME/services/HBASE > $TRAF_WORKDIR/traf_hbase_restart_temp if [ $? != 0 ]; then echo "***ERROR: Unable to stop HBase" @@ -298,8 +298,8 @@ echo "***DEBUG: Ambari command_id=$command_id" completed=0 while [ $completed -eq 0 ]; do sleep $poll_time - curl -u $ADMIN:$PASSWORD \ - http://$URL/api/v1/clusters/$CLUSTER_NAME/requests/$command_id \ + curl -k -u $ADMIN:$PASSWORD \ + $URL/api/v1/clusters/$CLUSTER_NAME/requests/$command_id \ > $LOCAL_WORKDIR/hbase_restart_status_temp cat $LOCAL_WORKDIR/hbase_restart_status_temp echo "***INFO: ...polling every $poll_time seconds until HBase stop is completed." @@ -313,10 +313,10 @@ echo "***INFO: HBase stop completed" #Stop Zookeeper echo "***INFO: Stopping Zookeeper..." -curl --user $ADMIN:$PASSWORD \ +curl -k --user $ADMIN:$PASSWORD \ -H "X-Requested-By: Trafodion" \ -X PUT -d '{"ServiceInfo": { "state" : "INSTALLED" }}' \ - http://$URL/api/v1/clusters/$CLUSTER_NAME/services/ZOOKEEPER > $TRAF_WORKDIR/traf_zoo_restart_temp + $URL/api/v1/clusters/$CLUSTER_NAME/services/ZOOKEEPER > $TRAF_WORKDIR/traf_zoo_restart_temp if [ $? != 0 ]; then echo "***ERROR: Unable to restart Zookeeper" @@ -340,8 +340,8 @@ echo "***DEBUG: Ambari command_id=$command_id" completed=0 while [ $completed -eq 0 ]; do sleep $poll_time - curl --user $ADMIN:$PASSWORD \ - http://$URL/api/v1/clusters/$CLUSTER_NAME/requests/$command_id \ + curl -k --user $ADMIN:$PASSWORD \ + $URL/api/v1/clusters/$CLUSTER_NAME/requests/$command_id \ > $LOCAL_WORKDIR/traf_zoo_restart_temp cat $LOCAL_WORKDIR/traf_zoo_restart_temp echo "***INFO: ...polling every $poll_time seconds until Zookeeper stop is completed." @@ -354,10 +354,10 @@ done echo "***INFO: Restarting HDFS to pick up config changes for Trafodion" echo "***INFO: Stopping HDFS..." -curl --user $ADMIN:$PASSWORD \ +curl -k --user $ADMIN:$PASSWORD \ -H "X-Requested-By: Trafodion" \ -X PUT -d '{"ServiceInfo": { "state" : "INSTALLED" }}' \ - http://$URL/api/v1/clusters/$CLUSTER_NAME/services/HDFS > $TRAF_WORKDIR/traf_hdfs_restart_temp + $URL/api/v1/clusters/$CLUSTER_NAME/services/HDFS > $TRAF_WORKDIR/traf_hdfs_restart_temp if [ $? != 0 ]; then echo "***ERROR: Unable to restart HDFS" @@ -381,8 +381,8 @@ echo "***DEBUG: Ambari command_id=$command_id" completed=0 while [ $completed -eq 0 ]; do sleep $poll_time - curl --user $ADMIN:$PASSWORD \ - http://$URL/api/v1/clusters/$CLUSTER_NAME/requests/$command_id \ + curl -k --user $ADMIN:$PASSWORD \ + $URL/api/v1/clusters/$CLUSTER_NAME/requests/$command_id \ > $LOCAL_WORKDIR/traf_hdfs_restart_temp cat $LOCAL_WORKDIR/traf_hdfs_restart_temp echo "***INFO: ...polling every $poll_time seconds until HDFS stop is completed." @@ -391,10 +391,10 @@ while [ $completed -eq 0 ]; do done echo "***INFO: Starting HDFS..." -curl --user $ADMIN:$PASSWORD \ +curl -k --user $ADMIN:$PASSWORD \ -H "X-Requested-By: Trafodion" \ -X PUT -d '{"ServiceInfo": { "state" : "STARTED" }}' \ - http://$URL/api/v1/clusters/$CLUSTER_NAME/services/HDFS > $TRAF_WORKDIR/traf_hdfs_restart_temp + $URL/api/v1/clusters/$CLUSTER_NAME/services/HDFS > $TRAF_WORKDIR/traf_hdfs_restart_temp if [ $? != 0 ]; then echo "***ERROR: Unable to restart HDFS" @@ -417,8 +417,8 @@ echo "***DEBUG: Ambari command_id=$command_id" completed=0 while [ $completed -eq 0 ]; do sleep $poll_time - curl --user $ADMIN:$PASSWORD \ - http://$URL/api/v1/clusters/$CLUSTER_NAME/requests/$command_id \ + curl -k --user $ADMIN:$PASSWORD \ + $URL/api/v1/clusters/$CLUSTER_NAME/requests/$command_id \ > $LOCAL_WORKDIR/traf_hdfs_restart_temp cat $LOCAL_WORKDIR/traf_hdfs_restart_temp echo "***INFO: ...polling every $poll_time seconds until HDFS start is completed." @@ -436,10 +436,10 @@ sudo su hdfs --command "hdfs dfsadmin -safemode wait" # Start Zookeeper to pick up all the changes just made echo "***INFO: Starting Zookeeper..." -curl --user $ADMIN:$PASSWORD \ +curl -k --user $ADMIN:$PASSWORD \ -H "X-Requested-By: Trafodion" \ -X PUT -d '{"ServiceInfo": { "state" : "STARTED" }}' \ - http://$URL/api/v1/clusters/$CLUSTER_NAME/services/ZOOKEEPER > $TRAF_WORKDIR/traf_zoo_restart_temp + $URL/api/v1/clusters/$CLUSTER_NAME/services/ZOOKEEPER > $TRAF_WORKDIR/traf_zoo_restart_temp if [ $? != 0 ]; then echo "***ERROR: Unable to restart Zookeeper" @@ -462,8 +462,8 @@ echo "***DEBUG: Ambari command_id=$command_id" completed=0 while [ $completed -eq 0 ]; do sleep $poll_time - curl --user $ADMIN:$PASSWORD \ - http://$URL/api/v1/clusters/$CLUSTER_NAME/requests/$command_id \ + curl -k --user $ADMIN:$PASSWORD \ + $URL/api/v1/clusters/$CLUSTER_NAME/requests/$command_id \ > $LOCAL_WORKDIR/traf_zoo_restart_temp cat $LOCAL_WORKDIR/traf_zoo_restart_temp echo "***INFO: ...polling every $poll_time seconds until Zookeeper start is completed." @@ -478,10 +478,10 @@ echo "***INFO: Zookeeper start completed" echo "***INFO: Restarting HBase to pick up config changes for Trafodion" echo "***INFO: Starting HBase..." -curl -u $ADMIN:$PASSWORD \ +curl -k -u $ADMIN:$PASSWORD \ -H "X-Requested-By: Trafodion" \ -X PUT -d '{"ServiceInfo": { "state" : "STARTED" }}' \ - http://$URL/api/v1/clusters/$CLUSTER_NAME/services/HBASE > $TRAF_WORKDIR/traf_hbase_restart_temp + $URL/api/v1/clusters/$CLUSTER_NAME/services/HBASE > $TRAF_WORKDIR/traf_hbase_restart_temp if [ $? != 0 ]; then echo "***ERROR: Unable to restart HBase" @@ -504,8 +504,8 @@ echo "***DEBUG: Ambari command_id=$command_id" completed=0 while [ $completed -eq 0 ]; do sleep $poll_time - curl -u $ADMIN:$PASSWORD \ - http://$URL/api/v1/clusters/$CLUSTER_NAME/requests/$command_id \ + curl -k -u $ADMIN:$PASSWORD \ + $URL/api/v1/clusters/$CLUSTER_NAME/requests/$command_id \ > $LOCAL_WORKDIR/hbase_restart_status_temp cat $LOCAL_WORKDIR/hbase_restart_status_temp echo "***INFO: ...polling every $poll_time seconds until HBase start is completed." @@ -545,6 +545,32 @@ if [ $? != 0 ]; then exit -1 fi +ssh -q -n $AMBARI_HOST 'sudo su hdfs --command "hdfs dfs -mkdir -p /apps/hbase/data/archive/data"' +if [ $? != 0 ]; then + echo "***ERROR: (hdfs dfs -mkdir -p /apps/hbase/data/archive/data) command failed" + exit -1 +fi + +ssh -q -n $AMBARI_HOST 'sudo su hdfs --command "hdfs dfs -chown hbase:hdfs /apps/hbase/data/archive/data"' +if [ $? != 0 ]; then + echo "***ERROR: (hdfs dfs -chown hbase:hdfs /apps/hbase/data/archive/data) command failed" + exit -1 +fi + +ssh -q -n $AMBARI_HOST 'sudo su hdfs --command "hdfs dfs -mkdir -p /apps/hbase/data/archive/data/default"' +if [ $? != 0 ]; then + echo "***ERROR: (hdfs dfs -mkdir -p /apps/hbase/data/archive/data/default) command failed" + exit -1 +fi + +ssh -q -n $AMBARI_HOST 'sudo su hdfs --command "hdfs dfs -chown hbase:hdfs /apps/hbase/data/archive/data/default"' +if [ $? != 0 ]; then + echo "***ERROR: (hdfs dfs -chown hbase:hdfs /apps/hbase/data/archive/data/default) command failed" + exit -1 +fi + + + # clean up files generated by Ambari's config.sh script ssh -q -n $AMBARI_HOST 'rm $HOME/doSet_version*' @@ -556,3 +582,14 @@ echo "export MODS_COMPLETE=\"$MODS_COMPLETE\"" >> $TRAF_CONFIG sudo chmod 777 $TRAF_CONFIG source $TRAF_CONFIG +TRAF_CONFIG="trafodion_config" +TRAF_CONFIG_DIR="/etc/trafodion" + +if [ $node_count -ne 1 ]; then + cp $TRAF_CONFIG $LOCAL_WORKDIR + $TRAF_PDCP $LOCAL_WORKDIR/$TRAF_CONFIG_FILE $HOME + $TRAF_PDSH sudo mkdir -p $TRAF_CONFIG_DIR + $TRAF_PDSH sudo cp $HOME/$TRAF_CONFIG_FILE $TRAF_CONFIG_DIR + $TRAF_PDSH sudo chmod 777 $TRAF_CONFIG +fi + http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ee4c2539/install/installer/traf_package_setup ---------------------------------------------------------------------- diff --git a/install/installer/traf_package_setup b/install/installer/traf_package_setup index 7f515f1..4e10834 100755 --- a/install/installer/traf_package_setup +++ b/install/installer/traf_package_setup @@ -40,12 +40,12 @@ echo >> $INSTALL_LOG echo "***INFO: Starting Trafodion Package Setup ($timestamp)" #======================================== #Setting up pdsh variables -if [ $node_count -eq 1 ]; then +if [ $all_node_count -eq 1 ]; then TRAF_PDSH="" TRAF_PDCP="" else - TRAF_PDSH="pdsh -R exec $MY_NODES ssh -q -n %h" - TRAF_PDCP="pdcp -R ssh $MY_NODES" + TRAF_PDSH="pdsh -R exec $ALL_NODES ssh -q -n %h" + TRAF_PDCP="pdcp -R ssh $ALL_NODES" fi #========================================== echo "***INFO: Installing required packages" @@ -60,7 +60,7 @@ internetAccess="true" #================================ #Checking to see if epel package is installed. if [[ $SUSE_LINUX == "false" ]]; then - for node in $NODE_LIST; + for node in $ALL_NODE_LIST; do EPEL_INSTALLED=$(ssh -q -n $node rpm -qa | grep epel | wc -l) if [[ $EPEL_INSTALLED == 0 ]]; then @@ -81,8 +81,8 @@ if [[ $SUSE_LINUX == "false" ]]; then exit -1 fi - if [ $node_count -ne 1 ]; then - for node in $NODE_LIST + if [ $all_node_count -ne 1 ]; then + for node in $ALL_NODE_LIST do scp -q $LOCAL_WORKDIR/$epel_rpm $(whoami)@$node:$HOME if [[ $? -gt 1 ]]; then @@ -95,7 +95,7 @@ if [[ $SUSE_LINUX == "false" ]]; then fi fi - for node in $NODE_LIST; + for node in $ALL_NODE_LIST; do EPEL_INSTALLED=$( ssh -q -n $node rpm -qa | grep epel | wc -l) if [[ $EPEL_INSTALLED == 0 ]]; then @@ -120,8 +120,8 @@ fi #install pdsh if not on SUSE Linux if [[ $SUSE_LINUX == "false" ]]; then - if [ $node_count -ne 1 ]; then - for node in $NODE_LIST + if [ $all_node_count -ne 1 ]; then + for node in $ALL_NODE_LIST do echo "***INFO: ... pdsh on node $node" #if not already installed on this node, then install it @@ -165,7 +165,7 @@ if [[ $SUSE_LINUX == "false" ]]; then for package in $package_list do echo "***INFO: Checking if $package is installed ..." - for node in $NODE_LIST + for node in $ALL_NODE_LIST do installed=`ssh -q -n $node sudo rpm -qa | grep $package | wc -l` if [[ $installed -eq "0" ]]; then @@ -179,7 +179,7 @@ if [[ $SUSE_LINUX == "false" ]]; then done done else - for node in $NODE_LIST + for node in $ALL_NODE_LIST do echo "***ERROR: No internet access on $node" echo "***ERROR: Missing packages will need to be installed." http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ee4c2539/install/installer/traf_setup ---------------------------------------------------------------------- diff --git a/install/installer/traf_setup b/install/installer/traf_setup index 2d4dbc2..84dd919 100755 --- a/install/installer/traf_setup +++ b/install/installer/traf_setup @@ -69,45 +69,17 @@ if [ "$USER" == "$TRAF_USER" ]; then fi #================================ -# validate specified nodes and that we can get to them with passwordless ssh -echo "***INFO: Checking all nodes in specified node list" -bad_nodes="" -MY_NODES="" - -for node in $NODE_LIST -do - # use BatchMode so it will just give an error instead of prompting for password - ssh -q -oStrictHostKeyChecking=no -oBatchMode=yes $node hostname - if [ $? != 0 ]; then - bad_nodes="$bad_nodes $node" - fi - # build MY_NODES env var to be used for pdsh & pdcp commands - MY_NODES="$MY_NODES -w $node" -done -if [ -n "$bad_nodes" ]; then - echo "***ERROR: unable to access all nodes in the node list with passwordless ssh" - echo "***ERROR: problem nodes: $bad_nodes" - exit -1 -fi - -echo "***INFO: Total number of nodes = $node_count" -if [ $node_count -eq 1 ]; then +if [ $all_node_count -eq 1 ]; then TRAF_PDSH="" TRAF_PDCP="" else # use the -S option to cause pdsh to return largest of # the remote command return values so we can tell if one # or more of the remote commands failed - TRAF_PDSH="pdsh -R exec $MY_NODES ssh -q -n %h" - TRAF_PDCP="pdcp -R ssh $MY_NODES" + TRAF_PDSH="pdsh -R exec $ALL_NODES ssh -q -n %h" + TRAF_PDCP="pdcp -R ssh $ALL_NODES" fi #========================================== -#Determine Internet Access - -#>>>>>Check for internet access -internetAccess=true - -#========================================== #installing required packages" $LOCAL_WORKDIR/traf_package_setup @@ -140,7 +112,7 @@ echo "$TRAF_USER soft nofile 8192" >> $TRAF_LIMITS_CONF echo "$TRAF_USER hard nofile 65535" >> $TRAF_LIMITS_CONF # copy to all nodes -if [ $node_count -eq 1 ]; then +if [ $all_node_count -eq 1 ]; then sudo cp $TRAF_LIMITS_CONF /etc/security/limits.d/trafodion.conf if [ $? != 0 ]; then echo "***ERROR: unable to copy $TRAF_LIMITS_CONF to /etc/security/limits.d/trafodion.conf" http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ee4c2539/install/installer/traf_sqgen ---------------------------------------------------------------------- diff --git a/install/installer/traf_sqgen b/install/installer/traf_sqgen new file mode 100755 index 0000000..d0feb01 --- /dev/null +++ b/install/installer/traf_sqgen @@ -0,0 +1,96 @@ +#!/bin/bash +# @@@ START COPYRIGHT @@@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# @@@ END COPYRIGHT @@@ + +export TRAF_CONFIG=/etc/trafodion/trafodion_config + +source $TRAF_CONFIG + +STARTING_NODE=$(hostname) + +#========================================== +echo "***INFO: starting sqgen" | tee -a $INSTALL_LOG +cd $SQ_ROOT/sql/scripts +$SQ_ROOT/sql/scripts/sqgen +SQGEN_RC=$? +if [ "$SQGEN_RC" != "0" ]; then + echo "***ERROR: sqgen failed with RC=$SQGEN_RC. Check install log file for details." | tee -a $INSTALL_LOG + exit -1; +fi +#========================================== + +if [ "$node_count" -ne "1" ]; then + echo "***INFO: copying $HOME/sqcert directory to all nodes" | tee -a $INSTALL_LOG + $PDCP $ALL_NODES -x $HOSTNAME -r $HOME/sqcert $HOME +fi + +#========================================== + +if [ "$all_node_count" -ne "1" ]; then + echo "***INFO: copying install to all nodes" | tee -a $INSTALL_LOG + $PDCP $ALL_NODES -x $HOSTNAME -r $SQ_ROOT $SQ_ROOT/.. + + if [ "$?" != "0" ]; then + echo "***ERROR: Unable to copy Trafodion install to all machines in this cluster. Check install log files for details." | tee -a $INSTALL_LOG + exit -1; + fi +fi + +#========================================= +#Copying traf_authentication_conf to scripts directory +if [[ "$LDAP_SECURITY" == "Y" ]]; then + if [ "$all_node_count" -ne "1" ]; then + echo "***INFO: Copying $LDAP_AUTH_FILE to all nodes" | tee -a $INSTALL_LOG + $PDCP $ALL_NODES -r $HOME/$LDAP_AUTH_FILE $SQ_ROOT/sql/scripts/traf_authentication_config + if [ "$?" != "0" ]; then + echo "***ERROR: Unable to copy $LDAP_AUTH_FILE to all machines in this cluster. Check install log files for details." | tee -a $INSTALL_LOG + exit -1; + fi + else + echo "***INFO: Copying $LDAP_AUTH_FILE to scripts directory." | tee -a $INSTALL_LOG + cp -rf $HOME_DIR/$TRAF_USER/$LDAP_AUTH_FILE $SQ_ROOT/sql/scripts/traf_authentication_config + fi + + #Check traf_authentication_config for errors + echo "***INFO: Checking LDAP Configuration file for errors." + ldapconfigcheck -file $SQ_ROOT/sql/scripts/traf_authentication_config + if [ "$?" != "0" ]; then + echo "***ERROR: traf_authentication_config not configured correctly." + echo "***ERROR: Install will continue WITHOUT simple security turned on." + echo "***ERROR: Please review wiki for manual steps to setup simple security." + else + if [[ $LDAP_LEVEL == "1" ]]; then + echo "TLS_CACERT $HOME/$LDAP_CERT_BASE" > $HOME_DIR/$TRAF_USER/.ldaprc + echo "TLS_REQCERT demand" >> $HOME_DIR/$TRAF_USER/.ldaprc + fi + echo "***INFO: Enabling security. Running traf_authentication_setup" + ssh $STARTING_NODE "cd $MY_SQROOT/sql/scripts; traf_authentication_setup --setup --file traf_authentication_config" + + if [[ $? != "0" ]]; then + echo "***WARNING: Error during script traf_authentication_setup" + echo "***WARNING: Install will continue WITHOUT simple security turned on." + echo "***WARNING: Please review wiki for manual steps to setup simple security." + fi + fi +fi + +#========================================== + http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ee4c2539/install/installer/traf_start ---------------------------------------------------------------------- diff --git a/install/installer/traf_start b/install/installer/traf_start index 5f1b44b..8c459c0 100755 --- a/install/installer/traf_start +++ b/install/installer/traf_start @@ -1,7 +1,5 @@ #!/bin/bash -#!/bin/bash - # @@@ START COPYRIGHT @@@ # # Licensed to the Apache Software Foundation (ASF) under one @@ -23,21 +21,19 @@ # # @@@ END COPYRIGHT @@@ -#================================================= -#Start Trafodion -#================================================= export TRAF_CONFIG=/etc/trafodion/trafodion_config source $TRAF_CONFIG -if [[ "$node_count" -eq "1" ]]; then +if [[ "$all_node_count" -eq "1" ]]; then STARTING_NODE=$NODE_LIST else STARTING_NODE=$(echo $MY_NODES | awk '{print $2}') fi +#========================================== if [ "$START" == "N" ]; then echo "***INFO: Trafodion will not be started, skipping sqstart." | tee -a $INSTALL_LOG @@ -48,10 +44,15 @@ else SQSTART_RC=$? if [ "$SQSTART_RC" != "0" ]; then echo "***ERROR: sqstart failed with RC=$SQSTART_RC. Check $SQ_ROOT/sqmon.log file for details." | tee -a $INSTALL_LOG - echo "***ERROR: Consider running trafodion_scanner, to assist in debugging." | tee -a $INSTALL_LOG exit -1; fi + if [[ $MANAGE_ENABLED == "Y" ]]; then + ssh $STARTING_NODE "dcsstop" + sleep 15 + ssh $MANAGE_NODES "dcsstart" + fi + # Do init_trafodion if specified if [ "$INIT_TRAFODION" == "Y" ] && [ "$UPGRADE_TRAF" == "false" ]; then ssh $STARTING_NODE "cd $SQ_ROOT/sql/scripts; echo \"initialize Trafodion;\" | sqlci" | tee -a $SQ_ROOT/initializeErrors.txt @@ -78,3 +79,5 @@ else fi fi fi +#========================================== +echo "***INFO: Installation setup completed successfully." | tee -a $INSTALL_LOG http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ee4c2539/install/installer/trafodion_config_default ---------------------------------------------------------------------- diff --git a/install/installer/trafodion_config_default b/install/installer/trafodion_config_default index 8e32879..c807752 100755 --- a/install/installer/trafodion_config_default +++ b/install/installer/trafodion_config_default @@ -96,7 +96,8 @@ export REST_BUILD="" # Either "cloudera" or "hortonworks" (all lowercase) export HADOOP_TYPE="" -# The URL for Cloudera/Hortonworks REST API (i.e. node1.hp.com:8080) +# The URL for Cloudera/Hortonworks REST API (i.e. http://node1.hp.com:8080) +# Include http or https export URL="" # Cloudera/Hortonworks UI admin's userid and password @@ -152,10 +153,11 @@ export FLOATING_IP="" #If HA enabled the set the DCS Backup nodes export BACKUP_DCS_NODES="" + #If HA enabled then set INTERFACE to match the interface of the floating ip export INTERFACE="" -#Define if cloud environment is being used, if specified "Y" for CLOUD_CONFIG +#Define if cloud environment is being used, if specified "Y" for CLOUD_CONFIG #the please define the type of cloud (CLOUD_TYPE) being used export CLOUD_CONFIG="N" @@ -169,3 +171,4 @@ export CLOUD_OTHER="" #Specify the node name of primary DcsMaster node, Default is the host where #dcs start script is run export DCS_PRIMARY_MASTER_NODE=`hostname -f` + http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ee4c2539/install/installer/trafodion_install ---------------------------------------------------------------------- diff --git a/install/installer/trafodion_install b/install/installer/trafodion_install index 8cc2224..9e8f00e 100755 --- a/install/installer/trafodion_install +++ b/install/installer/trafodion_install @@ -49,6 +49,32 @@ Options: EOF } +function copyConfig { + +if [ $all_node_count -ne 1 ]; then + cp $TRAF_CONFIG $LOCAL_WORKDIR + $TRAF_PDCP $LOCAL_WORKDIR/$TRAF_CONFIG_FILE $HOME + $TRAF_PDSH sudo mkdir -p $TRAF_CONFIG_DIR + $TRAF_PDSH sudo cp $HOME/$TRAF_CONFIG_FILE $TRAF_CONFIG_DIR + $TRAF_PDSH sudo chmod 777 $TRAF_CONFIG +fi + +} + +function removePassword { + +export TRAF_CONFIG="/etc/trafodion/trafodion_config" +source $TRAF_CONFIG + +$TRAF_PDSH sudo chmod 777 $TRAF_CONFIG +$TRAF_PDSH sudo sed -i '/TRAF_USER_PASSWORD\=/d' $TRAF_CONFIG +$TRAF_PDSH sudo sed -i '/PASSWORD\=/d' $TRAF_CONFIG +$TRAF_PDSH sudo chmod 777 $TRAF_CONFIG +echo "here" + +} + + function checkHBaseVersion { for node in $HADOOP_NODES; @@ -111,7 +137,7 @@ do done -HBASE=$(curl -su $ADMIN:$PASSWORD http://$URL/api/v1/clusters/$CLUSTER_NAME/services | grep name | grep hbase | sed -e 's@[,]@@'g | awk '{print $3}' | sed "s/\"//g") +HBASE=$(curl -k -su $ADMIN:$PASSWORD $URL/api/v1/clusters/$CLUSTER_NAME/services | grep name | grep hbase | sed -e 's@[,]@@'g | awk '{print $3}' | sed "s/\"//g") sudo chmod 777 $TRAF_CONFIG @@ -125,7 +151,7 @@ sudo rm -rf hbaseVersion.txt trafodionFullName=$(basename $TRAF_BUILD) echo "***DEBUG: trafodionFullName=$trafodionFullName" #Debug version -trafodionVersion=$(echo $trafodionFullName | sed 's/trafodion_server-//' | sed 's/-debug//' | sed 's/.tgz//') +trafodionVersion=$(echo $trafodionFullName | sed 's/trafodion_.*server-//' | sed 's/-debug//' | sed 's/.tgz//') echo "***INFO: Trafodion version = $trafodionVersion" @@ -323,8 +349,7 @@ if [ $? -ne 0 ]; then echo "***ERROR: unable to source $TRAF_CONFIG" exit -1 fi - -#============================================== +#============================================= #Check Config File for Errors $LOCAL_WORKDIR/traf_config_check | tee -a $INSTALL_LOG @@ -347,8 +372,7 @@ echo "export START_ALL=\"$NO_START\"" >> $TRAF_CONFIG sudo chmod 777 $TRAF_CONFIG source $TRAF_CONFIG - -#============================================== +#============================================= #Check if doing an upgrade if [[ $CONFIG_COMPLETE ]]; then @@ -371,7 +395,7 @@ fi #Run Trafodion scanner if [[ $UPGRADE_TRAF == "true" ]]; then - echo "***INFO: Trafodion scanner will not be run." + echo else if [[ "$RUN_SCANNER" == "Y" ]]; then echo "***INFO: Running Trafodion Scanner" @@ -400,12 +424,12 @@ fi #============================================= -if [ $node_count -eq 1 ]; then +if [ $all_node_count -eq 1 ]; then TRAF_PDSH="" TRAF_PDCP="" else - TRAF_PDSH="pdsh -R exec $MY_NODES $PDSH_SSH_CMD" - TRAF_PDCP="pdcp -R ssh $MY_NODES" + TRAF_PDSH="pdsh -R exec $ALL_NODES $PDSH_SSH_CMD" + TRAF_PDCP="pdcp -R ssh $ALL_NODES" fi sed -i '/ACCEPT_LICENSE\=/d' $TRAF_CONFIG @@ -429,7 +453,7 @@ if [ ! -z $TRAF_PACKAGE ]; then # untar the package sudo tar -xzf $TRAF_PACKAGE --directory=$UNTAR_DIR - build_file=$(ls $UNTAR_DIR | grep "trafodion_server") + build_file=$(ls $UNTAR_DIR | grep "trafodion_.*server") if [[ -z $build_file ]]; then echo "***ERROR: trafodion_server-*.tgz could not be found in $UNTAR_DIR" exit -1 @@ -523,8 +547,8 @@ fi # parse the version string from the trafodion_server-n.n.n.jar filename traf_version=$(sudo ls $UNTAR_DIR | \ - grep --max-count=1 trafodion_server | \ - sed -e 's@trafodion_server-\([0-9*].*\).tgz@\1@') + grep --max-count=1 trafodion_.*server | \ + sed -e 's@trafodion_.*server-\([0-9*].*\).tgz@\1@') TRAF_VERSION=$(echo $traf_version | sed -e 's@\([0-9*].*\)-debug@\1@') # Detect java version @@ -597,7 +621,7 @@ if [[ "$UPGRADE_TRAF" == "true" ]]; then sudo sed -i -e "s@export NODE_LIST=.*@export NODE_LIST=\"$NODE_LIST\"@" $HOME_DIR/$TRAF_USER/.bashrc sudo sed -i -e "s@export MY_NODES=.*@export MY_NODES=\"$MY_NODES\"@" $HOME_DIR/$TRAF_USER/.bashrc - for node in $NODE_LIST + for node in $ALL_NODE_LIST do userForTrafodion=$(ssh -q -n $node grep "$TRAF_USER:" /etc/passwd | wc -l) if [[ "$userForTrafodion" == 0 ]]; then @@ -612,7 +636,6 @@ if [[ "$UPGRADE_TRAF" == "true" ]]; then fi fi done - echo "***INFO: Installing required RPM packages" | tee -a $INSTALL_LOG $LOCAL_WORKDIR/traf_package_setup | tee -a $INSTALL_LOG if [ $? != 0 ]; then @@ -640,13 +663,8 @@ $TRAF_PDSH sudo mkdir -p $LOCAL_WORKDIR $TRAF_PDSH sudo chmod 777 $LOCAL_WORKDIR # copy config file to all nodes -if [ $node_count -ne 1 ]; then - cp $TRAF_CONFIG $LOCAL_WORKDIR - $TRAF_PDCP $LOCAL_WORKDIR/$TRAF_CONFIG_FILE $LOCAL_WORKDIR - $TRAF_PDSH sudo mkdir -p $TRAF_CONFIG_DIR - $TRAF_PDSH sudo cp $LOCAL_WORKDIR/$TRAF_CONFIG_FILE $TRAF_CONFIG_DIR - $TRAF_PDSH sudo chmod 777 $TRAF_CONFIG -fi + +copyConfig $LOCAL_WORKDIR/traf_sqconfig @@ -661,7 +679,7 @@ echo "***INFO: Reserving DCS ports" trafodion_ports=${DCS_MASTER_PORT:-23400},${DCS_MASTER_INFO_PORT:-24400} reservedTrafodionPorts="net.ipv4.ip_local_reserved_ports = ${trafodion_ports}" -for node in $NODE_LIST +for node in $ALL_NODE_LIST do redhatSystem=$(ssh -q -n $node '/sbin/sysctl net.ipv4.ip_local_reserved_ports | wc -l') if [[ "$redhatSystem" -ge "1" ]]; then @@ -682,6 +700,17 @@ do fi done +sudo chmod 777 $TRAF_CONFIG +sed -i '/DCS_PORT\=/d' $TRAF_CONFIG +echo "export DCS_PORT=\"23400\"" >> $TRAF_CONFIG +sudo chmod 777 $TRAF_CONFIG + +sudo chmod 777 $TRAF_CONFIG +sed -i '/DCS_INFO_PORT\=/d' $TRAF_CONFIG +echo "export DCS_INFO_PORT=\"24400\"" >> $TRAF_CONFIG +sudo chmod 777 $TRAF_CONFIG +source $TRAF_CONFIG +copyConfig if [[ $ENABLE_HA == "true" ]]; then $LOCAL_WORKDIR/traf_add_sudoAccess @@ -690,10 +719,23 @@ fi #Set Max kernel pid $TRAF_PDSH sudo /sbin/sysctl -w kernel.pid_max=65535 2>&1 > /dev/null +sudo sed -i '/kernel\.pid\_max*/d' /etc/sysctl.conf 2>&1 > /dev/null +sudo chmod 777 /etc/sysctl.conf +sudo echo "kernel.pid_max = 65535" >> /etc/sysctl.conf +sudo chmod 777 /etc/sysctl.conf +sudo chown $(whoami).$(whoami) /etc/sysctl.conf +cp /etc/sysctl.conf $HOME 2>&1 > /dev/null + +if [[ $node_count -ne "1" ]]; then + $TRAF_PDCP $HOME/sysctl.conf $HOME 2>&1 > /dev/null +fi +$TRAF_PDSH sudo mv $HOME/sysctl.conf /etc/sysctl.conf +$TRAF_PDSH sudo chown root.root /etc/sysctl.conf +$TRAF_PDSH sudo chmod 644 /etc/sysctl.conf #Create install directory -if [[ "$node_count" -ne "1" ]]; then +if [[ "$all_node_count" -ne "1" ]]; then $TRAF_PDSH sudo mkdir -p /$HOME_DIR/$TRAF_USER/installer &> /dev/null fi @@ -701,7 +743,7 @@ fi if [[ "$LDAP_SECURITY" == "Y" ]]; then echo "***INFO: Copying $LDAP_AUTH_FILE to Trafodion home dir" - if [[ "$node_count" -eq "1" ]]; then + if [[ "$all_node_count" -eq "1" ]]; then sudo cp -r $LOCAL_WORKDIR/$LDAP_AUTH_FILE $HOME_DIR/$TRAF_USER sudo chown trafodion.trafodion $HOME_DIR/$TRAF_USER/$LDAP_AUTH_FILE sudo chmod 750 $HOME_DIR/$TRAF_USER/$LDAP_AUTH_FILE @@ -718,7 +760,7 @@ if [[ "$LDAP_SECURITY" == "Y" ]]; then $TRAF_PDSH sudo chmod 750 $HOME_DIR/$TRAF_USER/$LDAP_AUTH_FILE if [[ $LDAP_LEVEL == "2" ]] || [[ $LDAP_LEVEL == "1" ]]; then echo "***INFO: Copying $LDAP_CERT to Trafodion home dir" - $TRAF_PDCP $LDAP_CERT $HOME + $TRAF_PCP $LDAP_CERT $HOME $TRAF_PDSH sudo cp -r $HOME/$LDAP_CERT_BASE $HOME_DIR/$TRAF_USER $TRAF_PDSH sudo chown trafodion.trafodion $HOME_DIR/$TRAF_USER/$LDAP_CERT_BASE $TRAF_PDSH sudo chmod 750 $HOME_DIR/$TRAF_USER/$LDAP_CERT_BASE @@ -773,7 +815,7 @@ sudo mkdir -p $SQ_ROOT sudo chown $TRAF_USER.trafodion $SQ_ROOT /bin/cp -rf $LOCAL_WORKDIR/traf_config $TRAF_WORKDIR/installer/traf_config -sudo su $TRAF_USER --command "$TRAF_WORKDIR/installer/traf_config" 2>&1 | tee -a $INSTALL_LOG +sudo su $TRAF_USER --login --command "$TRAF_WORKDIR/installer/traf_config" 2>&1 | tee -a $INSTALL_LOG if [ ${PIPESTATUS[0]} != "0" ]; then echo "***ERROR: Error while running traf_config." | tee -a $INSTALL_LOG @@ -784,12 +826,13 @@ fi # Install and configure CLI for Cloud environments if [ "$ENABLE_HA" == "true" ] && [[ $CLOUD_CONFIG == "Y" ]]; then - echo "***INFO: Install and configure CLI for Cloud" + echo "***INFO: Install and configure CLI for Cloud" $TRAF_WORKDIR/installer/cloud_cli_setup fi if [[ $START_ALL == "Y" ]]; then - sudo su $TRAF_USER --command "$TRAF_WORKDIR/installer/traf_start" 2>&1 | tee -a $INSTALL_LOG + sudo su $TRAF_USER --login --command "$TRAF_WORKDIR/installer/traf_sqgen" 2>&1 | tee -a $INSTALL_LOG + sudo su $TRAF_USER --login --command "$TRAF_WORKDIR/installer/traf_start" 2>&1 | tee -a $INSTALL_LOG if [ ${PIPESTATUS[0]} != "0" ]; then echo "***ERROR: Error while running traf_start." | tee -a $INSTALL_LOG @@ -806,8 +849,8 @@ sed -i '/MODS_COMPLETE\=/d' $TRAF_CONFIG echo "export MODS_COMPLETE=\"$MODS_COMPLETE\"" >> $TRAF_CONFIG sudo chmod 777 $TRAF_CONFIG source $TRAF_CONFIG +copyConfig +removePassword echo -echo "*********************************" -echo " TRAFODION INSTALLATION COMPLETE" -echo "*********************************" echo + http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ee4c2539/install/installer/trafodion_uninstaller ---------------------------------------------------------------------- diff --git a/install/installer/trafodion_uninstaller b/install/installer/trafodion_uninstaller index 6a92db6..62ec6aa 100755 --- a/install/installer/trafodion_uninstaller +++ b/install/installer/trafodion_uninstaller @@ -88,13 +88,13 @@ export PDSH_SSH_CMD="ssh -q -n %h" export PDCP="pdcp -R ssh" -if [ $node_count -eq 1 ]; then +if [ $all_node_count -eq 1 ]; then TRAF_PDSH="" else # use the -S option to cause pdsh to return largest of # the remote command return values so we can tell if one # or more of the remote commands failed - TRAF_PDSH="pdsh -R exec $MY_NODES $PDSH_SSH_CMD" + TRAF_PDSH="pdsh -R exec $ALL_NODES $PDSH_SSH_CMD" fi #Check sudo access which is required for "--all" option @@ -114,7 +114,7 @@ sudo su $TRAF_USER --login --command "vncserver -kill :1" echo "***INFO: restoring linux system files that were changed" echo "***INFO: removing $HBASE_TRX from Hadoop directories" -if [ $node_count -eq 1 ]; then +if [ $all_node_count -eq 1 ]; then sudo rm /etc/security/limits.d/trafodion.conf 2>/dev/null sudo rm /usr/share/cmf/lib/plugins/$HBASE_TRX 2>/dev/null sudo rm /usr/lib/hbase/lib/$HBASE_TRX 2>/dev/null
