http://git-wip-us.apache.org/repos/asf/ambari/blob/23b7c110/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hadoop-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hadoop-env.xml b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hadoop-env.xml deleted file mode 100644 index c06ed20..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hadoop-env.xml +++ /dev/null @@ -1,201 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> - -<configuration> - <property> - <name>hdfs_log_dir_prefix</name> - <value>/var/log/hadoop</value> - <description>Hadoop Log Dir Prefix</description> - </property> - <property> - <name>hadoop_pid_dir_prefix</name> - <value>/var/run/hadoop</value> - <description>Hadoop PID Dir Prefix</description> - </property> - <property> - <name>hadoop_root_logger</name> - <value>INFO,RFA</value> - <description>Hadoop Root Logger</description> - </property> - <property> - <name>hadoop_heapsize</name> - <value>1024</value> - <description>Hadoop maximum Java heap size</description> - </property> - <property> - <name>namenode_heapsize</name> - <value>1024</value> - <description>NameNode Java heap size</description> - </property> - <property> - <name>namenode_opt_newsize</name> - <value>200</value> - <description>NameNode new generation size</description> - </property> - <property> - <name>namenode_opt_maxnewsize</name> - <value>200</value> - <description>NameNode maximum new generation size</description> - </property> - <property> - <name>namenode_opt_permsize</name> - <value>128</value> - <description>NameNode permanent generation size</description> - </property> - <property> - <name>namenode_opt_maxpermsize</name> - <value>256</value> - <description>NameNode maximum permanent generation size</description> - </property> - <property> - <name>dtnode_heapsize</name> - <value>1024</value> - <description>DataNode maximum Java heap size</description> - </property> - <property> - <name>proxyuser_group</name> - <value>users</value> - <property-type>GROUP</property-type> - <description>Proxy user group.</description> - </property> - <property> - <name>hdfs_user</name> - <value>hdfs</value> - <property-type>USER</property-type> - <description>User to run HDFS as</description> - </property> - <property> - <name>dfs.datanode.data.dir.mount.file</name> - <value>/etc/hadoop/conf/dfs_data_dir_mount.hist</value> - <description>File path that contains the last known mount point for each data dir. This file is used to avoid creating a DFS data dir on the root drive (and filling it up) if a path was previously mounted on a drive.</description> - </property> - - <!-- hadoop-env.sh --> - <property> - <name>content</name> - <description>This is the jinja template for hadoop-env.sh file</description> - <value> -# Set Hadoop-specific environment variables here. - -# The only required environment variable is JAVA_HOME. All others are -# optional. When running a distributed configuration it is best to -# set JAVA_HOME in this file, so that it is correctly defined on -# remote nodes. - -# The java implementation to use. Required. -export JAVA_HOME={{java_home}} -export HADOOP_HOME_WARN_SUPPRESS=1 - -# Hadoop Configuration Directory -#TODO: if env var set that can cause problems -export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-{{hadoop_conf_dir}}} - -# Hadoop home directory -export HADOOP_HOME=${HADOOP_HOME:-/usr/lib/hadoop} - -# The maximum amount of heap to use, in MB. Default is 1000. -export HADOOP_HEAPSIZE="{{hadoop_heapsize}}" - -export HADOOP_NAMENODE_INIT_HEAPSIZE="-Xms{{namenode_heapsize}}" - -# Extra Java runtime options. Empty by default. -export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}" - -# History server logs -export HADOOP_MAPRED_LOG_DIR={{hdfs_log_dir_prefix}}/$USER - -# Command specific options appended to HADOOP_OPTS when specified -export HADOOP_NAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSize={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_NAMENODE_OPTS}" -export HADOOP_JOBTRACKER_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx{{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA -Dmapred.log.dir=$HADOOP_MAPRED_LOG_DIR ${HADOOP_JOBTRACKER_OPTS}" - -HADOOP_TASKTRACKER_OPTS="-server -Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}" -HADOOP_DATANODE_OPTS="-Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=ERROR,DRFAS ${HADOOP_DATANODE_OPTS}" -HADOOP_BALANCER_OPTS="-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCER_OPTS}" - -export HADOOP_SECONDARYNAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSize={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps ${HADOOP_NAMENODE_INIT_HEAPSIZE} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_SECONDARYNAMENODE_OPTS}" - -# The following applies to multiple commands (fs, dfs, fsck, distcp etc) -export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS" -# On secure datanodes, user to run the datanode as after dropping privileges -export HADOOP_SECURE_DN_USER={{hdfs_user}} - -# Extra ssh options. Empty by default. -export HADOOP_SSH_OPTS="-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR" - -# Where log files are stored. $HADOOP_HOME/logs by default. -export HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER - -# Where log files are stored in the secure data environment. -export HADOOP_SECURE_DN_LOG_DIR={{hdfs_log_dir_prefix}}/$HADOOP_SECURE_DN_USER - -# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default. -# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves - -# host:path where hadoop code should be rsync'd from. Unset by default. -# export HADOOP_MASTER=master:/home/$USER/src/hadoop - -# Seconds to sleep between slave commands. Unset by default. This -# can be useful in large clusters, where, e.g., slave rsyncs can -# otherwise arrive faster than the master can service them. -# export HADOOP_SLAVE_SLEEP=0.1 - -# The directory where pid files are stored. /tmp by default. -export HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}/$USER -export HADOOP_SECURE_DN_PID_DIR={{hadoop_pid_dir_prefix}}/$HADOOP_SECURE_DN_USER - -# History server pid -export HADOOP_MAPRED_PID_DIR={{mapred_pid_dir_prefix}}/$USER - -YARN_RESOURCEMANAGER_OPTS="-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY" - -# A string representing this instance of hadoop. $USER by default. -export HADOOP_IDENT_STRING=$USER - -# The scheduling priority for daemon processes. See 'man nice'. - -# export HADOOP_NICENESS=10 - -# Use libraries from standard classpath -JAVA_JDBC_LIBS="" -#Add libraries required by mysql connector -for jarFile in `ls /usr/share/java/*mysql* 2>/dev/null` -do - JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile -done -#Add libraries required by oracle connector -for jarFile in `ls /usr/share/java/*ojdbc* 2>/dev/null` -do - JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile -done -#Add libraries required by nodemanager -MAPREDUCE_LIBS={{mapreduce_libs_path}} -export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS} - -# Setting path to hdfs command line -export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}} - -#Mostly required for hadoop 2.0 -export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/lib/hadoop/lib/native/Linux-amd64-64 - </value> - </property> - -</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/23b7c110/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hadoop-policy.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hadoop-policy.xml b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hadoop-policy.xml deleted file mode 100644 index def654b..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hadoop-policy.xml +++ /dev/null @@ -1,134 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> - -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<!-- Put site-specific property overrides in this file. --> - -<configuration supports_final="true"> - <property> - <name>security.client.protocol.acl</name> - <value>*</value> - <description>ACL for ClientProtocol, which is used by user code - via the DistributedFileSystem. - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed.</description> - </property> - - <property> - <name>security.client.datanode.protocol.acl</name> - <value>*</value> - <description>ACL for ClientDatanodeProtocol, the client-to-datanode protocol - for block recovery. - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed.</description> - </property> - - <property> - <name>security.datanode.protocol.acl</name> - <value>*</value> - <description>ACL for DatanodeProtocol, which is used by datanodes to - communicate with the namenode. - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed.</description> - </property> - - <property> - <name>security.inter.datanode.protocol.acl</name> - <value>*</value> - <description>ACL for InterDatanodeProtocol, the inter-datanode protocol - for updating generation timestamp. - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed.</description> - </property> - - <property> - <name>security.namenode.protocol.acl</name> - <value>*</value> - <description>ACL for NamenodeProtocol, the protocol used by the secondary - namenode to communicate with the namenode. - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed.</description> - </property> - - <property> - <name>security.inter.tracker.protocol.acl</name> - <value>*</value> - <description>ACL for InterTrackerProtocol, used by the tasktrackers to - communicate with the jobtracker. - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed.</description> - </property> - - <property> - <name>security.job.submission.protocol.acl</name> - <value>*</value> - <description>ACL for JobSubmissionProtocol, used by job clients to - communciate with the jobtracker for job submission, querying job status etc. - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed.</description> - </property> - - <property> - <name>security.task.umbilical.protocol.acl</name> - <value>*</value> - <description>ACL for TaskUmbilicalProtocol, used by the map and reduce - tasks to communicate with the parent tasktracker. - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed.</description> - </property> - - <property> - <name>security.admin.operations.protocol.acl</name> - <value>*</value> - <description>ACL for AdminOperationsProtocol. Used for admin commands. - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed.</description> - </property> - - <property> - <name>security.refresh.usertogroups.mappings.protocol.acl</name> - <value>*</value> - <description>ACL for RefreshUserMappingsProtocol. Used to refresh - users mappings. The ACL is a comma-separated list of user and - group names. The user and group list is separated by a blank. For - e.g. "alice,bob users,wheel". A special value of "*" means all - users are allowed.</description> - </property> - -<property> - <name>security.refresh.policy.protocol.acl</name> - <value>*</value> - <description>ACL for RefreshAuthorizationPolicyProtocol, used by the - dfsadmin and mradmin commands to refresh the security policy in-effect. - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed.</description> - </property> - - -</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/23b7c110/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hdfs-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hdfs-log4j.xml b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hdfs-log4j.xml deleted file mode 100644 index 172f66e..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hdfs-log4j.xml +++ /dev/null @@ -1,197 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> - -<configuration supports_final="false"> - - <property> - <name>content</name> - <description>Custom log4j.properties</description> - <value> -# Copyright 2011 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Define some default values that can be overridden by system properties -# To change daemon root logger use hadoop_root_logger in hadoop-env -hadoop.root.logger=INFO,console -hadoop.log.dir=. -hadoop.log.file=hadoop.log - - -# Define the root logger to the system property "hadoop.root.logger". -log4j.rootLogger=${hadoop.root.logger}, EventCounter - -# Logging Threshold -log4j.threshhold=ALL - -# -# Daily Rolling File Appender -# - -log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender -log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file} - -# Rollver at midnight -log4j.appender.DRFA.DatePattern=.yyyy-MM-dd - -# 30-day backup -#log4j.appender.DRFA.MaxBackupIndex=30 -log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout - -# Pattern format: Date LogLevel LoggerName LogMessage -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n -# Debugging Pattern format -#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n - - -# -# console -# Add "console" to rootlogger above if you want to use this -# - -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n - -# -# TaskLog Appender -# - -#Default values -hadoop.tasklog.taskid=null -hadoop.tasklog.iscleanup=false -hadoop.tasklog.noKeepSplits=4 -hadoop.tasklog.totalLogFileSize=100 -hadoop.tasklog.purgeLogSplits=true -hadoop.tasklog.logsRetainHours=12 - -log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender -log4j.appender.TLA.taskId=${hadoop.tasklog.taskid} -log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup} -log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize} - -log4j.appender.TLA.layout=org.apache.log4j.PatternLayout -log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n - -# -#Security audit appender -# -hadoop.security.logger=INFO,console -hadoop.security.log.maxfilesize=256MB -hadoop.security.log.maxbackupindex=20 -log4j.category.SecurityLogger=${hadoop.security.logger} -hadoop.security.log.file=SecurityAuth.audit -log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender -log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} -log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout -log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n -log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd - -log4j.appender.RFAS=org.apache.log4j.RollingFileAppender -log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} -log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout -log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n -log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize} -log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex} - -# -# hdfs audit logging -# -hdfs.audit.logger=INFO,console -log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger} -log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false -log4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender -log4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log -log4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout -log4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n -log4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd - -# -# mapred audit logging -# -mapred.audit.logger=INFO,console -log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger} -log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false -log4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender -log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log -log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout -log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n -log4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd - -# -# Rolling File Appender -# - -log4j.appender.RFA=org.apache.log4j.RollingFileAppender -log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file} - -# Logfile size and and 30-day backups -log4j.appender.RFA.MaxFileSize=256MB -log4j.appender.RFA.MaxBackupIndex=10 - -log4j.appender.RFA.layout=org.apache.log4j.PatternLayout -log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n -log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n - - -# Custom Logging levels - -hadoop.metrics.log.level=INFO -#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG -#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG -#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG -log4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level} - -# Jets3t library -log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR - -# -# Null Appender -# Trap security logger on the hadoop client side -# -log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender - -# -# Event Counter Appender -# Sends counts of logging messages at different severity levels to Hadoop Metrics. -# -log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter - - </value> - </property> - -</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/23b7c110/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hdfs-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hdfs-site.xml b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hdfs-site.xml deleted file mode 100644 index 0075015..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hdfs-site.xml +++ /dev/null @@ -1,369 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> - -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<!-- Put site-specific property overrides in this file. --> - -<configuration supports_final="true"> - - <!-- file system properties --> - - <property> - <name>dfs.name.dir</name> - <!-- cluster variant --> - <value>/hadoop/hdfs/namenode</value> - <description>Determines where on the local filesystem the DFS name node - should store the name table. If this is a comma-delimited list - of directories then the name table is replicated in all of the - directories, for redundancy. </description> - <final>true</final> - </property> - - <property> - <name>dfs.support.append</name> - <value>true</value> - <description>to enable dfs append</description> - <final>true</final> - </property> - - <property> - <name>dfs.webhdfs.enabled</name> - <value>true</value> - <description>to enable webhdfs</description> - <final>true</final> - </property> - - <property> - <name>dfs.datanode.socket.write.timeout</name> - <value>0</value> - <description>DFS Client write socket timeout</description> - </property> - - <property> - <name>dfs.datanode.failed.volumes.tolerated</name> - <value>0</value> - <description>#of failed disks dn would tolerate</description> - <final>true</final> - </property> - - <property> - <name>dfs.block.local-path-access.user</name> - <value>hbase</value> - <description>the user who is allowed to perform short - circuit reads. - </description> - <final>true</final> - </property> - - <property> - <name>dfs.data.dir</name> - <value>/hadoop/hdfs/data</value> - <description>Determines where on the local filesystem an DFS data node - should store its blocks. If this is a comma-delimited - list of directories, then data will be stored in all named - directories, typically on different devices. - Directories that do not exist are ignored. - </description> - <final>true</final> - </property> - - <property> - <name>dfs.hosts.exclude</name> - <value>/etc/hadoop/conf/dfs.exclude</value> - <description>Names a file that contains a list of hosts that are - not permitted to connect to the namenode. The full pathname of the - file must be specified. If the value is empty, no hosts are - excluded.</description> - </property> - - <property> - <name>dfs.hosts</name> - <value>/etc/hadoop/conf/dfs.include</value> - <description>Names a file that contains a list of hosts that are - permitted to connect to the namenode. The full pathname of the file - must be specified. If the value is empty, all hosts are - permitted.</description> - </property> - - <property> - <name>dfs.replication.max</name> - <value>50</value> - <description>Maximal block replication. - </description> - </property> - - <property> - <name>dfs.replication</name> - <value>3</value> - <description>Default block replication. - </description> - </property> - - <property> - <name>dfs.heartbeat.interval</name> - <value>3</value> - <description>Determines datanode heartbeat interval in seconds.</description> - </property> - - <property> - <name>dfs.safemode.threshold.pct</name> - <value>1.0f</value> - <description> - Specifies the percentage of blocks that should satisfy - the minimal replication requirement defined by dfs.replication.min. - Values less than or equal to 0 mean not to start in safe mode. - Values greater than 1 will make safe mode permanent. - </description> - </property> - - <property> - <name>dfs.balance.bandwidthPerSec</name> - <value>6250000</value> - <description> - Specifies the maximum amount of bandwidth that each datanode - can utilize for the balancing purpose in term of - the number of bytes per second. - </description> - </property> - - <property> - <name>dfs.datanode.address</name> - <value>0.0.0.0:50010</value> - <description> - The datanode server address and port for data transfer. - </description> - </property> - - - <property> - <name>dfs.datanode.http.address</name> - <value>0.0.0.0:50075</value> - <description> - The datanode http server address and port. - </description> - </property> - - <property> - <name>dfs.datanode.https.address</name> - <value>0.0.0.0:50475</value> - <description> - The datanode https server address and port. - </description> - </property> - - <property> - <name>dfs.block.size</name> - <value>134217728</value> - <description>The default block size for new files.</description> - </property> - - <property> - <name>dfs.http.address</name> - <value>localhost:50070</value> - <description>The name of the default file system. Either the - literal string "local" or a host:port for NDFS.</description> - <final>true</final> - </property> - - <property> - <name>dfs.datanode.du.reserved</name> - <!-- cluster variant --> - <value>1073741824</value> - <description>Reserved space in bytes per volume. Always leave this much space free for non dfs use. - </description> - </property> - - <property> - <name>dfs.datanode.ipc.address</name> - <value>0.0.0.0:8010</value> - <description> - The datanode ipc server address and port. - If the port is 0 then the server will start on a free port. - </description> - </property> - - <property> - <name>dfs.blockreport.initialDelay</name> - <value>120</value> - <description>Delay for first block report in seconds.</description> - </property> - - <property> - <name>dfs.namenode.handler.count</name> - <value>40</value> - <description>The number of server threads for the namenode.</description> - </property> - - <property> - <name>dfs.datanode.max.xcievers</name> - <value>4096</value> - <description>PRIVATE CONFIG VARIABLE</description> - </property> - - <!-- Permissions configuration --> - - <property> - <name>dfs.umaskmode</name> - <value>077</value> - <description> - The octal umask used when creating files and directories. - </description> - </property> - - <property> - <name>dfs.web.ugi</name> - <!-- cluster variant --> - <value>gopher,gopher</value> - <description>The user account used by the web interface. - Syntax: USERNAME,GROUP1,GROUP2, ... - </description> - </property> - - <property> - <name>dfs.permissions</name> - <value>true</value> - <description> - If "true", enable permission checking in HDFS. - If "false", permission checking is turned off, - but all other behavior is unchanged. - Switching from one parameter value to the other does not change the mode, - owner or group of files or directories. - </description> - </property> - - <property> - <name>dfs.permissions.supergroup</name> - <value>hdfs</value> - <description>The name of the group of super-users.</description> - </property> - - <property> - <name>dfs.namenode.handler.count</name> - <value>100</value> - <description>Added to grow Queue size so that more client connections are allowed</description> - </property> - - <property> - <name>ipc.server.max.response.size</name> - <value>5242880</value> - </property> - <property> - <name>dfs.block.access.token.enable</name> - <value>true</value> - <description> - If "true", access tokens are used as capabilities for accessing datanodes. - If "false", no access tokens are checked on accessing datanodes. - </description> - </property> - - <property> - <!-- cluster variant --> - <name>dfs.secondary.http.address</name> - <value>localhost:50090</value> - <description>Address of secondary namenode web server</description> - </property> - - <property> - <name>dfs.secondary.https.port</name> - <value>50490</value> - <description>The https port where secondary-namenode binds</description> - </property> - - <property> - <name>dfs.https.port</name> - <value>50470</value> - <description>The https port where namenode binds</description> - - </property> - - <property> - <name>dfs.https.address</name> - <value>localhost:50470</value> - <description>The https address where namenode binds</description> - - </property> - - <property> - <name>dfs.datanode.data.dir.perm</name> - <value>750</value> - <description>The permissions that should be there on dfs.data.dir - directories. The datanode will not come up if the permissions are - different on existing dfs.data.dir directories. If the directories - don't exist, they will be created with this permission.</description> - </property> - - <property> - <name>dfs.access.time.precision</name> - <value>0</value> - <description>The access time for HDFS file is precise upto this value. - The default value is 1 hour. Setting a value of 0 disables - access times for HDFS. - </description> - </property> - - <property> - <name>dfs.cluster.administrators</name> - <value> hdfs</value> - <description>ACL for who all can view the default servlets in the HDFS</description> - </property> - - <property> - <name>ipc.server.read.threadpool.size</name> - <value>5</value> - <description></description> - </property> - - <property> - <name>dfs.datanode.failed.volumes.tolerated</name> - <value>0</value> - <description>Number of failed disks datanode would tolerate</description> - </property> - - <property> - <name>dfs.namenode.avoid.read.stale.datanode</name> - <value>true</value> - <description> - Indicate whether or not to avoid reading from stale datanodes whose - heartbeat messages have not been received by the namenode for more than a - specified time interval. - </description> - </property> - <property> - <name>dfs.namenode.avoid.write.stale.datanode</name> - <value>true</value> - <description> - Indicate whether or not to avoid writing to stale datanodes whose - heartbeat messages have not been received by the namenode for more than a - specified time interval. - </description> - </property> - <property> - <name>dfs.namenode.write.stale.datanode.ratio</name> - <value>1.0f</value> - <description>When the ratio of number stale datanodes to total datanodes marked is greater - than this ratio, stop avoiding writing to stale nodes so as to prevent causing hotspots. - </description> - </property> - <property> - <name>dfs.namenode.stale.datanode.interval</name> - <value>30000</value> - <description>Datanode is stale after not getting a heartbeat in this interval in ms</description> - </property> - -</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/23b7c110/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/metainfo.xml deleted file mode 100644 index 2aa0590..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/metainfo.xml +++ /dev/null @@ -1,163 +0,0 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<metainfo> - <schemaVersion>2.0</schemaVersion> - <services> - <service> - <name>HDFS</name> - <displayName>HDFS</displayName> - <comment>Apache Hadoop Distributed File System</comment> - <version>1.2.0.1.3.3.0</version> - - <components> - <component> - <name>NAMENODE</name> - <displayName>NameNode</displayName> - <category>MASTER</category> - <cardinality>1-2</cardinality> - <commandScript> - <script>scripts/namenode.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - <customCommands> - <customCommand> - <name>DECOMMISSION</name> - <commandScript> - <script>scripts/namenode.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - </customCommand> - </customCommands> - </component> - - <component> - <name>DATANODE</name> - <displayName>DataNode</displayName> - <category>SLAVE</category> - <cardinality>1+</cardinality> - <commandScript> - <script>scripts/datanode.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - </component> - - <component> - <name>SECONDARY_NAMENODE</name> - <displayName>SNameNode</displayName> - <category>MASTER</category> - <cardinality>1</cardinality> - <commandScript> - <script>scripts/snamenode.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - </component> - - <component> - <name>HDFS_CLIENT</name> - <displayName>HDFS Client</displayName> - <category>CLIENT</category> - <cardinality>1+</cardinality> - <commandScript> - <script>scripts/hdfs_client.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - <configFiles> - <configFile> - <type>xml</type> - <fileName>hdfs-site.xml</fileName> - <dictionaryName>hdfs-site</dictionaryName> - </configFile> - <configFile> - <type>xml</type> - <fileName>core-site.xml</fileName> - <dictionaryName>core-site</dictionaryName> - </configFile> - <configFile> - <type>env</type> - <fileName>hadoop-env.sh</fileName> - <dictionaryName>hadoop-env</dictionaryName> - </configFile> - <configFile> - <type>env</type> - <fileName>log4j.properties</fileName> - <dictionaryName>hdfs-log4j,mapreduce-log4j</dictionaryName> - </configFile> - </configFiles> - </component> - </components> - <osSpecifics> - <osSpecific> - <osFamily>any</osFamily> - <packages> - <package> - <name>lzo</name> - </package> - <package> - <name>hadoop</name> - </package> - <package> - <name>hadoop-libhdfs</name> - </package> - <package> - <name>hadoop-native</name> - </package> - <package> - <name>hadoop-pipes</name> - </package> - <package> - <name>hadoop-sbin</name> - </package> - <package> - <name>hadoop-lzo-0.5.0</name> - </package> - <package> - <name>hadoop-lzo-native-0.5.0</name> - </package> - <package> - <name>snappy</name> - </package> - <package> - <name>snappy-devel</name> - </package> - <package> - <name>ambari-log4j</name> - </package> - </packages> - </osSpecific> - </osSpecifics> - <commandScript> - <script>scripts/service_check.py</script> - <scriptType>PYTHON</scriptType> - <timeout>300</timeout> - </commandScript> - - <configuration-dependencies> - <config-type>core-site</config-type> - <config-type>hdfs-site</config-type> - <config-type>hadoop-env</config-type> - <config-type>hadoop-policy</config-type> - <config-type>hdfs-log4j</config-type> - </configuration-dependencies> - </service> - </services> -</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/23b7c110/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/files/checkForFormat.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/files/checkForFormat.sh b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/files/checkForFormat.sh deleted file mode 100644 index c5af7de..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/files/checkForFormat.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/sh -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# - -export hdfs_user=$1 -shift -export conf_dir=$1 -shift -export mark_dir=$1 -shift -export name_dirs=$* - -export EXIT_CODE=0 -export command="namenode -format" -export list_of_non_empty_dirs="" - -mark_file=/var/run/hadoop/hdfs/namenode-formatted -if [[ -f ${mark_file} ]] ; then - sudo rm -f ${mark_file} - sudo mkdir -p ${mark_dir} -fi - -if [[ ! -d $mark_dir ]] ; then - for dir in `echo $name_dirs | tr ',' ' '` ; do - echo "NameNode Dirname = $dir" - cmd="ls $dir | wc -l | grep -q ^0$" - eval $cmd - if [[ $? -ne 0 ]] ; then - (( EXIT_CODE = $EXIT_CODE + 1 )) - list_of_non_empty_dirs="$list_of_non_empty_dirs $dir" - fi - done - - if [[ $EXIT_CODE == 0 ]] ; then - sudo su ${hdfs_user} - -s /bin/bash -c "yes Y | hadoop --config ${conf_dir} ${command}" - (( EXIT_CODE = $EXIT_CODE | $? )) - else - echo "ERROR: Namenode directory(s) is non empty. Will not format the namenode. List of non-empty namenode dirs ${list_of_non_empty_dirs}" - fi -else - echo "${mark_dir} exists. Namenode DFS already formatted" -fi - -exit $EXIT_CODE - http://git-wip-us.apache.org/repos/asf/ambari/blob/23b7c110/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/files/checkWebUI.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/files/checkWebUI.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/files/checkWebUI.py deleted file mode 100644 index f8e9c1a..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/files/checkWebUI.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python - -''' -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -''' - -import optparse -import httplib - -# -# Main. -# -def main(): - parser = optparse.OptionParser(usage="usage: %prog [options] component ") - parser.add_option("-m", "--hosts", dest="hosts", help="Comma separated hosts list for WEB UI to check it availability") - parser.add_option("-p", "--port", dest="port", help="Port of WEB UI to check it availability") - - (options, args) = parser.parse_args() - - hosts = options.hosts.split(',') - port = options.port - - for host in hosts: - try: - conn = httplib.HTTPConnection(host, port) - # This can be modified to get a partial url part to be sent with request - conn.request("GET", "/") - httpCode = conn.getresponse().status - conn.close() - except Exception: - httpCode = 404 - - if httpCode != 200: - print "Cannot access WEB UI on: http://"; + host + ":" + port - exit(1) - - -if __name__ == "__main__": - main() http://git-wip-us.apache.org/repos/asf/ambari/blob/23b7c110/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/datanode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/datanode.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/datanode.py deleted file mode 100644 index e8037d5..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/datanode.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management import * -from hdfs_datanode import datanode -from hdfs import hdfs - - -class DataNode(Script): - def install(self, env): - import params - - self.install_packages(env, params.exclude_packages) - env.set_params(params) - - def start(self, env): - import params - - env.set_params(params) - self.configure(env) - datanode(action="start") - - def stop(self, env): - import params - - env.set_params(params) - datanode(action="stop") - - def configure(self, env): - import params - - env.set_params(params) - hdfs() - datanode(action="configure") - - def status(self, env): - import status_params - - env.set_params(status_params) - check_process_status(status_params.datanode_pid_file) - - -if __name__ == "__main__": - DataNode().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/23b7c110/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs.py deleted file mode 100644 index 751ed9a..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs.py +++ /dev/null @@ -1,74 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -from resource_management import * -import sys -import os - - -def hdfs(name=None): - import params - - # On some OS this folder could be not exists, so we will create it before pushing there files - Directory(params.limits_conf_dir, - recursive=True, - owner='root', - group='root' - ) - - File(os.path.join(params.limits_conf_dir, 'hdfs.conf'), - owner='root', - group='root', - mode=0644, - content=Template("hdfs.conf.j2") - ) - - if params.security_enabled: - tc_mode = 0644 - tc_owner = "root" - else: - tc_mode = None - tc_owner = params.hdfs_user - - if "hadoop-policy" in params.config['configurations']: - XmlConfig("hadoop-policy.xml", - conf_dir=params.hadoop_conf_dir, - configurations=params.config['configurations']['hadoop-policy'], - configuration_attributes=params.config['configuration_attributes']['hadoop-policy'], - owner=params.hdfs_user, - group=params.user_group - ) - - XmlConfig("hdfs-site.xml", - conf_dir=params.hadoop_conf_dir, - configurations=params.config['configurations']['hdfs-site'], - configuration_attributes=params.config['configuration_attributes']['hdfs-site'], - owner=params.hdfs_user, - group=params.user_group - ) - - File(os.path.join(params.hadoop_conf_dir, 'slaves'), - owner=tc_owner, - content=Template("slaves.j2") - ) - if params.lzo_enabled: - Package(params.lzo_packages_for_current_host) - http://git-wip-us.apache.org/repos/asf/ambari/blob/23b7c110/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs_client.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs_client.py deleted file mode 100644 index 9f783fa..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs_client.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management import * -from hdfs import hdfs -from utils import service - - -class HdfsClient(Script): - def install(self, env): - import params - - self.install_packages(env, params.exclude_packages) - env.set_params(params) - self.configure(env) - - def start(self, env): - import params - - env.set_params(params) - - def stop(self, env): - import params - - env.set_params(params) - - def status(self, env): - raise ClientComponentHasNoStatus() - - def configure(self, env): - import params - hdfs() - - def generate_configs_get_template_file_content(self, filename, dicts): - import params - content = super(HdfsClient,self).generate_configs_get_template_file_content(filename, dicts) - if filename == 'log4j.properties': - content += params.rca_properties - return content - -if __name__ == "__main__": - HdfsClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/23b7c110/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs_datanode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs_datanode.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs_datanode.py deleted file mode 100644 index 0a106a7..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs_datanode.py +++ /dev/null @@ -1,63 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import os - -from resource_management import * -from resource_management.libraries.functions.dfs_datanode_helper import handle_dfs_data_dir -from utils import service - - -def create_dirs(data_dir, params): - """ - :param data_dir: The directory to create - :param params: parameters - """ - Directory(os.path.dirname(data_dir), - recursive=True, - mode=0755, - ignore_failures=True - ) - Directory(data_dir, - recursive=False, - mode=0750, - owner=params.hdfs_user, - group=params.user_group, - ignore_failures=True - ) - - -def datanode(action=None): - import params - - if action == "configure": - Directory(params.dfs_domain_socket_dir, - recursive=True, - mode=0751, - owner=params.hdfs_user, - group=params.user_group) - - handle_dfs_data_dir(create_dirs, params) - - elif action == "start" or action == "stop": - service( - action=action, name="datanode", - user=params.hdfs_user, - create_pid_dir=True, - create_log_dir=True - ) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/23b7c110/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs_namenode.py deleted file mode 100644 index 5a023eb..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs_namenode.py +++ /dev/null @@ -1,129 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management import * -from utils import service - - -def namenode(action=None, do_format=True): - import params - - if action == "configure": - create_name_dirs(params.dfs_name_dir) - - if action == "start": - if do_format: - format_namenode() - pass - service( - action="start", name="namenode", user=params.hdfs_user, - create_pid_dir=True, - create_log_dir=True - ) - - namenode_safe_mode_off = format("hadoop dfsadmin -safemode get | grep 'Safe mode is OFF'") - if params.security_enabled: - Execute(format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}"), - user = params.hdfs_user) - Execute(namenode_safe_mode_off, - tries=40, - try_sleep=10, - user=params.hdfs_user - ) - create_hdfs_directories() - - if action == "stop": - service( - action="stop", name="namenode", - user=params.hdfs_user, - ) - - if action == "decommission": - decommission() - -def create_name_dirs(directories): - import params - - dirs = directories.split(",") - Directory(dirs, - mode=0755, - owner=params.hdfs_user, - group=params.user_group, - recursive=True - ) - -def create_hdfs_directories(): - import params - - params.HdfsDirectory("/tmp", - action="create_delayed", - owner=params.hdfs_user, - mode=0777 - ) - params.HdfsDirectory(params.smoke_hdfs_user_dir, - action="create_delayed", - owner=params.smoke_user, - mode=params.smoke_hdfs_user_mode - ) - params.HdfsDirectory(None, action="create") - -def format_namenode(force=None): - import params - - mark_dir = params.namenode_formatted_mark_dir - dfs_name_dir = params.dfs_name_dir - hdfs_user = params.hdfs_user - hadoop_conf_dir = params.hadoop_conf_dir - - if True: - if force: - ExecuteHadoop('namenode -format', - kinit_override=True) - else: - File(format("{tmp_dir}/checkForFormat.sh"), - content=StaticFile("checkForFormat.sh"), - mode=0755) - Execute(format( - "{tmp_dir}/checkForFormat.sh {hdfs_user} {hadoop_conf_dir} {mark_dir} " - "{dfs_name_dir}"), - not_if=format("test -d {mark_dir}"), - path="/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin") - Directory(mark_dir, - recursive = True - ) - - -def decommission(): - import params - - hdfs_user = params.hdfs_user - conf_dir = params.hadoop_conf_dir - - File(params.exclude_file_path, - content=Template("exclude_hosts_list.j2"), - owner=hdfs_user, - group=params.user_group - ) - - if params.update_exclude_file_only == False: - ExecuteHadoop('dfsadmin -refreshNodes', - user=hdfs_user, - conf_dir=conf_dir, - kinit_override=True) - pass http://git-wip-us.apache.org/repos/asf/ambari/blob/23b7c110/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs_snamenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs_snamenode.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs_snamenode.py deleted file mode 100644 index 49241b4..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs_snamenode.py +++ /dev/null @@ -1,45 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management import * -from utils import service -from utils import hdfs_directory - - -def snamenode(action=None, format=False): - import params - - if action == "configure": - Directory(params.fs_checkpoint_dir, - recursive=True, - mode=0755, - owner=params.hdfs_user, - group=params.user_group) - File(params.exclude_file_path, - content=Template("exclude_hosts_list.j2"), - owner=params.hdfs_user, - group=params.user_group) - elif action == "start" or action == "stop": - service( - action=action, - name="secondarynamenode", - user=params.hdfs_user, - create_pid_dir=True, - create_log_dir=True - ) http://git-wip-us.apache.org/repos/asf/ambari/blob/23b7c110/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/namenode.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/namenode.py deleted file mode 100644 index 847e536..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/namenode.py +++ /dev/null @@ -1,68 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management import * -from hdfs_namenode import namenode -from hdfs import hdfs - - -class NameNode(Script): - def install(self, env): - import params - - self.install_packages(env, params.exclude_packages) - env.set_params(params) - - def start(self, env): - import params - - env.set_params(params) - self.configure(env) - namenode(action="start") - - def stop(self, env): - import params - - env.set_params(params) - namenode(action="stop") - - def configure(self, env): - import params - - env.set_params(params) - hdfs() - namenode(action="configure") - pass - - def status(self, env): - import status_params - - env.set_params(status_params) - check_process_status(status_params.namenode_pid_file) - pass - - def decommission(self, env): - import params - - env.set_params(params) - namenode(action="decommission") - pass - -if __name__ == "__main__": - NameNode().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/23b7c110/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/params.py deleted file mode 100644 index ae0a05d..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/params.py +++ /dev/null @@ -1,200 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management import * -import status_params -import os -import itertools - -config = Script.get_config() -tmp_dir = Script.get_tmp_dir() - -ulimit_cmd = "ulimit -c unlimited && " - -#security params -security_enabled = config['configurations']['cluster-env']['security_enabled'] -smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] -hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] - -#exclude file -hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", []) -exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude'] -update_exclude_file_only = default("/commandParams/update_exclude_file_only",False) - -kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) -#hosts -hostname = config["hostname"] -rm_host = default("/clusterHostInfo/rm_host", []) -slave_hosts = default("/clusterHostInfo/slave_hosts", []) -oozie_servers = default("/clusterHostInfo/oozie_server", []) -hcat_server_hosts = default("/clusterHostInfo/webhcat_server_host", []) -hive_server_host = default("/clusterHostInfo/hive_server_host", []) -hbase_master_hosts = default("/clusterHostInfo/hbase_master_hosts", []) -hs_host = default("/clusterHostInfo/hs_host", []) -jtnode_host = default("/clusterHostInfo/jtnode_host", []) -namenode_host = default("/clusterHostInfo/namenode_host", []) -nm_host = default("/clusterHostInfo/nm_host", []) -ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", []) -journalnode_hosts = default("/clusterHostInfo/journalnode_hosts", []) -zkfc_hosts = default("/clusterHostInfo/zkfc_hosts", []) - -has_ganglia_server = not len(ganglia_server_hosts) == 0 -has_namenodes = not len(namenode_host) == 0 -has_jobtracker = not len(jtnode_host) == 0 -has_resourcemanager = not len(rm_host) == 0 -has_histroryserver = not len(hs_host) == 0 -has_hbase_masters = not len(hbase_master_hosts) == 0 -has_slaves = not len(slave_hosts) == 0 -has_oozie_server = not len(oozie_servers) == 0 -has_hcat_server_host = not len(hcat_server_hosts) == 0 -has_hive_server_host = not len(hive_server_host) == 0 -has_journalnode_hosts = not len(journalnode_hosts) == 0 -has_zkfc_hosts = not len(zkfc_hosts) == 0 - - -is_namenode_master = hostname in namenode_host -is_jtnode_master = hostname in jtnode_host -is_rmnode_master = hostname in rm_host -is_hsnode_master = hostname in hs_host -is_hbase_master = hostname in hbase_master_hosts -is_slave = hostname in slave_hosts - -if has_ganglia_server: - ganglia_server_host = ganglia_server_hosts[0] - -#users and groups -hbase_user = config['configurations']['hbase-env']['hbase_user'] -oozie_user = config['configurations']['oozie-env']['oozie_user'] -webhcat_user = config['configurations']['hive-env']['hcat_user'] -hcat_user = config['configurations']['hive-env']['hcat_user'] -hive_user = config['configurations']['hive-env']['hive_user'] -smoke_user = config['configurations']['cluster-env']['smokeuser'] -mapred_user = config['configurations']['mapred-env']['mapred_user'] -hdfs_user = status_params.hdfs_user - -user_group = config['configurations']['cluster-env']['user_group'] -proxyuser_group = config['configurations']['hadoop-env']['proxyuser_group'] - -#hadoop params -hadoop_conf_dir = "/etc/hadoop/conf" -hadoop_pid_dir_prefix = status_params.hadoop_pid_dir_prefix -hadoop_bin = "/usr/lib/hadoop/bin" - -hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix'] -hadoop_root_logger = config['configurations']['hadoop-env']['hadoop_root_logger'] - -dfs_domain_socket_path = "/var/lib/hadoop-hdfs/dn_socket" -dfs_domain_socket_dir = os.path.dirname(dfs_domain_socket_path) - -hadoop_libexec_dir = "/usr/lib/hadoop/libexec" - -jn_edits_dir = config['configurations']['hdfs-site']['dfs.journalnode.edits.dir'] - -dfs_name_dir = config['configurations']['hdfs-site']['dfs.name.dir'] - -namenode_dirs_created_stub_dir = format("{hdfs_log_dir_prefix}/{hdfs_user}") -namenode_dirs_stub_filename = "namenode_dirs_created" - -smoke_hdfs_user_dir = format("/user/{smoke_user}") -smoke_hdfs_user_mode = 0770 - -namenode_formatted_mark_dir = format("{hadoop_pid_dir_prefix}/hdfs/namenode/formatted/") - -fs_checkpoint_dir = config['configurations']['core-site']['fs.checkpoint.dir'] - -dfs_data_dir = config['configurations']['hdfs-site']['dfs.data.dir'] -data_dir_mount_file = config['configurations']['hadoop-env']['dfs.datanode.data.dir.mount.file'] - -#for create_hdfs_directory -hostname = config["hostname"] -hadoop_conf_dir = "/etc/hadoop/conf" -hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] -hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] -hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] -kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) -import functools -#create partial functions with common arguments for every HdfsDirectory call -#to create hdfs directory we need to call params.HdfsDirectory in code -HdfsDirectory = functools.partial( - HdfsDirectory, - conf_dir=hadoop_conf_dir, - hdfs_user=hdfs_user, - security_enabled = security_enabled, - keytab = hdfs_user_keytab, - kinit_path_local = kinit_path_local -) -limits_conf_dir = "/etc/security/limits.d" - -io_compression_codecs = config['configurations']['core-site']['io.compression.codecs'] -lzo_enabled = "com.hadoop.compression.lzo" in io_compression_codecs - -lzo_packages_to_family = { - "any": ["hadoop-lzo"], - "redhat": ["lzo", "hadoop-lzo-native"], - "suse": ["lzo", "hadoop-lzo-native"], - "ubuntu": ["liblzo2-2"] -} -lzo_packages_for_current_host = lzo_packages_to_family['any'] + lzo_packages_to_family[System.get_instance().os_family] -all_lzo_packages = set(itertools.chain(*lzo_packages_to_family.values())) - -exclude_packages = [] -if not lzo_enabled: - exclude_packages += all_lzo_packages - -java_home = config['hostLevelParams']['java_home'] -#hadoop params - -hadoop_conf_empty_dir = "/etc/hadoop/conf.empty" -hadoop_env_sh_template = config['configurations']['hadoop-env']['content'] - -#hadoop-env.sh -if System.get_instance().os_family == "suse": - jsvc_path = "/usr/lib/bigtop-utils" -else: - jsvc_path = "/usr/libexec/bigtop-utils" -hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize'] -namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize'] -namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize'] -namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize'] -namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m") -namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m") - -jtnode_opt_newsize = default("/configurations/mapred-env/jtnode_opt_newsize","200m") -jtnode_opt_maxnewsize = default("/configurations/mapred-env/jtnode_opt_maxnewsize","200m") -jtnode_heapsize = default("/configurations/mapred-env/jtnode_heapsize","1024m") -ttnode_heapsize = default("/configurations/mapred-env/ttnode_heapsize","1024m") - -dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize'] - -mapred_pid_dir_prefix = default("/configurations/hadoop-env/mapred_pid_dir_prefix","/var/run/hadoop-mapreduce") -mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*" - -rca_enabled = False -if 'mapred-env' in config['configurations']: - rca_enabled = config['configurations']['mapred-env']['rca_enabled'] - -ambari_db_rca_url = config['hostLevelParams']['ambari_db_rca_url'] -ambari_db_rca_driver = config['hostLevelParams']['ambari_db_rca_driver'] -ambari_db_rca_username = config['hostLevelParams']['ambari_db_rca_username'] -ambari_db_rca_password = config['hostLevelParams']['ambari_db_rca_password'] - -rca_properties = '' -if rca_enabled and 'mapreduce-log4j' in config['configurations'] \ - and 'rca_properties' in config['configurations']['mapred-env']: - rca_properties = format(config['configurations']['mapred-env']['rca_properties']) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/23b7c110/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/service_check.py deleted file mode 100644 index be869fd..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/service_check.py +++ /dev/null @@ -1,104 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management import * - - -class HdfsServiceCheck(Script): - def service_check(self, env): - import params - - env.set_params(params) - unique = functions.get_unique_id_and_date() - dir = '/tmp' - tmp_file = format("{dir}/{unique}") - - safemode_command = "dfsadmin -safemode get | grep OFF" - - create_dir_cmd = format("fs -mkdir {dir} ; hadoop fs -chmod 777 {dir}") - test_dir_exists = as_user(format("hadoop fs -test -e {dir}"), params.smoke_user) - cleanup_cmd = format("fs -rm {tmp_file}") - #cleanup put below to handle retries; if retrying there wil be a stale file - #that needs cleanup; exit code is fn of second command - create_file_cmd = format( - "{cleanup_cmd}; hadoop fs -put /etc/passwd {tmp_file}") - test_cmd = format("fs -test -e {tmp_file}") - if params.security_enabled: - Execute(format("{kinit_path_local} -kt {smoke_user_keytab} {smoke_user}"), - user=params.smoke_user, - ) - ExecuteHadoop(safemode_command, - user=params.smoke_user, - logoutput=True, - conf_dir=params.hadoop_conf_dir, - try_sleep=15, - tries=20 - ) - ExecuteHadoop(create_dir_cmd, - user=params.smoke_user, - logoutput=True, - not_if=test_dir_exists, - conf_dir=params.hadoop_conf_dir, - try_sleep=3, - tries=5 - ) - ExecuteHadoop(create_file_cmd, - user=params.smoke_user, - logoutput=True, - conf_dir=params.hadoop_conf_dir, - try_sleep=3, - tries=5 - ) - ExecuteHadoop(test_cmd, - user=params.smoke_user, - logoutput=True, - conf_dir=params.hadoop_conf_dir, - try_sleep=3, - tries=5 - ) - if params.has_journalnode_hosts: - journalnode_port = params.journalnode_port - checkWebUIFileName = "checkWebUI.py" - checkWebUIFilePath = format("{tmp_dir}/{checkWebUIFileName}") - comma_sep_jn_hosts = ",".join(params.journalnode_hosts) - checkWebUICmd = format("python {checkWebUIFilePath} -m {comma_sep_jn_hosts} -p {journalnode_port}") - File(checkWebUIFilePath, - content=StaticFile(checkWebUIFileName)) - - Execute(checkWebUICmd, - logoutput=True, - try_sleep=3, - tries=5, - user=params.smoke_user - ) - - if params.has_zkfc_hosts: - pid_dir = format("{hadoop_pid_dir_prefix}/{hdfs_user}") - pid_file = format("{pid_dir}/hadoop-{hdfs_user}-zkfc.pid") - check_zkfc_process_cmd = format( - "ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1") - Execute(check_zkfc_process_cmd, - logoutput=True, - try_sleep=3, - tries=5 - ) - - -if __name__ == "__main__": - HdfsServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/23b7c110/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/snamenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/snamenode.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/snamenode.py deleted file mode 100644 index 5eb25d2..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/snamenode.py +++ /dev/null @@ -1,65 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management import * -from hdfs_snamenode import snamenode -from hdfs import hdfs - - -class SNameNode(Script): - def install(self, env): - import params - - env.set_params(params) - - self.install_packages(env, params.exclude_packages) - - - def start(self, env): - import params - - env.set_params(params) - - self.configure(env) - snamenode(action="start") - - def stop(self, env): - import params - - env.set_params(params) - - snamenode(action="stop") - - def configure(self, env): - import params - - env.set_params(params) - hdfs() - snamenode(action="configure") - - def status(self, env): - import status_params - - env.set_params(status_params) - - check_process_status(status_params.snamenode_pid_file) - - -if __name__ == "__main__": - SNameNode().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/23b7c110/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/status_params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/status_params.py deleted file mode 100644 index 0027a4c..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/status_params.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management import * - -config = Script.get_config() - -hadoop_pid_dir_prefix = config['configurations']['hadoop-env']['hadoop_pid_dir_prefix'] -hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] -hdp_pid_dir = format("{hadoop_pid_dir_prefix}/{hdfs_user}") -datanode_pid_file = format("{hdp_pid_dir}/hadoop-{hdfs_user}-datanode.pid") -namenode_pid_file = format("{hdp_pid_dir}/hadoop-{hdfs_user}-namenode.pid") -snamenode_pid_file = format("{hdp_pid_dir}/hadoop-{hdfs_user}-secondarynamenode.pid") -journalnode_pid_file = format("{hdp_pid_dir}/hadoop-{hdfs_user}-journalnode.pid") -zkfc_pid_file = format("{hdp_pid_dir}/hadoop-{hdfs_user}-zkfc.pid") http://git-wip-us.apache.org/repos/asf/ambari/blob/23b7c110/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/utils.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/utils.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/utils.py deleted file mode 100644 index f92225f..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/utils.py +++ /dev/null @@ -1,71 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management import * - - -def service(action=None, name=None, user=None, create_pid_dir=False, - create_log_dir=False): - import params - - pid_dir = format("{hadoop_pid_dir_prefix}/{user}") - pid_file = format("{pid_dir}/hadoop-{user}-{name}.pid") - log_dir = format("{hdfs_log_dir_prefix}/{user}") - check_process = format( - "ls {pid_file} >/dev/null 2>&1 &&" - " ps -p `cat {pid_file}` >/dev/null 2>&1") - hadoop_daemon = format("{hadoop_bin}/hadoop-daemon.sh") - hadoop_env_exports = {'HADOOP_LIBEXEC_DIR': params.hadoop_libexec_dir} - - if create_pid_dir: - Directory(pid_dir, - owner=user, - recursive=True) - if create_log_dir: - Directory(log_dir, - owner=user, - recursive=True) - - if params.security_enabled and name == "datanode": - user = "root" - pid_file = format( - "{hadoop_pid_dir_prefix}/{hdfs_user}/hadoop-{hdfs_user}-{name}.pid") - - if user == "root": - cmd = [hadoop_daemon, "--config", params.hadoop_conf_dir] - daemon_cmd = as_sudo(cmd + [action, name]) - else: - cmd = format("{hadoop_daemon} --config {hadoop_conf_dir}") - daemon_cmd = as_user(format("{ulimit_cmd} {cmd} {action} {name}"), user) - - service_is_up = check_process if action == "start" else None - #remove pid file from dead process - File(pid_file, - action="delete", - not_if=check_process, - - ) - Execute(daemon_cmd, - not_if=service_is_up, - environment=hadoop_env_exports, - ) - if action == "stop": - File(pid_file, - action="delete", - ) http://git-wip-us.apache.org/repos/asf/ambari/blob/23b7c110/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/templates/exclude_hosts_list.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/templates/exclude_hosts_list.j2 b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/templates/exclude_hosts_list.j2 deleted file mode 100644 index 1ec8cfe..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/templates/exclude_hosts_list.j2 +++ /dev/null @@ -1,22 +0,0 @@ -{# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - - -{% for host in hdfs_exclude_file %} -{{host}} -{% endfor %} \ No newline at end of file
