AMBARI-19093. HDP 3.0 support for HDFS with configs, kerberos, widgets, metrics, quicklinks, and themes (alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2a710b35 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2a710b35 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2a710b35 Branch: refs/heads/branch-feature-AMBARI-18456 Commit: 2a710b3587965c09e7b034ea79a7eca31fc35c25 Parents: aed9839 Author: Alejandro Fernandez <[email protected]> Authored: Mon Dec 5 15:00:07 2016 -0800 Committer: Alejandro Fernandez <[email protected]> Committed: Tue Dec 6 16:53:31 2016 -0800 ---------------------------------------------------------------------- .../services/HDFS/configuration/core-site.xml | 56 ++ .../services/HDFS/configuration/hadoop-env.xml | 200 ++++++ .../services/HDFS/configuration/hdfs-log4j.xml | 226 +++++++ .../services/HDFS/configuration/hdfs-site.xml | 153 +++++ .../HDFS/configuration/ranger-hdfs-audit.xml | 217 +++++++ .../ranger-hdfs-plugin-properties.xml | 98 +++ .../configuration/ranger-hdfs-policymgr-ssl.xml | 67 ++ .../HDFS/configuration/ranger-hdfs-security.xml | 65 ++ .../services/HDFS/configuration/widgets.json | 649 +++++++++++++++++++ .../stacks/HDP/3.0/services/HDFS/kerberos.json | 246 +++++++ .../stacks/HDP/3.0/services/HDFS/metainfo.xml | 190 ++++++ .../services/HDFS/quicklinks/quicklinks.json | 80 +++ .../HDP/3.0/services/HDFS/themes/theme.json | 179 +++++ 13 files changed, 2426 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/2a710b35/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/core-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/core-site.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/core-site.xml new file mode 100644 index 0000000..9dcf561 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/core-site.xml @@ -0,0 +1,56 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> +<!-- Put site-specific property overrides in this file. --> +<configuration xmlns:xi="http://www.w3.org/2001/XInclude"; supports_final="true"> + <!-- These configs were inherited from HDP 2.2 --> + <property> + <name>hadoop.http.authentication.simple.anonymous.allowed</name> + <value>true</value> + <description> + Indicates if anonymous requests are allowed when using 'simple' authentication. + </description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>hadoop.security.key.provider.path</name> + <value/> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <depends-on> + <property> + <type>hadoop-env</type> + <name>keyserver_host</name> + </property> + <property> + <type>hadoop-env</type> + <name>keyserver_port</name> + </property> + <property> + <type>kms-env</type> + <name>kms_port</name> + </property> + <property> + <type>ranger-kms-site</type> + <name>ranger.service.https.attrib.ssl.enabled</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/2a710b35/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hadoop-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hadoop-env.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hadoop-env.xml new file mode 100644 index 0000000..8697740 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hadoop-env.xml @@ -0,0 +1,200 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_adding_forbidden="true"> + <!-- These configs were inherited from HDP 2.2 --> + <property> + <name>keyserver_host</name> + <value> </value> + <display-name>Key Server Host</display-name> + <description>Hostnames where Key Management Server is installed</description> + <value-attributes> + <type>string</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>keyserver_port</name> + <value/> + <display-name>Key Server Port</display-name> + <description>Port number where Key Management Server is available</description> + <value-attributes> + <type>int</type> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + + <!-- These configs were inherited from HDP 2.3 --> + <!-- hadoop-env.sh --> + <property> + <name>content</name> + <display-name>hadoop-env template</display-name> + <description>This is the jinja template for hadoop-env.sh file</description> + <value> +# Set Hadoop-specific environment variables here. + +# The only required environment variable is JAVA_HOME. All others are +# optional. When running a distributed configuration it is best to +# set JAVA_HOME in this file, so that it is correctly defined on +# remote nodes. + +# The java implementation to use. Required. +export JAVA_HOME={{java_home}} +export HADOOP_HOME_WARN_SUPPRESS=1 + +# Hadoop home directory +export HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}} + +# Hadoop Configuration Directory + +{# this is different for HDP1 #} +# Path to jsvc required by secure HDP 2.0 datanode +export JSVC_HOME={{jsvc_path}} + + +# The maximum amount of heap to use, in MB. Default is 1000. +export HADOOP_HEAPSIZE="{{hadoop_heapsize}}" + +export HADOOP_NAMENODE_INIT_HEAPSIZE="-Xms{{namenode_heapsize}}" + +# Extra Java runtime options. Empty by default. +export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}" + +# Command specific options appended to HADOOP_OPTS when specified +HADOOP_JOBTRACKER_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx{{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}" + +HADOOP_TASKTRACKER_OPTS="-server -Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}" + +{% if java_version < 8 %} +SHARED_HADOOP_NAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSize={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT" +export HADOOP_NAMENODE_OPTS="${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\"/usr/hdp/current/hadoop-hdfs-namenode/bin/kill-name-node\" -Dorg.mortbay.jetty.Request.maxFormContentSize=-1 ${HADOOP_NAMENODE_OPTS}" +export HADOOP_DATANODE_OPTS="-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -XX:PermSize=128m -XX:MaxPermSize=256m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly" + +export HADOOP_SECONDARYNAMENODE_OPTS="${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\"/usr/hdp/current/hadoop-hdfs-secondarynamenode/bin/kill-secondary-name-node\" ${HADOOP_SECONDARYNAMENODE_OPTS}" + +# The following applies to multiple commands (fs, dfs, fsck, distcp etc) +export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m -XX:MaxPermSize=512m $HADOOP_CLIENT_OPTS" + +{% else %} +SHARED_HADOOP_NAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT" +export HADOOP_NAMENODE_OPTS="${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\"/usr/hdp/current/hadoop-hdfs-namenode/bin/kill-name-node\" -Dorg.mortbay.jetty.Request.maxFormContentSize=-1 ${HADOOP_NAMENODE_OPTS}" +export HADOOP_DATANODE_OPTS="-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly" + +export HADOOP_SECONDARYNAMENODE_OPTS="${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\"/usr/hdp/current/hadoop-hdfs-secondarynamenode/bin/kill-secondary-name-node\" ${HADOOP_SECONDARYNAMENODE_OPTS}" + +# The following applies to multiple commands (fs, dfs, fsck, distcp etc) +export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS" +{% endif %} + +HADOOP_NFS3_OPTS="-Xmx{{nfsgateway_heapsize}}m -Dhadoop.security.logger=ERROR,DRFAS ${HADOOP_NFS3_OPTS}" +HADOOP_BALANCER_OPTS="-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCER_OPTS}" + + +# On secure datanodes, user to run the datanode as after dropping privileges +export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER:-{{hadoop_secure_dn_user}}} + +# Extra ssh options. Empty by default. +export HADOOP_SSH_OPTS="-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR" + +# Where log files are stored. $HADOOP_HOME/logs by default. +export HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER + +# History server logs +export HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}/$USER + +# Where log files are stored in the secure data environment. +export HADOOP_SECURE_DN_LOG_DIR={{hdfs_log_dir_prefix}}/$HADOOP_SECURE_DN_USER + +# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default. +# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves + +# host:path where hadoop code should be rsync'd from. Unset by default. +# export HADOOP_MASTER=master:/home/$USER/src/hadoop + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export HADOOP_SLAVE_SLEEP=0.1 + +# The directory where pid files are stored. /tmp by default. +export HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}/$USER +export HADOOP_SECURE_DN_PID_DIR={{hadoop_pid_dir_prefix}}/$HADOOP_SECURE_DN_USER + +# History server pid +export HADOOP_MAPRED_PID_DIR={{mapred_pid_dir_prefix}}/$USER + +YARN_RESOURCEMANAGER_OPTS="-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY" + +# A string representing this instance of hadoop. $USER by default. +export HADOOP_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. + +# export HADOOP_NICENESS=10 + +# Add database libraries +JAVA_JDBC_LIBS="" +if [ -d "/usr/share/java" ]; then + for jarFile in `ls /usr/share/java | grep -E "(mysql|ojdbc|postgresql|sqljdbc)" 2>/dev/null` + do + JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile + done +fi + +# Add libraries to the hadoop classpath - some may not need a colon as they already include it +export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS} + +# Setting path to hdfs command line +export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}} + +# Mostly required for hadoop 2.0 +export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH} + +export HADOOP_OPTS="-Dhdp.version=$HDP_VERSION $HADOOP_OPTS" + + +# Fix temporary bug, when ulimit from conf files is not picked up, without full relogin. +# Makes sense to fix only when runing DN as root +if [ "$command" == "datanode" ] && [ "$EUID" -eq 0 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then + {% if is_datanode_max_locked_memory_set %} + ulimit -l {{datanode_max_locked_memory}} + {% endif %} + ulimit -n {{hdfs_user_nofile_limit}} +fi + </value> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>nfsgateway_heapsize</name> + <display-name>NFSGateway maximum Java heap size</display-name> + <value>1024</value> + <description>Maximum Java heap size for NFSGateway (Java option -Xmx)</description> + <value-attributes> + <type>int</type> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/2a710b35/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hdfs-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hdfs-log4j.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hdfs-log4j.xml new file mode 100644 index 0000000..215a6ee --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hdfs-log4j.xml @@ -0,0 +1,226 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <!-- These configs were inherited from HDP 2.2 --> + <property> + <name>content</name> + <display-name>hdfs-log4j template</display-name> + <description>Custom log4j.properties</description> + <value> +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +# Define some default values that can be overridden by system properties +# To change daemon root logger use hadoop_root_logger in hadoop-env +hadoop.root.logger=INFO,console +hadoop.log.dir=. +hadoop.log.file=hadoop.log + + +# Define the root logger to the system property "hadoop.root.logger". +log4j.rootLogger=${hadoop.root.logger}, EventCounter + +# Logging Threshold +log4j.threshhold=ALL + +# +# Daily Rolling File Appender +# + +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +# Debugging Pattern format +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + + +# +# console +# Add "console" to rootlogger above if you want to use this +# + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n + +# +# TaskLog Appender +# + +#Default values +hadoop.tasklog.taskid=null +hadoop.tasklog.iscleanup=false +hadoop.tasklog.noKeepSplits=4 +hadoop.tasklog.totalLogFileSize=100 +hadoop.tasklog.purgeLogSplits=true +hadoop.tasklog.logsRetainHours=12 + +log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender +log4j.appender.TLA.taskId=${hadoop.tasklog.taskid} +log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup} +log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize} + +log4j.appender.TLA.layout=org.apache.log4j.PatternLayout +log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n + +# +#Security audit appender +# +hadoop.security.logger=INFO,console +hadoop.security.log.maxfilesize=256MB +hadoop.security.log.maxbackupindex=20 +log4j.category.SecurityLogger=${hadoop.security.logger} +hadoop.security.log.file=SecurityAuth.audit +log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} +log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout +log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd + +log4j.appender.RFAS=org.apache.log4j.RollingFileAppender +log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} +log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout +log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize} +log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex} + +# +# hdfs audit logging +# +hdfs.audit.logger=INFO,console +log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger} +log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false +log4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log +log4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout +log4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n +log4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd + +# +# NameNode metrics logging. +# The default is to retain two namenode-metrics.log files up to 64MB each. +# +namenode.metrics.logger=INFO,NullAppender +log4j.logger.NameNodeMetricsLog=${namenode.metrics.logger} +log4j.additivity.NameNodeMetricsLog=false +log4j.appender.NNMETRICSRFA=org.apache.log4j.RollingFileAppender +log4j.appender.NNMETRICSRFA.File=${hadoop.log.dir}/namenode-metrics.log +log4j.appender.NNMETRICSRFA.layout=org.apache.log4j.PatternLayout +log4j.appender.NNMETRICSRFA.layout.ConversionPattern=%d{ISO8601} %m%n +log4j.appender.NNMETRICSRFA.MaxBackupIndex=1 +log4j.appender.NNMETRICSRFA.MaxFileSize=64MB + +# +# mapred audit logging +# +mapred.audit.logger=INFO,console +log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger} +log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false +log4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender +log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log +log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout +log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n +log4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd + +# +# Rolling File Appender +# + +log4j.appender.RFA=org.apache.log4j.RollingFileAppender +log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file} + +# Logfile size and and 30-day backups +log4j.appender.RFA.MaxFileSize=256MB +log4j.appender.RFA.MaxBackupIndex=10 + +log4j.appender.RFA.layout=org.apache.log4j.PatternLayout +log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n +log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + + +# Custom Logging levels + +hadoop.metrics.log.level=INFO +#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG +#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG +#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG +log4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level} + +# Jets3t library +log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR + +# +# Null Appender +# Trap security logger on the hadoop client side +# +log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender + +# +# Event Counter Appender +# Sends counts of logging messages at different severity levels to Hadoop Metrics. +# +log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter + +# Removes "deprecated" messages +log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN + +# +# HDFS block state change log from block manager +# +# Uncomment the following to suppress normal block state change +# messages from BlockManager in NameNode. +#log4j.logger.BlockStateChange=WARN + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/2a710b35/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hdfs-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hdfs-site.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hdfs-site.xml new file mode 100644 index 0000000..ac141d1 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/hdfs-site.xml @@ -0,0 +1,153 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<!-- Put site-specific property overrides in this file. --> +<configuration supports_final="true"> + <!-- These configs were inherited from HDP 2.1 --> + <property> + <name>dfs.namenode.audit.log.async</name> + <value>true</value> + <description>Whether to enable async auditlog</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>dfs.namenode.fslock.fair</name> + <value>false</value> + <description>Whether fsLock is fair</description> + <on-ambari-upgrade add="true"/> + </property> + + <!-- These configs were inherited from HDP 2.2 --> + <property> + <name>dfs.namenode.startup.delay.block.deletion.sec</name> + <value>3600</value> + <description> + The delay in seconds at which we will pause the blocks deletion + after Namenode startup. By default it's disabled. + In the case a directory has large number of directories and files are + deleted, suggested delay is one hour to give the administrator enough time + to notice large number of pending deletion blocks and take corrective + action. + </description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>dfs.journalnode.edits.dir</name> + <value>/hadoop/hdfs/journalnode</value> + <description>The path where the JournalNode daemon will store its local state. </description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>dfs.client.retry.policy.enabled</name> + <value>false</value> + <description>Enables HDFS client retry in the event of a NameNode failure.</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>dfs.content-summary.limit</name> + <value>5000</value> + <description>Dfs content summary limit.</description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>dfs.encryption.key.provider.uri</name> + <description> + The KeyProvider to use when interacting with encryption keys used + when reading and writing to an encryption zone. + </description> + <value/> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <depends-on> + <property> + <type>hadoop-env</type> + <name>keyserver_host</name> + </property> + <property> + <type>hadoop-env</type> + <name>keyserver_port</name> + </property> + <property> + <type>kms-env</type> + <name>kms_port</name> + </property> + <property> + <type>ranger-kms-site</type> + <name>ranger.service.https.attrib.ssl.enabled</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + + <!-- These configs were inherited from HDP 2.3 --> + <property> + <name>nfs.file.dump.dir</name> + <value>/tmp/.hdfs-nfs</value> + <display-name>NFSGateway dump directory</display-name> + <description> + This directory is used to temporarily save out-of-order writes before + writing to HDFS. For each file, the out-of-order writes are dumped after + they are accumulated to exceed certain threshold (e.g., 1MB) in memory. + One needs to make sure the directory has enough space. + </description> + <value-attributes> + <type>directory</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>nfs.exports.allowed.hosts</name> + <value>* rw</value> + <description> + By default, the export can be mounted by any client. To better control the access, + users can update the following property. The value string contains machine name and access privilege, + separated by whitespace characters. Machine name format can be single host, wildcards, and IPv4 + networks.The access privilege uses rw or ro to specify readwrite or readonly access of the machines + to exports. If the access privilege is not provided, the default is read-only. Entries are separated + by ";". For example: "192.168.0.0/22 rw ; host*.example.com ; host1.test.org ro;". + </description> + <display-name>Allowed hosts</display-name> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>dfs.encrypt.data.transfer.cipher.suites</name> + <value>AES/CTR/NoPadding</value> + <description> + This value may be either undefined or AES/CTR/NoPadding. If defined, then + dfs.encrypt.data.transfer uses the specified cipher suite for data encryption. + If not defined, then only the algorithm specified in dfs.encrypt.data.transfer.algorithm + is used. By default, the property is not defined. + </description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>dfs.namenode.inode.attributes.provider.class</name> + <description>Enable ranger hdfs plugin</description> + <depends-on> + <property> + <type>ranger-hdfs-plugin-properties</type> + <name>ranger-hdfs-plugin-enabled</name> + </property> + </depends-on> + <value-attributes> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/2a710b35/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-audit.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-audit.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-audit.xml new file mode 100644 index 0000000..fd41817 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-audit.xml @@ -0,0 +1,217 @@ +<?xml version="1.0" encoding="UTF-8"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <!-- These configs were inherited from HDP 2.3 --> + <property> + <name>xasecure.audit.is.enabled</name> + <value>true</value> + <description>Is Audit enabled?</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.audit.destination.db</name> + <value>false</value> + <display-name>Audit to DB</display-name> + <description>Is Audit to DB enabled?</description> + <value-attributes> + <type>boolean</type> + </value-attributes> + <depends-on> + <property> + <type>ranger-env</type> + <name>xasecure.audit.destination.db</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.audit.destination.db.jdbc.url</name> + <value>{{audit_jdbc_url}}</value> + <description>Audit DB JDBC URL</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.audit.destination.db.user</name> + <value>{{xa_audit_db_user}}</value> + <description>Audit DB JDBC User</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.audit.destination.db.password</name> + <value>crypted</value> + <property-type>PASSWORD</property-type> + <description>Audit DB JDBC Password</description> + <value-attributes> + <type>password</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.audit.destination.db.jdbc.driver</name> + <value>{{jdbc_driver}}</value> + <description>Audit DB JDBC Driver</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.audit.credential.provider.file</name> + <value>jceks://file{{credential_file}}</value> + <description>Credential file store</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.audit.destination.db.batch.filespool.dir</name> + <value>/var/log/hadoop/hdfs/audit/db/spool</value> + <description>/var/log/hadoop/hdfs/audit/db/spool</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.audit.destination.hdfs</name> + <value>true</value> + <display-name>Audit to HDFS</display-name> + <description>Is Audit to HDFS enabled?</description> + <value-attributes> + <type>boolean</type> + </value-attributes> + <depends-on> + <property> + <type>ranger-env</type> + <name>xasecure.audit.destination.hdfs</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.audit.destination.hdfs.dir</name> + <value>hdfs://NAMENODE_HOSTNAME:8020/ranger/audit</value> + <description>HDFS folder to write audit to, make sure the service user has requried permissions</description> + <depends-on> + <property> + <type>ranger-env</type> + <name>xasecure.audit.destination.hdfs.dir</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.audit.destination.hdfs.batch.filespool.dir</name> + <value>/var/log/hadoop/hdfs/audit/hdfs/spool</value> + <description>/var/log/hadoop/hdfs/audit/hdfs/spool</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.audit.destination.solr</name> + <value>false</value> + <display-name>Audit to SOLR</display-name> + <description>Is Solr audit enabled?</description> + <value-attributes> + <type>boolean</type> + </value-attributes> + <depends-on> + <property> + <type>ranger-env</type> + <name>xasecure.audit.destination.solr</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.audit.destination.solr.urls</name> + <value/> + <description>Solr URL</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <depends-on> + <property> + <type>ranger-admin-site</type> + <name>ranger.audit.solr.urls</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.audit.destination.solr.zookeepers</name> + <value>NONE</value> + <description>Solr Zookeeper string</description> + <depends-on> + <property> + <type>ranger-admin-site</type> + <name>ranger.audit.solr.zookeepers</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.audit.destination.solr.batch.filespool.dir</name> + <value>/var/log/hadoop/hdfs/audit/solr/spool</value> + <description>/var/log/hadoop/hdfs/audit/solr/spool</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.audit.provider.summary.enabled</name> + <value>false</value> + <display-name>Audit provider summary enabled</display-name> + <description>Enable Summary audit?</description> + <value-attributes> + <type>boolean</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + + <!-- These configs are deleted in HDP 2.5. --> + <property> + <name>xasecure.audit.destination.db</name> + <deleted>true</deleted> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.audit.destination.db.jdbc.url</name> + <deleted>true</deleted> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.audit.destination.db.user</name> + <deleted>true</deleted> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.audit.destination.db.password</name> + <deleted>true</deleted> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.audit.destination.db.jdbc.driver</name> + <deleted>true</deleted> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.audit.credential.provider.file</name> + <deleted>true</deleted> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.audit.destination.db.batch.filespool.dir</name> + <deleted>true</deleted> + <on-ambari-upgrade add="false"/> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/2a710b35/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-plugin-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-plugin-properties.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-plugin-properties.xml new file mode 100644 index 0000000..b31742c --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-plugin-properties.xml @@ -0,0 +1,98 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="true"> + <!-- These configs were inherited from HDP 2.2 --> + <property> + <name>policy_user</name> + <value>ambari-qa</value> + <display-name>Policy user for HDFS</display-name> + <description>This user must be system user and also present at Ranger + admin portal</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>hadoop.rpc.protection</name> + <value/> + <description>Used for repository creation on ranger admin + </description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>common.name.for.certificate</name> + <value/> + <description>Common name for certificate, this value should match what is specified in repo within ranger admin</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>ranger-hdfs-plugin-enabled</name> + <value>No</value> + <display-name>Enable Ranger for HDFS</display-name> + <description>Enable ranger hdfs plugin</description> + <depends-on> + <property> + <type>ranger-env</type> + <name>ranger-hdfs-plugin-enabled</name> + </property> + </depends-on> + <value-attributes> + <type>boolean</type> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>REPOSITORY_CONFIG_USERNAME</name> + <value>hadoop</value> + <display-name>Ranger repository config user</display-name> + <description>Used for repository creation on ranger admin + </description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>REPOSITORY_CONFIG_PASSWORD</name> + <value>hadoop</value> + <display-name>Ranger repository config password</display-name> + <property-type>PASSWORD</property-type> + <description>Used for repository creation on ranger admin + </description> + <value-attributes> + <type>password</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + + <!-- These configs were inherited from HDP 2.5 --> + <property> + <name>hadoop.rpc.protection</name> + <value>authentication</value> + <description>Used for repository creation on ranger admin</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="false" /> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/2a710b35/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-policymgr-ssl.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-policymgr-ssl.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-policymgr-ssl.xml new file mode 100644 index 0000000..1bc83df --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-policymgr-ssl.xml @@ -0,0 +1,67 @@ +<?xml version="1.0"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <!-- These configs were inherited from HDP 2.3 --> + <property> + <name>xasecure.policymgr.clientssl.keystore</name> + <value>/usr/hdp/current/hadoop-client/conf/ranger-plugin-keystore.jks</value> + <description>Java Keystore files</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.policymgr.clientssl.keystore.password</name> + <value>myKeyFilePassword</value> + <property-type>PASSWORD</property-type> + <description>password for keystore</description> + <value-attributes> + <type>password</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.policymgr.clientssl.truststore</name> + <value>/usr/hdp/current/hadoop-client/conf/ranger-plugin-truststore.jks</value> + <description>java truststore file</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.policymgr.clientssl.truststore.password</name> + <value>changeit</value> + <property-type>PASSWORD</property-type> + <description>java truststore password</description> + <value-attributes> + <type>password</type> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.policymgr.clientssl.keystore.credential.file</name> + <value>jceks://file{{credential_file}}</value> + <description>java keystore credential file</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.policymgr.clientssl.truststore.credential.file</name> + <value>jceks://file{{credential_file}}</value> + <description>java truststore credential file</description> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/2a710b35/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-security.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-security.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-security.xml new file mode 100644 index 0000000..1b0a821 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/ranger-hdfs-security.xml @@ -0,0 +1,65 @@ +<?xml version="1.0"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <!-- These configs were inherited from HDP 2.3 --> + <property> + <name>ranger.plugin.hdfs.service.name</name> + <value>{{repo_name}}</value> + <description>Name of the Ranger service containing Hdfs policies</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>ranger.plugin.hdfs.policy.source.impl</name> + <value>org.apache.ranger.admin.client.RangerAdminRESTClient</value> + <description>Class to retrieve policies from the source</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>ranger.plugin.hdfs.policy.rest.url</name> + <value>{{policymgr_mgr_url}}</value> + <description>URL to Ranger Admin</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>ranger.plugin.hdfs.policy.rest.ssl.config.file</name> + <value>/etc/hadoop/conf/ranger-policymgr-ssl.xml</value> + <description>Path to the file containing SSL details to contact Ranger Admin</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>ranger.plugin.hdfs.policy.pollIntervalMs</name> + <value>30000</value> + <description>How often to poll for changes in policies?</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>ranger.plugin.hdfs.policy.cache.dir</name> + <value>/etc/ranger/{{repo_name}}/policycache</value> + <description>Directory where Ranger policies are cached after successful retrieval from the source</description> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>xasecure.add-hadoop-authorization</name> + <value>true</value> + <description>Enable/Disable the default hadoop authorization (based on rwxrwxrwx permission on the resource) if Ranger Authorization fails.</description> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/2a710b35/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/widgets.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/widgets.json b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/widgets.json new file mode 100644 index 0000000..4a645b0 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/configuration/widgets.json @@ -0,0 +1,649 @@ +{ + "layouts": [ + { + "layout_name": "default_hdfs_dashboard", + "display_name": "Standard HDFS Dashboard", + "section_name": "HDFS_SUMMARY", + "widgetLayoutInfo": [ + { + "widget_name": "NameNode GC count", + "description": "Count of total garbage collections and count of major type garbage collections of the JVM.", + "widget_type": "GRAPH", + "is_visible": true, + "metrics": [ + { + "name": "jvm.JvmMetrics.GcCount._rate", + "metric_path": "metrics/jvm/gcCount._rate", + "service_name": "HDFS", + "component_name": "NAMENODE", + "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active" + }, + { + "name": "jvm.JvmMetrics.GcCountConcurrentMarkSweep._rate", + "metric_path": "metrics/jvm/GcCountConcurrentMarkSweep._rate", + "service_name": "HDFS", + "component_name": "NAMENODE", + "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active" + } + ], + "values": [ + { + "name": "GC total count", + "value": "${jvm.JvmMetrics.GcCount._rate}" + }, + { + "name": "GC count of type major collection", + "value": "${jvm.JvmMetrics.GcCountConcurrentMarkSweep._rate}" + } + ], + "properties": { + "graph_type": "LINE", + "time_range": "1" + } + }, + { + "widget_name": "NameNode GC time", + "description": "Total time taken by major type garbage collections in milliseconds.", + "widget_type": "GRAPH", + "is_visible": true, + "metrics": [ + { + "name": "jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep._rate", + "metric_path": "metrics/jvm/GcTimeMillisConcurrentMarkSweep._rate", + "service_name": "HDFS", + "component_name": "NAMENODE", + "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active" + } + ], + "values": [ + { + "name": "GC time in major collection", + "value": "${jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep._rate}" + } + ], + "properties": { + "display_unit": "ms", + "graph_type": "LINE", + "time_range": "1" + } + }, + { + "widget_name": "NN Connection Load", + "description": "Number of open RPC connections being managed by NameNode.", + "widget_type": "GRAPH", + "is_visible": true, + "metrics": [ + { + "name": "rpc.rpc.client.NumOpenConnections", + "metric_path": "metrics/rpc/client/NumOpenConnections", + "category": "", + "service_name": "HDFS", + "component_name": "NAMENODE", + "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active" + }, + { + "name": "rpc.rpc.datanode.NumOpenConnections", + "metric_path": "metrics/rpc/datanode/NumOpenConnections", + "category": "", + "service_name": "HDFS", + "component_name": "NAMENODE", + "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active" + } + ], + "values": [ + { + "name": "Open Client Connections", + "value": "${rpc.rpc.client.NumOpenConnections}" + }, + { + "name": "Open Datanode Connections", + "value": "${rpc.rpc.datanode.NumOpenConnections}" + } + ], + "properties": { + "graph_type": "LINE", + "time_range": "1" + } + }, + { + "widget_name": "NameNode Heap", + "description": "Heap memory committed and Heap memory used with respect to time.", + "widget_type": "GRAPH", + "is_visible": true, + "metrics": [ + { + "name": "jvm.JvmMetrics.MemHeapCommittedM", + "metric_path": "metrics/jvm/memHeapCommittedM", + "service_name": "HDFS", + "component_name": "NAMENODE", + "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active" + }, + { + "name": "jvm.JvmMetrics.MemHeapUsedM", + "metric_path": "metrics/jvm/memHeapUsedM", + "service_name": "HDFS", + "component_name": "NAMENODE", + "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active" + } + ], + "values": [ + { + "name": "JVM heap committed", + "value": "${jvm.JvmMetrics.MemHeapCommittedM}" + }, + { + "name": "JVM heap used", + "value": "${jvm.JvmMetrics.MemHeapUsedM}" + } + ], + "properties": { + "display_unit": "MB", + "graph_type": "LINE", + "time_range": "1" + } + }, + { + "widget_name": "NameNode Host Load", + "description": "Percentage of CPU and Memory resources being consumed on NameNode host.", + "widget_type": "GRAPH", + "is_visible": true, + "metrics": [ + { + "name": "cpu_system", + "metric_path": "metrics/cpu/cpu_system", + "service_name": "HDFS", + "component_name": "NAMENODE", + "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active" + }, + { + "name": "cpu_user", + "metric_path": "metrics/cpu/cpu_user", + "service_name": "HDFS", + "component_name": "NAMENODE", + "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active" + }, + { + "name": "cpu_nice", + "metric_path": "metrics/cpu/cpu_nice", + "service_name": "HDFS", + "component_name": "NAMENODE", + "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active" + }, + { + "name": "cpu_idle", + "metric_path": "metrics/cpu/cpu_idle", + "service_name": "HDFS", + "component_name": "NAMENODE", + "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active" + }, + { + "name": "cpu_wio", + "metric_path": "metrics/cpu/cpu_wio", + "service_name": "HDFS", + "component_name": "NAMENODE", + "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active" + }, + { + "name": "mem_total", + "metric_path": "metrics/memory/mem_total", + "service_name": "HDFS", + "component_name": "NAMENODE", + "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active" + }, + { + "name": "mem_free", + "metric_path": "metrics/memory/mem_free", + "service_name": "HDFS", + "component_name": "NAMENODE", + "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active" + } + ], + "values": [ + { + "name": "CPU utilization", + "value": "${((cpu_system + cpu_user + cpu_nice)/(cpu_system + cpu_user + cpu_nice + cpu_idle + cpu_wio)) * 100}" + }, + { + "name": "Memory utilization", + "value": "${((mem_total - mem_free)/mem_total) * 100}" + } + ], + "properties": { + "graph_type": "LINE", + "time_range": "1", + "display_unit": "%" + } + }, + { + "widget_name": "NameNode RPC", + "description": "Compares the average time spent for RPC request in a queue and RPC request being processed.", + "widget_type": "GRAPH", + "is_visible": true, + "metrics": [ + { + "name": "rpc.rpc.client.RpcQueueTimeAvgTime", + "metric_path": "metrics/rpc/client/RpcQueueTime_avg_time", + "service_name": "HDFS", + "component_name": "NAMENODE", + "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active" + }, + { + "name": "rpc.rpc.client.RpcProcessingTimeAvgTime", + "metric_path": "metrics/rpc/client/RpcProcessingTime_avg_time", + "service_name": "HDFS", + "component_name": "NAMENODE", + "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active" + }, + { + "name": "rpc.rpc.datanode.RpcQueueTimeAvgTime", + "metric_path": "metrics/rpc/datanode/RpcQueueTime_avg_time", + "service_name": "HDFS", + "component_name": "NAMENODE", + "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active" + }, + { + "name": "rpc.rpc.datanode.RpcProcessingTimeAvgTime", + "metric_path": "metrics/rpc/datanode/RpcProcessingTime_avg_time", + "service_name": "HDFS", + "component_name": "NAMENODE", + "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active" + } + ], + "values": [ + { + "name": "Client RPC Queue Wait time", + "value": "${rpc.rpc.client.RpcQueueTimeAvgTime}" + }, + { + "name": "Client RPC Processing time", + "value": "${rpc.rpc.client.RpcProcessingTimeAvgTime}" + }, + { + "name": "Datanode RPC Queue Wait time", + "value": "${rpc.rpc.datanode.RpcQueueTimeAvgTime}" + }, + { + "name": "Datanode RPC Processing time", + "value": "${rpc.rpc.datanode.RpcProcessingTimeAvgTime}" + } + ], + "properties": { + "graph_type": "LINE", + "time_range": "1", + "display_unit": "ms" + } + }, + { + "widget_name": "NameNode Operations", + "description": "Rate per second of number of file operation over time.", + "widget_type": "GRAPH", + "is_visible": false, + "metrics": [ + { + "name": "dfs.namenode.TotalFileOps._rate", + "metric_path": "metrics/dfs/namenode/TotalFileOps._rate", + "service_name": "HDFS", + "component_name": "NAMENODE", + "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active" + } + ], + "values": [ + { + "name": "NameNode File Operations", + "value": "${dfs.namenode.TotalFileOps._rate}" + } + ], + "properties": { + "graph_type": "LINE", + "time_range": "1" + } + }, + { + "widget_name": "Failed disk volumes", + "description": "Number of Failed disk volumes across all DataNodes. Its indicative of HDFS bad health.", + "widget_type": "NUMBER", + "is_visible": true, + "metrics": [ + { + "name": "FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.NumFailedVolumes._sum", + "metric_path": "metrics/dfs/datanode/NumFailedVolumes", + "service_name": "HDFS", + "component_name": "DATANODE" + } + ], + "values": [ + { + "name": "Failed disk volumes", + "value": "${FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.NumFailedVolumes._sum}" + } + ], + "properties": { + "display_unit": "" + } + }, + { + "widget_name": "Blocks With Corrupted Replicas", + "description": "Number represents data blocks with at least one corrupted replica (but not all of them). Its indicative of HDFS bad health.", + "widget_type": "NUMBER", + "is_visible": true, + "metrics": [ + { + "name": "Hadoop:service=NameNode,name=FSNamesystem.CorruptBlocks", + "metric_path": "metrics/dfs/FSNamesystem/CorruptBlocks", + "service_name": "HDFS", + "component_name": "NAMENODE", + "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active" + } + ], + "values": [ + { + "name": "Blocks With Corrupted Replicas", + "value": "${Hadoop:service=NameNode,name=FSNamesystem.CorruptBlocks}" + } + ], + "properties": { + "warning_threshold": "0", + "error_threshold": "50" + } + }, + { + "widget_name": "Under Replicated Blocks", + "description": "Number represents file blocks that does not meet the replication factor criteria. Its indicative of HDFS bad health.", + "widget_type": "NUMBER", + "is_visible": true, + "metrics": [ + { + "name": "Hadoop:service=NameNode,name=FSNamesystem.UnderReplicatedBlocks", + "metric_path": "metrics/dfs/FSNamesystem/UnderReplicatedBlocks", + "service_name": "HDFS", + "component_name": "NAMENODE", + "host_component_criteria": "host_components/metrics/dfs/FSNamesystem/HAState=active" + } + ], + "values": [ + { + "name": "Under Replicated Blocks", + "value": "${Hadoop:service=NameNode,name=FSNamesystem.UnderReplicatedBlocks}" + } + ], + "properties": { + "warning_threshold": "0", + "error_threshold": "50" + } + }, + { + "widget_name": "HDFS Space Utilization", + "description": "Percentage of available space used in the DFS.", + "widget_type": "GAUGE", + "is_visible": true, + "metrics": [ + { + "name": "FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Remaining", + "metric_path": "metrics/FSDatasetState/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl/Remaining", + "service_name": "HDFS", + "component_name": "DATANODE" + }, + { + "name": "FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Capacity", + "metric_path": "metrics/dfs/datanode/Capacity", + "service_name": "HDFS", + "component_name": "DATANODE" + } + ], + "values": [ + { + "name": "HDFS Space Utilization", + "value": "${(FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Capacity - FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Remaining)/FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Capacity}" + } + ], + "properties": { + "warning_threshold": "0.75", + "error_threshold": "0.9" + } + } + ] + }, + { + "layout_name": "default_hdfs_heatmap", + "section_name": "HDFS_HEATMAPS", + "display_name": "HDFS Heatmaps", + "widgetLayoutInfo": [ + { + "widget_name": "HDFS Bytes Read", + "default_section_name": "HDFS_HEATMAPS", + "description": "", + "widget_type": "HEATMAP", + "is_visible": true, + "metrics": [ + { + "name": "dfs.datanode.BytesRead._rate", + "metric_path": "metrics/dfs/datanode/bytes_read._rate", + "service_name": "HDFS", + "component_name": "DATANODE" + } + ], + "values": [ + { + "name": "HDFS Bytes Read", + "value": "${dfs.datanode.BytesRead._rate}" + } + ], + "properties": { + "display_unit": "MB", + "max_limit": "1024" + } + }, + { + "widget_name": "HDFS Bytes Written", + "description": "", + "widget_type": "HEATMAP", + "is_visible": false, + "metrics": [ + { + "name": "dfs.datanode.BytesWritten._rate", + "metric_path": "metrics/dfs/datanode/bytes_written._rate", + "service_name": "HDFS", + "component_name": "DATANODE" + } + ], + "values": [ + { + "name": "HDFS Bytes Written", + "value": "${dfs.datanode.BytesWritten._rate}" + } + ], + "properties": { + "display_unit": "MB", + "max_limit": "1024" + } + }, + { + "widget_name": "DataNode Garbage Collection Time", + "description": "", + "widget_type": "HEATMAP", + "is_visible": false, + "metrics": [ + { + "name": "Hadoop:service=DataNode,name=JvmMetrics.GcTimeMillis", + "metric_path": "metrics/jvm/gcTimeMillis", + "service_name": "HDFS", + "component_name": "DATANODE" + } + ], + "values": [ + { + "name": "DataNode Garbage Collection Time", + "value": "${Hadoop:service=DataNode,name=JvmMetrics.GcTimeMillis}" + } + ], + "properties": { + "display_unit": "ms", + "max_limit": "10000" + } + }, + { + "widget_name": "DataNode JVM Heap Memory Used", + "description": "", + "widget_type": "HEATMAP", + "is_visible": false, + "metrics": [ + { + "name": "Hadoop:service=DataNode,name=JvmMetrics.MemHeapUsedM", + "metric_path": "metrics/jvm/memHeapUsedM", + "service_name": "HDFS", + "component_name": "DATANODE" + } + ], + "values": [ + { + "name": "DataNode JVM Heap Memory Used", + "value": "${Hadoop:service=DataNode,name=JvmMetrics.MemHeapUsedM}" + } + ], + "properties": { + "display_unit": "MB", + "max_limit": "512" + } + }, + { + "widget_name": "DataNode JVM Heap Memory Committed", + "description": "", + "widget_type": "HEATMAP", + "is_visible": false, + "metrics": [ + { + "name": "Hadoop:service=DataNode,name=JvmMetrics.MemHeapCommittedM", + "metric_path": "metrics/jvm/memHeapCommittedM", + "service_name": "HDFS", + "component_name": "DATANODE" + } + ], + "values": [ + { + "name": "DataNode JVM Heap Memory Committed", + "value": "${Hadoop:service=DataNode,name=JvmMetrics.MemHeapCommittedM}" + } + ], + "properties": { + "display_unit": "MB", + "max_limit": "512" + } + }, + { + "widget_name": "DataNode Process Disk I/O Utilization", + "default_section_name": "HDFS_HEATMAPS", + "description": "", + "widget_type": "HEATMAP", + "is_visible": false, + "metrics": [ + { + "name": "dfs.datanode.BytesRead._rate", + "metric_path": "metrics/dfs/datanode/bytes_read._rate", + "service_name": "HDFS", + "component_name": "DATANODE" + }, + { + "name": "dfs.datanode.BytesWritten._rate", + "metric_path": "metrics/dfs/datanode/bytes_written._rate", + "service_name": "HDFS", + "component_name": "DATANODE" + }, + { + "name": "dfs.datanode.TotalReadTime._rate", + "metric_path": "metrics/dfs/datanode/TotalReadTime._rate", + "service_name": "HDFS", + "component_name": "DATANODE" + }, + { + "name": "dfs.datanode.TotalWriteTime._rate", + "metric_path": "metrics/dfs/datanode/TotalWriteTime._rate", + "service_name": "HDFS", + "component_name": "DATANODE" + } + ], + "values": [ + { + "name": "DataNode Process Disk I/O Utilization", + "value": "${((dfs.datanode.BytesRead._rate/dfs.datanode.TotalReadTime._rate)+(dfs.datanode.BytesWritten._rate/dfs.datanode.TotalWriteTime._rate))*50}" + } + ], + "properties": { + "display_unit": "%", + "max_limit": "100" + } + }, + { + "widget_name": "DataNode Process Network I/O Utilization", + "description": "", + "widget_type": "HEATMAP", + "is_visible": false, + "metrics": [ + { + "name": "dfs.datanode.RemoteBytesRead._rate", + "metric_path": "metrics/dfs/datanode/RemoteBytesRead._rate", + "service_name": "HDFS", + "component_name": "DATANODE" + }, + { + "name": "dfs.datanode.ReadsFromRemoteClient._rate", + "metric_path": "metrics/dfs/datanode/reads_from_remote_client._rate", + "service_name": "HDFS", + "component_name": "DATANODE" + }, + { + "name": "dfs.datanode.RemoteBytesWritten._rate", + "metric_path": "metrics/dfs/datanode/RemoteBytesWritten._rate", + "service_name": "HDFS", + "component_name": "DATANODE" + }, + { + "name": "dfs.datanode.WritesFromRemoteClient._rate", + "metric_path": "metrics/dfs/datanode/writes_from_remote_client._rate", + "service_name": "HDFS", + "component_name": "DATANODE" + } + ], + "values": [ + { + "name": "DataNode Process Network I/O Utilization", + "value": "${((dfs.datanode.RemoteBytesRead._rate/dfs.datanode.ReadsFromRemoteClient._rate)+(dfs.datanode.RemoteBytesWritten._rate/dfs.datanode.WritesFromRemoteClient._rate))*50}" + } + ], + "properties": { + "display_unit": "%", + "max_limit": "100" + } + }, + { + "widget_name": "HDFS Space Utilization", + "widget_type": "HEATMAP", + "is_visible": false, + "metrics": [ + { + "name": "FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Remaining", + "metric_path": "metrics/FSDatasetState/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl/Remaining", + "service_name": "HDFS", + "component_name": "DATANODE" + }, + { + "name": "FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Capacity", + "metric_path": "metrics/dfs/datanode/Capacity", + "service_name": "HDFS", + "component_name": "DATANODE" + } + ], + "values": [ + { + "name": "HDFS Space Utilization", + "value": "${((FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Capacity - FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Remaining)/FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Capacity) * 100}" + } + ], + "properties": { + "display_unit": "%", + "max_limit": "100" + } + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/ambari/blob/2a710b35/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/kerberos.json new file mode 100644 index 0000000..9000e95 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/HDFS/kerberos.json @@ -0,0 +1,246 @@ +{ + "services": [ + { + "name": "HDFS", + "identities": [ + { + "name": "/spnego", + "principal": { + "configuration": "hdfs-site/dfs.web.authentication.kerberos.principal" + }, + "keytab": { + "configuration": "hdfs-site/dfs.web.authentication.kerberos.keytab" + } + }, + { + "name": "/smokeuser" + } + ], + "auth_to_local_properties" : [ + "core-site/hadoop.security.auth_to_local" + ], + "configurations": [ + { + "core-site": { + "hadoop.security.authentication": "kerberos", + "hadoop.security.authorization": "true", + "hadoop.proxyuser.HTTP.groups": "${hadoop-env/proxyuser_group}" + } + }, + { + "ranger-hdfs-audit": { + "xasecure.audit.jaas.Client.loginModuleName": "com.sun.security.auth.module.Krb5LoginModule", + "xasecure.audit.jaas.Client.loginModuleControlFlag": "required", + "xasecure.audit.jaas.Client.option.useKeyTab": "true", + "xasecure.audit.jaas.Client.option.storeKey": "false", + "xasecure.audit.jaas.Client.option.serviceName": "solr", + "xasecure.audit.destination.solr.force.use.inmemory.jaas.config": "true" + } + } + ], + "components": [ + { + "name": "HDFS_CLIENT", + "identities": [ + { + "name": "/HDFS/NAMENODE/hdfs" + } + ] + }, + { + "name": "NAMENODE", + "identities": [ + { + "name": "hdfs", + "principal": { + "value": "${hadoop-env/hdfs_user}-${cluster_name|toLower()}@${realm}", + "type" : "user" , + "configuration": "hadoop-env/hdfs_principal_name", + "local_username" : "${hadoop-env/hdfs_user}" + }, + "keytab": { + "file": "${keytab_dir}/hdfs.headless.keytab", + "owner": { + "name": "${hadoop-env/hdfs_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "hadoop-env/hdfs_user_keytab" + } + }, + { + "name": "namenode_nn", + "principal": { + "value": "nn/_HOST@${realm}", + "type" : "service", + "configuration": "hdfs-site/dfs.namenode.kerberos.principal", + "local_username" : "${hadoop-env/hdfs_user}" + }, + "keytab": { + "file": "${keytab_dir}/nn.service.keytab", + "owner": { + "name": "${hadoop-env/hdfs_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "hdfs-site/dfs.namenode.keytab.file" + } + }, + { + "name": "/spnego", + "principal": { + "configuration": "hdfs-site/dfs.namenode.kerberos.internal.spnego.principal" + } + }, + { + "name": "/HDFS/NAMENODE/namenode_nn", + "principal": { + "configuration": "ranger-hdfs-audit/xasecure.audit.jaas.Client.option.principal" + }, + "keytab": { + "configuration": "ranger-hdfs-audit/xasecure.audit.jaas.Client.option.keyTab" + } + } + ], + "configurations": [ + { + "hdfs-site": { + "dfs.block.access.token.enable": "true" + } + } + ] + }, + { + "name": "DATANODE", + "identities": [ + { + "name": "datanode_dn", + "principal": { + "value": "dn/_HOST@${realm}", + "type" : "service", + "configuration": "hdfs-site/dfs.datanode.kerberos.principal", + "local_username" : "${hadoop-env/hdfs_user}" + }, + "keytab": { + "file": "${keytab_dir}/dn.service.keytab", + "owner": { + "name": "${hadoop-env/hdfs_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "hdfs-site/dfs.datanode.keytab.file" + } + } + ], + "configurations" : [ + { + "hdfs-site" : { + "dfs.datanode.address" : "0.0.0.0:1019", + "dfs.datanode.http.address": "0.0.0.0:1022" + } + } + ] + }, + { + "name": "SECONDARY_NAMENODE", + "identities": [ + { + "name": "secondary_namenode_nn", + "principal": { + "value": "nn/_HOST@${realm}", + "type" : "service", + "configuration": "hdfs-site/dfs.secondary.namenode.kerberos.principal", + "local_username" : "${hadoop-env/hdfs_user}" + }, + "keytab": { + "file": "${keytab_dir}/nn.service.keytab", + "owner": { + "name": "${hadoop-env/hdfs_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "hdfs-site/dfs.secondary.namenode.keytab.file" + } + }, + { + "name": "/spnego", + "principal": { + "configuration": "hdfs-site/dfs.secondary.namenode.kerberos.internal.spnego.principal" + } + } + ] + }, + { + "name": "NFS_GATEWAY", + "identities": [ + { + "name": "nfsgateway", + "principal": { + "value": "nfs/_HOST@${realm}", + "type" : "service", + "configuration": "hdfs-site/nfs.kerberos.principal", + "local_username" : "${hadoop-env/hdfs_user}" + }, + "keytab": { + "file": "${keytab_dir}/nfs.service.keytab", + "owner": { + "name": "${hadoop-env/hdfs_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "hdfs-site/nfs.keytab.file" + } + } + ] + }, + { + "name": "JOURNALNODE", + "identities": [ + { + "name": "journalnode_jn", + "principal": { + "value": "jn/_HOST@${realm}", + "type" : "service", + "configuration": "hdfs-site/dfs.journalnode.kerberos.principal", + "local_username" : "${hadoop-env/hdfs_user}" + }, + "keytab": { + "file": "${keytab_dir}/jn.service.keytab", + "owner": { + "name": "${hadoop-env/hdfs_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "hdfs-site/dfs.journalnode.keytab.file" + } + }, + { + "name": "/spnego", + "principal": { + "configuration": "hdfs-site/dfs.journalnode.kerberos.internal.spnego.principal" + } + } + ] + } + ] + } + ] +}
