AMBARI-7119. log4j does not get used by hadoop as settings are present in hadoop.config.sh (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/15e40320 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/15e40320 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/15e40320 Branch: refs/heads/branch-alerts-dev Commit: 15e4032038a48b3f1472f5a1e4940ccb4f7d4102 Parents: ea58eab Author: Andrew Onishuk <[email protected]> Authored: Tue Sep 2 22:08:14 2014 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Tue Sep 2 22:08:14 2014 +0300 ---------------------------------------------------------------------- .../ambari/server/upgrade/UpgradeCatalog170.java | 4 ++++ .../HDP/1.3.2/hooks/after-INSTALL/scripts/params.py | 1 + .../1.3.2/services/HDFS/configuration/hadoop-env.xml | 8 ++++++++ .../1.3.2/services/HDFS/configuration/hdfs-log4j.xml | 1 + .../HDP/1.3.2/services/HDFS/package/scripts/params.py | 1 + .../HDP/2.0.6/hooks/after-INSTALL/scripts/params.py | 1 + .../2.0.6/services/HDFS/configuration/hadoop-env.xml | 8 ++++++++ .../2.0.6/services/HDFS/configuration/hdfs-log4j.xml | 1 + .../HDP/2.0.6/services/HDFS/package/scripts/params.py | 1 + .../ambari/server/upgrade/UpgradeCatalog170Test.java | 4 ++++ ambari-web/app/data/HDP2/site_properties.js | 13 +++++++++++++ ambari-web/app/data/site_properties.js | 13 +++++++++++++ 12 files changed, 56 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/15e40320/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java index c2dec57..801249f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java @@ -769,6 +769,10 @@ public class UpgradeCatalog170 extends AbstractUpgradeCatalog { false); updateConfigurationProperties("sqoop-env", Collections.singletonMap("sqoop_user", "sqoop"), false, false); + + updateConfigurationProperties("hadoop-env", + Collections.singletonMap("hadoop_root_logger", "INFO,RFA"), false, + false); } /** http://git-wip-us.apache.org/repos/asf/ambari/blob/15e40320/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/after-INSTALL/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/after-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/after-INSTALL/scripts/params.py index a4fb172..9fc0322 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/after-INSTALL/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/after-INSTALL/scripts/params.py @@ -33,6 +33,7 @@ hadoop_conf_dir = "/etc/hadoop/conf" hadoop_conf_empty_dir = "/etc/hadoop/conf.empty" hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix'] hadoop_pid_dir_prefix = config['configurations']['hadoop-env']['hadoop_pid_dir_prefix'] +hadoop_root_logger = config['configurations']['hadoop-env']['hadoop_root_logger'] hadoop_env_sh_template = config['configurations']['hadoop-env']['content'] #hadoop-env.sh http://git-wip-us.apache.org/repos/asf/ambari/blob/15e40320/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hadoop-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hadoop-env.xml b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hadoop-env.xml index a8e6243..3e42d2f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hadoop-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hadoop-env.xml @@ -32,6 +32,11 @@ <description>Hadoop PID Dir Prefix</description> </property> <property> + <name>hadoop_root_logger</name> + <value>INFO,RFA</value> + <description>Hadoop Root Logger</description> + </property> + <property> <name>hadoop_heapsize</name> <value>1024</value> <description>Hadoop maximum Java heap size</description> @@ -209,6 +214,9 @@ export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}} #Mostly required for hadoop 2.0 export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/lib/hadoop/lib/native/Linux-amd64-64 + +#Hadoop logging options +export HADOOP_ROOT_LOGGER={{hadoop_root_logger}} </value> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/15e40320/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hdfs-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hdfs-log4j.xml b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hdfs-log4j.xml index 1366c5d..172f66e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hdfs-log4j.xml +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hdfs-log4j.xml @@ -45,6 +45,7 @@ # limitations under the License. # Define some default values that can be overridden by system properties +# To change daemon root logger use hadoop_root_logger in hadoop-env hadoop.root.logger=INFO,console hadoop.log.dir=. hadoop.log.file=hadoop.log http://git-wip-us.apache.org/repos/asf/ambari/blob/15e40320/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/params.py index 7d23680..e1a8b8f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/params.py @@ -101,6 +101,7 @@ hadoop_pid_dir_prefix = status_params.hadoop_pid_dir_prefix hadoop_bin = "/usr/lib/hadoop/bin" hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix'] +hadoop_root_logger = config['configurations']['hadoop-env']['hadoop_root_logger'] dfs_domain_socket_path = "/var/lib/hadoop-hdfs/dn_socket" dfs_domain_socket_dir = os.path.dirname(dfs_domain_socket_path) http://git-wip-us.apache.org/repos/asf/ambari/blob/15e40320/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py index 3fe5905..e741247 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py @@ -33,6 +33,7 @@ hadoop_conf_dir = "/etc/hadoop/conf" hadoop_conf_empty_dir = "/etc/hadoop/conf.empty" hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix'] hadoop_pid_dir_prefix = config['configurations']['hadoop-env']['hadoop_pid_dir_prefix'] +hadoop_root_logger = config['configurations']['hadoop-env']['hadoop_root_logger'] hadoop_env_sh_template = config['configurations']['hadoop-env']['content'] #hadoop-env.sh http://git-wip-us.apache.org/repos/asf/ambari/blob/15e40320/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hadoop-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hadoop-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hadoop-env.xml index 817be91..1bd08d5 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hadoop-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hadoop-env.xml @@ -32,6 +32,11 @@ <description>Hadoop PID Dir Prefix</description> </property> <property> + <name>hadoop_root_logger</name> + <value>INFO,RFA</value> + <description>Hadoop Root Logger</description> + </property> + <property> <name>hadoop_heapsize</name> <value>1024</value> <description>Hadoop maximum Java heap size</description> @@ -211,6 +216,9 @@ export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}} #Mostly required for hadoop 2.0 export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/lib/hadoop/lib/native/Linux-amd64-64 + +#Hadoop logging options +export HADOOP_ROOT_LOGGER={{hadoop_root_logger}} </value> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/15e40320/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hdfs-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hdfs-log4j.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hdfs-log4j.xml index 976a3c2..08822eb 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hdfs-log4j.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hdfs-log4j.xml @@ -47,6 +47,7 @@ # Define some default values that can be overridden by system properties +# To change daemon root logger use hadoop_root_logger in hadoop-env hadoop.root.logger=INFO,console hadoop.log.dir=. hadoop.log.file=hadoop.log http://git-wip-us.apache.org/repos/asf/ambari/blob/15e40320/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py index ee6b64f..f6905e8 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py @@ -106,6 +106,7 @@ hadoop_pid_dir_prefix = status_params.hadoop_pid_dir_prefix hadoop_bin = "/usr/lib/hadoop/sbin" hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix'] +hadoop_root_logger = config['configurations']['hadoop-env']['hadoop_root_logger'] dfs_domain_socket_path = config['configurations']['hdfs-site']['dfs.domain.socket.path'] dfs_domain_socket_dir = os.path.dirname(dfs_domain_socket_path) http://git-wip-us.apache.org/repos/asf/ambari/blob/15e40320/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java index cc97114..9668ed5 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java @@ -300,6 +300,10 @@ public class UpgradeCatalog170Test { upgradeCatalog.updateConfigurationProperties("sqoop-env", Collections.singletonMap("sqoop_user", "sqoop"), false, false); expectLastCall(); + upgradeCatalog.updateConfigurationProperties("hadoop-env", + Collections.singletonMap("hadoop_root_logger", "INFO,RFA"), false, false); + expectLastCall(); + expect(dbAccessor.executeSelect("SELECT role_name, user_id FROM user_roles")).andReturn(userRolesResultSet).once(); expect(entityManager.getTransaction()).andReturn(trans).anyTimes(); expect(entityManager.getCriteriaBuilder()).andReturn(cb).anyTimes(); http://git-wip-us.apache.org/repos/asf/ambari/blob/15e40320/ambari-web/app/data/HDP2/site_properties.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/data/HDP2/site_properties.js b/ambari-web/app/data/HDP2/site_properties.js index 3ba0246..2498d94 100644 --- a/ambari-web/app/data/HDP2/site_properties.js +++ b/ambari-web/app/data/HDP2/site_properties.js @@ -1763,6 +1763,19 @@ module.exports = }, { "id": "puppet var", + "name": "hadoop_root_logger", + "displayName": "Hadoop Root Logger", + "description": "Hadoop logging options", + "defaultValue": "INFO,RFA", + "displayType": "string", + "isOverridable": false, + "isVisible": true, + "serviceName": "HDFS", + "filename": "hadoop-env.xml", + "category": "Advanced hadoop-env" + }, + { + "id": "puppet var", "name": "security_enabled", "displayName": "Hadoop Security", "description": "Enable hadoop security", http://git-wip-us.apache.org/repos/asf/ambari/blob/15e40320/ambari-web/app/data/site_properties.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/data/site_properties.js b/ambari-web/app/data/site_properties.js index 1f6bb63..4fb4ac8 100644 --- a/ambari-web/app/data/site_properties.js +++ b/ambari-web/app/data/site_properties.js @@ -938,6 +938,19 @@ module.exports = }, { "id": "puppet var", + "name": "hadoop_root_logger", + "displayName": "Hadoop Root Logger", + "description": "Hadoop logging options", + "defaultValue": "INFO,RFA", + "displayType": "string", + "isOverridable": false, + "isVisible": true, + "serviceName": "HDFS", + "filename": "hadoop-env.xml", + "category": "Advanced hadoop-env" + }, + { + "id": "puppet var", "name": "namenode_opt_maxnewsize", "displayName": "NameNode maximum new generation size", "description": "",
