Author: tucu
Date: Fri Mar 9 05:07:17 2012
New Revision: 1298705
URL: http://svn.apache.org/viewvc?rev=1298705&view=rev
Log:
OOZIE-747 HadoopAccessorService hadoop-configs should be loaded from Hadoop
-site.xml files (tucu)
Added:
incubator/oozie/trunk/core/src/main/conf/hadoop-conf/
incubator/oozie/trunk/core/src/main/conf/hadoop-conf/hadoop-site.xml
Modified:
incubator/oozie/trunk/core/src/main/conf/oozie-site.xml
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
incubator/oozie/trunk/core/src/main/resources/oozie-default.xml
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/test/XFsTestCase.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java
incubator/oozie/trunk/core/src/test/resources/hadoop-config.xml
incubator/oozie/trunk/docs/src/site/twiki/AG_HadoopConfiguration.twiki
incubator/oozie/trunk/release-log.txt
Added: incubator/oozie/trunk/core/src/main/conf/hadoop-conf/hadoop-site.xml
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/conf/hadoop-conf/hadoop-site.xml?rev=1298705&view=auto
==============================================================================
--- incubator/oozie/trunk/core/src/main/conf/hadoop-conf/hadoop-site.xml (added)
+++ incubator/oozie/trunk/core/src/main/conf/hadoop-conf/hadoop-site.xml Fri
Mar 9 05:07:17 2012
@@ -0,0 +1,42 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+
+ <property>
+ <name>mapreduce.jobtracker.kerberos.principal</name>
+ <value>mapred/_HOST@LOCALREALM</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.principal</name>
+ <value>yarn/_HOST@LOCALREALM</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.kerberos.principal</name>
+ <value>hdfs/_HOST@LOCALREALM</value>
+ </property>
+
+ <property>
+ <name>mapreduce.framework.name</name>
+ <value>yarn</value>
+ </property>
+
+</configuration>
Modified: incubator/oozie/trunk/core/src/main/conf/oozie-site.xml
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/conf/oozie-site.xml?rev=1298705&r1=1298704&r2=1298705&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/conf/oozie-site.xml (original)
+++ incubator/oozie/trunk/core/src/main/conf/oozie-site.xml Fri Mar 9 05:07:17
2012
@@ -227,11 +227,14 @@
<property>
<name>oozie.service.HadoopAccessorService.hadoop.configurations</name>
- <value>*=hadoop-config.xml</value>
+ <value>*=hadoop-conf</value>
<description>
- Comma separated AUTHORITY=CONFIG_FILE, where AUTHORITY is the
HOST:PORT of
+ Comma separated AUTHORITY=HADOOP_CONF_DIR, where AUTHORITY is the
HOST:PORT of
the Hadoop service (JobTracker, HDFS). The wildcard '*'
configuration is
- used when there is no exact match for an authority.
+ used when there is no exact match for an authority. The
HADOOP_CONF_DIR contains
+ the relevant Hadoop *-site.xml files. If the path is relative is
looked within
+ the Oozie configuration directory; though the path can be absolute
(i.e. to point
+ to Hadoop client conf/ directories in the local filesystem.
</description>
</property>
Modified:
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java?rev=1298705&r1=1298704&r2=1298705&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
(original)
+++
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
Fri Mar 9 05:07:17 2012
@@ -35,6 +35,7 @@ import org.apache.oozie.util.XLog;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
@@ -138,23 +139,51 @@ public class HadoopAccessorService imple
}
}
+ private static final String[] HADOOP_CONF_FILES =
+ {"core-site.xml", "hdfs-site.xml", "mapred-site.xml", "yarn-site.xml",
"hadoop-site.xml"};
+
+
+ private Configuration loadHadoopConf(File dir) throws IOException {
+ Configuration hadoopConf = new XConfiguration();
+ for (String file : HADOOP_CONF_FILES) {
+ File f = new File(dir, file);
+ if (f.exists()) {
+ InputStream is = new FileInputStream(f);
+ Configuration conf = new XConfiguration(is);
+ is.close();
+ XConfiguration.copy(conf, hadoopConf);
+ }
+ }
+ return hadoopConf;
+ }
+
private void loadHadoopConfigs(Configuration serviceConf) throws
ServiceException {
try {
File configDir = new
File(ConfigurationService.getConfigurationDirectory());
- String[] confDefs = serviceConf.getStrings(HADOOP_CONFS,
"*=hadoop-config.xml");
+ String[] confDefs = serviceConf.getStrings(HADOOP_CONFS,
"*=hadoop-conf");
for (String confDef : confDefs) {
if (confDef.trim().length() > 0) {
String[] parts = confDef.split("=");
- String hostPort = parts[0];
- String confFile = parts[1];
- File configFile = new File(configDir, confFile);
- if (configFile.exists()) {
- Configuration conf = new XConfiguration(new
FileInputStream(configFile));
- hadoopConfigs.put(hostPort.toLowerCase(), conf);
+ if (parts.length == 2) {
+ String hostPort = parts[0];
+ String confDir = parts[1];
+ File dir = new File(confDir);
+ if (!dir.isAbsolute()) {
+ dir = new File(configDir, confDir);
+ }
+ if (dir.exists()) {
+ Configuration conf = loadHadoopConf(dir);
+ hadoopConfigs.put(hostPort.toLowerCase(), conf);
+ }
+ else {
+ throw new ServiceException(ErrorCode.E0100,
getClass().getName(),
+ "could not find hadoop
configuration directory: " +
+ dir.getAbsolutePath());
+ }
}
else {
throw new ServiceException(ErrorCode.E0100,
getClass().getName(),
- "could not find hadoop
configuration file: " + confFile);
+ "Incorrect hadoop
configuration definition: " + confDef);
}
}
}
Modified: incubator/oozie/trunk/core/src/main/resources/oozie-default.xml
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/resources/oozie-default.xml?rev=1298705&r1=1298704&r2=1298705&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/resources/oozie-default.xml (original)
+++ incubator/oozie/trunk/core/src/main/resources/oozie-default.xml Fri Mar 9
05:07:17 2012
@@ -1354,11 +1354,14 @@
<property>
<name>oozie.service.HadoopAccessorService.hadoop.configurations</name>
- <value>*=hadoop-config.xml</value>
+ <value>*=hadoop-conf</value>
<description>
- Comma separated AUTHORITY=CONFIG_FILE, where AUTHORITY is the
HOST:PORT of
+ Comma separated AUTHORITY=HADOOP_CONF_DIR, where AUTHORITY is the
HOST:PORT of
the Hadoop service (JobTracker, HDFS). The wildcard '*'
configuration is
- used when there is no exact match for an authority.
+ used when there is no exact match for an authority. The
HADOOP_CONF_DIR contains
+ the relevant Hadoop *-site.xml files. If the path is relative is
looked within
+ the Oozie configuration directory; though the path can be absolute
(i.e. to point
+ to Hadoop client conf/ directories in the local filesystem.
</description>
</property>
Modified:
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java?rev=1298705&r1=1298704&r2=1298705&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java
(original)
+++
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java
Fri Mar 9 05:07:17 2012
@@ -33,11 +33,12 @@ public class TestHadoopAccessorService e
protected void setUp() throws Exception {
super.setUp();
+ new File(getTestCaseConfDir(), "hadoop-confx").mkdir();
InputStream is =
Thread.currentThread().getContextClassLoader().getResourceAsStream("test-hadoop-config.xml");
- OutputStream os = new FileOutputStream(new File(getTestCaseConfDir(),
"test-hadoop-config.xml"));
+ OutputStream os = new FileOutputStream(new File(getTestCaseConfDir() +
"/hadoop-confx", "core-site.xml"));
IOUtils.copyStream(is, os);
setSystemProperty("oozie.service.HadoopAccessorService.hadoop.configurations",
- "*=hadoop-config.xml,test=test-hadoop-config.xml");
+ "*=hadoop-conf,test=hadoop-confx");
if (System.getProperty("oozie.test.hadoop.security",
"simple").equals("kerberos")) {
setSystemProperty("oozie.service.HadoopAccessorService.kerberos.enabled",
"true");
setSystemProperty("oozie.service.HadoopAccessorService.keytab.file",
getKeytabFile());
Modified:
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/test/XFsTestCase.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/test/XFsTestCase.java?rev=1298705&r1=1298704&r2=1298705&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/test/XFsTestCase.java
(original)
+++
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/test/XFsTestCase.java
Fri Mar 9 05:07:17 2012
@@ -64,7 +64,7 @@ public abstract class XFsTestCase extend
conf.set("local.realm", getRealm());
injectKerberosInfo(conf);
- conf.set("oozie.service.HadoopAccessorService.hadoop.configurations",
"*=hadoop-config.xml");
+ conf.set("oozie.service.HadoopAccessorService.hadoop.configurations",
"*=hadoop-conf");
has = new HadoopAccessorService();
has.init(conf);
Modified:
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java?rev=1298705&r1=1298704&r2=1298705&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java
(original)
+++
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java
Fri Mar 9 05:07:17 2012
@@ -253,8 +253,10 @@ public abstract class XTestCase extends
File target = new File(testCaseConfDir, "oozie-site.xml");
IOUtils.copyStream(new FileInputStream(source), new
FileOutputStream(target));
+ File hadoopConfDir = new File(testCaseConfDir, "hadoop-conf");
+ hadoopConfDir.mkdir();
source = new File(OOZIE_SRC_DIR,
"core/src/test/resources/hadoop-config.xml");
- target = new File(testCaseConfDir, "hadoop-config.xml");
+ target = new File(hadoopConfDir, "hadoop-site.xml");
IOUtils.copyStream(new FileInputStream(source), new
FileOutputStream(target));
if (System.getProperty("oozielocal.log") == null) {
Modified: incubator/oozie/trunk/core/src/test/resources/hadoop-config.xml
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/resources/hadoop-config.xml?rev=1298705&r1=1298704&r2=1298705&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/resources/hadoop-config.xml (original)
+++ incubator/oozie/trunk/core/src/test/resources/hadoop-config.xml Fri Mar 9
05:07:17 2012
@@ -21,12 +21,12 @@
<property>
<name>mapreduce.jobtracker.kerberos.principal</name>
- <value>map/_HOST@${localRealm}</value>
+ <value>mapred/_HOST@LOCALREALM</value>
</property>
<property>
<name>dfs.namenode.kerberos.principal</name>
- <value>hdfs/_HOST@${localRealm}</value>
+ <value>hdfs/_HOST@LOCALREALM</value>
</property>
<property>
Modified: incubator/oozie/trunk/docs/src/site/twiki/AG_HadoopConfiguration.twiki
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/docs/src/site/twiki/AG_HadoopConfiguration.twiki?rev=1298705&r1=1298704&r2=1298705&view=diff
==============================================================================
--- incubator/oozie/trunk/docs/src/site/twiki/AG_HadoopConfiguration.twiki
(original)
+++ incubator/oozie/trunk/docs/src/site/twiki/AG_HadoopConfiguration.twiki Fri
Mar 9 05:07:17 2012
@@ -34,34 +34,32 @@ If the value is empty any HOST:PORT is a
---++ Hadoop Default Configuration Values
-Oozie supports Hadoop configuration equivalent to the Hadoop =*-sitel.xml=
files.
-
-Instead supporting multiple files (core, hdfs, yarn, mapred), Oozie support a
single file per Hadoop service
-(JobTracker, HDFS).
+Oozie supports Hadoop configuration equivalent to the Hadoop =*-site.xml=
files.
The configuration property in the =oozie-site.xml= is
=oozie.service.HadoopAccessorService.hadoop.configurations=
-and its value must follow the pattern =[<AUTHORITY>=<CONFIG_FILE>,]*=. Where
=<AUTHORITY>= is the =HOST:PORT= of
-the Hadoop service (JobTracker, HDFS). The =<CONFIG_FILE>= is the Hadoop
configuration file to load for the
-specified Hadoop service. This configuration file is looked in the Oozie
configuration directory.
+and its value must follow the pattern =[<AUTHORITY>=<HADOOP_CONF_DIR>,]*=.
Where =<AUTHORITY>= is the =HOST:PORT= of
+the Hadoop service (JobTracker, HDFS). The =<HADOO_CONF_DIR>= is a Hadoop
configuration directory. If the specified
+ directory is a relative path, it will be looked under the Oozie configuration
directory. And absolute path can
+ also be specified. Oozie will load the Hadoop =*-site.xml= files in the
following order: core-site.xml, hdfs-site.xml,
+ mapred-site.xml, yarn-site.xml, hadoop-site.xml.
In addition to explicit authorities, a '*' wildcard is supported. The
configuration file associated with the wildcard
will be used as default if there is no configuration for the requested Hadoop
service.
-For example, the configuration in the =oozie-site.xml= would lookl like:
+For example, the configuration in the =oozie-site.xml= would look like:
<verbatim>
...
<property>
<name>oozie.service.HadoopAccessorService.hadoop.configurations</name>
-
<value>*=hadoop-config.xml,jt-bar:8021=bar.xml,nn-bar:8020=bar.xml</value>
+
<value>*=hadoop-conf,jt-bar:8021=bar-cluster,nn-bar:8020=bar-cluster</value>
</property>
...
</verbatim>
The Hadoop configuration files use the Hadoop configuration syntax.
-By default Oozie defines =*=hadoop-config.xml= and the default values of the
=hadoop-config.xml= file are:
-
+By default Oozie defines =*=hadoop-conf= and the default values of the
=hadoop-site.xml= file are:
<verbatim>
<configuration>
Modified: incubator/oozie/trunk/release-log.txt
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/release-log.txt?rev=1298705&r1=1298704&r2=1298705&view=diff
==============================================================================
--- incubator/oozie/trunk/release-log.txt (original)
+++ incubator/oozie/trunk/release-log.txt Fri Mar 9 05:07:17 2012
@@ -1,5 +1,6 @@
-- Oozie 3.2.0 release
+OOZIE-747 HadoopAccessorService hadoop-configs should be loaded from Hadoop
-site.xml files (tucu)
OOZIE-746 JobConf/Configuration creation is inconsistent and makes things fail
in odd ways (tucu)
OOZIE-744 HadoopAccessorService hadoop-configs loading logic is not 100%
correct (tucu)
OOZIE-743 HadoopAccessorService hadoop configurations should not use VARs from
properties defined oozie-site.xml (tucu)
