Author: mona
Date: Fri Jan 25 18:25:12 2013
New Revision: 1438631
URL: http://svn.apache.org/viewvc?rev=1438631&view=rev
Log:
Sync hcat-intre with trunk patches
Modified:
oozie/branches/hcat-intre/core/pom.xml
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/OoziePigStats.java
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/command/XCommand.java
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/service/ProxyUserService.java
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/servlet/HostnameFilter.java
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/util/ParamChecker.java
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/command/TestXCommand.java
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/service/TestProxyUserService.java
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/servlet/TestHostnameFilter.java
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java
oozie/branches/hcat-intre/docs/src/site/twiki/ENG_Custom_Authentication.twiki
oozie/branches/hcat-intre/minitest/pom.xml
oozie/branches/hcat-intre/release-log.txt
oozie/branches/hcat-intre/webapp/src/main/webapp/oozie-console.js
Modified: oozie/branches/hcat-intre/core/pom.xml
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/pom.xml?rev=1438631&r1=1438630&r2=1438631&view=diff
==============================================================================
--- oozie/branches/hcat-intre/core/pom.xml (original)
+++ oozie/branches/hcat-intre/core/pom.xml Fri Jan 25 18:25:12 2013
@@ -623,10 +623,9 @@
<configuration>
<includes>
<include>**/XTestCase.class</include>
- <include>**/XTestCase$1.class</include>
<include>**/XFsTestCase.class</include>
<include>**/MiniOozieTestCase.class</include>
-
<include>**/XTestCase$Predicate.class</include>
+ <include>**/XTestCase$*.class</include>
</includes>
</configuration>
</execution>
Modified:
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java?rev=1438631&r1=1438630&r2=1438631&view=diff
==============================================================================
---
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
(original)
+++
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
Fri Jan 25 18:25:12 2013
@@ -479,13 +479,15 @@ public class JavaActionExecutor extends
// files and archives defined in the action
for (Element eProp : (List<Element>) actionXml.getChildren()) {
if (eProp.getName().equals("file")) {
- String path = eProp.getTextTrim();
- addToCache(conf, appPath, path, false);
+ String[] pathes = eProp.getTextTrim().split(",");
+ for (String path : pathes) {
+ addToCache(conf, appPath, path.trim(), false);
+ }
}
- else {
- if (eProp.getName().equals("archive")) {
- String path = eProp.getTextTrim();
- addToCache(conf, appPath, path, true);
+ else if (eProp.getName().equals("archive")) {
+ String[] pathes = eProp.getTextTrim().split(",");
+ for (String path : pathes){
+ addToCache(conf, appPath, path.trim(), true);
}
}
}
@@ -1035,6 +1037,7 @@ public class JavaActionExecutor extends
log.warn(errorReason);
}
context.setExecutionData(FAILED_KILLED, null);
+ setActionCompletionData(context, actionFs);
}
}
else {
@@ -1212,4 +1215,14 @@ public class JavaActionExecutor extends
protected String getDefaultShareLibName(Element actionXml) {
return null;
}
+
+ /**
+ * Sets some data for the action on completion
+ *
+ * @param context executor context
+ * @param actionFs the FileSystem object
+ */
+ protected void setActionCompletionData(Context context, FileSystem
actionFs) throws IOException,
+ HadoopAccessorException, URISyntaxException {
+ }
}
Modified:
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java?rev=1438631&r1=1438630&r2=1438631&view=diff
==============================================================================
---
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
(original)
+++
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
Fri Jan 25 18:25:12 2013
@@ -504,13 +504,13 @@ public class LauncherMapper<K1, V1, K2,
System.out.println();
System.out.println("<<< Invocation of Main class
completed <<<");
System.out.println();
+ handleExternalChildIDs(reporter);
}
if (errorMessage == null) {
File outputData = new
File(System.getProperty("oozie.action.output.properties"));
- FileSystem fs = null;
if (outputData.exists()) {
URI actionDirUri = new Path(actionDir,
ACTION_OUTPUT_PROPS).toUri();
- fs = FileSystem.get(actionDirUri, getJobConf());
+ FileSystem fs = FileSystem.get(actionDirUri,
getJobConf());
fs.copyFromLocalFile(new
Path(outputData.toString()), new Path(actionDir,
ACTION_OUTPUT_PROPS));
reporter.incrCounter(COUNTER_GROUP,
COUNTER_OUTPUT_DATA, 1);
@@ -531,8 +531,8 @@ public class LauncherMapper<K1, V1, K2,
System.out.println("=======================");
System.out.println();
}
- handleActionStatsData(fs, reporter);
- handleExternalChildIDs(fs, reporter);
+ handleActionStatsData(reporter);
+ handleExternalChildIDs(reporter);
File newId = new
File(System.getProperty("oozie.action.newId.properties"));
if (newId.exists()) {
Properties props = new Properties();
@@ -541,7 +541,7 @@ public class LauncherMapper<K1, V1, K2,
throw new IllegalStateException("ID swap file
does not have [id] property");
}
URI actionDirUri = new Path(actionDir,
ACTION_NEW_ID_PROPS).toUri();
- fs = FileSystem.get(actionDirUri, getJobConf());
+ FileSystem fs = FileSystem.get(actionDirUri,
getJobConf());
fs.copyFromLocalFile(new Path(newId.toString()),
new Path(actionDir, ACTION_NEW_ID_PROPS));
reporter.incrCounter(COUNTER_GROUP,
COUNTER_DO_ID_SWAP, 1);
@@ -596,7 +596,7 @@ public class LauncherMapper<K1, V1, K2,
return jobConf;
}
- private void handleActionStatsData(FileSystem fs, Reporter reporter)
throws IOException, LauncherException {
+ private void handleActionStatsData(Reporter reporter) throws IOException,
LauncherException {
File actionStatsData = new
File(System.getProperty(EXTERNAL_ACTION_STATS));
// If stats are stored by the action, then stats file should exist
if (actionStatsData.exists()) {
@@ -611,19 +611,19 @@ public class LauncherMapper<K1, V1, K2,
}
// copy the stats file to hdfs path which can be accessed by Oozie
server
URI actionDirUri = new Path(actionDir, ACTION_STATS_PROPS).toUri();
- fs = FileSystem.get(actionDirUri, getJobConf());
+ FileSystem fs = FileSystem.get(actionDirUri, getJobConf());
fs.copyFromLocalFile(new Path(actionStatsData.toString()), new
Path(actionDir,
ACTION_STATS_PROPS));
}
}
- private void handleExternalChildIDs(FileSystem fs, Reporter reporter)
throws IOException {
+ private void handleExternalChildIDs(Reporter reporter) throws IOException {
File externalChildIDs = new
File(System.getProperty(EXTERNAL_CHILD_IDS));
// if external ChildIDs are stored by the action, then the file should
exist
if (externalChildIDs.exists()) {
// copy the externalChildIDs file to hdfs path which can be
accessed by Oozie server
URI actionDirUri = new Path(actionDir,
ACTION_EXTERNAL_CHILD_IDS_PROPS).toUri();
- fs = FileSystem.get(actionDirUri, getJobConf());
+ FileSystem fs = FileSystem.get(actionDirUri, getJobConf());
fs.copyFromLocalFile(new Path(externalChildIDs.toString()), new
Path(actionDir,
ACTION_EXTERNAL_CHILD_IDS_PROPS));
}
Modified:
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/OoziePigStats.java
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/OoziePigStats.java?rev=1438631&r1=1438630&r2=1438631&view=diff
==============================================================================
---
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/OoziePigStats.java
(original)
+++
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/OoziePigStats.java
Fri Jan 25 18:25:12 2013
@@ -17,14 +17,10 @@
*/
package org.apache.oozie.action.hadoop;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
import java.util.Map;
import org.apache.hadoop.mapred.Counters;
import org.apache.hadoop.mapred.Counters.Counter;
-import org.apache.oozie.util.ParamChecker;
import org.apache.pig.tools.pigstats.JobStats;
import org.apache.pig.tools.pigstats.PigStats;
import org.apache.pig.tools.pigstats.PigStatsUtil;
Modified:
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java?rev=1438631&r1=1438630&r2=1438631&view=diff
==============================================================================
---
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
(original)
+++
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
Fri Jan 25 18:25:12 2013
@@ -170,6 +170,16 @@ public class PigActionExecutor extends J
return stats;
}
+ @Override
+ protected void setActionCompletionData(Context context, FileSystem fs)
throws HadoopAccessorException, IOException,
+ URISyntaxException {
+ String data = getExternalChildIDs(context, fs);
+ if (data != null && !data.isEmpty()) {
+ context.setExternalChildIDs(data);
+ XLog.getLog(getClass()).info(XLog.STD, "Hadoop Jobs launched :
[{0}]", data);
+ }
+ }
+
private String getExternalChildIDs(Context context, FileSystem actionFs)
throws IOException,
HadoopAccessorException, URISyntaxException {
Path actionOutput =
LauncherMapper.getExternalChildIDsDataPath(context.getActionDir());
Modified:
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/PigMain.java?rev=1438631&r1=1438630&r2=1438631&view=diff
==============================================================================
---
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
(original)
+++
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
Fri Jan 25 18:25:12 2013
@@ -281,6 +281,12 @@ public class PigMain extends LauncherMai
if (pigRunnerExists) {
System.out.println("Run pig script using PigRunner.run() for Pig
version 0.8+");
PigStats stats = PigRunner.run(args, null);
+ String jobIds = getHadoopJobIds(stats);
+ if (jobIds != null && !jobIds.isEmpty()) {
+ System.out.println("Hadoop Job IDs executed by Pig: " +
jobIds);
+ File f = new File(System.getProperty(EXTERNAL_CHILD_IDS));
+ writeExternalData(jobIds, f);
+ }
// isSuccessful is the API from 0.9 supported by both PigStats and
// EmbeddedPigStats
if (!stats.isSuccessful()) {
@@ -295,12 +301,6 @@ public class PigMain extends LauncherMai
if (resetSecurityManager) {
return;
}
- String jobIds = getHadoopJobIds(stats);
- if (jobIds != null) {
- System.out.println(" Hadoop Job IDs executed by Pig: " +
jobIds);
- File f = new File(System.getProperty(EXTERNAL_CHILD_IDS));
- writeExternalData(jobIds, f);
- }
// Retrieve stats only if user has specified in workflow
// configuration
if (retrieveStats) {
Modified:
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/command/XCommand.java
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/command/XCommand.java?rev=1438631&r1=1438630&r2=1438631&view=diff
==============================================================================
---
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/command/XCommand.java
(original)
+++
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/command/XCommand.java
Fri Jan 25 18:25:12 2013
@@ -291,7 +291,7 @@ public abstract class XCommand<T> implem
return ret;
}
finally {
- if (isLockRequired()) {
+ if (isLockRequired() && !this.inInterruptMode()) {
releaseLock();
}
}
Modified:
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java?rev=1438631&r1=1438630&r2=1438631&view=diff
==============================================================================
---
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java
(original)
+++
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java
Fri Jan 25 18:25:12 2013
@@ -133,7 +133,7 @@ public class BundleSubmitXCommand extend
ParameterVerifier.verifyParameters(conf,
XmlUtils.parseXml(bundleBean.getOrigJobXml()));
-
XmlUtils.removeComments(this.bundleBean.getOrigJobXml().toString());
+ String jobXmlWithNoComment =
XmlUtils.removeComments(this.bundleBean.getOrigJobXml().toString());
// Resolving all variables in the job properties.
// This ensures the Hadoop Configuration semantics is preserved.
XConfiguration resolvedVarsConf = new XConfiguration();
@@ -142,7 +142,7 @@ public class BundleSubmitXCommand extend
}
conf = resolvedVarsConf;
- String resolvedJobXml = resolvedVars(bundleBean.getOrigJobXml(),
conf);
+ String resolvedJobXml = resolvedVars(jobXmlWithNoComment, conf);
//verify the uniqueness of coord names
verifyCoordNameUnique(resolvedJobXml);
Modified:
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/service/ProxyUserService.java
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/service/ProxyUserService.java?rev=1438631&r1=1438630&r2=1438631&view=diff
==============================================================================
---
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/service/ProxyUserService.java
(original)
+++
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/service/ProxyUserService.java
Fri Jan 25 18:25:12 2013
@@ -128,8 +128,14 @@ public class ProxyUserService implements
*/
public void validate(String proxyUser, String proxyHost, String doAsUser)
throws IOException,
AccessControlException {
- ParamChecker.notEmpty(proxyUser, "proxyUser");
- ParamChecker.notEmpty(proxyHost, "proxyHost");
+ ParamChecker.notEmpty(proxyUser, "proxyUser",
+ "If you're attempting to use user-impersonation via a proxy
user, please make sure that "
+ + "oozie.service.ProxyUserService.proxyuser.#USER#.hosts and "
+ + "oozie.service.ProxyUserService.proxyuser.#USER#.groups are
configured correctly");
+ ParamChecker.notEmpty(proxyHost, "proxyHost",
+ "If you're attempting to use user-impersonation via a proxy
user, please make sure that "
+ + "oozie.service.ProxyUserService.proxyuser." + proxyUser +
".hosts and "
+ + "oozie.service.ProxyUserService.proxyuser." + proxyUser +
".groups are configured correctly");
ParamChecker.notEmpty(doAsUser, "doAsUser");
LOG.debug("Authorization check proxyuser [{0}] host [{1}] doAs [{2}]",
new Object[]{proxyUser, proxyHost, doAsUser});
Modified:
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/servlet/HostnameFilter.java
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/servlet/HostnameFilter.java?rev=1438631&r1=1438630&r2=1438631&view=diff
==============================================================================
---
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/servlet/HostnameFilter.java
(original)
+++
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/servlet/HostnameFilter.java
Fri Jan 25 18:25:12 2013
@@ -18,6 +18,8 @@
package org.apache.oozie.servlet;
+import org.apache.oozie.util.XLog;
+
import javax.servlet.Filter;
import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
@@ -26,11 +28,14 @@ import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import java.io.IOException;
import java.net.InetAddress;
+import java.net.UnknownHostException;
/**
* Filter that resolves the requester hostname.
*/
public class HostnameFilter implements Filter {
+ private static final XLog LOG = XLog.getLog(HostnameFilter.class);
+
static final ThreadLocal<String> HOSTNAME_TL = new ThreadLocal<String>();
/**
@@ -62,7 +67,19 @@ public class HostnameFilter implements F
public void doFilter(ServletRequest request, ServletResponse response,
FilterChain chain)
throws IOException, ServletException {
try {
- String hostname =
InetAddress.getByName(request.getRemoteAddr()).getCanonicalHostName();
+ String hostname;
+ try {
+ String address = request.getRemoteAddr();
+ if (address != null) {
+ hostname =
InetAddress.getByName(address).getCanonicalHostName();
+ } else {
+ LOG.warn("Request remote address is NULL");
+ hostname = "???";
+ }
+ } catch (UnknownHostException ex) {
+ LOG.warn("Request remote address could not be resolved, {0}",
ex.toString(), ex);
+ hostname = "???";
+ }
HOSTNAME_TL.set(hostname);
chain.doFilter(request, response);
}
Modified:
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/util/ParamChecker.java
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/util/ParamChecker.java?rev=1438631&r1=1438630&r2=1438631&view=diff
==============================================================================
---
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/util/ParamChecker.java
(original)
+++
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/util/ParamChecker.java
Fri Jan 25 18:25:12 2013
@@ -64,11 +64,23 @@ public class ParamChecker {
* @return the given value.
*/
public static String notEmpty(String str, String name) {
+ return notEmpty(str, name, null);
+ }
+
+ /**
+ * Check that a string is not null and not empty. If null or emtpy throws
an IllegalArgumentException.
+ *
+ * @param str value.
+ * @param name parameter name for the exception message.
+ * @param info additional information to be printed with the exception
message
+ * @return the given value.
+ */
+ public static String notEmpty(String str, String name, String info) {
if (str == null) {
- throw new IllegalArgumentException(name + " cannot be null");
+ throw new IllegalArgumentException(name + " cannot be null" +
(info == null ? "" : ", " + info));
}
if (str.length() == 0) {
- throw new IllegalArgumentException(name + " cannot be empty");
+ throw new IllegalArgumentException(name + " cannot be empty" +
(info == null ? "" : ", " + info));
}
return str;
}
Modified:
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java?rev=1438631&r1=1438630&r2=1438631&view=diff
==============================================================================
---
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
(original)
+++
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
Fri Jan 25 18:25:12 2013
@@ -46,6 +46,7 @@ import org.apache.oozie.WorkflowActionBe
import org.apache.oozie.WorkflowJobBean;
import org.apache.oozie.action.ActionExecutor;
import org.apache.oozie.action.ActionExecutorException;
+import org.apache.oozie.action.hadoop.ActionExecutorTestCase.Context;
import org.apache.oozie.client.OozieClient;
import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.client.WorkflowJob;
@@ -724,6 +725,114 @@ public class TestJavaActionExecutor exte
}
}
+ /**
+ * https://issues.apache.org/jira/browse/OOZIE-87
+ * @throws Exception
+ */
+ public void testCommaSeparatedFilesAndArchives() throws Exception {
+ Path root = new Path(getFsTestCaseDir(), "root");
+
+ Path jar = new Path("jar.jar");
+ getFileSystem().create(new Path(getAppPath(), jar)).close();
+ Path rootJar = new Path(root, "rootJar.jar");
+ getFileSystem().create(rootJar).close();
+
+ Path file = new Path("file");
+ getFileSystem().create(new Path(getAppPath(), file)).close();
+ Path rootFile = new Path(root, "rootFile");
+ getFileSystem().create(rootFile).close();
+
+ Path so = new Path("soFile.so");
+ getFileSystem().create(new Path(getAppPath(), so)).close();
+ Path rootSo = new Path(root, "rootSoFile.so");
+ getFileSystem().create(rootSo).close();
+
+ Path so1 = new Path("soFile.so.1");
+ getFileSystem().create(new Path(getAppPath(), so1)).close();
+ Path rootSo1 = new Path(root, "rootSoFile.so.1");
+ getFileSystem().create(rootSo1).close();
+
+ Path archive = new Path("archive.tar");
+ getFileSystem().create(new Path(getAppPath(), archive)).close();
+ Path rootArchive = new Path(root, "rootArchive.tar");
+ getFileSystem().create(rootArchive).close();
+
+ String actionXml = "<java>" +
+ " <job-tracker>" + getJobTrackerUri() + "</job-tracker>" +
+ " <name-node>" + getNameNodeUri() + "</name-node>" +
+ " <main-class>CLASS</main-class>" +
+ " <file>" + jar.toString() +
+ "," + rootJar.toString() +
+ "," + file.toString() +
+ ", " + rootFile.toString() + // with leading and
trailing spaces
+ " ," + so.toString() +
+ "," + rootSo.toString() +
+ "," + so1.toString() +
+ "," + rootSo1.toString() + "</file>\n" +
+ " <archive>" + archive.toString() + ", "
+ + rootArchive.toString() + " </archive>\n" + //
with leading and trailing spaces
+ "</java>";
+
+ Element eActionXml = XmlUtils.parseXml(actionXml);
+
+ Context context = createContext(actionXml, null);
+
+ Path appPath = getAppPath();
+
+ JavaActionExecutor ae = new JavaActionExecutor();
+
+ Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
+ ae.setupActionConf(jobConf, context, eActionXml, appPath);
+ ae.setLibFilesArchives(context, eActionXml, appPath, jobConf);
+
+
+ assertTrue(DistributedCache.getSymlink(jobConf));
+
+ Path[] filesInClasspath = DistributedCache.getFileClassPaths(jobConf);
+ for (Path p : new Path[]{new Path(getAppPath(), jar), rootJar}) {
+ boolean found = false;
+ for (Path c : filesInClasspath) {
+ if (!found && p.toUri().getPath().equals(c.toUri().getPath()))
{
+ found = true;
+ }
+ }
+ assertTrue("file " + p.toUri().getPath() + " not found in
classpath", found);
+ }
+ for (Path p : new Path[]{new Path(getAppPath(), file), rootFile, new
Path(getAppPath(), so), rootSo,
+ new Path(getAppPath(), so1), rootSo1}) {
+ boolean found = false;
+ for (Path c : filesInClasspath) {
+ if (!found && p.toUri().getPath().equals(c.toUri().getPath()))
{
+ found = true;
+ }
+ }
+ assertFalse("file " + p.toUri().getPath() + " found in classpath",
found);
+ }
+
+ URI[] filesInCache = DistributedCache.getCacheFiles(jobConf);
+ for (Path p : new Path[]{new Path(getAppPath(), jar), rootJar, new
Path(getAppPath(), file), rootFile,
+ new Path(getAppPath(), so), rootSo, new
Path(getAppPath(), so1), rootSo1}) {
+ boolean found = false;
+ for (URI c : filesInCache) {
+ if (!found && p.toUri().getPath().equals(c.getPath())) {
+ found = true;
+ }
+ }
+ assertTrue("file " + p.toUri().getPath() + " not found in cache",
found);
+ }
+
+ URI[] archivesInCache = DistributedCache.getCacheArchives(jobConf);
+ for (Path p : new Path[]{new Path(getAppPath(), archive),
rootArchive}) {
+ boolean found = false;
+ for (URI c : archivesInCache) {
+ if (!found && p.toUri().getPath().equals(c.getPath())) {
+ found = true;
+ }
+ }
+ assertTrue("archive " + p.toUri().getPath() + " not found in
cache", found);
+ }
+ }
+
public void testPrepare() throws Exception {
FileSystem fs = getFileSystem();
Path mkdir = new Path(getFsTestCaseDir(), "mkdir");
Modified:
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/command/TestXCommand.java
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/command/TestXCommand.java?rev=1438631&r1=1438630&r2=1438631&view=diff
==============================================================================
---
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/command/TestXCommand.java
(original)
+++
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/command/TestXCommand.java
Fri Jan 25 18:25:12 2013
@@ -135,6 +135,14 @@ public class TestXCommand extends XTestC
execute = true;
return null;
}
+
+ private void resetCalledMethods() {
+ eagerLoadState = false;
+ eagerVerifyPrecondition = false;
+ loadState = false;
+ verifyPrecondition = false;
+ execute = false;
+ }
}
public void testXCommandGetters() throws Exception {
@@ -203,6 +211,18 @@ public class TestXCommand extends XTestC
}
}
+ public void testXCommandPossibleReleaseLockWithoutAcquireEdgeCase()
+ throws Exception {
+ AXCommand command = new AXCommand(true);
+ command.setInterruptMode(false);
+ command.call();
+ assertTrue(command.execute);
+ command.resetCalledMethods();
+ command.setInterruptMode(true);
+ command.call();
+ assertTrue(command.execute);
+ }
+
private static class LockGetter extends Thread {
@Override
Modified:
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/service/TestProxyUserService.java
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/service/TestProxyUserService.java?rev=1438631&r1=1438630&r2=1438631&view=diff
==============================================================================
---
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/service/TestProxyUserService.java
(original)
+++
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/service/TestProxyUserService.java
Fri Jan 25 18:25:12 2013
@@ -270,5 +270,52 @@ public class TestProxyUserService extend
}
}
+ public void testNullProxyUser() throws Exception {
+ Services services = new Services();
+ Configuration conf = services.getConf();
+ conf.set(Services.CONF_SERVICE_CLASSES, StringUtils.join(",",
Arrays.asList(GroupsService.class.getName(),
+
ProxyUserService.class.getName())));
+ services.init();
+ try {
+ ProxyUserService proxyUser = services.get(ProxyUserService.class);
+ Assert.assertNotNull(proxyUser);
+ proxyUser.validate(null, "localhost", "bar");
+ fail();
+ }
+ catch (IllegalArgumentException ex) {
+
assertTrue(ex.getMessage().contains("oozie.service.ProxyUserService.proxyuser.#USER#.hosts"));
+
assertTrue(ex.getMessage().contains("oozie.service.ProxyUserService.proxyuser.#USER#.groups"));
+ }
+ catch (Exception ex) {
+ fail(ex.toString());
+ }
+ finally {
+ services.destroy();
+ }
+ }
+
+ public void testNullHost() throws Exception {
+ Services services = new Services();
+ Configuration conf = services.getConf();
+ conf.set(Services.CONF_SERVICE_CLASSES, StringUtils.join(",",
Arrays.asList(GroupsService.class.getName(),
+
ProxyUserService.class.getName())));
+ services.init();
+ try {
+ ProxyUserService proxyUser = services.get(ProxyUserService.class);
+ Assert.assertNotNull(proxyUser);
+ proxyUser.validate("foo", null, "bar");
+ fail();
+ }
+ catch (IllegalArgumentException ex) {
+
assertTrue(ex.getMessage().contains("oozie.service.ProxyUserService.proxyuser.foo.hosts"));
+
assertTrue(ex.getMessage().contains("oozie.service.ProxyUserService.proxyuser.foo.groups"));
+ }
+ catch (Exception ex) {
+ fail(ex.toString());
+ }
+ finally {
+ services.destroy();
+ }
+ }
}
Modified:
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/servlet/TestHostnameFilter.java
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/servlet/TestHostnameFilter.java?rev=1438631&r1=1438630&r2=1438631&view=diff
==============================================================================
---
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/servlet/TestHostnameFilter.java
(original)
+++
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/servlet/TestHostnameFilter.java
Fri Jan 25 18:25:12 2013
@@ -59,4 +59,30 @@ public class TestHostnameFilter extends
filter.destroy();
}
+ public void testMissingHostname() throws Exception {
+ ServletRequest request = Mockito.mock(ServletRequest.class);
+ Mockito.when(request.getRemoteAddr()).thenReturn(null);
+
+ ServletResponse response = Mockito.mock(ServletResponse.class);
+
+ final AtomicBoolean invoked = new AtomicBoolean();
+
+ FilterChain chain = new FilterChain() {
+ @Override
+ public void doFilter(ServletRequest servletRequest, ServletResponse
servletResponse)
+ throws IOException, ServletException {
+ Assert.assertTrue(HostnameFilter.get().contains("???"));
+ invoked.set(true);
+ }
+ };
+
+ Filter filter = new HostnameFilter();
+ filter.init(null);
+ Assert.assertNull(HostnameFilter.get());
+ filter.doFilter(request, response, chain);
+ Assert.assertTrue(invoked.get());
+ Assert.assertNull(HostnameFilter.get());
+ filter.destroy();
+ }
+
}
Modified:
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java?rev=1438631&r1=1438630&r2=1438631&view=diff
==============================================================================
---
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java
(original)
+++
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/util/TestLogStreamer.java
Fri Jan 25 18:25:12 2013
@@ -25,11 +25,9 @@ import java.io.StringWriter;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
-import java.util.GregorianCalendar;
import java.util.zip.GZIPOutputStream;
import org.apache.oozie.test.XTestCase;
-import org.apache.oozie.util.XLogStreamer;
public class TestLogStreamer extends XTestCase {
@@ -51,7 +49,7 @@ public class TestLogStreamer extends XTe
xf.setParameter("JOB", "14-200904160239--example-forkjoinwf");
xf.setLogLevel("DEBUG|INFO");
- // This file will be included in the list of files for log retrieval,
provided the modification time lies
+ // This file will be included in the list of files for log retrieval,
because the modification time lies
// between the start and end times of the job
FileWriter fw1 = new FileWriter(getTestCaseDir() + "/oozie.log");
StringBuilder sb1 = new StringBuilder();
@@ -61,7 +59,7 @@ public class TestLogStreamer extends XTe
fw1.write(sb1.toString());
fw1.close();
File f1 = new File(getTestCaseDir() + "/oozie.log");
- f1.setLastModified(currTime - 9000);
+ f1.setLastModified(currTime - 9 * 3600000); // 9 hours ago
// This file will be included in the list of files for log retrieval,
provided the modification time lies
// between the start and end times of the job
@@ -75,7 +73,7 @@ public class TestLogStreamer extends XTe
fw2.write(sb2.toString());
fw2.close();
File f2 = new File(getTestCaseDir() + "/oozie.log.1");
- f2.setLastModified(currTime - 8000);
+ f2.setLastModified(currTime - 8 * 3600000); // 8 hours ago
// This file will be included in the list of files for log retrieval,
provided, the modification time lies
// between the start and end times of the job
@@ -102,12 +100,13 @@ public class TestLogStreamer extends XTe
fwerr.write(sberr.toString());
fwerr.close();
File ferr = new File(getTestCaseDir() + "/testerr.log");
- ferr.setLastModified(currTime - 8000);
+ ferr.setLastModified(currTime - 8 * 3600000); // 8 hours ago
// This GZip file would be included in list of files for log
retrieval, provided, there is an overlap between
// the two time windows i) time duration during which the GZipped log
file is modified ii) time window between
// start and end times of the job
- String outFilename = "oozie.log-" + filenameDateFormatter.format(new
Date(currTime)) + ".gz";
+ // filename date below is equivalent to floor(6 hours ago)
+ String outFilename = "oozie.log-" + filenameDateFormatter.format(new
Date(currTime - 6 * 3600000)) + ".gz";
File f = new File(getTestCaseDir() + "/" + outFilename);
StringBuilder sb = new StringBuilder();
sb.append("\n2009-06-24 02:43:13,958 DEBUG _L8_:323 -" + logStatement
+ "End workflow state change");
@@ -136,11 +135,11 @@ public class TestLogStreamer extends XTe
+ "command.WorkflowRunnerCallable] " + "released lock");
writeToGZFile(f,sb);
- // Test for the log retrieval of the job that began 10 seconds before
and ended 5 seconds before current time
+ // Test for the log retrieval of the job that began 10 hours before
and ended 5 hours before current time
// respectively
StringWriter sw = new StringWriter();
XLogStreamer str = new XLogStreamer(xf, sw, getTestCaseDir(),
"oozie.log", 1);
- str.streamLog(new Date(currTime - 10000), new Date(currTime - 5000));
+ str.streamLog(new Date(currTime - 10 * 3600000), new Date(currTime - 5
* 3600000));
String[] out = sw.toString().split("\n");
// Check if the retrieved log content is of length seven lines after
filtering based on time window, file name
// pattern and parameters like JobId, Username etc. and/or based on
log level like INFO, DEBUG, etc.
@@ -148,11 +147,11 @@ public class TestLogStreamer extends XTe
// Check if the lines of the log contain the expected strings
assertEquals(true, out[0].contains("_L10_"));
assertEquals(true, out[1].contains("_L11_"));
- assertEquals(true, out[2].contains("_L8_"));
- assertEquals(true, out[3].contains("_L9_"));
- assertEquals(true, out[4].contains("_L1_"));
- assertEquals(true, out[5].contains("_L2_"));
- assertEquals(true, out[6].contains("_L4_"));
+ assertEquals(true, out[2].contains("_L1_"));
+ assertEquals(true, out[3].contains("_L2_"));
+ assertEquals(true, out[4].contains("_L4_"));
+ assertEquals(true, out[5].contains("_L8_"));
+ assertEquals(true, out[6].contains("_L9_"));
// Test to check if the null values for startTime and endTime are
translated to 0 and current time respectively
// and corresponding log content is retrieved properly
@@ -166,11 +165,11 @@ public class TestLogStreamer extends XTe
// Check if the lines of the log contain the expected strings
assertEquals(true, out[0].contains("_L10"));
assertEquals(true, out[1].contains("_L11_"));
- assertEquals(true, out[2].contains("_L8_"));
- assertEquals(true, out[3].contains("_L9_"));
- assertEquals(true, out[4].contains("_L1_"));
- assertEquals(true, out[5].contains("_L2_"));
- assertEquals(true, out[6].contains("_L4_"));
+ assertEquals(true, out[2].contains("_L1_"));
+ assertEquals(true, out[3].contains("_L2_"));
+ assertEquals(true, out[4].contains("_L4_"));
+ assertEquals(true, out[5].contains("_L8_"));
+ assertEquals(true, out[6].contains("_L9_"));
assertEquals(true, out[7].contains("_L7_"));
}
Modified:
oozie/branches/hcat-intre/docs/src/site/twiki/ENG_Custom_Authentication.twiki
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/docs/src/site/twiki/ENG_Custom_Authentication.twiki?rev=1438631&r1=1438630&r2=1438631&view=diff
==============================================================================
---
oozie/branches/hcat-intre/docs/src/site/twiki/ENG_Custom_Authentication.twiki
(original)
+++
oozie/branches/hcat-intre/docs/src/site/twiki/ENG_Custom_Authentication.twiki
Fri Jan 25 18:25:12 2013
@@ -187,7 +187,7 @@ in this property are contained in the us
be used.
<verbatim>
<property>
- <name>alt-kerberos.non-browser.user-agents</name>
+ <name>oozie.authentication.alt-kerberos.non-browser.user-agents</name>
<value>java,curl,wget,perl</value>
</property>
</verbatim>
Modified: oozie/branches/hcat-intre/minitest/pom.xml
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/minitest/pom.xml?rev=1438631&r1=1438630&r2=1438631&view=diff
==============================================================================
--- oozie/branches/hcat-intre/minitest/pom.xml (original)
+++ oozie/branches/hcat-intre/minitest/pom.xml Fri Jan 25 18:25:12 2013
@@ -72,13 +72,13 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
- <version>0.20.104.2</version>
+ <version>1.1.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-test</artifactId>
- <version>0.20.104.2</version>
+ <version>1.1.1</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -93,6 +93,18 @@
<version>1.5.2</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.10</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>jsr311-api</artifactId>
+ <version>0.11</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
Modified: oozie/branches/hcat-intre/release-log.txt
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/release-log.txt?rev=1438631&r1=1438630&r2=1438631&view=diff
==============================================================================
--- oozie/branches/hcat-intre/release-log.txt (original)
+++ oozie/branches/hcat-intre/release-log.txt Fri Jan 25 18:25:12 2013
@@ -1,5 +1,15 @@
-- Oozie 3.4.0 release (trunk - unreleased)
+OOZIE-1188 Typo in documentation for using login server example (rkanter)
+OOZIE-1177 HostnameFilter should only catch UnknownHostException, not
Exception (rkanter)
+OOZIE-945 BundleSubmitXCommand.submit() doesn't properly remove comments
(jaoki via rkanter)
+OOZIE-1171 HostnameFilter should handle hostname resolution failures and
continue processing (tucu via rkanter)
+OOZIE-1166 Print a more helpful message when ProxyUserService is configured
wrong (rkanter)
+OOZIE-1136 Fix MiniOozie (rkanter)
+OOZIE-1051 Repeating Errors for workflows that were allreday Killed (rkanter)
+OOZIE-1140 TestLogStreamer.testStreamLog fails when its started within the
first 4 seconds after the hour (rkanter)
+OOZIE-1153 comma separated list in <archive> and <file> for
JavaActionExecutor. (jaoki via tucu)
+OOZIE-1160 Oozie web-console to display all job URLs spawned by Pig (mona)
OOZIE-1186 Image load for Job DAG visualization should handle resources better
(mona)
OOZIE-1185 Retry jms connections on failure (rohini via virag)
OOZIE-1158 Add hcataloglib sub-module (mona)
Modified: oozie/branches/hcat-intre/webapp/src/main/webapp/oozie-console.js
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/webapp/src/main/webapp/oozie-console.js?rev=1438631&r1=1438630&r2=1438631&view=diff
==============================================================================
--- oozie/branches/hcat-intre/webapp/src/main/webapp/oozie-console.js (original)
+++ oozie/branches/hcat-intre/webapp/src/main/webapp/oozie-console.js Fri Jan
25 18:25:12 2013
@@ -246,7 +246,7 @@ function jobDetailsPopup(response, reque
var appName = jobDetails["appName"];
var jobActionStatus = new Ext.data.JsonStore({
data: jobDetails["actions"],
- fields: ['id', 'name', 'type', 'startTime', 'retries', 'consoleUrl',
'endTime', 'externalId', 'status', 'trackerUri', 'workflowId', 'errorCode',
'errorMessage', 'conf', 'transition', 'externalStatus']
+ fields: ['id', 'name', 'type', 'startTime', 'retries', 'consoleUrl',
'endTime', 'externalId', 'status', 'trackerUri', 'workflowId', 'errorCode',
'errorMessage', 'conf', 'transition', 'externalStatus', 'externalChildIDs']
});
var formFieldSet = new Ext.form.FieldSet({
@@ -492,8 +492,8 @@ function jobDetailsPopup(response, reque
width: 400,
value: actionStatus["trackerUri"]
- }, ]
- });
+ }
+ ]});
var detail = new Ext.FormPanel({
frame: true,
labelAlign: 'right',
@@ -501,6 +501,8 @@ function jobDetailsPopup(response, reque
width: 540,
items: [formFieldSet]
});
+ var urlUnit = new Ext.FormPanel();
+ populateUrlUnit(actionStatus, urlUnit);
var win = new Ext.Window({
title: 'Action (Name: ' + actionStatus["name"] + '/JobId: ' +
workflowId + ')',
closable: true,
@@ -525,13 +527,76 @@ function jobDetailsPopup(response, reque
autoScroll: true,
value: actionStatus["conf"]
})
- }, ]
+ }, {
+ title: 'Child Job URLs',
+ autoScroll: true,
+ frame: true,
+ labelAlign: 'right',
+ labelWidth: 70,
+ items: urlUnit
+ }],
+ tbar: [{
+ text: " ",
+ icon:
'ext-2.2/resources/images/default/grid/refresh.gif',
+ handler: function() {
+
refreshActionDetails(workflowId+"@"+actionStatus["name"], detail, urlUnit);
+ }
+ }]
})]
});
win.setPosition(50, 50);
win.show();
}
}
+
+ function populateUrlUnit(actionStatus, urlUnit) {
+ var consoleUrl = actionStatus["consoleUrl"];
+ var externalChildIDs = actionStatus["externalChildIDs"];
+ if(undefined !== consoleUrl && null !== consoleUrl && undefined
!== externalChildIDs && null !== externalChildIDs) {
+ var urlPrefix = consoleUrl.trim().split(/_/)[0];
+ //externalChildIds is a comma-separated string of each child job
ID.
+ //Create URL list by appending jobID portion after stripping "job"
+ var jobIds = externalChildIDs.split(/,/);
+ var count = 1;
+ jobIds.forEach(function(jobId) {
+ jobId = jobId.trim().split(/job/)[1];
+ var jobUrl = new Ext.form.TriggerField({
+ fieldLabel: 'Child Job ' + count,
+ editable: false,
+ name: 'childJobURLs',
+ width: 400,
+ value: urlPrefix + jobId,
+ triggerClass: 'x-form-search-trigger',
+ onTriggerClick: function() {
+ window.open(urlPrefix + jobId);
+ }
+ });
+ if(jobId != undefined) {
+ urlUnit.add(jobUrl);
+ count++;
+ }
+ });
+ } else {
+ var note = new Ext.form.TextField({
+ fieldLabel: 'Child Job',
+ value: 'n/a'
+ });
+ urlUnit.add(note);
+ }
+ }
+
+ function refreshActionDetails(actionId, detail, urlUnit) {
+ Ext.Ajax.request({
+ url: getOozieBase() + 'job/' + actionId + "?timezone=" +
getTimeZone(),
+ success: function(response, request) {
+ var results = eval("(" + response.responseText + ")");
+ detail.getForm().setValues(results);
+ urlUnit.getForm().setValues(results);
+ populateUrlUnit(results, urlUnit);
+ }
+ });
+ }
+
var dagImg = new Ext.ux.Image({
id: 'dagImage',
url: getOozieBase() + 'job/' + workflowId + "?show=graph",
@@ -701,7 +766,7 @@ function coordJobDetailsPopup(response,
var appName = jobDetails["coordJobName"];
var jobActionStatus = new Ext.data.JsonStore({
data: jobDetails["actions"],
- fields: ['id', 'name', 'type', 'createdConf', 'runConf',
'actionNumber', 'createdTime', 'externalId', 'lastModifiedTime', 'nominalTime',
'status', 'missingDependencies', 'externalStatus', 'trackerUri', 'consoleUrl',
'errorCode', 'errorMessage', 'actions']
+ fields: ['id', 'name', 'type', 'createdConf', 'runConf',
'actionNumber', 'createdTime', 'externalId', 'lastModifiedTime', 'nominalTime',
'status', 'missingDependencies', 'externalStatus', 'trackerUri', 'consoleUrl',
'errorCode', 'errorMessage', 'actions', 'externalChildIDs']
});
