Author: rkanter
Date: Wed Jul 10 23:37:01 2013
New Revision: 1502059
URL: http://svn.apache.org/r1502059
Log:
OOZIE-1447 Sqoop actions that don't launch a map reduce job fail with an
IllegalArgumentException (jarcec via rkanter)
Modified:
oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
oozie/trunk/release-log.txt
oozie/trunk/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java
Modified:
oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java?rev=1502059&r1=1502058&r2=1502059&view=diff
==============================================================================
---
oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
(original)
+++
oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
Wed Jul 10 23:37:01 2013
@@ -142,26 +142,29 @@ public class SqoopActionExecutor extends
// Cumulative counters for all Sqoop mapreduce jobs
Counters counters = null;
+ // Sqoop do not have to create mapreduce job each time
String externalIds = action.getExternalChildIDs();
- String []jobIds = externalIds.split(",");
+ if (externalIds != null && !externalIds.trim().isEmpty()) {
+ String []jobIds = externalIds.split(",");
- for(String jobId : jobIds) {
- RunningJob runningJob =
jobClient.getJob(JobID.forName(jobId));
- if (runningJob == null) {
- throw new
ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "SQOOP001",
- "Unknown hadoop job [{0}] associated with action
[{1}]. Failing this action!", action
- .getExternalId(), action.getId());
- }
+ for(String jobId : jobIds) {
+ RunningJob runningJob =
jobClient.getJob(JobID.forName(jobId));
+ if (runningJob == null) {
+ throw new
ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "SQOOP001",
+ "Unknown hadoop job [{0}] associated with action
[{1}]. Failing this action!", action
+ .getExternalId(), action.getId());
+ }
- Counters taskCounters = runningJob.getCounters();
- if(taskCounters != null) {
- if(counters == null) {
- counters = taskCounters;
+ Counters taskCounters = runningJob.getCounters();
+ if(taskCounters != null) {
+ if(counters == null) {
+ counters = taskCounters;
+ } else {
+ counters.incrAllCounters(taskCounters);
+ }
} else {
- counters.incrAllCounters(taskCounters);
+ XLog.getLog(getClass()).warn("Could not find Hadoop
Counters for job: [{0}]", jobId);
}
- } else {
- XLog.getLog(getClass()).warn("Could not find Hadoop
Counters for job: [{0}]", jobId);
}
}
Modified: oozie/trunk/release-log.txt
URL:
http://svn.apache.org/viewvc/oozie/trunk/release-log.txt?rev=1502059&r1=1502058&r2=1502059&view=diff
==============================================================================
--- oozie/trunk/release-log.txt (original)
+++ oozie/trunk/release-log.txt Wed Jul 10 23:37:01 2013
@@ -1,5 +1,6 @@
-- Oozie 4.1.0 release (trunk - unreleased)
+OOZIE-1447 Sqoop actions that don't launch a map reduce job fail with an
IllegalArgumentException (jarcec via rkanter)
OOZIE-1440 Build fails in certain environments due to xerces OpenJPA issue
(mackrorysd via rkanter)
-- Oozie 4.0.0 release
Modified:
oozie/trunk/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java?rev=1502059&r1=1502058&r2=1502059&view=diff
==============================================================================
---
oozie/trunk/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java
(original)
+++
oozie/trunk/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java
Wed Jul 10 23:37:01 2013
@@ -95,6 +95,27 @@ public class TestSqoopActionExecutor ext
"<arg>I</arg>" +
"</sqoop>";
+ private static final String SQOOP_ACTION_EVAL_XML =
+ "<sqoop xmlns=\"uri:oozie:sqoop-action:0.1\">" +
+ "<job-tracker>{0}</job-tracker>" +
+ "<name-node>{1}</name-node>" +
+ "<configuration>" +
+ "<property>" +
+ "<name>oozie.sqoop.log.level</name>" +
+ "<value>INFO</value>" +
+ "</property>" +
+ "</configuration>" +
+ "<arg>eval</arg>" +
+ "<arg>--connect</arg>" +
+ "<arg>{2}</arg>" +
+ "<arg>--username</arg>" +
+ "<arg>sa</arg>" +
+ "<arg>--password</arg>" +
+ "<arg></arg>" +
+ "<arg>--verbose</arg>" +
+ "<arg>--query</arg>" +
+ "<arg>{3}</arg>" +
+ "</sqoop>";
@Override
protected void setSystemProps() throws Exception {
@@ -168,6 +189,12 @@ public class TestSqoopActionExecutor ext
"dummy", "dummyValue", command);
}
+ private String getActionXmlEval() {
+ String query = "select TT.I, TT.S from TT";
+ return MessageFormat.format(SQOOP_ACTION_EVAL_XML, getJobTrackerUri(),
getNameNodeUri(),
+ getActionJdbcUri(), query);
+ }
+
private String getActionXmlFreeFromQuery() {
String query = "select TT.I, TT.S from TT where $CONDITIONS";
return MessageFormat.format(SQOOP_ACTION_ARGS_XML, getJobTrackerUri(),
getNameNodeUri(),
@@ -233,6 +260,42 @@ public class TestSqoopActionExecutor ext
assertTrue(outputData.getProperty(LauncherMain.HADOOP_JOBS).trim().length() >
0);
}
+ public void testSqoopEval() throws Exception {
+ createDB();
+
+ Context context = createContext(getActionXmlEval());
+ final RunningJob launcherJob = submitAction(context);
+ String launcherId = context.getAction().getExternalId();
+ waitFor(120 * 1000, new Predicate() {
+ public boolean evaluate() throws Exception {
+ return launcherJob.isComplete();
+ }
+ });
+ assertTrue(launcherJob.isSuccessful());
+
+ assertFalse(LauncherMapperHelper.hasIdSwap(launcherJob));
+
+ SqoopActionExecutor ae = new SqoopActionExecutor();
+ ae.check(context, context.getAction());
+ assertTrue(launcherId.equals(context.getAction().getExternalId()));
+ assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
+ assertNotNull(context.getAction().getData());
+ assertNotNull(context.getAction().getExternalChildIDs());
+ assertEquals(0, context.getAction().getExternalChildIDs().length());
+ ae.end(context, context.getAction());
+ assertEquals(WorkflowAction.Status.OK,
context.getAction().getStatus());
+
+ String hadoopCounters =
context.getVar(MapReduceActionExecutor.HADOOP_COUNTERS);
+ assertNotNull(hadoopCounters);
+ assertTrue(hadoopCounters.isEmpty());
+
+ assertNotNull(context.getAction().getData());
+ Properties outputData = new Properties();
+ outputData.load(new StringReader(context.getAction().getData()));
+ assertTrue(outputData.containsKey(LauncherMain.HADOOP_JOBS));
+ assertEquals(0,
outputData.getProperty(LauncherMain.HADOOP_JOBS).trim().length());
+ }
+
public void testSqoopActionFreeFormQuery() throws Exception {
createDB();
