Author: mona
Date: Thu Nov 29 01:38:13 2012
New Revision: 1415025
URL: http://svn.apache.org/viewvc?rev=1415025&view=rev
Log:
sync patches with trunk
Modified:
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/command/wf/ActionCheckXCommand.java
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java
oozie/branches/hcat-intre/release-log.txt
oozie/branches/hcat-intre/workflowgenerator/README.txt
oozie/branches/hcat-intre/workflowgenerator/src/main/java/org/apache/oozie/tools/workflowgenerator/client/OozieWorkflowGenerator.java
Modified:
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java?rev=1415025&r1=1415024&r2=1415025&view=diff
==============================================================================
---
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
(original)
+++
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
Thu Nov 29 01:38:13 2012
@@ -79,10 +79,10 @@ import org.apache.hadoop.security.token.
public class JavaActionExecutor extends ActionExecutor {
private static final String HADOOP_USER = "user.name";
- private static final String HADOOP_JOB_TRACKER = "mapred.job.tracker";
- private static final String HADOOP_JOB_TRACKER_2 =
"mapreduce.jobtracker.address";
- private static final String HADOOP_YARN_RM =
"yarn.resourcemanager.address";
- private static final String HADOOP_NAME_NODE = "fs.default.name";
+ public static final String HADOOP_JOB_TRACKER = "mapred.job.tracker";
+ public static final String HADOOP_JOB_TRACKER_2 =
"mapreduce.jobtracker.address";
+ public static final String HADOOP_YARN_RM = "yarn.resourcemanager.address";
+ public static final String HADOOP_NAME_NODE = "fs.default.name";
private static final String HADOOP_JOB_NAME = "mapred.job.name";
public static final String OOZIE_COMMON_LIBDIR = "oozie";
public static final int MAX_EXTERNAL_STATS_SIZE_DEFAULT =
Integer.MAX_VALUE;
@@ -711,7 +711,7 @@ public class JavaActionExecutor extends
if (alreadyRunning && !isUserRetry) {
runningJob = jobClient.getJob(JobID.forName(launcherId));
if (runningJob == null) {
- String jobTracker =
launcherJobConf.get("mapred.job.tracker");
+ String jobTracker =
launcherJobConf.get(HADOOP_JOB_TRACKER);
throw new
ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "JA017",
"unknown job [{0}@{1}], cannot recover",
launcherId, jobTracker);
}
@@ -720,8 +720,8 @@ public class JavaActionExecutor extends
XLog.getLog(getClass()).debug("Submitting the job through Job
Client for action " + action.getId());
// setting up propagation of the delegation token.
- Token<DelegationTokenIdentifier> mrdt =
jobClient.getDelegationToken(new Text("mr token"));
- launcherJobConf.getCredentials().addToken(new Text("mr
token"), mrdt);
+ Token<DelegationTokenIdentifier> mrdt =
jobClient.getDelegationToken(new
Text(HadoopAccessorService.MR_DELEGATION_TOKEN));
+ launcherJobConf.getCredentials().addToken(new
Text(HadoopAccessorService.MR_DELEGATION_TOKEN), mrdt);
// insert credentials tokens to launcher job conf if needed
if (needInjectCredentials()) {
Modified:
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/command/wf/ActionCheckXCommand.java
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/command/wf/ActionCheckXCommand.java?rev=1415025&r1=1415024&r2=1415025&view=diff
==============================================================================
---
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/command/wf/ActionCheckXCommand.java
(original)
+++
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/command/wf/ActionCheckXCommand.java
Thu Nov 29 01:38:13 2012
@@ -39,6 +39,7 @@ import org.apache.oozie.executor.jpa.JPA
import org.apache.oozie.executor.jpa.WorkflowActionGetJPAExecutor;
import org.apache.oozie.executor.jpa.WorkflowActionUpdateJPAExecutor;
import org.apache.oozie.executor.jpa.WorkflowJobGetJPAExecutor;
+import org.apache.oozie.service.ActionCheckerService;
import org.apache.oozie.service.ActionService;
import org.apache.oozie.service.JPAService;
import org.apache.oozie.service.Services;
@@ -156,6 +157,10 @@ public class ActionCheckXCommand extends
protected Void execute() throws CommandException {
LOG.debug("STARTED ActionCheckXCommand for wf actionId=" + actionId +
" priority =" + getPriority());
+ long retryInterval =
Services.get().getConf().getLong(ActionCheckerService.CONF_ACTION_CHECK_INTERVAL,
executor
+ .getRetryInterval());
+ executor.setRetryInterval(retryInterval);
+
ActionExecutorContext context = null;
try {
boolean isRetry = false;
@@ -239,4 +244,9 @@ public class ActionCheckXCommand extends
InstrumentUtils.incrJobCounter(INSTR_FAILED_JOBS_COUNTER, 1,
getInstrumentation());
}
}
+
+ protected long getRetryInterval() {
+ return (executor != null) ? executor.getRetryInterval() :
ActionExecutor.RETRY_INTERVAL;
+ }
+
}
Modified:
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java?rev=1415025&r1=1415024&r2=1415025&view=diff
==============================================================================
---
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
(original)
+++
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
Thu Nov 29 01:38:13 2012
@@ -28,6 +28,7 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.security.token.Token;
import org.apache.oozie.ErrorCode;
+import org.apache.oozie.action.hadoop.JavaActionExecutor;
import org.apache.oozie.util.ParamChecker;
import org.apache.oozie.util.XConfiguration;
import org.apache.oozie.util.XLog;
@@ -62,6 +63,7 @@ public class HadoopAccessorService imple
public static final String KERBEROS_AUTH_ENABLED = CONF_PREFIX +
"kerberos.enabled";
public static final String KERBEROS_KEYTAB = CONF_PREFIX + "keytab.file";
public static final String KERBEROS_PRINCIPAL = CONF_PREFIX +
"kerberos.principal";
+ public static final String MR_DELEGATION_TOKEN = "oozie mr token";
private static final String OOZIE_HADOOP_ACCESSOR_SERVICE_CREATED =
"oozie.HadoopAccessorService.created";
@@ -351,7 +353,7 @@ public class HadoopAccessorService imple
if (!conf.getBoolean(OOZIE_HADOOP_ACCESSOR_SERVICE_CREATED, false)) {
throw new HadoopAccessorException(ErrorCode.E0903);
}
- String jobTracker = conf.get("mapred.job.tracker");
+ String jobTracker = conf.get(JavaActionExecutor.HADOOP_JOB_TRACKER);
validateJobTracker(jobTracker);
try {
UserGroupInformation ugi = getUGI(user);
@@ -360,8 +362,8 @@ public class HadoopAccessorService imple
return new JobClient(conf);
}
});
- Token<DelegationTokenIdentifier> mrdt =
jobClient.getDelegationToken(new Text("mr token"));
- conf.getCredentials().addToken(new Text("mr token"), mrdt);
+ Token<DelegationTokenIdentifier> mrdt =
jobClient.getDelegationToken(new Text(MR_DELEGATION_TOKEN));
+ conf.getCredentials().addToken(new Text(MR_DELEGATION_TOKEN),
mrdt);
return jobClient;
}
catch (InterruptedException ex) {
Modified:
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java?rev=1415025&r1=1415024&r2=1415025&view=diff
==============================================================================
---
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java
(original)
+++
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java
Thu Nov 29 01:38:13 2012
@@ -40,6 +40,7 @@ import org.apache.oozie.executor.jpa.JPA
import org.apache.oozie.executor.jpa.WorkflowActionGetJPAExecutor;
import org.apache.oozie.executor.jpa.WorkflowActionInsertJPAExecutor;
import org.apache.oozie.executor.jpa.WorkflowJobGetJPAExecutor;
+import org.apache.oozie.service.ActionCheckerService;
import org.apache.oozie.service.HadoopAccessorService;
import org.apache.oozie.service.InstrumentationService;
import org.apache.oozie.service.JPAService;
@@ -479,6 +480,28 @@ public class TestActionCheckXCommand ext
assertEquals("SUCCEEDED", action5.getExternalStatus());
}
+ /**
+ * This test case verifies if getRetryInterval picks up the
+ * overridden value.
+ *
+ * @throws Exception
+ */
+ public void testCheckInterval() throws Exception {
+ long testedValue = 10;
+
Services.get().getConf().setLong(ActionCheckerService.CONF_ACTION_CHECK_INTERVAL,
+ testedValue);
+
+ WorkflowJobBean job0 =
this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING,
WorkflowInstance.Status.RUNNING);
+ final String jobId = job0.getId();
+ WorkflowActionBean action0 = this.addRecordToWfActionTable(jobId, "1",
WorkflowAction.Status.RUNNING);
+ final String actionId = action0.getId();
+
+ ActionCheckXCommand checkCommand = new ActionCheckXCommand(actionId);
+ checkCommand.call();
+ long effectiveValue = checkCommand.getRetryInterval();
+ assertEquals(testedValue, effectiveValue);
+ }
+
@Override
protected WorkflowActionBean addRecordToWfActionTable(String wfId, String
actionName, WorkflowAction.Status status) throws Exception {
WorkflowActionBean action = createWorkflowActionSetPending(wfId,
status);
Modified: oozie/branches/hcat-intre/release-log.txt
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/release-log.txt?rev=1415025&r1=1415024&r2=1415025&view=diff
==============================================================================
--- oozie/branches/hcat-intre/release-log.txt (original)
+++ oozie/branches/hcat-intre/release-log.txt Thu Nov 29 01:38:13 2012
@@ -1,5 +1,9 @@
-- Oozie 3.4.0 release (trunk - unreleased)
+OOZIE-1096 Update wfgen README.txt to have the TLP mailing list (jun aoki via
rkanter)
+OOZIE-1015 HadoopAccessorService jobtracker validation should not have
hardcoded conf key (mona)
+OOZIE-1078 Help -> Documentation and Help -> Online Help should link to
oozie.apache.org/ (jun via mohammad)
+OOZIE-1057 Log message for retrying to connect to the JT always says 60,000
milliseconds (jiezhou via rkanter)
OOZIE-1080 Add a dryrun option for workflows (rkanter)
OOZIE-1086 Command to check the missing partitions directly against HCatalog
server (mohammad)
OOZIE-1050 Implement logic to update dependencies via push JMS message(mona
via mohammad)
Modified: oozie/branches/hcat-intre/workflowgenerator/README.txt
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/workflowgenerator/README.txt?rev=1415025&r1=1415024&r2=1415025&view=diff
==============================================================================
--- oozie/branches/hcat-intre/workflowgenerator/README.txt (original)
+++ oozie/branches/hcat-intre/workflowgenerator/README.txt Thu Nov 29 01:38:13
2012
@@ -131,8 +131,8 @@ if you want to kill process, Ctrl-C in c
If you have any questions/issues, please send an email to:
[email protected]
[email protected]
Subscribe using the link:
-http://incubator.apache.org/oozie/MailingLists.html
+http://oozie.apache.org/mail-lists.html
Modified:
oozie/branches/hcat-intre/workflowgenerator/src/main/java/org/apache/oozie/tools/workflowgenerator/client/OozieWorkflowGenerator.java
URL:
http://svn.apache.org/viewvc/oozie/branches/hcat-intre/workflowgenerator/src/main/java/org/apache/oozie/tools/workflowgenerator/client/OozieWorkflowGenerator.java?rev=1415025&r1=1415024&r2=1415025&view=diff
==============================================================================
---
oozie/branches/hcat-intre/workflowgenerator/src/main/java/org/apache/oozie/tools/workflowgenerator/client/OozieWorkflowGenerator.java
(original)
+++
oozie/branches/hcat-intre/workflowgenerator/src/main/java/org/apache/oozie/tools/workflowgenerator/client/OozieWorkflowGenerator.java
Thu Nov 29 01:38:13 2012
@@ -537,8 +537,20 @@ public class OozieWorkflowGenerator impl
MenuBar helpMenu = new MenuBar(true);
helpMenu.setAutoOpen(true);
helpMenu.setAnimationEnabled(true);
- helpMenu.addItem("Documentation", cmd);
- helpMenu.addItem("Online Help", cmd);
+
+ // TODO this should point to a workflowgenerator's maven site, however
there is no maven site available. (Not even in
+ // Workspace of the jenkins job at
https://builds.apache.org/job/oozie-trunk-precommit-build/ws/workflowgenerator/target/)
+ // where client, for example, has target/site/apidocs
+ // The ideal place is somewhere under http://oozie.apache.org/docs/
once it is generated.
+ Command openOozieTopPageComman = new Command() {
+ @Override
+ public void execute() {
+ Window.open("http://oozie.apache.org/";, "_blank", "");
+ }
+ };
+ helpMenu.addItem("Documentation", openOozieTopPageComman);
+ helpMenu.addItem("Online Help", openOozieTopPageComman);
+
helpMenu.addItem("About", cmd);
MenuBar menu = new MenuBar();
