Repository: oozie Updated Branches: refs/heads/branch-4.1 ae471035b -> cb3e5653b
OOZIE-1914 CLI should retry on timeout Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/cb3e5653 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/cb3e5653 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/cb3e5653 Branch: refs/heads/branch-4.1 Commit: cb3e5653b5202bd4da3ba5e778525ac9bf77ac97 Parents: ae47103 Author: Purshotam Shah <[email protected]> Authored: Fri Oct 17 12:36:25 2014 -0700 Committer: Purshotam Shah <[email protected]> Committed: Fri Oct 17 12:36:25 2014 -0700 ---------------------------------------------------------------------- .../java/org/apache/oozie/cli/OozieCLI.java | 61 ++++++++----- .../org/apache/oozie/client/OozieClient.java | 41 +++++++-- .../client/retry/ConnectionRetriableClient.java | 92 ++++++++++++++++++++ .../apache/oozie/TestLocalOozieClientCoord.java | 4 +- .../org/apache/oozie/client/TestOozieCLI.java | 75 ++++++++++++++++ docs/src/site/twiki/DG_CommandLineTool.twiki | 8 ++ release-log.txt | 1 + 7 files changed, 255 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/cb3e5653/client/src/main/java/org/apache/oozie/cli/OozieCLI.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java index 708d038..21dfd2d 100644 --- a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java +++ b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java @@ -79,6 +79,7 @@ public class OozieCLI { public static final String ENV_OOZIE_DEBUG = "OOZIE_DEBUG"; public static final String ENV_OOZIE_TIME_ZONE = "OOZIE_TIMEZONE"; public static final String ENV_OOZIE_AUTH = "OOZIE_AUTH"; + public static final String OOZIE_RETRY_COUNT = "oozie.connection.retry.count"; public static final String WS_HEADER_PREFIX = "header:"; public static final String HELP_CMD = "help"; @@ -165,6 +166,8 @@ public class OozieCLI { private static final String RULER; private static final int LINE_WIDTH = 132; + private static final int RETRY_COUNT = 4; + private boolean used; private static final String INSTANCE_SEPARATOR = "#"; @@ -533,24 +536,7 @@ public class OozieCLI { throw new IllegalStateException("CLI instance already used"); } used = true; - - final CLIParser parser = new CLIParser(OOZIE_OPTION, getCLIHelp()); - parser.addCommand(HELP_CMD, "", "display usage for all commands or specified command", new Options(), false); - parser.addCommand(VERSION_CMD, "", "show client version", new Options(), false); - parser.addCommand(JOB_CMD, "", "job operations", createJobOptions(), false); - parser.addCommand(JOBS_CMD, "", "jobs status", createJobsOptions(), false); - parser.addCommand(ADMIN_CMD, "", "admin operations", createAdminOptions(), false); - parser.addCommand(VALIDATE_CMD, "", "validate a workflow XML file", new Options(), true); - parser.addCommand(SLA_CMD, "", "sla operations (Deprecated with Oozie 4.0)", createSlaOptions(), false); - parser.addCommand(PIG_CMD, "-X ", "submit a pig job, everything after '-X' are pass-through parameters to pig, any '-D' " - + "arguments after '-X' are put in <configuration>", createScriptLanguageOptions(PIG_CMD), true); - parser.addCommand(HIVE_CMD, "-X ", "submit a hive job, everything after '-X' are pass-through parameters to hive, any '-D' " - + "arguments after '-X' are put in <configuration>", createScriptLanguageOptions(HIVE_CMD), true); - parser.addCommand(SQOOP_CMD, "-X ", "submit a sqoop job, everything after '-X' are pass-through parameters " + - "to sqoop, any '-D' arguments after '-X' are put in <configuration>", createSqoopCLIOptions(), true); - parser.addCommand(INFO_CMD, "", "get more detailed info about specific topics", createInfoOptions(), false); - parser.addCommand(MR_CMD, "", "submit a mapreduce job", createMROptions(), false); - + final CLIParser parser = getCLIParser(); try { final CLIParser.Command command = parser.parse(args); @@ -568,7 +554,6 @@ public class OozieCLI { else { processCommand(parser, command); } - return 0; } catch (OozieCLIException ex) { @@ -588,7 +573,28 @@ public class OozieCLI { } } - private void processCommand(CLIParser parser, CLIParser.Command command) throws Exception { + @VisibleForTesting + public CLIParser getCLIParser(){ + CLIParser parser = new CLIParser(OOZIE_OPTION, getCLIHelp()); + parser.addCommand(HELP_CMD, "", "display usage for all commands or specified command", new Options(), false); + parser.addCommand(VERSION_CMD, "", "show client version", new Options(), false); + parser.addCommand(JOB_CMD, "", "job operations", createJobOptions(), false); + parser.addCommand(JOBS_CMD, "", "jobs status", createJobsOptions(), false); + parser.addCommand(ADMIN_CMD, "", "admin operations", createAdminOptions(), false); + parser.addCommand(VALIDATE_CMD, "", "validate a workflow XML file", new Options(), true); + parser.addCommand(SLA_CMD, "", "sla operations (Deprecated with Oozie 4.0)", createSlaOptions(), false); + parser.addCommand(PIG_CMD, "-X ", "submit a pig job, everything after '-X' are pass-through parameters to pig, any '-D' " + + "arguments after '-X' are put in <configuration>", createScriptLanguageOptions(PIG_CMD), true); + parser.addCommand(HIVE_CMD, "-X ", "submit a hive job, everything after '-X' are pass-through parameters to hive, any '-D' " + + "arguments after '-X' are put in <configuration>", createScriptLanguageOptions(HIVE_CMD), true); + parser.addCommand(SQOOP_CMD, "-X ", "submit a sqoop job, everything after '-X' are pass-through parameters " + + "to sqoop, any '-D' arguments after '-X' are put in <configuration>", createSqoopCLIOptions(), true); + parser.addCommand(INFO_CMD, "", "get more detailed info about specific topics", createInfoOptions(), false); + parser.addCommand(MR_CMD, "", "submit a mapreduce job", createMROptions(), false); + return parser; + } + + public void processCommand(CLIParser parser, CLIParser.Command command) throws Exception { if (command.getName().equals(HELP_CMD)) { parser.showHelp(command.getCommandLine()); } @@ -834,6 +840,7 @@ public class OozieCLI { XOozieClient wc = new AuthOozieClient(getOozieUrl(commandLine), getAuthOption(commandLine)); addHeader(wc); setDebugMode(wc,commandLine.hasOption(DEBUG_OPTION)); + setRetryCount(wc); return wc; } @@ -856,6 +863,20 @@ public class OozieCLI { } } + protected void setRetryCount(OozieClient wc) { + String retryCount = System.getProperty(OOZIE_RETRY_COUNT); + if (retryCount != null && !retryCount.isEmpty()) { + try { + int retry = Integer.parseInt(retryCount.trim()); + wc.setRetryCount(retry); + } + catch (Exception ex) { + System.err.println("Unable to parse the retry settings. May be not an integer [" + retryCount + "]"); + ex.printStackTrace(); + } + } + } + private static String JOB_ID_PREFIX = "job: "; private void jobCommand(CommandLine commandLine) throws IOException, OozieCLIException { http://git-wip-us.apache.org/repos/asf/oozie/blob/cb3e5653/client/src/main/java/org/apache/oozie/client/OozieClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/client/OozieClient.java b/client/src/main/java/org/apache/oozie/client/OozieClient.java index 7204d60..a9d8818 100644 --- a/client/src/main/java/org/apache/oozie/client/OozieClient.java +++ b/client/src/main/java/org/apache/oozie/client/OozieClient.java @@ -21,6 +21,7 @@ import org.apache.oozie.BuildInfo; import org.apache.oozie.client.rest.JsonTags; import org.apache.oozie.client.rest.JsonToBean; import org.apache.oozie.client.rest.RestConstants; +import org.apache.oozie.client.retry.ConnectionRetriableClient; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.JSONValue; @@ -166,6 +167,9 @@ public class OozieClient { */ public int debugMode = 0; + private int retryCount = 4; + + private String baseUrl; private String protocolUrl; private boolean validatedVersion = false; @@ -253,6 +257,15 @@ public class OozieClient { this.debugMode = debugMode; } + public int getRetryCount() { + return retryCount; + } + + + public void setRetryCount(int retryCount) { + this.retryCount = retryCount; + } + private String getBaseURLForVersion(long protocolVersion) throws OozieClientException { try { if (supportedVersions == null) { @@ -320,8 +333,10 @@ public class OozieClient { private JSONArray getSupportedProtocolVersions() throws IOException, OozieClientException { JSONArray versions = null; - URL url = new URL(baseUrl + RestConstants.VERSIONS); - HttpURLConnection conn = createConnection(url, "GET"); + final URL url = new URL(baseUrl + RestConstants.VERSIONS); + + HttpURLConnection conn = createRetryableConnection(url, "GET"); + if (conn.getResponseCode() == HttpURLConnection.HTTP_OK) { versions = (JSONArray) JSONValue.parse(new InputStreamReader(conn.getInputStream())); } @@ -430,6 +445,24 @@ public class OozieClient { } return true; } + /** + * Create retryable http connection to oozie server. + * + * @param url + * @param method + * @return connection + * @throws IOException + * @throws OozieClientException + */ + protected HttpURLConnection createRetryableConnection(final URL url, final String method) throws IOException{ + return (HttpURLConnection) new ConnectionRetriableClient(getRetryCount()) { + @Override + public Object doExecute(URL url, String method) throws IOException, OozieClientException { + HttpURLConnection conn = createConnection(url, method); + return conn; + } + }.execute(url, method); + } /** * Create http connection to oozie server. @@ -478,8 +511,7 @@ public class OozieClient { if (getDebugMode() > 0) { System.out.println(method + " " + url); } - HttpURLConnection conn = createConnection(url, method); - return call(conn); + return call(createRetryableConnection(url, method)); } else { System.out.println("Option not supported in target server. Supported only on Oozie-2.0 or greater." @@ -490,7 +522,6 @@ public class OozieClient { catch (IOException ex) { throw new OozieClientException(OozieClientException.IO_ERROR, ex); } - } protected abstract T call(HttpURLConnection conn) throws IOException, OozieClientException; http://git-wip-us.apache.org/repos/asf/oozie/blob/cb3e5653/client/src/main/java/org/apache/oozie/client/retry/ConnectionRetriableClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/client/retry/ConnectionRetriableClient.java b/client/src/main/java/org/apache/oozie/client/retry/ConnectionRetriableClient.java new file mode 100644 index 0000000..3371036 --- /dev/null +++ b/client/src/main/java/org/apache/oozie/client/retry/ConnectionRetriableClient.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.client.retry; + +import java.io.IOException; +import java.net.ConnectException; +import java.net.SocketException; +import java.net.URL; + +/** + * HTTP connection retryable class. It retries =oozie.connection.retry.count= times for ConnectException. For + * SocketException all create connection are retried except =PUT= and =POST=. + */ +public abstract class ConnectionRetriableClient { + private final int retryCount; + + public ConnectionRetriableClient(int retryCount) { + this.retryCount = retryCount; + } + + public Object execute(URL url, String method) throws IOException { + int numTries = 0; + boolean stopRetry = false; + Exception cliException = null; + + while (numTries < retryCount && !stopRetry) { + try { + return doExecute(url, method); + } + catch (ConnectException e) { + sleep(e, numTries++); + cliException = e; + } + catch (SocketException e) { + if (method.equals("POST") || method.equals("PUT")) { + stopRetry = true; + } + else { + sleep(e, numTries++); + } + cliException = e; + } + catch (Exception e) { + stopRetry = true; + cliException = e; + numTries++; + // No retry for other exceptions + } + } + throw new IOException("Error while connecting Oozie server. No of retries = " + numTries + ". Exception = " + + cliException.getMessage(), cliException); + } + + private void sleep(Exception e, int numTries) { + try { + long wait = getWaitTimeExp(numTries); + System.err.println("Connection exception has occurred [ " + e.getClass().getName() + " " + e.getMessage() + + " ]. Trying after " + wait / 1000 + " sec." + " Retry count = " + (numTries + 1)); + Thread.sleep(wait); + } + catch (InterruptedException e1) { + // Ignore InterruptedException + } + } + + /* + * Returns the next wait interval, in milliseconds, using an exponential backoff algorithm. + */ + private long getWaitTimeExp(int retryCount) { + long waitTime = ((long) Math.pow(2, retryCount) * 1000L); + return waitTime; + } + + public abstract Object doExecute(URL url, String method) throws Exception; + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/cb3e5653/core/src/test/java/org/apache/oozie/TestLocalOozieClientCoord.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/TestLocalOozieClientCoord.java b/core/src/test/java/org/apache/oozie/TestLocalOozieClientCoord.java index 37ee256..5178e02 100644 --- a/core/src/test/java/org/apache/oozie/TestLocalOozieClientCoord.java +++ b/core/src/test/java/org/apache/oozie/TestLocalOozieClientCoord.java @@ -67,12 +67,12 @@ public class TestLocalOozieClientCoord extends XDataTestCase { assertEquals("localoozie", client.getOozieUrl()); } - public void testGetProtocolUrl() throws OozieClientException { + public void testGetProtocolUrl() throws IOException, OozieClientException { OozieClient client = LocalOozie.getCoordClient(); assertEquals("localoozie", client.getProtocolUrl()); } - public void testValidateWSVersion() throws OozieClientException { + public void testValidateWSVersion() throws IOException, OozieClientException { OozieClient client = LocalOozie.getCoordClient(); client.validateWSVersion(); } http://git-wip-us.apache.org/repos/asf/oozie/blob/cb3e5653/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java b/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java index a8f8cf9..9f6835e 100644 --- a/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java +++ b/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java @@ -28,6 +28,7 @@ import java.util.concurrent.Callable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.oozie.cli.CLIParser; import org.apache.oozie.cli.OozieCLI; import org.apache.oozie.client.rest.RestConstants; import org.apache.oozie.servlet.DagServletTestCase; @@ -1223,7 +1224,81 @@ public class TestOozieCLI extends DagServletTestCase { return null; } }); + } + public void testRetryForTimeout() throws Exception { + runTest(END_POINTS, SERVLET_CLASSES, false, new Callable<Void>() { + @Override + public Void call() throws Exception { + HeaderTestingVersionServlet.OOZIE_HEADERS.clear(); + String oozieUrl = "http://localhost:11/oozie";; + String[] args = new String[] { "job", "-update", "aaa", "-dryrun", "-oozie", oozieUrl, "-debug" }; + OozieCLI cli = new OozieCLI(); + CLIParser parser = cli.getCLIParser(); + try { + final CLIParser.Command command = parser.parse(args); + cli.processCommand(parser, command); + } + catch (Exception e) { + assertTrue(e.getMessage().contains( + "Error while connecting Oozie server. No of retries = 4. Exception = Connection refused")); + } + return null; + } + }); + } + + public void testNoRetryForError() throws Exception { + runTest(END_POINTS, SERVLET_CLASSES, false, new Callable<Void>() { + @Override + public Void call() throws Exception { + HeaderTestingVersionServlet.OOZIE_HEADERS.clear(); + String oozieUrl = getContextURL(); + String[] args = new String[] { "job", "-info", "aaa", "-oozie", oozieUrl, "-debug" }; + OozieCLI cli = new OozieCLI(); + CLIParser parser = cli.getCLIParser(); + try { + final CLIParser.Command command = parser.parse(args); + cli.processCommand(parser, command); + } + catch (Exception e) { + //Create connection will be successful, no retry + assertFalse(e.getMessage().contains("Error while connecting Oozie server")); + assertTrue(e.getMessage().contains("invalid job id [aaa]")); + } + return null; + } + }); } + public void testRetryWithRetryCount() throws Exception { + runTest(END_POINTS, SERVLET_CLASSES, false, new Callable<Void>() { + @Override + public Void call() throws Exception { + HeaderTestingVersionServlet.OOZIE_HEADERS.clear(); + String oozieUrl = "http://localhost:11/oozie";; + String[] args = new String[] { "job", "-update", "aaa", "-dryrun", "-oozie", oozieUrl, "-debug" }; + OozieCLI cli = new OozieCLI() { + protected void setRetryCount(OozieClient wc) { + wc.setRetryCount(2); + } + public CLIParser getCLIParser(){ + return super.getCLIParser(); + } + + }; + CLIParser parser = cli.getCLIParser(); + try { + final CLIParser.Command command = parser.parse(args); + cli.processCommand(parser, command); + } + catch (Exception e) { + assertTrue(e.getMessage().contains( + "Error while connecting Oozie server. No of retries = 2. Exception = Connection refused")); + } + return null; + } + }); + } + } http://git-wip-us.apache.org/repos/asf/oozie/blob/cb3e5653/docs/src/site/twiki/DG_CommandLineTool.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/DG_CommandLineTool.twiki b/docs/src/site/twiki/DG_CommandLineTool.twiki index c65acbd..763cffc 100644 --- a/docs/src/site/twiki/DG_CommandLineTool.twiki +++ b/docs/src/site/twiki/DG_CommandLineTool.twiki @@ -209,6 +209,14 @@ time zone, it will use GMT. If you export <code>OOZIE_DEBUG=1</code> then the Oozie CLI will output the Web Services API details used by any commands you execute. This is useful for debugging purposes to or see how the Oozie CLI works with the WS API. +---+++ CLI retry +Oozie CLI retries connection to Oozie servers for transparent high availability failover when one of the Oozie servers go down. +=Oozie= CLI command will retry for all commands in case of ConnectException. +In case of SocketException, all commands except =PUT= and =POST= will have retry logic. +All job submit are POST call, examples of PUT and POST commands can be find out from [[WebServicesAPI][WebServicesAPI]]. +Retry count can be configured with system property =oozie.connection.retry.count=. Default count is 4. + + ---++ Job Operations ---+++ Submitting a Workflow, Coordinator or Bundle Job http://git-wip-us.apache.org/repos/asf/oozie/blob/cb3e5653/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index e7bb24f..0890ae8 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.1.0 release (4.1 - unreleased) +OOZIE-1914 CLI should retry on timeout (puru) OOZIE-1973 ConcurrentModificationException in Sharelib service (puru) OOZIE-1896 ZKUUIDService - Too many job submission fails (puru) OOZIE-2005 Coordinator rerun fails to initialize error code and message (ryota)
