Repository: sqoop Updated Branches: refs/heads/sqoop2 9fea04e6f -> 07eb1f289
SQOOP-1531: Sqoop2: Support getting connectors by Direction (Erzsebet Szilagyi via Abraham Fine) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/07eb1f28 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/07eb1f28 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/07eb1f28 Branch: refs/heads/sqoop2 Commit: 07eb1f289d4a2a17338ca03fd5aa9541ac13f726 Parents: 9fea04e Author: Abraham Fine <[email protected]> Authored: Thu May 26 23:07:37 2016 -0700 Committer: Abraham Fine <[email protected]> Committed: Thu May 26 23:07:37 2016 -0700 ---------------------------------------------------------------------- .../org/apache/sqoop/client/SqoopClient.java | 24 +++++++++ .../apache/sqoop/client/TestSqoopClient.java | 34 +++++++++++++ .../sqoop/shell/ShowConnectorFunction.java | 32 ++++++++++++ .../org/apache/sqoop/shell/core/Constants.java | 4 ++ .../main/resources/shell-resource.properties | 1 + .../org/apache/sqoop/shell/TestShowCommand.java | 51 ++++++++++++++++++++ 6 files changed, 146 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/07eb1f28/client/src/main/java/org/apache/sqoop/client/SqoopClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/sqoop/client/SqoopClient.java b/client/src/main/java/org/apache/sqoop/client/SqoopClient.java index 1cf549e..549e6c1 100644 --- a/client/src/main/java/org/apache/sqoop/client/SqoopClient.java +++ b/client/src/main/java/org/apache/sqoop/client/SqoopClient.java @@ -23,12 +23,14 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.ResourceBundle; +import java.util.ArrayList; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; import org.apache.sqoop.classification.InterfaceAudience; import org.apache.sqoop.classification.InterfaceStability; import org.apache.sqoop.client.request.SqoopResourceRequests; +import org.apache.sqoop.common.Direction; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.json.*; import org.apache.sqoop.model.*; @@ -202,6 +204,28 @@ public class SqoopClient { } /** + * Get list of all connectors in one direction. + * + * @param direction the required directions of the connectors to be listed. + * @return Collection of connector models. + */ + public Collection<MConnector> getConnectorsByDirection(Direction direction) { + if (!isAllConnectors) { + getConnectors(); + } + + Collection<MConnector> filteredConnectors = new ArrayList<>(); + + for (MConnector connector : connectors.values()) { + if (connector.getSupportedDirections().isDirectionSupported(direction)) { + filteredConnectors.add(connector); + } + } + + return filteredConnectors; + } + + /** * Get resource bundle for given connector. * * @param connectorName Connector name. http://git-wip-us.apache.org/repos/asf/sqoop/blob/07eb1f28/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java ---------------------------------------------------------------------- diff --git a/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java b/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java index adf4eb5..16a4588 100644 --- a/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java +++ b/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java @@ -19,6 +19,7 @@ package org.apache.sqoop.client; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -34,6 +35,7 @@ import java.util.Map; import java.util.ResourceBundle; import org.apache.sqoop.client.request.SqoopResourceRequests; +import org.apache.sqoop.common.Direction; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.json.ConnectorBean; import org.apache.sqoop.json.DriverBean; @@ -186,6 +188,38 @@ public class TestSqoopClient { verifyNoMoreInteractions(resourceRequests); } + @Test + public void testGetConnectorsByDirection() { + Collection<MConnector> filteredConnectors; + + MConnector connectorFrom = new MConnector("from_connector", "", "", + new MLinkConfig(null, null), + new MFromConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()), + null); + + MConnector connectorTo = new MConnector("to_connector", "", "", + new MLinkConfig(null, null), + null, + new MToConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>())); + + MConnector connectorBothWays = new MConnector("both_ways_connector", "", "", + new MLinkConfig(null, null), + new MFromConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()), + new MToConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>())); + + when(resourceRequests.readConnector(null)).thenReturn(connectorBean(connectorFrom, connectorTo, connectorBothWays)); + + filteredConnectors = client.getConnectorsByDirection(Direction.FROM); + assertTrue(filteredConnectors.contains(connectorFrom)); + assertTrue(!filteredConnectors.contains(connectorTo)); + assertTrue(filteredConnectors.contains(connectorBothWays)); + + filteredConnectors = client.getConnectorsByDirection(Direction.TO); + assertTrue(!filteredConnectors.contains(connectorFrom)); + assertTrue(filteredConnectors.contains(connectorTo)); + assertTrue(filteredConnectors.contains(connectorBothWays)); + } + /** * Link for non-existing connector can't be created. */ http://git-wip-us.apache.org/repos/asf/sqoop/blob/07eb1f28/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java index c90fe94..772e9eb 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java +++ b/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java @@ -24,9 +24,11 @@ import java.util.List; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.OptionBuilder; import org.apache.sqoop.client.ClientError; +import org.apache.sqoop.common.Direction; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.model.MConnector; import org.apache.sqoop.shell.core.Constants; +import org.apache.sqoop.shell.core.ShellError; import org.apache.sqoop.validation.Status; import static org.apache.sqoop.shell.ShellEnvironment.*; @@ -46,6 +48,10 @@ public class ShowConnectorFunction extends SqoopFunction { .withDescription(resourceString(Constants.RES_SHOW_PROMPT_DISPLAY_CONNECTOR_NAME)) .withLongOpt(Constants.OPT_NAME) .create(Constants.OPT_NAME_CHAR)); + this.addOption(OptionBuilder.hasArg().withArgName(Constants.OPT_DIRECTION) + .withDescription(resourceString(Constants.RES_SHOW_PROMPT_DISPLAY_CONNECTOR_DIRECTION)) + .withLongOpt(Constants.OPT_DIRECTION) + .create(Constants.OPT_DIRECTION_CHAR)); } @Override @@ -54,6 +60,8 @@ public class ShowConnectorFunction extends SqoopFunction { showConnectors(); } else if (line.hasOption(Constants.OPT_NAME)) { showConnector(line); + } else if (line.hasOption(Constants.OPT_DIRECTION)) { + showConnectorsByDirection(line); } else { showSummary(); } @@ -95,6 +103,30 @@ public class ShowConnectorFunction extends SqoopFunction { } } + private void showConnectorsByDirection(CommandLine line) { + // Check if the command argument is a connector direction + String directionString = line.getOptionValue(Constants.OPT_DIRECTION); + Direction direction; + switch (directionString) { + case Constants.OPT_FROM: + direction = Direction.FROM; + break; + case Constants.OPT_TO: + direction = Direction.TO; + break; + default: + throw new SqoopException(ShellError.SHELL_0003, directionString); + } + + Collection<MConnector> connectors = client.getConnectorsByDirection(direction); + + printlnResource(Constants.RES_SHOW_PROMPT_CONNECTORS_TO_SHOW, connectors.size()); + + for (MConnector connector : connectors) { + displayConnector(connector); + } + } + private void showConnector(CommandLine line) { //Check if the command argument is a connector name String connectorName = line.getOptionValue(Constants.OPT_NAME); http://git-wip-us.apache.org/repos/asf/sqoop/blob/07eb1f28/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java b/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java index a5f5d4d..068352e 100644 --- a/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java +++ b/shell/src/main/java/org/apache/sqoop/shell/core/Constants.java @@ -35,6 +35,7 @@ public class Constants { // Options + public static final String OPT_DIRECTION = "direction"; public static final String OPT_FROM = "from"; public static final String OPT_TO = "to"; public static final String OPT_ALL = "all"; @@ -65,6 +66,7 @@ public class Constants { public static final String OPT_TRUSTSTORE_PASSWORD = "truststore-password"; public static final String OPT_TRUSTSTORE_PASSWORD_GENERATOR = "truststore-password-generator"; + public static final char OPT_DIRECTION_CHAR = 'd'; public static final char OPT_FROM_CHAR = 'f'; public static final char OPT_TO_CHAR = 't'; public static final char OPT_ALL_CHAR = 'a'; @@ -293,6 +295,8 @@ public class Constants { "show.prompt_display_all_connectors"; public static final String RES_SHOW_PROMPT_DISPLAY_CONNECTOR_NAME = "show.prompt_display_connector_name"; + public static final String RES_SHOW_PROMPT_DISPLAY_CONNECTOR_DIRECTION = + "show.prompt_display_connectors_by_direction"; public static final String RES_SHOW_PROMPT_CONNECTORS_TO_SHOW = "show.prompt_connectors_to_show"; public static final String RES_SHOW_PROMPT_CONNECTOR_INFO = http://git-wip-us.apache.org/repos/asf/sqoop/blob/07eb1f28/shell/src/main/resources/shell-resource.properties ---------------------------------------------------------------------- diff --git a/shell/src/main/resources/shell-resource.properties b/shell/src/main/resources/shell-resource.properties index 2c94c58..d75ea51 100644 --- a/shell/src/main/resources/shell-resource.properties +++ b/shell/src/main/resources/shell-resource.properties @@ -142,6 +142,7 @@ show.prompt_link_cn_info = Using Connector @|bold {0}|@ with name @|bold {1}|@ show.prompt_display_all_connectors = Display all connectors show.prompt_display_connector_name = Display the connector with name +show.prompt_display_connectors_by_direction = Display all connectors that support the given direction show.connector_usage = Usage: show connector show.prompt_connectors_to_show = @|bold {0} connector(s) to show: |@ show.prompt_connector_info = Connector with Name: {0} \n \ http://git-wip-us.apache.org/repos/asf/sqoop/blob/07eb1f28/shell/src/test/java/org/apache/sqoop/shell/TestShowCommand.java ---------------------------------------------------------------------- diff --git a/shell/src/test/java/org/apache/sqoop/shell/TestShowCommand.java b/shell/src/test/java/org/apache/sqoop/shell/TestShowCommand.java index 49affa3..a9d9be7 100644 --- a/shell/src/test/java/org/apache/sqoop/shell/TestShowCommand.java +++ b/shell/src/test/java/org/apache/sqoop/shell/TestShowCommand.java @@ -30,9 +30,11 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import org.apache.sqoop.client.SqoopClient; +import org.apache.sqoop.common.Direction; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.json.VersionBean; import org.apache.sqoop.model.MConfig; @@ -187,6 +189,55 @@ public class TestShowCommand { assertTrue(str.contains("Connector with Name: test_connector")); } + @Test + public void testShowConnectorsByDirection() { + Collection<MConnector> connectorsFrom = new ArrayList<MConnector>(); + connectorsFrom.add(new MConnector("from_connector", "", "", + new MLinkConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()), + new MFromConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()), + null)); + when(client.getConnectorsByDirection(Direction.FROM)).thenReturn(connectorsFrom); + + Collection<MConnector> connectorsTo = new ArrayList<MConnector>(); + connectorsTo.add(new MConnector("to_connector", "", "", + new MLinkConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()), + null, + new MToConfig(new ArrayList<MConfig>(), new ArrayList<MValidator>()))); + when(client.getConnectorsByDirection(Direction.TO)).thenReturn(connectorsTo); + + // show connector -direction from + out.reset(); + Status status = (Status) showCmd.execute(Arrays.asList(Constants.FN_CONNECTOR, "-direction", "from")); + assertTrue(status != null && status == Status.OK); + String str = new String(out.toByteArray()); + assertTrue(str.contains("Connector with Name: from_connector")); + + // show connector -direction to + out.reset(); + status = (Status) showCmd.execute(Arrays.asList(Constants.FN_CONNECTOR, "-direction", "to")); + assertTrue(status != null && status == Status.OK); + str = new String(out.toByteArray()); + assertTrue(str.contains("Connector with Name: to_connector")); + + // show connector -direction + try { + out.reset(); + showCmd.execute(Arrays.asList(Constants.FN_CONNECTOR, "-direction")); + fail("Show connector should fail as option direction is missing!"); + } catch (SqoopException e) { + assertEquals(e.getErrorCode(), ShellError.SHELL_0003); + } + + // show connector -direction misc + try { + out.reset(); + showCmd.execute(Arrays.asList(Constants.FN_CONNECTOR, "-direction", "misc")); + fail("Show connector should fail as option direction is invalid!"); + } catch (SqoopException e) { + assertEquals(e.getErrorCode(), ShellError.SHELL_0003); + } + } + @SuppressWarnings({ "rawtypes", "unchecked" }) @Test public void testShowDriver() {
