Repository: sqoop Updated Branches: refs/heads/sqoop2 700d55ac7 -> 598460f4a
SQOOP-2230: Sqoop2: Kite connector can use URIBuilder (Abraham Elmahrek via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/598460f4 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/598460f4 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/598460f4 Branch: refs/heads/sqoop2 Commit: 598460f4a5f81cd2dded2b2d3c99af99934de50f Parents: 700d55a Author: Jarek Jarcec Cecho <[email protected]> Authored: Wed Mar 18 18:35:56 2015 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Wed Mar 18 18:35:56 2015 -0700 ---------------------------------------------------------------------- .../connector/kite/KiteDatasetExecutor.java | 21 +++++++++++++++++++- .../kite/configuration/ConfigUtil.java | 16 ++++++++++++--- .../sqoop/connector/kite/TestKiteExecutor.java | 13 ++++++++++++ 3 files changed, 46 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/598460f4/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java ---------------------------------------------------------------------- diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java index b111b5c..883925b 100644 --- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java +++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java @@ -21,6 +21,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.io.Closeables; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; +import org.apache.commons.lang.StringUtils; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.connector.common.FileFormat; import org.apache.sqoop.connector.common.AvroDataTypeUtil; @@ -162,7 +163,25 @@ public class KiteDatasetExecutor { */ public static String suggestTemporaryDatasetUri(String uri) { if (uri.startsWith("dataset:hdfs:")) { - return uri + TEMPORARY_DATASET_PREFIX + UUID.randomUUID().toString().replace("-", ""); + int pathStart = uri.lastIndexOf(":") + 1; + int pathEnd = uri.lastIndexOf("?"); + String[] uriParts = null; + + // Get URI parts + if (pathEnd > -1) { + uriParts = new String[3]; + uriParts[2] = uri.substring(pathEnd, uri.length()); + } else { + pathEnd = uri.length(); + uriParts = new String[2]; + } + uriParts[1] = uri.substring(pathStart, pathEnd); + uriParts[0] = uri.substring(0, pathStart); + + // Add to path + uriParts[1] += TEMPORARY_DATASET_PREFIX + UUID.randomUUID().toString().replace("-", ""); + + return StringUtils.join(uriParts, ""); } else { throw new SqoopException( KiteConnectorError.GENERIC_KITE_CONNECTOR_0000, uri); http://git-wip-us.apache.org/repos/asf/sqoop/blob/598460f4/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/ConfigUtil.java ---------------------------------------------------------------------- diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/ConfigUtil.java b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/ConfigUtil.java index d48e250..1df1859 100644 --- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/ConfigUtil.java +++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/ConfigUtil.java @@ -18,6 +18,7 @@ package org.apache.sqoop.connector.kite.configuration; import com.google.common.base.Strings; +import org.kitesdk.data.URIBuilder; public class ConfigUtil { @@ -27,10 +28,19 @@ public class ConfigUtil { */ public static String buildDatasetUri(String fsLocation, String uri) { if (!Strings.isNullOrEmpty(fsLocation) && !uri.contains("://")) { - // Add fsLocation after the second colon - int p = uri.indexOf(":", uri.indexOf(":") + 1); - return uri.substring(0, p + 1) + "//" + fsLocation + uri.substring(p + 1); + URIBuilder builder = new URIBuilder(uri); + + String[] parts = fsLocation.split(":"); + if (parts.length > 0) { + builder.with("auth:host", parts[0]); + } + if (parts.length > 1) { + builder.with("auth:port", parts[1]); + } + + return builder.build().toString().replaceFirst("view:", "dataset:"); } + return uri; } http://git-wip-us.apache.org/repos/asf/sqoop/blob/598460f4/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java ---------------------------------------------------------------------- diff --git a/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java b/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java index eed6ee0..7173c18 100644 --- a/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java +++ b/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java @@ -147,6 +147,19 @@ public class TestKiteExecutor { assertTrue(executor.isReaderClosed()); } + @Test + public void testSuggestTemporaryDatasetUri() { + String uri = "dataset:hdfs:/tmp/sqoop/test"; + String suggestedUri = KiteDatasetExecutor.suggestTemporaryDatasetUri(uri); + assertTrue(suggestedUri.length() > uri.length()); + assertTrue(suggestedUri.contains(uri)); + + uri = "dataset:hdfs://namenode:8020/tmp/sqoop/test"; + suggestedUri = KiteDatasetExecutor.suggestTemporaryDatasetUri(uri); + assertTrue(suggestedUri.length() > uri.length()); + assertTrue(suggestedUri.contains(uri)); + } + private static Schema createTwoFieldSchema() { return new Schema.Parser().parse("{" + "\"name\":\"test\",\"type\":\"record\"," +
