Repository: sqoop Updated Branches: refs/heads/sqoop2 09a21649d -> 3668aea7d
SQOOP-2505: Sqoop2: Add ability to specify custom configuration properties for HDFS connector (Jarek Jarcec Cecho via Abraham Elmahrek) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/3668aea7 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/3668aea7 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/3668aea7 Branch: refs/heads/sqoop2 Commit: 3668aea7db65c63b539deea9518267890dd8c8bc Parents: 09a2164 Author: Abraham Elmahrek <[email protected]> Authored: Mon Aug 24 14:41:53 2015 -0700 Committer: Abraham Elmahrek <[email protected]> Committed: Mon Aug 24 14:41:53 2015 -0700 ---------------------------------------------------------------------- .../apache/sqoop/common/MutableMapContext.java | 4 ++ .../sqoop/common/TestMutableMapContext.java | 3 ++ .../connector/hdfs/HdfsFromInitializer.java | 1 + .../sqoop/connector/hdfs/HdfsToInitializer.java | 2 + .../hdfs/configuration/LinkConfig.java | 8 ++++ .../resources/hdfs-connector-config.properties | 3 ++ .../connector/hdfs/TestFromInitializer.java | 47 ++++++++++++++++++++ 7 files changed, 68 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/3668aea7/common/src/main/java/org/apache/sqoop/common/MutableMapContext.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/common/MutableMapContext.java b/common/src/main/java/org/apache/sqoop/common/MutableMapContext.java index 06ea7a9..7ff7269 100644 --- a/common/src/main/java/org/apache/sqoop/common/MutableMapContext.java +++ b/common/src/main/java/org/apache/sqoop/common/MutableMapContext.java @@ -64,6 +64,10 @@ public class MutableMapContext extends MapContext implements MutableContext { @Override public void setAll(Map<String, String> map) { + if(map == null) { + return; + } + getOptions().putAll(map); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/3668aea7/common/src/test/java/org/apache/sqoop/common/TestMutableMapContext.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/sqoop/common/TestMutableMapContext.java b/common/src/test/java/org/apache/sqoop/common/TestMutableMapContext.java index 9aab7f8..db7fa34 100644 --- a/common/src/test/java/org/apache/sqoop/common/TestMutableMapContext.java +++ b/common/src/test/java/org/apache/sqoop/common/TestMutableMapContext.java @@ -95,6 +95,9 @@ public class TestMutableMapContext { assertEquals(context.getLong("long", -1), 1L); assertEquals(context.getInt("integer", -1), 13); assertEquals(context.getBoolean("boolean", false), true); + + // Verify that we're resilient against null + context.setAll(null); } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/3668aea7/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsFromInitializer.java ---------------------------------------------------------------------- diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsFromInitializer.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsFromInitializer.java index 902549d..f5d9e1f 100644 --- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsFromInitializer.java +++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsFromInitializer.java @@ -38,5 +38,6 @@ public class HdfsFromInitializer extends Initializer<LinkConfiguration, FromJobC public void initialize(InitializerContext context, LinkConfiguration linkConfig, FromJobConfiguration jobConfig) { Configuration configuration = HdfsUtils.createConfiguration(linkConfig); HdfsUtils.configurationToContext(configuration, context.getContext()); + context.getContext().setAll(linkConfig.linkConfig.configOverrides); } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/3668aea7/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsToInitializer.java ---------------------------------------------------------------------- diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsToInitializer.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsToInitializer.java index 234bb71..5bb0928 100644 --- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsToInitializer.java +++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsToInitializer.java @@ -48,6 +48,8 @@ public class HdfsToInitializer extends Initializer<LinkConfiguration, ToJobConfi Configuration configuration = HdfsUtils.createConfiguration(linkConfig); HdfsUtils.configurationToContext(configuration, context.getContext()); + context.getContext().setAll(linkConfig.linkConfig.configOverrides); + boolean appendMode = Boolean.TRUE.equals(jobConfig.toJobConfig.appendMode); // Verification that given HDFS directory either don't exists or is empty http://git-wip-us.apache.org/repos/asf/sqoop/blob/3668aea7/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java ---------------------------------------------------------------------- diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java index b54ad15..39f3752 100644 --- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java +++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java @@ -24,6 +24,8 @@ import org.apache.sqoop.validation.Status; import org.apache.sqoop.validation.validators.AbstractValidator; import org.apache.sqoop.validation.validators.DirectoryExistsValidator; +import java.util.HashMap; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -34,6 +36,12 @@ public class LinkConfig { @Input(size = 255, validators = { @Validator(DirectoryExistsValidator.class)}) public String confDir; + @Input public Map<String, String> configOverrides; + + public LinkConfig() { + configOverrides = new HashMap<>(); + } + public static class ConfigValidator extends AbstractValidator<LinkConfig> { private static final Pattern URI_PATTERN = Pattern.compile("((?<=\\()[A-Za-z][A-Za-z0-9\\+\\.\\-]*:([A-Za-z0-9\\.\\-_~:/\\?#\\[\\]@!\\$&'\\(\\)\\*\\+,;=]|%[A-Fa-f0-9]{2})+(?=\\)))|([A-Za-z][A-Za-z0-9\\+\\.\\-]*:([A-Za-z0-9\\.\\-_~:/\\?#\\[\\]@!\\$&'\\(\\)\\*\\+,;=]|%[A-Fa-f0-9]{2})+)"); http://git-wip-us.apache.org/repos/asf/sqoop/blob/3668aea7/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties ---------------------------------------------------------------------- diff --git a/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties b/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties index 5feae54..db23a95 100644 --- a/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties +++ b/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties @@ -27,6 +27,9 @@ linkConfig.uri.help = HDFS URI used to connect to HDFS linkConfig.confDir.label = Hadoop conf directory linkConfig.confDir.help = Directory with Hadoop configuration files. The connector will load all -site.xml files. +linkConfig.configOverrides.label = Override configuration +linkConfig.configOverrides.help = Map of properties that that should be set for the Hadoop's configuration object on top of the files loaded from configuration directory. + # To Job Config # toJobConfig.label = To HDFS configuration http://git-wip-us.apache.org/repos/asf/sqoop/blob/3668aea7/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestFromInitializer.java ---------------------------------------------------------------------- diff --git a/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestFromInitializer.java b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestFromInitializer.java new file mode 100644 index 0000000..5215901 --- /dev/null +++ b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestFromInitializer.java @@ -0,0 +1,47 @@ +/** + * Created by jarcec on 8/18/15. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.connector.hdfs; + +import org.apache.sqoop.common.MutableMapContext; +import org.apache.sqoop.connector.hdfs.configuration.FromJobConfiguration; +import org.apache.sqoop.connector.hdfs.configuration.LinkConfiguration; +import org.apache.sqoop.job.etl.Initializer; +import org.apache.sqoop.job.etl.InitializerContext; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; + +public class TestFromInitializer { + + @Test + public void testConfigOverrides() { + LinkConfiguration linkConfig = new LinkConfiguration(); + FromJobConfiguration jobConfig = new FromJobConfiguration(); + + linkConfig.linkConfig.uri = "file:///"; + linkConfig.linkConfig.configOverrides.put("key", "value"); + + InitializerContext initializerContext = new InitializerContext(new MutableMapContext()); + + Initializer initializer = new HdfsFromInitializer(); + initializer.initialize(initializerContext, linkConfig, jobConfig); + + assertEquals(initializerContext.getString("key"), "value"); + } +}
