Repository: sqoop Updated Branches: refs/heads/sqoop2 58b2d1c1c -> 094c9c241
SQOOP-2491: Sqoop2: Common external JAR handling in configuration (Abraham Elmahrek via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/094c9c24 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/094c9c24 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/094c9c24 Branch: refs/heads/sqoop2 Commit: 094c9c241f728da8ee4d034c54bd0ed22b57ca38 Parents: 58b2d1c Author: Jarek Jarcec Cecho <[email protected]> Authored: Mon Aug 17 15:21:45 2015 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Mon Aug 17 15:21:45 2015 -0700 ---------------------------------------------------------------------- .../org/apache/sqoop/utils/ContextUtils.java | 81 ++++++++++ .../apache/sqoop/utils/TestContextUtils.java | 147 +++++++++++++++++++ .../apache/sqoop/core/SqoopConfiguration.java | 14 +- 3 files changed, 235 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/094c9c24/common/src/main/java/org/apache/sqoop/utils/ContextUtils.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/utils/ContextUtils.java b/common/src/main/java/org/apache/sqoop/utils/ContextUtils.java new file mode 100644 index 0000000..25d6fcb --- /dev/null +++ b/common/src/main/java/org/apache/sqoop/utils/ContextUtils.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.utils; + +import org.apache.commons.lang.StringUtils; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +/** + * Utils to be used ubiquitously for contexts. + */ +public class ContextUtils { + /** + * Return an array of strings for given value or null by default. + * + * @param value Non-empty value + * @return Array of values for given key or null in case of unknown key + */ + public static String[] getArrayOfStrings(String value) { + return getArrayOfStrings(value, ":"); + } + + /** + * Return an array of strings for given value or default value. + * + * @param value Non-empty value + * @param separator String separator + * @return Array of values for given key or default value in case of unknown key + */ + public static String[] getArrayOfStrings(String value, String separator) { + assert !StringUtils.isBlank(value); + + return value.split(separator); + } + + private static Set<String> getUniqueStrings(String[] strings) { + if (strings == null) { + return new HashSet<String>(); + } + + return new HashSet(Arrays.asList(strings)); + } + + /** + * Return an array of unique strings for given value or null by default. + * + * @param value Non-empty value + * @return Set of unique strings for given value or null in case of unknown key + */ + public static Set<String> getUniqueStrings(String value) { + return getUniqueStrings(getArrayOfStrings(value)); + } + + /** + * Return an array of unique strings for given value or default value. + * + * @param value Non-empty value + * @param separator String separator + * @return Set of unique strings for given value or default value in case of unknown key + */ + public static Set<String> getUniqueStrings(String value, String separator) { + return getUniqueStrings(getArrayOfStrings(value, separator)); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/094c9c24/common/src/test/java/org/apache/sqoop/utils/TestContextUtils.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/sqoop/utils/TestContextUtils.java b/common/src/test/java/org/apache/sqoop/utils/TestContextUtils.java new file mode 100644 index 0000000..779fc78 --- /dev/null +++ b/common/src/test/java/org/apache/sqoop/utils/TestContextUtils.java @@ -0,0 +1,147 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.utils; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; + +import org.apache.sqoop.common.MapContext; +import org.testng.annotations.Test; + +import static org.apache.sqoop.utils.ContextUtils.getArrayOfStrings; +import static org.apache.sqoop.utils.ContextUtils.getUniqueStrings; +import static org.testng.Assert.assertEquals; + +/** + * Test class for org.apache.sqoop.utils.ContextUtils + */ +public class TestContextUtils { + @Test + public void testGetArrayOfStrings() { + final String DEFAULT_SEPARATOR = ","; + + Map<String, String> options1 = new HashMap<String, String>(); + options1.put("sqooptest1", "value1:value2"); + options1.put("sqooptest2", "value1"); + + Map<String, String> options2 = new HashMap<String, String>(); + options2.put("sqooptest1", "value1,value2"); + options2.put("sqooptest2", "value1"); + + MapContext mc1 = new MapContext(options1); + MapContext mc2 = new MapContext(options2); + + assertEquals(getArrayOfStrings(mc1.getString("sqooptest1")), new String[]{"value1", "value2"}); + assertEquals(getArrayOfStrings(mc1.getString("sqooptest2")), new String[]{"value1"}); + + assertEquals(getArrayOfStrings(mc2.getString("sqooptest1"), DEFAULT_SEPARATOR), new String[]{"value1", "value2"}); + assertEquals(getArrayOfStrings(mc2.getString("sqooptest2"), DEFAULT_SEPARATOR), new String[]{"value1"}); + } + + @Test(expectedExceptions = {AssertionError.class}) + public void testGetArrayOfStringsNullValue() { + Map<String, String> options1 = new HashMap<String, String>(); + options1.put("sqooptest1", null); + MapContext mc1 = new MapContext(options1); + getArrayOfStrings(mc1.getString("sqooptest1")); + } + + @Test(expectedExceptions = {AssertionError.class}) + public void testGetArrayOfStringsWithSeparatorNullValue() { + Map<String, String> options1 = new HashMap<String, String>(); + options1.put("sqooptest1", null); + MapContext mc1 = new MapContext(options1); + getArrayOfStrings(mc1.getString("sqooptest1"), ","); + } + + @Test(expectedExceptions = {AssertionError.class}) + public void testGetArrayOfStringsEmptyValue() { + Map<String, String> options1 = new HashMap<String, String>(); + options1.put("sqooptest1", ""); + MapContext mc1 = new MapContext(options1); + getArrayOfStrings(mc1.getString("sqooptest1")); + } + + @Test(expectedExceptions = {AssertionError.class}) + public void testGetArrayOfStringsWithSeparatorEmptyValue() { + Map<String, String> options1 = new HashMap<String, String>(); + options1.put("sqooptest1", ""); + MapContext mc1 = new MapContext(options1); + getArrayOfStrings(mc1.getString("sqooptest1"), ","); + } + + @Test + public void testGetUniqueStrings() { + final String DEFAULT_SEPARATOR = ","; + + Map<String, String> options1 = new HashMap<String, String>(); + options1.put("sqooptest1", "value1:value2"); + options1.put("sqooptest2", "value1"); + options1.put("sqooptest3", "value1:value2:value1"); + + Map<String, String> options2 = new HashMap<String, String>(); + options2.put("sqooptest1", "value1,value2"); + options2.put("sqooptest2", "value1"); + options2.put("sqooptest3", "value1,value2,value1"); + + MapContext mc1 = new MapContext(options1); + MapContext mc2 = new MapContext(options2); + + assertEquals(getUniqueStrings(mc1.getString("sqooptest1")), new HashSet<String>(Arrays.asList(new String[]{"value1", "value2"}))); + assertEquals(getUniqueStrings(mc1.getString("sqooptest2")), new HashSet<String>(Arrays.asList(new String[]{"value1"}))); + assertEquals(getUniqueStrings(mc1.getString("sqooptest3")), new HashSet<String>(Arrays.asList(new String[]{"value1", "value2"}))); + + assertEquals(getUniqueStrings(mc2.getString("sqooptest1"), DEFAULT_SEPARATOR), new HashSet<String>(Arrays.asList(new String[]{"value1", "value2"}))); + assertEquals(getUniqueStrings(mc2.getString("sqooptest2"), DEFAULT_SEPARATOR), new HashSet<String>(Arrays.asList(new String[]{"value1"}))); + assertEquals(getUniqueStrings(mc2.getString("sqooptest3"), DEFAULT_SEPARATOR), new HashSet<String>(Arrays.asList(new String[]{"value1", "value2"}))); + } + + @Test(expectedExceptions = {AssertionError.class}) + public void testGetUniqueStringsNullValue() { + Map<String, String> options1 = new HashMap<String, String>(); + options1.put("sqooptest1", null); + MapContext mc1 = new MapContext(options1); + getUniqueStrings(mc1.getString("sqooptest1")); + } + + @Test(expectedExceptions = {AssertionError.class}) + public void testGetUniqueStringsWithSeparatorNullValue() { + Map<String, String> options1 = new HashMap<String, String>(); + options1.put("sqooptest1", null); + MapContext mc1 = new MapContext(options1); + getUniqueStrings(mc1.getString("sqooptest1"), ","); + } + + @Test(expectedExceptions = {AssertionError.class}) + public void testGetUniqueStringsEmptyValue() { + Map<String, String> options1 = new HashMap<String, String>(); + options1.put("sqooptest1", ""); + MapContext mc1 = new MapContext(options1); + getUniqueStrings(mc1.getString("sqooptest1")); + } + + @Test(expectedExceptions = {AssertionError.class}) + public void testGetUniqueStringsWithSeparatorEmptyValue() { + Map<String, String> options1 = new HashMap<String, String>(); + options1.put("sqooptest1", ""); + MapContext mc1 = new MapContext(options1); + getUniqueStrings(mc1.getString("sqooptest1"), ","); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/094c9c24/core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java b/core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java index 58c4c34..49f139c 100644 --- a/core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java +++ b/core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java @@ -24,8 +24,6 @@ import java.io.InputStream; import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; -import java.util.Arrays; -import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -41,6 +39,8 @@ import org.apache.sqoop.common.MapContext; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.error.code.CoreError; +import static org.apache.sqoop.utils.ContextUtils.getUniqueStrings; + /** * Configuration manager that loads Sqoop configuration. */ @@ -230,9 +230,10 @@ public class SqoopConfiguration implements Reconfigurable { private synchronized void configureClassLoader(String classpathProperty) { LOG.info("Adding jars to current classloader from property: " + classpathProperty); - String classpath = getContext().getString(classpathProperty); + // CSV URL list separated by ":". + String paths = getContext().getString(classpathProperty); - if (StringUtils.isEmpty(classpath)) { + if (StringUtils.isEmpty(paths)) { LOG.debug("Property " + classpathProperty + " is null or empty. Not adding any extra jars."); return; } @@ -242,11 +243,10 @@ public class SqoopConfiguration implements Reconfigurable { throw new SqoopException(CoreError.CORE_0009, "No thread context classloader to override."); } - // CSV URL list separated by ":". - Set<String> paths = new HashSet(Arrays.asList(classpath.split(":"))); + Set<String> pathSet = getUniqueStrings(paths); List<URL> urls = new LinkedList<URL>(); - for (String path : paths) { + for (String path : pathSet) { try { LOG.debug("Found jar in path: " + path); URL url = new File(path).toURI().toURL();
