Repository: sqoop Updated Branches: refs/heads/trunk dc2568431 -> 660f3e8ad
SQOOP-2910: Add capability to Sqoop to require an explicit option to be specified with --split-by for a String column (Attila Szabo via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/660f3e8a Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/660f3e8a Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/660f3e8a Branch: refs/heads/trunk Commit: 660f3e8ad07758aabf0a9b6ede3accdfac5fb1be Parents: dc25684 Author: Jarek Jarcec Cecho <[email protected]> Authored: Tue Apr 19 15:54:58 2016 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Tue Apr 19 15:54:58 2016 -0700 ---------------------------------------------------------------------- .../apache/sqoop/mapreduce/db/TextSplitter.java | 6 ++ .../TextSplitterHadoopConfIntegrationTest.java | 59 ++++++++++++++++++++ 2 files changed, 65 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/660f3e8a/src/java/org/apache/sqoop/mapreduce/db/TextSplitter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/db/TextSplitter.java b/src/java/org/apache/sqoop/mapreduce/db/TextSplitter.java index 9896d95..8c98d14 100644 --- a/src/java/org/apache/sqoop/mapreduce/db/TextSplitter.java +++ b/src/java/org/apache/sqoop/mapreduce/db/TextSplitter.java @@ -38,6 +38,8 @@ import org.apache.sqoop.validation.ValidationException; */ public class TextSplitter extends BigDecimalSplitter { + public static final String ALLOW_TEXT_SPLITTER_PROPERTY = "org.apache.sqoop.splitter.allow_text_splitter"; + private static final Log LOG = LogFactory.getLog(TextSplitter.class); private boolean useNCharStrings = false; @@ -61,6 +63,10 @@ public class TextSplitter extends BigDecimalSplitter { */ public List<InputSplit> split(Configuration conf, ResultSet results, String colName) throws SQLException, ValidationException { + if (!conf.getBoolean(ALLOW_TEXT_SPLITTER_PROPERTY, false)) { + throw new ValidationException("Generating splits for a textual index column " + "allowed only in case of \"-D" + + ALLOW_TEXT_SPLITTER_PROPERTY + "=true\" property " + "passed as a parameter"); + } LOG.warn("Generating splits for a textual index column."); LOG.warn("If your database sorts in a case-insensitive order, " http://git-wip-us.apache.org/repos/asf/sqoop/blob/660f3e8a/src/test/org/apache/sqoop/mapreduce/db/TextSplitterHadoopConfIntegrationTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/mapreduce/db/TextSplitterHadoopConfIntegrationTest.java b/src/test/org/apache/sqoop/mapreduce/db/TextSplitterHadoopConfIntegrationTest.java new file mode 100644 index 0000000..043130e --- /dev/null +++ b/src/test/org/apache/sqoop/mapreduce/db/TextSplitterHadoopConfIntegrationTest.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.mapreduce.db; + +import java.sql.ResultSet; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.Job; +import org.apache.sqoop.validation.ValidationException; + +import com.cloudera.sqoop.Sqoop; +import com.cloudera.sqoop.testutil.MockResultSet; + +import junit.framework.TestCase; + +public class TextSplitterHadoopConfIntegrationTest extends TestCase { + private static final String TEXT_COL_NAME = "text_col_name"; + + public void testDefaultValueOfUnsetBooleanParam() throws Exception { + Configuration conf = Job.getInstance().getConfiguration(); + TextSplitter splitter = new TextSplitter(); + ResultSet rs = new MockResultSet(); + try { + splitter.split(conf, rs, TEXT_COL_NAME); + fail(); + } catch (ValidationException e) { + // expected to throw ValidationException with the a message about the + // "i-know-what-i-am-doing" prop + assertTrue(e.getMessage().contains(TextSplitter.ALLOW_TEXT_SPLITTER_PROPERTY)); + } + } + + public void testBooleanParamValue() throws Exception { + Configuration conf = Job.getInstance().getConfiguration(); + conf.set(TextSplitter.ALLOW_TEXT_SPLITTER_PROPERTY, "true"); + TextSplitter splitter = new TextSplitter(); + ResultSet rs = new MockResultSet(); + List<InputSplit> splits = splitter.split(conf, rs, TEXT_COL_NAME); + assertFalse(splits.isEmpty()); + } +} +
