Repository: sqoop Updated Branches: refs/heads/trunk 98c5ccb80 -> 24d08185c
SQOOP-3061: Improve query validation in connection with --options-file thus correct options/queries like "--query SELECT * FROM test WHERE a = 'b'" will not fail with "Malformed option in options file" error message (Eric Lin via Attila Szabo) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/24d08185 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/24d08185 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/24d08185 Branch: refs/heads/trunk Commit: 24d08185c502e416265f1fa78300347ad02a4e1f Parents: 98c5ccb Author: Attila Szabo <[email protected]> Authored: Wed Feb 15 09:30:16 2017 +0100 Committer: Attila Szabo <[email protected]> Committed: Wed Feb 15 09:30:16 2017 +0100 ---------------------------------------------------------------------- .../org/apache/sqoop/util/OptionsFileUtil.java | 24 ++- .../sqoop/util/TestOptionsFileExpansion.java | 156 +++++++++++++++++++ 2 files changed, 173 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/24d08185/src/java/org/apache/sqoop/util/OptionsFileUtil.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/util/OptionsFileUtil.java b/src/java/org/apache/sqoop/util/OptionsFileUtil.java index c476e00..993ac1b 100644 --- a/src/java/org/apache/sqoop/util/OptionsFileUtil.java +++ b/src/java/org/apache/sqoop/util/OptionsFileUtil.java @@ -24,7 +24,9 @@ import java.io.FileReader; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.regex.Pattern; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -147,10 +149,10 @@ public final class OptionsFileUtil { * Removes the surrounding quote characters from the given string. The quotes * are identified by the quote parameter, the given string by option. The * fileName parameter is used for raising exceptions with relevant message. - * @param fileName - * @param option - * @param quote - * @return + * @param fileName String The name of the file that contains sqoop options + * @param option String the actual options in the options file + * @param quote char The quote that we need to validate on, either single or double quotes + * @return String The validated options string * @throws Exception */ private static String removeQuoteCharactersIfNecessary(String fileName, @@ -167,11 +169,19 @@ public final class OptionsFileUtil { } if (startingQuote || endingQuote) { - throw new Exception("Malformed option in options file(" - + fileName + "): " + option); + // Regular expression looks like below: + // .*=\s*".*"$ OR .*=\s*'.*'$ + // it tries to match the following: + // .... column_name = "values" OR .... column_name = 'values' + // so that the query like: + // SELECT * FROM table WHERE column = "values" + // is valid even though it ends with double quote but no starting double quote + if (!Pattern.matches(".*=\\s*"+quote+".*"+quote+"$", option)) { + throw new Exception("Malformed option in options file(" + + fileName + "): " + option); + } } return option; } - } http://git-wip-us.apache.org/repos/asf/sqoop/blob/24d08185/src/test/com/cloudera/sqoop/util/TestOptionsFileExpansion.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/util/TestOptionsFileExpansion.java b/src/test/com/cloudera/sqoop/util/TestOptionsFileExpansion.java index 6d3f0f3..d403f3b 100644 --- a/src/test/com/cloudera/sqoop/util/TestOptionsFileExpansion.java +++ b/src/test/com/cloudera/sqoop/util/TestOptionsFileExpansion.java @@ -165,6 +165,162 @@ public class TestOptionsFileExpansion { } } + @Test + public void testValidFreeFormQueryNoQuotes() throws Exception { + String[] input = new String[]{ + "--query", + "SELECT * FROM table", + }; + + String[] output = new String[] { + "--query", + "SELECT * FROM table", + }; + + checkOptionsFile(input, output); + } + + @Test + public void testValidFreeFormQuerySingleQuotesStartAndEnd() throws Exception { + String[] input = new String[]{ + "--query", + "'SELECT * FROM table'", + }; + + String[] output = new String[]{ + "--query", + "SELECT * FROM table", + }; + + checkOptionsFile(input, output); + } + + @Test + public void testValidFreeFormQueryDoubleQuotesStartAndEnd() throws Exception { + String[] input = new String[]{ + "--query", + "\"SELECT * FROM table\"", + }; + + String[] output = new String[]{ + "--query", + "SELECT * FROM table", + }; + + checkOptionsFile(input, output); + } + + @Test + public void testValidFreeFormQuerySingleQuotesInWhere() throws Exception { + String[] input = new String[]{ + "--query", + "SELECT * FROM table WHERE a = '1'", + }; + + String[] output = new String[]{ + "--query", + "SELECT * FROM table WHERE a = '1'", + }; + + checkOptionsFile(input, output); + } + + @Test + public void testValidFreeFormQuerySingleAndDoubleQuotesInWhere() throws Exception { + String[] input = new String[] { + "--query", + "SELECT * FROM table WHERE a = '1' AND b = \"testing\"", + }; + + String[] output = new String[] { + "--query", + "SELECT * FROM table WHERE a = '1' AND b = \"testing\"", + }; + + checkOptionsFile(input, output); + } + + @Test + public void testValidFreeFormQueryQuotesInTableNameAndColumnName() throws Exception { + String[] input = new String[] { + "--query", + "select * from `test\"test` where `c'c` = 'a'", + }; + + String[] output = new String[] { + "--query", + "select * from `test\"test` where `c'c` = 'a'", + }; + + checkOptionsFile(input, output); + } + + @Test + public void testValidFreeFormQueryQuotesInTableNameAndColumnName2() throws Exception { + String[] input = new String[] { + "--query", + "select * from `test\"test` where `c'c` = 'a\"'", + }; + + String[] output = new String[] { + "--query", + "select * from `test\"test` where `c'c` = 'a\"'", + }; + + checkOptionsFile(input, output); + } + + @Test + public void testValidFreeFormQueryQuotesInTableNameAndColumnName3() throws Exception { + String[] input = new String[] { + "--query", + "select * from `test\"test` where `c'c` = \"\"", + }; + + String[] output = new String[] { + "--query", + "select * from `test\"test` where `c'c` = \"\"", + }; + + checkOptionsFile(input, output); + } + + @Test + public void testValidFreeFormQueryQuotesInTableNameAndColumnName4() throws Exception { + String[] input = new String[] { + "--query", + "select * from test where a = \"\\\"\"", + }; + + String[] output = new String[] { + "--query", + "select * from test where a = \"\\\"\"", + }; + + checkOptionsFile(input, output); + } + + @Test + public void testInvalidFreeFormQueryEndingSingleQuoteOnly() throws Exception { + String[] input = new String[]{ + "--query", + "SELECT * FROM table'", + }; + + checkInvalidOptionsFile(input); + } + + @Test + public void testInvalidFreeFormQuerySingleQuoteStartDoubleQuoteEnd() throws Exception { + + String[] input = new String[]{ + "--query", + "'SELECT * FROM table\"", + }; + + checkInvalidOptionsFile(input); + } + private void checkInvalidOptionsFile(String[] fileContents) { try { checkOptionsFile(fileContents, new String[] {});
