This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch 4.x
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x by this push:
new 43b56d4 PHOENIX-3499 Enable null value for quote character for
CSVBulkLoad tool
43b56d4 is described below
commit 43b56d40e9d38a4120865be4bac098236fb5fe26
Author: Sergey Soldatov <s...@apache.org>
AuthorDate: Fri Nov 18 03:24:00 2016 -0800
PHOENIX-3499 Enable null value for quote character for CSVBulkLoad tool
---
.../apache/phoenix/end2end/CsvBulkLoadToolIT.java | 37 +++++++++++++++++++++-
.../phoenix/mapreduce/CsvBulkImportUtil.java | 10 +++---
.../apache/phoenix/mapreduce/CsvBulkLoadTool.java | 26 +++++++++------
.../phoenix/mapreduce/CsvToKeyValueMapper.java | 2 +-
4 files changed, 60 insertions(+), 15 deletions(-)
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java
index 6f10d71..2de2c82 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java
@@ -139,7 +139,6 @@ public class CsvBulkLoadToolIT extends BaseOwnClusterIT {
stmt.close();
}
-
@Test
public void testImportWithTabs() throws Exception {
@@ -173,6 +172,42 @@ public class CsvBulkLoadToolIT extends BaseOwnClusterIT {
}
@Test
+ public void testImportWithTabsAndEmptyQuotes() throws Exception {
+
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE TABLE TABLE8 (ID INTEGER NOT NULL PRIMARY KEY, " +
+ "NAME1 VARCHAR, NAME2 VARCHAR)");
+
+ FileSystem fs = FileSystem.get(getUtility().getConfiguration());
+ FSDataOutputStream outputStream = fs.create(new
Path("/tmp/input8.csv"));
+ PrintWriter printWriter = new PrintWriter(outputStream);
+ printWriter.println("1\t\"\\t123\tName 2a");
+ printWriter.println("2\tName 2a\tName 2b");
+ printWriter.close();
+
+ CsvBulkLoadTool csvBulkLoadTool = new CsvBulkLoadTool();
+ csvBulkLoadTool.setConf(getUtility().getConfiguration());
+ int exitCode = csvBulkLoadTool.run(new String[] {
+ "--input", "/tmp/input8.csv",
+ "--table", "table8",
+ "--zookeeper", zkQuorum,
+ "-q", "\"\"",
+ "-e", "\"\"",
+ "--delimiter", "\\t"
+ });
+ assertEquals(0, exitCode);
+
+ ResultSet rs = stmt.executeQuery("SELECT id, name1, name2 FROM table8
ORDER BY id");
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+ assertEquals("\"\\t123", rs.getString(2));
+ assertEquals("Name 2a", rs.getString(3));
+
+ rs.close();
+ stmt.close();
+ }
+
+ @Test
public void testFullOptionImport() throws Exception {
Statement stmt = conn.createStatement();
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkImportUtil.java
b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkImportUtil.java
index ff9ff72..4ea2d21 100644
---
a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkImportUtil.java
+++
b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkImportUtil.java
@@ -41,8 +41,8 @@ public class CsvBulkImportUtil {
* @param arrayDelimiter array delimiter character, can be null
* @param binaryEncoding
*/
- public static void initCsvImportJob(Configuration conf, char
fieldDelimiter, char quoteChar,
- char escapeChar, String arrayDelimiter, String binaryEncoding) {
+ public static void initCsvImportJob(Configuration conf, char
fieldDelimiter, Character quoteChar,
+ Character escapeChar, String arrayDelimiter, String
binaryEncoding) {
setChar(conf, CsvToKeyValueMapper.FIELD_DELIMITER_CONFKEY,
fieldDelimiter);
setChar(conf, CsvToKeyValueMapper.QUOTE_CHAR_CONFKEY, quoteChar);
setChar(conf, CsvToKeyValueMapper.ESCAPE_CHAR_CONFKEY, escapeChar);
@@ -67,8 +67,10 @@ public class CsvBulkImportUtil {
}
@VisibleForTesting
- static void setChar(Configuration conf, String confKey, char charValue) {
- conf.set(confKey,
Base64.encodeBytes(Character.toString(charValue).getBytes()));
+ static void setChar(Configuration conf, String confKey, Character
charValue) {
+ if(charValue!=null) {
+ conf.set(confKey,
Base64.encodeBytes(charValue.toString().getBytes()));
+ }
}
@VisibleForTesting
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkLoadTool.java
b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkLoadTool.java
index 14b8c34..a00dcb8 100644
---
a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkLoadTool.java
+++
b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkLoadTool.java
@@ -55,31 +55,39 @@ public class CsvBulkLoadTool extends AbstractBulkLoadTool {
// we don't parse ZK_QUORUM_OPT here because we need it in order to
// create the connection we need to build importColumns.
- char delimiterChar = ',';
+ Character delimiterChar = ',';
if (cmdLine.hasOption(DELIMITER_OPT.getOpt())) {
- String delimString =
StringEscapeUtils.unescapeJava(cmdLine.getOptionValue(DELIMITER_OPT.getOpt()));
+ String delimString =
StringEscapeUtils.unescapeJava(cmdLine.getOptionValue
+ (DELIMITER_OPT.getOpt()));
if (delimString.length() != 1) {
throw new IllegalArgumentException("Illegal delimiter
character: " + delimString);
}
delimiterChar = delimString.charAt(0);
}
- char quoteChar = '"';
+ Character quoteChar = '"';
if (cmdLine.hasOption(QUOTE_OPT.getOpt())) {
- String quoteString = cmdLine.getOptionValue(QUOTE_OPT.getOpt());
- if (quoteString.length() != 1) {
+ String quoteString =
StringEscapeUtils.unescapeJava(cmdLine.getOptionValue(QUOTE_OPT
+ .getOpt()));
+ if(quoteString.length() == 0) {
+ quoteChar = null;
+ } else if (quoteString.length() != 1) {
throw new IllegalArgumentException("Illegal quote character: "
+ quoteString);
+ } else {
+ quoteChar = quoteString.charAt(0);
}
- quoteChar = quoteString.charAt(0);
}
- char escapeChar = '\\';
+ Character escapeChar = '\\';
if (cmdLine.hasOption(ESCAPE_OPT.getOpt())) {
String escapeString = cmdLine.getOptionValue(ESCAPE_OPT.getOpt());
- if (escapeString.length() != 1) {
+ if(escapeString.length() == 0) {
+ escapeChar = null;
+ } else if (escapeString.length() != 1) {
throw new IllegalArgumentException("Illegal escape character:
" + escapeString);
+ } else {
+ escapeChar = escapeString.charAt(0);
}
- escapeChar = escapeString.charAt(0);
}
String binaryEncoding = null;
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java
b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java
index e5a6e5b..3b2293e 100644
---
a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java
+++
b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java
@@ -92,7 +92,7 @@ public class CsvToKeyValueMapper extends
FormatToBytesWritableMapper<CSVRecord>
static class CsvLineParser implements LineParser<CSVRecord> {
private final CSVFormat csvFormat;
- CsvLineParser(char fieldDelimiter, char quote, char escape) {
+ CsvLineParser(Character fieldDelimiter, Character quote, Character
escape) {
this.csvFormat = CSVFormat.DEFAULT
.withIgnoreEmptyLines(true)
.withDelimiter(fieldDelimiter)
