Repository: incubator-carbondata Updated Branches: refs/heads/master b91be7759 -> cb480e0dd
[CARBONDATA-316] Change BAD_RECORDS_LOGGER_ACTION to BAD_RECORDS_ACTION Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/af0cb585 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/af0cb585 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/af0cb585 Branch: refs/heads/master Commit: af0cb5857c426a2192508ee05b28a50a0fbb68fd Parents: b91be77 Author: mohammadshahidkhan <mohdshahidkhan1...@gmail.com> Authored: Thu Oct 13 17:59:10 2016 +0530 Committer: jackylk <jacky.li...@huawei.com> Committed: Fri Oct 14 14:15:08 2016 +0800 ---------------------------------------------------------------------- .../apache/carbondata/hadoop/test/util/StoreCreator.java | 2 +- .../main/scala/org/apache/spark/sql/CarbonSqlParser.scala | 4 ++-- .../spark/sql/execution/command/carbonTableSchema.scala | 4 ++-- .../testsuite/badrecordloger/BadRecordLoggerTest.scala | 10 +++++----- .../csvbased/CarbonCSVBasedSeqGenStep.java | 8 ++++---- 5 files changed, 14 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af0cb585/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java ---------------------------------------------------------------------- diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java index 5caa96a..5661888 100644 --- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java +++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java @@ -360,7 +360,7 @@ public class StoreCreator { info.setTableName(tableName); info.setSerializationNullFormat("serialization_null_format" + "," + "\\N"); info.setBadRecordsLoggerEnable("bad_records_logger_enable"+","+"false"); - info.setBadRecordsLoggerEnable("bad_records_logger_action"+","+"force"); + info.setBadRecordsLoggerEnable("bad_records_action"+","+"force"); generateGraph(schmaModel, info, loadModel.getTableName(), "0", loadModel.getSchema(), null, loadModel.getLoadMetadataDetails()); http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af0cb585/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala index 775d48d..0247aec 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala @@ -89,7 +89,7 @@ class CarbonSqlParser() protected val FILEHEADER = carbonKeyWord("FILEHEADER") protected val SERIALIZATION_NULL_FORMAT = carbonKeyWord("SERIALIZATION_NULL_FORMAT") protected val BAD_RECORDS_LOGGER_ENABLE = carbonKeyWord("BAD_RECORDS_LOGGER_ENABLE") - protected val BAD_RECORDS_LOGGER_ACTION = carbonKeyWord("BAD_RECORDS_LOGGER_ACTION") + protected val BAD_RECORDS_ACTION = carbonKeyWord("BAD_RECORDS_ACTION") protected val FILES = carbonKeyWord("FILES") protected val FROM = carbonKeyWord("FROM") protected val HIERARCHIES = carbonKeyWord("HIERARCHIES") @@ -935,7 +935,7 @@ class CarbonSqlParser() val options = partionDataOptions.get.groupBy(x => x._1) val supportedOptions = Seq("DELIMITER", "QUOTECHAR", "FILEHEADER", "ESCAPECHAR", "MULTILINE", "COMPLEX_DELIMITER_LEVEL_1", "COMPLEX_DELIMITER_LEVEL_2", "COLUMNDICT", - "SERIALIZATION_NULL_FORMAT", "BAD_RECORDS_LOGGER_ENABLE", "BAD_RECORDS_LOGGER_ACTION", + "SERIALIZATION_NULL_FORMAT", "BAD_RECORDS_LOGGER_ENABLE", "BAD_RECORDS_ACTION", "ALL_DICTIONARY_PATH", "MAXCOLUMNS", "COMMENTCHAR" ) var isSupported = true http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af0cb585/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala index 4ade2be..6dc6668 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala @@ -1118,7 +1118,7 @@ case class LoadTable( val columnDict = partionValues.getOrElse("columndict", null) val serializationNullFormat = partionValues.getOrElse("serialization_null_format", "\\N") val badRecordsLoggerEnable = partionValues.getOrElse("bad_records_logger_enable", "false") - val badRecordsLoggerRedirect = partionValues.getOrElse("bad_records_logger_action", "force") + val badRecordsLoggerRedirect = partionValues.getOrElse("bad_records_action", "force") val allDictionaryPath = partionValues.getOrElse("all_dictionary_path", "") val complex_delimiter_level_1 = partionValues.getOrElse("complex_delimiter_level_1", "\\$") val complex_delimiter_level_2 = partionValues.getOrElse("complex_delimiter_level_2", "\\:") @@ -1140,7 +1140,7 @@ case class LoadTable( carbonLoadModel .setBadRecordsLoggerEnable("bad_records_logger_enable" + "," + badRecordsLoggerEnable) carbonLoadModel - .setBadRecordsLoggerRedirect("bad_records_logger_action" + "," + badRecordsLoggerRedirect) + .setBadRecordsLoggerRedirect("bad_records_action" + "," + badRecordsLoggerRedirect) if (delimiter.equalsIgnoreCase(complex_delimiter_level_1) || complex_delimiter_level_1.equalsIgnoreCase(complex_delimiter_level_2) || http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af0cb585/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerTest.scala index 2e2132e..a77d314 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerTest.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerTest.scala @@ -66,7 +66,7 @@ class BadRecordLoggerTest extends QueryTest with BeforeAndAfterAll { var csvFilePath = currentDirectory + "/src/test/resources/badrecords/datasample.csv" sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE sales OPTIONS" + - "('bad_records_logger_enable'='true','bad_records_logger_action'='redirect', 'DELIMITER'=" + + "('bad_records_logger_enable'='true','bad_records_action'='redirect', 'DELIMITER'=" + " ',', 'QUOTECHAR'= '\"')"); // 1.0 "\N" which should be treated as NULL @@ -79,7 +79,7 @@ class BadRecordLoggerTest extends QueryTest with BeforeAndAfterAll { """) sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE serializable_values OPTIONS" + - "('bad_records_logger_enable'='true', 'bad_records_logger_action'='ignore', " + + "('bad_records_logger_enable'='true', 'bad_records_action'='ignore', " + "'DELIMITER'= ',', 'QUOTECHAR'= '\"')"); // load with bad_records_logger_enable false sql( @@ -99,7 +99,7 @@ class BadRecordLoggerTest extends QueryTest with BeforeAndAfterAll { "/src/test/resources/badrecords/insufficientColumns.csv" sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE insufficientColumn OPTIONS" + - "('bad_records_logger_enable'='true', 'bad_records_logger_action'='ignore', " + + "('bad_records_logger_enable'='true', 'bad_records_action'='ignore', " + "'DELIMITER'= ',', 'QUOTECHAR'= '\"')"); // load with bad_records_logger_enable false sql( @@ -121,7 +121,7 @@ class BadRecordLoggerTest extends QueryTest with BeforeAndAfterAll { "/src/test/resources/badrecords/emptyValues.csv" sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE emptyColumnValues OPTIONS" + - "('bad_records_logger_enable'='true', 'bad_records_logger_action'='ignore', " + + "('bad_records_logger_enable'='true', 'bad_records_action'='ignore', " + "'DELIMITER'= ',', 'QUOTECHAR'= '\"')"); // load with bad_records_logger_enable to false sql( @@ -147,7 +147,7 @@ class BadRecordLoggerTest extends QueryTest with BeforeAndAfterAll { "/src/test/resources/badrecords/emptyTimeStampValue.csv" sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE empty_timestamp OPTIONS" + - "('bad_records_logger_enable'='true', 'bad_records_logger_action'='ignore', " + + "('bad_records_logger_enable'='true', 'bad_records_action'='ignore', " + "'DELIMITER'= ',', 'QUOTECHAR'= '\"')"); // load with bad_records_logger_enable to false sql( http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af0cb585/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java index 84a1f51..dc7dd22 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java +++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java @@ -443,12 +443,12 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep { badRecordsLoggerEnable = Boolean.parseBoolean(meta.getTableOptionWrapper().get("bad_records_logger_enable")); badRecordConvertNullDisable = true; - String bad_records_logger_action = - meta.getTableOptionWrapper().get("bad_records_logger_action"); - if(null != bad_records_logger_action) { + String bad_records_action = + meta.getTableOptionWrapper().get("bad_records_action"); + if(null != bad_records_action) { LoggerAction loggerAction = null; try { - loggerAction = LoggerAction.valueOf(bad_records_logger_action.toUpperCase()); + loggerAction = LoggerAction.valueOf(bad_records_action.toUpperCase()); } catch (IllegalArgumentException e) { loggerAction = LoggerAction.FORCE; }
