Repository: carbondata Updated Branches: refs/heads/master 3d8f09bac -> 4e3151087
[CARBONDATA-1831] BAD_RECORDS: Data Loading with Action as Redirect & logger enable is not logging the logs in the defined path. Current system expects the carbon.badRecords.location to be absolute path with filescheme. But, it should support absolute path without file scheme(eg.: hdfs://, viewfs://) This closes #1609 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4e315108 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4e315108 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4e315108 Branch: refs/heads/master Commit: 4e315108763858781d0ef2c340e3343e9860b14a Parents: 3d8f09b Author: dhatchayani <[email protected]> Authored: Mon Dec 4 20:01:06 2017 +0530 Committer: Jacky Li <[email protected]> Committed: Wed Dec 6 00:29:10 2017 +0800 ---------------------------------------------------------------------- .../carbondata/core/datastore/filesystem/LocalCarbonFile.java | 2 ++ .../scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/4e315108/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java index 5b2cc68..30eb7fe 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java @@ -123,6 +123,7 @@ public class LocalCarbonFile implements CarbonFile { } public boolean renameTo(String changetoName) { + changetoName = getUpdatedFilePath(changetoName); return file.renameTo(new File(changetoName)); } @@ -318,6 +319,7 @@ public class LocalCarbonFile implements CarbonFile { @Override public DataOutputStream getDataOutputStream(String path, FileFactory.FileType fileType) throws IOException { path = path.replace("\\", "/"); + path = getUpdatedFilePath(path); return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path))); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/4e315108/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala index 77f75f2..74ed6a6 100644 --- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala +++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala @@ -209,7 +209,7 @@ object DataLoadingUtil { val single_pass = optionsFinal("single_pass") val bad_records_logger_enable = optionsFinal("bad_records_logger_enable") val bad_records_action = optionsFinal("bad_records_action") - val bad_record_path = optionsFinal("bad_record_path") + var bad_record_path = optionsFinal("bad_record_path") val global_sort_partitions = optionsFinal("global_sort_partitions") val timestampformat = optionsFinal("timestampformat") val dateFormat = optionsFinal("dateformat") @@ -224,6 +224,7 @@ object DataLoadingUtil { if (bad_records_logger_enable.toBoolean || LoggerAction.REDIRECT.name().equalsIgnoreCase(bad_records_action)) { + bad_record_path = CarbonUtil.checkAndAppendHDFSUrl(bad_record_path) if (!CarbonUtil.isValidBadStorePath(bad_record_path)) { CarbonException.analysisException("Invalid bad records location.") }
