Handle all dictionary exception more properly modifed by suggestions
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/eedfe596 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/eedfe596 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/eedfe596 Branch: refs/heads/branch-0.1 Commit: eedfe5969edbcca8ea9ec0bc18651af7320ed831 Parents: b53a4db Author: foryou2030 <foryou2...@126.com> Authored: Sat Aug 27 14:42:16 2016 +0800 Committer: ravipesala <ravi.pes...@gmail.com> Committed: Thu Sep 22 10:30:16 2016 +0530 ---------------------------------------------------------------------- .../core/constants/CarbonCommonConstants.java | 4 +- .../store/filesystem/LocalCarbonFile.java | 5 +- .../spark/util/GlobalDictionaryUtil.scala | 54 ++++++++++++-------- 3 files changed, 39 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eedfe596/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java index 1e603b5..6050719 100644 --- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java +++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java @@ -199,9 +199,9 @@ public final class CarbonCommonConstants { */ public static final String MEMBER_DEFAULT_VAL = "@NU#LL$!"; /** - * BLANK_LINE_FLAG + * DEFAULT_COLUMN_NAME */ - public static final String BLANK_LINE_FLAG = "@NU#LL$!BLANKLINE"; + public static final String DEFAULT_COLUMN_NAME = "@NU#LL$!COLUMN"; /** * FILE STATUS IN-PROGRESS */ http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eedfe596/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/LocalCarbonFile.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/LocalCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/LocalCarbonFile.java index f46aeed..406f6d1 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/LocalCarbonFile.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/LocalCarbonFile.java @@ -86,7 +86,10 @@ public class LocalCarbonFile implements CarbonFile { } @Override public boolean exists() { - return file.exists(); + if (file != null) { + return file.exists(); + } + return false; } @Override public String getCanonicalPath() { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eedfe596/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala index cabedfd..bdd8adc 100644 --- a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala +++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala @@ -17,7 +17,7 @@ package org.apache.carbondata.spark.util -import java.io.IOException +import java.io.{FileNotFoundException, IOException} import java.nio.charset.Charset import java.util.regex.Pattern @@ -602,29 +602,31 @@ object GlobalDictionaryUtil extends Logging { val basicRdd = sqlContext.sparkContext.textFile(allDictionaryPath) .map(x => { val tokens = x.split("" + CSVWriter.DEFAULT_SEPARATOR) - var index: Int = 0 + if (tokens.size != 2) { + logError("Read a bad dictionary record: " + x) + } + var columnName: String = CarbonCommonConstants.DEFAULT_COLUMN_NAME var value: String = "" try { - index = tokens(0).toInt + columnName = csvFileColumns(tokens(0).toInt) value = tokens(1) } catch { case ex: Exception => - logError("read a bad dictionary record" + x) + logError("Reset bad dictionary record as default value") } - (index, value) + (columnName, value) }) + // group by column index, and filter required columns val requireColumnsList = requireColumns.toList allDictionaryRdd = basicRdd .groupByKey() - .map(x => (csvFileColumns(x._1), x._2)) .filter(x => requireColumnsList.contains(x._1)) } catch { case ex: Exception => - logError("read local dictionary files failed") + logError("Read dictionary files failed. Caused by: " + ex.getMessage) throw ex } - allDictionaryRdd } @@ -640,22 +642,32 @@ object GlobalDictionaryUtil extends Logging { // filepath regex, look like "/path/*.dictionary" if (filePath.getName.startsWith("*")) { val dictExt = filePath.getName.substring(1) - val listFiles = filePath.getParentFile.listFiles() - if (listFiles.exists(file => - file.getName.endsWith(dictExt) && file.getSize > 0)) { - true + if (filePath.getParentFile.exists()) { + val listFiles = filePath.getParentFile.listFiles() + if (listFiles.exists(file => + file.getName.endsWith(dictExt) && file.getSize > 0)) { + true + } else { + logWarning("No dictionary files found or empty dictionary files! " + + "Won't generate new dictionary.") + false + } } else { - logInfo("No dictionary files found or empty dictionary files! " + - "Won't generate new dictionary.") - false + throw new FileNotFoundException( + "The given dictionary file path is not found!") } } else { - if (filePath.exists() && filePath.getSize > 0) { - true + if (filePath.exists()) { + if (filePath.getSize > 0) { + true + } else { + logWarning("No dictionary files found or empty dictionary files! " + + "Won't generate new dictionary.") + false + } } else { - logInfo("No dictionary files found or empty dictionary files! " + - "Won't generate new dictionary.") - false + throw new FileNotFoundException( + "The given dictionary file path is not found!") } } } @@ -768,7 +780,7 @@ object GlobalDictionaryUtil extends Logging { } } } else { - logInfo("Generate global dictionary from all dictionary files!") + logInfo("Generate global dictionary from dictionary files!") val isNonempty = validateAllDictionaryPath(allDictionaryPath) if(isNonempty) { var headers = if (StringUtils.isEmpty(carbonLoadModel.getCsvHeader)) {