Repository: carbondata Updated Branches: refs/heads/master 52bf7c81c -> 75ec79e3f
[CARBONDATA-1741] Remove AKSK in log when saving to S3 This closes #1511 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/75ec79e3 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/75ec79e3 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/75ec79e3 Branch: refs/heads/master Commit: 75ec79e3f7ddbbfcddb1fbfe1413aae2ee13b6e2 Parents: 52bf7c8 Author: Jacky Li <jacky.li...@qq.com> Authored: Fri Nov 17 14:57:15 2017 +0800 Committer: QiangCai <qiang...@qq.com> Committed: Fri Nov 17 16:22:11 2017 +0800 ---------------------------------------------------------------------- .../apache/carbondata/core/util/CarbonUtil.java | 19 ++++++++++++++++++- .../scala/org/apache/spark/util/FileUtils.scala | 8 +++++--- 2 files changed, 23 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/75ec79e3/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index 1e402d0..3c177dc 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -764,6 +764,23 @@ public final class CarbonUtil { lowerPath.startsWith(CarbonCommonConstants.S3A_PREFIX); } + public static String removeAKSK(String filePath) { + if (null == filePath) { + return ""; + } + String lowerPath = filePath.toLowerCase(Locale.getDefault()); + if (lowerPath.startsWith(CarbonCommonConstants.S3N_PREFIX) || + lowerPath.startsWith(CarbonCommonConstants.S3A_PREFIX) || + lowerPath.startsWith(CarbonCommonConstants.S3_PREFIX)) { + int prefixLength = filePath.indexOf(":", 0) + 3; + int pathOffset = filePath.indexOf("@"); + if (pathOffset > 0) { + return filePath.substring(0, prefixLength) + filePath.substring(pathOffset + 1); + } + } + return filePath; + } + /** * This method will check the existence of a file at a given path */ @@ -774,7 +791,7 @@ public final class CarbonUtil { return true; } } catch (IOException e) { - LOGGER.error("@@@@@@ File not found at a given location @@@@@@ : " + fileName); + LOGGER.error("@@@@@@ File not found at a given location @@@@@@ : " + removeAKSK(fileName)); } return false; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/75ec79e3/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala b/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala index 7345d54..46311db 100644 --- a/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala +++ b/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala @@ -21,6 +21,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.datastore.filesystem.CarbonFile import org.apache.carbondata.core.datastore.impl.FileFactory +import org.apache.carbondata.core.util.CarbonUtil import org.apache.carbondata.processing.exception.DataLoadingException object FileUtils { @@ -38,11 +39,11 @@ object FileUtils { val fileName = carbonFile.getName if (carbonFile.getSize == 0) { LogServiceFactory.getLogService(this.getClass.getCanonicalName) - .warn(s"skip empty input file: $path") + .warn(s"skip empty input file: ${CarbonUtil.removeAKSK(path)}") } else if (fileName.startsWith(CarbonCommonConstants.UNDERSCORE) || fileName.startsWith(CarbonCommonConstants.POINT)) { LogServiceFactory.getLogService(this.getClass.getCanonicalName) - .warn(s"skip invisible input file: $path") + .warn(s"skip invisible input file: ${CarbonUtil.removeAKSK(path)}") } else { stringBuild.append(path.replace('\\', '/')).append(CarbonCommonConstants.COMMA) } @@ -63,7 +64,8 @@ object FileUtils { val fileType = FileFactory.getFileType(filePaths(i)) val carbonFile = FileFactory.getCarbonFile(filePaths(i), fileType) if (!carbonFile.exists()) { - throw new DataLoadingException(s"The input file does not exist: ${filePaths(i)}" ) + throw new DataLoadingException( + s"The input file does not exist: ${CarbonUtil.removeAKSK(filePaths(i))}" ) } getPathsFromCarbonFile(carbonFile, stringBuild) }
