[CARBONDATA-1236] Support absolute path without scheme in loading - change in logic
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c671c5b6 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c671c5b6 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c671c5b6 Branch: refs/heads/datamap Commit: c671c5b60ad4989643e9b6f92540b60c2c540145 Parents: 126a041 Author: dhatchayani <[email protected]> Authored: Thu Jun 29 17:59:33 2017 +0530 Committer: dhatchayani <[email protected]> Committed: Thu Jun 29 18:02:50 2017 +0530 ---------------------------------------------------------------------- .../apache/carbondata/core/util/CarbonUtil.java | 50 +++++------ .../carbondata/core/util/CarbonUtilTest.java | 89 +++++++++++++++++++- 2 files changed, 112 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/c671c5b6/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index 6e0348b..8298600 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -697,32 +697,32 @@ public final class CarbonUtil { */ public static String checkAndAppendHDFSUrl(String filePath) { String currentPath = filePath; - if (null != filePath && filePath.length() != 0 - && FileFactory.getFileType(filePath) != FileFactory.FileType.HDFS - && FileFactory.getFileType(filePath) != FileFactory.FileType.VIEWFS) { - if (!filePath.startsWith("/")) { - filePath = "/" + filePath; - } - String baseDFSUrl = CarbonProperties.getInstance() - .getProperty(CarbonCommonConstants.CARBON_DDL_BASE_HDFS_URL); - String dfsUrl = conf.get(FS_DEFAULT_FS); - if (null != baseDFSUrl) { - if (!baseDFSUrl.startsWith("/")) { - baseDFSUrl = "/" + baseDFSUrl; - } - if (null != dfsUrl && (dfsUrl.startsWith(HDFS_PREFIX) || dfsUrl - .startsWith(VIEWFS_PREFIX))) { - baseDFSUrl = dfsUrl + baseDFSUrl; - } - if (baseDFSUrl.endsWith("/")) { - baseDFSUrl = baseDFSUrl.substring(0, baseDFSUrl.length() - 1); - } - currentPath = baseDFSUrl + filePath; - } else { - currentPath = dfsUrl + filePath; - } + String defaultFsUrl = conf.get(FS_DEFAULT_FS); + String baseDFSUrl = CarbonProperties.getInstance() + .getProperty(CarbonCommonConstants.CARBON_DDL_BASE_HDFS_URL, ""); + if (checkIfPrefixExists(filePath)) { + return currentPath; } - return currentPath; + if (baseDFSUrl.endsWith("/")) { + baseDFSUrl = baseDFSUrl.substring(0, baseDFSUrl.length() - 1); + } + if (!filePath.startsWith("/")) { + filePath = "/" + filePath; + } + currentPath = baseDFSUrl + filePath; + if (checkIfPrefixExists(currentPath)) { + return currentPath; + } + if (defaultFsUrl == null) { + return currentPath; + } + return defaultFsUrl + currentPath; + } + + private static boolean checkIfPrefixExists(String path) { + final String lowerPath = path.toLowerCase(); + return lowerPath.startsWith(HDFS_PREFIX) || lowerPath.startsWith(VIEWFS_PREFIX) || lowerPath + .startsWith("file://") || lowerPath.startsWith(ALLUXIO_PREFIX); } public static String getCarbonStorePath() { http://git-wip-us.apache.org/repos/asf/carbondata/blob/c671c5b6/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java index b42c57e..776059f 100644 --- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java +++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java @@ -342,7 +342,7 @@ public class CarbonUtilTest { } }; String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("../core/src/test/resources/testDatabase"); - assertEquals(hdfsURL, "/BASE_URL/../core/src/test/resources/testDatabase"); + assertEquals(hdfsURL, "file:///BASE_URL/../core/src/test/resources/testDatabase"); } @Test public void testToCheckAndAppendHDFSUrlWithBlackSlash() { @@ -357,7 +357,7 @@ public class CarbonUtilTest { } }; String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("../core/src/test/resources/testDatabase"); - assertEquals(hdfsURL, "/BASE_URL/../core/src/test/resources/testDatabase"); + assertEquals(hdfsURL, "file:///BASE_URL/../core/src/test/resources/testDatabase"); } @Test public void testToCheckAndAppendHDFSUrlWithNull() { @@ -375,6 +375,91 @@ public class CarbonUtilTest { assertEquals(hdfsURL, "file:////../core/src/test/resources/testDatabase"); } + @Test public void testToCheckAndAppendHDFSUrlWithHdfs() { + new MockUp<FileFactory>() { + @SuppressWarnings("unused") @Mock public FileFactory.FileType getFileType(String path) { + return FileFactory.FileType.HDFS; + } + }; + new MockUp<org.apache.hadoop.conf.Configuration>() { + @SuppressWarnings("unused") @Mock public String get(String name) { + return "hdfs://"; + } + }; + String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("hdfs://ha/core/src/test/resources/testDatabase"); + assertEquals(hdfsURL, "hdfs://ha/core/src/test/resources/testDatabase"); + } + + @Test public void testToCheckAndAppendHDFSUrlWithDoubleSlashLocal() { + new MockUp<FileFactory>() { + @SuppressWarnings("unused") @Mock public FileFactory.FileType getFileType(String path) { + return FileFactory.FileType.LOCAL; + } + }; + new MockUp<CarbonProperties>() { + @SuppressWarnings("unused") @Mock public String getProperty(String key) { + return "/opt/"; + } + }; + String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("/core/src/test/resources/testDatabase"); + assertEquals(hdfsURL, "file:////opt/core/src/test/resources/testDatabase"); + } + + @Test public void testToCheckAndAppendHDFSUrlWithDoubleSlashHDFS() { + new MockUp<FileFactory>() { + @SuppressWarnings("unused") @Mock public FileFactory.FileType getFileType(String path) { + return FileFactory.FileType.HDFS; + } + }; + new MockUp<org.apache.hadoop.conf.Configuration>() { + @SuppressWarnings("unused") @Mock public String get(String name) { + return "hdfs://"; + } + }; + new MockUp<CarbonProperties>() { + @SuppressWarnings("unused") @Mock public String getProperty(String key) { + return "/opt/"; + } + }; + String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("/core/src/test/resources/testDatabase"); + assertEquals(hdfsURL, "hdfs:///opt/core/src/test/resources/testDatabase"); + } + + @Test public void testToCheckAndAppendHDFSUrlWithBaseURLPrefix() { + new MockUp<FileFactory>() { + @SuppressWarnings("unused") @Mock public FileFactory.FileType getFileType(String path) { + return FileFactory.FileType.HDFS; + } + }; + new MockUp<CarbonProperties>() { + @SuppressWarnings("unused") @Mock public String getProperty(String key) { + return "hdfs://ha/opt/"; + } + }; + String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("/core/src/test/resources/testDatabase"); + assertEquals(hdfsURL, "hdfs://ha/opt/core/src/test/resources/testDatabase"); + } + + @Test public void testToCheckAndAppendHDFSUrlWithBaseURLFile() { + new MockUp<FileFactory>() { + @SuppressWarnings("unused") @Mock public FileFactory.FileType getFileType(String path) { + return FileFactory.FileType.HDFS; + } + }; + new MockUp<CarbonProperties>() { + @SuppressWarnings("unused") @Mock public String getProperty(String key) { + return "file:///"; + } + }; + String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("/core/src/test/resources/testDatabase"); + assertEquals(hdfsURL, "file:///core/src/test/resources/testDatabase"); + } + + @Test public void testToCheckAndAppendHDFSUrlWithFilepathPrefix() { + String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("file:///core/src/test/resources/testDatabase"); + assertEquals(hdfsURL, "file:///core/src/test/resources/testDatabase"); + } + @Test public void testForisFileExists() { assertTrue(CarbonUtil.isFileExists("../core/src/test/resources/testFile.txt")); }
