Repository: kylin Updated Branches: refs/heads/KYLIN-3052 [created] fb2c9a5b6
KYLIN-3052 Support Redshift as data source Signed-off-by: shaofengshi <shaofeng...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fb2c9a5b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fb2c9a5b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fb2c9a5b Branch: refs/heads/KYLIN-3052 Commit: fb2c9a5b6e033907e5109d2b0cb0027fdd5e5647 Parents: 6474e22 Author: ZhansShaoxiong <shaoxiong0...@gmail.com> Authored: Sat Dec 2 15:28:00 2017 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Tue Dec 5 17:55:11 2017 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/job/constant/ExecutableConstants.java | 1 + .../java/org/apache/kylin/rest/service/TableService.java | 8 ++++---- .../java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java | 3 ++- .../kylin/source/jdbc/metadata/MySQLJdbcMetadata.java | 6 ++---- .../kylin/source/jdbc/metadata/MySQLJdbcMetadataTest.java | 9 ++------- 5 files changed, 11 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/fb2c9a5b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java index f3caf3b..cf61722 100644 --- a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java +++ b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java @@ -35,6 +35,7 @@ public final class ExecutableConstants { public static final String STEP_NAME_BUILD_DICTIONARY = "Build Dimension Dictionary"; public static final String STEP_NAME_BUILD_UHC_DICTIONARY = "Build UHC Dictionary"; public static final String STEP_NAME_CREATE_FLAT_HIVE_TABLE = "Create Intermediate Flat Hive Table"; + public static final String STEP_NAME_SQOOP_TO_FLAT_HIVE_TABLE = "Sqoop To Flat Hive Table"; public static final String STEP_NAME_MATERIALIZE_HIVE_VIEW_IN_LOOKUP = "Materialize Hive View in Lookup Tables"; public static final String STEP_NAME_FACT_DISTINCT_COLUMNS = "Extract Fact Table Distinct Columns"; public static final String STEP_NAME_CALCULATE_STATS_FROM_BASE_CUBOID = "Calculate Statistics from Base Cuboid"; http://git-wip-us.apache.org/repos/asf/kylin/blob/fb2c9a5b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java index ffb71a6..901ac46 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java @@ -119,7 +119,7 @@ public class TableService extends BasicService { SetMultimap<String, String> db2tables = LinkedHashMultimap.create(); for (String fullTableName : tables) { String[] parts = HadoopUtil.parseHiveTableName(fullTableName); - db2tables.put(parts[0].toUpperCase(), parts[1].toUpperCase()); + db2tables.put(parts[0], parts[1]); } // load all tables first @@ -128,9 +128,9 @@ public class TableService extends BasicService { for (Map.Entry<String, String> entry : db2tables.entries()) { Pair<TableDesc, TableExtDesc> pair = explr.loadTableMetadata(entry.getKey(), entry.getValue(), project); TableDesc tableDesc = pair.getFirst(); - Preconditions.checkState(tableDesc.getDatabase().equals(entry.getKey())); - Preconditions.checkState(tableDesc.getName().equals(entry.getValue())); - Preconditions.checkState(tableDesc.getIdentity().equals(entry.getKey() + "." + entry.getValue())); + Preconditions.checkState(tableDesc.getDatabase().equals(entry.getKey().toUpperCase())); + Preconditions.checkState(tableDesc.getName().equals(entry.getValue().toUpperCase())); + Preconditions.checkState(tableDesc.getIdentity().equals(entry.getKey().toUpperCase() + "." + entry.getValue().toUpperCase())); TableExtDesc extDesc = pair.getSecond(); Preconditions.checkState(tableDesc.getIdentity().equals(extDesc.getIdentity())); allMeta.add(pair); http://git-wip-us.apache.org/repos/asf/kylin/blob/fb2c9a5b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java index 15259cc..0bfc145 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java +++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java @@ -74,6 +74,7 @@ public class JdbcHiveMRInput extends HiveMRInput { HiveCmdStep step = new HiveCmdStep(); step.setCmd(hiveInitStatements + dropTableHql + createTableHql); + step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE); return step; } @@ -171,7 +172,7 @@ public class JdbcHiveMRInput extends HiveMRInput { logger.debug(String.format("sqoop cmd:%s", cmd)); CmdStep step = new CmdStep(); step.setCmd(cmd); - step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE); + step.setName(ExecutableConstants.STEP_NAME_SQOOP_TO_FLAT_HIVE_TABLE); return step; } http://git-wip-us.apache.org/repos/asf/kylin/blob/fb2c9a5b/source-hive/src/main/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadata.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadata.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadata.java index 6404fd6..54c2a03 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadata.java +++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadata.java @@ -35,10 +35,8 @@ public class MySQLJdbcMetadata extends DefaultJdbcMetadata { @Override public List<String> listDatabases() throws SQLException { List<String> ret = new ArrayList<>(); - try (Connection con = SqlUtil.getConnection(dbconf); ResultSet res = con.getMetaData().getCatalogs()) { - while (res.next()) { - ret.add(res.getString("TABLE_CAT")); - } + try (Connection con = SqlUtil.getConnection(dbconf)) { + ret.add(con.getCatalog()); } return ret; } http://git-wip-us.apache.org/repos/asf/kylin/blob/fb2c9a5b/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadataTest.java ---------------------------------------------------------------------- diff --git a/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadataTest.java b/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadataTest.java index d0cb6c4..e461c15 100644 --- a/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadataTest.java +++ b/source-hive/src/test/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadataTest.java @@ -47,16 +47,11 @@ public class MySQLJdbcMetadataTest extends DefaultJdbcMetadataTest { @Test public void testListDatabases() throws SQLException { - ResultSet rs = mock(ResultSet.class); - when(rs.next()).thenReturn(true).thenReturn(true).thenReturn(false); - when(rs.getString("TABLE_CAT")).thenReturn("catalog1").thenReturn("catalog2"); - - when(connection.getMetaData()).thenReturn(dbmd); - when(dbmd.getCatalogs()).thenReturn(rs); + when(connection.getCatalog()).thenReturn("catalog1"); List<String> dbs = jdbcMetadata.listDatabases(); - Assert.assertEquals(2, dbs.size()); + Assert.assertEquals(1, dbs.size()); Assert.assertEquals("catalog1", dbs.get(0)); }