Repository: kylin Updated Branches: refs/heads/hive-load-refactor 0b66ef1d6 -> 877b538a5
refactor Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/877b538a Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/877b538a Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/877b538a Branch: refs/heads/hive-load-refactor Commit: 877b538a508f60ab72d97eaf7af6e39d55a4b4bd Parents: 0b66ef1 Author: Billy Liu <billy...@apache.org> Authored: Tue Dec 27 10:22:50 2016 +0800 Committer: Billy Liu <billy...@apache.org> Committed: Tue Dec 27 10:22:50 2016 +0800 ---------------------------------------------------------------------- .../kylin/rest/controller/TableController.java | 76 ++++++++++---------- .../apache/kylin/rest/service/TableService.java | 72 +++++++++++-------- webapp/app/js/controllers/sourceMeta.js | 16 ++--- webapp/app/js/services/tables.js | 1 - 4 files changed, 86 insertions(+), 79 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/877b538a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java index 0debad7..964cd4c 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java @@ -24,12 +24,11 @@ import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.rest.exception.InternalErrorException; +import org.apache.kylin.rest.exception.NotFoundException; import org.apache.kylin.rest.request.CardinalityRequest; import org.apache.kylin.rest.request.HiveTableRequest; -import org.apache.kylin.rest.request.StreamingRequest; import org.apache.kylin.rest.service.TableService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,14 +65,12 @@ public class TableController extends BasicController { @RequestMapping(value = "", method = { RequestMethod.GET }) @ResponseBody public List<TableDesc> getTableDesc(@RequestParam(value = "ext", required = false) boolean withExt, @RequestParam(value = "project", required = true) String project) throws IOException { - List<TableDesc> tables = null; try { - tables = tableService.getTableDescByProject(project, withExt); + return tableService.getTableDescByProject(project, withExt); } catch (IOException e) { logger.error("Failed to get Hive Tables", e); throw new InternalErrorException(e.getLocalizedMessage()); } - return tables; } /** @@ -85,21 +82,28 @@ public class TableController extends BasicController { @RequestMapping(value = "/{tableName:.+}", method = { RequestMethod.GET }) @ResponseBody public TableDesc getTableDesc(@PathVariable String tableName) { - return tableService.getTableDescByName(tableName); + TableDesc table = tableService.getTableDescByName(tableName, false); + if (table == null) + throw new NotFoundException("Could not find Hive table: " + tableName); + return table; } @RequestMapping(value = "/{tables}/{project}", method = { RequestMethod.POST }) @ResponseBody public Map<String, String[]> loadHiveTables(@PathVariable String tables, @PathVariable String project, @RequestBody HiveTableRequest request) throws IOException { String submitter = SecurityContextHolder.getContext().getAuthentication().getName(); + Map<String, String[]> result = new HashMap<String, String[]>(); String[] tableNames = tables.split(","); - String[] loaded = tableService.loadHiveTablesToProject(tableNames, project); - if (request.isCalculate()) { - tableService.calculateCardinalityIfNotPresent(loaded, submitter); + try { + String[] loaded = tableService.loadHiveTablesToProject(tableNames, project); + result.put("result.loaded", loaded); + if (request.isCalculate()) { + tableService.calculateCardinalityIfNotPresent(loaded, submitter); + } + } catch (Exception e) { + logger.error("Failed to load Hive Table", e); + throw new InternalErrorException(e.getLocalizedMessage()); } - Map<String, String[]> result = new HashMap<String, String[]>(); - result.put("result.loaded", loaded); - result.put("result.unloaded", new String[] {}); return result; } @@ -109,29 +113,23 @@ public class TableController extends BasicController { Set<String> unLoadSuccess = Sets.newHashSet(); Set<String> unLoadFail = Sets.newHashSet(); Map<String, String[]> result = new HashMap<String, String[]>(); - for (String tableName : tables.split(",")) { - if (tableService.unLoadHiveTable(tableName, project)) { - unLoadSuccess.add(tableName); - } else { - unLoadFail.add(tableName); + try { + for (String tableName : tables.split(",")) { + if (tableService.unLoadHiveTable(tableName, project)) { + unLoadSuccess.add(tableName); + } else { + unLoadFail.add(tableName); + } } + } catch (Exception e) { + logger.error("Failed to unload Hive Table", e); + throw new InternalErrorException(e.getLocalizedMessage()); } result.put("result.unload.success", (String[]) unLoadSuccess.toArray(new String[unLoadSuccess.size()])); result.put("result.unload.fail", (String[]) unLoadFail.toArray(new String[unLoadFail.size()])); return result; } - @RequestMapping(value = "/addStreamingSrc", method = { RequestMethod.POST }) - @ResponseBody - public Map<String, String> addStreamingTable(@RequestBody StreamingRequest request) throws IOException { - Map<String, String> result = new HashMap<String, String>(); - String project = request.getProject(); - TableDesc desc = JsonUtil.readValue(request.getTableData(), TableDesc.class); - tableService.addStreamingTable(desc, project); - result.put("success", "true"); - return result; - } - /** * Regenerate table cardinality * @@ -143,8 +141,13 @@ public class TableController extends BasicController { public CardinalityRequest generateCardinality(@PathVariable String tableNames, @RequestBody CardinalityRequest request) throws IOException { String submitter = SecurityContextHolder.getContext().getAuthentication().getName(); String[] tables = tableNames.split(","); - for (String table : tables) { - tableService.calculateCardinality(table.trim().toUpperCase(), submitter); + try { + for (String table : tables) { + tableService.calculateCardinality(table.trim().toUpperCase(), submitter); + } + } catch (IOException e) { + logger.error("Failed to calculate cardinality", e); + throw new InternalErrorException(e.getLocalizedMessage()); } return request; } @@ -158,13 +161,11 @@ public class TableController extends BasicController { @RequestMapping(value = "/hive", method = { RequestMethod.GET }) @ResponseBody private List<String> showHiveDatabases() throws IOException { - List<String> results = null; try { - results = tableService.getHiveDbNames(); + return tableService.getHiveDbNames(); } catch (Exception e) { - throw new IOException(e); + throw new InternalErrorException(e.getLocalizedMessage()); } - return results; } /** @@ -176,14 +177,11 @@ public class TableController extends BasicController { @RequestMapping(value = "/hive/{database}", method = { RequestMethod.GET }) @ResponseBody private List<String> showHiveTables(@PathVariable String database) throws IOException { - List<String> results = null; - try { - results = tableService.getHiveTableNames(database); + return tableService.getHiveTableNames(database); } catch (Exception e) { - throw new IOException(e); + throw new InternalErrorException(e.getLocalizedMessage()); } - return results; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/877b538a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java index 6accfdc..8868bff 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java @@ -83,8 +83,12 @@ public class TableService extends BasicService { return tables; } - public TableDesc getTableDescByName(String tableName) { - return getMetadataManager().getTableDesc(tableName); + public TableDesc getTableDescByName(String tableName, boolean withExt) { + TableDesc table = getMetadataManager().getTableDesc(tableName); + if(withExt){ + table = cloneTableDesc(table); + } + return table; } @PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN) @@ -128,7 +132,7 @@ public class TableService extends BasicService { //remove streaming info String[] dbTableName = HadoopUtil.parseHiveTableName(tableName); tableName = dbTableName[0] + "." + dbTableName[1]; - TableDesc desc = getTableDescByName(tableName); + TableDesc desc = getMetadataManager().getTableDesc(tableName); if (desc == null) return false; tableType = desc.getSourceType(); @@ -206,39 +210,45 @@ public class TableService extends BasicService { return results; } + private TableDescResponse cloneTableDesc(TableDesc table) { + TableExtDesc tableExtDesc = getMetadataManager().getTableExt(table.getIdentity()); + + // Clone TableDesc + TableDescResponse rtableDesc = new TableDescResponse(table); + Map<String, Long> cardinality = new HashMap<String, Long>(); + Map<String, String> dataSourceProp = new HashMap<>(); + String scard = tableExtDesc.getCardinality(); + if (!StringUtils.isEmpty(scard)) { + String[] cards = StringUtils.split(scard, ","); + ColumnDesc[] cdescs = rtableDesc.getColumns(); + for (int i = 0; i < cdescs.length; i++) { + ColumnDesc columnDesc = cdescs[i]; + if (cards.length > i) { + cardinality.put(columnDesc.getName(), Long.parseLong(cards[i])); + } else { + logger.error("The result cardinality is not identical with hive table metadata, cardinality : " + scard + " column array length: " + cdescs.length); + break; + } + } + rtableDesc.setCardinality(cardinality); + } + dataSourceProp.putAll(tableExtDesc.getDataSourceProp()); + dataSourceProp.put("location", tableExtDesc.getStorageLocation()); + dataSourceProp.put("owner", tableExtDesc.getOwner()); + dataSourceProp.put("last_access_time", tableExtDesc.getLastAccessTime()); + dataSourceProp.put("partition_column", tableExtDesc.getPartitionColumn()); + dataSourceProp.put("total_file_size", tableExtDesc.getTotalFileSize()); + rtableDesc.setDescExd(dataSourceProp); + return rtableDesc; + } + + private List<TableDesc> cloneTableDesc(List<TableDesc> tables) throws IOException { List<TableDesc> descs = new ArrayList<TableDesc>(); Iterator<TableDesc> it = tables.iterator(); while (it.hasNext()) { TableDesc table = it.next(); - TableExtDesc tableExtDesc = getMetadataManager().getTableExt(table.getIdentity()); - - // Clone TableDesc - TableDescResponse rtableDesc = new TableDescResponse(table); - Map<String, Long> cardinality = new HashMap<String, Long>(); - Map<String, String> dataSourceProp = new HashMap<>(); - String scard = tableExtDesc.getCardinality(); - if (!StringUtils.isEmpty(scard)) { - String[] cards = StringUtils.split(scard, ","); - ColumnDesc[] cdescs = rtableDesc.getColumns(); - for (int i = 0; i < cdescs.length; i++) { - ColumnDesc columnDesc = cdescs[i]; - if (cards.length > i) { - cardinality.put(columnDesc.getName(), Long.parseLong(cards[i])); - } else { - logger.error("The result cardinality is not identical with hive table metadata, cardinality : " + scard + " column array length: " + cdescs.length); - break; - } - } - rtableDesc.setCardinality(cardinality); - } - dataSourceProp.putAll(tableExtDesc.getDataSourceProp()); - dataSourceProp.put("location", tableExtDesc.getStorageLocation()); - dataSourceProp.put("owner", tableExtDesc.getOwner()); - dataSourceProp.put("last_access_time", tableExtDesc.getLastAccessTime()); - dataSourceProp.put("partition_column", tableExtDesc.getPartitionColumn()); - dataSourceProp.put("total_file_size", tableExtDesc.getTotalFileSize()); - rtableDesc.setDescExd(dataSourceProp); + TableDescResponse rtableDesc = cloneTableDesc(table); descs.add(rtableDesc); } http://git-wip-us.apache.org/repos/asf/kylin/blob/877b538a/webapp/app/js/controllers/sourceMeta.js ---------------------------------------------------------------------- diff --git a/webapp/app/js/controllers/sourceMeta.js b/webapp/app/js/controllers/sourceMeta.js index bbb9915..a53a35f 100755 --- a/webapp/app/js/controllers/sourceMeta.js +++ b/webapp/app/js/controllers/sourceMeta.js @@ -330,7 +330,7 @@ KylinApp } if ($scope.tableNames.trim() === "") { - SweetAlert.swal('', 'Please input table(s) you want to synchronize.', 'info'); + SweetAlert.swal('', 'Please input table(s) you want to load.', 'info'); return; } @@ -352,13 +352,13 @@ KylinApp }) if (result['result.unloaded'].length != 0 && result['result.loaded'].length == 0) { - SweetAlert.swal('Failed!', 'Failed to synchronize following table(s): ' + unloadedTableInfo, 'error'); + SweetAlert.swal('Failed!', 'Failed to load following table(s): ' + unloadedTableInfo, 'error'); } if (result['result.loaded'].length != 0 && result['result.unloaded'].length == 0) { - SweetAlert.swal('Success!', 'The following table(s) have been successfully synchronized: ' + loadTableInfo, 'success'); + SweetAlert.swal('Success!', 'The following table(s) have been successfully loaded: ' + loadTableInfo, 'success'); } if (result['result.loaded'].length != 0 && result['result.unloaded'].length != 0) { - SweetAlert.swal('Partial loaded!', 'The following table(s) have been successfully synchronized: ' + loadTableInfo + "\n\n Failed to synchronize following table(s):" + unloadedTableInfo, 'warning'); + SweetAlert.swal('Partial loaded!', 'The following table(s) have been successfully loaded: ' + loadTableInfo + "\n\n Failed to load following table(s):" + unloadedTableInfo, 'warning'); } loadingRequest.hide(); scope.aceSrcTbLoaded(true); @@ -378,7 +378,7 @@ KylinApp $scope.remove = function () { if ($scope.tableNames.trim() === "") { - SweetAlert.swal('', 'Please input table(s) you want to synchronize.', 'info'); + SweetAlert.swal('', 'Please input table(s) you want to unload.', 'info'); return; } @@ -400,13 +400,13 @@ KylinApp }) if (result['result.unload.fail'].length != 0 && result['result.unload.success'].length == 0) { - SweetAlert.swal('Failed!', 'Failed to synchronize following table(s): ' + unRemovedTableInfo, 'error'); + SweetAlert.swal('Failed!', 'Failed to unload following table(s): ' + unRemovedTableInfo, 'error'); } if (result['result.unload.success'].length != 0 && result['result.unload.fail'].length == 0) { - SweetAlert.swal('Success!', 'The following table(s) have been successfully synchronized: ' + removedTableInfo, 'success'); + SweetAlert.swal('Success!', 'The following table(s) have been successfully unloaded: ' + removedTableInfo, 'success'); } if (result['result.unload.success'].length != 0 && result['result.unload.fail'].length != 0) { - SweetAlert.swal('Partial unloaded!', 'The following table(s) have been successfully synchronized: ' + removedTableInfo + "\n\n Failed to synchronize following table(s):" + unRemovedTableInfo, 'warning'); + SweetAlert.swal('Partial unloaded!', 'The following table(s) have been successfully unloaded: ' + removedTableInfo + "\n\n Failed to unload following table(s):" + unRemovedTableInfo, 'warning'); } loadingRequest.hide(); scope.aceSrcTbLoaded(true); http://git-wip-us.apache.org/repos/asf/kylin/blob/877b538a/webapp/app/js/services/tables.js ---------------------------------------------------------------------- diff --git a/webapp/app/js/services/tables.js b/webapp/app/js/services/tables.js index 4199d6c..4e7a7c4 100755 --- a/webapp/app/js/services/tables.js +++ b/webapp/app/js/services/tables.js @@ -24,7 +24,6 @@ KylinApp.factory('TableService', ['$resource', function ($resource, config) { reload: {method: 'PUT', params: {action: 'reload'}, isArray: false}, loadHiveTable: {method: 'POST', params: {}, isArray: false}, unLoadHiveTable: {method: 'DELETE', params: {}, isArray: false}, - addStreamingSrc: {method: 'POST', params: {action:'addStreamingSrc'}, isArray: false}, genCardinality: {method: 'PUT', params: {action: 'cardinality'}, isArray: false}, showHiveDatabases: {method: 'GET', params: {action:'hive'}, cache: true, isArray: true}, showHiveTables: {method: 'GET', params: {action:'hive'}, cache: true, isArray: true}