This is an automated email from the ASF dual-hosted git repository. xxyu pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 97bf6b8bd557b10868855f90bd8f2d777bddfbf6 Author: Pengfei Zhan <pengfei.z...@kyligence.io> AuthorDate: Wed Oct 12 20:50:19 2022 +0800 KYLIN-5314 Check name conflict before export tds file --- .../org/apache/kylin/common/KylinConfigBase.java | 5 - .../common/exception/code/ErrorCodeServer.java | 2 + .../java/org/apache/kylin/common/msg/Message.java | 2 - .../resources/kylin_error_msg_conf_cn.properties | 2 + .../resources/kylin_error_msg_conf_en.properties | 2 + .../main/resources/kylin_errorcode_conf.properties | 2 + .../rest/controller/v2/NModelControllerV2.java | 1 - .../kylin/rest/service/AbstractModelService.java | 8 +- .../apache/kylin/rest/service/ModelService.java | 111 ------ .../kylin/rest/service/ModelServiceTest.java | 398 +-------------------- 10 files changed, 16 insertions(+), 517 deletions(-) diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index c1a6e84bb3..33f7f0b5f1 100644 --- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -3637,11 +3637,6 @@ public abstract class KylinConfigBase implements Serializable { return Boolean.parseBoolean(getOptional("kylin.build.resource.skip-resource-check", FALSE)); } - public boolean useTableIndexAnswerSelectStarEnabled() { - return Boolean.parseBoolean(getOptional("kylin.query.use-tableindex-answer-select-star.enabled", FALSE)); - } - - public int getSecondStorageSkippingIndexGranularity() { int granularity = Integer.parseInt(getOptional("kylin.second-storage.skipping-index.granularity", "3")); return granularity <= 0 ? 3 : granularity; diff --git a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java index 3a0ca0195d..c8abaa1142 100644 --- a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java +++ b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java @@ -32,6 +32,8 @@ public enum ErrorCodeServer implements ErrorCodeProducer { MODEL_NAME_DUPLICATE("KE-010002206"), SIMPLIFIED_MEASURES_MISSING_ID("KE-010002207"), MODEL_NOT_EXIST_SEGMENTS("KE-010002208"), + MODEL_TDS_EXPORT_DIM_COL_AND_MEASURE_NAME_CONFLICT("KE-010002301"), + MODEL_TDS_EXPORT_COLUMN_AND_MEASURE_NAME_CONFLICT("KE-010002302"), // 100252XX Cube CUBE_NOT_EXIST("KE-010025201"), diff --git a/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java b/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java index cad28ace11..d64b7179c1 100644 --- a/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java +++ b/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java @@ -38,8 +38,6 @@ public class Message { private static final String NON_EXISTEN_MODEL = "Model %s doesn't exist. Please confirm and try again later."; private static final String LACK_PROJECT = "Please fill in the project parameters."; private static final String NON_EXIST_PROJECT = "Project %s doesn't exist. Please confirm and try again later."; - private static final String DUP_MODCOL_MEASURE_NAME = "There are duplicated names among model column %s and measure name %s. Cannot export a valid TDS file. Please correct the duplicated names and try again."; - private static final String DUP_DIMCOL_MEASURE_NAME = "There are duplicated names among dimension column %s and measure name %s. Cannot export a valid TDS file. Please correct the duplicated names and try again."; private static final String MODIFY_PERMISSION_OF_SUPER_ADMIN = "Super Admin’s permission can’t be modified."; private static final String ILLEGAL_AUTHORIZING_USER = "Unable to modify. Only Super Admin or System Admin with query permission can modify query permission."; private static final String GRANT_TO_NON_SYSTEM_ADMIN = "Unable to modify. You can only modify the permission of the System Admin."; diff --git a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties index e6a11efcc1..759514ee40 100644 --- a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties +++ b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties @@ -32,6 +32,8 @@ KE-010002205=无效的模型名称 “%s”。请使用字母、数字或下划 KE-010002206=模型 “%s” 已存在。请重新命名。 KE-010002207=修改模型时,simplified_measures 参数中的每个度量必须传入 id 值。请为以下度量传入 id 之后重试:%s。 KE-010002208=模型上线必须存在 Segment。请重新输入。 +KE-010002301=维度的列名 %s 与度量名 %s 重复,无法导出 TDS。请去除重名后再重试。 +KE-010002302=模型中的列名 %s 与度量名 %s 重复,无法导出 TDS。请去除重名后再重试。 ## 100252XX Cube KE-010025201=无法找到相关 Cube。 diff --git a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties index b4130c7b2a..02ecccdb25 100644 --- a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties +++ b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties @@ -32,6 +32,8 @@ KE-010002205=The model name "%s" is invalid. Please use letters, numbers and und KE-010002206=Model "%s" already exists. Please rename it. KE-010002207=When modifying model, each measure id is required in simplified_measures parameter. Please pass ids for following measures and try again: %s. KE-010002208=The online model must have a segment. Please re-enter. +KE-010002301=There are duplicated names among dimension column %s and measure name %s. Cannot export a valid TDS file. Please correct the duplicated names and try again. +KE-010002302=There are duplicated names among model column %s and measure name %s. Cannot export a valid TDS file. Please correct the duplicated names and try again. ## 100252XX Cube KE-010025201=Can't find the cube. diff --git a/src/core-common/src/main/resources/kylin_errorcode_conf.properties b/src/core-common/src/main/resources/kylin_errorcode_conf.properties index 5b506d68fb..7f07828a6a 100644 --- a/src/core-common/src/main/resources/kylin_errorcode_conf.properties +++ b/src/core-common/src/main/resources/kylin_errorcode_conf.properties @@ -33,6 +33,8 @@ KE-010002205 KE-010002206 KE-010002207 KE-010002208 +KE-010002301 +KE-010002302 ## 100252XX Cube KE-010025201 diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NModelControllerV2.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NModelControllerV2.java index 224b68c6a8..ac9d46580f 100644 --- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NModelControllerV2.java +++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/v2/NModelControllerV2.java @@ -21,7 +21,6 @@ import static org.apache.kylin.common.constant.HttpConstant.HTTP_VND_APACHE_KYLI import static org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_ID_NOT_EXIST; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/AbstractModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/AbstractModelService.java index b46314b2e1..849214bd34 100644 --- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/AbstractModelService.java +++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/AbstractModelService.java @@ -57,7 +57,7 @@ public class AbstractModelService extends BasicService { @Autowired public AccessService accessService; - public void checkModelPermission(String project, String modelId) { + public final void checkModelPermission(String project, String modelId) { String userName = aclEvaluate.getCurrentUserName(); Set<String> groups = getCurrentUserGroups(); if (AclPermissionUtil.isAdmin() || AclPermissionUtil.isAdminInProject(project, groups)) { @@ -94,7 +94,7 @@ public class AbstractModelService extends BasicService { }); } - public NDataModel getModelById(String modelId, String project) { + public final NDataModel getModelById(String modelId, String project) { NDataModelManager modelManager = getManager(NDataModelManager.class, project); NDataModel nDataModel = modelManager.getDataModelDesc(modelId); if (null == nDataModel) { @@ -103,7 +103,7 @@ public class AbstractModelService extends BasicService { return nDataModel; } - public NDataModel getModelByAlias(String modelAlias, String project) { + public final NDataModel getModelByAlias(String modelAlias, String project) { NDataModelManager modelManager = getManager(NDataModelManager.class, project); NDataModel nDataModel = modelManager.getDataModelDescByAlias(modelAlias); if (null == nDataModel) { @@ -112,7 +112,7 @@ public class AbstractModelService extends BasicService { return nDataModel; } - public Set<String> listAllModelIdsInProject(String project) { + public final Set<String> listAllModelIdsInProject(String project) { NDataModelManager dataModelManager = getManager(NDataModelManager.class, project); return dataModelManager.listAllModelIds(); } diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java index b836937033..07cab15306 100644 --- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java +++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java @@ -4313,117 +4313,6 @@ public class ModelService extends AbstractModelService implements TableModelSupp return response; } - public void checkCCEmpty(ModelRequest modelRequest) { - List<ComputedColumnDesc> ccList = modelRequest.getComputedColumnDescs(); - if (CollectionUtils.isEmpty(ccList)) { - return; - } - boolean matchEmpty = ccList.stream() - .anyMatch(cc -> StringUtils.isEmpty(cc.getColumnName()) || StringUtils.isEmpty(cc.getExpression())); - if (matchEmpty) { - throw new KylinException(COMPUTED_COLUMN_NAME_OR_EXPR_EMPTY); - } - } - - public Pair<ModelRequest, ComputedColumnConflictResponse> checkCCConflict(ModelRequest modelRequest) { - String project = modelRequest.getProject(); - validatePartitionDateColumn(modelRequest); - - val dataModel = semanticUpdater.convertToDataModel(modelRequest); - val modelManager = getManager(NDataModelManager.class, project); - val ccRelatedModels = modelManager.getCCRelatedModels(dataModel); - // check cc conflict and return ccConflictInfo - val ccConflictInfo = dataModel.checkCCFailAtEnd(getConfig(), project, ccRelatedModels, true); - boolean autoAdjust = modelRequest.isComputedColumnNameAutoAdjust(); - - if (ccConflictInfo.noneConflict()) { - // No conflict, return - return Pair.newPair(modelRequest, new ComputedColumnConflictResponse()); - } - if (ccConflictInfo.hasSameNameConflict()) { - // have sameNameDiffExpr Conflict, all conflict messages need to be thrown - val response = handleOnConflictResponse(ccConflictInfo.getAllConflictException()); - throw new KylinException(COMPUTED_COLUMN_CONFLICT).withData(response); - } - // have sameExprDiffExprConflict Conflict - if (!autoAdjust) { - // AutoAdjust = false - // SameExpr conflict messages need to be thrown - val response = handleOnConflictResponse(ccConflictInfo.getSameExprConflictException()); - throw new KylinException(COMPUTED_COLUMN_CONFLICT).withData(response); - } - // AutoAdjust = true - List<ComputedColumnDesc> inputCCDescList = Lists.newArrayList(modelRequest.getComputedColumnDescs()); - // deal with conflicts - val pair = ccConflictInfo.getAdjustedCCList(inputCCDescList); - val adjustExceptions = pair.getSecond().stream() // - .map(ComputedColumnUtil.CCConflictDetail::getAdjustKylinException).collect(Collectors.toList()); - ModelRequest resultModelRequest = adjustModelRequestCCName(modelRequest, pair); - - return Pair.newPair(resultModelRequest, handleOnConflictResponse(adjustExceptions)); - } - - public ModelRequest adjustModelRequestCCName(ModelRequest modelRequest, - Pair<List<ComputedColumnDesc>, List<ComputedColumnUtil.CCConflictDetail>> pair) { - val adjustDetails = pair.getSecond(); - // adjust cc name - modelRequest.setComputedColumnDescs(pair.getFirst()); - - val dimensions = modelRequest.getSimplifiedDimensions(); - val measures = modelRequest.getSimplifiedMeasures(); - for (val detail : adjustDetails) { - String newCCFullName = detail.getNewCC().getFullName(); - String existingCCFullName = detail.getExistingCC().getFullName(); - - // adjust dimensions - dimensions.stream() // - .filter(NDataModel.NamedColumn::isExist) // - // column equals - .filter(d -> StringUtils.equalsIgnoreCase(d.getAliasDotColumn(), newCCFullName)) - .forEach(d -> d.setAliasDotColumn(existingCCFullName)); - - // adjust measures - measures.forEach(m -> m.getParameterValue().stream() // - // type = column - .filter(pr -> StringUtils.equalsIgnoreCase(pr.getType(), PARAMETER_TYPE_COLUMN)) - // value equals - .filter(pr -> StringUtils.equalsIgnoreCase(pr.getValue(), newCCFullName)) - .forEach(pr -> pr.setValue(existingCCFullName))); - } - - // adjust filter condition - String filterCondition = modelRequest.getFilterCondition(); - if (StringUtils.isEmpty(filterCondition)) { - return modelRequest; - } - for (val detail : adjustDetails) { - String newCCFullName = detail.getNewCC().getFullName(); - String existingCCFullName = detail.getExistingCC().getFullName(); - if (StringUtils.containsIgnoreCase(filterCondition, newCCFullName)) { - filterCondition = replaceAllIgnoreCase(filterCondition, newCCFullName, existingCCFullName); - } - } - modelRequest.setFilterCondition(filterCondition); - return modelRequest; - } - - public String replaceAllIgnoreCase(String input, String regex, String replacement) { - return Pattern.compile(regex, Pattern.CASE_INSENSITIVE).matcher(input).replaceAll(replacement); - } - - public ComputedColumnConflictResponse handleOnConflictResponse(List<KylinException> exceptionList) { - val response = new ComputedColumnConflictResponse(); - exceptionList.stream() // - .filter(Objects::nonNull) // - .forEach(e -> { - val producer = e.getErrorCodeProducer(); - val code = producer.getErrorCode().getCode(); - val msg = producer.getMsg(e.getArgs()); - response.addConflictDetail(code, msg); - }); - return response; - } - @Override public void onUpdateBrokenModel(NDataModel model, AffectedModelContext removeAffectedModel, AffectedModelContext changeTypeAffectedModel, String projectName) throws Exception { diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java index 696347e093..e9dc0f22c3 100644 --- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java +++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java @@ -522,8 +522,7 @@ public class ModelServiceTest extends SourceTestCase { Assert.assertEquals(1, models.size()); NDataModelResponse model = models.get(0); Assert.assertTrue(model.getSimpleTables().stream().map(SimplifiedTableResponse::getColumns) - .flatMap(List::stream) - .anyMatch(SimplifiedColumnResponse::isComputedColumn)); + .flatMap(List::stream).anyMatch(SimplifiedColumnResponse::isComputedColumn)); } @Test @@ -564,8 +563,7 @@ public class ModelServiceTest extends SourceTestCase { NIndexPlanManager indexPlanManager = NIndexPlanManager.getInstance(getTestConfig(), getProject()); val indexPlan = indexPlanManager.getIndexPlan(modelId); indexPlanManager.updateIndexPlan(modelId, copyForWrite -> { - copyForWrite.markIndexesToBeDeleted(modelId, - new HashSet<>(indexPlan.getAllLayouts())); + copyForWrite.markIndexesToBeDeleted(modelId, new HashSet<>(indexPlan.getAllLayouts())); copyForWrite.getIndexes().clear(); }); NDataflowManager dataflowManager = NDataflowManager.getInstance(KylinConfig.getInstanceFromEnv(), "default"); @@ -5083,394 +5081,6 @@ public class ModelServiceTest extends SourceTestCase { } } - @Test - public void testExportTDSByAdmin() throws Exception { - val project = "default"; - val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c"; - prepareBasic(project); - List<String> dimensions = Lists.newArrayList(); - dimensions.add("DEFAULT.TEST_MEASURE.FLAG"); - dimensions.add("DEFAULT.TEST_MEASURE.PRICE1"); - dimensions.add("DEFAULT.TEST_MEASURE.ID1"); - List<String> measurs = Lists.newArrayList(); - measurs.add("COUNT_STAR"); - measurs.add("SUM_1"); - SyncContext syncContext = modelService.getADMINSyncContext(project, modelId, - SyncContext.BI.TABLEAU_CONNECTOR_TDS, SyncContext.ModelElement.CUSTOM_COLS, "localhost", 8080); - TableauDatasourceModel datasource1 = (TableauDatasourceModel) modelService - .exportTDSDimensionsAndMeasuresByAdmin(syncContext, dimensions, measurs); - ByteArrayOutputStream outStream4 = new ByteArrayOutputStream(); - datasource1.dump(outStream4); - Assert.assertEquals(getExpectedTds("/bisync_tableau/nmodel_full_measure_test.connector_admin.tds"), - outStream4.toString(Charset.defaultCharset().name())); - } - - @Test - public void testExportTDSByUser() throws Exception { - val project = "default"; - val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c"; - prepareBasic(project); - List<String> dimensions = Lists.newArrayList(); - dimensions.add("TEST_MEASURE.ID1"); - dimensions.add("TEST_MEASURE.ID2"); - dimensions.add("TEST_MEASURE.ID3"); - dimensions.add("TEST_MEASURE1.ID1"); - dimensions.add("TEST_MEASURE1.NAME1"); - dimensions.add("TEST_MEASURE1.NAME2"); - dimensions.add("TEST_MEASURE1.NAME3"); - List<String> measurs = Lists.newArrayList(); - measurs.add("COUNT_STAR"); - measurs.add("SUM_1"); - SecurityContextHolder.getContext() - .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST)); - SyncContext syncContext = modelService.getSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS, - SyncContext.ModelElement.CUSTOM_COLS, "localhost", 8080); - TableauDatasourceModel datasource1 = (TableauDatasourceModel) modelService - .exportTDSDimensionsAndMeasuresByNormalUser(syncContext, dimensions, measurs); - ByteArrayOutputStream outStream4 = new ByteArrayOutputStream(); - datasource1.dump(outStream4); - Assert.assertEquals(getExpectedTds("/bisync_tableau/nmodel_full_measure_test.connector_user.tds"), - outStream4.toString(Charset.defaultCharset().name())); - } - - @Test - public void testExportTDSByUserAndElement() throws Exception { - val project = "default"; - val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c"; - prepareBasic(project); - try { - SecurityContextHolder.getContext() - .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST)); - SyncContext syncContext = modelService.getSyncContext(project, modelId, - SyncContext.BI.TABLEAU_CONNECTOR_TDS, SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080); - TableauDatasourceModel datasource1 = (TableauDatasourceModel) modelService - .exportTDSDimensionsAndMeasuresByNormalUser(syncContext, null, null); - ByteArrayOutputStream outStream4 = new ByteArrayOutputStream(); - datasource1.dump(outStream4); - Assert.assertEquals( - getExpectedTds("/bisync_tableau/nmodel_full_measure_test.connector_user_agg_index_col.tds"), - outStream4.toString(Charset.defaultCharset().name())); - - TableauDatasourceModel datasource = (TableauDatasourceModel) modelService - .exportTDSDimensionsAndMeasuresByNormalUser(syncContext, new ArrayList<>(), new ArrayList<>()); - } finally { - SecurityContextHolder.getContext() - .setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN)); - } - } - - @Test - public void testCheckModelExportPermissionException() { - val project = "default"; - val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c"; - prepareBasic(project); - modelService.getADMINSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS, - SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080); - try { - Mockito.when(accessService.getGroupsOfExecuteUser(Mockito.any(String.class))) - .thenReturn(Sets.newHashSet("ROLE_ANALYST")); - SecurityContextHolder.getContext() - .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST)); - thrown.expect(KylinException.class); - thrown.expectMessage("current user does not have full permission on requesting model"); - modelService.getADMINSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS, - SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080); - } finally { - SecurityContextHolder.getContext() - .setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN)); - } - } - - @Test - public void testCheckModelExportPermission() { - val project = "default"; - val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c"; - prepareBasic(project); - modelService.getADMINSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS, - SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080); - modelService.getADMINSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS, - SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080); - } - - @Test - public void testCheckModelExportPermissionWithCC() { - val project = "cc_test"; - val modelId = "0d146f1a-bdd3-4548-87ac-21c2c6f9a0da"; - AclTCRManager manager = AclTCRManager.getInstance(getTestConfig(), project); - { - AclTCR u1a1 = new AclTCR(); - manager.updateAclTCR(u1a1, "u1", true); - SecurityContextHolder.getContext() - .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST)); - Mockito.when(accessService.getGroupsOfExecuteUser(Mockito.any(String.class))) - .thenReturn(Sets.newHashSet("ROLE_ANALYST")); - modelService.getADMINSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS, - SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080); - } - { - try { - AclTCR u1a1 = new AclTCR(); - AclTCR.Table u1t1 = new AclTCR.Table(); - AclTCR.ColumnRow u1cr1 = new AclTCR.ColumnRow(); - AclTCR.Column u1c1 = new AclTCR.Column(); - u1c1.add("ORDER_ID"); - u1cr1.setColumn(u1c1); - u1t1.put("SSB.LINEORDER", u1cr1); - u1a1.setTable(u1t1); - manager.updateAclTCR(u1a1, "u1", true); - thrown.expect(KylinException.class); - thrown.expectMessage("current user does not have full permission on requesting model"); - modelService.getADMINSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS, - SyncContext.ModelElement.AGG_INDEX_COL, "localhost", 8080); - } finally { - SecurityContextHolder.getContext() - .setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN)); - } - } - - } - - @Test - public void testExportTDSByBroken() { - val project = "test_broken_project"; - val modelId = "4b93b131-824e-6966-c4dd-5a4268d27095"; - List<String> dimensions = Lists.newArrayList(); - List<String> measurs = Lists.newArrayList(); - SyncContext syncContext = modelService.getSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS, - SyncContext.ModelElement.CUSTOM_COLS, "localhost", 8080); - Assert.assertThrows(KylinException.class, - () -> modelService.exportTDSDimensionsAndMeasuresByNormalUser(syncContext, dimensions, measurs)); - - Assert.assertThrows(KylinException.class, - () -> modelService.exportTDSDimensionsAndMeasuresByAdmin(syncContext, dimensions, measurs)); - } - - @Test - public void testExportTDSMeasurePermission() { - val project = "default"; - val modelId = "82fa7671-a935-45f5-8779-85703601f49a"; - prepareBasicByMeasure(project); - List<String> dimensions = Lists.newArrayList(); - //"ORDER_ID", "PRICE", "CAL_DT", "PRICE", "ITEM_COUNT", "LEAF_CATEG_ID" - dimensions.add("TEST_KYLIN_FACT.ORDER_ID"); - dimensions.add("TEST_KYLIN_FACT.PRICE"); - dimensions.add("TEST_KYLIN_FACT.CAL_DT"); - dimensions.add("TEST_KYLIN_FACT.PRICE"); - dimensions.add("TEST_KYLIN_FACT.ITEM_COUNT"); - dimensions.add("TEST_KYLIN_FACT.LEAF_CATEG_ID"); - //"ORDER_ID", "TEST_TIME_ENC", "TEST_DATE_ENC" - dimensions.add("TEST_ORDER.ORDER_ID"); - dimensions.add("TEST_ORDER.TEST_TIME_ENC"); - dimensions.add("TEST_ORDER.TEST_DATE_ENC"); - //"ORDER_ID", "PRICE", "CAL_DT", "TRANS_ID" - dimensions.add("TEST_MEASURE.ORDER_ID"); - dimensions.add("TEST_MEASURE.PRICE"); - dimensions.add("TEST_MEASURE.CAL_DT"); - dimensions.add("TEST_MEASURE.TRANS_ID"); - - List<String> measures = Lists.newArrayList(); - measures.add("TRANS_CNT"); - measures.add("GMV_SUM"); - measures.add("GMV_MIN"); - measures.add("GMV_MAX"); - measures.add("ITEM_COUNT_SUM"); - measures.add("ITEM_COUNT_MAX"); - measures.add("ITEM_COUNT_MIN"); - measures.add("SELLER_HLL"); - measures.add("COUNT_DISTINCT"); - measures.add("TOP_SELLER"); - measures.add("TEST_COUNT_DISTINCT_BITMAP"); - measures.add("GVM_PERCENTILE"); - SecurityContextHolder.getContext() - .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST)); - SyncContext syncContext = modelService.getSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS, - SyncContext.ModelElement.CUSTOM_COLS, "localhost", 8080); - Assert.assertThrows(KylinException.class, - () -> modelService.exportTDSDimensionsAndMeasuresByNormalUser(syncContext, dimensions, measures)); - } - - private void prepareBasicByMeasure(String project) { - AclTCRManager manager = AclTCRManager.getInstance(getTestConfig(), project); - - AclTCR u1a1 = new AclTCR(); - AclTCR.Table u1t1 = new AclTCR.Table(); - AclTCR.ColumnRow u1cr1 = new AclTCR.ColumnRow(); - AclTCR.Column u1c1 = new AclTCR.Column(); - u1c1.addAll(Arrays.asList("ORDER_ID", "PRICE", "CAL_DT", "PRICE", "ITEM_COUNT", "LEAF_CATEG_ID")); - u1cr1.setColumn(u1c1); - - AclTCR.ColumnRow u1cr2 = new AclTCR.ColumnRow(); - AclTCR.Column u1c2 = new AclTCR.Column(); - u1c2.addAll(Arrays.asList("ORDER_ID", "TEST_TIME_ENC", "TEST_DATE_ENC")); - u1cr2.setColumn(u1c2); - u1t1.put("DEFAULT.TEST_KYLIN_FACT", u1cr1); - u1t1.put("DEFAULT.TEST_ORDER", u1cr2); - u1a1.setTable(u1t1); - manager.updateAclTCR(u1a1, "u1", true); - } - - @Test - public void testExportModel() throws Exception { - val project = "default"; - val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c"; - prepareBasic(project); - TableauDatasourceModel datasource1 = (TableauDatasourceModel) modelService.exportModel(project, modelId, - SyncContext.BI.TABLEAU_CONNECTOR_TDS, SyncContext.ModelElement.AGG_INDEX_AND_TABLE_INDEX_COL, - "localhost", 8080); - ByteArrayOutputStream outStream4 = new ByteArrayOutputStream(); - datasource1.dump(outStream4); - Assert.assertEquals(getExpectedTds("/bisync_tableau/nmodel_full_measure_test.connector.tds"), - outStream4.toString(Charset.defaultCharset().name())); - } - - private String getExpectedTds(String path) throws IOException { - return CharStreams.toString(new InputStreamReader(getClass().getResourceAsStream(path), Charsets.UTF_8)); - } - - private void prepareBasic(String project) { - AclTCRManager manager = AclTCRManager.getInstance(getTestConfig(), project); - - AclTCR u1a1 = new AclTCR(); - AclTCR.Table u1t1 = new AclTCR.Table(); - AclTCR.ColumnRow u1cr1 = new AclTCR.ColumnRow(); - AclTCR.Column u1c1 = new AclTCR.Column(); - u1c1.addAll(Arrays.asList("ID1", "ID2", "ID3")); - u1cr1.setColumn(u1c1); - - AclTCR.ColumnRow u1cr2 = new AclTCR.ColumnRow(); - AclTCR.Column u1c2 = new AclTCR.Column(); - u1c2.addAll(Arrays.asList("ID1", "NAME1", "NAME2", "NAME3")); - u1cr2.setColumn(u1c2); - u1t1.put("DEFAULT.TEST_MEASURE", u1cr1); - u1t1.put("DEFAULT.TEST_MEASURE1", u1cr2); - u1a1.setTable(u1t1); - manager.updateAclTCR(u1a1, "u1", true); - - AclTCR g1a1 = new AclTCR(); - AclTCR.Table g1t1 = new AclTCR.Table(); - AclTCR.ColumnRow g1cr1 = new AclTCR.ColumnRow(); - AclTCR.Column g1c1 = new AclTCR.Column(); - g1c1.addAll(Arrays.asList("ID1", "ID2", "ID3", "ID4")); - g1cr1.setColumn(g1c1); - g1t1.put("DEFAULT.TEST_MEASURE", g1cr1); - g1a1.setTable(g1t1); - manager.updateAclTCR(g1a1, "g1", false); - } - - @Test - public void testCheckTablePermission() { - val project = "default"; - val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c"; - thrown.expect(KylinException.class); - thrown.expectMessage(MsgPicker.getMsg().getTableNoColumnsPermission()); - - AclTCRManager manager = AclTCRManager.getInstance(getTestConfig(), project); - Set<String> columns = new HashSet<>(); - columns.add("DEFAULT.TEST_MEASURE1.NAME1"); - columns.add("DEFAULT.TEST_MEASURE1.NAME2"); - columns.add("DEFAULT.TEST_MEASURE1.NAME3"); - - AclTCR u1a1 = new AclTCR(); - AclTCR.Table u1t1 = new AclTCR.Table(); - AclTCR.ColumnRow u1cr1 = new AclTCR.ColumnRow(); - AclTCR.Column u1c1 = new AclTCR.Column(); - u1cr1.setColumn(u1c1); - - AclTCR.ColumnRow u1cr2 = new AclTCR.ColumnRow(); - AclTCR.Column u1c2 = new AclTCR.Column(); - u1c2.addAll(Arrays.asList("NAME1", "NAME2", "NAME3")); - u1cr2.setColumn(u1c2); - u1t1.put("DEFAULT.TEST_MEASURE", u1cr1); - u1t1.put("DEFAULT.TEST_MEASURE1", u1cr2); - u1a1.setTable(u1t1); - manager.updateAclTCR(u1a1, "u1", true); - SecurityContextHolder.getContext() - .setAuthentication(new TestingAuthenticationToken("u1", "ANALYST", Constant.ROLE_ANALYST)); - List<String> dimensions = Lists.newArrayList(); - dimensions.add("TEST_MEASURE.FLAG"); - dimensions.add("TEST_MEASURE.PRICE1"); - dimensions.add("TEST_MEASURE.ID1"); - List<String> measurs = Lists.newArrayList(); - measurs.add("COUNT_STAR"); - measurs.add("SUM_1"); - modelService.checkTableHasColumnPermission(SyncContext.ModelElement.CUSTOM_COLS, project, modelId, columns, - dimensions, measurs); - - dimensions.add("TEST_MEASURE.ID4"); - Assert.assertThrows(KylinException.class, - () -> modelService.checkTableHasColumnPermission(SyncContext.ModelElement.CUSTOM_COLS, project, modelId, - columns, dimensions, measurs)); - } - - @Test - public void testExportTDSCheckColumnPermission() { - val project = "default"; - val modelId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa"; - - NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project); - NDataModel dataModel = modelManager.getDataModelDesc(modelId); - - Set<String> authColumns = Sets.newHashSet(); - List<String> dimensions = Lists.newArrayList(); - List<String> measurs = Lists.newArrayList(); - - Assert.assertTrue(modelService.checkColumnPermission(dataModel, authColumns, null, measurs)); - Assert.assertTrue(modelService.checkColumnPermission(dataModel, authColumns, null, null)); - Assert.assertTrue(modelService.checkColumnPermission(dataModel, authColumns, dimensions, null)); - Assert.assertTrue(modelService.checkColumnPermission(dataModel, authColumns, dimensions, measurs)); - - authColumns.add("DEFAULT.TEST_KYLIN_FACT.PRICE"); - authColumns.add("DEFAULT.TEST_KYLIN_FACT.ITEM_COUNT"); - authColumns.add("EDW.TEST_CAL_DT.CAL_DT"); - authColumns.add("DEFAULT.TEST_ACCOUNT.ACCOUNT_ID"); - - Set<String> newAuthColumns = Sets.newHashSet(); - dataModel.getAllTables().forEach(tableRef -> { - List<TblColRef> collect = tableRef.getColumns().stream() - .filter(column -> authColumns.contains(column.getCanonicalName())).collect(Collectors.toList()); - collect.forEach(x -> newAuthColumns.add(x.getAliasDotName())); - }); - - dimensions.add("TEST_KYLIN_FACT.DEAL_AMOUNT"); - dimensions.add("TEST_KYLIN_FACT.TRANS_ID"); - - Assert.assertFalse(modelService.checkColumnPermission(dataModel, newAuthColumns, dimensions, measurs)); - - newAuthColumns.add("TEST_KYLIN_FACT.TRANS_ID"); - - measurs.add("SUM_NEST4"); - measurs.add("COUNT_CAL_DT"); - Assert.assertTrue(modelService.checkColumnPermission(dataModel, newAuthColumns, dimensions, measurs)); - - Assert.assertTrue(modelService.checkColumnPermission(dataModel, newAuthColumns, dimensions, measurs)); - - } - - @Test - public void testConvertCCToNormalCols() { - val project = "default"; - val modelId = "89af4ee2-2cdb-4b07-b39e-4c29856309aa"; - NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project); - NDataModel dataModel = modelManager.getDataModelDesc(modelId); - NDataModel.Measure measure = dataModel.getEffectiveMeasures().values().stream() - .filter(x -> x.getName().equals("SUM_NEST4")).findFirst().get(); - Set<String> measureColumns = measure.getFunction().getParameters().stream() - .filter(parameterDesc -> parameterDesc.getColRef() != null) - .map(parameterDesc -> parameterDesc.getColRef().getCanonicalName()).collect(Collectors.toSet()); - ComputedColumnDesc sumNest4 = dataModel.getComputedColumnDescs().stream() - .filter(x -> measureColumns.contains(x.getIdentName())).findFirst().get(); - Set<String> strings = modelService.convertCCToNormalCols(dataModel, sumNest4); - Assert.assertEquals("TEST_KYLIN_FACT.PRICE, TEST_KYLIN_FACT.ITEM_COUNT", String.join(", ", strings)); - - sumNest4.setInnerExpression("1 + 2"); - Set<String> set = modelService.convertCCToNormalCols(dataModel, sumNest4); - Assert.assertEquals(Collections.emptySet(), set); - - HashSet<Object> authColumns = Sets.newHashSet(); - authColumns.add("DEFAULT.TEST_KYLIN_FACT.PRICE"); - Assert.assertTrue(authColumns.containsAll(set)); - } - @Test public void testBuildExceptionMessage() { NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), "default"); @@ -5507,8 +5117,8 @@ public class ModelServiceTest extends SourceTestCase { public void testBuildDuplicateCCException() { Set<String> set = Sets.newHashSet("test"); Assert.assertThrows("The computed column name \"test\" has been used in the current model. Please rename it.\n", - KylinException.class, () -> ReflectionTestUtils.invokeMethod(modelService, "buildDuplicateCCException", - set)); + KylinException.class, + () -> ReflectionTestUtils.invokeMethod(modelService, "buildDuplicateCCException", set)); } @Test