http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json index 221ffb6..cbf7a76 100644 --- a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json +++ b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json @@ -1,455 +1,627 @@ { - "uuid" : "629ab7a8-3929-4dff-b59d-2100aadccd1a", - "name" : "ci_left_join_cube", - "model_name" : "ci_left_join_model", - "description" : null, - "dimensions" : [ { - "name" : "CAL_DT", - "table" : "TEST_CAL_DT", - "column" : "{FK}", - "derived" : [ "WEEK_BEG_DT" ] - }, { - "name" : "ORDER_ID", - "table" : "TEST_KYLIN_FACT", - "column" : "ORDER_ID" - }, { - "name" : "TEST_DATE_ENC", - "table" : "TEST_ORDER", - "column" : "TEST_DATE_ENC" - }, { - "name" : "TEST_TIME_ENC", - "table" : "TEST_ORDER", - "column" : "TEST_TIME_ENC" - }, { - "name" : "CATEGORY", - "table" : "TEST_CATEGORY_GROUPINGS", - "column" : "{FK}", - "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ] - }, { - "name" : "CATEGORY_HIERARCHY", - "table" : "TEST_CATEGORY_GROUPINGS", - "column" : "META_CATEG_NAME", - "derived" : null - }, { - "name" : "CATEGORY_HIERARCHY", - "table" : "TEST_CATEGORY_GROUPINGS", - "column" : "CATEG_LVL2_NAME", - "derived" : null - }, { - "name" : "CATEGORY_HIERARCHY", - "table" : "TEST_CATEGORY_GROUPINGS", - "column" : "CATEG_LVL3_NAME", - "derived" : null - }, { - "name" : "LSTG_FORMAT_NAME", - "table" : "TEST_KYLIN_FACT", - "column" : "LSTG_FORMAT_NAME", - "derived" : null - }, { - "name" : "SITE_ID", - "table" : "TEST_SITES", - "column" : "{FK}", - "derived" : [ "SITE_NAME", "CRE_USER" ] - }, { - "name" : "SELLER_TYPE_CD", - "table" : "TEST_SELLER_TYPE_DIM", - "column" : "{FK}", - "derived" : [ "SELLER_TYPE_DESC" ] - }, { - "name" : "SELLER_ID", - "table" : "TEST_KYLIN_FACT", - "column" : "SELLER_ID" - }, { - "name" : "SELLER_BUYER_LEVEL", - "table" : "SELLER_ACCOUNT", - "column" : "ACCOUNT_BUYER_LEVEL" - }, { - "name" : "SELLER_SELLER_LEVEL", - "table" : "SELLER_ACCOUNT", - "column" : "ACCOUNT_SELLER_LEVEL" - }, { - "name" : "SELLER_COUNTRY", - "table" : "SELLER_ACCOUNT", - "column" : "ACCOUNT_COUNTRY" - }, { - "name" : "SELLER_COUNTRY_NAME", - "table" : "SELLER_COUNTRY", - "column" : "NAME" - }, { - "name" : "BUYER_ID", - "table" : "TEST_ORDER", - "column" : "BUYER_ID" - }, { - "name" : "BUYER_BUYER_LEVEL", - "table" : "BUYER_ACCOUNT", - "column" : "ACCOUNT_BUYER_LEVEL" - }, { - "name" : "BUYER_SELLER_LEVEL", - "table" : "BUYER_ACCOUNT", - "column" : "ACCOUNT_SELLER_LEVEL" - }, { - "name" : "BUYER_COUNTRY", - "table" : "BUYER_ACCOUNT", - "column" : "ACCOUNT_COUNTRY" - }, { - "name" : "BUYER_COUNTRY_NAME", - "table" : "BUYER_COUNTRY", - "column" : "NAME" - } ], - "measures" : [ { - "name" : "TRANS_CNT", - "function" : { - "expression" : "COUNT", - "parameter" : { - "type" : "constant", - "value" : "1" - }, - "returntype" : "bigint" + "uuid": "629ab7a8-3929-4dff-b59d-2100aadccd1a", + "name": "ci_left_join_cube", + "model_name": "ci_left_join_model", + "description": null, + "dimensions": [ + { + "name": "CAL_DT", + "table": "TEST_CAL_DT", + "column": "{FK}", + "derived": [ + "WEEK_BEG_DT" + ] + }, + { + "name": "ORDER_ID", + "table": "TEST_KYLIN_FACT", + "column": "ORDER_ID" + }, + { + "name": "TEST_DATE_ENC", + "table": "TEST_ORDER", + "column": "TEST_DATE_ENC" + }, + { + "name": "TEST_TIME_ENC", + "table": "TEST_ORDER", + "column": "TEST_TIME_ENC" + }, + { + "name": "CATEGORY", + "table": "TEST_CATEGORY_GROUPINGS", + "column": "{FK}", + "derived": [ + "USER_DEFINED_FIELD1", + "USER_DEFINED_FIELD3", + "UPD_DATE", + "UPD_USER" + ] + }, + { + "name": "CATEGORY_HIERARCHY", + "table": "TEST_CATEGORY_GROUPINGS", + "column": "META_CATEG_NAME", + "derived": null + }, + { + "name": "CATEGORY_HIERARCHY", + "table": "TEST_CATEGORY_GROUPINGS", + "column": "CATEG_LVL2_NAME", + "derived": null + }, + { + "name": "CATEGORY_HIERARCHY", + "table": "TEST_CATEGORY_GROUPINGS", + "column": "CATEG_LVL3_NAME", + "derived": null + }, + { + "name": "LSTG_FORMAT_NAME", + "table": "TEST_KYLIN_FACT", + "column": "LSTG_FORMAT_NAME", + "derived": null + }, + { + "name": "SITE_ID", + "table": "TEST_SITES", + "column": "{FK}", + "derived": [ + "SITE_NAME", + "CRE_USER" + ] + }, + { + "name": "SELLER_TYPE_CD", + "table": "TEST_SELLER_TYPE_DIM", + "column": "{FK}", + "derived": [ + "SELLER_TYPE_DESC" + ] + }, + { + "name": "SELLER_ID", + "table": "TEST_KYLIN_FACT", + "column": "SELLER_ID" + }, + { + "name": "SELLER_BUYER_LEVEL", + "table": "SELLER_ACCOUNT", + "column": "ACCOUNT_BUYER_LEVEL" + }, + { + "name": "SELLER_SELLER_LEVEL", + "table": "SELLER_ACCOUNT", + "column": "ACCOUNT_SELLER_LEVEL" + }, + { + "name": "SELLER_COUNTRY", + "table": "SELLER_ACCOUNT", + "column": "ACCOUNT_COUNTRY" + }, + { + "name": "SELLER_COUNTRY_NAME", + "table": "SELLER_COUNTRY", + "column": "NAME" + }, + { + "name": "BUYER_ID", + "table": "TEST_ORDER", + "column": "BUYER_ID" + }, + { + "name": "BUYER_BUYER_LEVEL", + "table": "BUYER_ACCOUNT", + "column": "ACCOUNT_BUYER_LEVEL" + }, + { + "name": "BUYER_SELLER_LEVEL", + "table": "BUYER_ACCOUNT", + "column": "ACCOUNT_SELLER_LEVEL" + }, + { + "name": "BUYER_COUNTRY", + "table": "BUYER_ACCOUNT", + "column": "ACCOUNT_COUNTRY" + }, + { + "name": "DEAL_YEAR", + "table": "TEST_KYLIN_FACT", + "column": "DEAL_YEAR" + }, + { + "name": "SELLER_COUNTRY_ABBR", + "table": "SELLER_ACCOUNT", + "column": "COUNTRY_ABBR" + }, + { + "name": "BUYER_COUNTRY_ABBR", + "table": "BUYER_ACCOUNT", + "column": "COUNTRY_ABBR" + }, + { + "name": "BUYER_COUNTRY_NAME", + "table": "BUYER_COUNTRY", + "column": "NAME" } - }, { - "name" : "ITEM_COUNT_SUM", - "function" : { - "expression" : "SUM", - "parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.ITEM_COUNT" - }, - "returntype" : "bigint" + ], + "measures": [ + { + "name": "TRANS_CNT", + "function": { + "expression": "COUNT", + "parameter": { + "type": "constant", + "value": "1" + }, + "returntype": "bigint" + } + }, + { + "name": "ITEM_COUNT_SUM", + "function": { + "expression": "SUM", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.ITEM_COUNT" + }, + "returntype": "bigint" + } + }, + { + "name": "GMV_SUM", + "function": { + "expression": "SUM", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.PRICE" + }, + "returntype": "decimal(19,4)" + } + }, + { + "name": "GMV_MIN", + "function": { + "expression": "MIN", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.PRICE" + }, + "returntype": "decimal(19,4)" + } + }, + { + "name": "GMV_MAX", + "function": { + "expression": "MAX", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.PRICE" + }, + "returntype": "decimal(19,4)" + } + }, + { + "name": "SELLER_HLL", + "function": { + "expression": "COUNT_DISTINCT", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.SELLER_ID" + }, + "returntype": "hllc(10)" + } + }, + { + "name": "SELLER_FORMAT_HLL", + "function": { + "expression": "COUNT_DISTINCT", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "next_parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.SELLER_ID" + } + }, + "returntype": "hllc(10)" + } + }, + { + "name": "TOP_SELLER", + "function": { + "expression": "TOP_N", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.PRICE", + "next_parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.SELLER_ID" + } + }, + "returntype": "topn(100, 4)", + "configuration": { + "topn.encoding.TEST_KYLIN_FACT.SELLER_ID": "int:4" + } + } + }, + { + "name": "TEST_COUNT_DISTINCT_BITMAP", + "function": { + "expression": "COUNT_DISTINCT", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP" + }, + "returntype": "bitmap" + } + }, + { + "name": "TEST_EXTENDED_COLUMN", + "function": { + "expression": "EXTENDED_COLUMN", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.ORDER_ID", + "next_parameter": { + "type": "column", + "value": "TEST_ORDER.TEST_EXTENDED_COLUMN" + } + }, + "returntype": "extendedcolumn(100)" + } + }, + { + "name": "BUYER_CONTACT", + "function": { + "expression": "EXTENDED_COLUMN", + "parameter": { + "type": "column", + "value": "TEST_ORDER.BUYER_ID", + "next_parameter": { + "type": "column", + "value": "BUYER_ACCOUNT.ACCOUNT_CONTACT" + } + }, + "returntype": "extendedcolumn(100)" + } + }, + { + "name": "SELLER_CONTACT", + "function": { + "expression": "EXTENDED_COLUMN", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.SELLER_ID", + "next_parameter": { + "type": "column", + "value": "SELLER_ACCOUNT.ACCOUNT_CONTACT" + } + }, + "returntype": "extendedcolumn(100)" + } + }, + { + "name": "TRANS_ID_RAW", + "function": { + "expression": "RAW", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.TRANS_ID" + }, + "returntype": "raw" + } + }, + { + "name": "PRICE_RAW", + "function": { + "expression": "RAW", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.PRICE" + }, + "returntype": "raw" + } + }, + { + "name": "CAL_DT_RAW", + "function": { + "expression": "RAW", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.CAL_DT" + }, + "returntype": "raw" + } + }, + { + "name": "GVM_PERCENTILE", + "function": { + "expression": "PERCENTILE", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.PRICE" + }, + "returntype": "percentile(100)" + } + }, + { + "name": "COMPUTED_COLUMN_MEASURE", + "function": { + "expression": "SUM", + "parameter": { + "type": "column", + "value": "TEST_KYLIN_FACT.DEAL_AMOUNT" + }, + "returntype": "decimal" + } } - }, { - "name" : "GMV_SUM", - "function" : { - "expression" : "SUM", - "parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.PRICE" - }, - "returntype" : "decimal(19,4)" + ], + "dictionaries": [ + { + "column": "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP", + "builder": "org.apache.kylin.dict.GlobalDictionaryBuilder" } - }, { - "name" : "GMV_MIN", - "function" : { - "expression" : "MIN", - "parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.PRICE" + ], + "rowkey": { + "rowkey_columns": [ + { + "column": "TEST_KYLIN_FACT.SELLER_ID", + "encoding": "int:4" }, - "returntype" : "decimal(19,4)" - } - }, { - "name" : "GMV_MAX", - "function" : { - "expression" : "MAX", - "parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.PRICE" + { + "column": "TEST_KYLIN_FACT.ORDER_ID", + "encoding": "int:4" }, - "returntype" : "decimal(19,4)" - } - }, { - "name" : "SELLER_HLL", - "function" : { - "expression" : "COUNT_DISTINCT", - "parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.SELLER_ID" + { + "column": "TEST_KYLIN_FACT.CAL_DT", + "encoding": "dict" }, - "returntype" : "hllc(10)" - } - }, { - "name" : "SELLER_FORMAT_HLL", - "function" : { - "expression" : "COUNT_DISTINCT", - "parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", - "next_parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.SELLER_ID" - } + { + "column": "TEST_KYLIN_FACT.LEAF_CATEG_ID", + "encoding": "dict" }, - "returntype" : "hllc(10)" - } - }, { - "name" : "TOP_SELLER", - "function" : { - "expression" : "TOP_N", - "parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.PRICE", - "next_parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.SELLER_ID" - } + { + "column": "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", + "encoding": "dict" }, - "returntype" : "topn(100, 4)", - "configuration": {"topn.encoding.TEST_KYLIN_FACT.SELLER_ID" : "int:4"} - } - }, { - "name" : "TEST_COUNT_DISTINCT_BITMAP", - "function" : { - "expression" : "COUNT_DISTINCT", - "parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP" + { + "column": "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "encoding": "dict" }, - "returntype" : "bitmap" - } - }, { - "name" : "TEST_EXTENDED_COLUMN", - "function" : { - "expression" : "EXTENDED_COLUMN", - "parameter": { - "type": "column", - "value": "TEST_KYLIN_FACT.ORDER_ID", - "next_parameter": { - "type": "column", - "value": "TEST_ORDER.TEST_EXTENDED_COLUMN" - } + { + "column": "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + "encoding": "dict" }, - "returntype": "extendedcolumn(100)" - } - }, { - "name" : "BUYER_CONTACT", - "function" : { - "expression" : "EXTENDED_COLUMN", - "parameter": { - "type": "column", - "value": "TEST_ORDER.BUYER_ID", - "next_parameter": { - "type": "column", - "value": "BUYER_ACCOUNT.ACCOUNT_CONTACT" - } + { + "column": "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "encoding": "fixed_length:12" }, - "returntype": "extendedcolumn(100)" - } - }, { - "name" : "SELLER_CONTACT", - "function" : { - "expression" : "EXTENDED_COLUMN", - "parameter": { - "type": "column", - "value": "TEST_KYLIN_FACT.SELLER_ID", - "next_parameter": { - "type": "column", - "value": "SELLER_ACCOUNT.ACCOUNT_CONTACT" - } + { + "column": "TEST_KYLIN_FACT.LSTG_SITE_ID", + "encoding": "dict" }, - "returntype": "extendedcolumn(100)" - } - }, { - "name" : "TRANS_ID_RAW", - "function" : { - "expression" : "RAW", - "parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.TRANS_ID" + { + "column": "TEST_KYLIN_FACT.SLR_SEGMENT_CD", + "encoding": "dict" }, - "returntype" : "raw" - } - }, { - "name" : "PRICE_RAW", - "function" : { - "expression" : "RAW", - "parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.PRICE" + { + "column": "TEST_ORDER.TEST_TIME_ENC", + "encoding": "time" }, - "returntype" : "raw" - } - }, { - "name" : "CAL_DT_RAW", - "function" : { - "expression" : "RAW", - "parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.CAL_DT" + { + "column": "TEST_ORDER.TEST_DATE_ENC", + "encoding": "date" }, - "returntype" : "raw" - } - }, { - "name" : "GVM_PERCENTILE", - "function" : { - "expression" : "PERCENTILE", - "parameter" : { - "type" : "column", - "value" : "TEST_KYLIN_FACT.PRICE" + { + "column": "TEST_ORDER.BUYER_ID", + "encoding": "int:4" }, - "returntype" : "percentile(100)" - } - } ], - "dictionaries": [ { - "column": "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP", - "builder": "org.apache.kylin.dict.GlobalDictionaryBuilder" - } ], - "rowkey" : { - "rowkey_columns" : [ { - "column" : "TEST_KYLIN_FACT.SELLER_ID", - "encoding" : "int:4" - }, { - "column" : "TEST_KYLIN_FACT.ORDER_ID", - "encoding" : "int:4" - }, { - "column" : "TEST_KYLIN_FACT.CAL_DT", - "encoding" : "dict" - }, { - "column" : "TEST_KYLIN_FACT.LEAF_CATEG_ID", - "encoding" : "dict" - }, { - "column" : "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", - "encoding" : "dict" - }, { - "column" : "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", - "encoding" : "dict" - }, { - "column" : "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", - "encoding" : "dict" - }, { - "column" : "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", - "encoding" : "fixed_length:12" - }, { - "column" : "TEST_KYLIN_FACT.LSTG_SITE_ID", - "encoding" : "dict" - }, { - "column" : "TEST_KYLIN_FACT.SLR_SEGMENT_CD", - "encoding" : "dict" - }, { - "column" : "TEST_ORDER.TEST_TIME_ENC", - "encoding" : "time" - }, { - "column" : "TEST_ORDER.TEST_DATE_ENC", - "encoding" : "date" - }, { - "column" : "TEST_ORDER.BUYER_ID", - "encoding" : "int:4" - }, { - "column" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", - "encoding" : "dict" - }, { - "column" : "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL", - "encoding" : "dict" - }, { - "column" : "BUYER_ACCOUNT.ACCOUNT_COUNTRY", - "encoding" : "dict" - }, { - "column" : "BUYER_COUNTRY.NAME", - "encoding" : "dict" - }, { - "column" : "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL", - "encoding" : "dict" - }, { - "column" : "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", - "encoding" : "dict" - }, { - "column" : "SELLER_ACCOUNT.ACCOUNT_COUNTRY", - "encoding" : "dict" - }, { - "column" : "SELLER_COUNTRY.NAME", - "encoding" : "dict" - } ] + { + "column": "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "encoding": "dict" + }, + { + "column": "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "encoding": "dict" + }, + { + "column": "BUYER_ACCOUNT.ACCOUNT_COUNTRY", + "encoding": "dict" + }, + { + "column": "BUYER_COUNTRY.NAME", + "encoding": "dict" + }, + { + "column": "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "encoding": "dict" + }, + { + "column": "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "encoding": "dict" + }, + { + "column": "SELLER_ACCOUNT.ACCOUNT_COUNTRY", + "encoding": "dict" + }, + { + "column": "TEST_KYLIN_FACT.DEAL_YEAR", + "encoding": "int:2" + }, + { + "column": "SELLER_COUNTRY.NAME", + "encoding": "dict" + }, + { + "column": "SELLER_ACCOUNT.COUNTRY_ABBR", + "encoding": "dict" + }, + { + "column": "BUYER_ACCOUNT.COUNTRY_ABBR", + "encoding": "dict" + } + ] }, - "signature" : null, - "last_modified" : 1448959801311, - "null_string" : null, - "hbase_mapping" : { - "column_family" : [ { - "name" : "f1", - "columns" : [ { - "qualifier" : "m", - "measure_refs" : [ "TRANS_CNT", "ITEM_COUNT_SUM", "GMV_SUM", "GMV_MIN", "GMV_MAX" ] - } ] - }, { - "name" : "f2", - "columns" : [ { - "qualifier" : "m", - "measure_refs" : [ "SELLER_HLL", "SELLER_FORMAT_HLL", "TOP_SELLER", "TEST_COUNT_DISTINCT_BITMAP" ] - } ] - }, { - "name" : "f3", - "columns" : [ { - "qualifier" : "m", - "measure_refs" : [ "TEST_EXTENDED_COLUMN", "TRANS_ID_RAW", "PRICE_RAW", "CAL_DT_RAW", "BUYER_CONTACT", "SELLER_CONTACT", "GVM_PERCENTILE" ] - } ] - } ] + "signature": null, + "last_modified": 1448959801311, + "null_string": null, + "hbase_mapping": { + "column_family": [ + { + "name": "f1", + "columns": [ + { + "qualifier": "m", + "measure_refs": [ + "TRANS_CNT", + "ITEM_COUNT_SUM", + "GMV_SUM", + "GMV_MIN", + "GMV_MAX", + "COMPUTED_COLUMN_MEASURE" + ] + } + ] + }, + { + "name": "f2", + "columns": [ + { + "qualifier": "m", + "measure_refs": [ + "SELLER_HLL", + "SELLER_FORMAT_HLL", + "TOP_SELLER", + "TEST_COUNT_DISTINCT_BITMAP" + ] + } + ] + }, + { + "name": "f3", + "columns": [ + { + "qualifier": "m", + "measure_refs": [ + "TEST_EXTENDED_COLUMN", + "TRANS_ID_RAW", + "PRICE_RAW", + "CAL_DT_RAW", + "BUYER_CONTACT", + "SELLER_CONTACT", + "GVM_PERCENTILE" + ] + } + ] + } + ] }, - "aggregation_groups" : [ { - "includes" : [ "TEST_KYLIN_FACT.CAL_DT", - "TEST_KYLIN_FACT.LEAF_CATEG_ID", - "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", - "TEST_KYLIN_FACT.LSTG_SITE_ID", - "TEST_KYLIN_FACT.SLR_SEGMENT_CD", - "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", - "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", - "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME" ], - "select_rule" : { - "hierarchy_dims" : [ [ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", - "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", - "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", - "TEST_KYLIN_FACT.LEAF_CATEG_ID" ] ], - "mandatory_dims" : [ ], - "joint_dims" : [ [ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", - "TEST_KYLIN_FACT.LSTG_SITE_ID", - "TEST_KYLIN_FACT.SLR_SEGMENT_CD" ] ] - } - }, { - "includes" : [ "TEST_KYLIN_FACT.CAL_DT", - "TEST_KYLIN_FACT.LEAF_CATEG_ID", - "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", - "TEST_KYLIN_FACT.LSTG_SITE_ID", - "TEST_KYLIN_FACT.SLR_SEGMENT_CD", - "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", - "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", - "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", - - "TEST_KYLIN_FACT.SELLER_ID", - "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL", - "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", - "SELLER_ACCOUNT.ACCOUNT_COUNTRY", - "SELLER_COUNTRY.NAME", - - "TEST_KYLIN_FACT.ORDER_ID", - "TEST_ORDER.TEST_DATE_ENC", - "TEST_ORDER.TEST_TIME_ENC", - "TEST_ORDER.BUYER_ID", - "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", - "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL", - "BUYER_ACCOUNT.ACCOUNT_COUNTRY", - "BUYER_COUNTRY.NAME" ], - "select_rule" : { - "hierarchy_dims" : [ ], - "mandatory_dims" : [ "TEST_KYLIN_FACT.CAL_DT" ], - - "joint_dims" : [ [ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", - "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", - "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", - "TEST_KYLIN_FACT.LEAF_CATEG_ID" ], - - [ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", - "TEST_KYLIN_FACT.LSTG_SITE_ID", - "TEST_KYLIN_FACT.SLR_SEGMENT_CD" ], - - [ "TEST_KYLIN_FACT.SELLER_ID", - "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL", - "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", - "SELLER_ACCOUNT.ACCOUNT_COUNTRY", - "SELLER_COUNTRY.NAME" ], - - [ "TEST_KYLIN_FACT.ORDER_ID", - "TEST_ORDER.TEST_DATE_ENC", - "TEST_ORDER.TEST_TIME_ENC", - "TEST_ORDER.BUYER_ID", - "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", - "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL", - "BUYER_ACCOUNT.ACCOUNT_COUNTRY", - "BUYER_COUNTRY.NAME" ] ] + "aggregation_groups": [ + { + "includes": [ + "TEST_KYLIN_FACT.CAL_DT", + "TEST_KYLIN_FACT.LEAF_CATEG_ID", + "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "TEST_KYLIN_FACT.LSTG_SITE_ID", + "TEST_KYLIN_FACT.SLR_SEGMENT_CD", + "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + "TEST_KYLIN_FACT.DEAL_YEAR" + ], + "select_rule": { + "hierarchy_dims": [ + [ + "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + "TEST_KYLIN_FACT.LEAF_CATEG_ID" + ] + ], + "mandatory_dims": [], + "joint_dims": [ + [ + "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "TEST_KYLIN_FACT.LSTG_SITE_ID", + "TEST_KYLIN_FACT.SLR_SEGMENT_CD", + "TEST_KYLIN_FACT.DEAL_YEAR" + ] + ] + } + }, + { + "includes": [ + "TEST_KYLIN_FACT.CAL_DT", + "TEST_KYLIN_FACT.LEAF_CATEG_ID", + "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "TEST_KYLIN_FACT.LSTG_SITE_ID", + "TEST_KYLIN_FACT.SLR_SEGMENT_CD", + "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + "TEST_KYLIN_FACT.SELLER_ID", + "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "SELLER_ACCOUNT.ACCOUNT_COUNTRY", + "SELLER_COUNTRY.NAME", + "TEST_KYLIN_FACT.ORDER_ID", + "TEST_ORDER.TEST_DATE_ENC", + "TEST_ORDER.TEST_TIME_ENC", + "TEST_ORDER.BUYER_ID", + "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "BUYER_ACCOUNT.ACCOUNT_COUNTRY", + "BUYER_COUNTRY.NAME", + "SELLER_ACCOUNT.COUNTRY_ABBR", + "BUYER_ACCOUNT.COUNTRY_ABBR" + ], + "select_rule": { + "hierarchy_dims": [], + "mandatory_dims": [ + "TEST_KYLIN_FACT.CAL_DT" + ], + "joint_dims": [ + [ + "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + "TEST_KYLIN_FACT.LEAF_CATEG_ID" + ], + [ + "TEST_KYLIN_FACT.LSTG_FORMAT_NAME", + "TEST_KYLIN_FACT.LSTG_SITE_ID", + "TEST_KYLIN_FACT.SLR_SEGMENT_CD", + "SELLER_ACCOUNT.COUNTRY_ABBR", + "BUYER_ACCOUNT.COUNTRY_ABBR" + ], + [ + "TEST_KYLIN_FACT.SELLER_ID", + "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "SELLER_ACCOUNT.ACCOUNT_COUNTRY", + "SELLER_COUNTRY.NAME" + ], + [ + "TEST_KYLIN_FACT.ORDER_ID", + "TEST_ORDER.TEST_DATE_ENC", + "TEST_ORDER.TEST_TIME_ENC", + "TEST_ORDER.BUYER_ID", + "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "BUYER_ACCOUNT.ACCOUNT_COUNTRY", + "BUYER_COUNTRY.NAME" + ] + ] + } } - } ], - "notify_list" : null, - "status_need_notify" : [ ], - "auto_merge_time_ranges" : null, - "retention_range" : 0, - "engine_type" : 2, - "storage_type" : 2, + ], + "notify_list": null, + "status_need_notify": [], + "auto_merge_time_ranges": null, + "retention_range": 0, + "engine_type": 2, + "storage_type": 2, "override_kylin_properties": { "kylin.cube.algorithm": "INMEM" },
http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json b/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json index b79d293..d084ce2 100644 --- a/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json +++ b/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json @@ -121,6 +121,29 @@ } } ], + "computed_columns": [ + { + "tableIdentity": "DEFAULT.TEST_KYLIN_FACT", + "columnName": "DEAL_AMOUNT", + "expression": "DEFAULT.TEST_KYLIN_FACT.PRICE * DEFAULT.TEST_KYLIN_FACT.ITEM_COUNT", + "datatype": "decimal", + "comment": "deal amount of inner join model" + }, + { + "tableIdentity": "DEFAULT.TEST_KYLIN_FACT", + "columnName": "DEAL_YEAR", + "expression": "year(DEFAULT.TEST_KYLIN_FACT.CAL_DT)", + "datatype": "integer", + "comment": "the year of the deal of the inner join model" + }, + { + "tableIdentity": "DEFAULT.TEST_ACCOUNT", + "columnName": "COUNTRY_ABBR", + "expression": "SUBSTR(DEFAULT.TEST_ACCOUNT.ACCOUNT_COUNTRY,0,1)", + "datatype": "string", + "comment": "first char of country of the inner join model" + } + ], "dimensions": [ { "table": "TEST_KYLIN_FACT", @@ -133,7 +156,8 @@ "LEAF_CATEG_ID", "SLR_SEGMENT_CD", "SELLER_ID", - "TEST_COUNT_DISTINCT_BITMAP" + "TEST_COUNT_DISTINCT_BITMAP", + "DEAL_YEAR" ] }, { @@ -153,7 +177,8 @@ "ACCOUNT_BUYER_LEVEL", "ACCOUNT_SELLER_LEVEL", "ACCOUNT_COUNTRY", - "ACCOUNT_CONTACT" + "ACCOUNT_CONTACT", + "COUNTRY_ABBR" ] }, { @@ -163,7 +188,8 @@ "ACCOUNT_BUYER_LEVEL", "ACCOUNT_SELLER_LEVEL", "ACCOUNT_COUNTRY", - "ACCOUNT_CONTACT" + "ACCOUNT_CONTACT", + "COUNTRY_ABBR" ] }, { @@ -219,7 +245,8 @@ ], "metrics": [ "TEST_KYLIN_FACT.PRICE", - "TEST_KYLIN_FACT.ITEM_COUNT" + "TEST_KYLIN_FACT.ITEM_COUNT", + "TEST_KYLIN_FACT.DEAL_AMOUNT" ], "last_modified": 1422435345352, "filter_condition": null, @@ -228,4 +255,4 @@ "partition_date_start": 0, "partition_type": "APPEND" } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json b/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json index bc5b444..4a10899 100644 --- a/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json +++ b/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json @@ -121,6 +121,29 @@ } } ], + "computed_columns": [ + { + "tableIdentity": "DEFAULT.TEST_KYLIN_FACT", + "columnName": "DEAL_AMOUNT", + "expression": "DEFAULT.TEST_KYLIN_FACT.PRICE * DEFAULT.TEST_KYLIN_FACT.ITEM_COUNT", + "datatype": "decimal", + "comment": "deal amount of left join model" + }, + { + "tableIdentity": "DEFAULT.TEST_KYLIN_FACT", + "columnName": "DEAL_YEAR", + "expression": "year(DEFAULT.TEST_KYLIN_FACT.CAL_DT)", + "datatype": "integer", + "comment": "the year of the deal of the left join model" + }, + { + "tableIdentity": "DEFAULT.TEST_ACCOUNT", + "columnName": "COUNTRY_ABBR", + "expression": "SUBSTR(DEFAULT.TEST_ACCOUNT.ACCOUNT_COUNTRY,0,1)", + "datatype": "string", + "comment": "first char of country of the left join model" + } + ], "dimensions": [ { "table": "TEST_KYLIN_FACT", @@ -133,7 +156,8 @@ "LEAF_CATEG_ID", "SLR_SEGMENT_CD", "SELLER_ID", - "TEST_COUNT_DISTINCT_BITMAP" + "TEST_COUNT_DISTINCT_BITMAP", + "DEAL_YEAR" ] }, { @@ -153,7 +177,8 @@ "ACCOUNT_BUYER_LEVEL", "ACCOUNT_SELLER_LEVEL", "ACCOUNT_COUNTRY", - "ACCOUNT_CONTACT" + "ACCOUNT_CONTACT", + "COUNTRY_ABBR" ] }, { @@ -163,7 +188,8 @@ "ACCOUNT_BUYER_LEVEL", "ACCOUNT_SELLER_LEVEL", "ACCOUNT_COUNTRY", - "ACCOUNT_CONTACT" + "ACCOUNT_CONTACT", + "COUNTRY_ABBR" ] }, { @@ -219,7 +245,8 @@ ], "metrics": [ "TEST_KYLIN_FACT.PRICE", - "TEST_KYLIN_FACT.ITEM_COUNT" + "TEST_KYLIN_FACT.ITEM_COUNT", + "TEST_KYLIN_FACT.DEAL_AMOUNT" ], "last_modified": 1422435345352, "filter_condition": null, @@ -228,4 +255,4 @@ "partition_date_start": 0, "partition_type": "APPEND" } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/examples/test_case_data/sandbox/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties index 684b4dd..13474f9 100644 --- a/examples/test_case_data/sandbox/kylin.properties +++ b/examples/test_case_data/sandbox/kylin.properties @@ -25,7 +25,7 @@ kylin.server.mode=all kylin.storage.hbase.owner-tag=who...@kylin.apache.org # List of web servers in use, this enables one web server instance to sync up with other servers. -#kylin.server.cluster-servers=localhost:7070 +kylin.server.cluster-servers=localhost:7070 # Display timezone on UI,format like[GMT+N or GMT-N] kylin.web.timezone=GMT+8 http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/kylin-it/src/test/java/org/apache/kylin/query/CompareQueryBySuffix.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/CompareQueryBySuffix.java b/kylin-it/src/test/java/org/apache/kylin/query/CompareQueryBySuffix.java new file mode 100644 index 0000000..2dbe6fd --- /dev/null +++ b/kylin-it/src/test/java/org/apache/kylin/query/CompareQueryBySuffix.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +package org.apache.kylin.query; + +import java.io.File; +import java.io.IOException; + +public class CompareQueryBySuffix implements ICompareQueryTranslator { + + public static CompareQueryBySuffix INSTANCE = new CompareQueryBySuffix(); + + @Override + public String transform(File f) { + File parentFile = f.getParentFile(); + File compareFile = new File(parentFile, f.getName() + ".compare"); + try { + return KylinTestBase.getTextFromFile(compareFile); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/kylin-it/src/test/java/org/apache/kylin/query/ICompareQueryTranslator.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ICompareQueryTranslator.java b/kylin-it/src/test/java/org/apache/kylin/query/ICompareQueryTranslator.java new file mode 100644 index 0000000..67473b1 --- /dev/null +++ b/kylin-it/src/test/java/org/apache/kylin/query/ICompareQueryTranslator.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +package org.apache.kylin.query; + +import java.io.File; + +public interface ICompareQueryTranslator { + /** + * + * @param f origin query file + * @return the compare query to submit to standard DB like H2 + */ + public String transform(File f); +} http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java index cdaa10b..629e861 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java @@ -235,6 +235,11 @@ public class ITKylinQueryTest extends KylinTestBase { } @Test + public void testComputedColumnsQuery() throws Exception { + execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_computedcolumn", null, true, CompareQueryBySuffix.INSTANCE); + } + + @Test public void testTopNQuery() throws Exception { if ("left".equalsIgnoreCase(joinType)) { this.execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_topn", null, true); http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java index fad7a94..656666f 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java @@ -160,7 +160,7 @@ public class KylinTestBase { return new ArrayList<>(set); } - protected static String getTextFromFile(File file) throws IOException { + public static String getTextFromFile(File file) throws IOException { BufferedReader reader = new BufferedReader(new FileReader(file)); String line = null; StringBuilder stringBuilder = new StringBuilder(); @@ -497,7 +497,21 @@ public class KylinTestBase { logger.info("Queries appended with limit: " + appendLimitQueries); } + protected void execAndCompQuery(String queryFolder, String[] exclusiveQuerys, boolean needSort) throws Exception { + execAndCompQuery(queryFolder, exclusiveQuerys, needSort, new ICompareQueryTranslator() { + @Override + public String transform(File f) { + try { + return KylinTestBase.getTextFromFile(f); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + } + + protected void execAndCompQuery(String queryFolder, String[] exclusiveQuerys, boolean needSort, ICompareQueryTranslator translator) throws Exception { logger.info("---------- test folder: " + new File(queryFolder).getAbsolutePath()); Set<String> exclusiveSet = buildExclusiveSet(exclusiveQuerys); @@ -507,17 +521,18 @@ public class KylinTestBase { if (exclusiveSet.contains(queryName)) { continue; } - String sql = getTextFromFile(sqlFile); + String sql1 = getTextFromFile(sqlFile); + String sql2 = translator.transform(sqlFile); // execute Kylin logger.info("Query Result from Kylin - " + queryName + " (" + queryFolder + ")"); IDatabaseConnection kylinConn = new DatabaseConnection(cubeConnection); - ITable kylinTable = executeQuery(kylinConn, queryName, sql, needSort); + ITable kylinTable = executeQuery(kylinConn, queryName, sql1, needSort); // execute H2 logger.info("Query Result from H2 - " + queryName); long currentTime = System.currentTimeMillis(); - ITable h2Table = executeQuery(newH2Connection(), queryName, sql, needSort); + ITable h2Table = executeQuery(newH2Connection(), queryName, sql2, needSort); logger.info("H2 spent " + (System.currentTimeMillis() - currentTime) + " mili-seconds."); try { @@ -530,7 +545,7 @@ public class KylinTestBase { compQueryCount++; if (kylinTable.getRowCount() == 0) { - zeroResultQueries.add(sql); + zeroResultQueries.add(sql1); } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/kylin-it/src/test/resources/query/sql_computedcolumn/query01.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_computedcolumn/query01.sql b/kylin-it/src/test/resources/query/sql_computedcolumn/query01.sql new file mode 100644 index 0000000..5661c79 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_computedcolumn/query01.sql @@ -0,0 +1,14 @@ +SELECT + +count(*) as cnt, sum(price) as sum_price, sum(DEAL_AMOUNT) as deal_amount, SELLER_COUNTRY.NAME, DEAL_YEAR as deal_year + +FROM TEST_KYLIN_FACT as TEST_KYLIN_FACT +INNER JOIN TEST_ACCOUNT as SELLER_ACCOUNT +ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID +INNER JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS +ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID +INNER JOIN TEST_COUNTRY as SELLER_COUNTRY +ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY + +where SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL=1 and SELLER_ACCOUNT.COUNTRY_ABBR in ('I', 'F') +group by SELLER_COUNTRY.NAME, DEAL_YEAR http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/kylin-it/src/test/resources/query/sql_computedcolumn/query01.sql.compare ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_computedcolumn/query01.sql.compare b/kylin-it/src/test/resources/query/sql_computedcolumn/query01.sql.compare new file mode 100644 index 0000000..3f21647 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_computedcolumn/query01.sql.compare @@ -0,0 +1,14 @@ +SELECT + +count(*) as cnt, sum(price) as sum_price, sum(TEST_KYLIN_FACT.PRICE * TEST_KYLIN_FACT.ITEM_COUNT) as deal_amount, SELLER_COUNTRY.NAME, year(TEST_KYLIN_FACT.CAL_DT) as deal_year + +FROM TEST_KYLIN_FACT as TEST_KYLIN_FACT +INNER JOIN TEST_ACCOUNT as SELLER_ACCOUNT +ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID +INNER JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS +ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID +INNER JOIN TEST_COUNTRY as SELLER_COUNTRY +ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY + +where SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL=1 and SUBSTR(SELLER_ACCOUNT.ACCOUNT_COUNTRY,0,1) in ('I', 'F') +group by SELLER_COUNTRY.NAME, year(TEST_KYLIN_FACT.CAL_DT) http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/kylin-it/src/test/resources/query/sql_computedcolumn/query02.sql ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_computedcolumn/query02.sql b/kylin-it/src/test/resources/query/sql_computedcolumn/query02.sql new file mode 100644 index 0000000..0e84301 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_computedcolumn/query02.sql @@ -0,0 +1,21 @@ +SELECT + +count(*) as cnt, sum(price) as sum_price, sum(DEAL_AMOUNT) as deal_amount, SELLER_COUNTRY.NAME, DEAL_YEAR as deal_year + +FROM TEST_KYLIN_FACT as TEST_KYLIN_FACT +INNER JOIN TEST_ORDER as TEST_ORDER +ON TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID +INNER JOIN TEST_ACCOUNT as BUYER_ACCOUNT +ON TEST_ORDER.BUYER_ID = BUYER_ACCOUNT.ACCOUNT_ID +INNER JOIN TEST_ACCOUNT as SELLER_ACCOUNT +ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID +INNER JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS +ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID +INNER JOIN TEST_COUNTRY as BUYER_COUNTRY +ON BUYER_ACCOUNT.ACCOUNT_COUNTRY = BUYER_COUNTRY.COUNTRY +INNER JOIN TEST_COUNTRY as SELLER_COUNTRY +ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY + + +where SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL=1 and SELLER_ACCOUNT.COUNTRY_ABBR in ('T', 'R') and BUYER_ACCOUNT.COUNTRY_ABBR in ('T', 'R') +group by SELLER_COUNTRY.NAME, DEAL_YEAR http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/kylin-it/src/test/resources/query/sql_computedcolumn/query02.sql.compare ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_computedcolumn/query02.sql.compare b/kylin-it/src/test/resources/query/sql_computedcolumn/query02.sql.compare new file mode 100644 index 0000000..7243d7e --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_computedcolumn/query02.sql.compare @@ -0,0 +1,21 @@ +SELECT + +count(*) as cnt, sum(price) as sum_price, sum(TEST_KYLIN_FACT.PRICE * TEST_KYLIN_FACT.ITEM_COUNT) as deal_amount, SELLER_COUNTRY.NAME, year(TEST_KYLIN_FACT.CAL_DT) as deal_year + +FROM TEST_KYLIN_FACT as TEST_KYLIN_FACT +INNER JOIN TEST_ORDER as TEST_ORDER +ON TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID +INNER JOIN TEST_ACCOUNT as BUYER_ACCOUNT +ON TEST_ORDER.BUYER_ID = BUYER_ACCOUNT.ACCOUNT_ID +INNER JOIN TEST_ACCOUNT as SELLER_ACCOUNT +ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID +INNER JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS +ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID +INNER JOIN TEST_COUNTRY as BUYER_COUNTRY +ON BUYER_ACCOUNT.ACCOUNT_COUNTRY = BUYER_COUNTRY.COUNTRY +INNER JOIN TEST_COUNTRY as SELLER_COUNTRY +ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY + + +where SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL=1 and SUBSTR(SELLER_ACCOUNT.ACCOUNT_COUNTRY,1,1) in ('T', 'R') and SUBSTR(BUYER_ACCOUNT.ACCOUNT_COUNTRY,1,1) in ('T', 'R') +group by SELLER_COUNTRY.NAME, year(TEST_KYLIN_FACT.CAL_DT) http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/kylin-it/src/test/resources/query/sql_verifyCount/query01.sql.expected ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_verifyCount/query01.sql.expected b/kylin-it/src/test/resources/query/sql_verifyCount/query01.sql.expected index 86242f2..11636e7 100644 --- a/kylin-it/src/test/resources/query/sql_verifyCount/query01.sql.expected +++ b/kylin-it/src/test/resources/query/sql_verifyCount/query01.sql.expected @@ -1,2 +1,2 @@ * -11 +13 http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/kylin-it/src/test/resources/query/sql_verifyCount/query03.sql.expected ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_verifyCount/query03.sql.expected b/kylin-it/src/test/resources/query/sql_verifyCount/query03.sql.expected index 093e327..349103a 100644 --- a/kylin-it/src/test/resources/query/sql_verifyCount/query03.sql.expected +++ b/kylin-it/src/test/resources/query/sql_verifyCount/query03.sql.expected @@ -1,2 +1,2 @@ 10 -13 +15 http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/kylin-it/src/test/resources/query/sql_verifyCount/query04.sql.expected ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/resources/query/sql_verifyCount/query04.sql.expected b/kylin-it/src/test/resources/query/sql_verifyCount/query04.sql.expected index 71011db..d10e701 100644 --- a/kylin-it/src/test/resources/query/sql_verifyCount/query04.sql.expected +++ b/kylin-it/src/test/resources/query/sql_verifyCount/query04.sql.expected @@ -1,2 +1,2 @@ 100 -11 +13 http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java index 27d456f..52023e4 100644 --- a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java +++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java @@ -78,7 +78,7 @@ public class OLAPSchema extends AbstractSchema { final String tableName = tableDesc.getName();//safe to use tableDesc.getName() here, it is in a DB context now final OLAPTable table = new OLAPTable(this, tableDesc); olapTables.put(tableName, table); - // logger.debug("Project " + projectName + " exposes table " + tableName); + //logger.debug("Project " + projectName + " exposes table " + tableName); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java index df9ecfb..4217f86 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java @@ -191,7 +191,7 @@ public class ModelController extends BasicController { newModelDesc = modelService.createModelDesc(project, newModelDesc); //reload avoid shallow - metaManager.reloadDataModelDesc(newModelName); + metaManager.reloadDataModelDescAt(DataModelDesc.concatResourcePath(newModelName)); } catch (IOException e) { throw new InternalErrorException("failed to clone DataModelDesc", e); } http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java index 7f7b6f0..b176dae 100644 --- a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java +++ b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java @@ -289,7 +289,6 @@ public class CacheServiceTest extends LocalFileMetadataTestCase { return tableDesc; } - @SuppressWarnings("deprecation") @Test public void testMetaCRUD() throws Exception { final MetadataManager metadataManager = MetadataManager.getInstance(configA); http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java new file mode 100644 index 0000000..550aeb8 --- /dev/null +++ b/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.rest.service; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.List; + +import org.apache.kylin.job.exception.JobException; +import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.model.ComputedColumnDesc; +import org.apache.kylin.metadata.model.DataModelDesc; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.springframework.beans.factory.annotation.Autowired; + +public class ModelServiceTest extends ServiceTestBase { + + @Autowired + ModelService modelService; + + @Rule + public ExpectedException expectedEx = ExpectedException.none(); + + @Test + public void testSuccessModelUpdate() throws IOException, JobException { + List<DataModelDesc> dataModelDescs = modelService.listAllModels("ci_inner_join_model", "default"); + Assert.assertTrue(dataModelDescs.size() == 1); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + MetadataManager.MODELDESC_SERIALIZER.serialize(dataModelDescs.get(0), new DataOutputStream(baos)); + ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); + DataModelDesc deserialize = MetadataManager.MODELDESC_SERIALIZER.deserialize(new DataInputStream(bais)); + + deserialize.setOwner("somebody"); + DataModelDesc dataModelDesc = modelService.updateModelAndDesc(deserialize); + Assert.assertTrue(dataModelDesc.getOwner().equals("somebody")); + } + + @Test + public void testSuccessModelUpdateOnComputedColumn() throws IOException, JobException, NoSuchFieldException, IllegalAccessException { + + List<DataModelDesc> dataModelDescs = modelService.listAllModels("ci_left_join_model", "default"); + Assert.assertTrue(dataModelDescs.size() == 1); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + MetadataManager.MODELDESC_SERIALIZER.serialize(dataModelDescs.get(0), new DataOutputStream(baos)); + ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); + DataModelDesc deserialize = MetadataManager.MODELDESC_SERIALIZER.deserialize(new DataInputStream(bais)); + + Field field = ComputedColumnDesc.class.getDeclaredField("comment"); + field.setAccessible(true); + field.set(deserialize.getComputedColumnDescs().get(0), "change on comment is okay"); + DataModelDesc dataModelDesc = modelService.updateModelAndDesc(deserialize); + } + + @Test + public void testFailureModelUpdateDueToComputedColumnConflict() throws IOException, JobException, NoSuchFieldException, IllegalAccessException { + expectedEx.expect(IllegalStateException.class); + expectedEx.expectMessage("Computed column named DEFAULT.TEST_KYLIN_FACT.DEAL_AMOUNT is already defined in other models: [DataModelDesc [name=ci_left_join_model], DataModelDesc [name=ci_inner_join_model]]. Please change another name, or try to keep consistent definition"); + + List<DataModelDesc> dataModelDescs = modelService.listAllModels("ci_left_join_model", "default"); + Assert.assertTrue(dataModelDescs.size() == 1); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + MetadataManager.MODELDESC_SERIALIZER.serialize(dataModelDescs.get(0), new DataOutputStream(baos)); + ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); + DataModelDesc deserialize = MetadataManager.MODELDESC_SERIALIZER.deserialize(new DataInputStream(bais)); + + Field field = ComputedColumnDesc.class.getDeclaredField("expression"); + field.setAccessible(true); + field.set(deserialize.getComputedColumnDescs().get(0), "another expression"); + DataModelDesc dataModelDesc = modelService.updateModelAndDesc(deserialize); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/9a812d5f/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java index 3488fbd..dbcfc7a 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java @@ -153,7 +153,7 @@ public class SchemaChecker { List<String> violateColumns = Lists.newArrayList(); for (ColumnDesc column : table.getColumns()) { - if (usedColumns.contains(column)) { + if (!column.isComputedColumnn() && usedColumns.contains(column)) { HiveTableMeta.HiveTableColumnMeta field = fieldsMap.get(column.getName()); if (field == null || !isColumnCompatible(column, field)) { violateColumns.add(column.getName());
