hi,shaofeng:
segment metadata??????
{
"uuid": "9e968e85-5f82-44e5-aeef-1679b6560da8",
"last_modified": 1493684106740,
"version": "1.6.0",
"name": "aaabbb",
"owner": "ADMIN",
"descriptor": "aaabbb",
"cost": 50,
"status": "DISABLED",
"segments": [
{
"uuid": "5c1d409e-da29-4ca1-9bb6-1fe77bbf1cf5",
"name": "20170406000000_20170412000000",
"storage_location_identifier": "KYLIN_ZCY1CIR704",
"date_range_start": 1491436800000,
"date_range_end": 1491955200000,
"source_offset_start": 0,
"source_offset_end": 0,
"status": "READY",
"size_kb": 72,
"input_records": 1877,
"input_records_size": 21352,
"last_build_time": 1493358964275,
"last_build_job_id": "65daf4d1-0efb-4205-9624-18581307b5df",
"create_time_utc": 1493358368080,
"cuboid_shard_nums": {},
"total_shards": 1,
"blackout_cuboids": [],
"binary_signature": null,
"dictionaries": {
"DEFAULT.OPTIONACTION/FDZ":
"/dict/DEFAULT.OPTIONACTION/FDZ/7c285980-3683-46e2-b794-000a2ca6766e.dict",
"DEFAULT.OPTIONACTION/AGENTID":
"/dict/DEFAULT.OPTIONACTION/AGENTID/2fa4bf2a-f959-458c-9fd7-25b6d9890274.dict",
"DEFAULT.OPTIONACTION/COURSEID":
"/dict/DEFAULT.OPTIONACTION/COURSEID/55f24172-0c32-4456-b381-09e6b9056fdf.dict",
"DEFAULT.OPTIONACTION/COURSEVERSION":
"/dict/DEFAULT.OPTIONACTION/COURSEVERSION/ca8663a4-ea11-4f81-97ae-fffc56fa470a.dict"
},
"snapshots": null,
"index_path":
"/kylin/kylin_metadata/kylin-65daf4d1-0efb-4205-9624-18581307b5df/aaabbb/secondary_index/",
"rowkey_stats": [
[
"FDZ",
27,
1
],
[
"AGENTID",
2,
1
],
[
"COURSEID",
21,
1
],
[
"COURSEVERSION",
21,
1
]
]
},
{
"uuid": "9fdc46fe-87d5-4a15-87d4-75877aebbf75",
"name": "20170406000000_20170413000000",
"storage_location_identifier": "KYLIN_OTUWGWA5L3",
"date_range_start": 1491436800000,
"date_range_end": 1492041600000,
"source_offset_start": 0,
"source_offset_end": 0,
"status": "NEW",
"size_kb": 0,
"input_records": 0,
"input_records_size": 0,
"last_build_time": 0,
"last_build_job_id": null,
"create_time_utc": 1493358970374,
"cuboid_shard_nums": {},
"total_shards": 1,
"blackout_cuboids": [],
"binary_signature": null,
"dictionaries": {
"DEFAULT.OPTIONACTION/FDZ":
"/dict/DEFAULT.OPTIONACTION/FDZ/e88ff5f7-1b99-4b97-beb2-54bfe21effc3.dict",
"DEFAULT.OPTIONACTION/AGENTID":
"/dict/DEFAULT.OPTIONACTION/AGENTID/6adc329d-4170-4386-beb7-a2d02e05a405.dict",
"DEFAULT.OPTIONACTION/COURSEID":
"/dict/DEFAULT.OPTIONACTION/COURSEID/be0448f3-8ad2-4f62-a53c-d7b8d2c30a12.dict",
"DEFAULT.OPTIONACTION/COURSEVERSION":
"/dict/DEFAULT.OPTIONACTION/COURSEVERSION/a2feb1c2-ce6f-4782-9d60-f5253c2b5d38.dict"
},
"snapshots": null,
"index_path": null,
"rowkey_stats": []
},
{
"uuid": "0d04b6e1-46b8-49c2-83bd-35c1a16c2f0b",
"name": "20170412000000_20170413000000",
"storage_location_identifier": "KYLIN_3RUC4BTTTP",
"date_range_start": 1491955200000,
"date_range_end": 1492041600000,
"source_offset_start": 0,
"source_offset_end": 0,
"status": "READY",
"size_kb": 46,
"input_records": 893,
"input_records_size": 11268,
"last_build_time": 1493358969356,
"last_build_job_id": "9591dcb3-ba35-40fa-88f4-c3a7e6203693",
"create_time_utc": 1493358378989,
"cuboid_shard_nums": {},
"total_shards": 1,
"blackout_cuboids": [],
"binary_signature": null,
"dictionaries": {
"DEFAULT.OPTIONACTION/FDZ":
"/dict/DEFAULT.OPTIONACTION/FDZ/39a0e4e7-5f64-4cd5-b510-200b0e291c30.dict",
"DEFAULT.OPTIONACTION/AGENTID":
"/dict/DEFAULT.OPTIONACTION/AGENTID/6adc329d-4170-4386-beb7-a2d02e05a405.dict",
"DEFAULT.OPTIONACTION/COURSEID":
"/dict/DEFAULT.OPTIONACTION/COURSEID/24514359-d66e-43d3-84ee-3cf99f8772c2.dict",
"DEFAULT.OPTIONACTION/COURSEVERSION":
"/dict/DEFAULT.OPTIONACTION/COURSEVERSION/3d18e4da-b950-4448-a0c2-6a1698322417.dict"
},
"snapshots": {},
"index_path":
"/kylin/kylin_metadata/kylin-9591dcb3-ba35-40fa-88f4-c3a7e6203693/aaabbb/secondary_index/",
"rowkey_stats": [
[
"FDZ",
16,
1
],
[
"AGENTID",
3,
1
],
[
"COURSEID",
14,
1
],
[
"COURSEVERSION",
14,
1
]
]
}
],
"create_time_utc": 1493263501746,
"size_kb": 118,
"input_records_count": 2770,
"input_records_size": 32620
}
------------------ ???????? ------------------
??????: "ShaoFeng Shi";<[email protected]>;
????????: 2017??4??28??(??????) ????9:52
??????: "dev"<[email protected]>;
????: Re: java.lang.NullPointerException
Hi,
The cube definition looks okay, although "AGENTID" and "COURSEID" appeared
both in dimension and TopN measure. They were defined to use "dict"
encoding so Kylin will build dictionary for them. Could you please share
the cube instance json, in which there has the segment metadata; all
dictionaries are recorded there; You can get that by click "view cube" in
the cube's "Admin actions".
2017-04-28 13:58 GMT+08:00 35925138 <[email protected]>:
> ??????
> ????cube json ????
>
>
> { "uuid": "9bc80d90-4bf2-4ae5-b3ee-dd8a563fcefd", "last_modified":
> 1493358357281, "version": "1.6.0", "name": "aaabbb", "model_name":
> "optionaction", "description": "", "null_string": null, "dimensions":
> [ { "name": "ACTION_TIME", "table":
> "DEFAULT.OPTIONACTION", "column": "ACTION_TIME", "derived":
> null }, { "name": "AGENTID", "table":
> "DEFAULT.OPTIONACTION", "column": "AGENTID", "derived": null
> }, { "name": "COURSEID", "table": "DEFAULT.OPTIONACTION",
> "column": "COURSEID", "derived": null }, { "name":
> "FDZ", "table": "DEFAULT.OPTIONACTION", "column": "FDZ",
> "derived": null } ], "measures": [ { "name": "_COUNT_",
> "function": { "expression": "COUNT", "parameter": {
> "type": "constant", "value": "1",
> "next_parameter": null }, "returntype": "bigint" },
> "dependent_measure_ref": null }, { "name": "COUNTD",
> "function": { "expression": "COUNT_DISTINCT",
> "parameter": { "type": "constant", "value": "1",
> "next_parameter": null }, "returntype": "hllc(16)"
> }, "dependent_measure_ref": null }, { "name":
> "TOPNTIMELONG", "function": { "expression": "TOP_N",
> "parameter": { "type": "column", "value": "TIMELONG",
> "next_parameter": { "type": "column",
> "value": "AGENTID", "next_parameter": { "type":
> "column", "value": "COURSEID",
> "next_parameter": { "type": "column",
> "value": "COURSEVERSION", "next_parameter": null
> } } } }, "returntype":
> "topn(100)", "configuration": { "topn.encoding.COURSEID":
> "dict", "topn.encoding.AGENTID": "dict",
> "topn.encoding.COURSEVERSION": "dict" } },
> "dependent_measure_ref": null } ], "dictionaries": [], "rowkey":
> { "rowkey_columns": [ { "column": "ACTION_TIME",
> "encoding": "time", "isShardBy": false }, {
> "column": "FDZ", "encoding": "dict", "isShardBy": false
> }, { "column": "AGENTID", "encoding": "dict",
> "isShardBy": false }, { "column": "COURSEID",
> "encoding": "dict", "isShardBy": false } ] },
> "hbase_mapping": { "column_family": [ { "name": "F1",
> "columns": [ { "qualifier": "M",
> "measure_refs": [ "_COUNT_", "TOPNTIMELONG"
> ] } ] }, { "name": "F2",
> "columns": [ { "qualifier": "M",
> "measure_refs": [ "COUNTD" ] }
> ] } ] }, "aggregation_groups": [ { "includes": [
> "ACTION_TIME", "FDZ", "AGENTID", "COURSEID"
> ], "select_rule": { "hierarchy_dims": [],
> "mandatory_dims": [ "ACTION_TIME", "FDZ",
> "AGENTID", "COURSEID" ], "joint_dims": []
> } } ], "signature": "55F0fLhStyeYlIRDyYUqOg==", "notify_list":
> [], "status_need_notify": [ "ERROR", "SUCCEED" ],
> "partition_date_start": 1491436800000, "partition_date_end":
> 3153600000000, "auto_merge_time_ranges": [ 604800000, 2419200000
> ], "retention_range": 0, "engine_type": 2, "storage_type": 2,
> "override_kylin_properties": {} }
>
>
>
>
> ------------------ ???????? ------------------
> ??????: "ShaoFeng Shi";<[email protected]>;
> ????????: 2017??4??26??(??????) ????7:44
> ??????: "dev"<[email protected]>;
>
> ????: Re: java.lang.NullPointerException
>
>
>
> It seems the dictionary wasn't found. What's the encoding method for the
> "group by" column in TopN (if not specified, it will use dictionary)? Is
> the TopN query working well? If you can provide the JSON of the cube, that
> would be easier.
>
> ?? 2017??4??26?? ????4:19??Li Yang <[email protected]>??????
>
> > Shaofeng knows this better..
> >
> > 2017-04-26 15:11 GMT+08:00 35925138 <[email protected]>:
> >
> > > ????????????????????????meansures
> > > ??????topn??????????????????????????????????
> ??????????????????sum????????????????????????????
> > ????????????????????????????????????kylin????1.6.0
> > > hive????1.2.1
> > >
> > >
> > > ------------------ ???????? ------------------
> > > *??????:* "35925138";<[email protected]>;
> > > *????????:* 2017??4??25??(??????) ????8:24
> > > *??????:* "dev"<[email protected]>;
> > > *????:* java.lang.NullPointerException
> > >
> > > ??????
> > > ????merge cube????????????????
> > > ??????????????????????????
> > > 2017-04-25 08:13:24,313 ERROR [IPC Server handler 3 on 41962]
> > > org.apache.hadoop.mapred.TaskAttemptListenerImpl: Task:
> > > attempt_1492659249668_0138_m_000004_0 - exited :
> > > java.lang.NullPointerException
> > > at org.apache.kylin.dimension.DictionaryDimEnc.<init>(
> > > DictionaryDimEnc.java:56)
> > > at org.apache.kylin.dimension.DictionaryDimEnc.<init>(
> > > DictionaryDimEnc.java:51)
> > > at org.apache.kylin.measure.topn.TopNMeasureType.
> > > getDimensionEncodings(TopNMeasureType.java:417)
> > > at org.apache.kylin.measure.topn.TopNMeasureType.access$100(
> > > TopNMeasureType.java:52)
> > > at org.apache.kylin.measure.topn.TopNMeasureType$1.
> > > reEncodeDictionary(TopNMeasureType.java:166)
> > > at org.apache.kylin.measure.topn.TopNMeasureType$1.
> > > reEncodeDictionary(TopNMeasureType.java:117)
> > > at org.apache.kylin.engine.mr.steps.MergeCuboidMapper.map(
> > > MergeCuboidMapper.java:239)
> > > at org.apache.kylin.engine.mr.steps.MergeCuboidMapper.map(
> > > MergeCuboidMapper.java:62)
> > > at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:145)
> > > at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.
> > java:784)
> > > at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341)
> > > at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.
> java:163)
> > > at java.security.AccessController.doPrivileged(Native Method)
> > > at javax.security.auth.Subject.doAs(Subject.java:422)
> > > at org.apache.hadoop.security.UserGroupInformation.doAs(
> > > UserGroupInformation.java:1628)
> > > at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)
> > >
> > > ????????????????????????????
> > >
> >
>
>
>
> --
> Best regards,
>
> Shaofeng Shi ??????
>
--
Best regards,
Shaofeng Shi ??????
