[ https://issues.apache.org/jira/browse/KYLIN-1568?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
liyang resolved KYLIN-1568. --------------------------- Resolution: Duplicate Assignee: liyang (was: Dong Li) Fix Version/s: v1.5.3 Dup with KYLIN-1718 > Calculate row value buffer size instead of hard coded ROWVALUE_BUFFER_SIZE > -------------------------------------------------------------------------- > > Key: KYLIN-1568 > URL: https://issues.apache.org/jira/browse/KYLIN-1568 > Project: Kylin > Issue Type: Bug > Components: Job Engine > Affects Versions: v1.5.0 > Environment: hadoop2.4.1 , hive1.0 , hbase0.98 > Reporter: Xingxing Di > Assignee: liyang > Labels: newbie > Fix For: v1.5.3 > > > I looked into the code , found this : > valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE); > It looks like the ROWVALUE_BUFFER_SIZE is smaller than the value bytes length. > The exeption is : > 2016-04-07 10:18:57,311 WARN [main] org.apache.hadoop.mapred.YarnChild: > Exception running child : java.nio.BufferOverflowException at > java.nio.HeapByteBuffer.put(HeapByteBuffer.java:200) at > org.apache.kylin.measure.bitmap.BitmapCounter.writeRegisters(BitmapCounter.java:85) > at > org.apache.kylin.measure.bitmap.BitmapSerializer.serialize(BitmapSerializer.java:40) > at > org.apache.kylin.measure.bitmap.BitmapSerializer.serialize(BitmapSerializer.java:30) > at > org.apache.kylin.measure.MeasureCodec.encode(MeasureCodec.java:89) at > org.apache.kylin.engine.mr.steps.CuboidReducer.reduce(CuboidReducer.java:105) > at > org.apache.kylin.engine.mr.steps.CuboidReducer.reduce(CuboidReducer.java:43) > at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) > at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:627) > at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389) > at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:167) at > java.security.AccessController.doPrivileged(Native Method) at > javax.security.auth.Subject.doAs(Subject.java:415) at > org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1556) > at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:162) > The cube info : > { > "uuid": "6990333d-ef59-44e3-b46a-7e571d002698", > "version": "1.5.0", > "name": "flw_test2_cube_1", > "description": "", > "dimensions": [ > { > "name": "TMP.TMP_FLW_MAIN_2.SITE_CLASS", > "table": "TMP.TMP_FLW_MAIN_2", > "column": "SITE_CLASS", > "derived": null > }, > { > "name": "TMP.TMP_FLW_MAIN_2.DEVICE_ID", > "table": "TMP.TMP_FLW_MAIN_2", > "column": "DEVICE_ID", > "derived": null > }, > { > "name": "TMP.TMP_FLW_MAIN_2.REF_DOMAIN", > "table": "TMP.TMP_FLW_MAIN_2", > "column": "REF_DOMAIN", > "derived": null > }, > { > "name": "TMP.TMP_FLW_MAIN_2.NEW_FLAG", > "table": "TMP.TMP_FLW_MAIN_2", > "column": "NEW_FLAG", > "derived": null > }, > { > "name": "TMP.TMP_FLW_MAIN_2.DEALER_ID", > "table": "TMP.TMP_FLW_MAIN_2", > "column": "DEALER_ID", > "derived": null > }, > { > "name": "TMP.TMP_FLW_MAIN_2.KEY_WORD", > "table": "TMP.TMP_FLW_MAIN_2", > "column": "KEY_WORD", > "derived": null > }, > { > "name": "TMP.OLAP_DIM_CITY_ALL_DERIVED", > "table": "TMP.OLAP_DIM_CITY_ALL", > "column": null, > "derived": [ > "ID", > "CITY_ID", > "CITY_NAME", > "PROVINCE_ID", > "PROVINCE_NAME" > ] > }, > { > "name": "TMP.OLAP_DIM_SITE_ALL_DERIVED", > "table": "TMP.OLAP_DIM_SITE_ALL", > "column": null, > "derived": [ > "ID", > "SITE_ID", > "SITE_NAME", > "CATEGORY_ID", > "CATEGORY_NAME", > "SUB_CATEGORY_ID", > "SUB_CATEGORY_NAME" > ] > }, > { > "name": "OLAP.OLAP_DIM_DAY_DERIVED", > "table": "OLAP.OLAP_DIM_DAY", > "column": null, > "derived": [ > "DIM_DAY_ID", > "DAY_DATE", > "WEEKDAY_ID", > "DIM_MONTH_ID" > ] > }, > { > "name": "TMP.OLAP_DIM_SPEC_ALL_DERIVED", > "table": "TMP.OLAP_DIM_SPEC_ALL", > "column": null, > "derived": [ > "ID", > "SPEC_ID", > "SPEC_NAME", > "SERIES_ID", > "SERIES_NAME", > "BRAND_ID", > "BRAND_NAME", > "LEVEL_ID", > "LEVEL_NAME", > "FACTORY_ID", > "FACTORY_NAME" > ] > } > ], > "measures": [ > { > "name": "_COUNT_", > "function": { > "expression": "COUNT", > "parameter": { > "type": "constant", > "value": "1", > "next_parameter": null > }, > "returntype": "bigint" > }, > "dependent_measure_ref": null > }, > { > "name": "PV", > "function": { > "expression": "SUM", > "parameter": { > "type": "column", > "value": "PV", > "next_parameter": null > }, > "returntype": "bigint" > }, > "dependent_measure_ref": null > }, > { > "name": "USER_ID", > "function": { > "expression": "COUNT_DISTINCT", > "parameter": { > "type": "column", > "value": "USER_ID", > "next_parameter": null > }, > "returntype": "bitmap" > }, > "dependent_measure_ref": null > } > ], > "rowkey": { > "rowkey_columns": [ > { > "column": "DT", > "encoding": "dict" > }, > { > "column": "SPEC_ID", > "encoding": "dict" > }, > { > "column": "SITE_ID", > "encoding": "dict" > }, > { > "column": "DEALER_ID", > "encoding": "dict" > }, > { > "column": "CITY_ID", > "encoding": "dict" > }, > { > "column": "KEY_WORD", > "encoding": "dict" > }, > { > "column": "REF_DOMAIN", > "encoding": "dict" > }, > { > "column": "SITE_CLASS", > "encoding": "dict" > }, > { > "column": "DEVICE_ID", > "encoding": "dict" > }, > { > "column": "NEW_FLAG", > "encoding": "dict" > } > ] > }, > "signature": "8O48Zjj6YujqabUmbVcg+A==", > "last_modified": 1459996931395, > "model_name": "flw_test2", > "null_string": null, > "hbase_mapping": { > "column_family": [ > { > "name": "F1", > "columns": [ > { > "qualifier": "M", > "measure_refs": [ > "_COUNT_", > "PV" > ] > } > ] > }, > { > "name": "F2", > "columns": [ > { > "qualifier": "M", > "measure_refs": [ > "USER_ID" > ] > } > ] > } > ] > }, > "aggregation_groups": [ > { > "includes": [ > "SPEC_ID", > "CITY_ID", > "DT", > "SITE_CLASS", > "SITE_ID" > ], > "select_rule": { > "hierarchy_dims": [], > "mandatory_dims": [ > "DT" > ], > "joint_dims": [ > [ > "SITE_CLASS", > "CITY_ID", > "SPEC_ID" > ] > ] > } > }, > { > "includes": [ > "DT", > "SITE_CLASS", > "DEVICE_ID", > "REF_DOMAIN", > "SITE_ID" > ], > "select_rule": { > "hierarchy_dims": [], > "mandatory_dims": [ > "DT" > ], > "joint_dims": [ > [ > "DEVICE_ID", > "REF_DOMAIN", > "SITE_ID" > ] > ] > } > }, > { > "includes": [ > "DT", > "NEW_FLAG", > "SPEC_ID", > "REF_DOMAIN", > "DEALER_ID" > ], > "select_rule": { > "hierarchy_dims": [], > "mandatory_dims": [ > "DT" > ], > "joint_dims": [] > } > }, > { > "includes": [ > "DT", > "KEY_WORD", > "SITE_ID", > "DEVICE_ID", > "REF_DOMAIN" > ], > "select_rule": { > "hierarchy_dims": [], > "mandatory_dims": [ > "DT" > ], > "joint_dims": [] > } > } > ], > "notify_list": [], > "status_need_notify": [], > "partition_date_start": 1459814400000, > "partition_date_end": 3153600000000, > "auto_merge_time_ranges": [ > 604800000, > 2419200000 > ], > "retention_range": 0, > "engine_type": 2, > "storage_type": 2, > "override_kylin_properties": {} > } -- This message was sent by Atlassian JIRA (v6.3.4#6332)