[
https://issues.apache.org/jira/browse/HIVE-18308?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Juliusz Skowron updated HIVE-18308:
-----------------------------------
Attachment: sample.gz
Sample dataset
> Error inserting data into many partitions
> -----------------------------------------
>
> Key: HIVE-18308
> URL: https://issues.apache.org/jira/browse/HIVE-18308
> Project: Hive
> Issue Type: Bug
> Components: Hive
> Affects Versions: 2.3.1
> Reporter: Juliusz Skowron
> Attachments: sample.gz
>
>
> I am trying to partition data by a field of high cardinality (~75K). Is this
> even possible?
> Tried different ways and it keeps failing. Even on very small sample data
> (12MB gzipped).
> Below are queries and errors they throw. Anything I can do to work this out?
> {noformat}
> CREATE EXTERNAL TABLE IF NOT EXISTS history_sample_hdfs(
> ts bigint,
> sp float,
> cr float,
> id string)
> ROW FORMAT DELIMITED
> FIELDS TERMINATED BY ','
> STORED AS TEXTFILE
> LOCATION '/history_sample';
> CREATE EXTERNAL TABLE IF NOT EXISTS history_sample_partitioned_hdfs(
> ts bigint,
> sp float,
> cr float) PARTITIONED BY(id string)
> ROW FORMAT DELIMITED
> FIELDS TERMINATED BY ','
> STORED AS TEXTFILE
> LOCATION
> '/history_sample_partitioned/';
> {noformat}
> {noformat}
> set mapred.reduce.tasks=1000;
> insert overwrite table history_sample_partitioned_hdfs
> PARTITION(id)
> select ts,
> sp,
> cr,
> id
> from history_sample_hdfs
> distribute by id;
> {noformat}
> {noformat}
> Vertex failed, vertexName=Reducer 2, vertexId=vertex_1513674323124_0019_5_01,
> diagnostics=[Task failed, taskId=task_1513674323124_0019_5_01_000000,
> diagnostics=[TaskAttempt 0 failed, info=[Error: Error while running task (
> failure ) :
> attempt_1513674323124_0019_5_01_000000_0:java.lang.RuntimeException:
> org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while
> processing row (tag=0)
> {"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:211)
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:168)
> at
> org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37)
> at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime
> Error while processing row (tag=0)
> {"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:365)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:250)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.run(ReduceRecordProcessor.java:317)
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:185)
> ... 14 more
> Caused by: java.lang.NullPointerException
> at
> org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:762)
> at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:897)
> at
> org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:356)
> ... 17 more
> ], TaskAttempt 1 failed, info=[Error: Error while running task ( failure ) :
> attempt_1513674323124_0019_5_01_000000_1:java.lang.RuntimeException:
> org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while
> processing row (tag=0)
> {"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:211)
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:168)
> at
> org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37)
> at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime
> Error while processing row (tag=0)
> {"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:365)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:250)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.run(ReduceRecordProcessor.java:317)
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:185)
> ... 14 more
> Caused by: java.lang.NullPointerException
> at
> org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:762)
> at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:897)
> at
> org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:356)
> ... 17 more
> ], TaskAttempt 2 failed, info=[Error: Error while running task ( failure ) :
> attempt_1513674323124_0019_5_01_000000_2:java.lang.RuntimeException:
> org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while
> processing row (tag=0)
> {"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:211)
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:168)
> at
> org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37)
> at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime
> Error while processing row (tag=0)
> {"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:365)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:250)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.run(ReduceRecordProcessor.java:317)
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:185)
> ... 14 more
> Caused by: java.lang.NullPointerException
> at
> org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:762)
> at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:897)
> at
> org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:356)
> ... 17 more
> ], TaskAttempt 3 failed, info=[Error: Error while running task ( failure ) :
> attempt_1513674323124_0019_5_01_000000_3:java.lang.RuntimeException:
> org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while
> processing row (tag=0)
> {"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:211)
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:168)
> at
> org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37)
> at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime
> Error while processing row (tag=0)
> {"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:365)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:250)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.run(ReduceRecordProcessor.java:317)
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:185)
> ... 14 more
> Caused by: java.lang.NullPointerException
> at
> org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:762)
> at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:897)
> at
> org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:356)
> ... 17 more
> ]], Vertex did not succeed due to OWN_TASK_FAILURE, failedTasks:1
> killedTasks:999, Vertex vertex_1513674323124_0019_5_01 [Reducer 2]
> killed/failed due to:OWN_TASK_FAILURE]
> DAG did not succeed due to VERTEX_FAILURE. failedVertices:1 killedVertices:0
> FAILED: Execution Error, return code 2 from
> org.apache.hadoop.hive.ql.exec.tez.TezTask. Vertex failed, vertexName=Reducer
> 2, vertexId=vertex_1513674323124_0019_5_01, diagnostics=[Task failed,
> taskId=task_1513674323124_0019_5_01_000000, diagnostics=[TaskAttempt 0
> failed, info=[Error: Error while running task ( failure ) :
> attempt_1513674323124_0019_5_01_000000_0:java.lang.RuntimeException:
> org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while
> processing row (tag=0)
> {"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:211)
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:168)
> at
> org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37)
> at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime
> Error while processing row (tag=0)
> {"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:365)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:250)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.run(ReduceRecordProcessor.java:317)
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:185)
> ... 14 more
> Caused by: java.lang.NullPointerException
> at
> org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:762)
> at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:897)
> at
> org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:356)
> ... 17 more
> ], TaskAttempt 1 failed, info=[Error: Error while running task ( failure ) :
> attempt_1513674323124_0019_5_01_000000_1:java.lang.RuntimeException:
> org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while
> processing row (tag=0)
> {"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:211)
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:168)
> at
> org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37)
> at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime
> Error while processing row (tag=0)
> {"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:365)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:250)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.run(ReduceRecordProcessor.java:317)
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:185)
> ... 14 more
> Caused by: java.lang.NullPointerException
> at
> org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:762)
> at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:897)
> at
> org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:356)
> ... 17 more
> ], TaskAttempt 2 failed, info=[Error: Error while running task ( failure ) :
> attempt_1513674323124_0019_5_01_000000_2:java.lang.RuntimeException:
> org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while
> processing row (tag=0)
> {"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:211)
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:168)
> at
> org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37)
> at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime
> Error while processing row (tag=0)
> {"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:365)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:250)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.run(ReduceRecordProcessor.java:317)
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:185)
> ... 14 more
> Caused by: java.lang.NullPointerException
> at
> org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:762)
> at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:897)
> at
> org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:356)
> ... 17 more
> ], TaskAttempt 3 failed, info=[Error: Error while running task ( failure ) :
> attempt_1513674323124_0019_5_01_000000_3:java.lang.RuntimeException:
> org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while
> processing row (tag=0)
> {"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:211)
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:168)
> at
> org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:370)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:73)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:61)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:61)
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:37)
> at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime
> Error while processing row (tag=0)
> {"key":{},"value":{"_col0":1510399139699,"_col1":0.0,"_col2":360.0,"_col3":"367633640"}}
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:365)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:250)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.run(ReduceRecordProcessor.java:317)
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:185)
> ... 14 more
> Caused by: java.lang.NullPointerException
> at
> org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:762)
> at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:897)
> at
> org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:356)
> ... 17 more
> ]], Vertex did not succeed due to OWN_TASK_FAILURE, failedTasks:1
> killedTasks:999, Vertex vertex_1513674323124_0019_5_01 [Reducer 2]
> killed/failed due to:OWN_TASK_FAILURE]DAG did not succeed due to
> VERTEX_FAILURE. failedVertices:1 killedVertices:0
> {noformat}
> {noformat}
> set mapred.reduce.tasks=1; --back to default
> insert overwrite table history_sample_partitioned_hdfs
> PARTITION(id)
> select ts,
> sp,
> cr,
> id
> from history_sample_hdfs;
> {noformat}
> {noformat}
> Vertex failed, vertexName=Reducer 2, vertexId=vertex_1513674323124_0019_6_01,
> diagnostics=[Task failed, taskId=task_1513674323124_0019_6_01_000000,
> diagnostics=[TaskAttempt 0 failed, info=[Container
> container_1513674323124_0019_01_000175 finished with diagnostics set to
> [Container failed, exitCode=-104. Container
> [pid=2746,containerID=container_1513674323124_0019_01_000175] is running
> beyond physical memory limits. Current usage: 1.4 GB of 1.4 GB physical
> memory used; 4.2 GB of 7.0 GB virtual memory used. Killing container.
> Dump of the process-tree for container_1513674323124_0019_01_000175 :
> |- PID PPID PGRPID SESSID CMD_NAME USER_MODE_TIME(MILLIS)
> SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) RSSMEM_USAGE(PAGES) FULL_CMD_LINE
> |- 2746 2744 2746 2746 (bash) 1 1 115818496 672 /bin/bash -c
> /usr/lib/jvm/java-openjdk/bin/java -server -Djava.net.preferIPv4Stack=true
> -Dhadoop.metrics.log.level=WARN
> -Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator
> -Dlog4j.configuration=tez-container-log4j.properties
> -Dyarn.app.container.log.dir=/var/log/hadoop-yarn/containers/application_1513674323124_0019/container_1513674323124_0019_01_000175
> -Dtez.root.logger=INFO,CLA -Djava.net.preferIPv4Stack=true
> -Dhadoop.metrics.log.level=WARN -Xmx1152m
> -Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator
> -Dlog4j.configuration=tez-container-log4j.properties
> -Dyarn.app.container.log.dir=/var/log/hadoop-yarn/containers/application_1513674323124_0019/container_1513674323124_0019_01_000175
> -Dtez.root.logger=INFO,CLA
> -Djava.io.tmpdir=/mnt1/yarn/usercache/hadoop/appcache/application_1513674323124_0019/container_1513674323124_0019_01_000175/tmp
> org.apache.tez.runtime.task.TezChild
> ip-10-22-1-155.eu-west-1.compute.internal 44339
> container_1513674323124_0019_01_000175 application_1513674323124_0019 1
> 1>/var/log/hadoop-yarn/containers/application_1513674323124_0019/container_1513674323124_0019_01_000175/stdout
>
> 2>/var/log/hadoop-yarn/containers/application_1513674323124_0019/container_1513674323124_0019_01_000175/stderr
>
> |- 2761 2746 2746 2746 (java) 4325 1233 4384346112 376642
> /usr/lib/jvm/java-openjdk/bin/java -server -Djava.net.preferIPv4Stack=true
> -Dhadoop.metrics.log.level=WARN
> -Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator
> -Dlog4j.configuration=tez-container-log4j.properties
> -Dyarn.app.container.log.dir=/var/log/hadoop-yarn/containers/application_1513674323124_0019/container_1513674323124_0019_01_000175
> -Dtez.root.logger=INFO,CLA -Djava.net.preferIPv4Stack=true
> -Dhadoop.metrics.log.level=WARN -Xmx1152m
> -Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator
> -Dlog4j.configuration=tez-container-log4j.properties
> -Dyarn.app.container.log.dir=/var/log/hadoop-yarn/containers/application_1513674323124_0019/container_1513674323124_0019_01_000175
> -Dtez.root.logger=INFO,CLA
> -Djava.io.tmpdir=/mnt1/yarn/usercache/hadoop/appcache/application_1513674323124_0019/container_1513674323124_0019_01_000175/tmp
> org.apache.tez.runtime.task.TezChild
> ip-10-22-1-155.eu-west-1.compute.internal 44339
> container_1513674323124_0019_01_000175 application_1513674323124_0019 1
> Container killed on request. Exit code is 143
> Container exited with a non-zero exit code 143
> ]], TaskAttempt 1 failed, info=[Container
> container_1513674323124_0019_01_000176 finished with diagnostics set to
> [Container failed, exitCode=-104. Container
> [pid=22630,containerID=container_1513674323124_0019_01_000176] is running
> beyond physical memory limits. Current usage: 1.4 GB of 1.4 GB physical
> memory used; 4.5 GB of 7.0 GB virtual memory used. Killing container.
> Dump of the process-tree for container_1513674323124_0019_01_000176 :
> |- PID PPID PGRPID SESSID CMD_NAME USER_MODE_TIME(MILLIS)
> SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) RSSMEM_USAGE(PAGES) FULL_CMD_LINE
> |- 22630 22628 22630 22630 (bash) 0 1 115818496 674 /bin/bash -c
> /usr/lib/jvm/java-openjdk/bin/java -server -Djava.net.preferIPv4Stack=true
> -Dhadoop.metrics.log.level=WARN
> -Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator
> -Dlog4j.configuration=tez-container-log4j.properties
> -Dyarn.app.container.log.dir=/var/log/hadoop-yarn/containers/application_1513674323124_0019/container_1513674323124_0019_01_000176
> -Dtez.root.logger=INFO,CLA -Djava.net.preferIPv4Stack=true
> -Dhadoop.metrics.log.level=WARN -Xmx1152m
> -Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator
> -Dlog4j.configuration=tez-container-log4j.properties
> -Dyarn.app.container.log.dir=/var/log/hadoop-yarn/containers/application_1513674323124_0019/container_1513674323124_0019_01_000176
> -Dtez.root.logger=INFO,CLA
> -Djava.io.tmpdir=/mnt1/yarn/usercache/hadoop/appcache/application_1513674323124_0019/container_1513674323124_0019_01_000176/tmp
> org.apache.tez.runtime.task.TezChild
> ip-10-22-1-155.eu-west-1.compute.internal 44339
> container_1513674323124_0019_01_000176 application_1513674323124_0019 1
> 1>/var/log/hadoop-yarn/containers/application_1513674323124_0019/container_1513674323124_0019_01_000176/stdout
>
> 2>/var/log/hadoop-yarn/containers/application_1513674323124_0019/container_1513674323124_0019_01_000176/stderr
>
> |- 22644 22630 22630 22630 (java) 4798 1395 4666630144 368835
> /usr/lib/jvm/java-openjdk/bin/java -server -Djava.net.preferIPv4Stack=true
> -Dhadoop.metrics.log.level=WARN
> -Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator
> -Dlog4j.configuration=tez-container-log4j.properties
> -Dyarn.app.container.log.dir=/var/log/hadoop-yarn/containers/application_1513674323124_0019/container_1513674323124_0019_01_000176
> -Dtez.root.logger=INFO,CLA -Djava.net.preferIPv4Stack=true
> -Dhadoop.metrics.log.level=WARN -Xmx1152m
> -Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator
> -Dlog4j.configuration=tez-container-log4j.properties
> -Dyarn.app.container.log.dir=/var/log/hadoop-yarn/containers/application_1513674323124_0019/container_1513674323124_0019_01_000176
> -Dtez.root.logger=INFO,CLA
> -Djava.io.tmpdir=/mnt1/yarn/usercache/hadoop/appcache/application_1513674323124_0019/container_1513674323124_0019_01_000176/tmp
> org.apache.tez.runtime.task.TezChild
> ip-10-22-1-155.eu-west-1.compute.internal 44339
> container_1513674323124_0019_01_000176 application_1513674323124_0019 1
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
