[
https://issues.apache.org/jira/browse/HIVE-27835?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Basapuram Kumar updated HIVE-27835:
-----------------------------------
Description:
Hello Team,
I am trying to use Druid storage handler to create a table.
Steps to repro
Create table
{code:java}
CREATE TABLE emp_zlib_3110_2038 (
`__time` TIMESTAMP,
id int,
name string,
age int,
gender string)
STORED AS ORC
TBLPROPERTIES ('transactional'='true',"orc.compress"="ZLIB");
{code}
insert data
{code:java}
insert into emp_zlib_3110_2038 values(CURRENT_TIMESTAMP(),10,'basa',30,'M');
{code}
Create a DruidStorageHandler table
{code:java}
CREATE TABLE druid_hive_table_3110_2038_02
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.datasource" = "druid_hive_table_3110_2038_02")
AS
select `__time`, id,name,age,gender from emp_zlib_3110_2038 ; {code}
Error:
{code:java}
INFO : Status: Running (Executing on YARN cluster with App id
application_1698753624381_0021)
----------------------------------------------------------------------------------------------
VERTICES MODE STATUS TOTAL COMPLETED RUNNING PENDING FAILED
KILLED
----------------------------------------------------------------------------------------------
Map 1 .......... container SUCCEEDED 1 1 0 0
0 0
Reducer 2 ... container RUNNING 2 1 0 1
4 0
----------------------------------------------------------------------------------------------
VERTICES: 01/02 [=================>>---------] 66% ELAPSED TIME: 37.91 s
----------------------------------------------------------------------------------------------
ERROR : Status: FailedERROR : Vertex failed, vertexName=Reducer 2,
vertexId=vertex_1698753624381_0021_3_01, diagnostics=[Task failed,
taskId=task_1698753624381_0021_3_01_000000, diagnostics=[TaskAttempt 0 failed,
info=[Error: Error while running task ( failure ) :
attempt_1698753624381_0021_3_01_000000_0:java.lang.RuntimeException:
java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException:
Hive Runtime Error while processing row
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:348)
at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:276)
at
org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:381)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:82)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:69)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1762)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:69)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:39)
at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
at
com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:125)
at
com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:57)
at
com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:78)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)Caused by: java.lang.RuntimeException:
org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while
processing row
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:311)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.run(ReduceRecordProcessor.java:291)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:293)
... 16 moreCaused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive
Runtime Error while processing row
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:385)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:301)
... 18 more
Caused by: java.lang.RuntimeException: java.util.concurrent.ExecutionException:
java.lang.NoClassDefFoundError: net/jpountz/lz4/LZ4Factory
at
org.apache.hive.druid.com.google.common.base.Throwables.propagate(Throwables.java:241)
at
org.apache.hadoop.hive.druid.io.DruidRecordWriter.pushSegments(DruidRecordWriter.java:209)
at
org.apache.hadoop.hive.druid.io.DruidRecordWriter.getSegmentIdentifierAndMaybePush(DruidRecordWriter.java:164)
at
org.apache.hadoop.hive.druid.io.DruidRecordWriter.write(DruidRecordWriter.java:258)
at
org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:1160)
at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:888)
at
org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:94)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:370)
... 19 moreCaused
by: java.util.concurrent.ExecutionException: java.lang.NoClassDefFoundError:
net/jpountz/lz4/LZ4Factory at
org.apache.hive.druid.com.google.common.util.concurrent.AbstractFuture.getDoneValue(AbstractFuture.java:552)
at
org.apache.hive.druid.com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:533)
at
org.apache.hive.druid.com.google.common.util.concurrent.FluentFuture$TrustedFuture.get(FluentFuture.java:82)
at
org.apache.hadoop.hive.druid.io.DruidRecordWriter.pushSegments(DruidRecordWriter.java:173)
... 25 more {code}
I see that lz4-java jar is available from hive/lib
{code:java}
$ grep -irn "net.jpountz.lz4.LZ4Factory" hive/lib/
Binary file hive/lib/lz4-java-1.7.1.jar matches {code}
Can u help us how to fix this.?
was:
Hello Team,
I am trying to use Druid storage handler to create a table.
Steps to repro
Create table
{code:java}
CREATE TABLE emp_zlib_3110_2038 (
`__time` TIMESTAMP,
id int,
name string,
age int,
gender string)
STORED AS ORC
TBLPROPERTIES ('transactional'='true',"orc.compress"="ZLIB");
{code}
insert data
{code:java}
insert into emp_zlib_3110_2038 values(CURRENT_TIMESTAMP(),10,'basa',30,'M');
{code}
Create a DruidStorageHandler table
{code:java}
CREATE TABLE druid_hive_table_3110_2038_02
STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
TBLPROPERTIES ("druid.datasource" = "druid_hive_table_3110_2038_02")
AS
select `__time`, id,name,age,gender from emp_zlib_3110_2038 ; {code}
Error:
{code:java}
INFO : Status: Running (Executing on YARN cluster with App id
application_1698753624381_0021)
----------------------------------------------------------------------------------------------
VERTICES MODE STATUS TOTAL COMPLETED RUNNING PENDING
FAILED
KILLED----------------------------------------------------------------------------------------------Map
1 .......... container SUCCEEDED 1 1 0 0
0 0Reducer 2 ... container RUNNING 2 1 0
1 4
0----------------------------------------------------------------------------------------------VERTICES:
01/02 [=================>>---------] 66% ELAPSED TIME: 37.91
s----------------------------------------------------------------------------------------------
ERROR : Status: FailedERROR : Vertex failed, vertexName=Reducer 2,
vertexId=vertex_1698753624381_0021_3_01, diagnostics=[Task failed,
taskId=task_1698753624381_0021_3_01_000000, diagnostics=[TaskAttempt 0 failed,
info=[Error: Error while running task ( failure ) :
attempt_1698753624381_0021_3_01_000000_0:java.lang.RuntimeException:
java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException:
Hive Runtime Error while processing row at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:348)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:276) at
org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:381)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:82)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:69)
at java.security.AccessController.doPrivileged(Native Method) at
javax.security.auth.Subject.doAs(Subject.java:422) at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1762)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:69)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:39)
at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36) at
com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:125)
at
com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:57)
at
com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:78)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)Caused by:
java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException:
Hive Runtime Error while processing row at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:311)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.run(ReduceRecordProcessor.java:291)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:293)
... 16 moreCaused by: org.apache.hadoop.hive.ql.metadata.HiveException:
Hive Runtime Error while processing row at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:385)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:301)
... 18 more
Caused by: java.lang.RuntimeException: java.util.concurrent.ExecutionException:
java.lang.NoClassDefFoundError: net/jpountz/lz4/LZ4Factory at
org.apache.hive.druid.com.google.common.base.Throwables.propagate(Throwables.java:241)
at
org.apache.hadoop.hive.druid.io.DruidRecordWriter.pushSegments(DruidRecordWriter.java:209)
at
org.apache.hadoop.hive.druid.io.DruidRecordWriter.getSegmentIdentifierAndMaybePush(DruidRecordWriter.java:164)
at
org.apache.hadoop.hive.druid.io.DruidRecordWriter.write(DruidRecordWriter.java:258)
at
org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:1160)
at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:888) at
org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:94)
at
org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:370)
... 19 moreCaused
by: java.util.concurrent.ExecutionException: java.lang.NoClassDefFoundError:
net/jpountz/lz4/LZ4Factory at
org.apache.hive.druid.com.google.common.util.concurrent.AbstractFuture.getDoneValue(AbstractFuture.java:552)
at
org.apache.hive.druid.com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:533)
at
org.apache.hive.druid.com.google.common.util.concurrent.FluentFuture$TrustedFuture.get(FluentFuture.java:82)
at
org.apache.hadoop.hive.druid.io.DruidRecordWriter.pushSegments(DruidRecordWriter.java:173)
... 25 more {code}
I see that lz4-java jar is available from hive/lib
{code:java}
$ grep -irn "net.jpountz.lz4.LZ4Factory" hive/lib/
Binary file hive/lib/lz4-java-1.7.1.jar matches {code}
Can u help us how to fix this.?
> java.lang.NoClassDefFoundError: net/jpountz/lz4/LZ4Factory for
> DruidStorageHandler
> ----------------------------------------------------------------------------------
>
> Key: HIVE-27835
> URL: https://issues.apache.org/jira/browse/HIVE-27835
> Project: Hive
> Issue Type: Task
> Components: Hive
> Affects Versions: 4.0.0-alpha-1
> Environment: hive-4-alpha-1
> Druid-27.0.0
> Reporter: Basapuram Kumar
> Priority: Major
>
> Hello Team,
> I am trying to use Druid storage handler to create a table.
> Steps to repro
> Create table
>
> {code:java}
> CREATE TABLE emp_zlib_3110_2038 (
> `__time` TIMESTAMP,
> id int,
> name string,
> age int,
> gender string)
> STORED AS ORC
> TBLPROPERTIES ('transactional'='true',"orc.compress"="ZLIB");
> {code}
> insert data
> {code:java}
> insert into emp_zlib_3110_2038 values(CURRENT_TIMESTAMP(),10,'basa',30,'M');
> {code}
> Create a DruidStorageHandler table
> {code:java}
> CREATE TABLE druid_hive_table_3110_2038_02
> STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
> TBLPROPERTIES ("druid.datasource" = "druid_hive_table_3110_2038_02")
> AS
> select `__time`, id,name,age,gender from emp_zlib_3110_2038 ; {code}
> Error:
> {code:java}
> INFO : Status: Running (Executing on YARN cluster with App id
> application_1698753624381_0021)
> ----------------------------------------------------------------------------------------------
>
> VERTICES MODE STATUS TOTAL COMPLETED RUNNING PENDING FAILED
> KILLED
> ----------------------------------------------------------------------------------------------
> Map 1 .......... container SUCCEEDED 1 1 0 0
> 0 0
> Reducer 2 ... container RUNNING 2 1 0 1
> 4 0
> ----------------------------------------------------------------------------------------------
> VERTICES: 01/02 [=================>>---------] 66% ELAPSED TIME: 37.91 s
> ----------------------------------------------------------------------------------------------
> ERROR : Status: FailedERROR : Vertex failed, vertexName=Reducer 2,
> vertexId=vertex_1698753624381_0021_3_01, diagnostics=[Task failed,
> taskId=task_1698753624381_0021_3_01_000000, diagnostics=[TaskAttempt 0
> failed, info=[Error: Error while running task ( failure ) :
> attempt_1698753624381_0021_3_01_000000_0:java.lang.RuntimeException:
> java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException:
> Hive Runtime Error while processing row
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:348)
>
> at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:276)
> at
> org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:381)
>
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:82)
>
> at
> org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:69)
>
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1762)
>
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:69)
>
> at
> org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:39)
>
> at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
> at
> com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:125)
>
> at
> com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:57)
>
> at
> com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:78)
>
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
>
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
>
> at java.lang.Thread.run(Thread.java:750)Caused by:
> java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException:
> Hive Runtime Error while processing row
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:311)
>
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordProcessor.run(ReduceRecordProcessor.java:291)
>
> at
> org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:293)
>
> ... 16 moreCaused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive
> Runtime Error while processing row
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:385)
>
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource.pushRecord(ReduceRecordSource.java:301)
>
> ... 18 more
> Caused by: java.lang.RuntimeException:
> java.util.concurrent.ExecutionException: java.lang.NoClassDefFoundError:
> net/jpountz/lz4/LZ4Factory
> at
> org.apache.hive.druid.com.google.common.base.Throwables.propagate(Throwables.java:241)
>
> at
> org.apache.hadoop.hive.druid.io.DruidRecordWriter.pushSegments(DruidRecordWriter.java:209)
>
> at
> org.apache.hadoop.hive.druid.io.DruidRecordWriter.getSegmentIdentifierAndMaybePush(DruidRecordWriter.java:164)
>
> at
> org.apache.hadoop.hive.druid.io.DruidRecordWriter.write(DruidRecordWriter.java:258)
>
> at
> org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:1160)
>
> at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:888)
> at
> org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:94)
>
> at
> org.apache.hadoop.hive.ql.exec.tez.ReduceRecordSource$GroupIterator.next(ReduceRecordSource.java:370)
>
> ... 19 moreCaused
> by: java.util.concurrent.ExecutionException: java.lang.NoClassDefFoundError:
> net/jpountz/lz4/LZ4Factory at
> org.apache.hive.druid.com.google.common.util.concurrent.AbstractFuture.getDoneValue(AbstractFuture.java:552)
>
> at
> org.apache.hive.druid.com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:533)
>
> at
> org.apache.hive.druid.com.google.common.util.concurrent.FluentFuture$TrustedFuture.get(FluentFuture.java:82)
>
> at
> org.apache.hadoop.hive.druid.io.DruidRecordWriter.pushSegments(DruidRecordWriter.java:173)
>
> ... 25 more {code}
> I see that lz4-java jar is available from hive/lib
> {code:java}
> $ grep -irn "net.jpountz.lz4.LZ4Factory" hive/lib/
> Binary file hive/lib/lz4-java-1.7.1.jar matches {code}
> Can u help us how to fix this.?
>
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
