Thomas Rebele created HIVE-30205:
------------------------------------
Summary: OpenCSVSerde without vectorization throws
UnsupportedOperationException
Key: HIVE-30205
URL: https://issues.apache.org/jira/browse/HIVE-30205
Project: Hive
Issue Type: Bug
Components: Serializers/Deserializers
Reporter: Thomas Rebele
The OpenCSVSerde only stores string columns. HIVE-26388 introduced an
auto-conversion, so that OpenCSVSerde can accept other types as input. It seems
that this auto-conversion works only for vectorized execution. Ideally, the
patch of HIVE-26388 would be extended so that tables created without
vectorization also apply the auto-conversion.
The following q file test:
{code:java}
set hive.vectorized.execution.enabled=true;
create table tab1_vec(abc float);
insert into table tab1_vec values (1.23);
create table tab2_vec row format serde
'org.apache.hadoop.hive.serde2.OpenCSVSerde' with serdeproperties
("separatorChar" = ',' , "quoteChar" = '"') stored as textfile as select * from
tab1_vec;
select * from tab2_vec;
set hive.vectorized.execution.enabled=false;
create table tab1_novec(abc float);
insert into table tab1_novec values (1.23);
create table tab2_novec row format serde
'org.apache.hadoop.hive.serde2.OpenCSVSerde' with serdeproperties
("separatorChar" = ',' , "quoteChar" = '"') stored as textfile as select * from
tab1_novec;
select * from tab2_novec;
{code}
executed with {{mvn test -Drat.skip=true -Pitests -pl serde,itests/qtest
-Dtest=TestMiniLlapLocalCliDriver -Dqfile=opencsvserde_vectorization.q
-Dtest.output.overwrite=true -T1C}} on the most recent commit
(32efdcd9e6cfdfbecd30a4bf454c66f76be6281d), leads to an
{{UnsupportedOperationException: Column type of float is not supported with
OpenCSVSerde}}
Here the complete stack trace:
{code:java}
create table tab2_novec row format serde
'org.apache.hadoop.hive.serde2.OpenCSVSerde' with serdeproperties
("separatorChar" = ',' , "quoteChar" = '"') stored as textfile as select * from
tab1_novec
fname=opencsvserde_vectorization.q
See ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, or
check ./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/
for specific test cases logs.
org.apache.hadoop.hive.ql.exec.tez.TezRuntimeException: Vertex failed,
vertexName=Map 1, vertexId=vertex_1792447139701_0001_4_00, diagnostics=[Task
failed, taskId=task_1792447139701_0001_4_00_000000, diagnostics=[TaskAttempt 0
failed, info=[Error: Error while running task ( failure ) :
attempt_1792447139701_0001_4_00_000000_0:java.lang.RuntimeException:
java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException:
Hive Runtime Error while processing row
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:342)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:277)
at
org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:381)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:86)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:72)
at
java.base/java.security.AccessController.doPrivileged(AccessController.java:714)
at java.base/javax.security.auth.Subject.doAs(Subject.java:525)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:72)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:42)
at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
at
org.apache.hadoop.hive.llap.daemon.impl.StatsRecordingThreadPool$WrappedCallable.call(StatsRecordingThreadPool.java:112)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.lang.RuntimeException:
org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while
processing row
at
org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.processRow(MapRecordSource.java:111)
at
org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.pushRecord(MapRecordSource.java:84)
at
org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.run(MapRecordProcessor.java:418)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:294)
... 15 more
Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error
while processing row
at
org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:580)
at
org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.processRow(MapRecordSource.java:102)
... 18 more
Caused by: java.lang.UnsupportedOperationException: Column type of float is not
supported with OpenCSVSerde
at
org.apache.hadoop.hive.serde2.OpenCSVSerde.doSerialize(OpenCSVSerde.java:136)
at
org.apache.hadoop.hive.serde2.AbstractEncodingAwareSerDe.serialize(AbstractEncodingAwareSerDe.java:54)
at
org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:1166)
at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:889)
at
org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:889)
at
org.apache.hadoop.hive.ql.exec.TableScanOperator.process(TableScanOperator.java:182)
at
org.apache.hadoop.hive.ql.exec.MapOperator$MapOpCtx.forward(MapOperator.java:157)
at
org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:559)
... 19 more
], TaskAttempt 1 failed, info=[Error: Error while running task ( failure ) :
attempt_1792447139701_0001_4_00_000000_1:java.lang.RuntimeException:
java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException:
Hive Runtime Error while processing row
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:342)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:277)
at
org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:381)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:86)
at
org.apache.tez.runtime.task.TaskRunner2Callable$1.run(TaskRunner2Callable.java:72)
at
java.base/java.security.AccessController.doPrivileged(AccessController.java:714)
at java.base/javax.security.auth.Subject.doAs(Subject.java:525)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:72)
at
org.apache.tez.runtime.task.TaskRunner2Callable.callInternal(TaskRunner2Callable.java:42)
at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36)
at
org.apache.hadoop.hive.llap.daemon.impl.StatsRecordingThreadPool$WrappedCallable.call(StatsRecordingThreadPool.java:112)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.lang.RuntimeException:
org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while
processing row
at
org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.processRow(MapRecordSource.java:111)
at
org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.pushRecord(MapRecordSource.java:84)
at
org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.run(MapRecordProcessor.java:418)
at
org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:294)
... 15 more
Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error
while processing row
at
org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:580)
at
org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.processRow(MapRecordSource.java:102)
... 18 more
Caused by: java.lang.UnsupportedOperationException: Column type of float is not
supported with OpenCSVSerde
at
org.apache.hadoop.hive.serde2.OpenCSVSerde.doSerialize(OpenCSVSerde.java:136)
at
org.apache.hadoop.hive.serde2.AbstractEncodingAwareSerDe.serialize(AbstractEncodingAwareSerDe.java:54)
at
org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:1166)
at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:889)
at
org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:889)
at
org.apache.hadoop.hive.ql.exec.TableScanOperator.process(TableScanOperator.java:182)
at
org.apache.hadoop.hive.ql.exec.MapOperator$MapOpCtx.forward(MapOperator.java:157)
at
org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:559)
... 19 more
]], Vertex did not succeed due to OWN_TASK_FAILURE, failedTasks:1
killedTasks:0, Vertex vertex_1792447139701_0001_4_00 [Map 1] killed/failed due
to:OWN_TASK_FAILURE]Vertex killed, vertexName=Reducer 2,
vertexId=vertex_1792447139701_0001_4_01, diagnostics=[Vertex received Kill
while in RUNNING state., Vertex did not succeed due to OTHER_VERTEX_FAILURE,
failedTasks:0 killedTasks:1, Vertex vertex_1792447139701_0001_4_01 [Reducer 2]
killed/failed due to:OTHER_VERTEX_FAILURE]DAG did not succeed due to
VERTEX_FAILURE. failedVertices:1 killedVertices:1
at org.apache.hadoop.hive.ql.exec.tez.TezTask.execute(TezTask.java:287)
at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:215)
at
org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:106)
at org.apache.hadoop.hive.ql.Executor.launchTask(Executor.java:355)
at org.apache.hadoop.hive.ql.Executor.launchTasks(Executor.java:328)
at org.apache.hadoop.hive.ql.Executor.runTasks(Executor.java:245)
at org.apache.hadoop.hive.ql.Executor.execute(Executor.java:106)
at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:347)
at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:192)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:144)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:139)
at
org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:191)
at
org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:236)
at
org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:260)
at org.apache.hadoop.hive.cli.CliDriver.processCmd1(CliDriver.java:204)
at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:130)
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:433)
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:361)
at
org.apache.hadoop.hive.ql.QTestUtil.executeClientInternal(QTestUtil.java:795)
at org.apache.hadoop.hive.ql.QTestUtil.executeClient(QTestUtil.java:765)
at
org.apache.hadoop.hive.cli.control.CoreCliDriver.runTest(CoreCliDriver.java:116)
at
org.apache.hadoop.hive.cli.control.CliAdapter.runTest(CliAdapter.java:140)
at
org.apache.hadoop.hive.cli.TestMiniLlapLocalCliDriver.testCliDriver(TestMiniLlapLocalCliDriver.java:63)
at
java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at
org.apache.hadoop.hive.cli.control.CliAdapter$2$1.evaluate(CliAdapter.java:119)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at
org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at
org.apache.hadoop.hive.cli.control.CliAdapter$1$1.evaluate(CliAdapter.java:90)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at
org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:316)
at
org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:240)
at
org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:214)
at
org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:155)
at
org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:385)
at
org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:162)
at
org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:507)
at
org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:495)
{code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
