[
https://issues.apache.org/jira/browse/HIVE-26615?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17615531#comment-17615531
]
László Bodor commented on HIVE-26615:
-------------------------------------
this is not as easy as I wished, a couple issues need to be addressed, e.g.
{code}
2022-10-11T00:01:40,127 ERROR [be3ae85f-c1a6-4c92-a694-2e73e4eb354c Listener at
localhost/50845] FileOperations: Failed to read stats from filesystem
java.util.concurrent.ExecutionException: java.lang.IndexOutOfBoundsException:
Index: 7735, Size: 0
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
~[?:1.8.0_292]
at java.util.concurrent.FutureTask.get(FutureTask.java:192)
~[?:1.8.0_292]
at
org.apache.hadoop.hive.ql.stats.fs.FSStatsAggregator.connect(FSStatsAggregator.java:94)
~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at
org.apache.hadoop.hive.ql.stats.BasicStatsTask.createStatsAggregator(BasicStatsTask.java:441)
~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at
org.apache.hadoop.hive.ql.stats.BasicStatsTask.aggregateStats(BasicStatsTask.java:282)
~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at
org.apache.hadoop.hive.ql.stats.BasicStatsTask.process(BasicStatsTask.java:108)
~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at org.apache.hadoop.hive.ql.exec.StatsTask.execute(StatsTask.java:107)
~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:214)
~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at
org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:105)
~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at org.apache.hadoop.hive.ql.Executor.launchTask(Executor.java:354)
~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:?]
at org.apache.hadoop.hive.ql.Executor.launchTasks(Executor.java:327)
~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:?]
at org.apache.hadoop.hive.ql.Executor.runTasks(Executor.java:244)
~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:?]
at org.apache.hadoop.hive.ql.Executor.execute(Executor.java:105)
~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:?]
at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:370)
~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:?]
at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:205)
~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:?]
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:154)
~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:?]
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:149)
~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:?]
at
org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:185)
~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at
org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:228)
~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at
org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:255)
~[hive-cli-4.0.0-alpha-2-SNAPSHOT.jar:?]
at org.apache.hadoop.hive.cli.CliDriver.processCmd1(CliDriver.java:200)
~[hive-cli-4.0.0-alpha-2-SNAPSHOT.jar:?]
at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:126)
~[hive-cli-4.0.0-alpha-2-SNAPSHOT.jar:?]
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:421)
~[hive-cli-4.0.0-alpha-2-SNAPSHOT.jar:?]
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:352)
~[hive-cli-4.0.0-alpha-2-SNAPSHOT.jar:?]
at
org.apache.hadoop.hive.ql.QTestUtil.executeClientInternal(QTestUtil.java:727)
~[classes/:?]
at
org.apache.hadoop.hive.ql.QTestUtil.executeClient(QTestUtil.java:697)
~[classes/:?]
at
org.apache.hadoop.hive.cli.control.CoreCliDriver.runTest(CoreCliDriver.java:114)
~[classes/:?]
at
org.apache.hadoop.hive.cli.control.CliAdapter.runTest(CliAdapter.java:157)
~[classes/:?]
at
org.apache.hadoop.hive.cli.TestEncryptedHDFSCliDriver.testCliDriver(TestEncryptedHDFSCliDriver.java:62)
~[test-classes/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
~[?:1.8.0_292]
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
~[?:1.8.0_292]
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
~[?:1.8.0_292]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_292]
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
~[junit-4.13.jar:4.13]
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
~[junit-4.13.jar:4.13]
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
~[junit-4.13.jar:4.13]
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
~[junit-4.13.jar:4.13]
at
org.apache.hadoop.hive.cli.control.CliAdapter$2$1.evaluate(CliAdapter.java:135)
~[classes/:?]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
~[junit-4.13.jar:4.13]
at
org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
~[junit-4.13.jar:4.13]
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
~[junit-4.13.jar:4.13]
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
~[junit-4.13.jar:4.13]
at org.junit.runners.Suite.runChild(Suite.java:128)
~[junit-4.13.jar:4.13]
at org.junit.runners.Suite.runChild(Suite.java:27)
~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
~[junit-4.13.jar:4.13]
at
org.apache.hadoop.hive.cli.control.CliAdapter$1$1.evaluate(CliAdapter.java:95)
~[classes/:?]
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
~[junit-4.13.jar:4.13]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
~[junit-4.13.jar:4.13]
at
org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:365)
~[surefire-junit4-3.0.0-M4.jar:3.0.0-M4]
at
org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:273)
~[surefire-junit4-3.0.0-M4.jar:3.0.0-M4]
at
org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238)
~[surefire-junit4-3.0.0-M4.jar:3.0.0-M4]
at
org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:159)
~[surefire-junit4-3.0.0-M4.jar:3.0.0-M4]
at
org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:377)
~[surefire-booter-3.0.0-M4.jar:3.0.0-M4]
at
org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:138)
~[surefire-booter-3.0.0-M4.jar:3.0.0-M4]
at
org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:465)
~[surefire-booter-3.0.0-M4.jar:3.0.0-M4]
at
org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:451)
~[surefire-booter-3.0.0-M4.jar:3.0.0-M4]
Caused by: java.lang.IndexOutOfBoundsException: Index: 7735, Size: 0
at java.util.ArrayList.rangeCheck(ArrayList.java:659) ~[?:1.8.0_292]
at java.util.ArrayList.get(ArrayList.java:435) ~[?:1.8.0_292]
at
org.apache.hive.com.esotericsoftware.kryo.util.MapReferenceResolver.getReadObject(MapReferenceResolver.java:83)
~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at
org.apache.hive.com.esotericsoftware.kryo.Kryo.readReferenceOrNull(Kryo.java:855)
~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at
org.apache.hive.com.esotericsoftware.kryo.Kryo.readObject(Kryo.java:705)
~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at
org.apache.hadoop.hive.ql.exec.SerializationUtilities$KryoWithHooks.readObject(SerializationUtilities.java:209)
~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at
org.apache.hadoop.hive.ql.stats.fs.FSStatsAggregator.lambda$connect$0(FSStatsAggregator.java:85)
~[hive-exec-4.0.0-alpha-2-SNAPSHOT.jar:4.0.0-alpha-2-SNAPSHOT]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
~[?:1.8.0_292]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
~[?:1.8.0_292]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
~[?:1.8.0_292]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_292]
{code}
> TestEncryptedHDFSCliDriver to use Tez
> -------------------------------------
>
> Key: HIVE-26615
> URL: https://issues.apache.org/jira/browse/HIVE-26615
> Project: Hive
> Issue Type: Improvement
> Reporter: László Bodor
> Assignee: László Bodor
> Priority: Major
> Labels: pull-request-available
> Attachments:
> org.apache.hadoop.hive.cli.TestEncryptedHDFSCliDriver-output.txt
>
> Time Spent: 0.5h
> Remaining Estimate: 0h
>
> https://github.com/apache/hive/blob/eb860203fcdc126788e62e932b40a5f3ff944458/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java#L255-L261
> we set it to MR and then check if it's tez, which is already confusing
> {code}
> setClusterType(MiniClusterType.MR);
> setFsType(QTestMiniClusters.FsType.ENCRYPTED_HDFS); // override
> default FsType.HDFS
> if (getClusterType() == MiniClusterType.TEZ) {
> setHiveConfDir("data/conf/tez");
> } else {
> setHiveConfDir("data/conf");
> }
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
