[
https://issues.apache.org/jira/browse/HIVE-27875?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
László Bodor updated HIVE-27875:
--------------------------------
Description:
This codepath:
{code}
hiveserver2 <14>1 2023-11-15T16:05:10.504Z hiveserver2-0 hiveserver2 1
a51a9165-623b-4837-b087-818cd7e78d88 [mdc@18060 class="s3a.S3AInputStream"
level="INFO" operationLogLevel="EXECUTION"
queryId="hive_20231115160510_cdab039d-efd4-4711-b75c-c382798b7640"
sessionId="e7f1b1b3-ad51-4823-8a9b-17228c2216ef"
thread="HiveServer2-Background-Pool: Thread-164"] Reopen called,
trace\rjava.lang.RuntimeException
at org.apache.hadoop.fs.s3a.S3AInputStream.reopen(S3AInputStream.java:262)
at
org.apache.hadoop.fs.s3a.S3AInputStream.lambda$lazySeek$1(S3AInputStream.java:437)
at org.apache.hadoop.fs.s3a.Invoker.lambda$maybeRetry$3(Invoker.java:284)
at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:122)
at org.apache.hadoop.fs.s3a.Invoker.lambda$maybeRetry$5(Invoker.java:408)
at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:468)
at org.apache.hadoop.fs.s3a.Invoker.maybeRetry(Invoker.java:404)
at org.apache.hadoop.fs.s3a.Invoker.maybeRetry(Invoker.java:282)
at org.apache.hadoop.fs.s3a.Invoker.maybeRetry(Invoker.java:326)
at org.apache.hadoop.fs.s3a.S3AInputStream.lazySeek(S3AInputStream.java:429)
at org.apache.hadoop.fs.s3a.S3AInputStream.read(S3AInputStream.java:547)
at
org.apache.hadoop.fs.s3a.S3AInputStream.readFully(S3AInputStream.java:838)
at
org.apache.hadoop.fs.FSDataInputStream.readFully(FSDataInputStream.java:118)
at org.apache.orc.impl.ReaderImpl.read(ReaderImpl.java:702)
at org.apache.orc.impl.ReaderImpl.extractFileTail(ReaderImpl.java:806)
at org.apache.orc.impl.ReaderImpl.<init>(ReaderImpl.java:567)
at org.apache.hadoop.hive.ql.io.orc.ReaderImpl.<init>(ReaderImpl.java:61)
at org.apache.hadoop.hive.ql.io.orc.OrcFile.createReader(OrcFile.java:112)
at
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.validateInput(OrcInputFormat.java:655)
at
org.apache.hadoop.hive.ql.io.HiveFileFormatUtils.checkInputFormat(HiveFileFormatUtils.java:207)
at
org.apache.hadoop.hive.ql.exec.MoveTask.checkFileFormats(MoveTask.java:826)
at org.apache.hadoop.hive.ql.exec.MoveTask.execute(MoveTask.java:493)
at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:213)
at
org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:105)
at org.apache.hadoop.hive.ql.Executor.launchTask(Executor.java:356)
at org.apache.hadoop.hive.ql.Executor.launchTasks(Executor.java:329)
at org.apache.hadoop.hive.ql.Executor.runTasks(Executor.java:246)
at org.apache.hadoop.hive.ql.Executor.execute(Executor.java:107)
at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:809)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:546)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:540)
at org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:190)
at
org.apache.hive.service.cli.operation.SQLOperation.runQuery(SQLOperation.java:235)
at
org.apache.hive.service.cli.operation.SQLOperation.access$700(SQLOperation.java:92)
at
org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork$1.run(SQLOperation.java:340)
at java.base/java.security.AccessController.doPrivileged(Native Method)
at java.base/javax.security.auth.Subject.doAs(Subject.java:423)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1899)
at
org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork.run(SQLOperation.java:360)
at
java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at
java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:829)\r
{code}
ReaderImpl.extractFileTail creates an FSDataInputStream but we never call close
on that reader from OrcInputFormat
> OrcInputFormat leaks a CLOSE_WAIT socket with an unclosed input stream
> ----------------------------------------------------------------------
>
> Key: HIVE-27875
> URL: https://issues.apache.org/jira/browse/HIVE-27875
> Project: Hive
> Issue Type: Improvement
> Reporter: László Bodor
> Assignee: László Bodor
> Priority: Major
>
> This codepath:
> {code}
> hiveserver2 <14>1 2023-11-15T16:05:10.504Z hiveserver2-0 hiveserver2 1
> a51a9165-623b-4837-b087-818cd7e78d88 [mdc@18060 class="s3a.S3AInputStream"
> level="INFO" operationLogLevel="EXECUTION"
> queryId="hive_20231115160510_cdab039d-efd4-4711-b75c-c382798b7640"
> sessionId="e7f1b1b3-ad51-4823-8a9b-17228c2216ef"
> thread="HiveServer2-Background-Pool: Thread-164"] Reopen called,
> trace\rjava.lang.RuntimeException
> at org.apache.hadoop.fs.s3a.S3AInputStream.reopen(S3AInputStream.java:262)
> at
> org.apache.hadoop.fs.s3a.S3AInputStream.lambda$lazySeek$1(S3AInputStream.java:437)
> at org.apache.hadoop.fs.s3a.Invoker.lambda$maybeRetry$3(Invoker.java:284)
> at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:122)
> at org.apache.hadoop.fs.s3a.Invoker.lambda$maybeRetry$5(Invoker.java:408)
> at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:468)
> at org.apache.hadoop.fs.s3a.Invoker.maybeRetry(Invoker.java:404)
> at org.apache.hadoop.fs.s3a.Invoker.maybeRetry(Invoker.java:282)
> at org.apache.hadoop.fs.s3a.Invoker.maybeRetry(Invoker.java:326)
> at
> org.apache.hadoop.fs.s3a.S3AInputStream.lazySeek(S3AInputStream.java:429)
> at org.apache.hadoop.fs.s3a.S3AInputStream.read(S3AInputStream.java:547)
> at
> org.apache.hadoop.fs.s3a.S3AInputStream.readFully(S3AInputStream.java:838)
> at
> org.apache.hadoop.fs.FSDataInputStream.readFully(FSDataInputStream.java:118)
> at org.apache.orc.impl.ReaderImpl.read(ReaderImpl.java:702)
> at org.apache.orc.impl.ReaderImpl.extractFileTail(ReaderImpl.java:806)
> at org.apache.orc.impl.ReaderImpl.<init>(ReaderImpl.java:567)
> at org.apache.hadoop.hive.ql.io.orc.ReaderImpl.<init>(ReaderImpl.java:61)
> at org.apache.hadoop.hive.ql.io.orc.OrcFile.createReader(OrcFile.java:112)
> at
> org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.validateInput(OrcInputFormat.java:655)
> at
> org.apache.hadoop.hive.ql.io.HiveFileFormatUtils.checkInputFormat(HiveFileFormatUtils.java:207)
> at
> org.apache.hadoop.hive.ql.exec.MoveTask.checkFileFormats(MoveTask.java:826)
> at org.apache.hadoop.hive.ql.exec.MoveTask.execute(MoveTask.java:493)
> at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:213)
> at
> org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:105)
> at org.apache.hadoop.hive.ql.Executor.launchTask(Executor.java:356)
> at org.apache.hadoop.hive.ql.Executor.launchTasks(Executor.java:329)
> at org.apache.hadoop.hive.ql.Executor.runTasks(Executor.java:246)
> at org.apache.hadoop.hive.ql.Executor.execute(Executor.java:107)
> at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:809)
> at org.apache.hadoop.hive.ql.Driver.run(Driver.java:546)
> at org.apache.hadoop.hive.ql.Driver.run(Driver.java:540)
> at
> org.apache.hadoop.hive.ql.reexec.ReExecDriver.run(ReExecDriver.java:190)
> at
> org.apache.hive.service.cli.operation.SQLOperation.runQuery(SQLOperation.java:235)
> at
> org.apache.hive.service.cli.operation.SQLOperation.access$700(SQLOperation.java:92)
> at
> org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork$1.run(SQLOperation.java:340)
> at java.base/java.security.AccessController.doPrivileged(Native Method)
> at java.base/javax.security.auth.Subject.doAs(Subject.java:423)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1899)
> at
> org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork.run(SQLOperation.java:360)
> at
> java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
> at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
> at
> java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
> at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> at java.base/java.lang.Thread.run(Thread.java:829)\r
> {code}
> ReaderImpl.extractFileTail creates an FSDataInputStream but we never call
> close on that reader from OrcInputFormat
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
