aokolnychyi commented on issue #104: ManifestReader is not properly closed in BaseTableScan URL: https://github.com/apache/incubator-iceberg/issues/104#issuecomment-478890258 I also see this exception from time to time in `TestParquetWrite`. ``` [Finalizer] WARN org.apache.iceberg.hadoop.HadoopStreams - Unclosed input stream created by: org.apache.iceberg.hadoop.HadoopStreams$HadoopSeekableInputStream.<init>(HadoopStreams.java:77) org.apache.iceberg.hadoop.HadoopStreams.wrap(HadoopStreams.java:52) org.apache.iceberg.hadoop.HadoopInputFile.newStream(HadoopInputFile.java:125) org.apache.iceberg.avro.AvroIterable.newFileReader(AvroIterable.java:95) org.apache.iceberg.avro.AvroIterable.iterator(AvroIterable.java:77) com.google.common.collect.Iterables$6.iterator(Iterables.java:585) com.google.common.collect.Iterables$8.iterator(Iterables.java:709) org.apache.iceberg.ManifestReader.iterator(ManifestReader.java:236) org.apache.iceberg.FilteredManifest.iterator(FilteredManifest.java:128) com.google.common.collect.Iterables$8.iterator(Iterables.java:709) com.google.common.collect.Iterables$3.transform(Iterables.java:508) com.google.common.collect.Iterables$3.transform(Iterables.java:505) com.google.common.collect.TransformedIterator.next(TransformedIterator.java:48) com.google.common.collect.Iterators$5.hasNext(Iterators.java:543) com.google.common.collect.TransformedIterator.hasNext(TransformedIterator.java:43) com.google.common.collect.TransformedIterator.hasNext(TransformedIterator.java:43) com.google.common.collect.Iterators$5.hasNext(Iterators.java:542) org.apache.iceberg.util.BinPacking$PackingIterator.hasNext(BinPacking.java:99) org.apache.iceberg.io.CloseableIterable$4$1.hasNext(CloseableIterable.java:92) com.google.common.collect.Iterators.addAll(Iterators.java:356) com.google.common.collect.Lists.newArrayList(Lists.java:147) com.google.common.collect.Lists.newArrayList(Lists.java:129) org.apache.iceberg.spark.source.Reader.tasks(Reader.java:226) org.apache.iceberg.spark.source.Reader.planInputPartitions(Reader.java:148) org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExec.partitions$lzycompute(DataSourceV2ScanExec.scala:76) org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExec.partitions(DataSourceV2ScanExec.scala:75) org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExec.outputPartitioning(DataSourceV2ScanExec.scala:65) org.apache.spark.sql.execution.exchange.EnsureRequirements$$anonfun$org$apache$spark$sql$execution$exchange$EnsureRequirements$$ensureDistributionAndOrdering$1.apply(EnsureRequirements.scala:150) org.apache.spark.sql.execution.exchange.EnsureRequirements$$anonfun$org$apache$spark$sql$execution$exchange$EnsureRequirements$$ensureDistributionAndOrdering$1.apply(EnsureRequirements.scala:149) scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) scala.collection.immutable.List.foreach(List.scala:392) scala.collection.TraversableLike$class.map(TraversableLike.scala:234) scala.collection.immutable.List.map(List.scala:296) org.apache.spark.sql.execution.exchange.EnsureRequirements.org$apache$spark$sql$execution$exchange$EnsureRequirements$$ensureDistributionAndOrdering(EnsureRequirements.scala:149) org.apache.spark.sql.execution.exchange.EnsureRequirements$$anonfun$apply$1.applyOrElse(EnsureRequirements.scala:304) org.apache.spark.sql.execution.exchange.EnsureRequirements$$anonfun$apply$1.applyOrElse(EnsureRequirements.scala:296) org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:278) org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:278) org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:70) org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:277) org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:275) org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:275) org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:326) org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:187) org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:324) org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:275) org.apache.spark.sql.execution.exchange.EnsureRequirements.apply(EnsureRequirements.scala:296) org.apache.spark.sql.execution.exchange.EnsureRequirements.apply(EnsureRequirements.scala:38) org.apache.spark.sql.execution.QueryExecution$$anonfun$prepareForExecution$1.apply(QueryExecution.scala:87) org.apache.spark.sql.execution.QueryExecution$$anonfun$prepareForExecution$1.apply(QueryExecution.scala:87) scala.collection.LinearSeqOptimized$class.foldLeft(LinearSeqOptimized.scala:124) scala.collection.immutable.List.foldLeft(List.scala:84) org.apache.spark.sql.execution.QueryExecution.prepareForExecution(QueryExecution.scala:87) org.apache.spark.sql.execution.QueryExecution.executedPlan$lzycompute(QueryExecution.scala:77) org.apache.spark.sql.execution.QueryExecution.executedPlan(QueryExecution.scala:77) org.apache.spark.sql.Dataset.withAction(Dataset.scala:3360) org.apache.spark.sql.Dataset.collectAsList(Dataset.scala:2794) org.apache.iceberg.spark.source.TestParquetWrite.testBasicWrite(TestParquetWrite.java:98) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48) org.junit.rules.RunRules.evaluate(RunRules.java:20) org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325) org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78) org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57) org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) org.junit.runners.ParentRunner.run(ParentRunner.java:363) org.junit.runner.JUnitCore.run(JUnitCore.java:137) com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:68) com.intellij.rt.execution.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:47) com.intellij.rt.execution.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:242) com.intellij.rt.execution.junit.JUnitStarter.main(JUnitStarter.java:70) ```
---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
