[ https://issues.apache.org/jira/browse/SPARK-29133?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Jungtaek Lim resolved SPARK-29133. ---------------------------------- Resolution: Invalid Just affected by problematic commit (had been reverted) [https://github.com/apache/spark/commit/850833fa177ec1f265e143fc383e40ec2c8341a6] See more: [https://github.com/apache/spark/pull/25790#issuecomment-531517800] > Flaky test: > org.apache.spark.sql.execution.datasources.orc.OrcV1SchemaPruningSuite > ---------------------------------------------------------------------------------- > > Key: SPARK-29133 > URL: https://issues.apache.org/jira/browse/SPARK-29133 > Project: Spark > Issue Type: Bug > Components: SQL, Tests > Affects Versions: 3.0.0 > Reporter: Jungtaek Lim > Priority: Major > > [https://amplab.cs.berkeley.edu/jenkins/view/Spark%20QA%20Test/job/spark-master-test-maven-hadoop-2.7-jdk-11-ubuntu-testing/1547/testReport/] > [https://amplab.cs.berkeley.edu/jenkins/view/Spark%20QA%20Test/job/spark-master-test-maven-hadoop-2.7-jdk-11-ubuntu-testing/1546/testReport/] > Bunch of tests in > org.apache.spark.sql.execution.datasources.orc.OrcV1SchemaPruningSuite fail > on CI build. Some tests fail due to SparkContext leak, other tests fail with > below stack trace: > {code:java} > java.util.NoSuchElementException: None.get > at scala.None$.get(Option.scala:366) > at scala.None$.get(Option.scala:364) > at > org.apache.spark.sql.execution.datasources.BasicWriteJobStatsTracker$.metrics(BasicWriteStatsTracker.scala:173) > at > org.apache.spark.sql.execution.command.DataWritingCommand.metrics(DataWritingCommand.scala:51) > at > org.apache.spark.sql.execution.command.DataWritingCommand.metrics$(DataWritingCommand.scala:51) > at > org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.metrics$lzycompute(InsertIntoHadoopFsRelationCommand.scala:47) > at > org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.metrics(InsertIntoHadoopFsRelationCommand.scala:47) > at > org.apache.spark.sql.execution.command.DataWritingCommandExec.metrics$lzycompute(commands.scala:100) > at > org.apache.spark.sql.execution.command.DataWritingCommandExec.metrics(commands.scala:100) > at > org.apache.spark.sql.execution.SparkPlanInfo$.fromSparkPlan(SparkPlanInfo.scala:61) > at > org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$4(SQLExecution.scala:98) > at > org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160) > at > org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:87) > at > org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:830) > at > org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:311) > at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:295) > at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:238) > at > org.apache.spark.sql.execution.datasources.FileBasedDataSourceTest.makeDataSourceFile(FileBasedDataSourceTest.scala:99) > at > org.apache.spark.sql.execution.datasources.FileBasedDataSourceTest.makeDataSourceFile$(FileBasedDataSourceTest.scala:96) > at > org.apache.spark.sql.execution.datasources.SchemaPruningSuite.makeDataSourceFile(SchemaPruningSuite.scala:33) > at > org.apache.spark.sql.execution.datasources.SchemaPruningSuite.$anonfun$withContacts$1(SchemaPruningSuite.scala:300) > at > org.apache.spark.sql.execution.datasources.SchemaPruningSuite.$anonfun$withContacts$1$adapted(SchemaPruningSuite.scala:297) > at > org.apache.spark.sql.catalyst.plans.SQLHelper.withTempPath(SQLHelper.scala:62) > at > org.apache.spark.sql.catalyst.plans.SQLHelper.withTempPath$(SQLHelper.scala:59) > at org.apache.spark.sql.QueryTest.withTempPath(QueryTest.scala:31) > at > org.apache.spark.sql.execution.datasources.SchemaPruningSuite.withContacts(SchemaPruningSuite.scala:297) > at > org.apache.spark.sql.execution.datasources.SchemaPruningSuite.$anonfun$testSchemaPruning$6(SchemaPruningSuite.scala:286) > at > org.apache.spark.sql.catalyst.plans.SQLHelper.withSQLConf(SQLHelper.scala:47) > at > org.apache.spark.sql.catalyst.plans.SQLHelper.withSQLConf$(SQLHelper.scala:31) > at > org.apache.spark.sql.execution.datasources.SchemaPruningSuite.org$apache$spark$sql$test$SQLTestUtilsBase$$super$withSQLConf(SchemaPruningSuite.scala:33) > at > org.apache.spark.sql.test.SQLTestUtilsBase.withSQLConf(SQLTestUtils.scala:231) > at > org.apache.spark.sql.test.SQLTestUtilsBase.withSQLConf$(SQLTestUtils.scala:229) > at > org.apache.spark.sql.execution.datasources.SchemaPruningSuite.withSQLConf(SchemaPruningSuite.scala:33) > at > org.apache.spark.sql.execution.datasources.SchemaPruningSuite.$anonfun$testSchemaPruning$5(SchemaPruningSuite.scala:286) > at > scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) > at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85) > at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83) > at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104) > at org.scalatest.Transformer.apply(Transformer.scala:22) > at org.scalatest.Transformer.apply(Transformer.scala:20) > at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:186) > at org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:149) > at > org.scalatest.FunSuiteLike.invokeWithFixture$1(FunSuiteLike.scala:184) > at org.scalatest.FunSuiteLike.$anonfun$runTest$1(FunSuiteLike.scala:196) > at org.scalatest.SuperEngine.runTestImpl(Engine.scala:289) > at org.scalatest.FunSuiteLike.runTest(FunSuiteLike.scala:196) > at org.scalatest.FunSuiteLike.runTest$(FunSuiteLike.scala:178) > at > org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterEach$$super$runTest(SparkFunSuite.scala:56) > at > org.scalatest.BeforeAndAfterEach.runTest(BeforeAndAfterEach.scala:221) > at > org.scalatest.BeforeAndAfterEach.runTest$(BeforeAndAfterEach.scala:214) > at org.apache.spark.SparkFunSuite.runTest(SparkFunSuite.scala:56) > at > org.scalatest.FunSuiteLike.$anonfun$runTests$1(FunSuiteLike.scala:229) > at > org.scalatest.SuperEngine.$anonfun$runTestsInBranch$1(Engine.scala:396) > at scala.collection.immutable.List.foreach(List.scala:392) > at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:384) > at org.scalatest.SuperEngine.runTestsInBranch(Engine.scala:379) > at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:461) > at org.scalatest.FunSuiteLike.runTests(FunSuiteLike.scala:229) > at org.scalatest.FunSuiteLike.runTests$(FunSuiteLike.scala:228) > at org.scalatest.FunSuite.runTests(FunSuite.scala:1560) > at org.scalatest.Suite.run(Suite.scala:1147) > at org.scalatest.Suite.run$(Suite.scala:1129) > at > org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1560) > at org.scalatest.FunSuiteLike.$anonfun$run$1(FunSuiteLike.scala:233) > at org.scalatest.SuperEngine.runImpl(Engine.scala:521) > at org.scalatest.FunSuiteLike.run(FunSuiteLike.scala:233) > at org.scalatest.FunSuiteLike.run$(FunSuiteLike.scala:232) > at > org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterAll$$super$run(SparkFunSuite.scala:56) > at > org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:213) > at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210) > at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208) > at org.apache.spark.SparkFunSuite.run(SparkFunSuite.scala:56) > at org.scalatest.Suite.callExecuteOnSuite$1(Suite.scala:1210) > at org.scalatest.Suite.$anonfun$runNestedSuites$1(Suite.scala:1257) > at > scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36) > at > scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33) > at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198) > at org.scalatest.Suite.runNestedSuites(Suite.scala:1255) > at org.scalatest.Suite.runNestedSuites$(Suite.scala:1189) > at > org.scalatest.tools.DiscoverySuite.runNestedSuites(DiscoverySuite.scala:30) > at org.scalatest.Suite.run(Suite.scala:1144) > at org.scalatest.Suite.run$(Suite.scala:1129) > at org.scalatest.tools.DiscoverySuite.run(DiscoverySuite.scala:30) > at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:45) > at > org.scalatest.tools.Runner$.$anonfun$doRunRunRunDaDoRunRun$13(Runner.scala:1346) > at > org.scalatest.tools.Runner$.$anonfun$doRunRunRunDaDoRunRun$13$adapted(Runner.scala:1340) > at scala.collection.immutable.List.foreach(List.scala:392) > at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:1340) > at > org.scalatest.tools.Runner$.$anonfun$runOptionallyWithPassFailReporter$24(Runner.scala:1031) > at > org.scalatest.tools.Runner$.$anonfun$runOptionallyWithPassFailReporter$24$adapted(Runner.scala:1010) > at > org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:1506) > at > org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1010) > at org.scalatest.tools.Runner$.main(Runner.scala:827) > at org.scalatest.tools.Runner.main(Runner.scala){code} -- This message was sent by Atlassian Jira (v8.3.4#803005) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org