Alexey Zinoviev created IGNITE-12435:
----------------------------------------
Summary: [Spark] Add support for saving to existing table via
saveAsTable
Key: IGNITE-12435
URL: https://issues.apache.org/jira/browse/IGNITE-12435
Project: Ignite
Issue Type: Sub-task
Components: spark
Reporter: Alexey Zinoviev
Assignee: Alexey Zinoviev
Fix For: 2.9
Tests in IgniteSQLDataFrameIgniteSessionWriteSpec are muted due to strange
error related to working with filesystems and schemas
All three tests generates the same error when you are trying to call
saveAsTable as a terminal operation on dataframe write:
java.io.IOException: No FileSystem for scheme: ignitejava.io.IOException: No
FileSystem for scheme: ignite
at org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:2586) at
org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2593) at
org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:91) at
org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2632) at
org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2614) at
org.apache.hadoop.fs.FileSystem.get(FileSystem.java:370) at
org.apache.hadoop.fs.Path.getFileSystem(Path.java:296) at
org.apache.spark.sql.catalyst.catalog.SessionCatalog.validateTableLocation(SessionCatalog.scala:333)
at
org.apache.spark.sql.execution.command.CreateDataSourceTableAsSelectCommand.run(createDataSourceTables.scala:170)
at
org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:104)
at
org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:102)
at
org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:122)
at
org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:131)
at
org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:127)
at
org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:155)
at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:152)
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:127) at
org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:80)
at
org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:80) at
org.apache.spark.sql.DataFrameWriter$$anonfun$runCommand$1.apply(DataFrameWriter.scala:676)
at
org.apache.spark.sql.DataFrameWriter$$anonfun$runCommand$1.apply(DataFrameWriter.scala:676)
at
org.apache.spark.sql.execution.SQLExecution$$anonfun$withNewExecutionId$1.apply(SQLExecution.scala:78)
at
org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:125)
at
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:73)
at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:676)
at org.apache.spark.sql.DataFrameWriter.createTable(DataFrameWriter.scala:474)
at org.apache.spark.sql.DataFrameWriter.saveAsTable(DataFrameWriter.scala:449)
at org.apache.spark.sql.DataFrameWriter.saveAsTable(DataFrameWriter.scala:409)
at
org.apache.ignite.spark.IgniteSQLDataFrameIgniteSessionWriteSpec$$anonfun$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(IgniteSQLDataFrameIgniteSessionWriteSpec.scala:45)
at
org.apache.ignite.spark.IgniteSQLDataFrameIgniteSessionWriteSpec$$anonfun$1$$anonfun$apply$mcV$sp$1.apply(IgniteSQLDataFrameIgniteSessionWriteSpec.scala:35)
at
org.apache.ignite.spark.IgniteSQLDataFrameIgniteSessionWriteSpec$$anonfun$1$$anonfun$apply$mcV$sp$1.apply(IgniteSQLDataFrameIgniteSessionWriteSpec.scala:35)
at
org.scalatest.Transformer$$anonfun$apply$1.apply$mcV$sp(Transformer.scala:22)
at org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85) at
org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104) at
org.scalatest.Transformer.apply(Transformer.scala:22) at
org.scalatest.Transformer.apply(Transformer.scala:20) at
org.scalatest.FunSpecLike$$anon$1.apply(FunSpecLike.scala:422) at
org.scalatest.Suite$class.withFixture(Suite.scala:1122) at
org.scalatest.FunSpec.withFixture(FunSpec.scala:1626) at
org.scalatest.FunSpecLike$class.invokeWithFixture$1(FunSpecLike.scala:419) at
org.scalatest.FunSpecLike$$anonfun$runTest$1.apply(FunSpecLike.scala:431) at
org.scalatest.FunSpecLike$$anonfun$runTest$1.apply(FunSpecLike.scala:431) at
org.scalatest.SuperEngine.runTestImpl(Engine.scala:306) at
org.scalatest.FunSpecLike$class.runTest(FunSpecLike.scala:431) at
org.apache.ignite.spark.AbstractDataFrameSpec.org$scalatest$BeforeAndAfter$$super$runTest(AbstractDataFrameSpec.scala:39)
at org.scalatest.BeforeAndAfter$class.runTest(BeforeAndAfter.scala:200) at
org.apache.ignite.spark.AbstractDataFrameSpec.runTest(AbstractDataFrameSpec.scala:39)
at org.scalatest.FunSpecLike$$anonfun$runTests$1.apply(FunSpecLike.scala:464)
at org.scalatest.FunSpecLike$$anonfun$runTests$1.apply(FunSpecLike.scala:464)
at
org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:413)
at
org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401)
at scala.collection.immutable.List.foreach(List.scala:392) at
org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401) at
org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:390)
at
org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:427)
at
org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401)
at scala.collection.immutable.List.foreach(List.scala:392) at
org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401) at
org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:396)
at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:483) at
org.scalatest.FunSpecLike$class.runTests(FunSpecLike.scala:464) at
org.scalatest.FunSpec.runTests(FunSpec.scala:1626) at
org.scalatest.Suite$class.run(Suite.scala:1424) at
org.scalatest.FunSpec.org$scalatest$FunSpecLike$$super$run(FunSpec.scala:1626)
at org.scalatest.FunSpecLike$$anonfun$run$1.apply(FunSpecLike.scala:468) at
org.scalatest.FunSpecLike$$anonfun$run$1.apply(FunSpecLike.scala:468) at
org.scalatest.SuperEngine.runImpl(Engine.scala:545) at
org.scalatest.FunSpecLike$class.run(FunSpecLike.scala:468) at
org.apache.ignite.spark.AbstractDataFrameSpec.org$scalatest$BeforeAndAfterAll$$super$run(AbstractDataFrameSpec.scala:39)
at
org.scalatest.BeforeAndAfterAll$class.liftedTree1$1(BeforeAndAfterAll.scala:257)
at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:256) at
org.apache.ignite.spark.AbstractDataFrameSpec.org$scalatest$BeforeAndAfter$$super$run(AbstractDataFrameSpec.scala:39)
at org.scalatest.BeforeAndAfter$class.run(BeforeAndAfter.scala:241) at
org.apache.ignite.spark.AbstractDataFrameSpec.run(AbstractDataFrameSpec.scala:39)
at
org.scalatest.junit.JUnitRunner.run(JUnitRunner.scala:99) at
org.junit.runner.JUnitCore.run(JUnitCore.java:160) at
com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:68)
at
com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:33)
at
com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:230)
at com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:58)
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
