Hi, I am putting data using spark-redshift connector("com.databricks" %% "spark-redshift" % "1.1.0") which uses s3. Basically when I use *fs.s3a.fast.upload=true* hadoop property then my program works fine.
But if this property is false then it causes issue and throws NullPointerException. Basically it tries to find* ${hadoop.tmp.dir}/s3a* property here https://github.com/gnudeep/hadoop-2.6/blob/master/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java#L267 Here: contextCfgItemName:- *${hadoop.tmp.dir}/s3a* *It looks strange that property is having forward slash in it.* Can someone explain why this is happening. Plz find the huge stack-trace down: org.apache.spark.SparkException: Job aborted. at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelation$$anonfun$run$1.apply$mcV$sp(InsertIntoHadoopFsRelation.scala:156) at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelation$$anonfun$run$1.apply(InsertIntoHadoopFsRelation.scala:108) at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelation$$anonfun$run$1.apply(InsertIntoHadoopFsRelation.scala:108) at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:56) at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelation.run(InsertIntoHadoopFsRelation.scala:108) at org.apache.spark.sql.execution.ExecutedCommand.sideEffectResult$lzycompute(commands.scala:58) at org.apache.spark.sql.execution.ExecutedCommand.sideEffectResult(commands.scala:56) at org.apache.spark.sql.execution.ExecutedCommand.doExecute(commands.scala:70) at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$5.apply(SparkPlan.scala:132) at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$5.apply(SparkPlan.scala:130) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:150) at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:130) at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:55) at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:55) at org.apache.spark.sql.execution.datasources.ResolvedDataSource$.apply(ResolvedDataSource.scala:256) at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:148) at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:139) at com.databricks.spark.redshift.RedshiftWriter.unloadData(RedshiftWriter.scala:278) at com.databricks.spark.redshift.RedshiftWriter.saveToRedshift(RedshiftWriter.scala:346) at com.databricks.spark.redshift.DefaultSource.createRelation(DefaultSource.scala:106) at org.apache.spark.sql.execution.datasources.ResolvedDataSource$.apply(ResolvedDataSource.scala:222) at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:148) at com.goibibo.sqlshift.commons.MySQLToRedshiftMigrator$.storeToRedshift(MySQLToRedshiftMigrator.scala:325) at com.goibibo.sqlshift.SQLShift$$anonfun$run$2.apply(SQLShift.scala:156) at com.goibibo.sqlshift.SQLShift$$anonfun$run$2.apply(SQLShift.scala:127) at scala.collection.immutable.List.foreach(List.scala:318) at com.goibibo.sqlshift.SQLShift$.run(SQLShift.scala:127) at com.goibibo.sqlshift.SQLShift$.start(SQLShift.scala:196) at com.goibibo.sqlshift.FullDump.startSqlShift(FullDump.scala:47) at com.goibibo.sqlshift.FullDump$$anonfun$1.apply$mcV$sp(FullDump.scala:52) at com.goibibo.sqlshift.FullDump$$anonfun$1.apply(FullDump.scala:50) at com.goibibo.sqlshift.FullDump$$anonfun$1.apply(FullDump.scala:50) at org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85) at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104) at org.scalatest.Transformer.apply(Transformer.scala:22) at org.scalatest.Transformer.apply(Transformer.scala:20) at org.scalatest.FlatSpecLike$$anon$1.apply(FlatSpecLike.scala:1682) at org.scalatest.TestSuite$class.withFixture(TestSuite.scala:196) at org.scalatest.FlatSpec.withFixture(FlatSpec.scala:1685) at org.scalatest.FlatSpecLike$class.invokeWithFixture$1(FlatSpecLike.scala:1679) at org.scalatest.FlatSpecLike$$anonfun$runTest$1.apply(FlatSpecLike.scala:1692) at org.scalatest.FlatSpecLike$$anonfun$runTest$1.apply(FlatSpecLike.scala:1692) at org.scalatest.SuperEngine.runTestImpl(Engine.scala:289) at org.scalatest.FlatSpecLike$class.runTest(FlatSpecLike.scala:1692) at org.scalatest.FlatSpec.runTest(FlatSpec.scala:1685) at org.scalatest.FlatSpecLike$$anonfun$runTests$1.apply(FlatSpecLike.scala:1750) at org.scalatest.FlatSpecLike$$anonfun$runTests$1.apply(FlatSpecLike.scala:1750) at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:396) at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:384) at scala.collection.immutable.List.foreach(List.scala:318) at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:384) at org.scalatest.SuperEngine.org $scalatest$SuperEngine$$runTestsInBranch(Engine.scala:373) at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:410) at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:384) at scala.collection.immutable.List.foreach(List.scala:318) at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:384) at org.scalatest.SuperEngine.org $scalatest$SuperEngine$$runTestsInBranch(Engine.scala:379) at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:461) at org.scalatest.FlatSpecLike$class.runTests(FlatSpecLike.scala:1750) at org.scalatest.FlatSpec.runTests(FlatSpec.scala:1685) at org.scalatest.Suite$class.run(Suite.scala:1147) at org.scalatest.FlatSpec.org $scalatest$FlatSpecLike$$super$run(FlatSpec.scala:1685) at org.scalatest.FlatSpecLike$$anonfun$run$1.apply(FlatSpecLike.scala:1795) at org.scalatest.FlatSpecLike$$anonfun$run$1.apply(FlatSpecLike.scala:1795) at org.scalatest.SuperEngine.runImpl(Engine.scala:521) at org.scalatest.FlatSpecLike$class.run(FlatSpecLike.scala:1795) at com.goibibo.sqlshift.FullDump.org $scalatest$BeforeAndAfterAll$$super$run(FullDump.scala:20) at org.scalatest.BeforeAndAfterAll$class.liftedTree1$1(BeforeAndAfterAll.scala:213) at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:210) at com.goibibo.sqlshift.FullDump.run(FullDump.scala:20) at org.scalatest.tools.Framework.org $scalatest$tools$Framework$$runSuite(Framework.scala:314) at org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:480) at sbt.TestRunner.runTest$1(TestFramework.scala:106) at sbt.TestRunner.run(TestFramework.scala:117) at sbt.TestFramework$$anon$2$$anonfun$$lessinit$greater$1.$anonfun$apply$1(TestFramework.scala:262) at sbt.TestFramework$.sbt$TestFramework$$withContextLoader(TestFramework.scala:233) at sbt.TestFramework$$anon$2$$anonfun$$lessinit$greater$1.apply(TestFramework.scala:262) at sbt.TestFramework$$anon$2$$anonfun$$lessinit$greater$1.apply(TestFramework.scala:262) at sbt.TestFunction.apply(TestFramework.scala:271) at sbt.Tests$.$anonfun$toTask$1(Tests.scala:281) at sbt.std.Transform$$anon$3.$anonfun$apply$2(System.scala:46) at sbt.std.Transform$$anon$4.work(System.scala:66) at sbt.Execute.$anonfun$submit$2(Execute.scala:262) at sbt.internal.util.ErrorHandling$.wideConvert(ErrorHandling.scala:16) at sbt.Execute.work(Execute.scala:271) at sbt.Execute.$anonfun$submit$1(Execute.scala:262) at sbt.ConcurrentRestrictions$$anon$4.$anonfun$submitValid$1(ConcurrentRestrictions.scala:174) at sbt.CompletionService$$anon$2.call(CompletionService.scala:36) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 0.0 failed 1 times, most recent failure: Lost task 0.0 in stage 0.0 (TID 0, localhost): java.lang.NullPointerException at org.apache.hadoop.fs.LocalDirAllocator$AllocatorPerContext.confChanged(LocalDirAllocator.java:268) at org.apache.hadoop.fs.LocalDirAllocator$AllocatorPerContext.getLocalPathForWrite(LocalDirAllocator.java:344) at org.apache.hadoop.fs.LocalDirAllocator$AllocatorPerContext.createTmpFileForWrite(LocalDirAllocator.java:412) at org.apache.hadoop.fs.LocalDirAllocator.createTmpFileForWrite(LocalDirAllocator.java:198) at org.apache.hadoop.fs.s3a.S3AOutputStream.<init>(S3AOutputStream.java:91) at org.apache.hadoop.fs.s3a.S3AFileSystem.create(S3AFileSystem.java:456) at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:905) at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:886) at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:783) at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:772) at com.databricks.spark.avro.AvroOutputWriter$$anon$1.getAvroFileOutputStream(AvroOutputWriter.scala:79) at org.apache.avro.mapreduce.AvroKeyOutputFormat.getRecordWriter(AvroKeyOutputFormat.java:105) at com.databricks.spark.avro.AvroOutputWriter.<init>(AvroOutputWriter.scala:82) at com.databricks.spark.avro.AvroOutputWriterFactory.newInstance(AvroOutputWriterFactory.scala:31) at org.apache.spark.sql.execution.datasources.BaseWriterContainer.newOutputWriter(WriterContainer.scala:129) at org.apache.spark.sql.execution.datasources.DefaultWriterContainer.writeRows(WriterContainer.scala:255) at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelation$$anonfun$run$1$$anonfun$apply$mcV$sp$3.apply(InsertIntoHadoopFsRelation.scala:150) at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelation$$anonfun$run$1$$anonfun$apply$mcV$sp$3.apply(InsertIntoHadoopFsRelation.scala:150) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) at org.apache.spark.scheduler.Task.run(Task.scala:89) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Thanks -- Shivam Sharma Data Engineer @ Goibibo Indian Institute Of Information Technology, Design and Manufacturing Jabalpur Mobile No- (+91) 8882114744 Email:- 28shivamsha...@gmail.com LinkedIn:-*https://www.linkedin.com/in/28shivamsharma <https://www.linkedin.com/in/28shivamsharma>*