Michael Han created SPARK-12123: ----------------------------------- Summary: Spark ava.lang.NullPointerException Key: SPARK-12123 URL: https://issues.apache.org/jira/browse/SPARK-12123 Project: Spark Issue Type: Question Affects Versions: 1.5.2 Reporter: Michael Han
Hi, I'm fresh to study Spark. I download Spark 1.5.2 into my windows C disk. Download the latest Eclipse and create a Java project with maven, The only java class is: package com.qad; import org.apache.spark.api.java.*; import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.Function; public class SparkTest1 { public static void main(String[] args) { String logFile = "README.md"; // Should be some file on your system SparkConf conf = new SparkConf().setMaster("spark://192.168.79.1:7077").setAppName("Simple Application"); JavaSparkContext sc = new JavaSparkContext(conf); // example 1 JavaRDD<String> logData = sc.textFile(logFile); long numAs, numBs; numAs = 0; numBs = 0; JavaRDD<String> logData2 = logData.filter(new Function<String,Boolean>() { private static final long serialVersionUID = 1L; public Boolean call(String s) { return s.contains("Spark is a fast"); } }); numAs = logData2.count(); String content = "Lines with a: " + numAs + ", lines with b: " + numBs; System.out.println(content); WriteText(content,"mh6log.txt"); sc.close(); } private static void WriteText(String content,String fileName) { try { File logFile=new File(fileName); BufferedWriter writer = new BufferedWriter(new FileWriter(logFile)); writer.write (content); //Close writer writer.close(); } catch(Exception e) { e.printStackTrace(); } } } The pom are: <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <groupId>Spark-Test</groupId> <artifactId>Spark-Test</artifactId> <version>1.0</version> <build> <sourceDirectory>src</sourceDirectory> <plugins> <plugin> <artifactId>maven-compiler-plugin</artifactId> <version>3.3</version> <configuration> <source>1.8</source> <target>1.8</target> </configuration> </plugin> </plugins> </build> <dependencies> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-core_2.10</artifactId> <version>1.5.2</version> </dependency> </dependencies> </project> I can run this java class correctly in Eclipse, but exceptions when I using the following command to commit it: spark-submit --master local --class com.qad.SparkTest1 Spark-Test-1.0.jar Who knows which step I was wrong? Thank you. The exceptions are: Exception in thread "main" org.apache.spark.SparkException: Job aborted due to s tage failure: Task 0 in stage 0.0 failed 1 times, most recent failure: Lost task 0.0 in stage 0.0 (TID 0, localhost): java.lang.NullPointerException at java.lang.ProcessBuilder.start(ProcessBuilder.java:1012) at org.apache.hadoop.util.Shell.runCommand(Shell.java:482) at org.apache.hadoop.util.Shell.run(Shell.java:455) at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java: 715) at org.apache.hadoop.fs.FileUtil.chmod(FileUtil.java:873) at org.apache.hadoop.fs.FileUtil.chmod(FileUtil.java:853) at org.apache.spark.util.Utils$.fetchFile(Utils.scala:381) at org.apache.spark.executor.Executor$$anonfun$org$apache$spark$executor $Executor$$updateDependencies$5.apply(Executor.scala:405) at org.apache.spark.executor.Executor$$anonfun$org$apache$spark$executor $Executor$$updateDependencies$5.apply(Executor.scala:397) at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply( TraversableLike.scala:772) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.sca la:98) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.sca la:98) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala :226) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:39) at scala.collection.mutable.HashMap.foreach(HashMap.scala:98) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.s cala:771) at org.apache.spark.executor.Executor.org$apache$spark$executor$Executor $$updateDependencies(Executor.scala:397) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:193) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor. java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor .java:617) at java.lang.Thread.run(Thread.java:745) Driver stacktrace: at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DA GScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1283) at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(D AGScheduler.scala:1271) at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(D AGScheduler.scala:1270) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray. scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47) at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala :1270) at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$ 1.apply(DAGScheduler.scala:697) at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$ 1.apply(DAGScheduler.scala:697) at scala.Option.foreach(Option.scala:236) at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGSchedu ler.scala:697) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(D AGScheduler.scala:1496) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAG Scheduler.scala:1458) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAG Scheduler.scala:1447) at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:567 ) at org.apache.spark.SparkContext.runJob(SparkContext.scala:1824) at org.apache.spark.SparkContext.runJob(SparkContext.scala:1837) at org.apache.spark.SparkContext.runJob(SparkContext.scala:1850) at org.apache.spark.SparkContext.runJob(SparkContext.scala:1921) at org.apache.spark.rdd.RDD.count(RDD.scala:1125) at org.apache.spark.api.java.JavaRDDLike$class.count(JavaRDDLike.scala:4 45) at org.apache.spark.api.java.AbstractJavaRDDLike.count(JavaRDDLike.scala :47) at com.qad.SparkTest1.main(SparkTest1.java:29) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl. java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAcces sorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSub mit$$runMain(SparkSubmit.scala:674) at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:18 0) at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:120) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) Caused by: java.lang.NullPointerException at java.lang.ProcessBuilder.start(ProcessBuilder.java:1012) at org.apache.hadoop.util.Shell.runCommand(Shell.java:482) at org.apache.hadoop.util.Shell.run(Shell.java:455) at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java: 715) at org.apache.hadoop.fs.FileUtil.chmod(FileUtil.java:873) at org.apache.hadoop.fs.FileUtil.chmod(FileUtil.java:853) at org.apache.spark.util.Utils$.fetchFile(Utils.scala:381) at org.apache.spark.executor.Executor$$anonfun$org$apache$spark$executor $Executor$$updateDependencies$5.apply(Executor.scala:405) at org.apache.spark.executor.Executor$$anonfun$org$apache$spark$executor $Executor$$updateDependencies$5.apply(Executor.scala:397) at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply( TraversableLike.scala:772) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.sca la:98) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.sca la:98) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala :226) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:39) at scala.collection.mutable.HashMap.foreach(HashMap.scala:98) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.s cala:771) at org.apache.spark.executor.Executor.org$apache$spark$executor$Executor $$updateDependencies(Executor.scala:397) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:193) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor. java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor .java:617) at java.lang.Thread.run(Thread.java:745) -- This message was sent by Atlassian JIRA (v6.3.4#6332) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org