Hi list,
I have a Spark cluster with 3 nodes. I'm calling spark-shell with some
packages to connect to AWS S3 and Cassandra:
spark-shell \
--packages
org.apache.hadoop:hadoop-aws:2.7.3,com.amazonaws:aws-java-sdk:1.7.4,datastax:spark-cassandra-connector:2.0.6-s_2.11
\
--conf spark.cassandra.connection.host=10.100.120.100,10.100.120.101 \
--conf spark.cassandra.auth.username=cassandra \
--conf spark.cassandra.auth.password=cassandra \
--master spark://10.100.120.104:7077
Then running this test app:
sc.stop
import org.apache.spark._
import org.apache.spark.sql._
import org.apache.spark.sql.types._
import org.apache.spark.sql.Row
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.functions.from_json
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext._
import java.sql.Timestamp
import java.io.File
import org.apache.commons.io.IOUtils
import java.net.URL
import java.nio.charset.Charset
import org.apache.hadoop.fs
System.setProperty("com.amazonaws.services.s3.enableV4", "true")
val region = "eu-central-1"
val conf = new SparkConf(true).setMaster("local[*]").setAppName("S3
connect")
val sc = new SparkContext(conf)
sc.setLocalProperty("spark.default.parallelism", "30")
sc.hadoopConfiguration.set("fs.s3a.impl",
"org.apache.hadoop.fs.s3a.S3AFileSystem")
sc.hadoopConfiguration.set("com.amazonaws.services.s3.enableV4", "true")
sc.hadoopConfiguration.set("fs.s3a.endpoint", "s3." + region +
".amazonaws.com")
val sqlContext = new SQLContext(sc)
val s3r = sqlContext.read.json("s3a://mybucket/folder/file.json")
s3r.take(1)
With .setMaster("local[*]") the application runs nice, but removing the
setmaster and let the entire cluster work I'm getting:
WARN TaskSchedulerImpl: Initial job has not accepted any resources;
check your cluster UI to ensure that workers are registered and have
sufficient resources
How can I make my extra packages available to the entire cluster?
Thanks in advance
---------------------------------------------------------------------
To unsubscribe e-mail: user-unsubscr...@spark.apache.org
