[
https://issues.apache.org/jira/browse/SPARK-16725?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15394311#comment-15394311
]
Min Wei commented on SPARK-16725:
---------------------------------
I am not sure if the Hadoop jar is the issue, as I am just using the spark
shell for some local testing.
I "fixed" my test after upgrading the Guava in the Spark core jar per the diff
file. Here is a way to repro it after building the spark-cassandra connector.
./bin/spark-shell --jars
./spark-cassandra-connector-assembly-1.6.0-27-g5760745.jar
The following exception goes away after the "upgrade". I agree that there would
be other cases where the hadoop jar could cause problems.
java.lang.IllegalStateException: Detected Guava issue #1635 which indicates
that a version of Guava less than 16.01 is in use. This introduces codec
resolution issues and potentially other incompatibility issues in the driver.
Please upgrade to Guava 16.01 or later.
at com.datastax.driver.core.SanityChecks.checkGuava(SanityChecks.java:62)
at com.datastax.driver.core.SanityChecks.check(SanityChecks.java:36)
at com.datastax.driver.core.Cluster.<clinit>(Cluster.java:68)
at
com.datastax.spark.connector.cql.DefaultConnectionFactory$.clusterBuilder(CassandraConnectionFactory.scala:37)
at
com.datastax.spark.connector.cql.DefaultConnectionFactory$.createCluster(CassandraConnectionFactory.scala:98)
at
com.datastax.spark.connector.cql.CassandraConnector$.com$datastax$spark$connector$cql$CassandraConnector$$createSession(CassandraConnector.scala:163)
at
com.datastax.spark.connector.cql.CassandraConnector$$anonfun$3.apply(CassandraConnector.scala:157)
at
com.datastax.spark.connector.cql.CassandraConnector$$anonfun$3.apply(CassandraConnector.scala:157)
at
com.datastax.spark.connector.cql.RefCountedCache.createNewValueAndKeys(RefCountedCache.scala:34)
at
com.datastax.spark.connector.cql.RefCountedCache.acquire(RefCountedCache.scala:60)
at
com.datastax.spark.connector.cql.CassandraConnector.openSession(CassandraConnector.scala:85)
at
com.datastax.spark.connector.cql.CassandraConnector.withSessionDo(CassandraConnector.scala:114)
at
com.datastax.spark.connector.cql.CassandraConnector.withClusterDo(CassandraConnector.scala:127)
at com.datastax.spark.connector.cql.Schema$.fromCassandra(Schema.scala:346)
at
com.datastax.spark.connector.cql.Schema$.tableFromCassandra(Schema.scala:366)
at
com.datastax.spark.connector.rdd.CassandraTableRowReaderProvider$class.tableDef(CassandraTableRowReaderProvider.scala:52)
at
com.datastax.spark.connector.rdd.CassandraTableScanRDD.tableDef$lzycompute(CassandraTableScanRDD.scala:60)
at
com.datastax.spark.connector.rdd.CassandraTableScanRDD.tableDef(CassandraTableScanRDD.scala:60)
at
com.datastax.spark.connector.rdd.CassandraTableRowReaderProvider$class.verify(CassandraTableRowReaderProvider.scala:140)
at
com.datastax.spark.connector.rdd.CassandraTableScanRDD.verify(CassandraTableScanRDD.scala:60)
at
com.datastax.spark.connector.rdd.CassandraTableScanRDD.getPartitions(CassandraTableScanRDD.scala:246)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:248)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:246)
at scala.Option.getOrElse(Option.scala:121)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:246)
at org.apache.spark.rdd.RDD$$anonfun$take$1.apply(RDD.scala:1280)
at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:358)
at org.apache.spark.rdd.RDD.take(RDD.scala:1275)
at com.datastax.spark.connector.rdd.CassandraRDD.take(CassandraRDD.scala:132)
at com.datastax.spark.connector.rdd.CassandraRDD.take(CassandraRDD.scala:133)
at org.apache.spark.rdd.RDD$$anonfun$first$1.apply(RDD.scala:1315)
at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:358)
at org.apache.spark.rdd.RDD.first(RDD.scala:1314)
... 52 elided
> Migrate Guava to 16+?
> ---------------------
>
> Key: SPARK-16725
> URL: https://issues.apache.org/jira/browse/SPARK-16725
> Project: Spark
> Issue Type: Improvement
> Components: Build
> Affects Versions: 2.0.1
> Reporter: Min Wei
> Priority: Minor
> Original Estimate: 12h
> Remaining Estimate: 12h
>
> Currently Spark depends on an old version of Guava, version 14. However
> Spark-cassandra driver asserts on Guava version 16 and above.
> It would be great to update the Guava dependency to version 16+
> diff --git a/core/src/main/scala/org/apache/spark/SecurityManager.scala
> b/core/src/main/scala/org/apache/spark/SecurityManager.scala
> index f72c7de..abddafe 100644
> --- a/core/src/main/scala/org/apache/spark/SecurityManager.scala
> +++ b/core/src/main/scala/org/apache/spark/SecurityManager.scala
> @@ -23,7 +23,7 @@ import java.security.{KeyStore, SecureRandom}
> import java.security.cert.X509Certificate
> import javax.net.ssl._
>
> -import com.google.common.hash.HashCodes
> +import com.google.common.hash.HashCode
> import com.google.common.io.Files
> import org.apache.hadoop.io.Text
>
> @@ -432,7 +432,7 @@ private[spark] class SecurityManager(sparkConf: SparkConf)
> val secret = new Array[Byte](length)
> rnd.nextBytes(secret)
>
> - val cookie = HashCodes.fromBytes(secret).toString()
> + val cookie = HashCode.fromBytes(secret).toString()
> SparkHadoopUtil.get.addSecretKeyToUserCredentials(SECRET_LOOKUP_KEY,
> cookie)
> cookie
> } else {
> diff --git a/core/src/main/scala/org/apache/spark/SparkEnv.scala
> b/core/src/main/scala/org/apache/spark/SparkEnv.scala
> index af50a6d..02545ae 100644
> --- a/core/src/main/scala/org/apache/spark/SparkEnv.scala
> +++ b/core/src/main/scala/org/apache/spark/SparkEnv.scala
> @@ -72,7 +72,7 @@ class SparkEnv (
>
> // A general, soft-reference map for metadata needed during HadoopRDD
> split computation
> // (e.g., HadoopFileRDD uses this to cache JobConfs and InputFormats).
> - private[spark] val hadoopJobMetadata = new
> MapMaker().softValues().makeMap[String, Any]()
> + private[spark] val hadoopJobMetadata = new
> MapMaker().weakValues().makeMap[String, Any]()
>
> private[spark] var driverTmpDir: Option[String] = None
>
> diff --git a/pom.xml b/pom.xml
> index d064cb5..7c3e036 100644
> --- a/pom.xml
> +++ b/pom.xml
> @@ -368,8 +368,7 @@
> <dependency>
> <groupId>com.google.guava</groupId>
> <artifactId>guava</artifactId>
> - <version>14.0.1</version>
> - <scope>provided</scope>
> + <version>19.0</version>
> </dependency>
> <!-- End of shaded deps -->
> <dependency>
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
