[
https://issues.apache.org/jira/browse/SPARK-4900?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14312769#comment-14312769
]
Mike Beyer commented on SPARK-4900:
-----------------------------------
put a snapshot test data 1000x1000 matrix to
https://dl.dropboxusercontent.com/u/8489998/test_matrix_1.zip
calling:
String filename = "/custompath/27637/test_matrix_1";
RDD<Vector> vectorRDD =
MLUtils.loadVectors(javaSparkContext.sc(), filename);
vectorRDD.cache();
System.out.println("trtRowRDD.count():\t" +
vectorRDD.count());
RowMatrix rowMatrix = new RowMatrix(vectorRDD);
System.out.println("rowMatrix.numRows():\t" +
rowMatrix.numRows());
System.out.println("rowMatrix.numCols():\t" +
rowMatrix.numCols());
{
int k = 10;
boolean computeU = true;
double rCond = 1.0E-9d;
SingularValueDecomposition<RowMatrix, Matrix>
svd = rowMatrix.computeSVD(k, computeU, rCond);
RowMatrix u = svd.U();
RDD<Vector> uRowsRDD = u.rows();
System.out.println("uRowsRDD.count():\t" +
uRowsRDD.count());
Vector s = svd.s();
System.out.println("s.size():\t" + s.size());
Matrix v = svd.V();
System.out.println("v.numRows():\t" +
v.numRows());
System.out.println("v.numCols():\t" +
v.numCols());
}
results in:
maxFeatureSpaceTermNumber: 1000
trtRowRDD.count(): 1000
rowMatrix.numRows(): 1000
rowMatrix.numCols(): 1000
15/02/09 19:56:59 WARN PrimaryRunnerSpark:
java.lang.IllegalStateException: ARPACK returns non-zero info = 3 Please refer
ARPACK user guide for error message.
at
org.apache.spark.mllib.linalg.EigenValueDecomposition$.symmetricEigs(EigenValueDecomposition.scala:120)
at
org.apache.spark.mllib.linalg.distributed.RowMatrix.computeSVD(RowMatrix.scala:258)
at
org.apache.spark.mllib.linalg.distributed.RowMatrix.computeSVD(RowMatrix.scala:190)
at
com.example.processing.spark.SVDProcessing2.createSVD_2(SVDProcessing2.java:184)
at
com.example.processing.spark.RunnerSpark.main(PrimaryRunnerSpark.java:27)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:483)
at sbt.Run.invokeMain(Run.scala:67)
at sbt.Run.run0(Run.scala:61)
at sbt.Run.sbt$Run$$execute$1(Run.scala:51)
at sbt.Run$$anonfun$run$1.apply$mcV$sp(Run.scala:55)
at sbt.Run$$anonfun$run$1.apply(Run.scala:55)
at sbt.Run$$anonfun$run$1.apply(Run.scala:55)
at sbt.Logger$$anon$4.apply(Logger.scala:85)
at sbt.TrapExit$App.run(TrapExit.scala:248)
at java.lang.Thread.run(Thread.java:745)
15/02/09 19:56:59 INFO TimeCounter: TIMER
[com.example.processing.spark.PrimaryRunnerSpark] : 13.0 Seconds
TIMER [com.example.processing.spark.PrimaryRunnerSpark] : 13.0 Seconds
15/02/09 19:56:59 ERROR ContextCleaner: Error cleaning broadcast 20
java.lang.InterruptedException
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
at
scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:208)
at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:218)
at
scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223)
at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107)
at
scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53)
at scala.concurrent.Await$.result(package.scala:107)
at
org.apache.spark.storage.BlockManagerMaster.removeBroadcast(BlockManagerMaster.scala:137)
at
org.apache.spark.broadcast.TorrentBroadcast$.unpersist(TorrentBroadcast.scala:227)
at
org.apache.spark.broadcast.TorrentBroadcastFactory.unbroadcast(TorrentBroadcastFactory.scala:45)
at
org.apache.spark.broadcast.BroadcastManager.unbroadcast(BroadcastManager.scala:66)
at
org.apache.spark.ContextCleaner.doCleanupBroadcast(ContextCleaner.scala:185)
at
org.apache.spark.ContextCleaner$$anonfun$org$apache$spark$ContextCleaner$$keepCleaning$1$$anonfun$apply$mcV$sp$2.apply(
ContextCleaner.scala:147)
at
org.apache.spark.ContextCleaner$$anonfun$org$apache$spark$ContextCleaner$$keepCleaning$1$$anonfun$apply$mcV$sp$2.apply(
ContextCleaner.scala:138)
at scala.Option.foreach(Option.scala:236)
at
org.apache.spark.ContextCleaner$$anonfun$org$apache$spark$ContextCleaner$$keepCleaning$1.apply$mcV$sp(ContextCleaner.sc
ala:138)
at
org.apache.spark.ContextCleaner$$anonfun$org$apache$spark$ContextCleaner$$keepCleaning$1.apply(ContextCleaner.scala:134
)
at
org.apache.spark.ContextCleaner$$anonfun$org$apache$spark$ContextCleaner$$keepCleaning$1.apply(ContextCleaner.scala:134
)
at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1550)
at
org.apache.spark.ContextCleaner.org$apache$spark$ContextCleaner$$keepCleaning(ContextCleaner.scala:133)
at org.apache.spark.ContextCleaner$$anon$3.run(ContextCleaner.scala:65)
15/02/09 19:56:59 ERROR Utils: Uncaught exception in thread SparkListenerBus
java.lang.InterruptedException
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:998)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
at java.util.concurrent.Semaphore.acquire(Semaphore.java:312)
at
org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1.apply$mcV$sp(LiveListenerBus.scala:48)
at
org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1.apply(LiveListenerBus.scala:47)
at
org.apache.spark.scheduler.LiveListenerBus$$anon$1$$anonfun$run$1.apply(LiveListenerBus.scala:47)
at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1550)
at
org.apache.spark.scheduler.LiveListenerBus$$anon$1.run(LiveListenerBus.scala:46)
[success] Total time: 20 s, completed Feb 9, 2015 7:56:59 PM
> MLlib SingularValueDecomposition ARPACK IllegalStateException
> --------------------------------------------------------------
>
> Key: SPARK-4900
> URL: https://issues.apache.org/jira/browse/SPARK-4900
> Project: Spark
> Issue Type: Bug
> Components: MLlib
> Affects Versions: 1.1.1, 1.2.0
> Environment: Ubuntu 1410, Java HotSpot(TM) 64-Bit Server VM (build
> 25.25-b02, mixed mode)
> spark local mode
> Reporter: Mike Beyer
>
> java.lang.reflect.InvocationTargetException
> ...
> Caused by: java.lang.IllegalStateException: ARPACK returns non-zero info = 3
> Please refer ARPACK user guide for error message.
> at
> org.apache.spark.mllib.linalg.EigenValueDecomposition$.symmetricEigs(EigenValueDecomposition.scala:120)
> at
> org.apache.spark.mllib.linalg.distributed.RowMatrix.computeSVD(RowMatrix.scala:235)
> at
> org.apache.spark.mllib.linalg.distributed.RowMatrix.computeSVD(RowMatrix.scala:171)
> ...
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
