Mohamed Elagamy created SPARK-20934:
---------------------------------------
Summary: Task is hung at inner join, would work with other kind of
joins
Key: SPARK-20934
URL: https://issues.apache.org/jira/browse/SPARK-20934
Project: Spark
Issue Type: Bug
Components: Shuffle, Spark Core
Affects Versions: 2.1.0
Reporter: Mohamed Elagamy
I am using spark 2.1.0 to read from parquets and inner join between different
dataframes, but it gets stuck at the inner join step and never show any
progress, here is the thread dump
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:208)
scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:218)
scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:153)
scala.concurrent.Await$$anonfun$ready$1.apply(package.scala:169)
scala.concurrent.Await$$anonfun$ready$1.apply(package.scala:169)
scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53)
scala.concurrent.Await$.ready(package.scala:169)
com.wd.perf.collector.spark.report.utils.GroupUtils$.generateReportInGroups(GroupUtils.scala:70)
com.wd.perf.collector.spark.report.jobs.definition.JobSummaryDefinitionParquetCreator$.com$wd$perf$collector$spark$report$jobs$definition$JobSummaryDefinitionParquetCreator$$generateHourlyBySWH(JobSummaryDefinitionParquetCreator.scala:82)
com.wd.perf.collector.spark.report.jobs.definition.JobSummaryDefinitionParquetCreator$$anonfun$1$$anonfun$apply$1.apply(JobSummaryDefinitionParquetCreator.scala:40)
com.wd.perf.collector.spark.report.jobs.definition.JobSummaryDefinitionParquetCreator$$anonfun$1$$anonfun$apply$1.apply(JobSummaryDefinitionParquetCreator.scala:39)
scala.collection.immutable.List.foreach(List.scala:381)
com.wd.perf.collector.spark.report.jobs.definition.JobSummaryDefinitionParquetCreator$$anonfun$1.apply(JobSummaryDefinitionParquetCreator.scala:39)
com.wd.perf.collector.spark.report.jobs.definition.JobSummaryDefinitionParquetCreator$$anonfun$1.apply(JobSummaryDefinitionParquetCreator.scala:38)
com.wd.perf.collector.spark.report.utils.ParquetReportInstaller$$anonfun$5.apply(ParquetReportInstaller.scala:148)
com.wd.perf.collector.spark.report.utils.ParquetReportInstaller$$anonfun$5.apply(ParquetReportInstaller.scala:110)
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
scala.collection.immutable.List.foreach(List.scala:381)
scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
scala.collection.immutable.List.map(List.scala:285)
com.wd.perf.collector.spark.report.utils.ParquetReportInstaller$.executeReportStep(ParquetReportInstaller.scala:110)
com.wd.perf.collector.spark.report.utils.ParquetReportInstaller$.executeByInstallOption(ParquetReportInstaller.scala:51)
com.wd.perf.collector.spark.report.jobs.definition.JobSummaryDefinitionParquetCreator$.generateReport(JobSummaryDefinitionParquetCreator.scala:44)
com.wd.perf.collector.spark.actor.report.parquet.ReportNameInstanceCreatorActor.com$wd$perf$collector$spark$actor$report$parquet$ReportNameInstanceCreatorActor$$processMessage(ReportNameInstanceCreatorActor.scala:67)
com.wd.perf.collector.spark.actor.report.parquet.ReportNameInstanceCreatorActor$$anonfun$receive$1$$anonfun$1$$anonfun$apply$1.apply$mcV$sp(ReportNameInstanceCreatorActor.scala:36)
com.wd.perf.collector.spark.actor.report.parquet.ReportNameInstanceCreatorActor$$anonfun$receive$1$$anonfun$1$$anonfun$apply$1.apply(ReportNameInstanceCreatorActor.scala:35)
com.wd.perf.collector.spark.actor.report.parquet.ReportNameInstanceCreatorActor$$anonfun$receive$1$$anonfun$1$$anonfun$apply$1.apply(ReportNameInstanceCreatorActor.scala:35)
scala.concurrent.impl.Future$PromiseCompletingRunnable.liftedTree1$1(Future.scala:24)
scala.concurrent.impl.Future$PromiseCompletingRunnable.run(Future.scala:24)
akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:39)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
