[
https://issues.apache.org/jira/browse/SPARK-20934?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Sean Owen resolved SPARK-20934.
-------------------------------
Resolution: Invalid
This doesn't provide any information on what you're executing or what is stuck
or not. This thread dump is irrelevant.
> Task is hung at inner join, would work with other kind of joins
> ---------------------------------------------------------------
>
> Key: SPARK-20934
> URL: https://issues.apache.org/jira/browse/SPARK-20934
> Project: Spark
> Issue Type: Bug
> Components: Shuffle, Spark Core
> Affects Versions: 2.1.0
> Reporter: Mohamed Elagamy
>
> I am using spark 2.1.0 to read from parquets and inner join between different
> dataframes, but it gets stuck at the inner join step and never show any
> progress, here is the thread dump
> sun.misc.Unsafe.park(Native Method)
> java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
> java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
> scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:208)
> scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:218)
> scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:153)
> scala.concurrent.Await$$anonfun$ready$1.apply(package.scala:169)
> scala.concurrent.Await$$anonfun$ready$1.apply(package.scala:169)
> scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53)
> scala.concurrent.Await$.ready(package.scala:169)
> com.wd.perf.collector.spark.report.utils.GroupUtils$.generateReportInGroups(GroupUtils.scala:70)
> com.wd.perf.collector.spark.report.jobs.definition.JobSummaryDefinitionParquetCreator$.com$wd$perf$collector$spark$report$jobs$definition$JobSummaryDefinitionParquetCreator$$generateHourlyBySWH(JobSummaryDefinitionParquetCreator.scala:82)
> com.wd.perf.collector.spark.report.jobs.definition.JobSummaryDefinitionParquetCreator$$anonfun$1$$anonfun$apply$1.apply(JobSummaryDefinitionParquetCreator.scala:40)
> com.wd.perf.collector.spark.report.jobs.definition.JobSummaryDefinitionParquetCreator$$anonfun$1$$anonfun$apply$1.apply(JobSummaryDefinitionParquetCreator.scala:39)
> scala.collection.immutable.List.foreach(List.scala:381)
> com.wd.perf.collector.spark.report.jobs.definition.JobSummaryDefinitionParquetCreator$$anonfun$1.apply(JobSummaryDefinitionParquetCreator.scala:39)
> com.wd.perf.collector.spark.report.jobs.definition.JobSummaryDefinitionParquetCreator$$anonfun$1.apply(JobSummaryDefinitionParquetCreator.scala:38)
> com.wd.perf.collector.spark.report.utils.ParquetReportInstaller$$anonfun$5.apply(ParquetReportInstaller.scala:148)
> com.wd.perf.collector.spark.report.utils.ParquetReportInstaller$$anonfun$5.apply(ParquetReportInstaller.scala:110)
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
> scala.collection.immutable.List.foreach(List.scala:381)
> scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
> scala.collection.immutable.List.map(List.scala:285)
> com.wd.perf.collector.spark.report.utils.ParquetReportInstaller$.executeReportStep(ParquetReportInstaller.scala:110)
> com.wd.perf.collector.spark.report.utils.ParquetReportInstaller$.executeByInstallOption(ParquetReportInstaller.scala:51)
> com.wd.perf.collector.spark.report.jobs.definition.JobSummaryDefinitionParquetCreator$.generateReport(JobSummaryDefinitionParquetCreator.scala:44)
> com.wd.perf.collector.spark.actor.report.parquet.ReportNameInstanceCreatorActor.com$wd$perf$collector$spark$actor$report$parquet$ReportNameInstanceCreatorActor$$processMessage(ReportNameInstanceCreatorActor.scala:67)
> com.wd.perf.collector.spark.actor.report.parquet.ReportNameInstanceCreatorActor$$anonfun$receive$1$$anonfun$1$$anonfun$apply$1.apply$mcV$sp(ReportNameInstanceCreatorActor.scala:36)
> com.wd.perf.collector.spark.actor.report.parquet.ReportNameInstanceCreatorActor$$anonfun$receive$1$$anonfun$1$$anonfun$apply$1.apply(ReportNameInstanceCreatorActor.scala:35)
> com.wd.perf.collector.spark.actor.report.parquet.ReportNameInstanceCreatorActor$$anonfun$receive$1$$anonfun$1$$anonfun$apply$1.apply(ReportNameInstanceCreatorActor.scala:35)
> scala.concurrent.impl.Future$PromiseCompletingRunnable.liftedTree1$1(Future.scala:24)
> scala.concurrent.impl.Future$PromiseCompletingRunnable.run(Future.scala:24)
> akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:39)
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> java.lang.Thread.run(Thread.java:745)
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
