Repository: spark Updated Branches: refs/heads/branch-2.4 15d2e9d7d -> 71f70130f
[SPARK-23820][CORE] Enable use of long form of callsite in logs This is a rework of #21433 to address some concerns there. Closes #22398 from michaelmior/long-callsite2. Authored-by: Michael Mior <mm...@uwaterloo.ca> Signed-off-by: Wenchen Fan <wenc...@databricks.com> (cherry picked from commit ab25c967905ca0973fc2f30b8523246bb9244206) Signed-off-by: Wenchen Fan <wenc...@databricks.com> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/71f70130 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/71f70130 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/71f70130 Branch: refs/heads/branch-2.4 Commit: 71f70130f1b2b4ec70595627f0a02a88e2c0e27d Parents: 15d2e9d Author: Michael Mior <mm...@uwaterloo.ca> Authored: Thu Sep 13 09:45:25 2018 +0800 Committer: Wenchen Fan <wenc...@databricks.com> Committed: Thu Sep 13 09:46:06 2018 +0800 ---------------------------------------------------------------------- .../scala/org/apache/spark/internal/config/package.scala | 3 +++ .../main/scala/org/apache/spark/storage/RDDInfo.scala | 11 ++++++++++- docs/configuration.md | 7 +++++++ 3 files changed, 20 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/71f70130/core/src/main/scala/org/apache/spark/internal/config/package.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index c74040c..7f1eb1e 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -72,6 +72,9 @@ package object config { private[spark] val EVENT_LOG_OVERWRITE = ConfigBuilder("spark.eventLog.overwrite").booleanConf.createWithDefault(false) + private[spark] val EVENT_LOG_CALLSITE_LONG_FORM = + ConfigBuilder("spark.eventLog.longForm.enabled").booleanConf.createWithDefault(false) + private[spark] val EXECUTOR_CLASS_PATH = ConfigBuilder(SparkLauncher.EXECUTOR_EXTRA_CLASSPATH).stringConf.createOptional http://git-wip-us.apache.org/repos/asf/spark/blob/71f70130/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala b/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala index e5abbf7..19f8656 100644 --- a/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala +++ b/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala @@ -17,7 +17,9 @@ package org.apache.spark.storage +import org.apache.spark.SparkEnv import org.apache.spark.annotation.DeveloperApi +import org.apache.spark.internal.config._ import org.apache.spark.rdd.{RDD, RDDOperationScope} import org.apache.spark.util.Utils @@ -53,10 +55,17 @@ class RDDInfo( } private[spark] object RDDInfo { + private val callsiteLongForm = SparkEnv.get.conf.get(EVENT_LOG_CALLSITE_LONG_FORM) + def fromRdd(rdd: RDD[_]): RDDInfo = { val rddName = Option(rdd.name).getOrElse(Utils.getFormattedClassName(rdd)) val parentIds = rdd.dependencies.map(_.rdd.id) + val callSite = if (callsiteLongForm) { + rdd.creationSite.longForm + } else { + rdd.creationSite.shortForm + } new RDDInfo(rdd.id, rddName, rdd.partitions.length, - rdd.getStorageLevel, parentIds, rdd.creationSite.shortForm, rdd.scope) + rdd.getStorageLevel, parentIds, callSite, rdd.scope) } } http://git-wip-us.apache.org/repos/asf/spark/blob/71f70130/docs/configuration.md ---------------------------------------------------------------------- diff --git a/docs/configuration.md b/docs/configuration.md index f344bcd..a3e59a0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -747,6 +747,13 @@ Apart from these, the following properties are also available, and may be useful </td> </tr> <tr> + <td><code>spark.eventLog.longForm.enabled</code></td> + <td>false</td> + <td> + If true, use the long form of call sites in the event log. Otherwise use the short form. + </td> +</tr> +<tr> <td><code>spark.eventLog.compress</code></td> <td>false</td> <td> --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
