Github user viirya commented on a diff in the pull request:
https://github.com/apache/spark/pull/21594#discussion_r196799526
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala ---
@@ -107,22 +107,35 @@ class CacheManager extends Logging {
/**
* Un-cache all the cache entries that refer to the given plan.
*/
- def uncacheQuery(query: Dataset[_], blocking: Boolean = true): Unit =
writeLock {
- uncacheQuery(query.sparkSession, query.logicalPlan, blocking)
+ def uncacheQuery(query: Dataset[_],
+ cascade: Boolean, blocking: Boolean = true): Unit = writeLock {
+ uncacheQuery(query.sparkSession, query.logicalPlan, cascade, blocking)
}
/**
* Un-cache all the cache entries that refer to the given plan.
*/
- def uncacheQuery(spark: SparkSession, plan: LogicalPlan, blocking:
Boolean): Unit = writeLock {
+ def uncacheQuery(spark: SparkSession, plan: LogicalPlan,
+ cascade: Boolean, blocking: Boolean): Unit = writeLock {
val it = cachedData.iterator()
+ val needToRecache =
scala.collection.mutable.ArrayBuffer.empty[CachedData]
while (it.hasNext) {
val cd = it.next()
if (cd.plan.find(_.sameResult(plan)).isDefined) {
- cd.cachedRepresentation.cacheBuilder.clearCache(blocking)
it.remove()
+ if (cascade || cd.plan.sameResult(plan)) {
+ cd.cachedRepresentation.cacheBuilder.clearCache(blocking)
+ } else {
+ val plan = spark.sessionState.executePlan(cd.plan).executedPlan
+ val newCache = InMemoryRelation(
--- End diff --
hmm, if the plan to uncache is iterated after a plan containing it, doesn't
this still use its cached plan?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
