cloud-fan commented on a change in pull request #26664:
[SPARK-9853][Doc][Follow-up] Update the description for all the SQLConf
affected by spark.sql.adaptive.enabled
URL: https://github.com/apache/spark/pull/26664#discussion_r350558598
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
##########
@@ -353,59 +353,67 @@ object SQLConf {
.booleanConf
.createWithDefault(false)
- val SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE =
- buildConf("spark.sql.adaptive.shuffle.targetPostShuffleInputSize")
- .doc("The target post-shuffle input size in bytes of a task.")
- .bytesConf(ByteUnit.BYTE)
- .createWithDefault(64 * 1024 * 1024)
-
val FETCH_SHUFFLE_BLOCKS_IN_BATCH_ENABLED =
buildConf("spark.sql.adaptive.shuffle.fetchShuffleBlocksInBatch.enabled")
.doc("Whether to fetch the continuous shuffle blocks in batch. Instead
of fetching blocks " +
"one by one, fetching continuous shuffle blocks for the same map task
in batch can " +
- "reduce IO and improve performance. Note, this feature also depends on
a relocatable " +
- "serializer and the concatenation support codec in use.")
+ "reduce IO and improve performance. Note, multiple continuous blocks
exist in single " +
+ s"fetch request only happen when '${ADAPTIVE_EXECUTION_ENABLED.key}'
is enabled, " +
+ "this feature also depends on a relocatable serializer and the
concatenation support " +
+ "codec in use.")
.booleanConf
.createWithDefault(true)
val REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED =
buildConf("spark.sql.adaptive.shuffle.reducePostShufflePartitions.enabled")
- .doc("When true and adaptive execution is enabled, this enables reducing
the number of " +
- "post-shuffle partitions based on map output statistics.")
+ .doc(s"When true and '${ADAPTIVE_EXECUTION_ENABLED.key}' is enabled, this
enables reducing " +
+ "the number of post-shuffle partitions based on map output statistics.")
.booleanConf
.createWithDefault(true)
val SHUFFLE_MIN_NUM_POSTSHUFFLE_PARTITIONS =
buildConf("spark.sql.adaptive.shuffle.minNumPostShufflePartitions")
- .doc("The advisory minimum number of post-shuffle partitions used in
adaptive execution.")
+ .doc("The advisory minimum number of post-shuffle partitions used when "
+
+ s"'${ADAPTIVE_EXECUTION_ENABLED.key}' and " +
+ s"'${REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED.key}' is enabled.")
.intConf
.checkValue(_ > 0, "The minimum shuffle partition number " +
"must be a positive integer.")
.createWithDefault(1)
+ val SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE =
+ buildConf("spark.sql.adaptive.shuffle.targetPostShuffleInputSize")
+ .doc("The target post-shuffle input size in bytes of a task. This
configuration only has " +
+ s"an effect when '${ADAPTIVE_EXECUTION_ENABLED.key}' and " +
+ s"'${REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED.key}' is enabled.")
+ .bytesConf(ByteUnit.BYTE)
+ .createWithDefault(64 * 1024 * 1024)
+
val SHUFFLE_MAX_NUM_POSTSHUFFLE_PARTITIONS =
buildConf("spark.sql.adaptive.shuffle.maxNumPostShufflePartitions")
.doc("The advisory maximum number of post-shuffle partitions used in
adaptive execution. " +
"This is used as the initial number of pre-shuffle partitions. By
default it equals to " +
- "spark.sql.shuffle.partitions")
+ "spark.sql.shuffle.partitions. This configuration only has an effect
when " +
+ s"'${ADAPTIVE_EXECUTION_ENABLED.key}' is enabled.")
Review comment:
and also `REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED` is enabled
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
