viirya commented on a change in pull request #31986:
URL: https://github.com/apache/spark/pull/31986#discussion_r632348736
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
##########
@@ -2000,6 +2000,26 @@ object SQLConf {
.intConf
.createWithDefault(SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD.defaultValue.get)
+ val SESSION_WINDOW_BUFFER_IN_MEMORY_THRESHOLD =
+ buildConf("spark.sql.sessionWindow.buffer.in.memory.threshold")
+ .internal()
+ .doc("Threshold for number of windows guaranteed to be held in memory by
the " +
+ "session window operator. Note that the buffer is used only for the
query Spark " +
+ "cannot apply aggregations on classifying session window.")
Review comment:
classifying -> determining?
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
##########
@@ -2000,6 +2000,26 @@ object SQLConf {
.intConf
.createWithDefault(SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD.defaultValue.get)
+ val SESSION_WINDOW_BUFFER_IN_MEMORY_THRESHOLD =
+ buildConf("spark.sql.sessionWindow.buffer.in.memory.threshold")
+ .internal()
+ .doc("Threshold for number of windows guaranteed to be held in memory by
the " +
+ "session window operator. Note that the buffer is used only for the
query Spark " +
+ "cannot apply aggregations on classifying session window.")
+ .version("3.2.0")
+ .intConf
+ .createWithDefault(4096)
+
+ val SESSION_WINDOW_BUFFER_SPILL_THRESHOLD =
+ buildConf("spark.sql.sessionWindow.buffer.spill.threshold")
+ .internal()
+ .doc("Threshold for number of rows to be spilled by window operator.
Note that " +
+ "the buffer is used only for the query Spark cannot apply aggregations
on classifying " +
Review comment:
ditto
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
