cloud-fan commented on a change in pull request #35789:
URL: https://github.com/apache/spark/pull/35789#discussion_r830937789
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
##########
@@ -341,6 +341,77 @@ object SQLConf {
.booleanConf
.createWithDefault(true)
+ val RUNTIME_FILTER_SEMI_JOIN_REDUCTION_ENABLED =
+ buildConf("spark.sql.optimizer.runtimeFilter.semiJoinReduction.enabled")
+ .doc("When true and if one side of a shuffle join has a selective
predicate, we attempt " +
+ "to insert a semi join in the other side to reduce the amount of
shuffle data.")
+ .version("3.3.0")
+ .booleanConf
+ .createWithDefault(false)
+
+ val RUNTIME_FILTER_NUMBER_THRESHOLD =
+ buildConf("spark.sql.optimizer.runtimeFilter.number.threshold")
+ .doc("The total number of injected runtime filters (non-DPP) for a
single " +
+ "query. This is to prevent driver OOMs with too many Bloom filters.")
+ .version("3.3.0")
+ .intConf
+ .checkValue(threshold => threshold >= 0, "The threshold should be >= 0")
+ .createWithDefault(10)
+
+ val RUNTIME_BLOOM_FILTER_ENABLED =
+ buildConf("spark.sql.optimizer.runtime.bloomFilter.enabled")
+ .doc("When true and if one side of a shuffle join has a selective
predicate, we attempt " +
+ "to insert a bloom filter in the other side to reduce the amount of
shuffle data.")
+ .version("3.3.0")
+ .booleanConf
+ .createWithDefault(false)
+
+ val RUNTIME_BLOOM_FILTER_CREATION_SIDE_THRESHOLD =
+ buildConf("spark.sql.optimizer.runtime.bloomFilter.creationSideThreshold")
+ .doc("Size threshold of the bloom filter creation side plan. Estimated
size needs to be " +
+ "under this value to try to inject bloom filter.")
+ .version("3.3.0")
+ .bytesConf(ByteUnit.BYTE)
+ .createWithDefaultString("10MB")
+
+ val RUNTIME_BLOOM_FILTER_APPLICATION_SIDE_SCAN_SIZE_THRESHOLD =
+
buildConf("spark.sql.optimizer.runtime.bloomFilter.applicationSideScanSizethreshold")
+ .doc("Byte size threshold of the Bloom filter application side plan's
aggregated scan " +
+ "size. Aggregated scan byte size of the Bloom filter application side
needs to be over " +
+ "this value to inject a bloom filter.")
+ .version("3.3.0")
+ .bytesConf(ByteUnit.BYTE)
+ .createWithDefaultString("10GB")
+
+ val RUNTIME_BLOOM_FILTER_EXPECTED_NUM_ITEMS =
+ buildConf("spark.sql.optimizer.runtime.bloomFilter.expectedNumItems")
+ .doc("The default number of expected items for the runtime bloomfilter")
+ .version("3.3.0")
+ .longConf
+ .createWithDefault(1000000L)
+
+ val RUNTIME_BLOOM_FILTER_MAX_NUM_ITEMS =
+ buildConf("spark.sql.optimizer.runtime.bloomFilter.maxNumItems")
+ .doc("The max allowed number of expected items for the runtime bloom
filter")
+ .version("3.3.0")
+ .longConf
+ .createWithDefault(4000000L)
+
+
+ val RUNTIME_BLOOM_FILTER_NUM_BITS =
+ buildConf("spark.sql.optimizer.runtime.bloomFilter.numBits")
Review comment:
hmm, doesn't the num bits depend on the num items?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
