This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 5ebb4b5 [SPARK-24783][SQL] spark.sql.shuffle.partitions=0 should
throw exception
5ebb4b5 is described below
commit 5ebb4b572318abf1b483b5a8f41aa9b021eb5327
Author: Sean Owen <[email protected]>
AuthorDate: Fri Mar 8 14:09:53 2019 +0900
[SPARK-24783][SQL] spark.sql.shuffle.partitions=0 should throw exception
## What changes were proposed in this pull request?
Throw an exception if spark.sql.shuffle.partitions=0
This takes over https://github.com/apache/spark/pull/23835
## How was this patch tested?
Existing tests.
Closes #24008 from srowen/SPARK-24783.2.
Lead-authored-by: Sean Owen <[email protected]>
Co-authored-by: WindCanDie <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
.../main/scala/org/apache/spark/sql/internal/SQLConf.scala | 1 +
.../scala/org/apache/spark/sql/internal/SQLConfSuite.scala | 12 +++++++++++-
2 files changed, 12 insertions(+), 1 deletion(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index e6cfd9d..2271bac 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -277,6 +277,7 @@ object SQLConf {
"Note: For structured streaming, this configuration cannot be changed
between query " +
"restarts from the same checkpoint location.")
.intConf
+ .checkValue(_ > 0, "The value of spark.sql.shuffle.partitions must be
positive")
.createWithDefault(200)
val SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE =
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
index 62cf705..5ecb79b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
@@ -20,7 +20,6 @@ package org.apache.spark.sql.internal
import org.apache.hadoop.fs.Path
import org.apache.spark.sql._
-import org.apache.spark.sql.execution.WholeStageCodegenExec
import org.apache.spark.sql.internal.StaticSQLConf._
import org.apache.spark.sql.test.{SharedSQLContext, TestSQLContext}
import org.apache.spark.util.Utils
@@ -310,4 +309,15 @@ class SQLConfSuite extends QueryTest with SharedSQLContext
{
SQLConf.unregister(fallback)
}
+ test("SPARK-24783: spark.sql.shuffle.partitions=0 should throw exception ") {
+ val e = intercept[IllegalArgumentException] {
+ spark.conf.set(SQLConf.SHUFFLE_PARTITIONS.key, 0)
+ }
+ assert(e.getMessage.contains("spark.sql.shuffle.partitions"))
+ val e2 = intercept[IllegalArgumentException] {
+ spark.conf.set(SQLConf.SHUFFLE_PARTITIONS.key, -1)
+ }
+ assert(e2.getMessage.contains("spark.sql.shuffle.partitions"))
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
