svn commit: r31918 - in /dev/spark/2.4.1-SNAPSHOT-2019_01_11_23_17-bb97459-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Sat Jan 12 07:34:36 2019 New Revision: 31918 Log: Apache Spark 2.4.1-SNAPSHOT-2019_01_11_23_17-bb97459 docs [This commit notification would consist of 1476 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
svn commit: r31917 - in /dev/spark/2.3.3-SNAPSHOT-2019_01_11_23_17-b6c4649-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Sat Jan 12 07:32:34 2019 New Revision: 31917 Log: Apache Spark 2.3.3-SNAPSHOT-2019_01_11_23_17-b6c4649 docs [This commit notification would consist of 1443 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[spark] branch branch-2.3 updated: [SPARK-26607][SQL][TEST] Remove Spark 2.2.x testing from HiveExternalCatalogVersionsSuite
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.3 by this push:
new b6c4649 [SPARK-26607][SQL][TEST] Remove Spark 2.2.x testing from
HiveExternalCatalogVersionsSuite
b6c4649 is described below
commit b6c464997e179bf648250435dfe6295c84e76784
Author: Dongjoon Hyun
AuthorDate: Fri Jan 11 22:53:58 2019 -0800
[SPARK-26607][SQL][TEST] Remove Spark 2.2.x testing from
HiveExternalCatalogVersionsSuite
The vote of final release of `branch-2.2` passed and the branch goes EOL.
This PR removes Spark 2.2.x from the testing coverage.
Pass the Jenkins.
Closes #23526 from dongjoon-hyun/SPARK-26607.
Authored-by: Dongjoon Hyun
Signed-off-by: Dongjoon Hyun
(cherry picked from commit 3587a9a2275615b82492b89204b141636542ce52)
Signed-off-by: Dongjoon Hyun
---
.../org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
index af15da6..6522f77 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
@@ -203,7 +203,7 @@ class HiveExternalCatalogVersionsSuite extends
SparkSubmitTestUtils {
object PROCESS_TABLES extends QueryTest with SQLTestUtils {
// Tests the latest version of every release line.
- val testingVersions = Seq("2.1.3", "2.2.2", "2.3.2")
+ val testingVersions = Seq("2.3.2")
protected var spark: SparkSession = _
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
[spark] branch master updated: [SPARK-26607][SQL][TEST] Remove Spark 2.2.x testing from HiveExternalCatalogVersionsSuite
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 3587a9a [SPARK-26607][SQL][TEST] Remove Spark 2.2.x testing from
HiveExternalCatalogVersionsSuite
3587a9a is described below
commit 3587a9a2275615b82492b89204b141636542ce52
Author: Dongjoon Hyun
AuthorDate: Fri Jan 11 22:53:58 2019 -0800
[SPARK-26607][SQL][TEST] Remove Spark 2.2.x testing from
HiveExternalCatalogVersionsSuite
## What changes were proposed in this pull request?
The vote of final release of `branch-2.2` passed and the branch goes EOL.
This PR removes Spark 2.2.x from the testing coverage.
## How was this patch tested?
Pass the Jenkins.
Closes #23526 from dongjoon-hyun/SPARK-26607.
Authored-by: Dongjoon Hyun
Signed-off-by: Dongjoon Hyun
---
.../org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
index f1e8423..dd0e1bd 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
@@ -206,7 +206,7 @@ class HiveExternalCatalogVersionsSuite extends
SparkSubmitTestUtils {
object PROCESS_TABLES extends QueryTest with SQLTestUtils {
// Tests the latest version of every release line.
- val testingVersions = Seq("2.2.2", "2.3.2", "2.4.0")
+ val testingVersions = Seq("2.3.2", "2.4.0")
protected var spark: SparkSession = _
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
[spark] branch branch-2.4 updated: [SPARK-26607][SQL][TEST] Remove Spark 2.2.x testing from HiveExternalCatalogVersionsSuite
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new bb97459 [SPARK-26607][SQL][TEST] Remove Spark 2.2.x testing from
HiveExternalCatalogVersionsSuite
bb97459 is described below
commit bb9745904f4c439be21fa435f2bb5c66de164f61
Author: Dongjoon Hyun
AuthorDate: Fri Jan 11 22:53:58 2019 -0800
[SPARK-26607][SQL][TEST] Remove Spark 2.2.x testing from
HiveExternalCatalogVersionsSuite
## What changes were proposed in this pull request?
The vote of final release of `branch-2.2` passed and the branch goes EOL.
This PR removes Spark 2.2.x from the testing coverage.
## How was this patch tested?
Pass the Jenkins.
Closes #23526 from dongjoon-hyun/SPARK-26607.
Authored-by: Dongjoon Hyun
Signed-off-by: Dongjoon Hyun
(cherry picked from commit 3587a9a2275615b82492b89204b141636542ce52)
Signed-off-by: Dongjoon Hyun
---
.../org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
index cce2f20..632a21a 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
@@ -203,7 +203,7 @@ class HiveExternalCatalogVersionsSuite extends
SparkSubmitTestUtils {
object PROCESS_TABLES extends QueryTest with SQLTestUtils {
// Tests the latest version of every release line.
- val testingVersions = Seq("2.2.2", "2.3.2", "2.4.0")
+ val testingVersions = Seq("2.3.2", "2.4.0")
protected var spark: SparkSession = _
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
[spark] branch master updated: [SPARK-26482][K8S][TEST][FOLLOWUP] Fix compile failure
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new e00ebd5 [SPARK-26482][K8S][TEST][FOLLOWUP] Fix compile failure
e00ebd5 is described below
commit e00ebd5c72e719500f3deb676afc5c99be20a95e
Author: Dongjoon Hyun
AuthorDate: Fri Jan 11 21:58:06 2019 -0800
[SPARK-26482][K8S][TEST][FOLLOWUP] Fix compile failure
## What changes were proposed in this pull request?
This fixes K8S integration test compilation failure introduced by #23423 .
```scala
$ build/sbt -Pkubernetes-integration-tests test:package
...
[error]
/Users/dongjoon/APACHE/spark/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala:71:
type mismatch;
[error] found :
org.apache.spark.internal.config.OptionalConfigEntry[Boolean]
[error] required: String
[error] .set(IS_TESTING, false)
[error]^
[error]
/Users/dongjoon/APACHE/spark/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala:71:
type mismatch;
[error] found : Boolean(false)
[error] required: String
[error] .set(IS_TESTING, false)
[error]^
[error] two errors found
```
## How was this patch tested?
Pass the K8S integration test.
Closes #23527 from dongjoon-hyun/SPARK-26482.
Authored-by: Dongjoon Hyun
Signed-off-by: Dongjoon Hyun
---
.../spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala
index d90c306..c869803 100644
---
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala
+++
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala
@@ -68,8 +68,8 @@ private[spark] class KubernetesTestComponents(defaultClient:
DefaultKubernetesCl
.set("spark.executor.cores", "1")
.set("spark.executors.instances", "1")
.set("spark.app.name", "spark-test-app")
- .set(IS_TESTING, false)
- .set(UI_ENABLED, true)
+ .set(IS_TESTING.key, "false")
+ .set(UI_ENABLED.key, "true")
.set("spark.kubernetes.submission.waitAppCompletion", "false")
.set("spark.kubernetes.authenticate.driver.serviceAccountName",
serviceAccountName)
}
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
svn commit: r31916 - in /dev/spark/3.0.0-SNAPSHOT-2019_01_11_21_14-19e17ac-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Sat Jan 12 05:27:11 2019 New Revision: 31916 Log: Apache Spark 3.0.0-SNAPSHOT-2019_01_11_21_14-19e17ac docs [This commit notification would consist of 1775 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[spark] branch master updated (ae382c9 -> 19e17ac)
This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/spark.git. from ae382c9 [SPARK-26586][SS] Fix race condition that causes streams to run with unexpected confs add 19e17ac [SPARK-25692][TEST] Increase timeout in fetchBothChunks test No new revisions were added by this update. Summary of changes: .../test/java/org/apache/spark/network/ChunkFetchIntegrationSuite.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
svn commit: r31914 - /dev/spark/v2.2.3-rc1-docs/
Author: dongjoon Date: Sat Jan 12 01:00:38 2019 New Revision: 31914 Log: Removing RC artifacts. Removed: dev/spark/v2.2.3-rc1-docs/ - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[spark] tag v2.2.3 created (now 4acb6ba)
This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a change to tag v2.2.3 in repository https://gitbox.apache.org/repos/asf/spark.git. at 4acb6ba (commit) No new revisions were added by this update. - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
svn commit: r31912 - /release/spark/KEYS
Author: dbtsai Date: Sat Jan 12 00:38:01 2019 New Revision: 31912 Log: Update KEYS Modified: release/spark/KEYS Modified: release/spark/KEYS == --- release/spark/KEYS (original) +++ release/spark/KEYS Sat Jan 12 00:38:01 2019 @@ -760,3 +760,130 @@ qmd2/zxUPGuOvX/e79ATOM8KMQFbGikLTcjW9gcU DFZYf6DnKNwDnBzBuAx2S+P0 =a0mL -END PGP PUBLIC KEY BLOCK- + +pub 4096R/34F0FC5C 2015-05-05 +uid Dongjoon Hyun (CODE SIGNING KEY) +sub 4096R/727240E9 2015-05-05 + +-BEGIN PGP PUBLIC KEY BLOCK- +Version: GnuPG v2.0.19 (FreeBSD) + +mQINBFVIx4YBEADCRN5gDmObS8L3e2YNFNlCrXxSVcUgJ6UOYt12wi337dEudEqk +s+0jZ6p9k3mz1hD3uTim+lT/O2ODRDNWYBfT2cF6E26QGieXrKTxeApwFrXyWuSm +49rlhgxi/ECGA59ovPMQvxrN1b8IFChWfXR9lmIllUKJ02zNnpossEPdHs6k2Efu +dgMlxhm0qFn5JULDfr/+ump9n9IeiojpQ+bsD/iZ/7DneK2BqtHA7NGOKCMjRzhe +OzG2lxwyLxB4zR3d2AUMlLDJUMAjR7MOEs7B/d9ES+ka2HI/CNc19F5cD5soT3EE +D7B6HwCRNPSyeytwYCafLCaxveKpDWfehUDNFrSqBaxyDh0/i8x50GrR++AsItZU +gh/qcYF2ju0uKZ4t5UVvoOOCBc11njzdk6VeE6VTDoRFXZ6QUP8HWR+F0PdLZKWf +e12Pp0dMwDcWppJRC6zzbfGizz/209VSkF4atIU9rJmB9T+rm2RxL4+l5QjADWRP +1U4NMC6Fh3aTR75KOtL50NMjk1H4o33KX33Ec2k6eJ+9kDcqhRb1+lA4I1JZB2pX ++crvehzPDUh9i+nVXPyxUhgC1SY3+I5IIdGGgz69KPU8da7l+MRLxL1YIZOnqQnt +3KdBRwpFzDXGK6bAhrboj4Hab+GIHBRpJ89pJz8uG5CaMFfWvapGbC85DQARAQAB +tDZEb25nam9vbiBIeXVuIChDT0RFIFNJR05JTkcgS0VZKSA8ZG9uZ2pvb25AYXBh +Y2hlLm9yZz6JAjcEEwEKACECGwMCHgECF4AFAlVQK2QFCwkIBwMFFQoJCAsFFgID +AQAACgkQ7aAM6DTw/FxSGA//eAzyfDp8RCD5IpPj3UeLs4J3zDsVMCj5IQNK0stv +XjMy7SBC4jhz8HdriYQC8qSj/fbnMfZuqtr2XV2lC8StDXFe8/G+JQFd25+BbrfR ++fNv97adR3lp97GxehXRLF4Aw+Am2EsbI0ej+ZhW/H6XS6jjSa/yxsLUtEIGDB4k +3wnlZ0gqH49nryHitXC7cCpXh7qY1K5ObXjkhnsdsIiKjVfsQBN4MzdknGuEhTfM +Aez4OkaNf9T7btmFI56ijTP6ClM7FaIbofLCJr8EVojR9HoErgdCO9byD4SIerc7 +rhuAyiyVFodMsqc/jvNvv8Usu1gFifP3YvuxL8TRrL04rJhOyg8oYU4gGTnipzii +AHcCvDP0zSwmhlL2wht8aExc3or8RvQluh7HSqGjDq+dUKIdqysIoxev7kOM4Gs3 +wBltyyV4Pwa3kzn3D2JjTm5E0rFEmP//3a58mbQL5zRXxg9iX9lGQ357GLRjz7se +eAuTpW+z9ZgmOhTRdFf1grVjiOmOCWMP7+Mqkq9R3/wvqkk1jjRJeSpwmR8ZtTP6 +IqduYhE609esFX65SeTTtbqpQIZ5J80wBybIYT/XPYWuFZdn/pgWpKVJWs+uEYjM +U7pONs6avNKguu92HLbpHCclQS5NgcjGb7PvuNIkJb8yQixeVoDZQXz/B6YN5iWS +uLuJAjgEEwECACIFAlVIx4YCGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJ +EO2gDOg08Pxc4OMP/jTvfPE9Z+n/Veddf/i8Gdxurf2LiYgzHFu84CfKE8vLxo13 +PP8DN4ZNBIRx2+a3OQ3/PLTpDn/D3Pf9uFriKXHNGksu153TAqMO5x81Hc2vaRbP +StUZrWLcIsBmGlZVR4rzuSF1ypFJ80m7bGyVYQFL4rq9sd2E9aPwy9zWGBjqZAQu +IfldLnOfkpnDS+X5J9/12kB8By7nvDydQIgTCJGxiLGh71cfmZDtTJuHLdfmxw+o +iqfXEthy6UqDayLJRGqeMJDCXZGj0HPQZHZZTDtF8czH0w0ezj6WePD0dnpm5w/Y +g3dwQ2jWfkRAe04Q3um/bTzcADWASQtoAOPCCSe0/JT5fXo86uwdYUBDivx9OABz +bgQZlq1Xj4YbRNF5QUvAq/k76yVhftNGaUBT7ubWyxBWBeh43W5g/r5ggvO9XfQj +r+Ox6efblvvJLMnYZe8Gmd2lbuFWfUpShYXqf33rfsKHcUkQuTEKFTVOtUsgDMng +Fg/K3R/Kg9pCX1BHRha1o89mBcwaTW0KaZ9sTZ6pjo58f7f7oWEKBfpGVWRFlQcq +iHTzCJVVWnk4u1AMk7+M5xqSHspX0YOHKFl4EXRs9bOtX+NUu2+7vbY2B3D1fzTY +IqEGdZ3SGh6yMyJgzi7/k+jMmx1O/rq9WNspO630JOBDH89PcxibDaVw5pPQuQIN +BFVIx4YBEADKRaIlOp9qlYhID8n7esMy75XfOliKPAxk9yeuZWQ+Bat2ZznfPVmL +kqnY4xSTqVVvFiJqKmaFnVa93YT5Z9MoIkd5z0A5gsSukOiYe4856FrluGzNsMJ+ +0i5rSQlWC1YozOQCBWKgfIW5Snam6rUrvIVnVEoZaTMS0QqIYndGuw0GkaCLn04V +ycu2W9w/1Mj5ypYdHMZERqIaCju70PDYS1+MVGKcv08D8jBOy45ZNKhIHny2N6NQ +f8+WmBqG+hTl7avE+E3mRx2Wy/XF6sHS6jsqwgSHBE80/bK1H2UMXNO9qHSFDxsZ +bcWYqC2zFFy0rM5poGlTk+9G6Dw0iUY1oKJ5hxTkgIN1yLz9qah8l5quMgAKcc+t +5H89xoK1gnb9CZJrNVD52phKZwDU30gh/CasRxVNSWmotwuJRwx+adBhklclAmjt +QGJQomlmBK6LzQcbl987BwWLQJfPx+CJRH4VqYaojP62vKQttQtCG/P/IQjux59I +5RwkiSYIRgi7/RvXfCEDPZRE+GP9b77TY7Rj7NW43ouGpTwP6i78CLVQ/XhJLA63 +6o1e0xNfAdtLT1pZbsnCT0sp4BovBfeiZGHdoy0oiAVH78bZH1W09oKK3HXnsNa5 +83VD6h+6L4NJS4cxagl3TYI5/YMnX/AVQxvZXW42tAFTDmqqmf2ioQARAQABiQIf +BBgBAgAJBQJVSMeGAhsMAAoJEO2gDOg08PxcKsYP/2etfI6tUOVK0PWEsnrwrPAf +C4hObPvQBjS1ccbMO12fgtaQlZkeUgBfkKEhimeWW2s3a2VnwCN3BCqTW2dE6Pii +84FrL9I5VQ11lWBMG5bq3+NEZKzzkTUUCg3blvSReDSUowTxwTzIF2/opT2IDRdB +/c40WOb/7mwgGb0zxxm4ye4X+Q3d8uxFhKQGUqssYz37Dg8PnNfqHjyOWWVPPXzP +HuKwaZlE7+6Jzd7mTuCghViwliVT88e0oCM85t6Dm0u9PgPgQnbTzS4Gf3Z9iZeZ +ecTc4VoXBFpFxr52luXJmvy/b3lXTgBwF0SsTfHg7VY4K3DGtcijg/fSVD4jCHej +/jIcbFgFD5MRxwvGbOG/c5F/KKqSW0/9E9cr8pJXJX+m5llCU4C+ag4CU41SFhxO +G8Q9v60ZdfXlUqVJp7Xxe9lXeRKZPHCjLNepZLDc4HwRtyXcWVreq2w3U/mN3Lqh +oQXTzdUCO+hwbMVC87C4Afzkt4LPEeNAdmwd00hb5eduva1BCUMo9zXsfYPmUd1w +ZQwuVzdFAiXHmPZIIb1u8MJPm8wrqPqZmCawLa7xACSGo8VuAfFL2jdzdV61sKrs +aI9kX8V9gl5PZLw+LchGX5H7HKoRxZM3UbPkY5Mv21YPQ5C1EVM47ERc5mUtl5O0 +ZIAzEigXrrsePyvHGf6H +=6YJg +-END PGP PUBLIC KEY BLOCK- + +pub rsa4096 2019-01-10 [SC] + 0E9925082727075EEE83D4B06EC5F1052DF08FF4 +uid [ultimate] Takeshi Yamamuro (CODE SIGNING KEY) +sub rsa4096 2019-01-10 [E] + +-BEGIN PGP PUBLIC KEY BLOCK- + +mQINBFw2q20BEADLW2BZbJO2YHmAmAumggCTm4aVWFRYH+NX0zqEX2bynA0GM5hR +euvLL6w5vq44S6zU+39o1s9wSDcBAqLNpPB2eDL8qqXKZa/AQTwCiitk9aDB1KZB +DzejoqtrtCK1WnCW7oB7mQIq+/txSyLgv1UgFijh2aAx0ChmMnb2WbeZAQz/5ids +ixMfZiRofZVJIjdNNe5kIBcc9uthoyLw3x16nLT3zrATtBSDA
svn commit: r31911 - /dev/spark/v2.2.3-rc1-bin/ /release/spark/spark-2.2.3/
Author: dbtsai Date: Sat Jan 12 00:14:38 2019 New Revision: 31911 Log: Publish Apache Spark 2.2.3 Added: release/spark/spark-2.2.3/ - copied from r31910, dev/spark/v2.2.3-rc1-bin/ Removed: dev/spark/v2.2.3-rc1-bin/ - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
svn commit: r31910 - /dev/spark/v2.4.0-rc5-docs/
Author: dongjoon Date: Fri Jan 11 23:29:23 2019 New Revision: 31910 Log: Remove old 2.4.0-rc5 docs Removed: dev/spark/v2.4.0-rc5-docs/ - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
svn commit: r31909 - in /dev/spark/2.4.1-SNAPSHOT-2019_01_11_15_03-bbf61eb-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Fri Jan 11 23:18:01 2019 New Revision: 31909 Log: Apache Spark 2.4.1-SNAPSHOT-2019_01_11_15_03-bbf61eb docs [This commit notification would consist of 1476 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
svn commit: r31904 - in /dev/spark/3.0.0-SNAPSHOT-2019_01_11_12_56-ae382c9-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Fri Jan 11 21:08:21 2019 New Revision: 31904 Log: Apache Spark 3.0.0-SNAPSHOT-2019_01_11_12_56-ae382c9 docs [This commit notification would consist of 1775 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[spark] branch branch-2.4 updated: [SPARK-26586][SS] Fix race condition that causes streams to run with unexpected confs
This is an automated email from the ASF dual-hosted git repository.
zsxwing pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new bbf61eb [SPARK-26586][SS] Fix race condition that causes streams to
run with unexpected confs
bbf61eb is described below
commit bbf61eb4222d7b46e71dc91eeedf82d27226fc2c
Author: Mukul Murthy
AuthorDate: Fri Jan 11 11:46:14 2019 -0800
[SPARK-26586][SS] Fix race condition that causes streams to run with
unexpected confs
## What changes were proposed in this pull request?
Fix race condition where streams can have unexpected conf values.
New streaming queries should run with isolated SparkSessions so that they
aren't affected by conf updates after they are started. In StreamExecution, the
parent SparkSession is cloned and used to run each batch, but this cloning
happens in a separate thread and may happen after DataStreamWriter.start()
returns. If a stream is started and a conf key is set immediately after, the
stream is likely to have the new value.
## How was this patch tested?
New unit test that fails prior to the production change and passes with it.
Please review http://spark.apache.org/contributing.html before opening a
pull request.
Closes #23513 from mukulmurthy/26586.
Authored-by: Mukul Murthy
Signed-off-by: Shixiong Zhu
(cherry picked from commit ae382c94dd10ff494dde4de44e66182bf6dbe8f8)
Signed-off-by: Shixiong Zhu
---
.../sql/execution/streaming/StreamExecution.scala | 5 +++--
.../test/DataStreamReaderWriterSuite.scala | 24 ++
2 files changed, 27 insertions(+), 2 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
index de33844..c1aa98a 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
@@ -176,6 +176,9 @@ abstract class StreamExecution(
lazy val streamMetrics = new MetricsReporter(
this, s"spark.streaming.${Option(name).getOrElse(id)}")
+ /** Isolated spark session to run the batches with. */
+ private val sparkSessionForStream = sparkSession.cloneSession()
+
/**
* The thread that runs the micro-batches of this stream. Note that this
thread must be
* [[org.apache.spark.util.UninterruptibleThread]] to workaround KAFKA-1894:
interrupting a
@@ -265,8 +268,6 @@ abstract class StreamExecution(
// force initialization of the logical plan so that the sources can be
created
logicalPlan
- // Isolated spark session to run the batches with.
- val sparkSessionForStream = sparkSession.cloneSession()
// Adaptive execution can change num shuffle partitions, disallow
sparkSessionForStream.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key,
"false")
// Disable cost-based join optimization as we do not want stateful
operations to be rearranged
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
index 8212fb9..569114a 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
@@ -18,6 +18,7 @@
package org.apache.spark.sql.streaming.test
import java.io.File
+import java.util.ConcurrentModificationException
import java.util.Locale
import java.util.concurrent.TimeUnit
@@ -651,4 +652,27 @@ class DataStreamReaderWriterSuite extends StreamTest with
BeforeAndAfter {
LastOptions.clear()
}
+
+ test("SPARK-26586: Streams should have isolated confs") {
+import testImplicits._
+val input = MemoryStream[Int]
+input.addData(1 to 10)
+spark.conf.set("testKey1", 0)
+val queries = (1 to 10).map { i =>
+ spark.conf.set("testKey1", i)
+ input.toDF().writeStream
+.foreachBatch { (df: Dataset[Row], id: Long) =>
+ val v = df.sparkSession.conf.get("testKey1").toInt
+ if (i != v) {
+throw new ConcurrentModificationException(s"Stream $i has the
wrong conf value $v")
+ }
+}
+.start()
+}
+try {
+ queries.foreach(_.processAllAvailable())
+} finally {
+ queries.foreach(_.stop())
+}
+ }
}
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
[spark] branch master updated: [SPARK-26586][SS] Fix race condition that causes streams to run with unexpected confs
This is an automated email from the ASF dual-hosted git repository.
zsxwing pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new ae382c9 [SPARK-26586][SS] Fix race condition that causes streams to
run with unexpected confs
ae382c9 is described below
commit ae382c94dd10ff494dde4de44e66182bf6dbe8f8
Author: Mukul Murthy
AuthorDate: Fri Jan 11 11:46:14 2019 -0800
[SPARK-26586][SS] Fix race condition that causes streams to run with
unexpected confs
## What changes were proposed in this pull request?
Fix race condition where streams can have unexpected conf values.
New streaming queries should run with isolated SparkSessions so that they
aren't affected by conf updates after they are started. In StreamExecution, the
parent SparkSession is cloned and used to run each batch, but this cloning
happens in a separate thread and may happen after DataStreamWriter.start()
returns. If a stream is started and a conf key is set immediately after, the
stream is likely to have the new value.
## How was this patch tested?
New unit test that fails prior to the production change and passes with it.
Please review http://spark.apache.org/contributing.html before opening a
pull request.
Closes #23513 from mukulmurthy/26586.
Authored-by: Mukul Murthy
Signed-off-by: Shixiong Zhu
---
.../sql/execution/streaming/StreamExecution.scala | 5 +++--
.../test/DataStreamReaderWriterSuite.scala | 24 ++
2 files changed, 27 insertions(+), 2 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
index 83824f4..90f7b47 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
@@ -181,6 +181,9 @@ abstract class StreamExecution(
lazy val streamMetrics = new MetricsReporter(
this, s"spark.streaming.${Option(name).getOrElse(id)}")
+ /** Isolated spark session to run the batches with. */
+ private val sparkSessionForStream = sparkSession.cloneSession()
+
/**
* The thread that runs the micro-batches of this stream. Note that this
thread must be
* [[org.apache.spark.util.UninterruptibleThread]] to workaround KAFKA-1894:
interrupting a
@@ -270,8 +273,6 @@ abstract class StreamExecution(
// force initialization of the logical plan so that the sources can be
created
logicalPlan
- // Isolated spark session to run the batches with.
- val sparkSessionForStream = sparkSession.cloneSession()
// Adaptive execution can change num shuffle partitions, disallow
sparkSessionForStream.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key,
"false")
// Disable cost-based join optimization as we do not want stateful
operations to be rearranged
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
index 4d3a54a..74ea0bf 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
@@ -18,6 +18,7 @@
package org.apache.spark.sql.streaming.test
import java.io.File
+import java.util.ConcurrentModificationException
import java.util.Locale
import java.util.concurrent.TimeUnit
@@ -651,4 +652,27 @@ class DataStreamReaderWriterSuite extends StreamTest with
BeforeAndAfter {
LastOptions.clear()
}
+
+ test("SPARK-26586: Streams should have isolated confs") {
+import testImplicits._
+val input = MemoryStream[Int]
+input.addData(1 to 10)
+spark.conf.set("testKey1", 0)
+val queries = (1 to 10).map { i =>
+ spark.conf.set("testKey1", i)
+ input.toDF().writeStream
+.foreachBatch { (df: Dataset[Row], id: Long) =>
+ val v = df.sparkSession.conf.get("testKey1").toInt
+ if (i != v) {
+throw new ConcurrentModificationException(s"Stream $i has the
wrong conf value $v")
+ }
+}
+.start()
+}
+try {
+ queries.foreach(_.processAllAvailable())
+} finally {
+ queries.foreach(_.stop())
+}
+ }
}
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
[spark] branch branch-2.4 updated: [SPARK-26551][SQL] Fix schema pruning error when selecting one complex field and having is not null predicate on another one
This is an automated email from the ASF dual-hosted git repository.
dbtsai pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 0e5b316 [SPARK-26551][SQL] Fix schema pruning error when selecting
one complex field and having is not null predicate on another one
0e5b316 is described below
commit 0e5b316dec8e3439a4020b3e5def2f5f2c0be5f2
Author: Liang-Chi Hsieh
AuthorDate: Fri Jan 11 19:23:32 2019 +
[SPARK-26551][SQL] Fix schema pruning error when selecting one complex
field and having is not null predicate on another one
## What changes were proposed in this pull request?
Schema pruning has errors when selecting one complex field and having is
not null predicate on another one:
```scala
val query = sql("select * from contacts")
.where("name.middle is not null")
.select(
"id",
"name.first",
"name.middle",
"name.last"
)
.where("last = 'Jones'")
.select(count("id"))
```
```
java.lang.IllegalArgumentException: middle does not exist. Available: last
[info] at
org.apache.spark.sql.types.StructType.$anonfun$fieldIndex$1(StructType.scala:303)
[info] at scala.collection.immutable.Map$Map1.getOrElse(Map.scala:119)
[info] at
org.apache.spark.sql.types.StructType.fieldIndex(StructType.scala:302)
[info] at
org.apache.spark.sql.execution.ProjectionOverSchema.$anonfun$getProjection$6(ProjectionOverSchema.scala:58)
[info] at scala.Option.map(Option.scala:163)
[info] at
org.apache.spark.sql.execution.ProjectionOverSchema.getProjection(ProjectionOverSchema.scala:56)
[info] at
org.apache.spark.sql.execution.ProjectionOverSchema.unapply(ProjectionOverSchema.scala:32)
[info] at
org.apache.spark.sql.execution.datasources.parquet.ParquetSchemaPruning$$anonfun$$nestedInanonfun$buildNewProjection$1$1.applyOrElse(Parque
tSchemaPruning.scala:153)
```
## How was this patch tested?
Added tests.
Closes #23474 from viirya/SPARK-26551.
Authored-by: Liang-Chi Hsieh
Signed-off-by: DB Tsai
(cherry picked from commit 50ebf3a43b84c8538ec60437189221c2c527990b)
Signed-off-by: DB Tsai
---
.../datasources/parquet/ParquetSchemaPruning.scala | 34 +++-
.../parquet/ParquetSchemaPruningSuite.scala| 36 ++
2 files changed, 62 insertions(+), 8 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruning.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruning.scala
index 91080b1..840fcae 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruning.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruning.scala
@@ -116,10 +116,28 @@ private[sql] object ParquetSchemaPruning extends
Rule[LogicalPlan] {
// For example, for a query `SELECT name.first FROM contacts WHERE name IS
NOT NULL`,
// we don't need to read nested fields of `name` struct other than `first`
field.
val (rootFields, optRootFields) = (projectionRootFields ++
filterRootFields)
- .distinct.partition(_.contentAccessed)
+ .distinct.partition(!_.prunedIfAnyChildAccessed)
optRootFields.filter { opt =>
- !rootFields.exists(_.field.name == opt.field.name)
+ !rootFields.exists { root =>
+root.field.name == opt.field.name && {
+ // Checking if current optional root field can be pruned.
+ // For each required root field, we merge it with the optional root
field:
+ // 1. If this optional root field has nested fields and any nested
field of it is used
+ //in the query, the merged field type must equal to the optional
root field type.
+ //We can prune this optional root field. For example, for
optional root field
+ //`struct>`, if its field
+ //`struct>` is used, we don't need to
add this optional
+ //root field.
+ // 2. If this optional root field has no nested fields, the merged
field type equals
+ //to the optional root field only if they are the same. If they
are, we can prune
+ //this optional root field too.
+ val rootFieldType = StructType(Array(root.field))
+ val optFieldType = StructType(Array(opt.field))
+ val merged = optFieldType.merge(rootFieldType)
+ merged.sameType(optFieldType)
+}
+ }
} ++ rootFields
}
@@ -213,11 +231,11 @@ private[sql] object ParquetSchemaPruning extends
Rule[LogicalPlan] {
// don't actually use any nested fields. These root field accesses might
be excluded later
// if there are any nested fields acce
[spark] branch master updated: [SPARK-26551][SQL] Fix schema pruning error when selecting one complex field and having is not null predicate on another one
This is an automated email from the ASF dual-hosted git repository.
dbtsai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 50ebf3a [SPARK-26551][SQL] Fix schema pruning error when selecting
one complex field and having is not null predicate on another one
50ebf3a is described below
commit 50ebf3a43b84c8538ec60437189221c2c527990b
Author: Liang-Chi Hsieh
AuthorDate: Fri Jan 11 19:23:32 2019 +
[SPARK-26551][SQL] Fix schema pruning error when selecting one complex
field and having is not null predicate on another one
## What changes were proposed in this pull request?
Schema pruning has errors when selecting one complex field and having is
not null predicate on another one:
```scala
val query = sql("select * from contacts")
.where("name.middle is not null")
.select(
"id",
"name.first",
"name.middle",
"name.last"
)
.where("last = 'Jones'")
.select(count("id"))
```
```
java.lang.IllegalArgumentException: middle does not exist. Available: last
[info] at
org.apache.spark.sql.types.StructType.$anonfun$fieldIndex$1(StructType.scala:303)
[info] at scala.collection.immutable.Map$Map1.getOrElse(Map.scala:119)
[info] at
org.apache.spark.sql.types.StructType.fieldIndex(StructType.scala:302)
[info] at
org.apache.spark.sql.execution.ProjectionOverSchema.$anonfun$getProjection$6(ProjectionOverSchema.scala:58)
[info] at scala.Option.map(Option.scala:163)
[info] at
org.apache.spark.sql.execution.ProjectionOverSchema.getProjection(ProjectionOverSchema.scala:56)
[info] at
org.apache.spark.sql.execution.ProjectionOverSchema.unapply(ProjectionOverSchema.scala:32)
[info] at
org.apache.spark.sql.execution.datasources.parquet.ParquetSchemaPruning$$anonfun$$nestedInanonfun$buildNewProjection$1$1.applyOrElse(Parque
tSchemaPruning.scala:153)
```
## How was this patch tested?
Added tests.
Closes #23474 from viirya/SPARK-26551.
Authored-by: Liang-Chi Hsieh
Signed-off-by: DB Tsai
---
.../datasources/parquet/ParquetSchemaPruning.scala | 34 +++-
.../parquet/ParquetSchemaPruningSuite.scala| 36 ++
2 files changed, 62 insertions(+), 8 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruning.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruning.scala
index 91080b1..840fcae 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruning.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruning.scala
@@ -116,10 +116,28 @@ private[sql] object ParquetSchemaPruning extends
Rule[LogicalPlan] {
// For example, for a query `SELECT name.first FROM contacts WHERE name IS
NOT NULL`,
// we don't need to read nested fields of `name` struct other than `first`
field.
val (rootFields, optRootFields) = (projectionRootFields ++
filterRootFields)
- .distinct.partition(_.contentAccessed)
+ .distinct.partition(!_.prunedIfAnyChildAccessed)
optRootFields.filter { opt =>
- !rootFields.exists(_.field.name == opt.field.name)
+ !rootFields.exists { root =>
+root.field.name == opt.field.name && {
+ // Checking if current optional root field can be pruned.
+ // For each required root field, we merge it with the optional root
field:
+ // 1. If this optional root field has nested fields and any nested
field of it is used
+ //in the query, the merged field type must equal to the optional
root field type.
+ //We can prune this optional root field. For example, for
optional root field
+ //`struct>`, if its field
+ //`struct>` is used, we don't need to
add this optional
+ //root field.
+ // 2. If this optional root field has no nested fields, the merged
field type equals
+ //to the optional root field only if they are the same. If they
are, we can prune
+ //this optional root field too.
+ val rootFieldType = StructType(Array(root.field))
+ val optFieldType = StructType(Array(opt.field))
+ val merged = optFieldType.merge(rootFieldType)
+ merged.sameType(optFieldType)
+}
+ }
} ++ rootFields
}
@@ -213,11 +231,11 @@ private[sql] object ParquetSchemaPruning extends
Rule[LogicalPlan] {
// don't actually use any nested fields. These root field accesses might
be excluded later
// if there are any nested fields accesses in the query plan.
case IsNotNull(SelectedField(field)) =>
-RootField(field, derivedFromA
svn commit: r31902 - in /dev/spark/2.4.1-SNAPSHOT-2019_01_11_10_40-b9eb0e8-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Fri Jan 11 18:57:29 2019 New Revision: 31902 Log: Apache Spark 2.4.1-SNAPSHOT-2019_01_11_10_40-b9eb0e8 docs [This commit notification would consist of 1476 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
svn commit: r31901 - in /dev/spark/2.3.3-SNAPSHOT-2019_01_11_10_40-87c2c11-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Fri Jan 11 18:55:32 2019 New Revision: 31901 Log: Apache Spark 2.3.3-SNAPSHOT-2019_01_11_10_40-87c2c11 docs [This commit notification would consist of 1443 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[spark] branch master updated: [SPARK-26482][CORE] Use ConfigEntry for hardcoded configs for ui categories
This is an automated email from the ASF dual-hosted git repository. vanzin pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git The following commit(s) were added to refs/heads/master by this push: new d9e4cf6 [SPARK-26482][CORE] Use ConfigEntry for hardcoded configs for ui categories d9e4cf6 is described below commit d9e4cf67c06b2d6daa4cd24b056e33dfb5eb35f5 Author: Jungtaek Lim (HeartSaVioR) AuthorDate: Fri Jan 11 10:18:07 2019 -0800 [SPARK-26482][CORE] Use ConfigEntry for hardcoded configs for ui categories ## What changes were proposed in this pull request? The PR makes hardcoded configs below to use `ConfigEntry`. * spark.ui * spark.ssl * spark.authenticate * spark.master.rest * spark.master.ui * spark.metrics * spark.admin * spark.modify.acl This patch doesn't change configs which are not relevant to SparkConf (e.g. system properties). ## How was this patch tested? Existing tests. Closes #23423 from HeartSaVioR/SPARK-26466. Authored-by: Jungtaek Lim (HeartSaVioR) Signed-off-by: Marcelo Vanzin --- .../scala/org/apache/spark/SecurityManager.scala | 54 +++ .../main/scala/org/apache/spark/SparkContext.scala | 5 +- .../apache/spark/deploy/LocalSparkCluster.scala| 4 +- .../org/apache/spark/deploy/SparkSubmit.scala | 1 + .../spark/deploy/history/FsHistoryProvider.scala | 47 +++--- .../spark/deploy/history/HistoryServer.scala | 10 +- .../org/apache/spark/deploy/master/Master.scala| 10 +- .../spark/deploy/master/MasterArguments.scala | 5 +- .../spark/deploy/master/ui/MasterWebUI.scala | 3 +- .../spark/deploy/rest/StandaloneRestServer.scala | 2 +- .../spark/deploy/worker/ExecutorRunner.scala | 3 +- .../org/apache/spark/deploy/worker/Worker.scala| 3 +- .../org/apache/spark/internal/config/History.scala | 12 +- .../org/apache/spark/internal/config/UI.scala | 145 +++ .../org/apache/spark/internal/config/package.scala | 50 +++ .../org/apache/spark/metrics/MetricsConfig.scala | 3 +- .../org/apache/spark/ui/ConsoleProgressBar.scala | 4 +- .../org/apache/spark/ui/HttpSecurityFilter.scala | 2 +- .../scala/org/apache/spark/ui/JettyUtils.scala | 4 +- .../main/scala/org/apache/spark/ui/SparkUI.scala | 6 +- .../org/apache/spark/ui/exec/ExecutorsTab.scala| 3 +- .../scala/org/apache/spark/ui/jobs/StagePage.scala | 3 +- .../main/scala/org/apache/spark/util/Utils.scala | 4 +- .../scala/org/apache/spark/CheckpointSuite.scala | 3 +- .../org/apache/spark/SecurityManagerSuite.scala| 160 +++-- .../scala/org/apache/spark/SparkContextSuite.scala | 5 +- .../org/apache/spark/deploy/SparkSubmitSuite.scala | 13 +- .../deploy/history/FsHistoryProviderSuite.scala| 23 +-- .../spark/deploy/history/HistoryServerSuite.scala | 7 +- .../apache/spark/deploy/master/MasterSuite.scala | 10 +- .../org/apache/spark/executor/ExecutorSuite.scala | 3 +- .../spark/launcher/LauncherBackendSuite.scala | 3 +- .../apache/spark/metrics/MetricsConfigSuite.scala | 9 +- .../apache/spark/metrics/MetricsSystemSuite.scala | 2 +- .../netty/NettyBlockTransferSecuritySuite.scala| 27 ++-- .../scala/org/apache/spark/rpc/RpcEnvSuite.scala | 29 ++-- .../apache/spark/scheduler/TaskContextSuite.scala | 3 +- .../storage/BlockManagerReplicationSuite.scala | 4 +- .../apache/spark/ui/HttpSecurityFilterSuite.scala | 10 +- .../org/apache/spark/ui/UISeleniumSuite.scala | 13 +- .../test/scala/org/apache/spark/ui/UISuite.scala | 3 +- .../k8s/features/BasicDriverFeatureStep.scala | 1 + .../k8s/features/BasicDriverFeatureStepSuite.scala | 3 +- .../integrationtest/KubernetesTestComponents.scala | 5 +- .../spark/deploy/yarn/ApplicationMaster.scala | 5 +- .../scheduler/cluster/YarnSchedulerBackend.scala | 5 +- .../spark/deploy/yarn/YarnClusterSuite.scala | 4 +- .../deploy/yarn/YarnSparkHadoopUtilSuite.scala | 10 +- .../spark/sql/SparkSessionBuilderSuite.scala | 3 +- .../sql/execution/ExchangeCoordinatorSuite.scala | 3 +- .../benchmark/DataSourceReadBenchmark.scala| 3 +- .../benchmark/FilterPushdownBenchmark.scala| 3 +- .../sql/hive/thriftserver/HiveThriftServer2.scala | 5 +- .../org/apache/spark/sql/hive/test/TestHive.scala | 3 +- .../spark/sql/hive/HiveSparkSubmitSuite.scala | 23 +-- .../sql/hive/execution/ConcurrentHiveSuite.scala | 3 +- .../org/apache/spark/streaming/Checkpoint.scala| 3 +- .../apache/spark/streaming/StreamingContext.scala | 3 +- .../org/apache/spark/streaming/ReceiverSuite.scala | 3 +- .../apache/spark/streaming/UISeleniumSuite.scala | 3 +- 60 files changed, 496 insertions(+), 305 deletions(-) diff --git a/core/src/main/scala/org/apache/spar
[spark] branch branch-2.3 updated: [SPARK-26576][SQL] Broadcast hint not applied to partitioned table
This is an automated email from the ASF dual-hosted git repository.
lixiao pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.3 by this push:
new 87c2c11 [SPARK-26576][SQL] Broadcast hint not applied to partitioned
table
87c2c11 is described below
commit 87c2c11e742a8b35699f68ec2002f817c56bef87
Author: John Zhuge
AuthorDate: Fri Jan 11 09:21:13 2019 -0800
[SPARK-26576][SQL] Broadcast hint not applied to partitioned table
## What changes were proposed in this pull request?
Make sure broadcast hint is applied to partitioned tables.
Since the issue exists in branch 2.0 to 2.4, but not in master, I created
this PR for branch-2.4.
## How was this patch tested?
- A new unit test in PruneFileSourcePartitionsSuite
- Unit test suites touched by SPARK-14581: JoinOptimizationSuite,
FilterPushdownSuite, ColumnPruningSuite, and PruneFiltersSuite
cloud-fan davies rxin
Closes #23507 from jzhuge/SPARK-26576.
Authored-by: John Zhuge
Signed-off-by: gatorsmile
(cherry picked from commit b9eb0e85de3317a7f4c89a90082f7793b645c6ea)
Signed-off-by: gatorsmile
---
.../apache/spark/sql/catalyst/planning/patterns.scala | 3 ---
.../execution/PruneFileSourcePartitionsSuite.scala| 19 ++-
2 files changed, 18 insertions(+), 4 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
index cc391aa..a91063b 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
@@ -65,9 +65,6 @@ object PhysicalOperation extends PredicateHelper {
val substitutedCondition = substitute(aliases)(condition)
(fields, filters ++ splitConjunctivePredicates(substitutedCondition),
other, aliases)
- case h: ResolvedHint =>
-collectProjectsAndFilters(h.child)
-
case other =>
(None, Nil, other, Map.empty)
}
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala
index 9438418..8a9adf7 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala
@@ -17,15 +17,20 @@
package org.apache.spark.sql.hive.execution
+import org.scalatest.Matchers._
+
import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
-import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan,
Project}
+import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan,
Project, ResolvedHint}
import org.apache.spark.sql.catalyst.rules.RuleExecutor
import org.apache.spark.sql.execution.datasources.{CatalogFileIndex,
HadoopFsRelation, LogicalRelation, PruneFileSourcePartitions}
import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
+import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec
+import org.apache.spark.sql.functions.broadcast
import org.apache.spark.sql.hive.test.TestHiveSingleton
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.sql.types.StructType
@@ -91,4 +96,16 @@ class PruneFileSourcePartitionsSuite extends QueryTest with
SQLTestUtils with Te
assert(size2 < tableStats.get.sizeInBytes)
}
}
+
+ test("SPARK-26576 Broadcast hint not applied to partitioned table") {
+withTable("tbl") {
+ withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+spark.range(10).selectExpr("id", "id % 3 as
p").write.partitionBy("p").saveAsTable("tbl")
+val df = spark.table("tbl")
+val qe = df.join(broadcast(df), "p").queryExecution
+qe.optimizedPlan.collect { case _: ResolvedHint => } should have size 1
+qe.sparkPlan.collect { case j: BroadcastHashJoinExec => j } should
have size 1
+ }
+}
+ }
}
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
[spark] branch branch-2.4 updated: [SPARK-26576][SQL] Broadcast hint not applied to partitioned table
This is an automated email from the ASF dual-hosted git repository.
lixiao pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new b9eb0e8 [SPARK-26576][SQL] Broadcast hint not applied to partitioned
table
b9eb0e8 is described below
commit b9eb0e85de3317a7f4c89a90082f7793b645c6ea
Author: John Zhuge
AuthorDate: Fri Jan 11 09:21:13 2019 -0800
[SPARK-26576][SQL] Broadcast hint not applied to partitioned table
## What changes were proposed in this pull request?
Make sure broadcast hint is applied to partitioned tables.
Since the issue exists in branch 2.0 to 2.4, but not in master, I created
this PR for branch-2.4.
## How was this patch tested?
- A new unit test in PruneFileSourcePartitionsSuite
- Unit test suites touched by SPARK-14581: JoinOptimizationSuite,
FilterPushdownSuite, ColumnPruningSuite, and PruneFiltersSuite
cloud-fan davies rxin
Closes #23507 from jzhuge/SPARK-26576.
Authored-by: John Zhuge
Signed-off-by: gatorsmile
---
.../apache/spark/sql/catalyst/planning/patterns.scala | 3 ---
.../execution/PruneFileSourcePartitionsSuite.scala| 19 ++-
2 files changed, 18 insertions(+), 4 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
index 84be677..d91b890 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
@@ -66,9 +66,6 @@ object PhysicalOperation extends PredicateHelper {
val substitutedCondition = substitute(aliases)(condition)
(fields, filters ++ splitConjunctivePredicates(substitutedCondition),
other, aliases)
- case h: ResolvedHint =>
-collectProjectsAndFilters(h.child)
-
case other =>
(None, Nil, other, Map.empty)
}
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala
index 9438418..8a9adf7 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala
@@ -17,15 +17,20 @@
package org.apache.spark.sql.hive.execution
+import org.scalatest.Matchers._
+
import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
-import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan,
Project}
+import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan,
Project, ResolvedHint}
import org.apache.spark.sql.catalyst.rules.RuleExecutor
import org.apache.spark.sql.execution.datasources.{CatalogFileIndex,
HadoopFsRelation, LogicalRelation, PruneFileSourcePartitions}
import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
+import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec
+import org.apache.spark.sql.functions.broadcast
import org.apache.spark.sql.hive.test.TestHiveSingleton
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.sql.types.StructType
@@ -91,4 +96,16 @@ class PruneFileSourcePartitionsSuite extends QueryTest with
SQLTestUtils with Te
assert(size2 < tableStats.get.sizeInBytes)
}
}
+
+ test("SPARK-26576 Broadcast hint not applied to partitioned table") {
+withTable("tbl") {
+ withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+spark.range(10).selectExpr("id", "id % 3 as
p").write.partitionBy("p").saveAsTable("tbl")
+val df = spark.table("tbl")
+val qe = df.join(broadcast(df), "p").queryExecution
+qe.optimizedPlan.collect { case _: ResolvedHint => } should have size 1
+qe.sparkPlan.collect { case j: BroadcastHashJoinExec => j } should
have size 1
+ }
+}
+ }
}
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
svn commit: r31900 - in /dev/spark/3.0.0-SNAPSHOT-2019_01_11_08_30-51a6ba0-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Fri Jan 11 16:43:16 2019 New Revision: 31900 Log: Apache Spark 3.0.0-SNAPSHOT-2019_01_11_08_30-51a6ba0 docs [This commit notification would consist of 1775 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[spark] branch master updated: [SPARK-26503][CORE] Get rid of spark.sql.legacy.timeParser.enabled
This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 51a6ba0 [SPARK-26503][CORE] Get rid of
spark.sql.legacy.timeParser.enabled
51a6ba0 is described below
commit 51a6ba0181a013f2b62b47184785a8b6f6a78f12
Author: Sean Owen
AuthorDate: Fri Jan 11 08:53:12 2019 -0600
[SPARK-26503][CORE] Get rid of spark.sql.legacy.timeParser.enabled
## What changes were proposed in this pull request?
Per discussion in #23391 (comment) this proposes to just remove the old
pre-Spark-3 time parsing behavior.
This is a rebase of https://github.com/apache/spark/pull/23411
## How was this patch tested?
Existing tests.
Closes #23495 from srowen/SPARK-26503.2.
Authored-by: Sean Owen
Signed-off-by: Sean Owen
---
.../spark/sql/catalyst/util/DateFormatter.scala| 43 +
.../sql/catalyst/util/TimestampFormatter.scala | 37 +
.../org/apache/spark/sql/internal/SQLConf.scala| 9 -
.../sql/catalyst/json/JsonInferSchemaSuite.scala | 77 -
.../sql/execution/datasources/json/JsonSuite.scala | 183 -
.../spark/sql/sources/HadoopFsRelationTest.scala | 107 ++--
6 files changed, 120 insertions(+), 336 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
index db92552..c47b087 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
@@ -20,12 +20,6 @@ package org.apache.spark.sql.catalyst.util
import java.time.{Instant, ZoneId}
import java.util.Locale
-import scala.util.Try
-
-import org.apache.commons.lang3.time.FastDateFormat
-
-import org.apache.spark.sql.internal.SQLConf
-
sealed trait DateFormatter extends Serializable {
def parse(s: String): Int // returns days since epoch
def format(days: Int): String
@@ -56,43 +50,8 @@ class Iso8601DateFormatter(
}
}
-class LegacyDateFormatter(pattern: String, locale: Locale) extends
DateFormatter {
- @transient
- private lazy val format = FastDateFormat.getInstance(pattern, locale)
-
- override def parse(s: String): Int = {
-val milliseconds = format.parse(s).getTime
-DateTimeUtils.millisToDays(milliseconds)
- }
-
- override def format(days: Int): String = {
-val date = DateTimeUtils.toJavaDate(days)
-format.format(date)
- }
-}
-
-class LegacyFallbackDateFormatter(
-pattern: String,
-locale: Locale) extends LegacyDateFormatter(pattern, locale) {
- override def parse(s: String): Int = {
-Try(super.parse(s)).orElse {
- // If it fails to parse, then tries the way used in 2.0 and 1.x for
backwards
- // compatibility.
- Try(DateTimeUtils.millisToDays(DateTimeUtils.stringToTime(s).getTime))
-}.getOrElse {
- // In Spark 1.5.0, we store the data as number of days since epoch in
string.
- // So, we just convert it to Int.
- s.toInt
-}
- }
-}
-
object DateFormatter {
def apply(format: String, locale: Locale): DateFormatter = {
-if (SQLConf.get.legacyTimeParserEnabled) {
- new LegacyFallbackDateFormatter(format, locale)
-} else {
- new Iso8601DateFormatter(format, locale)
-}
+new Iso8601DateFormatter(format, locale)
}
}
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
index 8042099..10c73b2 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
@@ -23,12 +23,6 @@ import java.time.format.DateTimeParseException
import java.time.temporal.TemporalQueries
import java.util.{Locale, TimeZone}
-import scala.util.Try
-
-import org.apache.commons.lang3.time.FastDateFormat
-
-import org.apache.spark.sql.internal.SQLConf
-
sealed trait TimestampFormatter extends Serializable {
/**
* Parses a timestamp in a string and converts it to microseconds.
@@ -79,37 +73,8 @@ class Iso8601TimestampFormatter(
}
}
-class LegacyTimestampFormatter(
-pattern: String,
-timeZone: TimeZone,
-locale: Locale) extends TimestampFormatter {
- @transient
- private lazy val format = FastDateFormat.getInstance(pattern, timeZone,
locale)
-
- protected def toMillis(s: String): Long = format.parse(s).getTime
-
- override def parse(s: String): Long = toMillis(s) *
DateTimeUtils.MICROS_PER_MILLIS
-
- override def format(us: Long): String = {
-format.format(DateTimeUtils.toJavaTimestamp(us))
- }
-}
-
-class Lega
