svn commit: r27891 - in /dev/spark/2.3.2-SNAPSHOT-2018_07_02_22_01-bc7ee75-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Tue Jul 3 05:15:55 2018 New Revision: 27891 Log: Apache Spark 2.3.2-SNAPSHOT-2018_07_02_22_01-bc7ee75 docs [This commit notification would consist of 1443 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24385][SQL] Resolve self-join condition ambiguity for EqualNullSafe
Repository: spark
Updated Branches:
refs/heads/branch-2.3 1cba0505e -> bc7ee7586
[SPARK-24385][SQL] Resolve self-join condition ambiguity for EqualNullSafe
## What changes were proposed in this pull request?
In Dataset.join we have a small hack for resolving ambiguity in the column name
for self-joins. The current code supports only `EqualTo`.
The PR extends the fix to `EqualNullSafe`.
Credit for this PR should be given to daniel-shields.
## How was this patch tested?
added UT
Author: Marco Gaido
Closes #21605 from mgaido91/SPARK-24385_2.
(cherry picked from commit a7c8f0c8cb144a026ea21e8780107e363ceacb8d)
Signed-off-by: Wenchen Fan
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/bc7ee758
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/bc7ee758
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/bc7ee758
Branch: refs/heads/branch-2.3
Commit: bc7ee758698911d75ffda386ee8048b42b0f2681
Parents: 1cba050
Author: Marco Gaido
Authored: Tue Jul 3 12:20:03 2018 +0800
Committer: Wenchen Fan
Committed: Tue Jul 3 12:28:58 2018 +0800
--
sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala | 5 +
.../test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala | 8
2 files changed, 13 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/bc7ee758/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index 21c9250..784abc8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -995,6 +995,11 @@ class Dataset[T] private[sql](
catalyst.expressions.EqualTo(
withPlan(plan.left).resolve(a.name),
withPlan(plan.right).resolve(b.name))
+ case catalyst.expressions.EqualNullSafe(a: AttributeReference, b:
AttributeReference)
+if a.sameRef(b) =>
+catalyst.expressions.EqualNullSafe(
+ withPlan(plan.left).resolve(a.name),
+ withPlan(plan.right).resolve(b.name))
}}
withPlan {
http://git-wip-us.apache.org/repos/asf/spark/blob/bc7ee758/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
--
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
index 0d9eeab..10d9a11 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
@@ -287,4 +287,12 @@ class DataFrameJoinSuite extends QueryTest with
SharedSQLContext {
dfOne.join(dfTwo, $"a" === $"b", "left").queryExecution.optimizedPlan
}
}
+
+ test("SPARK-24385: Resolve ambiguity in self-joins with EqualNullSafe") {
+withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "false") {
+ val df = spark.range(2)
+ // this throws an exception before the fix
+ df.join(df, df("id") <=> df("id")).queryExecution.optimizedPlan
+}
+ }
}
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24385][SQL] Resolve self-join condition ambiguity for EqualNullSafe
Repository: spark
Updated Branches:
refs/heads/master 85fe1297e -> a7c8f0c8c
[SPARK-24385][SQL] Resolve self-join condition ambiguity for EqualNullSafe
## What changes were proposed in this pull request?
In Dataset.join we have a small hack for resolving ambiguity in the column name
for self-joins. The current code supports only `EqualTo`.
The PR extends the fix to `EqualNullSafe`.
Credit for this PR should be given to daniel-shields.
## How was this patch tested?
added UT
Author: Marco Gaido
Closes #21605 from mgaido91/SPARK-24385_2.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a7c8f0c8
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a7c8f0c8
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a7c8f0c8
Branch: refs/heads/master
Commit: a7c8f0c8cb144a026ea21e8780107e363ceacb8d
Parents: 85fe129
Author: Marco Gaido
Authored: Tue Jul 3 12:20:03 2018 +0800
Committer: Wenchen Fan
Committed: Tue Jul 3 12:20:03 2018 +0800
--
sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala | 5 +
.../test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala | 8
2 files changed, 13 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/a7c8f0c8/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index 2ec236f..c97246f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -1016,6 +1016,11 @@ class Dataset[T] private[sql](
catalyst.expressions.EqualTo(
withPlan(plan.left).resolve(a.name),
withPlan(plan.right).resolve(b.name))
+ case catalyst.expressions.EqualNullSafe(a: AttributeReference, b:
AttributeReference)
+if a.sameRef(b) =>
+catalyst.expressions.EqualNullSafe(
+ withPlan(plan.left).resolve(a.name),
+ withPlan(plan.right).resolve(b.name))
}}
withPlan {
http://git-wip-us.apache.org/repos/asf/spark/blob/a7c8f0c8/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
--
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
index 0d9eeab..10d9a11 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
@@ -287,4 +287,12 @@ class DataFrameJoinSuite extends QueryTest with
SharedSQLContext {
dfOne.join(dfTwo, $"a" === $"b", "left").queryExecution.optimizedPlan
}
}
+
+ test("SPARK-24385: Resolve ambiguity in self-joins with EqualNullSafe") {
+withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "false") {
+ val df = spark.range(2)
+ // this throws an exception before the fix
+ df.join(df, df("id") <=> df("id")).queryExecution.optimizedPlan
+}
+ }
}
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
svn commit: r27887 - in /dev/spark/2.4.0-SNAPSHOT-2018_07_02_16_01-85fe129-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Mon Jul 2 23:16:06 2018 New Revision: 27887 Log: Apache Spark 2.4.0-SNAPSHOT-2018_07_02_16_01-85fe129 docs [This commit notification would consist of 1467 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24428][K8S] Fix unused code
Repository: spark
Updated Branches:
refs/heads/master 42815548c -> 85fe1297e
[SPARK-24428][K8S] Fix unused code
## What changes were proposed in this pull request?
Remove code that is misleading and is a leftover from a previous implementation.
## How was this patch tested?
Manually.
Author: Stavros Kontopoulos
Closes #21462 from skonto/fix-k8s-docs.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/85fe1297
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/85fe1297
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/85fe1297
Branch: refs/heads/master
Commit: 85fe1297e35bcff9cf86bd53fee615e140ee5bfb
Parents: 4281554
Author: Stavros Kontopoulos
Authored: Mon Jul 2 13:08:16 2018 -0700
Committer: Anirudh Ramanathan
Committed: Mon Jul 2 13:08:16 2018 -0700
--
.../scala/org/apache/spark/deploy/k8s/Constants.scala | 6 --
.../cluster/k8s/KubernetesClusterManager.scala | 2 --
.../docker/src/main/dockerfiles/spark/entrypoint.sh | 12 +---
3 files changed, 5 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/85fe1297/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
--
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
index 69bd03d..5ecdd3a 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
@@ -25,9 +25,6 @@ private[spark] object Constants {
val SPARK_POD_DRIVER_ROLE = "driver"
val SPARK_POD_EXECUTOR_ROLE = "executor"
- // Annotations
- val SPARK_APP_NAME_ANNOTATION = "spark-app-name"
-
// Credentials secrets
val DRIVER_CREDENTIALS_SECRETS_BASE_DIR =
"/mnt/secrets/spark-kubernetes-credentials"
@@ -50,17 +47,14 @@ private[spark] object Constants {
val DEFAULT_BLOCKMANAGER_PORT = 7079
val DRIVER_PORT_NAME = "driver-rpc-port"
val BLOCK_MANAGER_PORT_NAME = "blockmanager"
- val EXECUTOR_PORT_NAME = "executor"
// Environment Variables
- val ENV_EXECUTOR_PORT = "SPARK_EXECUTOR_PORT"
val ENV_DRIVER_URL = "SPARK_DRIVER_URL"
val ENV_EXECUTOR_CORES = "SPARK_EXECUTOR_CORES"
val ENV_EXECUTOR_MEMORY = "SPARK_EXECUTOR_MEMORY"
val ENV_APPLICATION_ID = "SPARK_APPLICATION_ID"
val ENV_EXECUTOR_ID = "SPARK_EXECUTOR_ID"
val ENV_EXECUTOR_POD_IP = "SPARK_EXECUTOR_POD_IP"
- val ENV_MOUNTED_CLASSPATH = "SPARK_MOUNTED_CLASSPATH"
val ENV_JAVA_OPT_PREFIX = "SPARK_JAVA_OPT_"
val ENV_CLASSPATH = "SPARK_CLASSPATH"
val ENV_DRIVER_BIND_ADDRESS = "SPARK_DRIVER_BIND_ADDRESS"
http://git-wip-us.apache.org/repos/asf/spark/blob/85fe1297/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala
--
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala
index c6e931a..de2a52b 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala
@@ -48,8 +48,6 @@ private[spark] class KubernetesClusterManager extends
ExternalClusterManager wit
sc: SparkContext,
masterURL: String,
scheduler: TaskScheduler): SchedulerBackend = {
-val executorSecretNamesToMountPaths =
KubernetesUtils.parsePrefixedKeyValuePairs(
- sc.conf, KUBERNETES_EXECUTOR_SECRETS_PREFIX)
val kubernetesClient = SparkKubernetesClientFactory.createKubernetesClient(
KUBERNETES_MASTER_INTERNAL_URL,
Some(sc.conf.get(KUBERNETES_NAMESPACE)),
http://git-wip-us.apache.org/repos/asf/spark/blob/85fe1297/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
--
diff --git
a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
index 2f4e115..8bdb0f7 100755
---
a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
+++
b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
@@ -51,12 +51,10 @@ esac
SPARK_CLASSPATH="$SPARK_CLASSPATH:${S
svn commit: r27873 - in /dev/spark/2.4.0-SNAPSHOT-2018_07_02_12_01-4281554-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Mon Jul 2 19:16:09 2018 New Revision: 27873 Log: Apache Spark 2.4.0-SNAPSHOT-2018_07_02_12_01-4281554 docs [This commit notification would consist of 1467 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24683][K8S] Fix k8s no resource
Repository: spark
Updated Branches:
refs/heads/master f599cde69 -> 42815548c
[SPARK-24683][K8S] Fix k8s no resource
## What changes were proposed in this pull request?
Make SparkSubmit pass in the main class even if `SparkLauncher.NO_RESOURCE` is
the primary resource.
## How was this patch tested?
New integration test written to capture this case.
Author: mcheah
Closes #21660 from mccheah/fix-k8s-no-resource.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/42815548
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/42815548
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/42815548
Branch: refs/heads/master
Commit: 42815548c7ef498439b9ba47134a6f3e1b519c83
Parents: f599cde
Author: mcheah
Authored: Mon Jul 2 10:24:04 2018 -0700
Committer: mcheah
Committed: Mon Jul 2 10:24:04 2018 -0700
--
.../main/scala/org/apache/spark/deploy/SparkSubmit.scala | 2 ++
.../spark/deploy/k8s/submit/KubernetesDriverBuilder.scala | 3 ++-
.../deploy/k8s/integrationtest/KubernetesSuite.scala | 10 --
3 files changed, 12 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/42815548/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
--
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
index e83d82f..2da778a 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -702,6 +702,8 @@ private[spark] class SparkSubmit extends Logging {
childArgs ++= Array("--primary-java-resource", args.primaryResource)
childArgs ++= Array("--main-class", args.mainClass)
}
+ } else {
+childArgs ++= Array("--main-class", args.mainClass)
}
if (args.childArgs != null) {
args.childArgs.foreach { arg =>
http://git-wip-us.apache.org/repos/asf/spark/blob/42815548/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala
--
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala
index 5762d82..0dd1c37 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesDriverBuilder.scala
@@ -64,7 +64,8 @@ private[spark] class KubernetesDriverBuilder(
case JavaMainAppResource(_) =>
provideJavaStep(kubernetesConf)
case PythonMainAppResource(_) =>
-
providePythonStep(kubernetesConf)}.getOrElse(provideJavaStep(kubernetesConf))
+ providePythonStep(kubernetesConf)}
+ .getOrElse(provideJavaStep(kubernetesConf))
val allFeatures: Seq[KubernetesFeatureConfigStep] =
(baseFeatures :+ bindingsStep) ++
http://git-wip-us.apache.org/repos/asf/spark/blob/42815548/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
--
diff --git
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
index 65c513c..6e334c8 100644
---
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
+++
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
@@ -21,17 +21,17 @@ import java.nio.file.{Path, Paths}
import java.util.UUID
import java.util.regex.Pattern
-import scala.collection.JavaConverters._
-
import com.google.common.io.PatternFilenameFilter
import io.fabric8.kubernetes.api.model.{Container, Pod}
import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll}
import org.scalatest.concurrent.{Eventually, PatienceConfiguration}
import org.scalatest.time.{Minutes, Seconds, Span}
+import scala.collection.JavaConverters._
import org.apache.spark.SparkFunSuite
import
org.apache.spark.deploy.k8s.integrationtest.backend.{IntegrationTestBackend,
IntegrationTestBackendFactory}
import org.apache.spark.deploy.k8s.integrationtest.config._
+import org.apache.spark.launcher.S
svn commit: r27870 - in /dev/spark/2.3.2-SNAPSHOT-2018_07_02_10_01-1cba050-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Mon Jul 2 17:16:29 2018 New Revision: 27870 Log: Apache Spark 2.3.2-SNAPSHOT-2018_07_02_10_01-1cba050 docs [This commit notification would consist of 1443 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
svn commit: r27859 - in /dev/spark/2.4.0-SNAPSHOT-2018_07_02_08_01-f599cde-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Mon Jul 2 15:16:20 2018 New Revision: 27859 Log: Apache Spark 2.4.0-SNAPSHOT-2018_07_02_08_01-f599cde docs [This commit notification would consist of 1467 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24507][DOCUMENTATION] Update streaming guide
Repository: spark
Updated Branches:
refs/heads/branch-2.3 3c0af793f -> 1cba0505e
[SPARK-24507][DOCUMENTATION] Update streaming guide
## What changes were proposed in this pull request?
Updated streaming guide for direct stream and link to integration guide.
## How was this patch tested?
jekyll build
Author: Rekha Joshi
Closes #21683 from rekhajoshm/SPARK-24507.
(cherry picked from commit f599cde69506a5aedeeec449cba9a8b5ab128282)
Signed-off-by: hyukjinkwon
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1cba0505
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1cba0505
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1cba0505
Branch: refs/heads/branch-2.3
Commit: 1cba0505ef36b83611c1f2ba0ad5e638980670ac
Parents: 3c0af79
Author: Rekha Joshi
Authored: Mon Jul 2 22:39:00 2018 +0800
Committer: hyukjinkwon
Committed: Mon Jul 2 22:39:22 2018 +0800
--
docs/streaming-programming-guide.md | 2 ++
1 file changed, 2 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/1cba0505/docs/streaming-programming-guide.md
--
diff --git a/docs/streaming-programming-guide.md
b/docs/streaming-programming-guide.md
index ffda36d..9efa9fc 100644
--- a/docs/streaming-programming-guide.md
+++ b/docs/streaming-programming-guide.md
@@ -2176,6 +2176,8 @@ the input data stream (using
`inputStream.repartition()`).
This distributes the received batches of data across the specified number of
machines in the cluster
before further processing.
+For direct stream, please refer to [Spark Streaming + Kafka Integration
Guide](streaming-kafka-integration.html)
+
### Level of Parallelism in Data Processing
{:.no_toc}
Cluster resources can be under-utilized if the number of parallel tasks used
in any stage of the
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24507][DOCUMENTATION] Update streaming guide
Repository: spark
Updated Branches:
refs/heads/master 8008f9cb8 -> f599cde69
[SPARK-24507][DOCUMENTATION] Update streaming guide
## What changes were proposed in this pull request?
Updated streaming guide for direct stream and link to integration guide.
## How was this patch tested?
jekyll build
Author: Rekha Joshi
Closes #21683 from rekhajoshm/SPARK-24507.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f599cde6
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f599cde6
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f599cde6
Branch: refs/heads/master
Commit: f599cde69506a5aedeeec449cba9a8b5ab128282
Parents: 8008f9c
Author: Rekha Joshi
Authored: Mon Jul 2 22:39:00 2018 +0800
Committer: hyukjinkwon
Committed: Mon Jul 2 22:39:00 2018 +0800
--
docs/streaming-programming-guide.md | 2 ++
1 file changed, 2 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/f599cde6/docs/streaming-programming-guide.md
--
diff --git a/docs/streaming-programming-guide.md
b/docs/streaming-programming-guide.md
index c309592..118b053 100644
--- a/docs/streaming-programming-guide.md
+++ b/docs/streaming-programming-guide.md
@@ -2176,6 +2176,8 @@ the input data stream (using
`inputStream.repartition()`).
This distributes the received batches of data across the specified number of
machines in the cluster
before further processing.
+For direct stream, please refer to [Spark Streaming + Kafka Integration
Guide](streaming-kafka-integration.html)
+
### Level of Parallelism in Data Processing
{:.no_toc}
Cluster resources can be under-utilized if the number of parallel tasks used
in any stage of the
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24715][BUILD] Override jline version as 2.14.3 in SBT
Repository: spark
Updated Branches:
refs/heads/master 8f91c697e -> 8008f9cb8
[SPARK-24715][BUILD] Override jline version as 2.14.3 in SBT
## What changes were proposed in this pull request?
During SPARK-24418 (Upgrade Scala to 2.11.12 and 2.12.6), we upgrade `jline`
version together. So, `mvn` works correctly. However, `sbt` brings old jline
library and is hitting `NoSuchMethodError` in `master` branch, see
https://github.com/apache/spark/pull/21495#issuecomment-401560826. This
overrides jline version in SBT to make sbt build work.
## How was this patch tested?
Manually test.
Author: Liang-Chi Hsieh
Closes #21692 from viirya/SPARK-24715.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8008f9cb
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8008f9cb
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8008f9cb
Branch: refs/heads/master
Commit: 8008f9cb82e7c228b94eade2e7cb484d6d17e6a4
Parents: 8f91c69
Author: Liang-Chi Hsieh
Authored: Mon Jul 2 22:09:47 2018 +0800
Committer: hyukjinkwon
Committed: Mon Jul 2 22:09:47 2018 +0800
--
project/SparkBuild.scala | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/8008f9cb/project/SparkBuild.scala
--
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index b606f93..f887e45 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -464,7 +464,8 @@ object DockerIntegrationTests {
*/
object DependencyOverrides {
lazy val settings = Seq(
-dependencyOverrides += "com.google.guava" % "guava" % "14.0.1")
+dependencyOverrides += "com.google.guava" % "guava" % "14.0.1",
+dependencyOverrides += "jline" % "jline" % "2.14.3")
}
/**
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
svn commit: r27854 - /dev/spark/KEYS
Author: jshao Date: Mon Jul 2 12:18:41 2018 New Revision: 27854 Log: Update KEYS Modified: dev/spark/KEYS Modified: dev/spark/KEYS == --- dev/spark/KEYS (original) +++ dev/spark/KEYS Mon Jul 2 12:18:41 2018 @@ -646,3 +646,60 @@ UiZuJIMHLPJK4sbOj5nMZE2163zUXz+gOVMeLqVx qG1EJoF+tteqemi1ZwYipD06wA== =cA5d -END PGP PUBLIC KEY BLOCK- +pub 4096R/12973FD0 2017-08-07 +uid Saisai Shao (CODE SIGNING KEY) +sig 312973FD0 2017-08-07 Saisai Shao (CODE SIGNING KEY) +sub 4096R/A0A12D58 2017-08-07 +sig 12973FD0 2017-08-07 Saisai Shao (CODE SIGNING KEY) + +-BEGIN PGP PUBLIC KEY BLOCK- + +mQINBFmH6vwBEADMnQZREQFXHjAcJxKh9o2rcgDJRQAZkE5kPVTFtkYskaNSDGP2 +A1TdKvOHRG79/9hLz3Ijvh6whwQ5a5JUiiCT06xeKLmJOQDOwYhEnUnTjxkNzAj0 +47Btno7vD1GKY7Yd67QortEXQxiWMoxDDYEDiHepuAZ+YgPOK0j0FBHyTHtikWLW +dDEJGWl5KGM3y1xxASGEUCaYpM/cpN+fP3rrvKoQ6H4/HlgSdGwGivnIhZSaZ9rM +/Uj5aWnBxhTOm4mkZDL26mtXH8UQsRA72kz1SC0wTQLl097VNmh626Hmhk7eUz/d +bW3fe2fL2ouXbyToOu8UV9+KPXJmZg6bLxejRo9CKLAtOMd+cxymhJYfzSUNh6vJ +VfStLYqySy8Oq6swe8M6W/2Asw6nsUdbItyReDvnGyEmxIuoiRoqtIo5hY5GT6WI +wwLn7wPIkozIyEd9GDbYGzDclff3i3lOXEfZ6lOwtYNSOZlPYW2/ZyxAV+sa/itd +fZJ8x1TH3jnNg/yELAVTaLKZ+XbDut3NDSjh8zzgwr8EYKr8UaZhzULfwDoGL6AJ +88aJqEUHKD0z/wgvow1Ts6mW5l2ego3235oeNMd/hJXbbGFn7EFNRxqF0xFcRwP0 +kfUC6twcENJ+jYD/bUQ0lt4PLHX6ziShAJynTdu1OabHSEsFa19XUpHWZwARAQAB +tDFTYWlzYWkgU2hhbyAoQ09ERSBTSUdOSU5HIEtFWSkgPGpzaGFvQGFwYWNoZS5v +cmc+iQI3BBMBCgAhBQJZh+r8AhsDBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJ +ENsLIaASlz/Q9uYP+wQ9jn5OqMHSv38HSWxBfuLbI1Jb90Ayx9Lg5KqEazBy5+Xc +2o5hKV7t5LrRY0ZW5P/mLibJC8tdakZCjMoSFbefVFUQzMw5ACHWXL1wYNcrlIcc +EA2muMRS8v6o1TYfLTMgP6QanAGseY7DFrk3lc38IA5KBJ5g7Q80I+pZswQOdiM6 +vsWskkRkdaoS5Ku1i8DENJgx9bJUrreGtrzRdVX2JfqRuufozTH5OHOFc2WHwosR ++scfmaAmScRxt1Xi6Vrm9NhoynWZro4XfhNzfk81CWMy2DT90CMTuK/DMCE3TH9n +Y0HG6gPL1gDJiGhhnhWK1k2RpCts0fOt81zNikdty5nhHnjCojaDdK79kYgPsWwT +0mBAMArKbHD650dpwS4wV/kqc0CFolL8Haj5f2caNuqAH5HH3yp2Qgdd5bHbUS/0 +CdTskLdCDPcJpsGGXjnLdBA6XOppjvaZtuaMS5SXJZ/mt6YwKzgasI6SrVkI4oEd +/o+YHm8HP0Xm/S5mURSflJGDRyFrkuHoKhID4LjLKOeTlP4NRLCy8TAYvVJ91BNL +u9fyvez9fNzdlWLRoBnt3grQRmf9dWxm6f1ZQVpVTYMls+ACwiHFzneNfli2LjGK +clsG2VyGRVaPykhvSCFuhQpaSpaSKiZK4Ux9AjsJnOVT9heUBJBDSuTOn5GRuQIN +BFmH6vwBEAC7jbxZY0TqNFNzxpx3BucUXOOe2Mhi52/6uPvXetFguRj12NkC5/vx +s2EpjyZR37YG9WXoHd9UZ0PI1FbeH0xBPF6yy7zNALxiSOpEciFumQMetzlWFHzC +yCUuo1vt9/Mmy0meSxnwSPq5juSBYGll2r7FZ8dpmJNUWE8njJttYDhse/b9WfGF +BOFckR1k3xnMNjTm8JXyVQC48HvZVUZLWWQnIp4lUD0UwXqjYT9LljNG3kRbP4H2 +SbW7ijL0ioeDUeGM2pavaW5C8nEONgEwcS4yi1uCSVm8dzRe4ZY/jw4NehkV1SSB +l51NB74WoGJ0tcjHs2+2Gvzr2DKxEqJNePdx1jQncnSnTrvlKAAnR4HbS61d7K/T +eI3DdQ5YSmM7Dyi/Bg/FJ3pWLpNMGKB1vX7yK2gguj4MoMvm9mYZf9vR6tIO1ZBt +YUKZjn9s0Q6aLXTk173u1gqfpzYigc5F5c+eGhRgZlmve0N1zT3I68buIzF/VO1g +AvOCwxC7qSqu07M7AUvwJwTzBplZNNj7QvoKqqSY0pT7pNwTMH1/jWHOi1jb/Oj2 +Y6DxVr6mDjsZd4/2nPZ4Nv3D+KP8q41tX3tThQ+prDUVSFXWWDxgU27EJLS9FnkU +2UY+FuxW34svUEpGHrISUbxDIzP2HL4hm1mSIk3ua4RqGQYcFR3ElwARAQABiQIf +BBgBCgAJBQJZh+r8AhsMAAoJENsLIaASlz/QmXMP/jLtuJhbFgIpztMa+Hv6Isxv +QQ2/6JdKLUCFLHMHAPoh1R97WnEr2l3DJCkwGNW2al3S75Zgj1KC0H/Nm5gxrLYp +1ne3aY7U90RwcXu5oqABdGQvLZmmo7O3BNZDbTf22XvSndS0sjE/+lJXzAUfpx2E +qu/0URnHxawOI8MPitoeszGrN1zJ9uTFhVvhLUMJ+JxlBmPMr1rUhuUddaQdldhk +JopwxcTQN4S+tCbFVz/PNcJUTL7VRABh5wHY5mOmvjxnG/2MNN6zmSfX5pCl6pgf +SliQ40wV2Sf4L+r/M/DvzHpycuzzLkLmcO9y5+/AtM9JrPRxNg32ycX8GuXnhUTZ +FlxfLw2nGCMRb5a9u15VfGvh037WYt7CmDUGh64f41m/wRangrPdicz3sy9TEKjs +OMA1OAVFovv5Nd3QS4IrM2lpVJSaBRvcL67NqJ+fE9CAsBUHxAjU/5wBAuT8xxiF +5r27fHM4Fq/Hy3aT16PJblVMbmf2V4z3Q7Je2xP0IP1V249a4nyXKPH+6FksUiG3 +Yv4vngYdI7Kw5/5kogYIVT1Es4cRW/9NxiK4360uIsecILpaDPtf7+EQH7DU5f2l +iI08G08QlT8LxLCvMSn81txtKAdPssT+pnMpTQo2vvrssG5ArImZc5InZDy+EutS +kyHyHY5kPG9HfDOSahPz +=SDAz +-END PGP PUBLIC KEY BLOCK- - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
svn commit: r27847 - in /dev/spark/2.4.0-SNAPSHOT-2018_07_02_00_01-8f91c69-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Mon Jul 2 07:17:44 2018 New Revision: 27847 Log: Apache Spark 2.4.0-SNAPSHOT-2018_07_02_00_01-8f91c69 docs [This commit notification would consist of 1467 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
