[spark] branch master updated: [SPARK-27748][SS] Kafka consumer/producer password/token redaction.
This is an automated email from the ASF dual-hosted git repository.
vanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 911fadf [SPARK-27748][SS] Kafka consumer/producer password/token
redaction.
911fadf is described below
commit 911fadf33a0af60bc0bb50cf293958609b877f46
Author: Gabor Somogyi
AuthorDate: Mon Jun 3 15:43:08 2019 -0700
[SPARK-27748][SS] Kafka consumer/producer password/token redaction.
## What changes were proposed in this pull request?
Kafka parameters are logged at several places and the following parameters
has to be redacted:
* Delegation token
* `ssl.truststore.password`
* `ssl.keystore.password`
* `ssl.key.password`
This PR contains:
* Spark central redaction framework used to redact passwords
(`spark.redaction.regex`)
* Custom redaction added to handle `sasl.jaas.config` (delegation token)
* Redaction code added into consumer/producer code
* Test refactor
## How was this patch tested?
Existing + additional unit tests.
Closes #24627 from gaborgsomogyi/SPARK-27748.
Authored-by: Gabor Somogyi
Signed-off-by: Marcelo Vanzin
---
.../spark/sql/kafka010/CachedKafkaProducer.scala | 26 +++--
.../apache/spark/kafka010/KafkaConfigUpdater.scala | 12 ++-
.../apache/spark/kafka010/KafkaRedactionUtil.scala | 51 +
.../org/apache/spark/kafka010/KafkaTokenUtil.scala | 10 +-
.../spark/kafka010/KafkaConfigUpdaterSuite.scala | 13 +--
.../spark/kafka010/KafkaDelegationTokenTest.scala | 32 ++
.../spark/kafka010/KafkaRedactionUtilSuite.scala | 120 +
.../spark/kafka010/KafkaTokenUtilSuite.scala | 44 ++--
8 files changed, 248 insertions(+), 60 deletions(-)
diff --git
a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/CachedKafkaProducer.scala
b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/CachedKafkaProducer.scala
index ce22e3f..fc177cd 100644
---
a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/CachedKafkaProducer.scala
+++
b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/CachedKafkaProducer.scala
@@ -28,7 +28,7 @@ import scala.util.control.NonFatal
import org.apache.spark.SparkEnv
import org.apache.spark.internal.Logging
-import org.apache.spark.kafka010.KafkaConfigUpdater
+import org.apache.spark.kafka010.{KafkaConfigUpdater, KafkaRedactionUtil}
private[kafka010] object CachedKafkaProducer extends Logging {
@@ -42,8 +42,7 @@ private[kafka010] object CachedKafkaProducer extends Logging {
private val cacheLoader = new CacheLoader[Seq[(String, Object)], Producer] {
override def load(config: Seq[(String, Object)]): Producer = {
- val configMap = config.map(x => x._1 -> x._2).toMap.asJava
- createKafkaProducer(configMap)
+ createKafkaProducer(config)
}
}
@@ -52,8 +51,11 @@ private[kafka010] object CachedKafkaProducer extends Logging
{
notification: RemovalNotification[Seq[(String, Object)], Producer]):
Unit = {
val paramsSeq: Seq[(String, Object)] = notification.getKey
val producer: Producer = notification.getValue
- logDebug(
-s"Evicting kafka producer $producer params: $paramsSeq, due to
${notification.getCause}")
+ if (log.isDebugEnabled()) {
+val redactedParamsSeq = KafkaRedactionUtil.redactParams(paramsSeq)
+logDebug(s"Evicting kafka producer $producer params:
$redactedParamsSeq, " +
+ s"due to ${notification.getCause}")
+ }
close(paramsSeq, producer)
}
}
@@ -63,9 +65,12 @@ private[kafka010] object CachedKafkaProducer extends Logging
{
.removalListener(removalListener)
.build[Seq[(String, Object)], Producer](cacheLoader)
- private def createKafkaProducer(producerConfiguration: ju.Map[String,
Object]): Producer = {
-val kafkaProducer: Producer = new Producer(producerConfiguration)
-logDebug(s"Created a new instance of KafkaProducer for
$producerConfiguration.")
+ private def createKafkaProducer(paramsSeq: Seq[(String, Object)]): Producer
= {
+val kafkaProducer: Producer = new Producer(paramsSeq.toMap.asJava)
+if (log.isDebugEnabled()) {
+ val redactedParamsSeq = KafkaRedactionUtil.redactParams(paramsSeq)
+ logDebug(s"Created a new instance of KafkaProducer for
$redactedParamsSeq.")
+}
kafkaProducer
}
@@ -103,7 +108,10 @@ private[kafka010] object CachedKafkaProducer extends
Logging {
/** Auto close on cache evict */
private def close(paramsSeq: Seq[(String, Object)], producer: Producer):
Unit = {
try {
- logInfo(s"Closing the KafkaProducer with params:
${paramsSeq.mkString("\n")}.")
+ if (log.isInfoEnabled())
[jira] [Commented] (SPARK-27891) Long running spark jobs fail because of HDFS delegation token expires
[ https://issues.apache.org/jira/browse/SPARK-27891?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16852420#comment-16852420 ] Marcelo Vanzin commented on SPARK-27891: Ok, the updated logs show the issue. But they're from Spark 2.2.0, which is EOL. If you can provide logs from the lastest 2.3 or 2.4 releases, that would be more helpful (since there's been a few changes in this area). > Long running spark jobs fail because of HDFS delegation token expires > - > > Key: SPARK-27891 > URL: https://issues.apache.org/jira/browse/SPARK-27891 > Project: Spark > Issue Type: Bug > Components: Security >Affects Versions: 2.0.1, 2.1.0, 2.3.1, 2.4.1 >Reporter: hemshankar sahu >Priority: Major > Attachments: application_1559242207407_0001.log > > > When the spark job runs on a secured cluster for longer then time that is > mentioned in the dfs.namenode.delegation.token.renew-interval property of > hdfs-site.xml the spark job fails. ** > Following command was used to submit the spark job > bin/spark-submit --principal acekrbuser --keytab ~/keytabs/acekrbuser.keytab > --master yarn --deploy-mode cluster examples/src/main/python/wordcount.py > /tmp/ff1.txt > > Application Logs attached > -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Commented] (SPARK-27891) Long running spark jobs fail because of HDFS delegation token expires
[
https://issues.apache.org/jira/browse/SPARK-27891?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16852408#comment-16852408
]
Marcelo Vanzin commented on SPARK-27891:
{{container_e48_1559242207407_0001_02_01}} tells me that's the second
attempt. That makes your problem the same as SPARK-23361.
> Long running spark jobs fail because of HDFS delegation token expires
> -
>
> Key: SPARK-27891
> URL: https://issues.apache.org/jira/browse/SPARK-27891
> Project: Spark
> Issue Type: Bug
> Components: Security
>Affects Versions: 2.0.1, 2.1.0, 2.3.1, 2.4.1
>Reporter: hemshankar sahu
>Priority: Major
>
> When the spark job runs on a secured cluster for longer then time that is
> mentioned in the dfs.namenode.delegation.token.renew-interval property of
> hdfs-site.xml the spark job fails. **
> Following command was used to submit the spark job
> bin/spark-submit --principal acekrbuser --keytab ~/keytabs/acekrbuser.keytab
> --master yarn --deploy-mode cluster examples/src/main/python/wordcount.py
> /tmp/ff1.txt
>
> Application Logs pasted in Docs Text
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Resolved] (SPARK-27891) Long running spark jobs fail because of HDFS delegation token expires
[ https://issues.apache.org/jira/browse/SPARK-27891?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin resolved SPARK-27891. Resolution: Not A Problem You have to provide a keytab for Spark for this to work. That's explained in the docs: http://spark.apache.org/docs/latest/security.html#kerberos > Long running spark jobs fail because of HDFS delegation token expires > - > > Key: SPARK-27891 > URL: https://issues.apache.org/jira/browse/SPARK-27891 > Project: Spark > Issue Type: Bug > Components: Security >Affects Versions: 2.0.1, 2.1.0, 2.3.1, 2.4.1 >Reporter: hemshankar sahu >Priority: Major > > When the spark job runs on a secured cluster for longer then time that is > mentioned in the dfs.namenode.delegation.token.renew-interval property of > hdfs-site.xml the spark job fails. ** > > > Application Logs pasted in Docs Text > -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Resolved] (SPARK-27773) Add shuffle service metric for number of exceptions caught in ExternalShuffleBlockHandler
[
https://issues.apache.org/jira/browse/SPARK-27773?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Marcelo Vanzin resolved SPARK-27773.
Resolution: Fixed
Fix Version/s: 3.0.0
Issue resolved by pull request 24645
[https://github.com/apache/spark/pull/24645]
> Add shuffle service metric for number of exceptions caught in
> ExternalShuffleBlockHandler
> -
>
> Key: SPARK-27773
> URL: https://issues.apache.org/jira/browse/SPARK-27773
> Project: Spark
> Issue Type: Improvement
> Components: Shuffle
>Affects Versions: 2.4.3
>Reporter: Steven Rand
>Assignee: Steven Rand
>Priority: Minor
> Fix For: 3.0.0
>
>
> The health of the external shuffle service is currently difficult to monitor.
> At least for the YARN shuffle service, the only current indication of health
> is whether or not the shuffle service threads are running in the NodeManager.
> However, we've seen that clients can sometimes experience elevated failure
> rates on requests to the shuffle service even when those threads are running.
> It would be helpful to have some indication of how often requests to the
> shuffle service are failing, as this could be monitored, alerted on, etc.
> One suggestion (implemented in the PR I'll attach to this ticket) is to add a
> metric to {{ExternalShuffleBlockHandler.ShuffleMetrics}} which keeps track of
> how many times we caught an exception in the shuffle service's RPC handler. I
> think that this gives us the insight into request failure rates that we're
> currently missing, but obviously I'm open to alternatives as well if people
> have other ideas.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[spark] branch master updated: [SPARK-27773][SHUFFLE] add metrics for number of exceptions caught in ExternalShuffleBlockHandler
This is an automated email from the ASF dual-hosted git repository.
vanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 568512c [SPARK-27773][SHUFFLE] add metrics for number of exceptions
caught in ExternalShuffleBlockHandler
568512c is described below
commit 568512cc82562f9f7d7da0a47bc59fafb86f3ebc
Author: Steven Rand
AuthorDate: Thu May 30 13:57:15 2019 -0700
[SPARK-27773][SHUFFLE] add metrics for number of exceptions caught in
ExternalShuffleBlockHandler
## What changes were proposed in this pull request?
Add a metric for number of exceptions caught in the
`ExternalShuffleBlockHandler`, the idea being that spikes in this metric over
some time window (or more desirably, the lack thereof) can be used as an
indicator of the health of an external shuffle service. (Where "health" refers
to its ability to successfully respond to client requests.)
## How was this patch tested?
Deployed a build of this PR to a YARN cluster, and confirmed that the
NodeManagers' JMX metrics include `numCaughtExceptions`.
Closes #24645 from sjrand/SPARK-27773.
Authored-by: Steven Rand
Signed-off-by: Marcelo Vanzin
---
.../network/shuffle/ExternalShuffleBlockHandler.java| 8
.../spark/network/yarn/YarnShuffleServiceMetrics.java | 17 ++---
.../network/yarn/YarnShuffleServiceMetricsSuite.scala | 2 +-
3 files changed, 19 insertions(+), 8 deletions(-)
diff --git
a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockHandler.java
b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockHandler.java
index cb2d01d..9b7bf25 100644
---
a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockHandler.java
+++
b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockHandler.java
@@ -150,6 +150,11 @@ public class ExternalShuffleBlockHandler extends
RpcHandler {
}
}
+ @Override
+ public void exceptionCaught(Throwable cause, TransportClient client) {
+metrics.caughtExceptions.inc();
+ }
+
public MetricSet getAllMetrics() {
return metrics;
}
@@ -215,6 +220,8 @@ public class ExternalShuffleBlockHandler extends RpcHandler
{
private Counter activeConnections = new Counter();
// Number of registered connections to the shuffle service
private Counter registeredConnections = new Counter();
+// Number of exceptions caught in connections to the shuffle service
+private Counter caughtExceptions = new Counter();
public ShuffleMetrics() {
allMetrics = new HashMap<>();
@@ -225,6 +232,7 @@ public class ExternalShuffleBlockHandler extends RpcHandler
{
(Gauge) () ->
blockManager.getRegisteredExecutorsSize());
allMetrics.put("numActiveConnections", activeConnections);
allMetrics.put("numRegisteredConnections", registeredConnections);
+ allMetrics.put("numCaughtExceptions", caughtExceptions);
}
@Override
diff --git
a/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleServiceMetrics.java
b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleServiceMetrics.java
index 5012374..6d4e568 100644
---
a/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleServiceMetrics.java
+++
b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleServiceMetrics.java
@@ -96,13 +96,13 @@ class YarnShuffleServiceMetrics implements MetricsSource {
} else if (metric instanceof Gauge) {
final Object gaugeValue = ((Gauge) metric).getValue();
if (gaugeValue instanceof Integer) {
-metricsRecordBuilder.addGauge(getShuffleServiceMetricsInfo(name),
(Integer) gaugeValue);
+
metricsRecordBuilder.addGauge(getShuffleServiceMetricsInfoForGauge(name),
(Integer) gaugeValue);
} else if (gaugeValue instanceof Long) {
-metricsRecordBuilder.addGauge(getShuffleServiceMetricsInfo(name),
(Long) gaugeValue);
+
metricsRecordBuilder.addGauge(getShuffleServiceMetricsInfoForGauge(name),
(Long) gaugeValue);
} else if (gaugeValue instanceof Float) {
-metricsRecordBuilder.addGauge(getShuffleServiceMetricsInfo(name),
(Float) gaugeValue);
+
metricsRecordBuilder.addGauge(getShuffleServiceMetricsInfoForGauge(name),
(Float) gaugeValue);
} else if (gaugeValue instanceof Double) {
-metricsRecordBuilder.addGauge(getShuffleServiceMetricsInfo(name),
(Double) gaugeValue);
+
metricsRecordBuilder.addGauge(getShuffleServiceMetricsInfoForGauge(name),
(Double) gaugeValue);
} else {
throw new IllegalStateException(
"Not supported c
[jira] [Assigned] (SPARK-27773) Add shuffle service metric for number of exceptions caught in ExternalShuffleBlockHandler
[
https://issues.apache.org/jira/browse/SPARK-27773?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Marcelo Vanzin reassigned SPARK-27773:
--
Assignee: Steven Rand
> Add shuffle service metric for number of exceptions caught in
> ExternalShuffleBlockHandler
> -
>
> Key: SPARK-27773
> URL: https://issues.apache.org/jira/browse/SPARK-27773
> Project: Spark
> Issue Type: Improvement
> Components: Shuffle
>Affects Versions: 2.4.3
>Reporter: Steven Rand
>Assignee: Steven Rand
>Priority: Minor
>
> The health of the external shuffle service is currently difficult to monitor.
> At least for the YARN shuffle service, the only current indication of health
> is whether or not the shuffle service threads are running in the NodeManager.
> However, we've seen that clients can sometimes experience elevated failure
> rates on requests to the shuffle service even when those threads are running.
> It would be helpful to have some indication of how often requests to the
> shuffle service are failing, as this could be monitored, alerted on, etc.
> One suggestion (implemented in the PR I'll attach to this ticket) is to add a
> metric to {{ExternalShuffleBlockHandler.ShuffleMetrics}} which keeps track of
> how many times we caught an exception in the shuffle service's RPC handler. I
> think that this gives us the insight into request failure rates that we're
> currently missing, but obviously I'm open to alternatives as well if people
> have other ideas.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Resolved] (SPARK-27378) spark-submit requests GPUs in YARN mode
[ https://issues.apache.org/jira/browse/SPARK-27378?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin resolved SPARK-27378. Resolution: Fixed Fix Version/s: 3.0.0 Issue resolved by pull request 24634 [https://github.com/apache/spark/pull/24634] > spark-submit requests GPUs in YARN mode > --- > > Key: SPARK-27378 > URL: https://issues.apache.org/jira/browse/SPARK-27378 > Project: Spark > Issue Type: Sub-task > Components: Spark Submit, YARN >Affects Versions: 3.0.0 >Reporter: Xiangrui Meng >Assignee: Thomas Graves >Priority: Major > Fix For: 3.0.0 > > -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[spark] branch master updated (955eef9 -> 0ced4c0)
This is an automated email from the ASF dual-hosted git repository. vanzin pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/spark.git. from 955eef9 Revert "[SPARK-27831][SQL][TEST][test-hadoop3.2] Move Hive test jars to maven dependency" add 0ced4c0 [SPARK-27378][YARN] YARN support for GPU-aware scheduling No new revisions were added by this update. Summary of changes: .../org/apache/spark/deploy/yarn/Client.scala | 6 ++- .../spark/deploy/yarn/ResourceRequestHelper.scala | 12 - .../apache/spark/deploy/yarn/YarnAllocator.scala | 3 +- .../spark/deploy/yarn/YarnSparkHadoopUtil.scala| 22 +++- .../org/apache/spark/deploy/yarn/ClientSuite.scala | 59 +- .../deploy/yarn/ResourceRequestTestHelper.scala| 11 .../spark/deploy/yarn/YarnAllocatorSuite.scala | 25 - 7 files changed, 132 insertions(+), 6 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[jira] [Assigned] (SPARK-27378) spark-submit requests GPUs in YARN mode
[ https://issues.apache.org/jira/browse/SPARK-27378?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin reassigned SPARK-27378: -- Assignee: Thomas Graves > spark-submit requests GPUs in YARN mode > --- > > Key: SPARK-27378 > URL: https://issues.apache.org/jira/browse/SPARK-27378 > Project: Spark > Issue Type: Sub-task > Components: Spark Submit, YARN >Affects Versions: 3.0.0 >Reporter: Xiangrui Meng >Assignee: Thomas Graves >Priority: Major > -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Created] (SPARK-27868) Better document shuffle / RPC listen backlog
Marcelo Vanzin created SPARK-27868:
--
Summary: Better document shuffle / RPC listen backlog
Key: SPARK-27868
URL: https://issues.apache.org/jira/browse/SPARK-27868
Project: Spark
Issue Type: Improvement
Components: Documentation, Spark Core
Affects Versions: 2.4.3
Reporter: Marcelo Vanzin
The option to control the listen socket backlog for RPC and shuffle servers is
not documented in our public docs.
The only piece of documentation is in a Java class, and even that documentation
is incorrect:
{code}
/** Requested maximum length of the queue of incoming connections. Default -1
for no backlog. */
public int backLog() { return conf.getInt(SPARK_NETWORK_IO_BACKLOG_KEY, -1); }
{code}
The default value actual causes the default value from the JRE to be used,
which is 50 according to the docs.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Commented] (SPARK-24149) Automatic namespaces discovery in HDFS federation
[
https://issues.apache.org/jira/browse/SPARK-24149?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16847926#comment-16847926
]
Marcelo Vanzin commented on SPARK-24149:
bq. If they are unrelated the user explicitly provides them.
So you basically have the opposite view of Marco's. He thinks that it's better
for users to get tokens automatically if the NN is in the configuration. You
could consider that if the user provides a configuration that references some
NN, then that's explicit enough, and Spark should get tokens for it.
I don't really care either way.
> Automatic namespaces discovery in HDFS federation
> -
>
> Key: SPARK-24149
> URL: https://issues.apache.org/jira/browse/SPARK-24149
> Project: Spark
> Issue Type: Improvement
> Components: YARN
>Affects Versions: 2.4.0
>Reporter: Marco Gaido
>Assignee: Marco Gaido
>Priority: Minor
> Fix For: 2.4.0
>
>
> Hadoop 3 introduced HDFS federation.
> Spark fails to write on different namespaces when Hadoop federation is turned
> on and the cluster is secure. This happens because Spark looks for the
> delegation token only for the defaultFS configured and not for all the
> available namespaces. A workaround is the usage of the property
> {{spark.yarn.access.hadoopFileSystems}}.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Commented] (SPARK-24149) Automatic namespaces discovery in HDFS federation
[
https://issues.apache.org/jira/browse/SPARK-24149?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16847888#comment-16847888
]
Marcelo Vanzin commented on SPARK-24149:
bq. I think we are duplicating the logic here
Do you have any suggestion for how not to duplicate the logic? Because as far
as I remember there's no HDFS API call for "get tokens for all configured name
nodes in my configuration".
> Automatic namespaces discovery in HDFS federation
> -
>
> Key: SPARK-24149
> URL: https://issues.apache.org/jira/browse/SPARK-24149
> Project: Spark
> Issue Type: Improvement
> Components: YARN
>Affects Versions: 2.4.0
>Reporter: Marco Gaido
>Assignee: Marco Gaido
>Priority: Minor
> Fix For: 2.4.0
>
>
> Hadoop 3 introduced HDFS federation.
> Spark fails to write on different namespaces when Hadoop federation is turned
> on and the cluster is secure. This happens because Spark looks for the
> delegation token only for the defaultFS configured and not for all the
> available namespaces. A workaround is the usage of the property
> {{spark.yarn.access.hadoopFileSystems}}.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[spark] branch master updated (de13f70 -> 1e87694)
This is an automated email from the ASF dual-hosted git repository. vanzin pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/spark.git. from de13f70 [SPARK-27824][SQL] Make rule EliminateResolvedHint idempotent add 1e87694 [SPARK-27677][CORE] Disable by default fetching of disk persisted RDD blocks via external shuffle service No new revisions were added by this update. Summary of changes: .../org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java | 2 +- .../apache/spark/network/shuffle/ExternalShuffleIntegrationSuite.java | 1 + core/src/main/scala/org/apache/spark/internal/config/package.scala | 2 +- core/src/test/scala/org/apache/spark/ExternalShuffleServiceSuite.scala | 3 ++- 4 files changed, 5 insertions(+), 3 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[jira] [Assigned] (SPARK-27677) Disk-persisted RDD blocks served by shuffle service, and ignored for Dynamic Allocation
[ https://issues.apache.org/jira/browse/SPARK-27677?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin reassigned SPARK-27677: -- Assignee: Attila Zsolt Piros > Disk-persisted RDD blocks served by shuffle service, and ignored for Dynamic > Allocation > --- > > Key: SPARK-27677 > URL: https://issues.apache.org/jira/browse/SPARK-27677 > Project: Spark > Issue Type: New Feature > Components: Block Manager, Spark Core >Affects Versions: 2.4.3 >Reporter: Imran Rashid >Assignee: Attila Zsolt Piros >Priority: Major > Fix For: 3.0.0 > > > Disk-cached RDD blocks are currently unavailable after an executor is > removed. However, when there is an external shuffle service, the data > remains available on disk and could be served by the shuffle service. This > would allow dynamic allocation to reclaim executors with only disk-cached > blocks more rapidly, but still keep the cached data available. -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Resolved] (SPARK-27677) Disk-persisted RDD blocks served by shuffle service, and ignored for Dynamic Allocation
[ https://issues.apache.org/jira/browse/SPARK-27677?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin resolved SPARK-27677. Resolution: Fixed Fix Version/s: 3.0.0 Issue resolved by pull request 24499 [https://github.com/apache/spark/pull/24499] > Disk-persisted RDD blocks served by shuffle service, and ignored for Dynamic > Allocation > --- > > Key: SPARK-27677 > URL: https://issues.apache.org/jira/browse/SPARK-27677 > Project: Spark > Issue Type: New Feature > Components: Block Manager, Spark Core >Affects Versions: 2.4.3 >Reporter: Imran Rashid >Priority: Major > Fix For: 3.0.0 > > > Disk-cached RDD blocks are currently unavailable after an executor is > removed. However, when there is an external shuffle service, the data > remains available on disk and could be served by the shuffle service. This > would allow dynamic allocation to reclaim executors with only disk-cached > blocks more rapidly, but still keep the cached data available. -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[spark] branch master updated (1a68fc3 -> e9f3f62)
This is an automated email from the ASF dual-hosted git repository.
vanzin pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git.
from 1a68fc3 [SPARK-27816][SQL] make TreeNode tag type safe
add e9f3f62 [SPARK-27677][CORE] Serve local disk persisted blocks by the
external service after releasing executor by dynamic allocation
No new revisions were added by this update.
Summary of changes:
.../network/server/ChunkFetchRequestHandler.java | 3 +
.../network/server/OneForOneStreamManager.java | 5 +-
.../network/ChunkFetchRequestHandlerSuite.java | 42 ++--
.../network/TransportRequestHandlerSuite.java | 47 ++--
.../server/OneForOneStreamManagerSuite.java| 56 -
.../apache/spark/network/shuffle/Constants.java| 14 +-
.../shuffle/ExternalShuffleBlockHandler.java | 77 +--
.../shuffle/ExternalShuffleBlockResolver.java | 79 +--
.../network/shuffle/ExternalShuffleClient.java | 42 +++-
.../spark/network/shuffle/ShuffleClient.java | 6 -
.../shuffle/protocol/BlockTransferMessage.java | 5 +-
.../{StreamHandle.java => BlocksRemoved.java} | 41 ++--
.../{OpenBlocks.java => RemoveBlocks.java} | 21 +-
.../CleanupNonShuffleServiceServedFilesSuite.java | 256 +
.../shuffle/ExternalShuffleBlockHandlerSuite.java | 63 +++--
.../shuffle/ExternalShuffleIntegrationSuite.java | 96 +++-
.../shuffle/NonShuffleFilesCleanupSuite.java | 221 --
.../network/shuffle/TestShuffleDataContext.java| 34 ++-
.../apache/spark/ExecutorAllocationManager.scala | 12 +-
.../src/main/scala/org/apache/spark/SparkEnv.scala | 37 ++-
.../org/apache/spark/deploy/worker/Worker.scala| 7 +-
.../org/apache/spark/internal/config/package.scala | 13 +-
.../org/apache/spark/storage/BlockManager.scala| 65 +++---
.../apache/spark/storage/BlockManagerMaster.scala | 9 +-
.../spark/storage/BlockManagerMasterEndpoint.scala | 159 ++---
.../spark/storage/BlockManagerMessages.scala | 3 +-
.../org/apache/spark/storage/StorageUtils.scala| 19 +-
.../apache/spark/ExternalShuffleServiceSuite.scala | 43 +++-
.../spark/storage/BlockManagerInfoSuite.scala | 160 +
.../storage/BlockManagerReplicationSuite.scala | 6 +-
.../apache/spark/storage/BlockManagerSuite.scala | 73 +-
.../streaming/ReceivedBlockHandlerSuite.scala | 4 +-
32 files changed, 1237 insertions(+), 481 deletions(-)
copy mllib-local/src/main/scala/org/apache/spark/ml/impl/Utils.scala =>
common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/Constants.java
(81%)
copy
common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/{StreamHandle.java
=> BlocksRemoved.java} (57%)
copy
common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/{OpenBlocks.java
=> RemoveBlocks.java} (84%)
create mode 100644
common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/CleanupNonShuffleServiceServedFilesSuite.java
delete mode 100644
common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/NonShuffleFilesCleanupSuite.java
create mode 100644
core/src/test/scala/org/apache/spark/storage/BlockManagerInfoSuite.scala
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
[jira] [Resolved] (SPARK-27804) LiveListenerBus#addToQueue : create multiple AsyncEventQueues under race condition
[
https://issues.apache.org/jira/browse/SPARK-27804?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Marcelo Vanzin resolved SPARK-27804.
Resolution: Not A Problem
> LiveListenerBus#addToQueue : create multiple AsyncEventQueues under race
> condition
> --
>
> Key: SPARK-27804
> URL: https://issues.apache.org/jira/browse/SPARK-27804
> Project: Spark
> Issue Type: Bug
> Components: Spark Core
>Affects Versions: 2.4.3
>Reporter: duyu
>Priority: Major
>
> LiveListenerBus.scala
> {code}
> private[spark] def addToQueue(
> listener: SparkListenerInterface,
> queue: String): Unit = synchronized {
> if (stopped.get()) {
> throw new IllegalStateException("LiveListenerBus is stopped.")
> }
> queues.asScala.find(_.name == queue) match {
> case Some(queue) =>
> queue.addListener(listener)
> case None =>
> // it will create multiple AsyncEventQueues with the same name when
> run in multi-thread scene and those created AsyncEventQueues will be added to
> queues
> val newQueue = new AsyncEventQueue(queue, conf, metrics, this)
> newQueue.addListener(listener)
> if (started.get()) {
> newQueue.start(sparkContext)
> }
> queues.add(newQueue)
> }
> }
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Commented] (SPARK-27797) Shuffle service metric "registeredConnections" not tracked correctly
[
https://issues.apache.org/jira/browse/SPARK-27797?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16846068#comment-16846068
]
Marcelo Vanzin commented on SPARK-27797:
You're right. Looks just like some dead code in that case.
> Shuffle service metric "registeredConnections" not tracked correctly
>
>
> Key: SPARK-27797
> URL: https://issues.apache.org/jira/browse/SPARK-27797
> Project: Spark
> Issue Type: Bug
> Components: Shuffle
>Affects Versions: 3.0.0
>Reporter: Marcelo Vanzin
>Priority: Minor
>
> In {{ExternalShuffleBlockHandler}}:
> {code}
> // Number of registered connections to the shuffle service
> private Counter registeredConnections = new Counter();
> public ShuffleMetrics() {
> ...
> allMetrics.put("numRegisteredConnections", registeredConnections);
> }
> {code}
> But the counter that's actually updated is in {{TransportContext}}. The call
> site is in {{TransportChannelHandler}}:
> {code}
> @Override
> public void channelRegistered(ChannelHandlerContext ctx) throws Exception {
> transportContext.getRegisteredConnections().inc();
> super.channelRegistered(ctx);
> }
> @Override
> public void channelUnregistered(ChannelHandlerContext ctx) throws Exception
> {
> transportContext.getRegisteredConnections().dec();
> super.channelUnregistered(ctx);
> }
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Created] (SPARK-27797) Shuffle service metric "registeredConnections" not tracked correctly
Marcelo Vanzin created SPARK-27797:
--
Summary: Shuffle service metric "registeredConnections" not
tracked correctly
Key: SPARK-27797
URL: https://issues.apache.org/jira/browse/SPARK-27797
Project: Spark
Issue Type: Bug
Components: Shuffle
Affects Versions: 3.0.0
Reporter: Marcelo Vanzin
In {{ExternalShuffleBlockHandler}}:
{code}
// Number of registered connections to the shuffle service
private Counter registeredConnections = new Counter();
public ShuffleMetrics() {
...
allMetrics.put("numRegisteredConnections", registeredConnections);
}
{code}
But the counter that's actually updated is in {{TransportContext}}. The call
site is in {{TransportChannelHandler}}:
{code}
@Override
public void channelRegistered(ChannelHandlerContext ctx) throws Exception {
transportContext.getRegisteredConnections().inc();
super.channelRegistered(ctx);
}
@Override
public void channelUnregistered(ChannelHandlerContext ctx) throws Exception {
transportContext.getRegisteredConnections().dec();
super.channelUnregistered(ctx);
}
{code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Updated] (SPARK-27726) Performance of InMemoryStore suffers under load
[ https://issues.apache.org/jira/browse/SPARK-27726?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin updated SPARK-27726: --- Fix Version/s: 2.4.4 > Performance of InMemoryStore suffers under load > --- > > Key: SPARK-27726 > URL: https://issues.apache.org/jira/browse/SPARK-27726 > Project: Spark > Issue Type: Umbrella > Components: Spark Core >Affects Versions: 2.3.3 >Reporter: David C Navas >Assignee: David C Navas >Priority: Major > Fix For: 2.4.4, 3.0.0 > > Attachments: GCRateIssues.pdf, PerformanceBeforeAndAfter.pdf > > > When our Spark system has been under load for an extended period of time, GC > remains highly active and the jobs page becomes unresponsive even when load > eases. Please see the attached GCRateIssues for more details regarding the > problem definition. > We found a number of separate issues which are detailed in the subtasks. I > anticipate committing a single PR for all subtasks whose commits roughly > align with the descriptions in the subtasks. > The performance of the code is measured before and after the change and is > attached in the document PerformanceBeforeAndAfter. tl;dr in our use case, > we saw about five (computed) orders of magnitude improvement. -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[spark] branch branch-2.4 updated: [SPARK-27726][CORE] Fix performance of ElementTrackingStore deletes when using InMemoryStore under high loads
This is an automated email from the ASF dual-hosted git repository.
vanzin pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 1e2b60f [SPARK-27726][CORE] Fix performance of ElementTrackingStore
deletes when using InMemoryStore under high loads
1e2b60f is described below
commit 1e2b60f6ae77497ac06f794c2e946e56e0a3ed1b
Author: David Navas
AuthorDate: Tue May 21 10:21:46 2019 -0700
[SPARK-27726][CORE] Fix performance of ElementTrackingStore deletes when
using InMemoryStore under high loads
The details of the PR are explored in-depth in the sub-tasks of the
umbrella jira SPARK-27726.
Briefly:
1. Stop issuing asynchronous requests to cleanup elements in the tracking
store when a request is already pending
2. Fix a couple of thread-safety issues (mutable state and mis-ordered
updates)
3. Move Summary deletion outside of Stage deletion loop like Tasks
already are
4. Reimplement multi-delete in a removeAllKeys call which allows
InMemoryStore to implement it in a performant manner.
5. Some generic typing and exception handling cleanup
We see about five orders of magnitude improvement in the deletion code,
which for us is the difference between a server that needs restarting daily,
and one that is stable over weeks.
Unit tests for the fire-once asynchronous code and the removeAll calls in
both LevelDB and InMemoryStore are supplied. It was noted that the testing
code for the LevelDB and InMemoryStore is highly repetitive, and should
probably be merged, but we did not attempt that in this PR.
A version of this code was run in our production 2.3.3 and we were able to
sustain higher throughput without going into GC overload (which was happening
on a daily basis some weeks ago).
A version of this code was also put under a purpose-built Performance Suite
of tests to verify performance under both types of Store implementations for
both before and after code streams and for both total and partial delete cases
(this code is not included in this PR).
Closes #24616 from davidnavas/PentaBugFix.
Authored-by: David Navas
Signed-off-by: Marcelo Vanzin
(cherry picked from commit 9e73be38a53214780512d0cafedfae9d472cdd05)
Signed-off-by: Marcelo Vanzin
---
.../apache/spark/util/kvstore/InMemoryStore.java | 246 ++---
.../org/apache/spark/util/kvstore/KVStore.java | 6 +
.../org/apache/spark/util/kvstore/KVStoreView.java | 6 -
.../org/apache/spark/util/kvstore/KVTypeInfo.java | 17 +-
.../org/apache/spark/util/kvstore/LevelDB.java | 29 ++-
.../apache/spark/util/kvstore/LevelDBIterator.java | 68 +++---
.../spark/util/kvstore/InMemoryStoreSuite.java | 46
.../apache/spark/util/kvstore/LevelDBSuite.java| 43
.../apache/spark/status/AppStatusListener.scala| 30 +--
.../apache/spark/status/ElementTrackingStore.scala | 68 +-
.../spark/status/ElementTrackingStoreSuite.scala | 49 +++-
11 files changed, 450 insertions(+), 158 deletions(-)
diff --git
a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
index 5ca4371..6af45ae 100644
---
a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
+++
b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
@@ -21,16 +21,18 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
+import java.util.HashSet;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.function.BiConsumer;
+import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
-import com.google.common.base.Throwables;
import org.apache.spark.annotation.Private;
@@ -43,7 +45,7 @@ import org.apache.spark.annotation.Private;
public class InMemoryStore implements KVStore {
private Object metadata;
- private ConcurrentMap, InstanceList> data = new
ConcurrentHashMap<>();
+ private InMemoryLists inMemoryLists = new InMemoryLists();
@Override
public T getMetadata(Class klass) {
@@ -57,13 +59,13 @@ public class InMemoryStore implements KVStore {
@Override
public long count(Class type) {
-InstanceList list = data.get(type);
+InstanceList list = inMemoryLists.get(type);
return list != null ? list.size() : 0;
}
@Override
public long count(Class type, String index, Object indexedValue) throws
Exception {
-InstanceList list = data.get(type);
+InstanceList list = inMemory
[jira] [Commented] (SPARK-27726) Performance of InMemoryStore suffers under load
[ https://issues.apache.org/jira/browse/SPARK-27726?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16845061#comment-16845061 ] Marcelo Vanzin commented on SPARK-27726: [~davidnavas] all of the sub-tasks were handled by the same PR, right? If so could you close them out as dupes of this one? > Performance of InMemoryStore suffers under load > --- > > Key: SPARK-27726 > URL: https://issues.apache.org/jira/browse/SPARK-27726 > Project: Spark > Issue Type: Umbrella > Components: Spark Core >Affects Versions: 2.3.3 >Reporter: David C Navas >Assignee: David C Navas >Priority: Major > Fix For: 3.0.0 > > Attachments: GCRateIssues.pdf, PerformanceBeforeAndAfter.pdf > > > When our Spark system has been under load for an extended period of time, GC > remains highly active and the jobs page becomes unresponsive even when load > eases. Please see the attached GCRateIssues for more details regarding the > problem definition. > We found a number of separate issues which are detailed in the subtasks. I > anticipate committing a single PR for all subtasks whose commits roughly > align with the descriptions in the subtasks. > The performance of the code is measured before and after the change and is > attached in the document PerformanceBeforeAndAfter. tl;dr in our use case, > we saw about five (computed) orders of magnitude improvement. -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Assigned] (SPARK-27726) Performance of InMemoryStore suffers under load
[ https://issues.apache.org/jira/browse/SPARK-27726?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin reassigned SPARK-27726: -- Assignee: David C Navas > Performance of InMemoryStore suffers under load > --- > > Key: SPARK-27726 > URL: https://issues.apache.org/jira/browse/SPARK-27726 > Project: Spark > Issue Type: Umbrella > Components: Spark Core >Affects Versions: 2.3.3 >Reporter: David C Navas >Assignee: David C Navas >Priority: Major > Fix For: 3.0.0 > > Attachments: GCRateIssues.pdf, PerformanceBeforeAndAfter.pdf > > > When our Spark system has been under load for an extended period of time, GC > remains highly active and the jobs page becomes unresponsive even when load > eases. Please see the attached GCRateIssues for more details regarding the > problem definition. > We found a number of separate issues which are detailed in the subtasks. I > anticipate committing a single PR for all subtasks whose commits roughly > align with the descriptions in the subtasks. > The performance of the code is measured before and after the change and is > attached in the document PerformanceBeforeAndAfter. tl;dr in our use case, > we saw about five (computed) orders of magnitude improvement. -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Resolved] (SPARK-27726) Performance of InMemoryStore suffers under load
[ https://issues.apache.org/jira/browse/SPARK-27726?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin resolved SPARK-27726. Resolution: Fixed Fix Version/s: 3.0.0 Issue resolved by pull request 24616 [https://github.com/apache/spark/pull/24616] > Performance of InMemoryStore suffers under load > --- > > Key: SPARK-27726 > URL: https://issues.apache.org/jira/browse/SPARK-27726 > Project: Spark > Issue Type: Umbrella > Components: Spark Core >Affects Versions: 2.3.3 >Reporter: David C Navas >Priority: Major > Fix For: 3.0.0 > > Attachments: GCRateIssues.pdf, PerformanceBeforeAndAfter.pdf > > > When our Spark system has been under load for an extended period of time, GC > remains highly active and the jobs page becomes unresponsive even when load > eases. Please see the attached GCRateIssues for more details regarding the > problem definition. > We found a number of separate issues which are detailed in the subtasks. I > anticipate committing a single PR for all subtasks whose commits roughly > align with the descriptions in the subtasks. > The performance of the code is measured before and after the change and is > attached in the document PerformanceBeforeAndAfter. tl;dr in our use case, > we saw about five (computed) orders of magnitude improvement. -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[spark] branch master updated (1e0facb -> 9e73be3)
This is an automated email from the ASF dual-hosted git repository. vanzin pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/spark.git. from 1e0facb [SQL][DOC][MINOR] update documents for Table and WriteBuilder add 9e73be3 [SPARK-27726][CORE] Fix performance of ElementTrackingStore deletes when using InMemoryStore under high loads No new revisions were added by this update. Summary of changes: .../apache/spark/util/kvstore/InMemoryStore.java | 246 ++--- .../org/apache/spark/util/kvstore/KVStore.java | 6 + .../org/apache/spark/util/kvstore/KVStoreView.java | 6 - .../org/apache/spark/util/kvstore/KVTypeInfo.java | 17 +- .../org/apache/spark/util/kvstore/LevelDB.java | 29 ++- .../apache/spark/util/kvstore/LevelDBIterator.java | 68 +++--- .../spark/util/kvstore/InMemoryStoreSuite.java | 46 .../apache/spark/util/kvstore/LevelDBSuite.java| 43 .../apache/spark/status/AppStatusListener.scala| 30 +-- .../apache/spark/status/ElementTrackingStore.scala | 68 +- .../spark/status/ElementTrackingStoreSuite.scala | 49 +++- 11 files changed, 450 insertions(+), 158 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[jira] [Resolved] (SPARK-27745) build/mvn take wrong scala version when compile for scala 2.12
[
https://issues.apache.org/jira/browse/SPARK-27745?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Marcelo Vanzin resolved SPARK-27745.
Resolution: Not A Bug
You need to run {{./dev/change-scala-version.sh}} first. Pretty sure this is in
the documentation.
> build/mvn take wrong scala version when compile for scala 2.12
> --
>
> Key: SPARK-27745
> URL: https://issues.apache.org/jira/browse/SPARK-27745
> Project: Spark
> Issue Type: Bug
> Components: Build
>Affects Versions: 2.4.3
>Reporter: Izek Greenfield
>Priority: Major
>
> in `build/mvn`
> line:
> local scala_binary_version=`grep "scala.binary.version" "${_DIR}/../pom.xml"
> | head -n1 | awk -F '[<>]' '{print $3}'`
> it grep the pom and there will be 2.11 and if I set -Pscala-2.12 it will take
> 2.11
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[spark] branch master updated (bfb3ffe -> d14e2d7)
This is an automated email from the ASF dual-hosted git repository. vanzin pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/spark.git. from bfb3ffe [SPARK-27682][CORE][GRAPHX][MLLIB] Replace use of collections and methods that will be removed in Scala 2.13 with work-alikes add d14e2d7 [SPARK-27678][UI] Allow user impersonation in the UI. No new revisions were added by this update. Summary of changes: .../scala/org/apache/spark/SecurityManager.scala | 42 +++-- .../org/apache/spark/ui/HttpSecurityFilter.scala | 26 +-- .../org/apache/spark/SecurityManagerSuite.scala| 8 +++- .../apache/spark/ui/HttpSecurityFilterSuite.scala | 53 -- 4 files changed, 96 insertions(+), 33 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[jira] [Assigned] (SPARK-27678) Support Knox user impersonation in UI
[ https://issues.apache.org/jira/browse/SPARK-27678?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin reassigned SPARK-27678: -- Assignee: Marcelo Vanzin > Support Knox user impersonation in UI > - > > Key: SPARK-27678 > URL: https://issues.apache.org/jira/browse/SPARK-27678 > Project: Spark > Issue Type: New Feature > Components: Web UI >Affects Versions: 3.0.0 > Reporter: Marcelo Vanzin > Assignee: Marcelo Vanzin >Priority: Minor > > When Spark is running behind a Knox proxy server, it would be useful to > support impersonation, so that Knox can properly identify the user who's > actually making the request. > This way, Knox authenticates to Spark, but Spark makes access control checks > against the user that Knox identifies as the actual requesting user, so that > proper access checks are performed, instead of performing the checks against > the more privileged Knox user. -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Resolved] (SPARK-27678) Support Knox user impersonation in UI
[ https://issues.apache.org/jira/browse/SPARK-27678?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin resolved SPARK-27678. Resolution: Fixed Fix Version/s: 3.0.0 Issue resolved by pull request 24582 [https://github.com/apache/spark/pull/24582] > Support Knox user impersonation in UI > - > > Key: SPARK-27678 > URL: https://issues.apache.org/jira/browse/SPARK-27678 > Project: Spark > Issue Type: New Feature > Components: Web UI >Affects Versions: 3.0.0 > Reporter: Marcelo Vanzin > Assignee: Marcelo Vanzin >Priority: Minor > Fix For: 3.0.0 > > > When Spark is running behind a Knox proxy server, it would be useful to > support impersonation, so that Knox can properly identify the user who's > actually making the request. > This way, Knox authenticates to Spark, but Spark makes access control checks > against the user that Knox identifies as the actual requesting user, so that > proper access checks are performed, instead of performing the checks against > the more privileged Knox user. -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Commented] (SPARK-27681) Use scala.collection.Seq explicitly instead of scala.Seq alias
[
https://issues.apache.org/jira/browse/SPARK-27681?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16839674#comment-16839674
]
Marcelo Vanzin commented on SPARK-27681:
bq. 2043 in method parameters, 1488 in method return types
Just a small fraction of those are public APIs, I'll bet. So probably not that
huge of a problem.
> Use scala.collection.Seq explicitly instead of scala.Seq alias
> --
>
> Key: SPARK-27681
> URL: https://issues.apache.org/jira/browse/SPARK-27681
> Project: Spark
> Issue Type: Sub-task
> Components: ML, MLlib, Spark Core, SQL, Structured Streaming
>Affects Versions: 3.0.0
>Reporter: Sean Owen
>Assignee: Sean Owen
>Priority: Major
>
> {{scala.Seq}} is widely used in the code, and is an alias for
> {{scala.collection.Seq}} in Scala 2.12. It will become an alias for
> {{scala.collection.immutable.Seq}} in Scala 2.13. In many cases, this will be
> fine, as Spark users using Scala 2.13 will also have this changed alias. In
> some cases it may be undesirable, as it will cause some code to compile in
> 2.12 but not in 2.13. In some cases, making the type {{scala.collection.Seq}}
> explicit so that it doesn't vary can help avoid this, so that Spark apps
> might cross-compile for 2.12 and 2.13 with the same source.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Commented] (SPARK-27681) Use scala.collection.Seq explicitly instead of scala.Seq alias
[
https://issues.apache.org/jira/browse/SPARK-27681?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16838799#comment-16838799
]
Marcelo Vanzin commented on SPARK-27681:
bq. The case to consider is, roughly, where Seq is an argument not a return
type.
If this change is restricted to that it's probably fine. But it's still a
tricky change. You'll probably have to explicitly use that class in the problem
spots (i.e. no importing at the top level or it may change things you don't
want to change). Then people will have to remember to do that (or reviewers
remember to catch it during review). It just feels a little too brittle.
> Use scala.collection.Seq explicitly instead of scala.Seq alias
> --
>
> Key: SPARK-27681
> URL: https://issues.apache.org/jira/browse/SPARK-27681
> Project: Spark
> Issue Type: Sub-task
> Components: ML, MLlib, Spark Core, SQL, Structured Streaming
>Affects Versions: 3.0.0
>Reporter: Sean Owen
>Assignee: Sean Owen
>Priority: Major
>
> {{scala.Seq}} is widely used in the code, and is an alias for
> {{scala.collection.Seq}} in Scala 2.12. It will become an alias for
> {{scala.collection.immutable.Seq}} in Scala 2.13. In many cases, this will be
> fine, as Spark users using Scala 2.13 will also have this changed alias. In
> some cases it may be undesirable, as it will cause some code to compile in
> 2.12 but not in 2.13. In some cases, making the type {{scala.collection.Seq}}
> explicit so that it doesn't vary can help avoid this, so that Spark apps
> might cross-compile for 2.12 and 2.13 with the same source.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Commented] (SPARK-27681) Use scala.collection.Seq explicitly instead of scala.Seq alias
[
https://issues.apache.org/jira/browse/SPARK-27681?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16838766#comment-16838766
]
Marcelo Vanzin commented on SPARK-27681:
bq. But, user code that passes a non-immutable Seq to a Spark API that accepts
scala.Seq now will no longer compile in 2.13.
That's true, but isn't that what the Scala developers intend anyway? That will
be true for any Scala code, not just Spark. That's assuming they didn't add any
implicit conversion from a mutable Seq to an immutable one, which would solve
that problem.
My problem with your suggestion is that now Spark developers will have to
always remember to import that different {{Seq}} type. And if they don't
remember, probably nothing will break until it's too late to notice. It's a
counter-intuitive change for developers, and I in particular am not seeing a
lot of benefits from it.
Here's an example of your proposed change that would break user code (just ran
it on Scala 2.13-RC1):
{code}
scala> def foo(): scala.collection.Seq[String] = Nil
foo: ()scala.collection.Seq[String]
scala> val s: Seq[String] = foo()
^
error: type mismatch;
found : Seq[String] (in scala.collection)
required: Seq[String] (in scala.collection.immutable)
{code}
So, aside from Spark developers having to remember to use the different {{Seq}}
type, user code might also have to change so that their internal APIs also use
the different type, or things like the above may occur.
BTW I also checked and there's no automatic promotion from mutable to immutable
seqs:
{code}
scala> val s: Seq[String] = scala.collection.mutable.ArrayBuffer[String]()
^
error: type mismatch;
found : scala.collection.mutable.ArrayBuffer[String]
required: Seq[String]
{code}
So I sort of understand your desire to keep things more similar, but I'm not
really seeing the advantages you see.
> Use scala.collection.Seq explicitly instead of scala.Seq alias
> --
>
> Key: SPARK-27681
> URL: https://issues.apache.org/jira/browse/SPARK-27681
> Project: Spark
> Issue Type: Sub-task
> Components: ML, MLlib, Spark Core, SQL, Structured Streaming
>Affects Versions: 3.0.0
>Reporter: Sean Owen
>Assignee: Sean Owen
>Priority: Major
>
> {{scala.Seq}} is widely used in the code, and is an alias for
> {{scala.collection.Seq}} in Scala 2.12. It will become an alias for
> {{scala.collection.immutable.Seq}} in Scala 2.13. In many cases, this will be
> fine, as Spark users using Scala 2.13 will also have this changed alias. In
> some cases it may be undesirable, as it will cause some code to compile in
> 2.12 but not in 2.13. In some cases, making the type {{scala.collection.Seq}}
> explicit so that it doesn't vary can help avoid this, so that Spark apps
> might cross-compile for 2.12 and 2.13 with the same source.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Commented] (SPARK-27681) Use scala.collection.Seq explicitly instead of scala.Seq alias
[
https://issues.apache.org/jira/browse/SPARK-27681?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16838715#comment-16838715
]
Marcelo Vanzin commented on SPARK-27681:
bq. varargs methods in Scala will change their signature as they are
implemented with Seq
Again, why is that a problem? That will only happen in the 2.13 version of
Spark.There are absolutely no backwards binary compatibility guarantees in that
case.
The 2.12 build will keep working just like it does today.
> Use scala.collection.Seq explicitly instead of scala.Seq alias
> --
>
> Key: SPARK-27681
> URL: https://issues.apache.org/jira/browse/SPARK-27681
> Project: Spark
> Issue Type: Sub-task
> Components: ML, MLlib, Spark Core, SQL, Structured Streaming
>Affects Versions: 3.0.0
>Reporter: Sean Owen
>Assignee: Sean Owen
>Priority: Major
>
> {{scala.Seq}} is widely used in the code, and is an alias for
> {{scala.collection.Seq}} in Scala 2.12. It will become an alias for
> {{scala.collection.immutable.Seq}} in Scala 2.13. To avoid API changes, we
> should simply explicit import and use {{scala.collection.Seq}}.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Commented] (SPARK-27681) Use scala.collection.Seq explicitly instead of scala.Seq alias
[
https://issues.apache.org/jira/browse/SPARK-27681?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16838693#comment-16838693
]
Marcelo Vanzin commented on SPARK-27681:
bq. scala.Seq is just a type def for scala.collection.Seq
In 2.12. If you force all the API in 2.13 to explicitly use
{{scala.collection.Seq}}, all the existing user code that does not explicitly
import that class will now be using a different type, and may run into issues.
Again, what I'm asking is why it is important for a method {{def foo(s:
Seq[Any])}} to be exposed as {{def foo(s: scala.collection.Seq[Any])}} in both
Scala 2.12 and 2.13, instead of using whatever is the default {{Seq}} alias for
the respective Scala version.
I really don't know how to word my question any more clearly.
> Use scala.collection.Seq explicitly instead of scala.Seq alias
> --
>
> Key: SPARK-27681
> URL: https://issues.apache.org/jira/browse/SPARK-27681
> Project: Spark
> Issue Type: Sub-task
> Components: ML, MLlib, Spark Core, SQL, Structured Streaming
>Affects Versions: 3.0.0
>Reporter: Sean Owen
>Assignee: Sean Owen
>Priority: Major
>
> {{scala.Seq}} is widely used in the code, and is an alias for
> {{scala.collection.Seq}} in Scala 2.12. It will become an alias for
> {{scala.collection.immutable.Seq}} in Scala 2.13. To avoid API changes, we
> should simply explicit import and use {{scala.collection.Seq}}.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Comment Edited] (SPARK-27681) Use scala.collection.Seq explicitly instead of scala.Seq alias
[
https://issues.apache.org/jira/browse/SPARK-27681?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16837752#comment-16837752
]
Marcelo Vanzin edited comment on SPARK-27681 at 5/11/19 4:28 AM:
-
Also the change you're proposing likely would break source compatibility, which
would make upgrading to Spark + Scala 2.13 harder than just a recompile...
was (Author: vanzin):
Also the change you're proposing likely would break source compatibility, which
would make upgrading to Spark 3 harder than just a recompile...
> Use scala.collection.Seq explicitly instead of scala.Seq alias
> --
>
> Key: SPARK-27681
> URL: https://issues.apache.org/jira/browse/SPARK-27681
> Project: Spark
> Issue Type: Sub-task
> Components: ML, MLlib, Spark Core, SQL, Structured Streaming
>Affects Versions: 3.0.0
>Reporter: Sean Owen
>Assignee: Sean Owen
>Priority: Major
>
> {{scala.Seq}} is widely used in the code, and is an alias for
> {{scala.collection.Seq}} in Scala 2.12. It will become an alias for
> {{scala.collection.immutable.Seq}} in Scala 2.13. To avoid API changes, we
> should simply explicit import and use {{scala.collection.Seq}}.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Commented] (SPARK-27681) Use scala.collection.Seq explicitly instead of scala.Seq alias
[
https://issues.apache.org/jira/browse/SPARK-27681?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16837753#comment-16837753
]
Marcelo Vanzin commented on SPARK-27681:
bq. we don't need to do this until we support Scala 2.13
I guess my question is why do we need to do it? Your explanation above
indicates that the {{Seq}} alias in 2.13 will just be a different type. That's
fine since 2.12 and 2.13 builds are not binary compatible anyway.
I'd understand this change if the {{Seq}} alias was being removed altogether
and you'd need an explicit import, but that doesn't seem to be the case.
> Use scala.collection.Seq explicitly instead of scala.Seq alias
> --
>
> Key: SPARK-27681
> URL: https://issues.apache.org/jira/browse/SPARK-27681
> Project: Spark
> Issue Type: Sub-task
> Components: ML, MLlib, Spark Core, SQL, Structured Streaming
>Affects Versions: 3.0.0
>Reporter: Sean Owen
>Assignee: Sean Owen
>Priority: Major
>
> {{scala.Seq}} is widely used in the code, and is an alias for
> {{scala.collection.Seq}} in Scala 2.12. It will become an alias for
> {{scala.collection.immutable.Seq}} in Scala 2.13. To avoid API changes, we
> should simply explicit import and use {{scala.collection.Seq}}.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Commented] (SPARK-27681) Use scala.collection.Seq explicitly instead of scala.Seq alias
[
https://issues.apache.org/jira/browse/SPARK-27681?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16837752#comment-16837752
]
Marcelo Vanzin commented on SPARK-27681:
Also the change you're proposing likely would break source compatibility, which
would make upgrading to Spark 3 harder than just a recompile...
> Use scala.collection.Seq explicitly instead of scala.Seq alias
> --
>
> Key: SPARK-27681
> URL: https://issues.apache.org/jira/browse/SPARK-27681
> Project: Spark
> Issue Type: Sub-task
> Components: ML, MLlib, Spark Core, SQL, Structured Streaming
>Affects Versions: 3.0.0
>Reporter: Sean Owen
>Assignee: Sean Owen
>Priority: Major
>
> {{scala.Seq}} is widely used in the code, and is an alias for
> {{scala.collection.Seq}} in Scala 2.12. It will become an alias for
> {{scala.collection.immutable.Seq}} in Scala 2.13. To avoid API changes, we
> should simply explicit import and use {{scala.collection.Seq}}.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Commented] (SPARK-27681) Use scala.collection.Seq explicitly instead of scala.Seq alias
[
https://issues.apache.org/jira/browse/SPARK-27681?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16837749#comment-16837749
]
Marcelo Vanzin commented on SPARK-27681:
OOC, what happens if we do nothing?
As far as I can see, Scala 2.12 builds will have scala.Seq in the API, Scala
2.13 builds will have scala.collection.immutable.Seq, and that should be fine,
right, since those builds are not binary compatible with each other anyway?
> Use scala.collection.Seq explicitly instead of scala.Seq alias
> --
>
> Key: SPARK-27681
> URL: https://issues.apache.org/jira/browse/SPARK-27681
> Project: Spark
> Issue Type: Sub-task
> Components: ML, MLlib, Spark Core, SQL, Structured Streaming
>Affects Versions: 3.0.0
>Reporter: Sean Owen
>Assignee: Sean Owen
>Priority: Major
>
> {{scala.Seq}} is widely used in the code, and is an alias for
> {{scala.collection.Seq}} in Scala 2.12. It will become an alias for
> {{scala.collection.immutable.Seq}} in Scala 2.13. To avoid API changes, we
> should simply explicit import and use {{scala.collection.Seq}}.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Created] (SPARK-27678) Support Knox user impersonation in UI
Marcelo Vanzin created SPARK-27678: -- Summary: Support Knox user impersonation in UI Key: SPARK-27678 URL: https://issues.apache.org/jira/browse/SPARK-27678 Project: Spark Issue Type: New Feature Components: Web UI Affects Versions: 3.0.0 Reporter: Marcelo Vanzin When Spark is running behind a Knox proxy server, it would be useful to support impersonation, so that Knox can properly identify the user who's actually making the request. This way, Knox authenticates to Spark, but Spark makes access control checks against the user that Knox identifies as the actual requesting user, so that proper access checks are performed, instead of performing the checks against the more privileged Knox user. -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Resolved] (SPARK-26632) Separate Thread Configurations of Driver and Executor
[ https://issues.apache.org/jira/browse/SPARK-26632?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin resolved SPARK-26632. Resolution: Fixed Fix Version/s: 3.0.0 Issue resolved by pull request 23560 [https://github.com/apache/spark/pull/23560] > Separate Thread Configurations of Driver and Executor > - > > Key: SPARK-26632 > URL: https://issues.apache.org/jira/browse/SPARK-26632 > Project: Spark > Issue Type: Improvement > Components: Spark Core >Affects Versions: 3.0.0 >Reporter: jiafu zhang >Priority: Minor > Fix For: 3.0.0 > > > During the benchmark of Spark 2.4.0 on HPC (High Performance Computing), we > identified an area can be optimized to improve RPC performance on large > number of HPC nodes with omini-path NIC. It's same thread configurations for > both driver and executor. From the test, we find driver and executor should > have different thread configurations because driver has far more RPC messages > than single executor. > These configurations are, > ||Config Key||for Driver||for Executor|| > |spark.rpc.io.serverThreads|spark.driver.rpc.io.serverThreads|spark.executor.rpc.io.serverThreads| > |spark.rpc.io.clientThreads|spark.driver.rpc.io.clientThreads|spark.executor.rpc.io.clientThreads| > |spark.rpc.netty.dispatcher.numThreads|spark.driver.rpc.netty.dispatcher.numThreads|spark.executor.rpc.netty.dispatcher.numThreads| > When Spark reads thread configurations, it tries to read driver's > configurations or executor's configurations first. Then fall back to the > common thread configurations. > After the separation, the performance is improved a lot in 256 nodes and 512 > nodes. see below test result of SimpleMapTask. > || > ||spark.driver.rpc.io.serverThreads||spark.driver.rpc.io.clientThreads||spark.driver.rpc.netty.dispatcher.numThreads||spark.executor.rpc.netty.dispatcher.numThreads||Overall > Time (s)||Overall Time without Separation (s)||Improvement|| > |128 nodes|15|15|10|30|107|108|0.9%| > |256 nodes|12|15|10|30|159|196|18.8%| > |512 nodes|12|15|10|30|283|377|24.9%| > > The implementation is almost done. We are working on the code merge. -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[spark] branch master updated (c71f217 -> fa5dc0a)
This is an automated email from the ASF dual-hosted git repository. vanzin pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/spark.git. from c71f217 [SPARK-27673][SQL] Add `since` info to random, regex, null expressions add fa5dc0a [SPARK-26632][CORE] Separate Thread Configurations of Driver and Executor No new revisions were added by this update. Summary of changes: .../executor/CoarseGrainedExecutorBackend.scala| 2 + .../spark/network/netty/SparkTransportConf.scala | 24 +-- .../org/apache/spark/rpc/netty/Dispatcher.scala| 19 +++-- .../org/apache/spark/rpc/netty/NettyRpcEnv.scala | 9 ++- .../network/netty/SparkTransportConfSuite.scala| 81 ++ docs/configuration.md | 38 ++ project/MimaExcludes.scala | 3 + .../spark/executor/MesosExecutorBackend.scala | 2 + 8 files changed, 165 insertions(+), 13 deletions(-) create mode 100644 core/src/test/scala/org/apache/spark/network/netty/SparkTransportConfSuite.scala - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[jira] [Assigned] (SPARK-26632) Separate Thread Configurations of Driver and Executor
[ https://issues.apache.org/jira/browse/SPARK-26632?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin reassigned SPARK-26632: -- Assignee: jiafu zhang > Separate Thread Configurations of Driver and Executor > - > > Key: SPARK-26632 > URL: https://issues.apache.org/jira/browse/SPARK-26632 > Project: Spark > Issue Type: Improvement > Components: Spark Core >Affects Versions: 3.0.0 >Reporter: jiafu zhang >Assignee: jiafu zhang >Priority: Minor > Fix For: 3.0.0 > > > During the benchmark of Spark 2.4.0 on HPC (High Performance Computing), we > identified an area can be optimized to improve RPC performance on large > number of HPC nodes with omini-path NIC. It's same thread configurations for > both driver and executor. From the test, we find driver and executor should > have different thread configurations because driver has far more RPC messages > than single executor. > These configurations are, > ||Config Key||for Driver||for Executor|| > |spark.rpc.io.serverThreads|spark.driver.rpc.io.serverThreads|spark.executor.rpc.io.serverThreads| > |spark.rpc.io.clientThreads|spark.driver.rpc.io.clientThreads|spark.executor.rpc.io.clientThreads| > |spark.rpc.netty.dispatcher.numThreads|spark.driver.rpc.netty.dispatcher.numThreads|spark.executor.rpc.netty.dispatcher.numThreads| > When Spark reads thread configurations, it tries to read driver's > configurations or executor's configurations first. Then fall back to the > common thread configurations. > After the separation, the performance is improved a lot in 256 nodes and 512 > nodes. see below test result of SimpleMapTask. > || > ||spark.driver.rpc.io.serverThreads||spark.driver.rpc.io.clientThreads||spark.driver.rpc.netty.dispatcher.numThreads||spark.executor.rpc.netty.dispatcher.numThreads||Overall > Time (s)||Overall Time without Separation (s)||Improvement|| > |128 nodes|15|15|10|30|107|108|0.9%| > |256 nodes|12|15|10|30|159|196|18.8%| > |512 nodes|12|15|10|30|283|377|24.9%| > > The implementation is almost done. We are working on the code merge. -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Assigned] (SPARK-27294) Multi-cluster Kafka delegation token support
[ https://issues.apache.org/jira/browse/SPARK-27294?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin reassigned SPARK-27294: -- Assignee: Gabor Somogyi > Multi-cluster Kafka delegation token support > > > Key: SPARK-27294 > URL: https://issues.apache.org/jira/browse/SPARK-27294 > Project: Spark > Issue Type: Improvement > Components: Structured Streaming >Affects Versions: 3.0.0 >Reporter: Gabor Somogyi >Assignee: Gabor Somogyi >Priority: Major > > Kafka delegation token only supports single cluster at the moment. > I've created a small document with the proposed Spark approach > [here|https://docs.google.com/document/d/1yuwIxKqUnzo5RxJDIqrWWC2s67hh5Tb1QtfIwEKVWtM/edit?usp=sharing]. -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Resolved] (SPARK-27294) Multi-cluster Kafka delegation token support
[ https://issues.apache.org/jira/browse/SPARK-27294?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin resolved SPARK-27294. Resolution: Fixed Fix Version/s: 3.0.0 Issue resolved by pull request 24305 [https://github.com/apache/spark/pull/24305] > Multi-cluster Kafka delegation token support > > > Key: SPARK-27294 > URL: https://issues.apache.org/jira/browse/SPARK-27294 > Project: Spark > Issue Type: Improvement > Components: Structured Streaming >Affects Versions: 3.0.0 >Reporter: Gabor Somogyi >Assignee: Gabor Somogyi >Priority: Major > Fix For: 3.0.0 > > > Kafka delegation token only supports single cluster at the moment. > I've created a small document with the proposed Spark approach > [here|https://docs.google.com/document/d/1yuwIxKqUnzo5RxJDIqrWWC2s67hh5Tb1QtfIwEKVWtM/edit?usp=sharing]. -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[spark] branch master updated: [SPARK-27294][SS] Add multi-cluster Kafka delegation token
This is an automated email from the ASF dual-hosted git repository.
vanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 2f55809 [SPARK-27294][SS] Add multi-cluster Kafka delegation token
2f55809 is described below
commit 2f558094257c38d26650049f2ac93be6d65d6d85
Author: Gabor Somogyi
AuthorDate: Tue May 7 11:40:43 2019 -0700
[SPARK-27294][SS] Add multi-cluster Kafka delegation token
## What changes were proposed in this pull request?
The actual implementation doesn't support multi-cluster Kafka connection
with delegation token. In this PR I've added this functionality.
What this PR contains:
* New way of configuration
* Multiple delegation token obtain/store/use functionality
* Documentation
* The change works on DStreams also
## How was this patch tested?
Existing + additional unit tests.
Additionally tested on cluster.
Test scenario:
* 2 * 4 node clusters
* The 4-4 nodes are in different kerberos realms
* Cross-Realm trust between the 2 realms
* Yarn
* Kafka broker version 2.1.0
* security.protocol = SASL_SSL
* sasl.mechanism = SCRAM-SHA-512
* Artificial exceptions during processing
* Source reads from realm1 sink writes to realm2
Kafka broker settings:
* delegation.token.expiry.time.ms=60 (10 min)
* delegation.token.max.lifetime.ms=120 (20 min)
* delegation.token.expiry.check.interval.ms=30 (5 min)
Closes #24305 from gaborgsomogyi/SPARK-27294.
Authored-by: Gabor Somogyi
Signed-off-by: Marcelo Vanzin
---
.../org/apache/spark/internal/config/Kafka.scala | 91 ---
docs/structured-streaming-kafka-integration.md | 103 -
.../spark/sql/kafka010/CachedKafkaProducer.scala | 14 +--
.../spark/sql/kafka010/ConsumerStrategy.scala | 10 +-
.../spark/sql/kafka010/KafkaDataConsumer.scala | 2 +-
.../apache/spark/kafka010/KafkaConfigUpdater.scala | 21 ++--
.../kafka010/KafkaDelegationTokenProvider.scala| 47 ++--
.../spark/kafka010/KafkaTokenSparkConf.scala | 96
.../org/apache/spark/kafka010/KafkaTokenUtil.scala | 102 ++--
.../spark/kafka010/KafkaConfigUpdaterSuite.scala | 45 ++--
.../spark/kafka010/KafkaDelegationTokenTest.scala | 7 +-
.../spark/kafka010/KafkaTokenSparkConfSuite.scala | 128 +
.../spark/kafka010/KafkaTokenUtilSuite.scala | 123
external/kafka-0-10/pom.xml| 5 +
.../kafka010/KafkaDataConsumerSuite.scala | 10 +-
15 files changed, 573 insertions(+), 231 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/internal/config/Kafka.scala
b/core/src/main/scala/org/apache/spark/internal/config/Kafka.scala
deleted file mode 100644
index e91ddd3..000
--- a/core/src/main/scala/org/apache/spark/internal/config/Kafka.scala
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.internal.config
-
-private[spark] object Kafka {
-
- val BOOTSTRAP_SERVERS =
-ConfigBuilder("spark.kafka.bootstrap.servers")
- .doc("A list of coma separated host/port pairs to use for establishing
the initial " +
-"connection to the Kafka cluster. For further details please see kafka
documentation. " +
-"Only used to obtain delegation token.")
- .stringConf
- .createOptional
-
- val SECURITY_PROTOCOL =
-ConfigBuilder("spark.kafka.security.protocol")
- .doc("Protocol used to communicate with brokers. For further details
please see kafka " +
-"documentation. Only used to obtain delegation token.")
- .stringConf
- .createWithDefault("SASL_SSL")
-
- val KERBEROS_SERVICE_NAME =
-ConfigBuilder("spark.kafka.sasl.kerberos.service.name")
- .doc("The Kerberos principal name that Kafka runs as. This can be
defined either in &q
[jira] [Resolved] (SPARK-27610) Yarn external shuffle service fails to start when spark.shuffle.io.mode=EPOLL
[
https://issues.apache.org/jira/browse/SPARK-27610?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Marcelo Vanzin resolved SPARK-27610.
Resolution: Fixed
Fix Version/s: 3.0.0
Issue resolved by pull request 24502
[https://github.com/apache/spark/pull/24502]
> Yarn external shuffle service fails to start when spark.shuffle.io.mode=EPOLL
> -
>
> Key: SPARK-27610
> URL: https://issues.apache.org/jira/browse/SPARK-27610
> Project: Spark
> Issue Type: Improvement
> Components: Shuffle
>Affects Versions: 2.4.2
>Reporter: Adrian Muraru
>Priority: Minor
> Fix For: 3.0.0
>
>
> Enabling netty epoll mode in yarn shuffle service
> ({{spark.shuffle.io.mode=EPOLL}}) makes the Yarn NodeManager to abort.
> Checking the stracktrace, it seems that while the io.netty package is
> shaded, the native libraries provided by netty-all are not:
>
> {noformat}
> Caused by: java.io.FileNotFoundException:
> META-INF/native/liborg_spark_project_netty_transport_native_epoll_x86_64.so{noformat}
> *Full stack trace:*
> {noformat}
> 2019-04-24 23:14:46,372 ERROR [main] nodemanager.NodeManager
> (NodeManager.java:initAndStartNodeManager(639)) - Error starting NodeManager
> java.lang.UnsatisfiedLinkError: failed to load the required native library
> at
> org.spark_project.io.netty.channel.epoll.Epoll.ensureAvailability(Epoll.java:81)
> at
> org.spark_project.io.netty.channel.epoll.EpollEventLoop.(EpollEventLoop.java:55)
> at
> org.spark_project.io.netty.channel.epoll.EpollEventLoopGroup.newChild(EpollEventLoopGroup.java:134)
> at
> org.spark_project.io.netty.channel.epoll.EpollEventLoopGroup.newChild(EpollEventLoopGroup.java:35)
> at
> org.spark_project.io.netty.util.concurrent.MultithreadEventExecutorGroup.(MultithreadEventExecutorGroup.java:84)
> at
> org.spark_project.io.netty.util.concurrent.MultithreadEventExecutorGroup.(MultithreadEventExecutorGroup.java:58)
> at
> org.spark_project.io.netty.util.concurrent.MultithreadEventExecutorGroup.(MultithreadEventExecutorGroup.java:47)
> at
> org.spark_project.io.netty.channel.MultithreadEventLoopGroup.(MultithreadEventLoopGroup.java:59)
> at
> org.spark_project.io.netty.channel.epoll.EpollEventLoopGroup.(EpollEventLoopGroup.java:104)
> at
> org.spark_project.io.netty.channel.epoll.EpollEventLoopGroup.(EpollEventLoopGroup.java:91)
> at
> org.spark_project.io.netty.channel.epoll.EpollEventLoopGroup.(EpollEventLoopGroup.java:68)
> at
> org.apache.spark.network.util.NettyUtils.createEventLoop(NettyUtils.java:52)
> at
> org.apache.spark.network.server.TransportServer.init(TransportServer.java:95)
> at
> org.apache.spark.network.server.TransportServer.(TransportServer.java:75)
> at
> org.apache.spark.network.TransportContext.createServer(TransportContext.java:108)
> at
> org.apache.spark.network.yarn.YarnShuffleService.serviceInit(YarnShuffleService.java:186)
> at
> org.apache.hadoop.service.AbstractService.init(AbstractService.java:163)
> at
> org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServices.serviceInit(AuxServices.java:147)
> at
> org.apache.hadoop.service.AbstractService.init(AbstractService.java:163)
> at
> org.apache.hadoop.service.CompositeService.serviceInit(CompositeService.java:107)
> at
> org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl.serviceInit(ContainerManagerImpl.java:268)
> at
> org.apache.hadoop.service.AbstractService.init(AbstractService.java:163)
> at
> org.apache.hadoop.service.CompositeService.serviceInit(CompositeService.java:107)
> at
> org.apache.hadoop.yarn.server.nodemanager.NodeManager.serviceInit(NodeManager.java:357)
> at
> org.apache.hadoop.service.AbstractService.init(AbstractService.java:163)
> at
> org.apache.hadoop.yarn.server.nodemanager.NodeManager.initAndStartNodeManager(NodeManager.java:636)
> at
> org.apache.hadoop.yarn.server.nodemanager.NodeManager.main(NodeManager.java:684)
> Caused by: java.lang.UnsatisfiedLinkError: could not load a native library:
> org_spark_project_netty_transport_native_epoll_x86_64
> at
> org.spark_project.io.netty.util.internal.NativeLibraryLoader.load(NativeLibraryLoader.java:205)
> at
> org.spark_project.io.netty.channel.epoll.Native.loadNativeLibrary(Native.java:207)
> at
> org.spark_project.io.netty.channel.epoll.Native.(Native.java:65)
> at org.spark_project.io.netty.channel.epoll.Epoll.(Epoll.
[jira] [Assigned] (SPARK-27610) Yarn external shuffle service fails to start when spark.shuffle.io.mode=EPOLL
[
https://issues.apache.org/jira/browse/SPARK-27610?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Marcelo Vanzin reassigned SPARK-27610:
--
Assignee: Adrian Muraru
> Yarn external shuffle service fails to start when spark.shuffle.io.mode=EPOLL
> -
>
> Key: SPARK-27610
> URL: https://issues.apache.org/jira/browse/SPARK-27610
> Project: Spark
> Issue Type: Improvement
> Components: Shuffle
>Affects Versions: 2.4.2
>Reporter: Adrian Muraru
>Assignee: Adrian Muraru
>Priority: Minor
> Fix For: 3.0.0
>
>
> Enabling netty epoll mode in yarn shuffle service
> ({{spark.shuffle.io.mode=EPOLL}}) makes the Yarn NodeManager to abort.
> Checking the stracktrace, it seems that while the io.netty package is
> shaded, the native libraries provided by netty-all are not:
>
> {noformat}
> Caused by: java.io.FileNotFoundException:
> META-INF/native/liborg_spark_project_netty_transport_native_epoll_x86_64.so{noformat}
> *Full stack trace:*
> {noformat}
> 2019-04-24 23:14:46,372 ERROR [main] nodemanager.NodeManager
> (NodeManager.java:initAndStartNodeManager(639)) - Error starting NodeManager
> java.lang.UnsatisfiedLinkError: failed to load the required native library
> at
> org.spark_project.io.netty.channel.epoll.Epoll.ensureAvailability(Epoll.java:81)
> at
> org.spark_project.io.netty.channel.epoll.EpollEventLoop.(EpollEventLoop.java:55)
> at
> org.spark_project.io.netty.channel.epoll.EpollEventLoopGroup.newChild(EpollEventLoopGroup.java:134)
> at
> org.spark_project.io.netty.channel.epoll.EpollEventLoopGroup.newChild(EpollEventLoopGroup.java:35)
> at
> org.spark_project.io.netty.util.concurrent.MultithreadEventExecutorGroup.(MultithreadEventExecutorGroup.java:84)
> at
> org.spark_project.io.netty.util.concurrent.MultithreadEventExecutorGroup.(MultithreadEventExecutorGroup.java:58)
> at
> org.spark_project.io.netty.util.concurrent.MultithreadEventExecutorGroup.(MultithreadEventExecutorGroup.java:47)
> at
> org.spark_project.io.netty.channel.MultithreadEventLoopGroup.(MultithreadEventLoopGroup.java:59)
> at
> org.spark_project.io.netty.channel.epoll.EpollEventLoopGroup.(EpollEventLoopGroup.java:104)
> at
> org.spark_project.io.netty.channel.epoll.EpollEventLoopGroup.(EpollEventLoopGroup.java:91)
> at
> org.spark_project.io.netty.channel.epoll.EpollEventLoopGroup.(EpollEventLoopGroup.java:68)
> at
> org.apache.spark.network.util.NettyUtils.createEventLoop(NettyUtils.java:52)
> at
> org.apache.spark.network.server.TransportServer.init(TransportServer.java:95)
> at
> org.apache.spark.network.server.TransportServer.(TransportServer.java:75)
> at
> org.apache.spark.network.TransportContext.createServer(TransportContext.java:108)
> at
> org.apache.spark.network.yarn.YarnShuffleService.serviceInit(YarnShuffleService.java:186)
> at
> org.apache.hadoop.service.AbstractService.init(AbstractService.java:163)
> at
> org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServices.serviceInit(AuxServices.java:147)
> at
> org.apache.hadoop.service.AbstractService.init(AbstractService.java:163)
> at
> org.apache.hadoop.service.CompositeService.serviceInit(CompositeService.java:107)
> at
> org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl.serviceInit(ContainerManagerImpl.java:268)
> at
> org.apache.hadoop.service.AbstractService.init(AbstractService.java:163)
> at
> org.apache.hadoop.service.CompositeService.serviceInit(CompositeService.java:107)
> at
> org.apache.hadoop.yarn.server.nodemanager.NodeManager.serviceInit(NodeManager.java:357)
> at
> org.apache.hadoop.service.AbstractService.init(AbstractService.java:163)
> at
> org.apache.hadoop.yarn.server.nodemanager.NodeManager.initAndStartNodeManager(NodeManager.java:636)
> at
> org.apache.hadoop.yarn.server.nodemanager.NodeManager.main(NodeManager.java:684)
> Caused by: java.lang.UnsatisfiedLinkError: could not load a native library:
> org_spark_project_netty_transport_native_epoll_x86_64
> at
> org.spark_project.io.netty.util.internal.NativeLibraryLoader.load(NativeLibraryLoader.java:205)
> at
> org.spark_project.io.netty.channel.epoll.Native.loadNativeLibrary(Native.java:207)
> at
> org.spark_project.io.netty.channel.epoll.Native.(Native.java:65)
> at org.spark_project.io.netty.channel.epoll.Epoll.(Epoll.java:33)
&g
[spark] branch master updated (d124ce9 -> 8ef4da7)
This is an automated email from the ASF dual-hosted git repository. vanzin pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/spark.git. from d124ce9 [SPARK-27590][CORE] do not consider skipped tasks when scheduling speculative tasks add 8ef4da7 [SPARK-27610][YARN] Shade netty native libraries No new revisions were added by this update. Summary of changes: common/network-yarn/pom.xml| 46 +- conf/log4j.properties.template | 4 +- .../org/apache/spark/log4j-defaults.properties | 4 +- core/src/test/resources/log4j.properties | 2 +- .../src/main/resources/log4j.properties| 4 +- .../src/test/resources/log4j.properties| 2 +- graphx/src/test/resources/log4j.properties | 2 +- launcher/src/test/resources/log4j.properties | 2 +- mllib/src/test/resources/log4j.properties | 2 +- pom.xml| 2 +- project/MimaExcludes.scala | 5 ++- repl/src/test/resources/log4j.properties | 2 +- .../core/src/test/resources/log4j.properties | 2 +- .../src/test/resources/log4j.properties| 2 +- .../yarn/src/test/resources/log4j.properties | 2 +- .../spark/deploy/yarn/BaseYarnClusterSuite.scala | 2 +- sql/catalyst/src/test/resources/log4j.properties | 2 +- streaming/src/test/resources/log4j.properties | 2 +- 18 files changed, 68 insertions(+), 21 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[jira] [Assigned] (SPARK-27194) Job failures when task attempts do not clean up spark-staging parquet files
[
https://issues.apache.org/jira/browse/SPARK-27194?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Marcelo Vanzin reassigned SPARK-27194:
--
Assignee: (was: Marcelo Vanzin)
> Job failures when task attempts do not clean up spark-staging parquet files
> ---
>
> Key: SPARK-27194
> URL: https://issues.apache.org/jira/browse/SPARK-27194
> Project: Spark
> Issue Type: Bug
> Components: Spark Core, SQL
>Affects Versions: 2.3.1, 2.3.2, 2.3.3
>Reporter: Reza Safi
>Priority: Major
>
> When a container fails for some reason (for example when killed by yarn for
> exceeding memory limits), the subsequent task attempts for the tasks that
> were running on that container all fail with a FileAlreadyExistsException.
> The original task attempt does not seem to successfully call abortTask (or at
> least its "best effort" delete is unsuccessful) and clean up the parquet file
> it was writing to, so when later task attempts try to write to the same
> spark-staging directory using the same file name, the job fails.
> Here is what transpires in the logs:
> The container where task 200.0 is running is killed and the task is lost:
> {code}
> 19/02/20 09:33:25 ERROR cluster.YarnClusterScheduler: Lost executor y on
> t.y.z.com: Container killed by YARN for exceeding memory limits. 8.1 GB of 8
> GB physical memory used. Consider boosting spark.yarn.executor.memoryOverhead.
> 19/02/20 09:33:25 WARN scheduler.TaskSetManager: Lost task 200.0 in stage
> 0.0 (TID xxx, t.y.z.com, executor 93): ExecutorLostFailure (executor 93
> exited caused by one of the running tasks) Reason: Container killed by YARN
> for exceeding memory limits. 8.1 GB of 8 GB physical memory used. Consider
> boosting spark.yarn.executor.memoryOverhead.
> {code}
> The task is re-attempted on a different executor and fails because the
> part-00200-blah-blah.c000.snappy.parquet file from the first task attempt
> already exists:
> {code}
> 19/02/20 09:35:01 WARN scheduler.TaskSetManager: Lost task 200.1 in stage 0.0
> (TID 594, tn.y.z.com, executor 70): org.apache.spark.SparkException: Task
> failed while writing rows.
> at
> org.apache.spark.sql.execution.datasources.FileFormatWriter$.org$apache$spark$sql$execution$datasources$FileFormatWriter$$executeTask(FileFormatWriter.scala:285)
> at
> org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply(FileFormatWriter.scala:197)
> at
> org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply(FileFormatWriter.scala:196)
> at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
> at org.apache.spark.scheduler.Task.run(Task.scala:109)
> at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
> Caused by: org.apache.hadoop.fs.FileAlreadyExistsException:
> /user/hive/warehouse/tmp_supply_feb1/.spark-staging-blah-blah-blah/dt=2019-02-17/part-00200-blah-blah.c000.snappy.parquet
> for client a.b.c.d already exists
> {code}
> The job fails when the the configured task attempts (spark.task.maxFailures)
> have failed with the same error:
> {code}
> org.apache.spark.SparkException: Job aborted due to stage failure: Task 200
> in stage 0.0 failed 20 times, most recent failure: Lost task 284.19 in stage
> 0.0 (TID yyy, tm.y.z.com, executor 16): org.apache.spark.SparkException: Task
> failed while writing rows.
> at
> org.apache.spark.sql.execution.datasources.FileFormatWriter$.org$apache$spark$sql$execution$datasources$FileFormatWriter$$executeTask(FileFormatWriter.scala:285)
> ...
> Caused by: org.apache.hadoop.fs.FileAlreadyExistsException:
> /user/hive/warehouse/tmp_supply_feb1/.spark-staging-blah-blah-blah/dt=2019-02-17/part-00200-blah-blah.c000.snappy.parquet
> for client i.p.a.d already exists
> {code}
> SPARK-26682 wasn't the root cause here, since there wasn't any stage
> reattempt.
> This issue seems to happen when
> spark.sql.sources.partitionOverwriteMode=dynamic.
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Assigned] (SPARK-27194) Job failures when task attempts do not clean up spark-staging parquet files
[
https://issues.apache.org/jira/browse/SPARK-27194?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Marcelo Vanzin reassigned SPARK-27194:
--
Assignee: Marcelo Vanzin
> Job failures when task attempts do not clean up spark-staging parquet files
> ---
>
> Key: SPARK-27194
> URL: https://issues.apache.org/jira/browse/SPARK-27194
> Project: Spark
> Issue Type: Bug
> Components: Spark Core, SQL
>Affects Versions: 2.3.1, 2.3.2, 2.3.3
>Reporter: Reza Safi
> Assignee: Marcelo Vanzin
>Priority: Major
>
> When a container fails for some reason (for example when killed by yarn for
> exceeding memory limits), the subsequent task attempts for the tasks that
> were running on that container all fail with a FileAlreadyExistsException.
> The original task attempt does not seem to successfully call abortTask (or at
> least its "best effort" delete is unsuccessful) and clean up the parquet file
> it was writing to, so when later task attempts try to write to the same
> spark-staging directory using the same file name, the job fails.
> Here is what transpires in the logs:
> The container where task 200.0 is running is killed and the task is lost:
> {code}
> 19/02/20 09:33:25 ERROR cluster.YarnClusterScheduler: Lost executor y on
> t.y.z.com: Container killed by YARN for exceeding memory limits. 8.1 GB of 8
> GB physical memory used. Consider boosting spark.yarn.executor.memoryOverhead.
> 19/02/20 09:33:25 WARN scheduler.TaskSetManager: Lost task 200.0 in stage
> 0.0 (TID xxx, t.y.z.com, executor 93): ExecutorLostFailure (executor 93
> exited caused by one of the running tasks) Reason: Container killed by YARN
> for exceeding memory limits. 8.1 GB of 8 GB physical memory used. Consider
> boosting spark.yarn.executor.memoryOverhead.
> {code}
> The task is re-attempted on a different executor and fails because the
> part-00200-blah-blah.c000.snappy.parquet file from the first task attempt
> already exists:
> {code}
> 19/02/20 09:35:01 WARN scheduler.TaskSetManager: Lost task 200.1 in stage 0.0
> (TID 594, tn.y.z.com, executor 70): org.apache.spark.SparkException: Task
> failed while writing rows.
> at
> org.apache.spark.sql.execution.datasources.FileFormatWriter$.org$apache$spark$sql$execution$datasources$FileFormatWriter$$executeTask(FileFormatWriter.scala:285)
> at
> org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply(FileFormatWriter.scala:197)
> at
> org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply(FileFormatWriter.scala:196)
> at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
> at org.apache.spark.scheduler.Task.run(Task.scala:109)
> at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
> Caused by: org.apache.hadoop.fs.FileAlreadyExistsException:
> /user/hive/warehouse/tmp_supply_feb1/.spark-staging-blah-blah-blah/dt=2019-02-17/part-00200-blah-blah.c000.snappy.parquet
> for client a.b.c.d already exists
> {code}
> The job fails when the the configured task attempts (spark.task.maxFailures)
> have failed with the same error:
> {code}
> org.apache.spark.SparkException: Job aborted due to stage failure: Task 200
> in stage 0.0 failed 20 times, most recent failure: Lost task 284.19 in stage
> 0.0 (TID yyy, tm.y.z.com, executor 16): org.apache.spark.SparkException: Task
> failed while writing rows.
> at
> org.apache.spark.sql.execution.datasources.FileFormatWriter$.org$apache$spark$sql$execution$datasources$FileFormatWriter$$executeTask(FileFormatWriter.scala:285)
> ...
> Caused by: org.apache.hadoop.fs.FileAlreadyExistsException:
> /user/hive/warehouse/tmp_supply_feb1/.spark-staging-blah-blah-blah/dt=2019-02-17/part-00200-blah-blah.c000.snappy.parquet
> for client i.p.a.d already exists
> {code}
> SPARK-26682 wasn't the root cause here, since there wasn't any stage
> reattempt.
> This issue seems to happen when
> spark.sql.sources.partitionOverwriteMode=dynamic.
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Resolved] (SPARK-16367) Wheelhouse Support for PySpark
[
https://issues.apache.org/jira/browse/SPARK-16367?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Marcelo Vanzin resolved SPARK-16367.
Resolution: Duplicate
This is somewhat similar to SPARK-13587 so let's keep the discussion in one
place.
> Wheelhouse Support for PySpark
> --
>
> Key: SPARK-16367
> URL: https://issues.apache.org/jira/browse/SPARK-16367
> Project: Spark
> Issue Type: New Feature
> Components: Deploy, PySpark
>Affects Versions: 1.6.1, 1.6.2, 2.0.0
>Reporter: gsemet
>Priority: Major
> Labels: newbie, python, python-wheel, wheelhouse
> Original Estimate: 168h
> Remaining Estimate: 168h
>
> *Rational*
> Is it recommended, in order to deploying Scala packages written in Scala, to
> build big fat jar files. This allows to have all dependencies on one package
> so the only "cost" is copy time to deploy this file on every Spark Node.
> On the other hand, Python deployment is more difficult once you want to use
> external packages, and you don't really want to mess with the IT to deploy
> the packages on the virtualenv of each nodes.
> This ticket proposes to allow users the ability to deploy their job as
> "Wheels" packages. The Python community is strongly advocating to promote
> this way of packaging and distributing Python application as a "standard way
> of deploying Python App". In other word, this is the "Pythonic Way of
> Deployment".
> *Previous approaches*
> I based the current proposal over the two following bugs related to this
> point:
> - SPARK-6764 ("Wheel support for PySpark")
> - SPARK-13587("Support virtualenv in PySpark")
> First part of my proposal was to merge, in order to support wheels install
> and virtualenv creation
> *Virtualenv, wheel support and "Uber Fat Wheelhouse" for PySpark*
> In Python, the packaging standard is now the "wheels" file format, which goes
> further that good old ".egg" files. With a wheel file (".whl"), the package
> is already prepared for a given architecture. You can have several wheels for
> a given package version, each specific to an architecture, or environment.
> For example, look at https://pypi.python.org/pypi/numpy all the different
> version of Wheel available.
> The {{pip}} tools knows how to select the right wheel file matching the
> current system, and how to install this package in a light speed (without
> compilation). Said otherwise, package that requires compilation of a C
> module, for instance "numpy", does *not* compile anything when installing
> from wheel file.
> {{pypi.pypthon.org}} already provided wheels for major python version. It the
> wheel is not available, pip will compile it from source anyway. Mirroring of
> Pypi is possible through projects such as http://doc.devpi.net/latest/
> (untested) or the Pypi mirror support on Artifactory (tested personnally).
> {{pip}} also provides the ability to generate easily all wheels of all
> packages used for a given project which is inside a "virtualenv". This is
> called "wheelhouse". You can even don't mess with this compilation and
> retrieve it directly from pypi.python.org.
> *Use Case 1: no internet connectivity*
> Here my first proposal for a deployment workflow, in the case where the Spark
> cluster does not have any internet connectivity or access to a Pypi mirror.
> In this case the simplest way to deploy a project with several dependencies
> is to build and then send to complete "wheelhouse":
> - you are writing a PySpark script that increase in term of size and
> dependencies. Deploying on Spark for example requires to build numpy or
> Theano and other dependencies
> - to use "Big Fat Wheelhouse" support of Pyspark, you need to turn his script
> into a standard Python package:
> -- write a {{requirements.txt}}. I recommend to specify all package version.
> You can use [pip-tools|https://github.com/nvie/pip-tools] to maintain the
> requirements.txt
> {code}
> astroid==1.4.6 # via pylint
> autopep8==1.2.4
> click==6.6 # via pip-tools
> colorama==0.3.7 # via pylint
> enum34==1.1.6 # via hypothesis
> findspark==1.0.0 # via spark-testing-base
> first==2.0.1 # via pip-tools
> hypothesis==3.4.0 # via spark-testing-base
> lazy-object-proxy==1.2.2 # via astroid
> linecache2==1.0.0 # via traceback2
> pbr==1.10.0
> pep8==1.7.0 # via autopep8
> pip-tools==1.6.5
> py==1.4.31 # via pytest
> pyflakes=
[jira] [Assigned] (SPARK-13587) Support virtualenv in PySpark
[ https://issues.apache.org/jira/browse/SPARK-13587?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin reassigned SPARK-13587: -- Assignee: Marcelo Vanzin > Support virtualenv in PySpark > - > > Key: SPARK-13587 > URL: https://issues.apache.org/jira/browse/SPARK-13587 > Project: Spark > Issue Type: New Feature > Components: PySpark >Affects Versions: 1.6.3, 2.0.2, 2.1.2, 2.2.1, 2.3.0 >Reporter: Jeff Zhang > Assignee: Marcelo Vanzin >Priority: Major > > Currently, it's not easy for user to add third party python packages in > pyspark. > * One way is to using --py-files (suitable for simple dependency, but not > suitable for complicated dependency, especially with transitive dependency) > * Another way is install packages manually on each node (time wasting, and > not easy to switch to different environment) > Python has now 2 different virtualenv implementation. One is native > virtualenv another is through conda. This jira is trying to migrate these 2 > tools to distributed environment -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Updated] (SPARK-13587) Support virtualenv in PySpark
[ https://issues.apache.org/jira/browse/SPARK-13587?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin updated SPARK-13587: --- Target Version/s: (was: 3.0.0) > Support virtualenv in PySpark > - > > Key: SPARK-13587 > URL: https://issues.apache.org/jira/browse/SPARK-13587 > Project: Spark > Issue Type: New Feature > Components: PySpark >Affects Versions: 1.6.3, 2.0.2, 2.1.2, 2.2.1, 2.3.0 >Reporter: Jeff Zhang >Priority: Major > > Currently, it's not easy for user to add third party python packages in > pyspark. > * One way is to using --py-files (suitable for simple dependency, but not > suitable for complicated dependency, especially with transitive dependency) > * Another way is install packages manually on each node (time wasting, and > not easy to switch to different environment) > Python has now 2 different virtualenv implementation. One is native > virtualenv another is through conda. This jira is trying to migrate these 2 > tools to distributed environment -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Assigned] (SPARK-13587) Support virtualenv in PySpark
[ https://issues.apache.org/jira/browse/SPARK-13587?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin reassigned SPARK-13587: -- Assignee: (was: Marcelo Vanzin) > Support virtualenv in PySpark > - > > Key: SPARK-13587 > URL: https://issues.apache.org/jira/browse/SPARK-13587 > Project: Spark > Issue Type: New Feature > Components: PySpark >Affects Versions: 1.6.3, 2.0.2, 2.1.2, 2.2.1, 2.3.0 >Reporter: Jeff Zhang >Priority: Major > > Currently, it's not easy for user to add third party python packages in > pyspark. > * One way is to using --py-files (suitable for simple dependency, but not > suitable for complicated dependency, especially with transitive dependency) > * Another way is install packages manually on each node (time wasting, and > not easy to switch to different environment) > Python has now 2 different virtualenv implementation. One is native > virtualenv another is through conda. This jira is trying to migrate these 2 > tools to distributed environment -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Resolved] (SPARK-27575) Spark overwrites existing value of spark.yarn.dist.* instead of merging value
[ https://issues.apache.org/jira/browse/SPARK-27575?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin resolved SPARK-27575. Resolution: Fixed Fix Version/s: 3.0.0 Issue resolved by pull request 24465 [https://github.com/apache/spark/pull/24465] > Spark overwrites existing value of spark.yarn.dist.* instead of merging value > - > > Key: SPARK-27575 > URL: https://issues.apache.org/jira/browse/SPARK-27575 > Project: Spark > Issue Type: Bug > Components: Spark Submit, YARN >Affects Versions: 2.4.0 >Reporter: Jungtaek Lim >Assignee: Jungtaek Lim >Priority: Major > Fix For: 3.0.0 > > > If we specify `--files` arg when submitting app where configuration has > "spark.yarn.dist.files", SparkSubmit overwrites the new files (files provided > in arg) instead of merging existing value in configuration. > Same issue happens also on "spark.yarn.dist.pyFiles", "spark.yarn.dist.jars", > "spark.yarn.dist.archives". > > While I encountered the issue in Spark 2.4.0, I can see the issue from > codebase in master branch and also branch-2.3 as well. -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Assigned] (SPARK-27575) Spark overwrites existing value of spark.yarn.dist.* instead of merging value
[ https://issues.apache.org/jira/browse/SPARK-27575?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin reassigned SPARK-27575: -- Assignee: Jungtaek Lim > Spark overwrites existing value of spark.yarn.dist.* instead of merging value > - > > Key: SPARK-27575 > URL: https://issues.apache.org/jira/browse/SPARK-27575 > Project: Spark > Issue Type: Bug > Components: Spark Submit, YARN >Affects Versions: 2.4.0 >Reporter: Jungtaek Lim >Assignee: Jungtaek Lim >Priority: Major > > If we specify `--files` arg when submitting app where configuration has > "spark.yarn.dist.files", SparkSubmit overwrites the new files (files provided > in arg) instead of merging existing value in configuration. > Same issue happens also on "spark.yarn.dist.pyFiles", "spark.yarn.dist.jars", > "spark.yarn.dist.archives". > > While I encountered the issue in Spark 2.4.0, I can see the issue from > codebase in master branch and also branch-2.3 as well. -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[spark] branch master updated: [SPARK-27575][CORE][YARN] Yarn file-related confs should merge new value with existing value
This is an automated email from the ASF dual-hosted git repository.
vanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 75b40a5 [SPARK-27575][CORE][YARN] Yarn file-related confs should
merge new value with existing value
75b40a5 is described below
commit 75b40a53d3fb49ccadf032dd8e75c554c67676d2
Author: Jungtaek Lim (HeartSaVioR)
AuthorDate: Mon Apr 29 10:14:59 2019 -0700
[SPARK-27575][CORE][YARN] Yarn file-related confs should merge new value
with existing value
## What changes were proposed in this pull request?
This patch fixes a bug which YARN file-related configurations are being
overwritten when there're some values to assign - e.g. if `--file` is specified
as an argument, `spark.yarn.dist.files` is overwritten with the value of
argument. After this patch the existing value and new value will be merged
before assigning to the value of configuration.
## How was this patch tested?
Added UT, and manually tested with below command:
> ./bin/spark-submit --verbose --files
/etc/spark2/conf/spark-defaults.conf.template --master yarn-cluster --class
org.apache.spark.examples.SparkPi examples/jars/spark-examples_2.11-2.4.0.jar 10
where the spark conf file has
`spark.yarn.dist.files=file:/etc/spark2/conf/atlas-application.properties.yarn#atlas-application.properties`
```
Spark config:
...
(spark.yarn.dist.files,file:/etc/spark2/conf/atlas-application.properties.yarn#atlas-application.properties,file:///etc/spark2/conf/spark-defaults.conf.template)
...
```
Closes #24465 from HeartSaVioR/SPARK-27575.
Authored-by: Jungtaek Lim (HeartSaVioR)
Signed-off-by: Marcelo Vanzin
---
.../org/apache/spark/deploy/SparkSubmit.scala | 23 +--
.../org/apache/spark/deploy/SparkSubmitSuite.scala | 80 +-
2 files changed, 95 insertions(+), 8 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
index 9efaaa7..49d9395 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -544,10 +544,14 @@ private[spark] class SparkSubmit extends Logging {
// Yarn only
OptionAssigner(args.queue, YARN, ALL_DEPLOY_MODES, confKey =
"spark.yarn.queue"),
- OptionAssigner(args.pyFiles, YARN, ALL_DEPLOY_MODES, confKey =
"spark.yarn.dist.pyFiles"),
- OptionAssigner(args.jars, YARN, ALL_DEPLOY_MODES, confKey =
"spark.yarn.dist.jars"),
- OptionAssigner(args.files, YARN, ALL_DEPLOY_MODES, confKey =
"spark.yarn.dist.files"),
- OptionAssigner(args.archives, YARN, ALL_DEPLOY_MODES, confKey =
"spark.yarn.dist.archives"),
+ OptionAssigner(args.pyFiles, YARN, ALL_DEPLOY_MODES, confKey =
"spark.yarn.dist.pyFiles",
+mergeFn = Some(mergeFileLists(_, _))),
+ OptionAssigner(args.jars, YARN, ALL_DEPLOY_MODES, confKey =
"spark.yarn.dist.jars",
+mergeFn = Some(mergeFileLists(_, _))),
+ OptionAssigner(args.files, YARN, ALL_DEPLOY_MODES, confKey =
"spark.yarn.dist.files",
+mergeFn = Some(mergeFileLists(_, _))),
+ OptionAssigner(args.archives, YARN, ALL_DEPLOY_MODES, confKey =
"spark.yarn.dist.archives",
+mergeFn = Some(mergeFileLists(_, _))),
// Other options
OptionAssigner(args.numExecutors, YARN | KUBERNETES, ALL_DEPLOY_MODES,
@@ -608,7 +612,13 @@ private[spark] class SparkSubmit extends Logging {
(deployMode & opt.deployMode) != 0 &&
(clusterManager & opt.clusterManager) != 0) {
if (opt.clOption != null) { childArgs += (opt.clOption, opt.value) }
-if (opt.confKey != null) { sparkConf.set(opt.confKey, opt.value) }
+if (opt.confKey != null) {
+ if (opt.mergeFn.isDefined && sparkConf.contains(opt.confKey)) {
+sparkConf.set(opt.confKey,
opt.mergeFn.get.apply(sparkConf.get(opt.confKey), opt.value))
+ } else {
+sparkConf.set(opt.confKey, opt.value)
+ }
+}
}
}
@@ -1381,7 +1391,8 @@ private case class OptionAssigner(
clusterManager: Int,
deployMode: Int,
clOption: String = null,
-confKey: String = null)
+confKey: String = null,
+mergeFn: Option[(String, String) => String] = None)
private[spark] trait SparkSubmitOperation {
diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
index 2a17245..ef6213e 100644
--- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy
[jira] [Resolved] (SPARK-23014) Migrate MemorySink fully to v2
[ https://issues.apache.org/jira/browse/SPARK-23014?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin resolved SPARK-23014. Resolution: Fixed Fix Version/s: 3.0.0 Issue resolved by pull request 24403 [https://github.com/apache/spark/pull/24403] > Migrate MemorySink fully to v2 > -- > > Key: SPARK-23014 > URL: https://issues.apache.org/jira/browse/SPARK-23014 > Project: Spark > Issue Type: Sub-task > Components: Structured Streaming >Affects Versions: 2.4.0 >Reporter: Jose Torres >Assignee: Gabor Somogyi >Priority: Major > Fix For: 3.0.0 > > > There's already a MemorySinkV2, but its use is controlled by a flag. We need > to remove the V1 sink and always use it. -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Assigned] (SPARK-23014) Migrate MemorySink fully to v2
[ https://issues.apache.org/jira/browse/SPARK-23014?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin reassigned SPARK-23014: -- Assignee: Gabor Somogyi > Migrate MemorySink fully to v2 > -- > > Key: SPARK-23014 > URL: https://issues.apache.org/jira/browse/SPARK-23014 > Project: Spark > Issue Type: Sub-task > Components: Structured Streaming >Affects Versions: 2.4.0 >Reporter: Jose Torres >Assignee: Gabor Somogyi >Priority: Major > > There's already a MemorySinkV2, but its use is controlled by a flag. We need > to remove the V1 sink and always use it. -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[spark] branch master updated: [SPARK-23014][SS] Fully remove V1 memory sink.
This is an automated email from the ASF dual-hosted git repository.
vanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new fb6b19a [SPARK-23014][SS] Fully remove V1 memory sink.
fb6b19a is described below
commit fb6b19ab7c38aa0e1b2e208da86897bf3c07ae00
Author: Gabor Somogyi
AuthorDate: Mon Apr 29 09:44:23 2019 -0700
[SPARK-23014][SS] Fully remove V1 memory sink.
## What changes were proposed in this pull request?
There is a MemorySink v2 already so v1 can be removed. In this PR I've
removed it completely.
What this PR contains:
* V1 memory sink removal
* V2 memory sink renamed to become the only implementation
* Since DSv2 sends exceptions in a chained format (linking them with cause
field) I've made python side compliant
* Adapted all the tests
## How was this patch tested?
Existing unit tests.
Closes #24403 from gaborgsomogyi/SPARK-23014.
Authored-by: Gabor Somogyi
Signed-off-by: Marcelo Vanzin
---
.../main/scala/org/apache/spark/TestUtils.scala| 14
.../spark/sql/kafka010/KafkaContinuousTest.scala | 1 -
.../apache/spark/ml/recommendation/ALSSuite.scala | 10 ++-
.../scala/org/apache/spark/ml/util/MLTest.scala| 10 +--
python/pyspark/sql/streaming.py| 2 +-
python/pyspark/sql/tests/test_streaming.py | 12 ++-
python/pyspark/sql/utils.py| 49
.../spark/sql/execution/SparkStrategies.scala | 5 +-
.../spark/sql/execution/streaming/memory.scala | 92 +-
.../sources/{memoryV2.scala => memory.scala} | 14 ++--
.../spark/sql/streaming/DataStreamWriter.scala | 12 +--
.../sql/execution/streaming/MemorySinkSuite.scala | 88 -
.../execution/streaming/MemorySinkV2Suite.scala| 66
.../sql/streaming/EventTimeWatermarkSuite.scala| 1 +
.../sql/streaming/FileStreamSourceSuite.scala | 1 +
.../apache/spark/sql/streaming/StreamSuite.scala | 16 ++--
.../apache/spark/sql/streaming/StreamTest.scala| 16 ++--
.../sql/streaming/StreamingAggregationSuite.scala | 1 +
.../streaming/StreamingQueryListenerSuite.scala| 2 +-
.../spark/sql/streaming/StreamingQuerySuite.scala | 22 +++---
.../ContinuousQueryStatusAndProgressSuite.scala| 2 +-
.../sql/streaming/continuous/ContinuousSuite.scala | 9 +--
22 files changed, 181 insertions(+), 264 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/TestUtils.scala
b/core/src/main/scala/org/apache/spark/TestUtils.scala
index c2ebd38..c97b10e 100644
--- a/core/src/main/scala/org/apache/spark/TestUtils.scala
+++ b/core/src/main/scala/org/apache/spark/TestUtils.scala
@@ -193,6 +193,20 @@ private[spark] object TestUtils {
}
/**
+ * Asserts that exception message contains the message. Please note this
checks all
+ * exceptions in the tree.
+ */
+ def assertExceptionMsg(exception: Throwable, msg: String): Unit = {
+var e = exception
+var contains = e.getMessage.contains(msg)
+while (e.getCause != null && !contains) {
+ e = e.getCause
+ contains = e.getMessage.contains(msg)
+}
+assert(contains, s"Exception tree doesn't contain the expected message:
$msg")
+ }
+
+ /**
* Test if a command is available.
*/
def testCommandAvailable(command: String): Boolean = {
diff --git
a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousTest.scala
b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousTest.scala
index ad1c2c5..9ee8cbf 100644
---
a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousTest.scala
+++
b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousTest.scala
@@ -30,7 +30,6 @@ import org.apache.spark.sql.test.TestSparkSession
// Trait to configure StreamTest for kafka continuous execution tests.
trait KafkaContinuousTest extends KafkaSourceTest {
override val defaultTrigger = Trigger.Continuous(1000)
- override val defaultUseV2Sink = true
// We need more than the default local[2] to be able to schedule all
partitions simultaneously.
override protected def createSparkSession = new TestSparkSession(
diff --git
a/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
b/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
index 6d0321c..5ba3928 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
@@ -695,12 +695,14 @@ class ALSSuite extends MLTest with DefaultReadWriteTest
with Logging {
withClue("transform should fail when ids exceed integer range. ") {
val model = als.fit
[jira] [Assigned] (SPARK-27477) Kafka token provider should have provided dependency on Spark
[ https://issues.apache.org/jira/browse/SPARK-27477?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin reassigned SPARK-27477: -- Assignee: koert kuipers > Kafka token provider should have provided dependency on Spark > - > > Key: SPARK-27477 > URL: https://issues.apache.org/jira/browse/SPARK-27477 > Project: Spark > Issue Type: Bug > Components: SQL >Affects Versions: 3.0.0 > Environment: spark 3.0.0-SNAPSHOT > commit 38fc8e2484aa4971d1f2c115da61fc96f36e7868 > Author: Sean Owen > Date: Sat Apr 13 22:27:25 2019 +0900 > [MINOR][DOCS] Fix some broken links in docs >Reporter: koert kuipers >Assignee: koert kuipers >Priority: Trivial > Fix For: 3.0.0 > > > currently the external module spark-token-provider-kafka-0-10 has a compile > dependency on spark-core. this means spark-sql-kafka-0-10 also has a > transitive compile dependency on spark-core. > since spark-sql-kafka-0-10 is not bundled with spark but instead has to be > added to an application that runs on spark this dependency should be > provided, not compile. -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Resolved] (SPARK-27477) Kafka token provider should have provided dependency on Spark
[ https://issues.apache.org/jira/browse/SPARK-27477?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin resolved SPARK-27477. Resolution: Fixed Issue resolved by pull request 24384 [https://github.com/apache/spark/pull/24384] > Kafka token provider should have provided dependency on Spark > - > > Key: SPARK-27477 > URL: https://issues.apache.org/jira/browse/SPARK-27477 > Project: Spark > Issue Type: Bug > Components: SQL >Affects Versions: 3.0.0 > Environment: spark 3.0.0-SNAPSHOT > commit 38fc8e2484aa4971d1f2c115da61fc96f36e7868 > Author: Sean Owen > Date: Sat Apr 13 22:27:25 2019 +0900 > [MINOR][DOCS] Fix some broken links in docs >Reporter: koert kuipers >Priority: Trivial > Fix For: 3.0.0 > > > currently the external module spark-token-provider-kafka-0-10 has a compile > dependency on spark-core. this means spark-sql-kafka-0-10 also has a > transitive compile dependency on spark-core. > since spark-sql-kafka-0-10 is not bundled with spark but instead has to be > added to an application that runs on spark this dependency should be > provided, not compile. -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[spark] branch master updated: [SPARK-27477][BUILD] Kafka token provider should have provided dependency on Spark
This is an automated email from the ASF dual-hosted git repository.
vanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 7b367bf [SPARK-27477][BUILD] Kafka token provider should have
provided dependency on Spark
7b367bf is described below
commit 7b367bfc86dbe7f61c0dda4f4811508137bbd0cc
Author: Koert Kuipers
AuthorDate: Fri Apr 26 11:52:08 2019 -0700
[SPARK-27477][BUILD] Kafka token provider should have provided dependency
on Spark
## What changes were proposed in this pull request?
Change spark-token-provider-kafka-0-10 dependency on spark-core to be
provided
## How was this patch tested?
Ran existing unit tests
Closes #24384 from koertkuipers/feat-kafka-token-provider-fix-deps.
Authored-by: Koert Kuipers
Signed-off-by: Marcelo Vanzin
---
external/kafka-0-10-token-provider/pom.xml | 1 +
1 file changed, 1 insertion(+)
diff --git a/external/kafka-0-10-token-provider/pom.xml
b/external/kafka-0-10-token-provider/pom.xml
index 40ef1f7..01ca96b 100644
--- a/external/kafka-0-10-token-provider/pom.xml
+++ b/external/kafka-0-10-token-provider/pom.xml
@@ -39,6 +39,7 @@
org.apache.spark
spark-core_${scala.binary.version}
${project.version}
+ provided
org.apache.spark
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
Re: Is it possible to obtain the full command to be invoked by SparkLauncher?
BTW the SparkLauncher API has hooks to capture the stderr of the spark-submit process into the logging system of the parent process. Check the API javadocs since it's been forever since I looked at that. On Wed, Apr 24, 2019 at 1:58 PM Marcelo Vanzin wrote: > > Setting the SPARK_PRINT_LAUNCH_COMMAND env variable to 1 in the > launcher env will make Spark code print the command to stderr. Not > optimal but I think it's the only current option. > > On Wed, Apr 24, 2019 at 1:55 PM Jeff Evans > wrote: > > > > The org.apache.spark.launcher.SparkLauncher is used to construct a > > spark-submit invocation programmatically, via a builder pattern. In > > our application, which uses a SparkLauncher internally, I would like > > to log the full spark-submit command that it will invoke to our log > > file, in order to aid in debugging/support. However, I can't figure > > out a way to do this. This snippet would work, except for the fact > > that the createBuilder method is private. > > > > sparkLauncher.createBuilder().command() > > > > Is there an alternate way of doing this? The Spark version is > > 2.11:2.4.0. Thanks. > > > > - > > To unsubscribe e-mail: user-unsubscr...@spark.apache.org > > > > > -- > Marcelo -- Marcelo - To unsubscribe e-mail: user-unsubscr...@spark.apache.org
Re: Is it possible to obtain the full command to be invoked by SparkLauncher?
Setting the SPARK_PRINT_LAUNCH_COMMAND env variable to 1 in the launcher env will make Spark code print the command to stderr. Not optimal but I think it's the only current option. On Wed, Apr 24, 2019 at 1:55 PM Jeff Evans wrote: > > The org.apache.spark.launcher.SparkLauncher is used to construct a > spark-submit invocation programmatically, via a builder pattern. In > our application, which uses a SparkLauncher internally, I would like > to log the full spark-submit command that it will invoke to our log > file, in order to aid in debugging/support. However, I can't figure > out a way to do this. This snippet would work, except for the fact > that the createBuilder method is private. > > sparkLauncher.createBuilder().command() > > Is there an alternate way of doing this? The Spark version is > 2.11:2.4.0. Thanks. > > - > To unsubscribe e-mail: user-unsubscr...@spark.apache.org > -- Marcelo - To unsubscribe e-mail: user-unsubscr...@spark.apache.org
[jira] [Resolved] (SPARK-27515) [Deploy] When application master retry after a long time running, the hdfs delegation token may be expired
[ https://issues.apache.org/jira/browse/SPARK-27515?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin resolved SPARK-27515. Resolution: Duplicate > [Deploy] When application master retry after a long time running, the hdfs > delegation token may be expired > -- > > Key: SPARK-27515 > URL: https://issues.apache.org/jira/browse/SPARK-27515 > Project: Spark > Issue Type: Bug > Components: Deploy >Affects Versions: 2.3.2 >Reporter: feiwang >Priority: Major > -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Commented] (SPARK-11033) Launcher: add support for monitoring standalone/cluster apps
[ https://issues.apache.org/jira/browse/SPARK-11033?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16819274#comment-16819274 ] Marcelo Vanzin commented on SPARK-11033: It's definitely not that simple. > Launcher: add support for monitoring standalone/cluster apps > > > Key: SPARK-11033 > URL: https://issues.apache.org/jira/browse/SPARK-11033 > Project: Spark > Issue Type: Sub-task > Components: Spark Core >Affects Versions: 1.6.0 >Reporter: Marcelo Vanzin >Priority: Major > > The backend for app monitoring in the launcher library was added in > SPARK-8673, but the code currently does not support standalone cluster mode. -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Commented] (SPARK-11033) Launcher: add support for monitoring standalone/cluster apps
[ https://issues.apache.org/jira/browse/SPARK-11033?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16819211#comment-16819211 ] Marcelo Vanzin commented on SPARK-11033: I just filed the bug. I have no intention of working on this. > Launcher: add support for monitoring standalone/cluster apps > > > Key: SPARK-11033 > URL: https://issues.apache.org/jira/browse/SPARK-11033 > Project: Spark > Issue Type: Sub-task > Components: Spark Core >Affects Versions: 1.6.0 > Reporter: Marcelo Vanzin >Priority: Major > > The backend for app monitoring in the launcher library was added in > SPARK-8673, but the code currently does not support standalone cluster mode. -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Assigned] (SPARK-27270) Add Kafka dynamic JAAS authentication debug possibility
[ https://issues.apache.org/jira/browse/SPARK-27270?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin reassigned SPARK-27270: -- Assignee: Gabor Somogyi > Add Kafka dynamic JAAS authentication debug possibility > --- > > Key: SPARK-27270 > URL: https://issues.apache.org/jira/browse/SPARK-27270 > Project: Spark > Issue Type: Improvement > Components: Structured Streaming >Affects Versions: 3.0.0 >Reporter: Gabor Somogyi >Assignee: Gabor Somogyi >Priority: Major > -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Resolved] (SPARK-27270) Add Kafka dynamic JAAS authentication debug possibility
[ https://issues.apache.org/jira/browse/SPARK-27270?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin resolved SPARK-27270. Resolution: Fixed Fix Version/s: 3.0.0 Issue resolved by pull request 24204 [https://github.com/apache/spark/pull/24204] > Add Kafka dynamic JAAS authentication debug possibility > --- > > Key: SPARK-27270 > URL: https://issues.apache.org/jira/browse/SPARK-27270 > Project: Spark > Issue Type: Improvement > Components: Structured Streaming >Affects Versions: 3.0.0 >Reporter: Gabor Somogyi >Assignee: Gabor Somogyi >Priority: Major > Fix For: 3.0.0 > > -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[spark] branch master updated: [SPARK-27270][SS] Add Kafka dynamic JAAS authentication debug possibility
This is an automated email from the ASF dual-hosted git repository.
vanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 94adffa [SPARK-27270][SS] Add Kafka dynamic JAAS authentication debug
possibility
94adffa is described below
commit 94adffa8b160c0f0317df9675d0a1534e5f804cd
Author: Gabor Somogyi
AuthorDate: Thu Apr 11 16:39:40 2019 -0700
[SPARK-27270][SS] Add Kafka dynamic JAAS authentication debug possibility
## What changes were proposed in this pull request?
`Krb5LoginModule` supports debug parameter which is not yet supported from
Spark side. This configuration makes it easier to debug authentication issues
against Kafka.
In this PR `Krb5LoginModule` debug flag controlled by either
`sun.security.krb5.debug` or `com.ibm.security.krb5.Krb5Debug`.
Additionally found some hardcoded values like `ssl.truststore.location`,
etc... which could be error prone if Kafka changes it so in such cases Kafka
define used.
## How was this patch tested?
Existing + additional unit tests + on cluster.
Closes #24204 from gaborgsomogyi/SPARK-27270.
Authored-by: Gabor Somogyi
Signed-off-by: Marcelo Vanzin
---
.../org/apache/spark/kafka010/KafkaTokenUtil.scala | 24 +
.../spark/kafka010/KafkaTokenUtilSuite.scala | 40 +-
2 files changed, 42 insertions(+), 22 deletions(-)
diff --git
a/external/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaTokenUtil.scala
b/external/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaTokenUtil.scala
index e5604f2..e0825e5 100644
---
a/external/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaTokenUtil.scala
+++
b/external/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaTokenUtil.scala
@@ -28,7 +28,7 @@ import org.apache.hadoop.security.token.{Token,
TokenIdentifier}
import
org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier
import org.apache.kafka.clients.CommonClientConfigs
import org.apache.kafka.clients.admin.{AdminClient,
CreateDelegationTokenOptions}
-import org.apache.kafka.common.config.SaslConfigs
+import org.apache.kafka.common.config.{SaslConfigs, SslConfigs}
import org.apache.kafka.common.security.JaasContext
import org.apache.kafka.common.security.auth.SecurityProtocol.{SASL_PLAINTEXT,
SASL_SSL, SSL}
import org.apache.kafka.common.security.scram.ScramLoginModule
@@ -136,22 +136,22 @@ private[spark] object KafkaTokenUtil extends Logging {
private def setTrustStoreProperties(sparkConf: SparkConf, properties:
ju.Properties): Unit = {
sparkConf.get(Kafka.TRUSTSTORE_LOCATION).foreach { truststoreLocation =>
- properties.put("ssl.truststore.location", truststoreLocation)
+ properties.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG,
truststoreLocation)
}
sparkConf.get(Kafka.TRUSTSTORE_PASSWORD).foreach { truststorePassword =>
- properties.put("ssl.truststore.password", truststorePassword)
+ properties.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG,
truststorePassword)
}
}
private def setKeyStoreProperties(sparkConf: SparkConf, properties:
ju.Properties): Unit = {
sparkConf.get(Kafka.KEYSTORE_LOCATION).foreach { keystoreLocation =>
- properties.put("ssl.keystore.location", keystoreLocation)
+ properties.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, keystoreLocation)
}
sparkConf.get(Kafka.KEYSTORE_PASSWORD).foreach { keystorePassword =>
- properties.put("ssl.keystore.password", keystorePassword)
+ properties.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, keystorePassword)
}
sparkConf.get(Kafka.KEY_PASSWORD).foreach { keyPassword =>
- properties.put("ssl.key.password", keyPassword)
+ properties.put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, keyPassword)
}
}
@@ -159,6 +159,7 @@ private[spark] object KafkaTokenUtil extends Logging {
val params =
s"""
|${getKrb5LoginModuleName} required
+ | debug=${isGlobalKrbDebugEnabled()}
| useKeyTab=true
| serviceName="${sparkConf.get(Kafka.KERBEROS_SERVICE_NAME)}"
| keyTab="${sparkConf.get(KEYTAB).get}"
@@ -175,6 +176,7 @@ private[spark] object KafkaTokenUtil extends Logging {
val params =
s"""
|${getKrb5LoginModuleName} required
+ | debug=${isGlobalKrbDebugEnabled()}
| useTicketCache=true
| serviceName="${sparkConf.get(Kafka.KERBEROS_SERVICE_NAME)}";
""".stripMargin.replace("\n", "")
@@ -194,6 +196,16 @@ private[spark] object KafkaTokenUtil extends Logging {
}
}
+ private def isGlobalKrbDebugEnabl
[jira] [Assigned] (LIVY-563) RSCConf take no effect in RSCDriver
[ https://issues.apache.org/jira/browse/LIVY-563?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin reassigned LIVY-563: --- Assignee: Marcelo Vanzin > RSCConf take no effect in RSCDriver > > > Key: LIVY-563 > URL: https://issues.apache.org/jira/browse/LIVY-563 > Project: Livy > Issue Type: Bug > Components: Server >Affects Versions: 0.5.0 >Reporter: holmes.infra > Assignee: Marcelo Vanzin >Priority: Major > Fix For: 0.7.0 > > Time Spent: 0.5h > Remaining Estimate: 0h > > When create InterActiveSessin, prepareBuilderProp did not add all > RSCConf(livy.rsc.) to BuilderProp. After RSCServer started, the livy.rsc.* > related configure which modified by user take no effect. For example, > livy.rsc.sql.num-rows and livy.rsc.rpc.max.size and so on > -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Resolved] (LIVY-563) RSCConf take no effect in RSCDriver
[ https://issues.apache.org/jira/browse/LIVY-563?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin resolved LIVY-563. - Resolution: Fixed Fix Version/s: 0.7.0 Issue resolved by pull request 168 [https://github.com/apache/incubator-livy/pull/168] > RSCConf take no effect in RSCDriver > > > Key: LIVY-563 > URL: https://issues.apache.org/jira/browse/LIVY-563 > Project: Livy > Issue Type: Bug > Components: Server >Affects Versions: 0.5.0 >Reporter: holmes.infra >Priority: Major > Fix For: 0.7.0 > > Time Spent: 0.5h > Remaining Estimate: 0h > > When create InterActiveSessin, prepareBuilderProp did not add all > RSCConf(livy.rsc.) to BuilderProp. After RSCServer started, the livy.rsc.* > related configure which modified by user take no effect. For example, > livy.rsc.sql.num-rows and livy.rsc.rpc.max.size and so on > -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[incubator-livy] branch master updated: [LIVY-563] Propagate RSC configuration when creating sessions.
This is an automated email from the ASF dual-hosted git repository.
vanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-livy.git
The following commit(s) were added to refs/heads/master by this push:
new 5abc043 [LIVY-563] Propagate RSC configuration when creating sessions.
5abc043 is described below
commit 5abc043708b443ba36612a4f4e2f5137bd63621a
Author: Marcelo Vanzin
AuthorDate: Thu Apr 11 09:29:39 2019 -0700
[LIVY-563] Propagate RSC configuration when creating sessions.
Even though not all RSC configs apply to the remote driver, a few do,
so propagate all of them when starting a new session.
Includes new unit test.
Author: Marcelo Vanzin
Closes #168 from vanzin/LIVY-563.
---
rsc/src/main/java/org/apache/livy/rsc/RSCConf.java | 2 +-
.../livy/server/interactive/InteractiveSession.scala | 9 +
.../livy/server/interactive/InteractiveSessionSpec.scala | 14 +-
3 files changed, 23 insertions(+), 2 deletions(-)
diff --git a/rsc/src/main/java/org/apache/livy/rsc/RSCConf.java
b/rsc/src/main/java/org/apache/livy/rsc/RSCConf.java
index 7c76164..d2496b5 100644
--- a/rsc/src/main/java/org/apache/livy/rsc/RSCConf.java
+++ b/rsc/src/main/java/org/apache/livy/rsc/RSCConf.java
@@ -35,7 +35,7 @@ public class RSCConf extends ClientConf {
public static final String SPARK_CONF_PREFIX = "spark.";
public static final String LIVY_SPARK_PREFIX = SPARK_CONF_PREFIX +
"__livy__.";
- private static final String RSC_CONF_PREFIX = "livy.rsc.";
+ public static final String RSC_CONF_PREFIX = "livy.rsc.";
public static enum Entry implements ConfEntry {
CLIENT_ID("client.auth.id", null),
diff --git
a/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSession.scala
b/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSession.scala
index 6ec2d75..9529ed3 100644
---
a/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSession.scala
+++
b/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSession.scala
@@ -346,6 +346,15 @@ object InteractiveSession extends Logging {
mergeHiveSiteAndHiveDeps(sparkMajorVersion)
}
+// Pick all the RSC-specific configs that have not been explicitly set
otherwise, and
+// put them in the resulting properties, so that the remote driver can use
them.
+livyConf.iterator().asScala.foreach { e =>
+ val (key, value) = (e.getKey(), e.getValue())
+ if (key.startsWith(RSCConf.RSC_CONF_PREFIX) &&
!builderProperties.contains(key)) {
+builderProperties(key) = value
+ }
+}
+
builderProperties
}
}
diff --git
a/server/src/test/scala/org/apache/livy/server/interactive/InteractiveSessionSpec.scala
b/server/src/test/scala/org/apache/livy/server/interactive/InteractiveSessionSpec.scala
index 2a99abb..2e21483 100644
---
a/server/src/test/scala/org/apache/livy/server/interactive/InteractiveSessionSpec.scala
+++
b/server/src/test/scala/org/apache/livy/server/interactive/InteractiveSessionSpec.scala
@@ -130,7 +130,6 @@ class InteractiveSessionSpec extends FunSpec
"dummy.jar"))
}
-
it("should set rsc jars through livy conf") {
val rscJars = Set(
"dummy.jar",
@@ -177,6 +176,19 @@ class InteractiveSessionSpec extends FunSpec
session.state should (be(SessionState.Starting) or be(SessionState.Idle))
}
+it("should propagate RSC configuration properties") {
+ val livyConf = new LivyConf(false)
+.set(LivyConf.REPL_JARS, "dummy.jar")
+.set(RSCConf.Entry.SASL_QOP.key(), "foo")
+.set(RSCConf.Entry.RPC_CHANNEL_LOG_LEVEL.key(), "TRACE")
+.set(LivyConf.LIVY_SPARK_VERSION, sys.env("LIVY_SPARK_VERSION"))
+.set(LivyConf.LIVY_SPARK_SCALA_VERSION, "2.10")
+
+ val properties = InteractiveSession.prepareBuilderProp(Map.empty, Spark,
livyConf)
+ assert(properties(RSCConf.Entry.SASL_QOP.key()) === "foo")
+ assert(properties(RSCConf.Entry.RPC_CHANNEL_LOG_LEVEL.key()) === "TRACE")
+}
+
withSession("should execute `1 + 2` == 3") { session =>
val pyResult = executeStatement("1 + 2", Some("pyspark"))
pyResult should equal (Extraction.decompose(Map(
[jira] [Created] (LIVY-587) Remove Guava dependency
Marcelo Vanzin created LIVY-587: --- Summary: Remove Guava dependency Key: LIVY-587 URL: https://issues.apache.org/jira/browse/LIVY-587 Project: Livy Issue Type: Task Components: Core Affects Versions: 0.6.0 Reporter: Marcelo Vanzin It seems Guava has crept back into Livy at some point. Guava is kind of a pain to maintain and update. We should avoid using it, especially since it doesn't seem to be used for anything important. -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Resolved] (SPARK-27394) The staleness of UI may last minutes or hours when no tasks start or finish
[
https://issues.apache.org/jira/browse/SPARK-27394?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Marcelo Vanzin resolved SPARK-27394.
Resolution: Fixed
Fix Version/s: 3.0.0
> The staleness of UI may last minutes or hours when no tasks start or finish
> ---
>
> Key: SPARK-27394
> URL: https://issues.apache.org/jira/browse/SPARK-27394
> Project: Spark
> Issue Type: Bug
> Components: Web UI
>Affects Versions: 2.4.0, 2.4.1
>Reporter: Shixiong Zhu
>Assignee: Shixiong Zhu
>Priority: Major
> Fix For: 3.0.0
>
>
> Run the following codes on a cluster that has at least 2 cores.
> {code}
> sc.makeRDD(1 to 1000, 1000).foreach { i =>
> Thread.sleep(30)
> }
> {code}
> The jobs page will just show one running task.
> This is because when the second task event calls
> "AppStatusListener.maybeUpdate" for a job, it will just ignore since the gap
> between two events is smaller than `spark.ui.liveUpdate.period`.
> After the second task event, in the above case, because there won't be any
> other task events, the Spark UI will be always stale until the next task
> event gets fired (after 300 seconds).
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[spark] branch master updated: [SPARK-27394][WEBUI] Flush LiveEntity if necessary when receiving SparkListenerExecutorMetricsUpdate
This is an automated email from the ASF dual-hosted git repository.
vanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 5ff39cd [SPARK-27394][WEBUI] Flush LiveEntity if necessary when
receiving SparkListenerExecutorMetricsUpdate
5ff39cd is described below
commit 5ff39cd5ee92da0d08380c7a680d350ff6f4b5db
Author: Shixiong Zhu
AuthorDate: Tue Apr 9 08:26:00 2019 -0700
[SPARK-27394][WEBUI] Flush LiveEntity if necessary when receiving
SparkListenerExecutorMetricsUpdate
## What changes were proposed in this pull request?
This PR updates `AppStatusListener` to flush `LiveEntity` if necessary when
receiving `SparkListenerExecutorMetricsUpdate`. This will ensure the staleness
of Spark UI doesn't last more than the executor heartbeat interval.
## How was this patch tested?
The new unit test.
Closes #24303 from zsxwing/SPARK-27394.
Authored-by: Shixiong Zhu
Signed-off-by: Marcelo Vanzin
---
.../org/apache/spark/internal/config/Status.scala | 6
.../apache/spark/status/AppStatusListener.scala| 39 --
.../org/apache/spark/ui/UISeleniumSuite.scala | 33 --
docs/configuration.md | 8 +
4 files changed, 73 insertions(+), 13 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/internal/config/Status.scala
b/core/src/main/scala/org/apache/spark/internal/config/Status.scala
index c561572..3e6a4e9 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Status.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Status.scala
@@ -29,6 +29,12 @@ private[spark] object Status {
.timeConf(TimeUnit.NANOSECONDS)
.createWithDefaultString("100ms")
+ val LIVE_ENTITY_UPDATE_MIN_FLUSH_PERIOD =
ConfigBuilder("spark.ui.liveUpdate.minFlushPeriod")
+.doc("Minimum time elapsed before stale UI data is flushed. This avoids UI
staleness when " +
+ "incoming task events are not fired frequently.")
+.timeConf(TimeUnit.NANOSECONDS)
+.createWithDefaultString("1s")
+
val MAX_RETAINED_JOBS = ConfigBuilder("spark.ui.retainedJobs")
.intConf
.createWithDefault(1000)
diff --git
a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
index a3e8242..b085f21 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
@@ -58,6 +58,12 @@ private[spark] class AppStatusListener(
// operations that we can live without when rapidly processing incoming task
events.
private val liveUpdatePeriodNs = if (live)
conf.get(LIVE_ENTITY_UPDATE_PERIOD) else -1L
+ /**
+ * Minimum time elapsed before stale UI data is flushed. This avoids UI
staleness when incoming
+ * task events are not fired frequently.
+ */
+ private val liveUpdateMinFlushPeriod =
conf.get(LIVE_ENTITY_UPDATE_MIN_FLUSH_PERIOD)
+
private val maxTasksPerStage = conf.get(MAX_RETAINED_TASKS_PER_STAGE)
private val maxGraphRootNodes = conf.get(MAX_RETAINED_ROOT_NODES)
@@ -76,6 +82,9 @@ private[spark] class AppStatusListener(
// around liveExecutors.
@volatile private var activeExecutorCount = 0
+ /** The last time when flushing `LiveEntity`s. This is to avoid flushing too
frequently. */
+ private var lastFlushTimeNs = System.nanoTime()
+
kvstore.addTrigger(classOf[ExecutorSummaryWrapper],
conf.get(MAX_RETAINED_DEAD_EXECUTORS))
{ count => cleanupExecutors(count) }
@@ -89,7 +98,8 @@ private[spark] class AppStatusListener(
kvstore.onFlush {
if (!live) {
- flush()
+ val now = System.nanoTime()
+ flush(update(_, now))
}
}
@@ -831,6 +841,14 @@ private[spark] class AppStatusListener(
}
}
}
+// Flush updates if necessary. Executor heartbeat is an event that happens
periodically. Flush
+// here to ensure the staleness of Spark UI doesn't last more than
+// `max(heartbeat interval, liveUpdateMinFlushPeriod)`.
+if (now - lastFlushTimeNs > liveUpdateMinFlushPeriod) {
+ flush(maybeUpdate(_, now))
+ // Re-get the current system time because `flush` may be slow and `now`
is stale.
+ lastFlushTimeNs = System.nanoTime()
+}
}
override def onStageExecutorMetrics(executorMetrics:
SparkListenerStageExecutorMetrics): Unit = {
@@ -856,18 +874,17 @@ private[spark] class AppStatusListener(
}
}
- /** Flush all live entities' data to the underlying store. */
- private def flush(): Unit = {
-val now = System.nanoTime()
+ /** Go through all `LiveEntity`s and use `entityFlushFunc(entity)` to flush
them. */
+ private def flush(entityFlushFunc: LiveEntity =
Re: Spark 2.4.0 tests fail with hadoop-3.1 profile: NoClassDefFoundError org.apache.hadoop.hive.conf.HiveConf
You can always try. But Hadoop 3 is not yet supported by Spark. On Fri, Apr 5, 2019 at 11:13 AM Anton Kirillov wrote: > > Marcelo, Sean, thanks for the clarification. So in order to support Hadoop 3+ > the preferred way would be to use Hadoop-free builds and provide Hadoop > dependencies in the classpath, is that correct? > > On Fri, Apr 5, 2019 at 10:57 AM Marcelo Vanzin wrote: >> >> The hadoop-3 profile doesn't really work yet, not even on master. >> That's being worked on still. >> >> On Fri, Apr 5, 2019 at 10:53 AM akirillov >> wrote: >> > >> > Hi there! I'm trying to run Spark unit tests with the following profiles: >> > >> > And 'core' module fails with the following test failing with >> > NoClassDefFoundError: >> > >> > In the meantime building a distribution works fine when running: >> > >> > Also, there are no problems with running tests using Hadoop 2.7 profile. >> > Does this issue look familiar? Any help appreciated! >> > >> > >> > >> > -- >> > Sent from: http://apache-spark-developers-list.1001551.n3.nabble.com/ >> > >> > - >> > To unsubscribe e-mail: dev-unsubscr...@spark.apache.org >> > >> >> >> -- >> Marcelo -- Marcelo - To unsubscribe e-mail: dev-unsubscr...@spark.apache.org
Re: Spark 2.4.0 tests fail with hadoop-3.1 profile: NoClassDefFoundError org.apache.hadoop.hive.conf.HiveConf
The hadoop-3 profile doesn't really work yet, not even on master. That's being worked on still. On Fri, Apr 5, 2019 at 10:53 AM akirillov wrote: > > Hi there! I'm trying to run Spark unit tests with the following profiles: > > And 'core' module fails with the following test failing with > NoClassDefFoundError: > > In the meantime building a distribution works fine when running: > > Also, there are no problems with running tests using Hadoop 2.7 profile. > Does this issue look familiar? Any help appreciated! > > > > -- > Sent from: http://apache-spark-developers-list.1001551.n3.nabble.com/ > > - > To unsubscribe e-mail: dev-unsubscr...@spark.apache.org > -- Marcelo - To unsubscribe e-mail: dev-unsubscr...@spark.apache.org
[jira] [Assigned] (LIVY-489) Expose a JDBC endpoint for Livy
[ https://issues.apache.org/jira/browse/LIVY-489?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin reassigned LIVY-489: --- Assignee: Marco Gaido > Expose a JDBC endpoint for Livy > --- > > Key: LIVY-489 > URL: https://issues.apache.org/jira/browse/LIVY-489 > Project: Livy > Issue Type: New Feature > Components: API, Server >Affects Versions: 0.6.0 >Reporter: Marco Gaido >Assignee: Marco Gaido >Priority: Major > Fix For: 0.6.0 > > > Many users and BI tools use JDBC connections in order to retrieve data. As > Livy exposes only a REST API, this is a limitation in its adoption. Hence, > adding a JDBC endpoint may be a very useful feature, which could also make > Livy a more attractive solution for end user to adopt. > Moreover, currently, Spark exposes a JDBC interface, but this has many > limitations, including that all the queries are submitted to the same > application, therefore there is no isolation/security, which can be offered > by Livy, making a Livy JDBC API a better solution for companies/users who > want to use Spark in order to run they queries through JDBC. > In order to make the transition from existing solutions to the new JDBC > server seamless, the proposal is to use the Hive thrift-server and extend it > as it was done by the STS. > [Here, you can find the design > doc.|https://docs.google.com/document/d/18HAR_VnQLegbYyzGg8f4zwD4GtDP5q_t3K21eXecZC4/edit] > -- This message was sent by Atlassian JIRA (v7.6.3#76005)
Re: [ANNOUNCE] Apache Livy 0.6.0-incubating released
They are, it just takes a while for mirrors to pick them up. https://repository.apache.org/content/groups/public/org/apache/livy/livy-api/ On Wed, Apr 3, 2019 at 10:07 AM kant kodali wrote: > > Looks like the maven artifacts are not published yet. > > https://mvnrepository.com/artifact/org.apache.livy/livy-api > > On Wed, Apr 3, 2019 at 9:54 AM Marcelo Vanzin wrote: >> >> I closed the bug and will fix the website link. Thanks for pointing it out! >> >> On Wed, Apr 3, 2019 at 4:55 AM Alexey Romanenko >> wrote: >> > >> > Great, thanks to all who worked on this! >> > >> > Just a couple of minor notes I noticed: >> > - Page http://livy.incubator.apache.org/history/ , wrong link >> > "0.6.0-incubating”, it leads to 0.5.0 tag instead of 0.6.0 >> > - LIVY-489 (umbrella jira for Hive-compatible JDBC / ODBC server feature) >> > is not resolved and not linked to 0.6.0 release. Do we need to properly >> > close this jira? >> > >> > Alexey >> > >> > On 3 Apr 2019, at 07:05, Jeff Zhang wrote: >> > >> > Thanks Marcelo >> > >> > >> > 于2019年4月3日周三 下午12:15写道： >> >> >> >> Nice!! >> >> >> >> Sent from my iPhone >> >> >> >> On Apr 2, 2019, at 8:21 PM, Ruslan Dautkhanov >> >> wrote: >> >> >> >> Thanks a lot Marcelo ! >> >> >> >> Ruslan >> >> >> >> >> >> >> >> On Tue, Apr 2, 2019 at 12:24 PM Marcelo Vanzin >> >> wrote: >> >>> >> >>> The Apache Livy team is proud to announce the release of Apache Livy >> >>> 0.6.0-incubating. >> >>> >> >>> Livy is web service that exposes a REST interface for managing long >> >>> running Apache Spark contexts in your cluster. Livy enables >> >>> programmatic, fault-tolerant, multi-tenant submission of Spark jobs >> >>> from web/mobile apps (no Spark client needed). So, multiple users can >> >>> interact with your Spark cluster concurrently and reliably. >> >>> >> >>> Download Apache Livy 0.6.0-incubating: >> >>> http://livy.incubator.apache.org/download/ >> >>> >> >>> Release Notes: >> >>> http://livy.incubator.apache.org/history/ >> >>> >> >>> For more about Livy check our website: >> >>> http://livy.incubator.apache.org/ >> >>> >> >>> We would like to thank the contributors that made the release possible! >> >>> >> >>> >> >>> -- >> >>> Marcelo >> >> >> >> -- >> >> >> >> -- >> >> Ruslan Dautkhanov >> > >> > >> > >> > -- >> > Best Regards >> > >> > Jeff Zhang >> > >> > >> >> >> -- >> Marcelo -- Marcelo
Re: [ANNOUNCE] Apache Livy 0.6.0-incubating released
I closed the bug and will fix the website link. Thanks for pointing it out! On Wed, Apr 3, 2019 at 4:55 AM Alexey Romanenko wrote: > > Great, thanks to all who worked on this! > > Just a couple of minor notes I noticed: > - Page http://livy.incubator.apache.org/history/ , wrong link > "0.6.0-incubating”, it leads to 0.5.0 tag instead of 0.6.0 > - LIVY-489 (umbrella jira for Hive-compatible JDBC / ODBC server feature) is > not resolved and not linked to 0.6.0 release. Do we need to properly close > this jira? > > Alexey > > On 3 Apr 2019, at 07:05, Jeff Zhang wrote: > > Thanks Marcelo > > > 于2019年4月3日周三 下午12:15写道： >> >> Nice!! >> >> Sent from my iPhone >> >> On Apr 2, 2019, at 8:21 PM, Ruslan Dautkhanov wrote: >> >> Thanks a lot Marcelo ! >> >> Ruslan >> >> >> >> On Tue, Apr 2, 2019 at 12:24 PM Marcelo Vanzin wrote: >>> >>> The Apache Livy team is proud to announce the release of Apache Livy >>> 0.6.0-incubating. >>> >>> Livy is web service that exposes a REST interface for managing long >>> running Apache Spark contexts in your cluster. Livy enables >>> programmatic, fault-tolerant, multi-tenant submission of Spark jobs >>> from web/mobile apps (no Spark client needed). So, multiple users can >>> interact with your Spark cluster concurrently and reliably. >>> >>> Download Apache Livy 0.6.0-incubating: >>> http://livy.incubator.apache.org/download/ >>> >>> Release Notes: >>> http://livy.incubator.apache.org/history/ >>> >>> For more about Livy check our website: >>> http://livy.incubator.apache.org/ >>> >>> We would like to thank the contributors that made the release possible! >>> >>> >>> -- >>> Marcelo >> >> -- >> >> -- >> Ruslan Dautkhanov > > > > -- > Best Regards > > Jeff Zhang > > -- Marcelo
[incubator-livy-website] branch master updated: Fix 0.6 tag link.
This is an automated email from the ASF dual-hosted git repository.
vanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-livy-website.git
The following commit(s) were added to refs/heads/master by this push:
new f40a695 Fix 0.6 tag link.
f40a695 is described below
commit f40a6959add52283fd6f323bd16f94377a16e728
Author: Marcelo Vanzin
AuthorDate: Wed Apr 3 09:50:37 2019 -0700
Fix 0.6 tag link.
---
site/history.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/site/history.md b/site/history.md
index 1a6b3e8..26583ed 100644
--- a/site/history.md
+++ b/site/history.md
@@ -32,7 +32,7 @@ For a full list of releases, see
Downloads are available on the
[downloads page]({{ site.baseurl }}/download).
-## https://github.com/apache/{{ site.data.project.incubator_name
}}/releases/tag/v0.5.0-incubating">0.6.0-incubating / 2019-04-01
+## https://github.com/apache/{{ site.data.project.incubator_name
}}/releases/tag/v0.6.0-incubating">0.6.0-incubating / 2019-04-01
{: #v0-6-0-incubating}
New features
[incubator-livy-website] branch asf-site updated: Publishing from f40a6959add52283fd6f323bd16f94377a16e728
This is an automated email from the ASF dual-hosted git repository. vanzin pushed a commit to branch asf-site in repository https://gitbox.apache.org/repos/asf/incubator-livy-website.git The following commit(s) were added to refs/heads/asf-site by this push: new 024e58f Publishing from f40a6959add52283fd6f323bd16f94377a16e728 024e58f is described below commit 024e58fe0e594d4950611688d429f24ff1c0f7d4 Author: Marcelo Vanzin AuthorDate: Wed Apr 3 09:51:27 2019 -0700 Publishing from f40a6959add52283fd6f323bd16f94377a16e728 --- content/feed.xml | 4 ++-- content/history/index.html | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/content/feed.xml b/content/feed.xml index bfead6c..19e4d93 100644 --- a/content/feed.xml +++ b/content/feed.xml @@ -5,8 +5,8 @@ / -Tue, 02 Apr 2019 11:14:18 -0700 -Tue, 02 Apr 2019 11:14:18 -0700 +Wed, 03 Apr 2019 09:51:26 -0700 +Wed, 03 Apr 2019 09:51:26 -0700 Jekyll v3.4.5 diff --git a/content/history/index.html b/content/history/index.html index d7fdf24..30f3a8b 100644 --- a/content/history/index.html +++ b/content/history/index.html @@ -162,7 +162,7 @@ Downloads are available on the downloads page. -https://github.com/apache/incubator-livy/releases/tag/v0.5.0-incubating";>0.6.0-incubating / 2019-04-01 +https://github.com/apache/incubator-livy/releases/tag/v0.6.0-incubating";>0.6.0-incubating / 2019-04-01 New features
[jira] [Resolved] (LIVY-489) Expose a JDBC endpoint for Livy
[ https://issues.apache.org/jira/browse/LIVY-489?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin resolved LIVY-489. - Resolution: Done Fix Version/s: 0.6.0 > Expose a JDBC endpoint for Livy > --- > > Key: LIVY-489 > URL: https://issues.apache.org/jira/browse/LIVY-489 > Project: Livy > Issue Type: New Feature > Components: API, Server >Affects Versions: 0.6.0 >Reporter: Marco Gaido >Priority: Major > Fix For: 0.6.0 > > > Many users and BI tools use JDBC connections in order to retrieve data. As > Livy exposes only a REST API, this is a limitation in its adoption. Hence, > adding a JDBC endpoint may be a very useful feature, which could also make > Livy a more attractive solution for end user to adopt. > Moreover, currently, Spark exposes a JDBC interface, but this has many > limitations, including that all the queries are submitted to the same > application, therefore there is no isolation/security, which can be offered > by Livy, making a Livy JDBC API a better solution for companies/users who > want to use Spark in order to run they queries through JDBC. > In order to make the transition from existing solutions to the new JDBC > server seamless, the proposal is to use the Hive thrift-server and extend it > as it was done by the STS. > [Here, you can find the design > doc.|https://docs.google.com/document/d/18HAR_VnQLegbYyzGg8f4zwD4GtDP5q_t3K21eXecZC4/edit] > -- This message was sent by Atlassian JIRA (v7.6.3#76005)
[jira] [Resolved] (SPARK-11702) Guava ClassLoading Issue When Using Different Hive Metastore Version
[
https://issues.apache.org/jira/browse/SPARK-11702?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Marcelo Vanzin resolved SPARK-11702.
Resolution: Cannot Reproduce
Pretty sure this does not happen in the 2.x line.
> Guava ClassLoading Issue When Using Different Hive Metastore Version
>
>
> Key: SPARK-11702
> URL: https://issues.apache.org/jira/browse/SPARK-11702
> Project: Spark
> Issue Type: Bug
> Components: Spark Core
>Affects Versions: 1.5.1
>Reporter: Joey Paskhay
>Priority: Major
>
> A Guava classloading error can occur when using a different version of the
> Hive metastore.
> Running the latest version of Spark at this time (1.5.1) and patched versions
> of Hadoop 2.2.0 and Hive 1.0.0. We set "spark.sql.hive.metastore.version" to
> "1.0.0" and "spark.sql.hive.metastore.jars" to
> "/lib/*:". When trying to
> launch the spark-shell, the sqlContext would fail to initialize with:
> {code}
> java.lang.ClassNotFoundException: java.lang.NoClassDefFoundError:
> com/google/common/base/Predicate when creating Hive client using classpath:
>
> Please make sure that jars for your version of hive and hadoop are included
> in the paths passed to SQLConfEntry(key = spark.sql.hive.metastore.jars,
> defaultValue=builtin, doc=...
> {code}
> We verified the Guava libraries are in the huge list of the included jars,
> but we saw that in the
> org.apache.spark.sql.hive.client.IsolatedClientLoader.isSharedClass method it
> seems to assume that *all* "com.google" (excluding "com.google.cloud")
> classes should be loaded from the base class loader. The Spark libraries seem
> to have *some* "com.google.common.base" classes shaded in but not all.
> See
> [https://mail-archives.apache.org/mod_mbox/spark-user/201511.mbox/%3CCAB51Vx4ipV34e=eishlg7bzldm0uefd_mpyqfe4dodbnbv9...@mail.gmail.com%3E]
> and its replies.
> The work-around is to add the guava JAR to the "spark.driver.extraClassPath"
> and "spark.executor.extraClassPath" properties.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Resolved] (SPARK-25762) Upgrade guava version in spark dependency lists due to CVE issue
[ https://issues.apache.org/jira/browse/SPARK-25762?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin resolved SPARK-25762. Resolution: Duplicate Target Version/s: (was: 3.0.0) > Upgrade guava version in spark dependency lists due to CVE issue > - > > Key: SPARK-25762 > URL: https://issues.apache.org/jira/browse/SPARK-25762 > Project: Spark > Issue Type: Dependency upgrade > Components: Spark Core >Affects Versions: 2.2.1, 2.2.2, 2.3.1, 2.3.2 >Reporter: Debojyoti >Priority: Major > > In spark2.x dependency list we have guava-14.0.1.jar. However there are lot > vulnerabilities exists in this version.eg. CVE-2018-10237 > [https://www.cvedetails.com/cve/CVE-2018-10237/] > Do we have any solution to resolve it or is there any plan to upgrade guava > version any of the spark's future release? -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
Re: [ANNOUNCE] Apache Livy 0.6.0-incubating released
(Apologies to all, this is the same announcement as previously, but including the missing disclaimer.) The Apache Livy team is proud to announce the release of Apache Livy 0.6.0-incubating. Livy is web service that exposes a REST interface for managing long running Apache Spark contexts in your cluster. Livy enables programmatic, fault-tolerant, multi-tenant submission of Spark jobs from web/mobile apps (no Spark client needed). So, multiple users can interact with your Spark cluster concurrently and reliably. Download Apache Livy 0.6.0-incubating: http://livy.incubator.apache.org/download/ Release Notes: http://livy.incubator.apache.org/history/ For more about Livy check our website: http://livy.incubator.apache.org/ We would like to thank the contributors that made the release possible! = *Disclaimer* Apache Livy (incubating) is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the name of Apache Incubator PMC. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF. -- Marcelo - To unsubscribe, e-mail: general-unsubscr...@incubator.apache.org For additional commands, e-mail: general-h...@incubator.apache.org
Re: [ANNOUNCE] Apache Livy 0.6.0-incubating released
(Apologies to all, this is the same announcement as previously, but including the missing disclaimer.) The Apache Livy team is proud to announce the release of Apache Livy 0.6.0-incubating. Livy is web service that exposes a REST interface for managing long running Apache Spark contexts in your cluster. Livy enables programmatic, fault-tolerant, multi-tenant submission of Spark jobs from web/mobile apps (no Spark client needed). So, multiple users can interact with your Spark cluster concurrently and reliably. Download Apache Livy 0.6.0-incubating: http://livy.incubator.apache.org/download/ Release Notes: http://livy.incubator.apache.org/history/ For more about Livy check our website: http://livy.incubator.apache.org/ We would like to thank the contributors that made the release possible! = *Disclaimer* Apache Livy (incubating) is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the name of Apache Incubator PMC. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF. -- Marcelo
Re: [ANNOUNCE] Apache Livy 0.6.0-incubating released
(Apologies to all, this is the same announcement as previously, but including the missing disclaimer.) The Apache Livy team is proud to announce the release of Apache Livy 0.6.0-incubating. Livy is web service that exposes a REST interface for managing long running Apache Spark contexts in your cluster. Livy enables programmatic, fault-tolerant, multi-tenant submission of Spark jobs from web/mobile apps (no Spark client needed). So, multiple users can interact with your Spark cluster concurrently and reliably. Download Apache Livy 0.6.0-incubating: http://livy.incubator.apache.org/download/ Release Notes: http://livy.incubator.apache.org/history/ For more about Livy check our website: http://livy.incubator.apache.org/ We would like to thank the contributors that made the release possible! = *Disclaimer* Apache Livy (incubating) is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the name of Apache Incubator PMC. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF. -- Marcelo
Re: [ANNOUNCE] Apache Livy 0.6.0-incubating released
Oops, my bad. Must have missed it when I copy & pasted a previous announcement. Will send an amended e-mail. On Tue, Apr 2, 2019 at 12:03 PM sebb wrote: > > On Tue, 2 Apr 2019 at 19:28, Marcelo Vanzin > wrote: > > > > The Apache Livy team is proud to announce the release of Apache Livy > > 0.6.0-incubating. > > > > Livy is web service that exposes a REST interface for managing long > > running Apache Spark contexts in your cluster. Livy enables > > programmatic, fault-tolerant, multi-tenant submission of Spark jobs > > from web/mobile apps (no Spark client needed). So, multiple users can > > interact with your Spark cluster concurrently and reliably. > > > > Download Apache Livy 0.6.0-incubating: > > http://livy.incubator.apache.org/download/ > > > > Release Notes: > > http://livy.incubator.apache.org/history/ > > > > For more about Livy check our website: > > http://livy.incubator.apache.org/ > > > > We would like to thank the contributors that made the release possible! > > This is an incubator release, and should include the standard > incubator disclaimer. > > See for example: > > https://lists.apache.org/thread.html/d94b89e6c4e83f2364607b47020ca715aaa7fae1a3b97b374b93ca7f@%3Cgeneral.incubator.apache.org%3E > https://lists.apache.org/thread.html/ec3d6b90a2b0054eb2f0a2e4ba2e09a594463720e2b28e75b315c693@%3Cgeneral.incubator.apache.org%3E > https://lists.apache.org/thread.html/8993ecb33e0f73e69faabba8c1f07d3a9965a178df1698c42fc9f4b2@%3Cgeneral.incubator.apache.org%3E > > Thanks. -- Marcelo
[ANNOUNCE] Apache Livy 0.6.0-incubating released
The Apache Livy team is proud to announce the release of Apache Livy 0.6.0-incubating. Livy is web service that exposes a REST interface for managing long running Apache Spark contexts in your cluster. Livy enables programmatic, fault-tolerant, multi-tenant submission of Spark jobs from web/mobile apps (no Spark client needed). So, multiple users can interact with your Spark cluster concurrently and reliably. Download Apache Livy 0.6.0-incubating: http://livy.incubator.apache.org/download/ Release Notes: http://livy.incubator.apache.org/history/ For more about Livy check our website: http://livy.incubator.apache.org/ We would like to thank the contributors that made the release possible! -- Marcelo - To unsubscribe, e-mail: general-unsubscr...@incubator.apache.org For additional commands, e-mail: general-h...@incubator.apache.org
[ANNOUNCE] Apache Livy 0.6.0-incubating released
The Apache Livy team is proud to announce the release of Apache Livy 0.6.0-incubating. Livy is web service that exposes a REST interface for managing long running Apache Spark contexts in your cluster. Livy enables programmatic, fault-tolerant, multi-tenant submission of Spark jobs from web/mobile apps (no Spark client needed). So, multiple users can interact with your Spark cluster concurrently and reliably. Download Apache Livy 0.6.0-incubating: http://livy.incubator.apache.org/download/ Release Notes: http://livy.incubator.apache.org/history/ For more about Livy check our website: http://livy.incubator.apache.org/ We would like to thank the contributors that made the release possible! -- Marcelo
[ANNOUNCE] Apache Livy 0.6.0-incubating released
The Apache Livy team is proud to announce the release of Apache Livy 0.6.0-incubating. Livy is web service that exposes a REST interface for managing long running Apache Spark contexts in your cluster. Livy enables programmatic, fault-tolerant, multi-tenant submission of Spark jobs from web/mobile apps (no Spark client needed). So, multiple users can interact with your Spark cluster concurrently and reliably. Download Apache Livy 0.6.0-incubating: http://livy.incubator.apache.org/download/ Release Notes: http://livy.incubator.apache.org/history/ For more about Livy check our website: http://livy.incubator.apache.org/ We would like to thank the contributors that made the release possible! -- Marcelo
[spark] branch branch-2.3 updated: [SPARK-26998][CORE] Remove SSL configuration from executors
This is an automated email from the ASF dual-hosted git repository.
vanzin pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.3 by this push:
new 96c2c3b [SPARK-26998][CORE] Remove SSL configuration from executors
96c2c3b is described below
commit 96c2c3b4ddc2911a8c7f93bd0fbe6b42e9ac5a39
Author: Gabor Somogyi
AuthorDate: Tue Apr 2 09:18:43 2019 -0700
[SPARK-26998][CORE] Remove SSL configuration from executors
## What changes were proposed in this pull request?
Different SSL passwords shown up as command line argument on executor side
in standalone mode:
* keyStorePassword
* keyPassword
* trustStorePassword
In this PR I've removed SSL configurations from executors.
## How was this patch tested?
Existing + additional unit tests.
Additionally tested with standalone mode and checked the command line
arguments:
```
[gaborsomogyi:~/spark] SPARK-26998(+4/-0,3)+ ± jps
94803 CoarseGrainedExecutorBackend
94818 Jps
90149 RemoteMavenServer
91925 Nailgun
94793 SparkSubmit
94680 Worker
94556 Master
398
[gaborsomogyi:~/spark] SPARK-26998(+4/-1,3)+ ± ps -ef | egrep
"94556|94680|94793|94803"
502 94556 1 0 2:02PM ttys0070:07.39
/Library/Java/JavaVirtualMachines/jdk1.8.0_152.jdk/Contents/Home/bin/java -cp
/Users/gaborsomogyi/spark/conf/:/Users/gaborsomogyi/spark/assembly/target/scala-2.12/jars/*
-Xmx1g org.apache.spark.deploy.master.Master --host gsomogyi-MBP.local --port
7077 --webui-port 8080 --properties-file conf/spark-defaults.conf
502 94680 1 0 2:02PM ttys0070:07.27
/Library/Java/JavaVirtualMachines/jdk1.8.0_152.jdk/Contents/Home/bin/java -cp
/Users/gaborsomogyi/spark/conf/:/Users/gaborsomogyi/spark/assembly/target/scala-2.12/jars/*
-Xmx1g org.apache.spark.deploy.worker.Worker --webui-port 8081
--properties-file conf/spark-defaults.conf spark://gsomogyi-MBP.local:7077
502 94793 94782 0 2:02PM ttys0070:35.52
/Library/Java/JavaVirtualMachines/jdk1.8.0_152.jdk/Contents/Home/bin/java -cp
/Users/gaborsomogyi/spark/conf/:/Users/gaborsomogyi/spark/assembly/target/scala-2.12/jars/*
-Dscala.usejavacp=true -Xmx1g org.apache.spark.deploy.SparkSubmit --master
spark://gsomogyi-MBP.local:7077 --class org.apache.spark.repl.Main --name Spark
shell spark-shell
502 94803 94680 0 2:03PM ttys0070:05.20
/Library/Java/JavaVirtualMachines/jdk1.8.0_152.jdk/Contents/Home/bin/java -cp
/Users/gaborsomogyi/spark/conf/:/Users/gaborsomogyi/spark/assembly/target/scala-2.12/jars/*
-Xmx1024M -Dspark.ssl.ui.port=0 -Dspark.driver.port=60902
org.apache.spark.executor.CoarseGrainedExecutorBackend --driver-url
spark://CoarseGrainedScheduler172.30.65.186:60902 --executor-id 0 --hostname
172.30.65.186 --cores 8 --app-id app-20190326140311- --worker-u [...]
502 94910 57352 0 2:05PM ttys0080:00.00 egrep
94556|94680|94793|94803
```
Closes #24170 from gaborgsomogyi/SPARK-26998.
Authored-by: Gabor Somogyi
Signed-off-by: Marcelo Vanzin
(cherry picked from commit 57aff93886ac7d02b88294672ce0d2495b0942b8)
Signed-off-by: Marcelo Vanzin
---
core/src/main/scala/org/apache/spark/SparkConf.scala | 1 -
core/src/test/scala/org/apache/spark/SparkConfSuite.scala | 11 +++
2 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala
b/core/src/main/scala/org/apache/spark/SparkConf.scala
index cfceedf..5670f7d 100644
--- a/core/src/main/scala/org/apache/spark/SparkConf.scala
+++ b/core/src/main/scala/org/apache/spark/SparkConf.scala
@@ -703,7 +703,6 @@ private[spark] object SparkConf extends Logging {
*/
def isExecutorStartupConf(name: String): Boolean = {
(name.startsWith("spark.auth") && name !=
SecurityManager.SPARK_AUTH_SECRET_CONF) ||
-name.startsWith("spark.ssl") ||
name.startsWith("spark.rpc") ||
name.startsWith("spark.network") ||
isSparkPortConf(name)
diff --git a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
index 1f68a0d..cc67b7f 100644
--- a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
@@ -339,6 +339,17 @@ class SparkConfSuite extends SparkFunSuite with
LocalSparkContext with ResetSyst
}
}
+ test("SPARK-26998: SSL configuration not needed on executors") {
+val conf = new SparkConf(false)
+conf.set("spark.ssl.enabled", "true")
+conf.set("spark.ssl.keyPassword", "password")
+conf.set("spark.ssl.keyStorePassword", "password")
+conf.set("spark.ssl.trustStorePassword", "password")
[jira] [Resolved] (SPARK-26998) spark.ssl.keyStorePassword in plaintext on 'ps -ef' output of executor processes in Standalone mode
[ https://issues.apache.org/jira/browse/SPARK-26998?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin resolved SPARK-26998. Resolution: Fixed Fix Version/s: 2.3.4 2.4.2 3.0.0 Issue resolved by pull request 24170 [https://github.com/apache/spark/pull/24170] > spark.ssl.keyStorePassword in plaintext on 'ps -ef' output of executor > processes in Standalone mode > --- > > Key: SPARK-26998 > URL: https://issues.apache.org/jira/browse/SPARK-26998 > Project: Spark > Issue Type: Bug > Components: Scheduler, Security, Spark Core >Affects Versions: 2.3.3, 2.4.0 >Reporter: t oo >Assignee: Gabor Somogyi >Priority: Major > Labels: SECURITY, Security, secur, security, security-issue > Fix For: 3.0.0, 2.4.2, 2.3.4 > > > Run spark standalone mode, then start a spark-submit requiring at least 1 > executor. Do a 'ps -ef' on linux (ie putty terminal) and you will be able to > see spark.ssl.keyStorePassword value in plaintext! > > spark.ssl.keyStorePassword and spark.ssl.keyPassword don't need to be passed > to CoarseGrainedExecutorBackend. Only spark.ssl.trustStorePassword is used. > > Can be resolved if below PR is merged: > [[Github] Pull Request #21514 > (tooptoop4)|https://github.com/apache/spark/pull/21514] -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Assigned] (SPARK-26998) spark.ssl.keyStorePassword in plaintext on 'ps -ef' output of executor processes in Standalone mode
[ https://issues.apache.org/jira/browse/SPARK-26998?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Marcelo Vanzin reassigned SPARK-26998: -- Assignee: Gabor Somogyi > spark.ssl.keyStorePassword in plaintext on 'ps -ef' output of executor > processes in Standalone mode > --- > > Key: SPARK-26998 > URL: https://issues.apache.org/jira/browse/SPARK-26998 > Project: Spark > Issue Type: Bug > Components: Scheduler, Security, Spark Core >Affects Versions: 2.3.3, 2.4.0 >Reporter: t oo >Assignee: Gabor Somogyi >Priority: Major > Labels: SECURITY, Security, secur, security, security-issue > > Run spark standalone mode, then start a spark-submit requiring at least 1 > executor. Do a 'ps -ef' on linux (ie putty terminal) and you will be able to > see spark.ssl.keyStorePassword value in plaintext! > > spark.ssl.keyStorePassword and spark.ssl.keyPassword don't need to be passed > to CoarseGrainedExecutorBackend. Only spark.ssl.trustStorePassword is used. > > Can be resolved if below PR is merged: > [[Github] Pull Request #21514 > (tooptoop4)|https://github.com/apache/spark/pull/21514] -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[spark] branch branch-2.4 updated: [SPARK-26998][CORE] Remove SSL configuration from executors
This is an automated email from the ASF dual-hosted git repository.
vanzin pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 55e6f7a [SPARK-26998][CORE] Remove SSL configuration from executors
55e6f7a is described below
commit 55e6f7ab2cc5b9e1a816929504471390fd0872c2
Author: Gabor Somogyi
AuthorDate: Tue Apr 2 09:18:43 2019 -0700
[SPARK-26998][CORE] Remove SSL configuration from executors
## What changes were proposed in this pull request?
Different SSL passwords shown up as command line argument on executor side
in standalone mode:
* keyStorePassword
* keyPassword
* trustStorePassword
In this PR I've removed SSL configurations from executors.
## How was this patch tested?
Existing + additional unit tests.
Additionally tested with standalone mode and checked the command line
arguments:
```
[gaborsomogyi:~/spark] SPARK-26998(+4/-0,3)+ ± jps
94803 CoarseGrainedExecutorBackend
94818 Jps
90149 RemoteMavenServer
91925 Nailgun
94793 SparkSubmit
94680 Worker
94556 Master
398
[gaborsomogyi:~/spark] SPARK-26998(+4/-1,3)+ ± ps -ef | egrep
"94556|94680|94793|94803"
502 94556 1 0 2:02PM ttys0070:07.39
/Library/Java/JavaVirtualMachines/jdk1.8.0_152.jdk/Contents/Home/bin/java -cp
/Users/gaborsomogyi/spark/conf/:/Users/gaborsomogyi/spark/assembly/target/scala-2.12/jars/*
-Xmx1g org.apache.spark.deploy.master.Master --host gsomogyi-MBP.local --port
7077 --webui-port 8080 --properties-file conf/spark-defaults.conf
502 94680 1 0 2:02PM ttys0070:07.27
/Library/Java/JavaVirtualMachines/jdk1.8.0_152.jdk/Contents/Home/bin/java -cp
/Users/gaborsomogyi/spark/conf/:/Users/gaborsomogyi/spark/assembly/target/scala-2.12/jars/*
-Xmx1g org.apache.spark.deploy.worker.Worker --webui-port 8081
--properties-file conf/spark-defaults.conf spark://gsomogyi-MBP.local:7077
502 94793 94782 0 2:02PM ttys0070:35.52
/Library/Java/JavaVirtualMachines/jdk1.8.0_152.jdk/Contents/Home/bin/java -cp
/Users/gaborsomogyi/spark/conf/:/Users/gaborsomogyi/spark/assembly/target/scala-2.12/jars/*
-Dscala.usejavacp=true -Xmx1g org.apache.spark.deploy.SparkSubmit --master
spark://gsomogyi-MBP.local:7077 --class org.apache.spark.repl.Main --name Spark
shell spark-shell
502 94803 94680 0 2:03PM ttys0070:05.20
/Library/Java/JavaVirtualMachines/jdk1.8.0_152.jdk/Contents/Home/bin/java -cp
/Users/gaborsomogyi/spark/conf/:/Users/gaborsomogyi/spark/assembly/target/scala-2.12/jars/*
-Xmx1024M -Dspark.ssl.ui.port=0 -Dspark.driver.port=60902
org.apache.spark.executor.CoarseGrainedExecutorBackend --driver-url
spark://CoarseGrainedScheduler172.30.65.186:60902 --executor-id 0 --hostname
172.30.65.186 --cores 8 --app-id app-20190326140311- --worker-u [...]
502 94910 57352 0 2:05PM ttys0080:00.00 egrep
94556|94680|94793|94803
```
Closes #24170 from gaborgsomogyi/SPARK-26998.
Authored-by: Gabor Somogyi
Signed-off-by: Marcelo Vanzin
(cherry picked from commit 57aff93886ac7d02b88294672ce0d2495b0942b8)
Signed-off-by: Marcelo Vanzin
---
core/src/main/scala/org/apache/spark/SparkConf.scala | 1 -
core/src/test/scala/org/apache/spark/SparkConfSuite.scala | 11 +++
2 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala
b/core/src/main/scala/org/apache/spark/SparkConf.scala
index ccafe16..0b24fe2 100644
--- a/core/src/main/scala/org/apache/spark/SparkConf.scala
+++ b/core/src/main/scala/org/apache/spark/SparkConf.scala
@@ -751,7 +751,6 @@ private[spark] object SparkConf extends Logging {
*/
def isExecutorStartupConf(name: String): Boolean = {
(name.startsWith("spark.auth") && name !=
SecurityManager.SPARK_AUTH_SECRET_CONF) ||
-name.startsWith("spark.ssl") ||
name.startsWith("spark.rpc") ||
name.startsWith("spark.network") ||
isSparkPortConf(name)
diff --git a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
index dcbfc93..66462de 100644
--- a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
@@ -339,6 +339,17 @@ class SparkConfSuite extends SparkFunSuite with
LocalSparkContext with ResetSyst
}
}
+ test("SPARK-26998: SSL configuration not needed on executors") {
+val conf = new SparkConf(false)
+conf.set("spark.ssl.enabled", "true")
+conf.set("spark.ssl.keyPassword", "password")
+conf.set("spark.ssl.keyStorePassword", "password")
+conf.set("spark.ssl.trustStorePassword", "password")
[spark] branch master updated: [SPARK-26998][CORE] Remove SSL configuration from executors
This is an automated email from the ASF dual-hosted git repository.
vanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 57aff93 [SPARK-26998][CORE] Remove SSL configuration from executors
57aff93 is described below
commit 57aff93886ac7d02b88294672ce0d2495b0942b8
Author: Gabor Somogyi
AuthorDate: Tue Apr 2 09:18:43 2019 -0700
[SPARK-26998][CORE] Remove SSL configuration from executors
## What changes were proposed in this pull request?
Different SSL passwords shown up as command line argument on executor side
in standalone mode:
* keyStorePassword
* keyPassword
* trustStorePassword
In this PR I've removed SSL configurations from executors.
## How was this patch tested?
Existing + additional unit tests.
Additionally tested with standalone mode and checked the command line
arguments:
```
[gaborsomogyi:~/spark] SPARK-26998(+4/-0,3)+ ± jps
94803 CoarseGrainedExecutorBackend
94818 Jps
90149 RemoteMavenServer
91925 Nailgun
94793 SparkSubmit
94680 Worker
94556 Master
398
[gaborsomogyi:~/spark] SPARK-26998(+4/-1,3)+ ± ps -ef | egrep
"94556|94680|94793|94803"
502 94556 1 0 2:02PM ttys0070:07.39
/Library/Java/JavaVirtualMachines/jdk1.8.0_152.jdk/Contents/Home/bin/java -cp
/Users/gaborsomogyi/spark/conf/:/Users/gaborsomogyi/spark/assembly/target/scala-2.12/jars/*
-Xmx1g org.apache.spark.deploy.master.Master --host gsomogyi-MBP.local --port
7077 --webui-port 8080 --properties-file conf/spark-defaults.conf
502 94680 1 0 2:02PM ttys0070:07.27
/Library/Java/JavaVirtualMachines/jdk1.8.0_152.jdk/Contents/Home/bin/java -cp
/Users/gaborsomogyi/spark/conf/:/Users/gaborsomogyi/spark/assembly/target/scala-2.12/jars/*
-Xmx1g org.apache.spark.deploy.worker.Worker --webui-port 8081
--properties-file conf/spark-defaults.conf spark://gsomogyi-MBP.local:7077
502 94793 94782 0 2:02PM ttys0070:35.52
/Library/Java/JavaVirtualMachines/jdk1.8.0_152.jdk/Contents/Home/bin/java -cp
/Users/gaborsomogyi/spark/conf/:/Users/gaborsomogyi/spark/assembly/target/scala-2.12/jars/*
-Dscala.usejavacp=true -Xmx1g org.apache.spark.deploy.SparkSubmit --master
spark://gsomogyi-MBP.local:7077 --class org.apache.spark.repl.Main --name Spark
shell spark-shell
502 94803 94680 0 2:03PM ttys0070:05.20
/Library/Java/JavaVirtualMachines/jdk1.8.0_152.jdk/Contents/Home/bin/java -cp
/Users/gaborsomogyi/spark/conf/:/Users/gaborsomogyi/spark/assembly/target/scala-2.12/jars/*
-Xmx1024M -Dspark.ssl.ui.port=0 -Dspark.driver.port=60902
org.apache.spark.executor.CoarseGrainedExecutorBackend --driver-url
spark://CoarseGrainedScheduler172.30.65.186:60902 --executor-id 0 --hostname
172.30.65.186 --cores 8 --app-id app-20190326140311- --worker-u [...]
502 94910 57352 0 2:05PM ttys0080:00.00 egrep
94556|94680|94793|94803
```
Closes #24170 from gaborgsomogyi/SPARK-26998.
Authored-by: Gabor Somogyi
Signed-off-by: Marcelo Vanzin
---
core/src/main/scala/org/apache/spark/SparkConf.scala | 1 -
core/src/test/scala/org/apache/spark/SparkConfSuite.scala | 11 +++
2 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala
b/core/src/main/scala/org/apache/spark/SparkConf.scala
index 8499246..4117aea 100644
--- a/core/src/main/scala/org/apache/spark/SparkConf.scala
+++ b/core/src/main/scala/org/apache/spark/SparkConf.scala
@@ -738,7 +738,6 @@ private[spark] object SparkConf extends Logging {
*/
def isExecutorStartupConf(name: String): Boolean = {
(name.startsWith("spark.auth") && name !=
SecurityManager.SPARK_AUTH_SECRET_CONF) ||
-name.startsWith("spark.ssl") ||
name.startsWith("spark.rpc") ||
name.startsWith("spark.network") ||
isSparkPortConf(name)
diff --git a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
index 4b2dd9a..9f759a5 100644
--- a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
@@ -353,6 +353,17 @@ class SparkConfSuite extends SparkFunSuite with
LocalSparkContext with ResetSyst
}
}
+ test("SPARK-26998: SSL configuration not needed on executors") {
+val conf = new SparkConf(false)
+conf.set("spark.ssl.enabled", "true")
+conf.set("spark.ssl.keyPassword", "password")
+conf.set("spark.ssl.keyStorePassword", "password")
+conf.set("spark.ssl.trustStorePassword", "password")
+
+val filtered = conf.getAll.filter { case (k, _) =>
SparkConf.isExecutorStartupConf(k) }
+assert(f
