Github user xuanyuanking commented on a diff in the pull request:
https://github.com/apache/spark/pull/21293#discussion_r188277683
--- Diff:
core/src/main/scala/org/apache/spark/scheduler/ContinuousShuffleMapTask.scala
---
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache
Github user xuanyuanking commented on a diff in the pull request:
https://github.com/apache/spark/pull/21293#discussion_r188273722
--- Diff: core/src/main/scala/org/apache/spark/MapOutputTracker.scala ---
@@ -769,6 +796,43 @@ private[spark] class MapOutputTrackerWorker(conf:
Github user xuanyuanking commented on a diff in the pull request:
https://github.com/apache/spark/pull/21293#discussion_r188270290
--- Diff: core/src/main/scala/org/apache/spark/Dependency.scala ---
@@ -88,14 +90,53 @@ class ShuffleDependency[K: ClassTag, V: ClassTag, C:
Github user xuanyuanking commented on a diff in the pull request:
https://github.com/apache/spark/pull/21293#discussion_r188269208
--- Diff: core/src/main/scala/org/apache/spark/Dependency.scala ---
@@ -65,15 +65,17 @@ abstract class NarrowDependency[T](_rdd: RDD[T])
extends
Github user jose-torres commented on a diff in the pull request:
https://github.com/apache/spark/pull/21293#discussion_r187646713
--- Diff:
core/src/main/scala/org/apache/spark/scheduler/ContinuousShuffleMapTask.scala
---
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache
Github user jose-torres commented on a diff in the pull request:
https://github.com/apache/spark/pull/21293#discussion_r187641941
--- Diff: core/src/main/scala/org/apache/spark/MapOutputTracker.scala ---
@@ -769,6 +796,43 @@ private[spark] class MapOutputTrackerWorker(conf:
Github user jose-torres commented on a diff in the pull request:
https://github.com/apache/spark/pull/21293#discussion_r187641292
--- Diff: core/src/main/scala/org/apache/spark/MapOutputTracker.scala ---
@@ -233,6 +239,28 @@ private[spark] class MapOutputTrackerMasterEndpoint(
Github user jose-torres commented on a diff in the pull request:
https://github.com/apache/spark/pull/21293#discussion_r187639422
--- Diff: core/src/main/scala/org/apache/spark/Dependency.scala ---
@@ -88,14 +90,53 @@ class ShuffleDependency[K: ClassTag, V: ClassTag, C:
ClassTag](
Github user jose-torres commented on a diff in the pull request:
https://github.com/apache/spark/pull/21293#discussion_r187638899
--- Diff: core/src/main/scala/org/apache/spark/Dependency.scala ---
@@ -65,15 +65,17 @@ abstract class NarrowDependency[T](_rdd: RDD[T])
extends
Github user xuanyuanking commented on a diff in the pull request:
https://github.com/apache/spark/pull/21293#discussion_r187599741
--- Diff: core/src/main/scala/org/apache/spark/MapOutputTracker.scala ---
@@ -769,6 +796,43 @@ private[spark] class MapOutputTrackerWorker(conf:
Github user xuanyuanking commented on a diff in the pull request:
https://github.com/apache/spark/pull/21293#discussion_r187598787
--- Diff:
core/src/main/scala/org/apache/spark/scheduler/ContinuousShuffleMapTask.scala
---
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache
Github user xuanyuanking commented on a diff in the pull request:
https://github.com/apache/spark/pull/21293#discussion_r187598365
--- Diff: core/src/main/scala/org/apache/spark/SparkEnv.scala ---
@@ -227,6 +228,7 @@ object SparkEnv extends Logging {
Github user xuanyuanking commented on a diff in the pull request:
https://github.com/apache/spark/pull/21293#discussion_r187598100
--- Diff: core/src/main/scala/org/apache/spark/MapOutputTracker.scala ---
@@ -233,6 +239,28 @@ private[spark] class MapOutputTrackerMasterEndpoint(
Github user xuanyuanking commented on a diff in the pull request:
https://github.com/apache/spark/pull/21293#discussion_r187597922
--- Diff: core/src/main/scala/org/apache/spark/Dependency.scala ---
@@ -88,14 +90,53 @@ class ShuffleDependency[K: ClassTag, V: ClassTag, C:
Github user xuanyuanking commented on a diff in the pull request:
https://github.com/apache/spark/pull/21293#discussion_r187596748
--- Diff: core/src/main/scala/org/apache/spark/Dependency.scala ---
@@ -65,15 +65,17 @@ abstract class NarrowDependency[T](_rdd: RDD[T])
extends
Github user jose-torres commented on a diff in the pull request:
https://github.com/apache/spark/pull/21293#discussion_r187366974
--- Diff: core/src/main/scala/org/apache/spark/Dependency.scala ---
@@ -88,14 +90,53 @@ class ShuffleDependency[K: ClassTag, V: ClassTag, C:
ClassTag](
Github user jose-torres commented on a diff in the pull request:
https://github.com/apache/spark/pull/21293#discussion_r187368178
--- Diff: core/src/main/scala/org/apache/spark/SparkEnv.scala ---
@@ -227,6 +228,7 @@ object SparkEnv extends Logging {
Github user jose-torres commented on a diff in the pull request:
https://github.com/apache/spark/pull/21293#discussion_r187367699
--- Diff: core/src/main/scala/org/apache/spark/MapOutputTracker.scala ---
@@ -233,6 +239,28 @@ private[spark] class MapOutputTrackerMasterEndpoint(
Github user jose-torres commented on a diff in the pull request:
https://github.com/apache/spark/pull/21293#discussion_r187371412
--- Diff:
core/src/main/scala/org/apache/spark/scheduler/ContinuousShuffleMapTask.scala
---
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache
Github user jose-torres commented on a diff in the pull request:
https://github.com/apache/spark/pull/21293#discussion_r187366150
--- Diff: core/src/main/scala/org/apache/spark/Dependency.scala ---
@@ -65,15 +65,17 @@ abstract class NarrowDependency[T](_rdd: RDD[T])
extends
Github user jose-torres commented on a diff in the pull request:
https://github.com/apache/spark/pull/21293#discussion_r187367770
--- Diff: core/src/main/scala/org/apache/spark/MapOutputTracker.scala ---
@@ -233,6 +239,28 @@ private[spark] class MapOutputTrackerMasterEndpoint(
Github user jose-torres commented on a diff in the pull request:
https://github.com/apache/spark/pull/21293#discussion_r187371707
--- Diff: core/src/main/scala/org/apache/spark/MapOutputTracker.scala ---
@@ -769,6 +796,43 @@ private[spark] class MapOutputTrackerWorker(conf:
Github user jose-torres commented on a diff in the pull request:
https://github.com/apache/spark/pull/21293#discussion_r187372104
--- Diff:
core/src/main/scala/org/apache/spark/scheduler/ContinuousShuffleMapTask.scala
---
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache
GitHub user xuanyuanking opened a pull request:
https://github.com/apache/spark/pull/21293
[SPARK-24237][SS] Continuous shuffle dependency and map output tracker
## What changes were proposed in this pull request?
As our disscussion in [jira
24 matches
Mail list logo
