This is an automated email from the ASF dual-hosted git repository.
yangzy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 6521ffa854 [CORE] Move scala file to scala package and fix several
typos (#7635)
6521ffa854 is described below
commit 6521ffa854cc40a3ea157c9d777c23b649434745
Author: Xuedong Luan <[email protected]>
AuthorDate: Tue Oct 22 18:38:29 2024 +0800
[CORE] Move scala file to scala package and fix several typos (#7635)
---
.../org/apache/gluten/softaffinity/SoftAffinityManager.scala | 11 ++++++++---
.../softaffinity/strategy/SoftAffinityAllocationTrait.scala | 0
.../gluten/softaffinity/strategy/SoftAffinityStrategy.scala | 0
.../{java => scala}/org/apache/gluten/task/TaskListener.scala | 0
.../org/apache/spark/softaffinity/SoftAffinity.scala | 1 +
.../org/apache/spark/softaffinity/SoftAffinityListener.scala | 1 +
.../org/apache/spark/util/SparkDirectoryUtil.scala | 5 +++--
.../org/apache/spark/util/SparkShutdownManagerUtil.scala | 0
.../src/main/scala/org/apache/gluten/GlutenConfig.scala | 4 ++--
9 files changed, 15 insertions(+), 7 deletions(-)
diff --git
a/gluten-core/src/main/java/org/apache/gluten/softaffinity/SoftAffinityManager.scala
b/gluten-core/src/main/scala/org/apache/gluten/softaffinity/SoftAffinityManager.scala
similarity index 98%
rename from
gluten-core/src/main/java/org/apache/gluten/softaffinity/SoftAffinityManager.scala
rename to
gluten-core/src/main/scala/org/apache/gluten/softaffinity/SoftAffinityManager.scala
index 576697bdb8..44ff5bb9bc 100644
---
a/gluten-core/src/main/java/org/apache/gluten/softaffinity/SoftAffinityManager.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/softaffinity/SoftAffinityManager.scala
@@ -17,17 +17,20 @@
package org.apache.gluten.softaffinity
import org.apache.gluten.GlutenConfig
+import org.apache.gluten.logging.LogLevelUtil
import org.apache.gluten.softaffinity.strategy.SoftAffinityStrategy
import org.apache.gluten.sql.shims.SparkShimLoader
+
import org.apache.spark.SparkEnv
import org.apache.spark.internal.Logging
import org.apache.spark.scheduler.{SparkListenerStageCompleted,
SparkListenerStageSubmitted, SparkListenerTaskEnd}
import org.apache.spark.sql.execution.datasources.FilePartition
+
import com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache}
-import org.apache.gluten.logging.LogLevelUtil
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.locks.ReentrantReadWriteLock
+
import scala.collection.mutable
import scala.util.Random
@@ -190,7 +193,8 @@ abstract class AffinityManager extends LogLevelUtil with
Logging {
} else {
(originalValues ++ value)
}
- logOnLevel(GlutenConfig.getConf.softAffinityLogLevel,
+ logOnLevel(
+ GlutenConfig.getConf.softAffinityLogLevel,
s"update host for $key: ${values.mkString(",")}")
duplicateReadingInfos.put(key, values)
}
@@ -272,7 +276,8 @@ abstract class AffinityManager extends LogLevelUtil with
Logging {
if (!hosts.isEmpty) {
rand.shuffle(hosts)
- logOnLevel(GlutenConfig.getConf.softAffinityLogLevel,
+ logOnLevel(
+ GlutenConfig.getConf.softAffinityLogLevel,
s"get host for $f: ${hosts.distinct.mkString(",")}")
}
hosts.distinct.toSeq
diff --git
a/gluten-core/src/main/java/org/apache/gluten/softaffinity/strategy/SoftAffinityAllocationTrait.scala
b/gluten-core/src/main/scala/org/apache/gluten/softaffinity/strategy/SoftAffinityAllocationTrait.scala
similarity index 100%
rename from
gluten-core/src/main/java/org/apache/gluten/softaffinity/strategy/SoftAffinityAllocationTrait.scala
rename to
gluten-core/src/main/scala/org/apache/gluten/softaffinity/strategy/SoftAffinityAllocationTrait.scala
diff --git
a/gluten-core/src/main/java/org/apache/gluten/softaffinity/strategy/SoftAffinityStrategy.scala
b/gluten-core/src/main/scala/org/apache/gluten/softaffinity/strategy/SoftAffinityStrategy.scala
similarity index 100%
rename from
gluten-core/src/main/java/org/apache/gluten/softaffinity/strategy/SoftAffinityStrategy.scala
rename to
gluten-core/src/main/scala/org/apache/gluten/softaffinity/strategy/SoftAffinityStrategy.scala
diff --git
a/gluten-core/src/main/java/org/apache/gluten/task/TaskListener.scala
b/gluten-core/src/main/scala/org/apache/gluten/task/TaskListener.scala
similarity index 100%
rename from gluten-core/src/main/java/org/apache/gluten/task/TaskListener.scala
rename to gluten-core/src/main/scala/org/apache/gluten/task/TaskListener.scala
diff --git
a/gluten-core/src/main/java/org/apache/spark/softaffinity/SoftAffinity.scala
b/gluten-core/src/main/scala/org/apache/spark/softaffinity/SoftAffinity.scala
similarity index 99%
rename from
gluten-core/src/main/java/org/apache/spark/softaffinity/SoftAffinity.scala
rename to
gluten-core/src/main/scala/org/apache/spark/softaffinity/SoftAffinity.scala
index 65a0bfd078..66e4456d46 100644
--- a/gluten-core/src/main/java/org/apache/spark/softaffinity/SoftAffinity.scala
+++
b/gluten-core/src/main/scala/org/apache/spark/softaffinity/SoftAffinity.scala
@@ -19,6 +19,7 @@ package org.apache.spark.softaffinity
import org.apache.gluten.GlutenConfig
import org.apache.gluten.logging.LogLevelUtil
import org.apache.gluten.softaffinity.{AffinityManager, SoftAffinityManager}
+
import org.apache.spark.internal.Logging
import org.apache.spark.scheduler.ExecutorCacheTaskLocation
import org.apache.spark.sql.execution.datasources.FilePartition
diff --git
a/gluten-core/src/main/java/org/apache/spark/softaffinity/SoftAffinityListener.scala
b/gluten-core/src/main/scala/org/apache/spark/softaffinity/SoftAffinityListener.scala
similarity index 99%
rename from
gluten-core/src/main/java/org/apache/spark/softaffinity/SoftAffinityListener.scala
rename to
gluten-core/src/main/scala/org/apache/spark/softaffinity/SoftAffinityListener.scala
index d6b9a8d70c..eba7e9f9fa 100644
---
a/gluten-core/src/main/java/org/apache/spark/softaffinity/SoftAffinityListener.scala
+++
b/gluten-core/src/main/scala/org/apache/spark/softaffinity/SoftAffinityListener.scala
@@ -17,6 +17,7 @@
package org.apache.spark.softaffinity
import org.apache.gluten.softaffinity.SoftAffinityManager
+
import org.apache.spark.SparkContext
import org.apache.spark.internal.Logging
import org.apache.spark.scheduler._
diff --git
a/gluten-core/src/main/java/org/apache/spark/util/SparkDirectoryUtil.scala
b/gluten-core/src/main/scala/org/apache/spark/util/SparkDirectoryUtil.scala
similarity index 99%
rename from
gluten-core/src/main/java/org/apache/spark/util/SparkDirectoryUtil.scala
rename to
gluten-core/src/main/scala/org/apache/spark/util/SparkDirectoryUtil.scala
index 06d5eb8ae8..de25973a95 100644
--- a/gluten-core/src/main/java/org/apache/spark/util/SparkDirectoryUtil.scala
+++ b/gluten-core/src/main/scala/org/apache/spark/util/SparkDirectoryUtil.scala
@@ -16,11 +16,12 @@
*/
package org.apache.spark.util
+import org.apache.spark.SparkConf
+import org.apache.spark.internal.Logging
+
import _root_.org.apache.gluten.exception.GlutenException
import org.apache.commons.io.FileUtils
import org.apache.commons.lang3.StringUtils
-import org.apache.spark.SparkConf
-import org.apache.spark.internal.Logging
import java.io.{File, IOException}
import java.nio.file.Paths
diff --git
a/gluten-core/src/main/java/org/apache/spark/util/SparkShutdownManagerUtil.scala
b/gluten-core/src/main/scala/org/apache/spark/util/SparkShutdownManagerUtil.scala
similarity index 100%
rename from
gluten-core/src/main/java/org/apache/spark/util/SparkShutdownManagerUtil.scala
rename to
gluten-core/src/main/scala/org/apache/spark/util/SparkShutdownManagerUtil.scala
diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
index 41431f1a11..089f695958 100644
--- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
+++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
@@ -561,11 +561,11 @@ object GlutenConfig {
val GLUTEN_SOFT_AFFINITY_MIN_TARGET_HOSTS =
"spark.gluten.soft-affinity.min.target-hosts"
val GLUTEN_SOFT_AFFINITY_MIN_TARGET_HOSTS_DEFAULT_VALUE = 1
- // Enable Soft Affinity duplicate reading detection, defalut value is true
+ // Enable Soft Affinity duplicate reading detection, default value is false
val GLUTEN_SOFT_AFFINITY_DUPLICATE_READING_DETECT_ENABLED =
"spark.gluten.soft-affinity.duplicateReadingDetect.enabled"
val GLUTEN_SOFT_AFFINITY_DUPLICATE_READING_DETECT_ENABLED_DEFAULT_VALUE =
false
- // Enable Soft Affinity duplicate reading detection, defalut value is 10000
+ // Enable Soft Affinity duplicate reading detection, default value is 10000
val GLUTEN_SOFT_AFFINITY_DUPLICATE_READING_MAX_CACHE_ITEMS =
"spark.gluten.soft-affinity.duplicateReading.maxCacheItems"
val GLUTEN_SOFT_AFFINITY_DUPLICATE_READING_MAX_CACHE_ITEMS_DEFAULT_VALUE =
10000
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
