vanzin commented on a change in pull request #23951: [SPARK-13704][CORE][YARN]
Re-implement RackResolver to reduce resolving time
URL: https://github.com/apache/spark/pull/23951#discussion_r268351807
##########
File path:
resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/SparkRackResolver.scala
##########
@@ -17,24 +17,110 @@
package org.apache.spark.deploy.yarn
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+
+import com.google.common.base.Strings
import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic
+import org.apache.hadoop.net._
+import org.apache.hadoop.util.ReflectionUtils
import org.apache.hadoop.yarn.util.RackResolver
import org.apache.log4j.{Level, Logger}
+import org.apache.spark.internal.Logging
+
/**
- * Wrapper around YARN's [[RackResolver]]. This allows Spark tests to easily
override the
+ * Re-implement YARN's [[RackResolver]]. This allows Spark tests to easily
override the
* default behavior, since YARN's class self-initializes the first time it's
called, and
* future calls all use the initial configuration.
*/
-private[yarn] class SparkRackResolver {
+private[yarn] class SparkRackResolver(conf: Configuration) extends Logging {
// RackResolver logs an INFO message whenever it resolves a rack, which is
way too often.
if (Logger.getLogger(classOf[RackResolver]).getLevel == null) {
Logger.getLogger(classOf[RackResolver]).setLevel(Level.WARN)
}
+ private val dnsToSwitchMapping: DNSToSwitchMapping = {
+ val dnsToSwitchMappingClass =
+
conf.getClass(CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
+ classOf[ScriptBasedMapping], classOf[DNSToSwitchMapping])
+ ReflectionUtils.newInstance(dnsToSwitchMappingClass, conf)
+ .asInstanceOf[DNSToSwitchMapping] match {
+ case c: CachedDNSToSwitchMapping => c
+ case o => new CachedDNSToSwitchMapping(o)
+ }
+ }
+
def resolve(conf: Configuration, hostName: String): String = {
- RackResolver.resolve(conf, hostName).getNetworkLocation()
+ SparkRackResolver(conf).coreResolve(Seq(hostName)).head.getNetworkLocation
+ }
+
+ /**
+ * Added in SPARK-13704.
+ * This should be changed to `RackResolver.resolve(conf, hostNames)`
+ * in hadoop releases with YARN-9332.
+ */
+ def resolve(conf: Configuration, hostNames: Seq[String]): Seq[Node] = {
+ SparkRackResolver(conf).coreResolve(hostNames)
Review comment:
Same question, why is the class calling the singleton?
The point of having a class is so that you can have more than one instance
of it. You're defeating that by doing this.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
