Github user rxin commented on a diff in the pull request:
https://github.com/apache/spark/pull/5430#discussion_r28928119
--- Diff:
core/src/main/scala/org/apache/spark/storage/TachyonBlockManager.scala ---
@@ -32,32 +34,97 @@ import org.apache.spark.util.Utils
* Creates and maintains the logical mapping between logical blocks and
tachyon fs locations. By
* default, one block is mapped to one file with a name given by its
BlockId.
*
- * @param rootDirs The directories to use for storing block files. Data
will be hashed among these.
*/
-private[spark] class TachyonBlockManager(
- blockManager: BlockManager,
- rootDirs: String,
- val master: String)
- extends Logging {
+private[spark] class TachyonBlockManager() extends ExtBlockManager with
Logging {
- val client = if (master != null && master != "") TachyonFS.get(master)
else null
-
- if (client == null) {
- logError("Failed to connect to the Tachyon as the master address is
not configured")
- System.exit(ExecutorExitCode.TACHYON_STORE_FAILED_TO_INITIALIZE)
- }
-
- private val MAX_DIR_CREATION_ATTEMPTS = 10
- private val subDirsPerTachyonDir =
- blockManager.conf.get("spark.tachyonStore.subDirectories", "64").toInt
+ var blockManager: BlockManager =_
+ var rootDirs: String = _
+ var master: String = _
+ var client: tachyon.client.TachyonFS = _
+ private var subDirsPerTachyonDir: Int = _
// Create one Tachyon directory for each path mentioned in
spark.tachyonStore.folderName;
// then, inside this directory, create multiple subdirectories that we
will hash files into,
// in order to avoid having really large inodes at the top level in
Tachyon.
- private val tachyonDirs: Array[TachyonFile] = createTachyonDirs()
- private val subDirs = Array.fill(tachyonDirs.length)(new
Array[TachyonFile](subDirsPerTachyonDir))
+ private var tachyonDirs: Array[TachyonFile] = _
+ private var subDirs: Array[Array[tachyon.client.TachyonFile]] = _
+
+
+ override def init(blockManager: BlockManager, executorId: String): Unit
= {
+ this.blockManager = blockManager
+ val storeDir = blockManager.conf.get(ExtBlockStore.BASE_DIR,
"/tmp_spark_tachyon")
+ val appFolderName = blockManager.conf.get(ExtBlockStore.FOLD_NAME)
+ rootDirs = s"$storeDir/$appFolderName/$executorId"
+ master = blockManager.conf.get(ExtBlockStore.MASTER_URL,
"tachyon://localhost:19998")
+ init0();
+ }
+
+ // move all original initialization here just for easy to review.
+ private def init0() = {
--- End diff --
thanks for doing this for review. i think you can just put this in init now
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
