[28/50] [abbrv] hadoop git commit: HDFS-12911. [SPS]: Modularize the SPS code and expose necessary interfaces for external/internal implementations. Contributed by Uma Maheswara Rao G
HDFS-12911. [SPS]: Modularize the SPS code and expose necessary interfaces for
external/internal implementations. Contributed by Uma Maheswara Rao G
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8d4f74e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8d4f74e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8d4f74e7
Branch: refs/heads/trunk
Commit: 8d4f74e7339abc77dc0daa162d7bd2814bd79b3d
Parents: 05d4daf
Author: Rakesh Radhakrishnan
Authored: Fri Jan 19 08:51:49 2018 +0530
Committer: Uma Maheswara Rao Gangumalla
Committed: Sun Aug 12 03:06:03 2018 -0700
--
.../server/blockmanagement/BlockManager.java| 61 +-
.../namenode/FSDirSatisfyStoragePolicyOp.java | 16 +-
.../hdfs/server/namenode/FSDirectory.java | 6 +-
.../hdfs/server/namenode/FSNamesystem.java | 10 +-
.../namenode/sps/BlockMoveTaskHandler.java | 44
.../namenode/sps/BlockMovementListener.java | 40
.../sps/BlockStorageMovementAttemptedItems.java | 28 +--
.../sps/BlockStorageMovementNeeded.java | 207 ---
.../hdfs/server/namenode/sps/Context.java | 43 ++--
.../server/namenode/sps/FileIdCollector.java| 43
.../IntraSPSNameNodeBlockMoveTaskHandler.java | 62 ++
.../namenode/sps/IntraSPSNameNodeContext.java | 62 ++
.../sps/IntraSPSNameNodeFileIdCollector.java| 178
.../hdfs/server/namenode/sps/ItemInfo.java | 81
.../hdfs/server/namenode/sps/SPSPathIds.java| 63 ++
.../hdfs/server/namenode/sps/SPSService.java| 107 ++
.../namenode/sps/StoragePolicySatisfier.java| 175 +++-
.../TestBlockStorageMovementAttemptedItems.java | 19 +-
.../sps/TestStoragePolicySatisfier.java | 111 ++
...stStoragePolicySatisfierWithStripedFile.java | 19 +-
20 files changed, 938 insertions(+), 437 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d4f74e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 5ee4026..d12cb01 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -93,8 +93,8 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.Namesystem;
import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
-import org.apache.hadoop.hdfs.server.namenode.sps.Context;
-import org.apache.hadoop.hdfs.server.namenode.sps.IntraSPSNameNodeContext;
+import org.apache.hadoop.hdfs.server.namenode.sps.SPSPathIds;
+import org.apache.hadoop.hdfs.server.namenode.sps.SPSService;
import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfier;
import org.apache.hadoop.hdfs.server.protocol.BlockCommand;
import org.apache.hadoop.hdfs.server.protocol.BlockReportContext;
@@ -434,7 +434,8 @@ public class BlockManager implements BlockStatsMXBean {
private final StoragePolicySatisfier sps;
private final boolean storagePolicyEnabled;
private boolean spsEnabled;
- private Context spsctxt = null;
+ private final SPSPathIds spsPaths;
+
/** Minimum live replicas needed for the datanode to be transitioned
* from ENTERING_MAINTENANCE to IN_MAINTENANCE.
*/
@@ -481,8 +482,8 @@ public class BlockManager implements BlockStatsMXBean {
conf.getBoolean(
DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_ENABLED_KEY,
DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_ENABLED_DEFAULT);
-spsctxt = new IntraSPSNameNodeContext(namesystem, this, conf);
-sps = new StoragePolicySatisfier(spsctxt);
+sps = new StoragePolicySatisfier(conf);
+spsPaths = new SPSPathIds();
blockTokenSecretManager = createBlockTokenSecretManager(conf);
providedStorageMap = new ProvidedStorageMap(namesystem, this, conf);
@@ -5041,8 +5042,7 @@ public class BlockManager implements BlockStatsMXBean {
LOG.info("Storage policy satisfier is already running.");
return;
}
-// TODO: FSDirectory will get removed via HDFS-12911 modularization work
-sps.start(false, namesystem.getFSDirectory());
+sps.start(false);
}
/**
@@ -5078,8 +5078,7 @@ public class BlockManager implements BlockStatsMXBean {
[28/50] [abbrv] hadoop git commit: HDFS-12911. [SPS]: Modularize the SPS code and expose necessary interfaces for external/internal implementations. Contributed by Uma Maheswara Rao G
HDFS-12911. [SPS]: Modularize the SPS code and expose necessary interfaces for
external/internal implementations. Contributed by Uma Maheswara Rao G
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8d4f74e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8d4f74e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8d4f74e7
Branch: refs/heads/HDFS-10285
Commit: 8d4f74e7339abc77dc0daa162d7bd2814bd79b3d
Parents: 05d4daf
Author: Rakesh Radhakrishnan
Authored: Fri Jan 19 08:51:49 2018 +0530
Committer: Uma Maheswara Rao Gangumalla
Committed: Sun Aug 12 03:06:03 2018 -0700
--
.../server/blockmanagement/BlockManager.java| 61 +-
.../namenode/FSDirSatisfyStoragePolicyOp.java | 16 +-
.../hdfs/server/namenode/FSDirectory.java | 6 +-
.../hdfs/server/namenode/FSNamesystem.java | 10 +-
.../namenode/sps/BlockMoveTaskHandler.java | 44
.../namenode/sps/BlockMovementListener.java | 40
.../sps/BlockStorageMovementAttemptedItems.java | 28 +--
.../sps/BlockStorageMovementNeeded.java | 207 ---
.../hdfs/server/namenode/sps/Context.java | 43 ++--
.../server/namenode/sps/FileIdCollector.java| 43
.../IntraSPSNameNodeBlockMoveTaskHandler.java | 62 ++
.../namenode/sps/IntraSPSNameNodeContext.java | 62 ++
.../sps/IntraSPSNameNodeFileIdCollector.java| 178
.../hdfs/server/namenode/sps/ItemInfo.java | 81
.../hdfs/server/namenode/sps/SPSPathIds.java| 63 ++
.../hdfs/server/namenode/sps/SPSService.java| 107 ++
.../namenode/sps/StoragePolicySatisfier.java| 175 +++-
.../TestBlockStorageMovementAttemptedItems.java | 19 +-
.../sps/TestStoragePolicySatisfier.java | 111 ++
...stStoragePolicySatisfierWithStripedFile.java | 19 +-
20 files changed, 938 insertions(+), 437 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d4f74e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 5ee4026..d12cb01 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -93,8 +93,8 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.Namesystem;
import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
-import org.apache.hadoop.hdfs.server.namenode.sps.Context;
-import org.apache.hadoop.hdfs.server.namenode.sps.IntraSPSNameNodeContext;
+import org.apache.hadoop.hdfs.server.namenode.sps.SPSPathIds;
+import org.apache.hadoop.hdfs.server.namenode.sps.SPSService;
import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfier;
import org.apache.hadoop.hdfs.server.protocol.BlockCommand;
import org.apache.hadoop.hdfs.server.protocol.BlockReportContext;
@@ -434,7 +434,8 @@ public class BlockManager implements BlockStatsMXBean {
private final StoragePolicySatisfier sps;
private final boolean storagePolicyEnabled;
private boolean spsEnabled;
- private Context spsctxt = null;
+ private final SPSPathIds spsPaths;
+
/** Minimum live replicas needed for the datanode to be transitioned
* from ENTERING_MAINTENANCE to IN_MAINTENANCE.
*/
@@ -481,8 +482,8 @@ public class BlockManager implements BlockStatsMXBean {
conf.getBoolean(
DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_ENABLED_KEY,
DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_ENABLED_DEFAULT);
-spsctxt = new IntraSPSNameNodeContext(namesystem, this, conf);
-sps = new StoragePolicySatisfier(spsctxt);
+sps = new StoragePolicySatisfier(conf);
+spsPaths = new SPSPathIds();
blockTokenSecretManager = createBlockTokenSecretManager(conf);
providedStorageMap = new ProvidedStorageMap(namesystem, this, conf);
@@ -5041,8 +5042,7 @@ public class BlockManager implements BlockStatsMXBean {
LOG.info("Storage policy satisfier is already running.");
return;
}
-// TODO: FSDirectory will get removed via HDFS-12911 modularization work
-sps.start(false, namesystem.getFSDirectory());
+sps.start(false);
}
/**
@@ -5078,8 +5078,7 @@ public class BlockManager implements BlockStatsMXBean {
[28/50] [abbrv] hadoop git commit: HDFS-12911. [SPS]: Modularize the SPS code and expose necessary interfaces for external/internal implementations. Contributed by Uma Maheswara Rao G
HDFS-12911. [SPS]: Modularize the SPS code and expose necessary interfaces for
external/internal implementations. Contributed by Uma Maheswara Rao G
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7bcf24f9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7bcf24f9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7bcf24f9
Branch: refs/heads/HDFS-10285
Commit: 7bcf24f9407afd6a32983d6776624ad5f76b7f5c
Parents: 87b11d0
Author: Rakesh Radhakrishnan
Authored: Fri Jan 19 08:51:49 2018 +0530
Committer: Uma Maheswara Rao Gangumalla
Committed: Fri Aug 10 13:23:02 2018 -0700
--
.../server/blockmanagement/BlockManager.java| 61 +-
.../namenode/FSDirSatisfyStoragePolicyOp.java | 16 +-
.../hdfs/server/namenode/FSDirectory.java | 6 +-
.../hdfs/server/namenode/FSNamesystem.java | 10 +-
.../namenode/sps/BlockMoveTaskHandler.java | 44
.../namenode/sps/BlockMovementListener.java | 40
.../sps/BlockStorageMovementAttemptedItems.java | 28 +--
.../sps/BlockStorageMovementNeeded.java | 207 ---
.../hdfs/server/namenode/sps/Context.java | 43 ++--
.../server/namenode/sps/FileIdCollector.java| 43
.../IntraSPSNameNodeBlockMoveTaskHandler.java | 62 ++
.../namenode/sps/IntraSPSNameNodeContext.java | 62 ++
.../sps/IntraSPSNameNodeFileIdCollector.java| 178
.../hdfs/server/namenode/sps/ItemInfo.java | 81
.../hdfs/server/namenode/sps/SPSPathIds.java| 63 ++
.../hdfs/server/namenode/sps/SPSService.java| 107 ++
.../namenode/sps/StoragePolicySatisfier.java| 175 +++-
.../TestBlockStorageMovementAttemptedItems.java | 19 +-
.../sps/TestStoragePolicySatisfier.java | 111 ++
...stStoragePolicySatisfierWithStripedFile.java | 19 +-
20 files changed, 938 insertions(+), 437 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bcf24f9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 5ee4026..d12cb01 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -93,8 +93,8 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.Namesystem;
import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
-import org.apache.hadoop.hdfs.server.namenode.sps.Context;
-import org.apache.hadoop.hdfs.server.namenode.sps.IntraSPSNameNodeContext;
+import org.apache.hadoop.hdfs.server.namenode.sps.SPSPathIds;
+import org.apache.hadoop.hdfs.server.namenode.sps.SPSService;
import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfier;
import org.apache.hadoop.hdfs.server.protocol.BlockCommand;
import org.apache.hadoop.hdfs.server.protocol.BlockReportContext;
@@ -434,7 +434,8 @@ public class BlockManager implements BlockStatsMXBean {
private final StoragePolicySatisfier sps;
private final boolean storagePolicyEnabled;
private boolean spsEnabled;
- private Context spsctxt = null;
+ private final SPSPathIds spsPaths;
+
/** Minimum live replicas needed for the datanode to be transitioned
* from ENTERING_MAINTENANCE to IN_MAINTENANCE.
*/
@@ -481,8 +482,8 @@ public class BlockManager implements BlockStatsMXBean {
conf.getBoolean(
DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_ENABLED_KEY,
DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_ENABLED_DEFAULT);
-spsctxt = new IntraSPSNameNodeContext(namesystem, this, conf);
-sps = new StoragePolicySatisfier(spsctxt);
+sps = new StoragePolicySatisfier(conf);
+spsPaths = new SPSPathIds();
blockTokenSecretManager = createBlockTokenSecretManager(conf);
providedStorageMap = new ProvidedStorageMap(namesystem, this, conf);
@@ -5041,8 +5042,7 @@ public class BlockManager implements BlockStatsMXBean {
LOG.info("Storage policy satisfier is already running.");
return;
}
-// TODO: FSDirectory will get removed via HDFS-12911 modularization work
-sps.start(false, namesystem.getFSDirectory());
+sps.start(false);
}
/**
@@ -5078,8 +5078,7 @@ public class BlockManager implements BlockStatsMXBean {
[28/50] [abbrv] hadoop git commit: HDFS-12911. [SPS]: Modularize the SPS code and expose necessary interfaces for external/internal implementations. Contributed by Uma Maheswara Rao G
HDFS-12911. [SPS]: Modularize the SPS code and expose necessary interfaces for
external/internal implementations. Contributed by Uma Maheswara Rao G
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a0c8e48c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a0c8e48c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a0c8e48c
Branch: refs/heads/HDFS-10285
Commit: a0c8e48c40e31d741b192a649d1a7773e2384437
Parents: 113185e
Author: Rakesh Radhakrishnan
Authored: Fri Jan 19 08:51:49 2018 +0530
Committer: Rakesh Radhakrishnan
Committed: Tue Jul 31 12:10:15 2018 +0530
--
.../server/blockmanagement/BlockManager.java| 61 +-
.../namenode/FSDirSatisfyStoragePolicyOp.java | 16 +-
.../hdfs/server/namenode/FSDirectory.java | 6 +-
.../hdfs/server/namenode/FSNamesystem.java | 10 +-
.../namenode/sps/BlockMoveTaskHandler.java | 44
.../namenode/sps/BlockMovementListener.java | 40
.../sps/BlockStorageMovementAttemptedItems.java | 28 +--
.../sps/BlockStorageMovementNeeded.java | 207 ---
.../hdfs/server/namenode/sps/Context.java | 43 ++--
.../server/namenode/sps/FileIdCollector.java| 43
.../IntraSPSNameNodeBlockMoveTaskHandler.java | 62 ++
.../namenode/sps/IntraSPSNameNodeContext.java | 62 ++
.../sps/IntraSPSNameNodeFileIdCollector.java| 178
.../hdfs/server/namenode/sps/ItemInfo.java | 81
.../hdfs/server/namenode/sps/SPSPathIds.java| 63 ++
.../hdfs/server/namenode/sps/SPSService.java| 107 ++
.../namenode/sps/StoragePolicySatisfier.java| 175 +++-
.../TestBlockStorageMovementAttemptedItems.java | 19 +-
.../sps/TestStoragePolicySatisfier.java | 111 ++
...stStoragePolicySatisfierWithStripedFile.java | 19 +-
20 files changed, 938 insertions(+), 437 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0c8e48c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index c2d5162..63117ce 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -93,8 +93,8 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.Namesystem;
import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
-import org.apache.hadoop.hdfs.server.namenode.sps.Context;
-import org.apache.hadoop.hdfs.server.namenode.sps.IntraSPSNameNodeContext;
+import org.apache.hadoop.hdfs.server.namenode.sps.SPSPathIds;
+import org.apache.hadoop.hdfs.server.namenode.sps.SPSService;
import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfier;
import org.apache.hadoop.hdfs.server.protocol.BlockCommand;
import org.apache.hadoop.hdfs.server.protocol.BlockReportContext;
@@ -434,7 +434,8 @@ public class BlockManager implements BlockStatsMXBean {
private final StoragePolicySatisfier sps;
private final boolean storagePolicyEnabled;
private boolean spsEnabled;
- private Context spsctxt = null;
+ private final SPSPathIds spsPaths;
+
/** Minimum live replicas needed for the datanode to be transitioned
* from ENTERING_MAINTENANCE to IN_MAINTENANCE.
*/
@@ -481,8 +482,8 @@ public class BlockManager implements BlockStatsMXBean {
conf.getBoolean(
DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_ENABLED_KEY,
DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_ENABLED_DEFAULT);
-spsctxt = new IntraSPSNameNodeContext(namesystem, this, conf);
-sps = new StoragePolicySatisfier(spsctxt);
+sps = new StoragePolicySatisfier(conf);
+spsPaths = new SPSPathIds();
blockTokenSecretManager = createBlockTokenSecretManager(conf);
providedStorageMap = new ProvidedStorageMap(namesystem, this, conf);
@@ -5033,8 +5034,7 @@ public class BlockManager implements BlockStatsMXBean {
LOG.info("Storage policy satisfier is already running.");
return;
}
-// TODO: FSDirectory will get removed via HDFS-12911 modularization work
-sps.start(false, namesystem.getFSDirectory());
+sps.start(false);
}
/**
@@ -5070,8 +5070,7 @@ public class BlockManager implements BlockStatsMXBean {
