[49/50] [abbrv] hadoop git commit: HDFS-13381 : [SPS]: Use DFSUtilClient#makePathFromFileId() to prepare satisfier file path. Contributed by Rakesh R.
HDFS-13381 : [SPS]: Use DFSUtilClient#makePathFromFileId() to prepare satisfier
file path. Contributed by Rakesh R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/66e8f9b3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/66e8f9b3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/66e8f9b3
Branch: refs/heads/trunk
Commit: 66e8f9b31529226309c924226a53dead3e6fcf11
Parents: 2acc50b
Author: Uma Maheswara Rao G
Authored: Mon Jul 2 17:22:00 2018 -0700
Committer: Uma Maheswara Rao Gangumalla
Committed: Sun Aug 12 03:06:06 2018 -0700
--
.../NamenodeProtocolServerSideTranslatorPB.java | 2 +-
.../NamenodeProtocolTranslatorPB.java | 2 +-
.../server/blockmanagement/BlockManager.java| 2 +-
.../hdfs/server/namenode/FSNamesystem.java | 11 ---
.../hdfs/server/namenode/NameNodeRpcServer.java | 8 +-
.../hadoop/hdfs/server/namenode/Namesystem.java | 9 ---
.../sps/BlockStorageMovementAttemptedItems.java | 72 +++--
.../sps/BlockStorageMovementNeeded.java | 61 ++
.../hdfs/server/namenode/sps/Context.java | 45 ---
.../namenode/sps/DatanodeCacheManager.java | 4 +-
.../hdfs/server/namenode/sps/FileCollector.java | 13 +--
.../namenode/sps/IntraSPSNameNodeContext.java | 54 +
.../sps/IntraSPSNameNodeFileIdCollector.java| 14 ++--
.../hdfs/server/namenode/sps/ItemInfo.java | 34
.../hdfs/server/namenode/sps/SPSService.java| 31 +++
.../namenode/sps/StoragePolicySatisfier.java| 61 +-
.../sps/StoragePolicySatisfyManager.java| 20 ++---
.../hdfs/server/protocol/NamenodeProtocol.java | 2 +-
.../sps/ExternalSPSBlockMoveTaskHandler.java| 4 +-
.../hdfs/server/sps/ExternalSPSContext.java | 85
.../sps/ExternalSPSFilePathCollector.java | 36 +
.../sps/ExternalStoragePolicySatisfier.java | 30 +--
.../src/main/proto/NamenodeProtocol.proto | 2 +-
.../TestBlockStorageMovementAttemptedItems.java | 16 ++--
.../sps/TestStoragePolicySatisfier.java | 66 +--
...stStoragePolicySatisfierWithStripedFile.java | 41 --
.../sps/TestExternalStoragePolicySatisfier.java | 35 +++-
27 files changed, 346 insertions(+), 414 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66e8f9b3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
index e4283c6..d9367fb 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
@@ -265,7 +265,7 @@ public class NamenodeProtocolServerSideTranslatorPB
implements
RpcController controller, GetNextSPSPathRequestProto request)
throws ServiceException {
try {
- String nextSPSPath = impl.getNextSPSPath();
+ Long nextSPSPath = impl.getNextSPSPath();
if (nextSPSPath == null) {
return GetNextSPSPathResponseProto.newBuilder().build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66e8f9b3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
index 97dee9b..3bd5986 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
@@ -267,7 +267,7 @@ public class NamenodeProtocolTranslatorPB implements
NamenodeProtocol,
}
@Override
- public String getNextSPSPath() throws IOException {
+ public Long getNextSPSPath() throws IOException {
GetNextSPSPathRequestProto req =
GetNextSPSPathRequestProto.newBuilder().build();
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66e8f9b3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
[49/50] [abbrv] hadoop git commit: HDFS-13381 : [SPS]: Use DFSUtilClient#makePathFromFileId() to prepare satisfier file path. Contributed by Rakesh R.
HDFS-13381 : [SPS]: Use DFSUtilClient#makePathFromFileId() to prepare satisfier
file path. Contributed by Rakesh R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/66e8f9b3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/66e8f9b3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/66e8f9b3
Branch: refs/heads/HDFS-10285
Commit: 66e8f9b31529226309c924226a53dead3e6fcf11
Parents: 2acc50b
Author: Uma Maheswara Rao G
Authored: Mon Jul 2 17:22:00 2018 -0700
Committer: Uma Maheswara Rao Gangumalla
Committed: Sun Aug 12 03:06:06 2018 -0700
--
.../NamenodeProtocolServerSideTranslatorPB.java | 2 +-
.../NamenodeProtocolTranslatorPB.java | 2 +-
.../server/blockmanagement/BlockManager.java| 2 +-
.../hdfs/server/namenode/FSNamesystem.java | 11 ---
.../hdfs/server/namenode/NameNodeRpcServer.java | 8 +-
.../hadoop/hdfs/server/namenode/Namesystem.java | 9 ---
.../sps/BlockStorageMovementAttemptedItems.java | 72 +++--
.../sps/BlockStorageMovementNeeded.java | 61 ++
.../hdfs/server/namenode/sps/Context.java | 45 ---
.../namenode/sps/DatanodeCacheManager.java | 4 +-
.../hdfs/server/namenode/sps/FileCollector.java | 13 +--
.../namenode/sps/IntraSPSNameNodeContext.java | 54 +
.../sps/IntraSPSNameNodeFileIdCollector.java| 14 ++--
.../hdfs/server/namenode/sps/ItemInfo.java | 34
.../hdfs/server/namenode/sps/SPSService.java| 31 +++
.../namenode/sps/StoragePolicySatisfier.java| 61 +-
.../sps/StoragePolicySatisfyManager.java| 20 ++---
.../hdfs/server/protocol/NamenodeProtocol.java | 2 +-
.../sps/ExternalSPSBlockMoveTaskHandler.java| 4 +-
.../hdfs/server/sps/ExternalSPSContext.java | 85
.../sps/ExternalSPSFilePathCollector.java | 36 +
.../sps/ExternalStoragePolicySatisfier.java | 30 +--
.../src/main/proto/NamenodeProtocol.proto | 2 +-
.../TestBlockStorageMovementAttemptedItems.java | 16 ++--
.../sps/TestStoragePolicySatisfier.java | 66 +--
...stStoragePolicySatisfierWithStripedFile.java | 41 --
.../sps/TestExternalStoragePolicySatisfier.java | 35 +++-
27 files changed, 346 insertions(+), 414 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66e8f9b3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
index e4283c6..d9367fb 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
@@ -265,7 +265,7 @@ public class NamenodeProtocolServerSideTranslatorPB
implements
RpcController controller, GetNextSPSPathRequestProto request)
throws ServiceException {
try {
- String nextSPSPath = impl.getNextSPSPath();
+ Long nextSPSPath = impl.getNextSPSPath();
if (nextSPSPath == null) {
return GetNextSPSPathResponseProto.newBuilder().build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66e8f9b3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
index 97dee9b..3bd5986 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
@@ -267,7 +267,7 @@ public class NamenodeProtocolTranslatorPB implements
NamenodeProtocol,
}
@Override
- public String getNextSPSPath() throws IOException {
+ public Long getNextSPSPath() throws IOException {
GetNextSPSPathRequestProto req =
GetNextSPSPathRequestProto.newBuilder().build();
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66e8f9b3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
[49/50] [abbrv] hadoop git commit: HDFS-13381 : [SPS]: Use DFSUtilClient#makePathFromFileId() to prepare satisfier file path. Contributed by Rakesh R.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc782499/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfier.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfier.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfier.java
index b05717a..ec5307b 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfier.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfier.java
@@ -108,8 +108,6 @@ public class TestStoragePolicySatisfier {
public static final long CAPACITY = 2 * 256 * 1024 * 1024;
public static final String FILE = "/testMoveToSatisfyStoragePolicy";
public static final int DEFAULT_BLOCK_SIZE = 1024;
- private ExternalBlockMovementListener blkMoveListener =
- new ExternalBlockMovementListener();
/**
* Sets hdfs cluster.
@@ -1282,8 +1280,8 @@ public class TestStoragePolicySatisfier {
//Queue limit can control the traverse logic to wait for some free
//entry in queue. After 10 files, traverse control will be on U.
-StoragePolicySatisfier sps = new
StoragePolicySatisfier(config);
-Context ctxt = new IntraSPSNameNodeContext(
+StoragePolicySatisfier sps = new StoragePolicySatisfier(config);
+Context ctxt = new IntraSPSNameNodeContext(
hdfsCluster.getNamesystem(),
hdfsCluster.getNamesystem().getBlockManager(), sps) {
@Override
@@ -1297,8 +1295,7 @@ public class TestStoragePolicySatisfier {
}
};
-FileCollector fileIDCollector = createFileIdCollector(sps, ctxt);
-sps.init(ctxt, fileIDCollector, null, null);
+sps.init(ctxt);
sps.getStorageMovementQueue().activate();
INode rootINode = fsDir.getINode("/root");
@@ -1314,13 +1311,6 @@ public class TestStoragePolicySatisfier {
dfs.delete(new Path("/root"), true);
}
- public FileCollector createFileIdCollector(
- StoragePolicySatisfier sps, Context ctxt) {
-FileCollector fileIDCollector = new IntraSPSNameNodeFileIdCollector(
-hdfsCluster.getNamesystem().getFSDirectory(), sps);
-return fileIDCollector;
- }
-
/**
* Test traverse when root parent got deleted.
* 1. Delete L when traversing Q
@@ -1351,8 +1341,8 @@ public class TestStoragePolicySatisfier {
// Queue limit can control the traverse logic to wait for some free
// entry in queue. After 10 files, traverse control will be on U.
-StoragePolicySatisfier sps = new
StoragePolicySatisfier(config);
-Context ctxt = new IntraSPSNameNodeContext(
+StoragePolicySatisfier sps = new StoragePolicySatisfier(config);
+Context ctxt = new IntraSPSNameNodeContext(
hdfsCluster.getNamesystem(),
hdfsCluster.getNamesystem().getBlockManager(), sps) {
@Override
@@ -1365,8 +1355,7 @@ public class TestStoragePolicySatisfier {
return true;
}
};
-FileCollector fileIDCollector = createFileIdCollector(sps, ctxt);
-sps.init(ctxt, fileIDCollector, null, null);
+sps.init(ctxt);
sps.getStorageMovementQueue().activate();
INode rootINode = fsDir.getINode("/root");
@@ -1383,12 +1372,12 @@ public class TestStoragePolicySatisfier {
}
private void assertTraversal(List expectedTraverseOrder,
- FSDirectory fsDir, StoragePolicySatisfier sps)
+ FSDirectory fsDir, StoragePolicySatisfier sps)
throws InterruptedException {
// Remove 10 element and make queue free, So other traversing will start.
for (int i = 0; i < 10; i++) {
String path = expectedTraverseOrder.remove(0);
- ItemInfo itemInfo = sps.getStorageMovementQueue().get();
+ ItemInfo itemInfo = sps.getStorageMovementQueue().get();
if (itemInfo == null) {
continue;
}
@@ -1403,7 +1392,7 @@ public class TestStoragePolicySatisfier {
// Check other element traversed in order and E, M, U, R, S should not be
// added in queue which we already removed from expected list
for (String path : expectedTraverseOrder) {
- ItemInfo itemInfo = sps.getStorageMovementQueue().get();
+ ItemInfo itemInfo = sps.getStorageMovementQueue().get();
if (itemInfo == null) {
continue;
}
@@ -1717,17 +1706,17 @@ public class TestStoragePolicySatisfier {
public void waitForAttemptedItems(long expectedBlkMovAttemptedCount,
int timeout) throws TimeoutException, InterruptedException {
BlockManager blockManager = hdfsCluster.getNamesystem().getBlockManager();
-final StoragePolicySatisfier sps =
-(StoragePolicySatisfier) blockManager.getSPSManager()
+final StoragePolicySatisfier sps =
+
[49/50] [abbrv] hadoop git commit: HDFS-13381 : [SPS]: Use DFSUtilClient#makePathFromFileId() to prepare satisfier file path. Contributed by Rakesh R.
HDFS-13381 : [SPS]: Use DFSUtilClient#makePathFromFileId() to prepare satisfier
file path. Contributed by Rakesh R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d3f163e8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d3f163e8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d3f163e8
Branch: refs/heads/HDFS-10285
Commit: d3f163e8cd4c2c57e3eb542edf2142911f67f72d
Parents: 8bc7252
Author: Uma Maheswara Rao G
Authored: Mon Jul 2 17:22:00 2018 -0700
Committer: Rakesh Radhakrishnan
Committed: Thu Jul 19 22:50:13 2018 +0530
--
.../NamenodeProtocolServerSideTranslatorPB.java | 2 +-
.../NamenodeProtocolTranslatorPB.java | 2 +-
.../server/blockmanagement/BlockManager.java| 2 +-
.../hdfs/server/namenode/FSNamesystem.java | 11 ---
.../hdfs/server/namenode/NameNodeRpcServer.java | 8 +-
.../hadoop/hdfs/server/namenode/Namesystem.java | 9 ---
.../sps/BlockStorageMovementAttemptedItems.java | 72 +++--
.../sps/BlockStorageMovementNeeded.java | 61 ++
.../hdfs/server/namenode/sps/Context.java | 45 ---
.../namenode/sps/DatanodeCacheManager.java | 4 +-
.../hdfs/server/namenode/sps/FileCollector.java | 13 +--
.../namenode/sps/IntraSPSNameNodeContext.java | 54 +
.../sps/IntraSPSNameNodeFileIdCollector.java| 14 ++--
.../hdfs/server/namenode/sps/ItemInfo.java | 34
.../hdfs/server/namenode/sps/SPSService.java| 31 +++
.../namenode/sps/StoragePolicySatisfier.java| 61 +-
.../sps/StoragePolicySatisfyManager.java| 20 ++---
.../hdfs/server/protocol/NamenodeProtocol.java | 2 +-
.../sps/ExternalSPSBlockMoveTaskHandler.java| 4 +-
.../hdfs/server/sps/ExternalSPSContext.java | 85
.../sps/ExternalSPSFilePathCollector.java | 36 +
.../sps/ExternalStoragePolicySatisfier.java | 30 +--
.../src/main/proto/NamenodeProtocol.proto | 2 +-
.../TestBlockStorageMovementAttemptedItems.java | 16 ++--
.../sps/TestStoragePolicySatisfier.java | 66 +--
...stStoragePolicySatisfierWithStripedFile.java | 41 --
.../sps/TestExternalStoragePolicySatisfier.java | 35 +++-
27 files changed, 346 insertions(+), 414 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3f163e8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
index e4283c6..d9367fb 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
@@ -265,7 +265,7 @@ public class NamenodeProtocolServerSideTranslatorPB
implements
RpcController controller, GetNextSPSPathRequestProto request)
throws ServiceException {
try {
- String nextSPSPath = impl.getNextSPSPath();
+ Long nextSPSPath = impl.getNextSPSPath();
if (nextSPSPath == null) {
return GetNextSPSPathResponseProto.newBuilder().build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3f163e8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
index 97dee9b..3bd5986 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
@@ -267,7 +267,7 @@ public class NamenodeProtocolTranslatorPB implements
NamenodeProtocol,
}
@Override
- public String getNextSPSPath() throws IOException {
+ public Long getNextSPSPath() throws IOException {
GetNextSPSPathRequestProto req =
GetNextSPSPathRequestProto.newBuilder().build();
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3f163e8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
[49/50] [abbrv] hadoop git commit: HDFS-13381 : [SPS]: Use DFSUtilClient#makePathFromFileId() to prepare satisfier file path. Contributed by Rakesh R.
HDFS-13381 : [SPS]: Use DFSUtilClient#makePathFromFileId() to prepare satisfier
file path. Contributed by Rakesh R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cb20be0e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cb20be0e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cb20be0e
Branch: refs/heads/HDFS-10285
Commit: cb20be0e9d42ac0330f729aa057b24ce0df3cb64
Parents: f446203
Author: Uma Maheswara Rao G
Authored: Mon Jul 2 17:22:00 2018 -0700
Committer: Rakesh Radhakrishnan
Committed: Sun Jul 15 20:23:44 2018 +0530
--
.../NamenodeProtocolServerSideTranslatorPB.java | 2 +-
.../NamenodeProtocolTranslatorPB.java | 2 +-
.../server/blockmanagement/BlockManager.java| 2 +-
.../hdfs/server/namenode/FSNamesystem.java | 11 ---
.../hdfs/server/namenode/NameNodeRpcServer.java | 8 +-
.../hadoop/hdfs/server/namenode/Namesystem.java | 9 ---
.../sps/BlockStorageMovementAttemptedItems.java | 72 +++--
.../sps/BlockStorageMovementNeeded.java | 61 ++
.../hdfs/server/namenode/sps/Context.java | 45 ---
.../namenode/sps/DatanodeCacheManager.java | 4 +-
.../hdfs/server/namenode/sps/FileCollector.java | 13 +--
.../namenode/sps/IntraSPSNameNodeContext.java | 54 +
.../sps/IntraSPSNameNodeFileIdCollector.java| 14 ++--
.../hdfs/server/namenode/sps/ItemInfo.java | 34
.../hdfs/server/namenode/sps/SPSService.java| 31 +++
.../namenode/sps/StoragePolicySatisfier.java| 61 +-
.../sps/StoragePolicySatisfyManager.java| 20 ++---
.../hdfs/server/protocol/NamenodeProtocol.java | 2 +-
.../sps/ExternalSPSBlockMoveTaskHandler.java| 4 +-
.../hdfs/server/sps/ExternalSPSContext.java | 85
.../sps/ExternalSPSFilePathCollector.java | 36 +
.../sps/ExternalStoragePolicySatisfier.java | 30 +--
.../src/main/proto/NamenodeProtocol.proto | 2 +-
.../TestBlockStorageMovementAttemptedItems.java | 16 ++--
.../sps/TestStoragePolicySatisfier.java | 66 +--
...stStoragePolicySatisfierWithStripedFile.java | 41 --
.../sps/TestExternalStoragePolicySatisfier.java | 35 +++-
27 files changed, 346 insertions(+), 414 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb20be0e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
index e4283c6..d9367fb 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
@@ -265,7 +265,7 @@ public class NamenodeProtocolServerSideTranslatorPB
implements
RpcController controller, GetNextSPSPathRequestProto request)
throws ServiceException {
try {
- String nextSPSPath = impl.getNextSPSPath();
+ Long nextSPSPath = impl.getNextSPSPath();
if (nextSPSPath == null) {
return GetNextSPSPathResponseProto.newBuilder().build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb20be0e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
index 97dee9b..3bd5986 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
@@ -267,7 +267,7 @@ public class NamenodeProtocolTranslatorPB implements
NamenodeProtocol,
}
@Override
- public String getNextSPSPath() throws IOException {
+ public Long getNextSPSPath() throws IOException {
GetNextSPSPathRequestProto req =
GetNextSPSPathRequestProto.newBuilder().build();
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb20be0e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
[49/50] [abbrv] hadoop git commit: HDFS-13381 : [SPS]: Use DFSUtilClient#makePathFromFileId() to prepare satisfier file path. Contributed by Rakesh R.
HDFS-13381 : [SPS]: Use DFSUtilClient#makePathFromFileId() to prepare satisfier
file path. Contributed by Rakesh R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/24bcc8c3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/24bcc8c3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/24bcc8c3
Branch: refs/heads/HDFS-10285
Commit: 24bcc8c3894622c692233bb3704be6a452168577
Parents: 02ebf5d
Author: Uma Maheswara Rao G
Authored: Mon Jul 2 17:22:00 2018 -0700
Committer: Rakesh Radhakrishnan
Committed: Thu Jul 12 17:04:55 2018 +0530
--
.../NamenodeProtocolServerSideTranslatorPB.java | 2 +-
.../NamenodeProtocolTranslatorPB.java | 2 +-
.../server/blockmanagement/BlockManager.java| 2 +-
.../hdfs/server/namenode/FSNamesystem.java | 11 ---
.../hdfs/server/namenode/NameNodeRpcServer.java | 8 +-
.../hadoop/hdfs/server/namenode/Namesystem.java | 9 ---
.../sps/BlockStorageMovementAttemptedItems.java | 72 +++--
.../sps/BlockStorageMovementNeeded.java | 61 ++
.../hdfs/server/namenode/sps/Context.java | 45 ---
.../namenode/sps/DatanodeCacheManager.java | 4 +-
.../hdfs/server/namenode/sps/FileCollector.java | 13 +--
.../namenode/sps/IntraSPSNameNodeContext.java | 54 +
.../sps/IntraSPSNameNodeFileIdCollector.java| 14 ++--
.../hdfs/server/namenode/sps/ItemInfo.java | 34
.../hdfs/server/namenode/sps/SPSService.java| 31 +++
.../namenode/sps/StoragePolicySatisfier.java| 61 +-
.../sps/StoragePolicySatisfyManager.java| 20 ++---
.../hdfs/server/protocol/NamenodeProtocol.java | 2 +-
.../sps/ExternalSPSBlockMoveTaskHandler.java| 4 +-
.../hdfs/server/sps/ExternalSPSContext.java | 85
.../sps/ExternalSPSFilePathCollector.java | 36 +
.../sps/ExternalStoragePolicySatisfier.java | 30 +--
.../src/main/proto/NamenodeProtocol.proto | 2 +-
.../TestBlockStorageMovementAttemptedItems.java | 16 ++--
.../sps/TestStoragePolicySatisfier.java | 66 +--
...stStoragePolicySatisfierWithStripedFile.java | 41 --
.../sps/TestExternalStoragePolicySatisfier.java | 35 +++-
27 files changed, 346 insertions(+), 414 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24bcc8c3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
index e4283c6..d9367fb 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
@@ -265,7 +265,7 @@ public class NamenodeProtocolServerSideTranslatorPB
implements
RpcController controller, GetNextSPSPathRequestProto request)
throws ServiceException {
try {
- String nextSPSPath = impl.getNextSPSPath();
+ Long nextSPSPath = impl.getNextSPSPath();
if (nextSPSPath == null) {
return GetNextSPSPathResponseProto.newBuilder().build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24bcc8c3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
index 97dee9b..3bd5986 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
@@ -267,7 +267,7 @@ public class NamenodeProtocolTranslatorPB implements
NamenodeProtocol,
}
@Override
- public String getNextSPSPath() throws IOException {
+ public Long getNextSPSPath() throws IOException {
GetNextSPSPathRequestProto req =
GetNextSPSPathRequestProto.newBuilder().build();
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24bcc8c3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
[49/50] [abbrv] hadoop git commit: HDFS-13381 : [SPS]: Use DFSUtilClient#makePathFromFileId() to prepare satisfier file path. Contributed by Rakesh R.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e2cb0ce/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfier.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfier.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfier.java
index b05717a..ec5307b 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfier.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfier.java
@@ -108,8 +108,6 @@ public class TestStoragePolicySatisfier {
public static final long CAPACITY = 2 * 256 * 1024 * 1024;
public static final String FILE = "/testMoveToSatisfyStoragePolicy";
public static final int DEFAULT_BLOCK_SIZE = 1024;
- private ExternalBlockMovementListener blkMoveListener =
- new ExternalBlockMovementListener();
/**
* Sets hdfs cluster.
@@ -1282,8 +1280,8 @@ public class TestStoragePolicySatisfier {
//Queue limit can control the traverse logic to wait for some free
//entry in queue. After 10 files, traverse control will be on U.
-StoragePolicySatisfier sps = new
StoragePolicySatisfier(config);
-Context ctxt = new IntraSPSNameNodeContext(
+StoragePolicySatisfier sps = new StoragePolicySatisfier(config);
+Context ctxt = new IntraSPSNameNodeContext(
hdfsCluster.getNamesystem(),
hdfsCluster.getNamesystem().getBlockManager(), sps) {
@Override
@@ -1297,8 +1295,7 @@ public class TestStoragePolicySatisfier {
}
};
-FileCollector fileIDCollector = createFileIdCollector(sps, ctxt);
-sps.init(ctxt, fileIDCollector, null, null);
+sps.init(ctxt);
sps.getStorageMovementQueue().activate();
INode rootINode = fsDir.getINode("/root");
@@ -1314,13 +1311,6 @@ public class TestStoragePolicySatisfier {
dfs.delete(new Path("/root"), true);
}
- public FileCollector createFileIdCollector(
- StoragePolicySatisfier sps, Context ctxt) {
-FileCollector fileIDCollector = new IntraSPSNameNodeFileIdCollector(
-hdfsCluster.getNamesystem().getFSDirectory(), sps);
-return fileIDCollector;
- }
-
/**
* Test traverse when root parent got deleted.
* 1. Delete L when traversing Q
@@ -1351,8 +1341,8 @@ public class TestStoragePolicySatisfier {
// Queue limit can control the traverse logic to wait for some free
// entry in queue. After 10 files, traverse control will be on U.
-StoragePolicySatisfier sps = new
StoragePolicySatisfier(config);
-Context ctxt = new IntraSPSNameNodeContext(
+StoragePolicySatisfier sps = new StoragePolicySatisfier(config);
+Context ctxt = new IntraSPSNameNodeContext(
hdfsCluster.getNamesystem(),
hdfsCluster.getNamesystem().getBlockManager(), sps) {
@Override
@@ -1365,8 +1355,7 @@ public class TestStoragePolicySatisfier {
return true;
}
};
-FileCollector fileIDCollector = createFileIdCollector(sps, ctxt);
-sps.init(ctxt, fileIDCollector, null, null);
+sps.init(ctxt);
sps.getStorageMovementQueue().activate();
INode rootINode = fsDir.getINode("/root");
@@ -1383,12 +1372,12 @@ public class TestStoragePolicySatisfier {
}
private void assertTraversal(List expectedTraverseOrder,
- FSDirectory fsDir, StoragePolicySatisfier sps)
+ FSDirectory fsDir, StoragePolicySatisfier sps)
throws InterruptedException {
// Remove 10 element and make queue free, So other traversing will start.
for (int i = 0; i < 10; i++) {
String path = expectedTraverseOrder.remove(0);
- ItemInfo itemInfo = sps.getStorageMovementQueue().get();
+ ItemInfo itemInfo = sps.getStorageMovementQueue().get();
if (itemInfo == null) {
continue;
}
@@ -1403,7 +1392,7 @@ public class TestStoragePolicySatisfier {
// Check other element traversed in order and E, M, U, R, S should not be
// added in queue which we already removed from expected list
for (String path : expectedTraverseOrder) {
- ItemInfo itemInfo = sps.getStorageMovementQueue().get();
+ ItemInfo itemInfo = sps.getStorageMovementQueue().get();
if (itemInfo == null) {
continue;
}
@@ -1717,17 +1706,17 @@ public class TestStoragePolicySatisfier {
public void waitForAttemptedItems(long expectedBlkMovAttemptedCount,
int timeout) throws TimeoutException, InterruptedException {
BlockManager blockManager = hdfsCluster.getNamesystem().getBlockManager();
-final StoragePolicySatisfier sps =
-(StoragePolicySatisfier) blockManager.getSPSManager()
+final StoragePolicySatisfier sps =
+
