This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new dbb2809 HDFS-15272. Backport HDFS-12862 to branch-3.1. Contributed by
Xiaoqiao He, Wang XL.
dbb2809 is described below
commit dbb28090262c5adb92fb346b34d290ad1934f72e
Author: He Xiaoqiao <[email protected]>
AuthorDate: Mon May 4 18:12:13 2020 -0700
HDFS-15272. Backport HDFS-12862 to branch-3.1. Contributed by Xiaoqiao He,
Wang XL.
Signed-off-by: Wei-Chiu Chuang <[email protected]>
---
.../hdfs/server/namenode/FSImageSerialization.java | 4 +-
.../hdfs/server/namenode/TestCacheDirectives.java | 66 +++++++++++++++++++++-
2 files changed, 67 insertions(+), 3 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
index 4d8b627..706b2bd 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
@@ -538,7 +538,7 @@ public class FSImageSerialization {
writeString(directive.getPool(), out);
}
if (directive.getExpiration() != null) {
- writeLong(directive.getExpiration().getMillis(), out);
+ writeLong(directive.getExpiration().getAbsoluteMillis(), out);
}
}
@@ -610,7 +610,7 @@ public class FSImageSerialization {
}
if (directive.getExpiration() != null) {
XMLUtils.addSaxString(contentHandler, "EXPIRATION",
- "" + directive.getExpiration().getMillis());
+ "" + directive.getExpiration().getAbsoluteMillis());
}
}
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
index 613aedf..1338886 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
@@ -55,12 +55,14 @@ import org.apache.hadoop.fs.InvalidRequestException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.client.impl.BlockReaderTestUtil;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.CacheDirective;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo.Expiration;
@@ -78,6 +80,7 @@ import
org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.CachedBl
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.io.nativeio.NativeIO.POSIX.CacheManipulator;
@@ -1560,4 +1563,65 @@ public class TestCacheDirectives {
cm.setCachedLocations(locations);
Mockito.verifyZeroInteractions(locations);
}
-}
+
+ @Test(timeout=120000)
+ public void testExpiryTimeConsistency() throws Exception {
+ conf.setInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
+ MiniDFSCluster dfsCluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES)
+ .nnTopology(MiniDFSNNTopology.simpleHATopology())
+ .build();
+ dfsCluster.transitionToActive(0);
+
+ DistributedFileSystem fs = dfsCluster.getFileSystem(0);
+ final NameNode ann = dfsCluster.getNameNode(0);
+
+ final Path filename = new Path("/file");
+ final short replication = (short) 3;
+ DFSTestUtil.createFile(fs, filename, 1, replication, 0x0BAC);
+ fs.addCachePool(new CachePoolInfo("pool"));
+ long id = fs.addCacheDirective(
+ new CacheDirectiveInfo.Builder().setPool("pool").setPath(filename)
+ .setExpiration(CacheDirectiveInfo.Expiration.newRelative(86400000))
+ .setReplication(replication).build());
+ fs.modifyCacheDirective(new CacheDirectiveInfo.Builder()
+ .setId(id)
+ .setExpiration(CacheDirectiveInfo.Expiration.newRelative(172800000))
+ .build());
+ final NameNode sbn = dfsCluster.getNameNode(1);
+ final CacheManager annCachemanager = ann.getNamesystem().getCacheManager();
+ final CacheManager sbnCachemanager = sbn.getNamesystem().getCacheManager();
+ HATestUtil.waitForStandbyToCatchUp(ann, sbn);
+ GenericTestUtils.waitFor(() -> {
+ boolean isConsistence = false;
+ ann.getNamesystem().readLock();
+ try {
+ sbn.getNamesystem().readLock();
+ try {
+ Iterator<CacheDirective> annDirectivesIt = annCachemanager.
+ getCacheDirectives().iterator();
+ Iterator<CacheDirective> sbnDirectivesIt = sbnCachemanager.
+ getCacheDirectives().iterator();
+ if (annDirectivesIt.hasNext() && sbnDirectivesIt.hasNext()) {
+ CacheDirective annDirective = annDirectivesIt.next();
+ CacheDirective sbnDirective = sbnDirectivesIt.next();
+ if (annDirective.getExpiryTimeString().
+ equals(sbnDirective.getExpiryTimeString())) {
+ isConsistence = true;
+ }
+ }
+ } finally {
+ sbn.getNamesystem().readUnlock();
+ }
+ } finally {
+ ann.getNamesystem().readUnlock();
+ }
+ if (!isConsistence) {
+ LOG.info("testEexpiryTimeConsistency:"
+ + "ANN CacheDirective Status is inconsistent with SBN");
+ }
+ return isConsistence;
+ }, 500, 120000);
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
