Repository: hadoop Updated Branches: refs/heads/HDFS-7240 04b39066d -> f6ed18633
HDFS-12170. Ozone: OzoneFileSystem: KSM should maintain key creation time and modification time. Contributed by Mukul Kumar Singh. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f6ed1863 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f6ed1863 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f6ed1863 Branch: refs/heads/HDFS-7240 Commit: f6ed186330fbc0d58374e1f167e73c0de870fdde Parents: 04b3906 Author: Yiqun Lin <yq...@apache.org> Authored: Thu Jul 27 11:43:01 2017 +0800 Committer: Yiqun Lin <yq...@apache.org> Committed: Thu Jul 27 11:43:01 2017 +0800 ---------------------------------------------------------------------- .../main/java/org/apache/hadoop/util/Time.java | 9 +++++ .../apache/hadoop/ksm/helpers/KsmKeyInfo.java | 35 ++++++++++++++++++-- .../main/proto/KeySpaceManagerProtocol.proto | 2 ++ .../apache/hadoop/ozone/ksm/KeyManagerImpl.java | 4 +++ .../hadoop/ozone/web/handlers/KeyHandler.java | 2 +- .../hadoop/ozone/web/response/KeyInfo.java | 19 +++++++++++ .../web/storage/DistributedStorageHandler.java | 4 ++- .../hadoop/ozone/ksm/TestKeySpaceManager.java | 8 ++++- 8 files changed, 77 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ed1863/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Time.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Time.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Time.java index db5a567..e099c8f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Time.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Time.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.util; +import java.text.ParseException; import java.text.SimpleDateFormat; import org.apache.hadoop.classification.InterfaceAudience; @@ -82,4 +83,12 @@ public final class Time { public static String formatTime(long millis) { return DATE_FORMAT.get().format(millis); } + + /** + * Convert time in human readable format to millisecond. + * @return time in milliseconds + */ + public static long formatDate(String date) throws ParseException { + return DATE_FORMAT.get().parse(date).getTime(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ed1863/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmKeyInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmKeyInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmKeyInfo.java index 76d22be..05748e8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmKeyInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmKeyInfo.java @@ -35,10 +35,13 @@ public final class KsmKeyInfo { private final String blockID; private final long dataSize; private final boolean shouldCreateContainer; + private final long creationTime; + private final long modificationTime; private KsmKeyInfo(String volumeName, String bucketName, String keyName, long dataSize, String blockID, String containerName, - boolean shouldCreateContainer) { + boolean shouldCreateContainer, long creationTime, + long modificationTime) { this.volumeName = volumeName; this.bucketName = bucketName; this.keyName = keyName; @@ -46,6 +49,8 @@ public final class KsmKeyInfo { this.blockID = blockID; this.dataSize = dataSize; this.shouldCreateContainer = shouldCreateContainer; + this.creationTime = creationTime; + this.modificationTime = modificationTime; } public String getVolumeName() { @@ -76,6 +81,14 @@ public final class KsmKeyInfo { return shouldCreateContainer; } + public long getCreationTime() { + return creationTime; + } + + public long getModificationTime() { + return modificationTime; + } + /** * Builder of KsmKeyInfo. */ @@ -87,6 +100,8 @@ public final class KsmKeyInfo { private String blockID; private long dataSize; private boolean shouldCreateContainer; + private long creationTime; + private long modificationTime; public Builder setVolumeName(String volume) { this.volumeName = volume; @@ -123,10 +138,20 @@ public final class KsmKeyInfo { return this; } + public Builder setCreationTime(long creationTime) { + this.creationTime = creationTime; + return this; + } + + public Builder setModificationTime(long modificationTime) { + this.modificationTime = modificationTime; + return this; + } + public KsmKeyInfo build() { return new KsmKeyInfo( volumeName, bucketName, keyName, dataSize, blockID, containerName, - shouldCreateContainer); + shouldCreateContainer, creationTime, modificationTime); } } @@ -139,6 +164,8 @@ public final class KsmKeyInfo { .setBlockKey(blockID) .setContainerName(containerName) .setShouldCreateContainer(shouldCreateContainer) + .setCreationTime(creationTime) + .setModificationTime(modificationTime) .build(); } @@ -150,7 +177,9 @@ public final class KsmKeyInfo { keyInfo.getDataSize(), keyInfo.getBlockKey(), keyInfo.getContainerName(), - keyInfo.getShouldCreateContainer()); + keyInfo.getShouldCreateContainer(), + keyInfo.getCreationTime(), + keyInfo.getModificationTime()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ed1863/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto index b8dbafc..f641898 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto @@ -233,6 +233,8 @@ message KeyInfo { required string blockKey = 5; required string containerName = 6; required bool shouldCreateContainer = 7; + required uint64 creationTime = 8; + required uint64 modificationTime = 9; } message LocateKeyRequest { http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ed1863/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java index aee3f7c..70e687b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java @@ -26,6 +26,7 @@ import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.Del import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock; import org.apache.hadoop.scm.container.common.helpers.DeleteBlockResult; import org.apache.hadoop.scm.protocol.ScmBlockLocationProtocol; +import org.apache.hadoop.util.Time; import org.iq80.leveldb.DBException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -95,6 +96,7 @@ public class KeyManagerImpl implements KeyManager { // metadata entry in case of 0 length key. AllocatedBlock allocatedBlock = scmBlockClient.allocateBlock(Math.max(args.getDataSize(), 1)); + long currentTime = Time.now(); KsmKeyInfo keyBlock = new KsmKeyInfo.Builder() .setVolumeName(args.getVolumeName()) .setBucketName(args.getBucketName()) @@ -103,6 +105,8 @@ public class KeyManagerImpl implements KeyManager { .setBlockID(allocatedBlock.getKey()) .setContainerName(allocatedBlock.getPipeline().getContainerName()) .setShouldCreateContainer(allocatedBlock.getCreateContainer()) + .setCreationTime(currentTime) + .setModificationTime(currentTime) .build(); metadataManager.put(keyKey, keyBlock.getProtobuf().toByteArray()); LOG.debug("Key {} allocated in volume {} bucket {}", http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ed1863/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyHandler.java index 10528e0..0655a0f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyHandler.java @@ -73,7 +73,7 @@ public class KeyHandler implements Keys { * @param input - The body as an Input Stream * @param request - Http request * @param headers - Parsed http Headers. - * @param info - UriInfo + * @param uriInfo - UriInfo * * @return Response * http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ed1863/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java index e5cfd21..34885f6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java @@ -69,6 +69,7 @@ public class KeyInfo implements Comparable<KeyInfo> { private long version; private String md5hash; private String createdOn; + private String modifiedOn; private long size; private String keyName; @@ -84,6 +85,15 @@ public class KeyInfo implements Comparable<KeyInfo> { } /** + * When this key was modified. + * + * @return Date String + */ + public String getModifiedOn() { + return modifiedOn; + } + + /** * When this key was created. * * @param createdOn - Date String @@ -93,6 +103,15 @@ public class KeyInfo implements Comparable<KeyInfo> { } /** + * When this key was modified. + * + * @param modifiedOn - Date String + */ + public void setModifiedOn(String modifiedOn) { + this.modifiedOn = modifiedOn; + } + + /** * Full path to where the actual data for this key is stored. * * @return String http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ed1863/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java index 57dcc32..1bf5b11 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java @@ -66,6 +66,7 @@ import org.apache.hadoop.scm.storage.ChunkInputStream; import org.apache.hadoop.scm.storage.ChunkOutputStream; import org.apache.hadoop.scm.storage.ContainerProtocolCalls; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -482,7 +483,8 @@ public final class DistributedStorageHandler implements StorageHandler { keyInfo.setVersion(0); keyInfo.setKeyName(ksmKeyInfo.getKeyName()); keyInfo.setSize(ksmKeyInfo.getDataSize()); - + keyInfo.setCreatedOn(Time.formatTime(ksmKeyInfo.getCreationTime())); + keyInfo.setModifiedOn(Time.formatTime(ksmKeyInfo.getModificationTime())); return keyInfo; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ed1863/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java index 18a5114..5936ccd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java @@ -44,6 +44,7 @@ import org.apache.hadoop.ozone.web.handlers.ListArgs; import org.apache.hadoop.ozone.web.response.ListBuckets; import org.apache.hadoop.ozone.web.response.ListKeys; import org.apache.hadoop.ozone.web.response.ListVolumes; +import org.apache.hadoop.util.Time; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -55,6 +56,7 @@ import org.junit.rules.ExpectedException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.text.ParseException; import java.util.HashSet; import java.util.LinkedList; import java.util.Random; @@ -941,11 +943,13 @@ public class TestKeySpaceManager { * @throws OzoneException */ @Test - public void testGetKeyInfo() throws IOException, OzoneException { + public void testGetKeyInfo() throws IOException, + OzoneException, ParseException { String userName = "user" + RandomStringUtils.randomNumeric(5); String adminName = "admin" + RandomStringUtils.randomNumeric(5); String volumeName = "volume" + RandomStringUtils.randomNumeric(5); String bucketName = "bucket" + RandomStringUtils.randomNumeric(5); + long currentTime = Time.monotonicNow(); VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs); createVolumeArgs.setUserName(userName); @@ -966,6 +970,8 @@ public class TestKeySpaceManager { stream.close(); KeyInfo keyInfo = storageHandler.getKeyInfo(keyArgs); + Assert.assertTrue(Time.formatDate(keyInfo.getCreatedOn()) >= currentTime); + Assert.assertTrue(Time.formatDate(keyInfo.getModifiedOn()) >= currentTime); Assert.assertEquals(keyName, keyInfo.getKeyName()); Assert.assertEquals(4096, keyInfo.getSize()); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
