Repository: bookkeeper Updated Branches: refs/heads/master 9c937f5d8 -> bac3ed360
BOOKKEEPER-879: Record ledger creation time Author: eolivelli <eolive...@gmail.com> Reviewers: Sijie Guo <si...@apache.org>, Flavio Junqueira <f...@apache.org> Closes #2 from eolivelli/master and squashes the following commits: 509ad4f [eolivelli] BOOKKEEPER-879: Record ledger creation time 227d50c [eolivelli] BOOKKEEPER-879: Record ledger creation time 94cdee6 [eolivelli] BOOKKEEPER-879: Record ledger creation time 2a8e700 [eolivelli] BOOKKEEPER-879: Record ledger creation time c43f0b4 [eolivelli] BOOKKEEPER-879: Record ledger creation time e33f2e0 [eolivelli] BOOKKEEPER-879: Record ledger creation time Project: http://git-wip-us.apache.org/repos/asf/bookkeeper/repo Commit: http://git-wip-us.apache.org/repos/asf/bookkeeper/commit/bac3ed36 Tree: http://git-wip-us.apache.org/repos/asf/bookkeeper/tree/bac3ed36 Diff: http://git-wip-us.apache.org/repos/asf/bookkeeper/diff/bac3ed36 Branch: refs/heads/master Commit: bac3ed3603f9dc9864beb610bea215c186ef4fab Parents: 9c937f5 Author: eolivelli <eolive...@gmail.com> Authored: Tue Feb 23 10:16:06 2016 -0800 Committer: Sijie Guo <si...@apache.org> Committed: Tue Feb 23 10:16:06 2016 -0800 ---------------------------------------------------------------------- .../bookkeeper/client/LedgerMetadata.java | 26 +++++- .../meta/AbstractZkLedgerManager.java | 9 +- .../bookkeeper/meta/MSLedgerManagerFactory.java | 3 +- .../apache/bookkeeper/proto/DataFormats.java | 86 ++++++++++++++++---- .../src/main/proto/DataFormats.proto | 2 + .../bookkeeper/client/ListLedgersTest.java | 25 ++++++ 6 files changed, 131 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/bac3ed36/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java ---------------------------------------------------------------------- diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java index a20f34a..a58adba 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java @@ -37,6 +37,7 @@ import java.util.SortedMap; import java.util.TreeMap; import static com.google.common.base.Charsets.UTF_8; +import com.google.common.base.Optional; /** * This class encapsulates all the ledger metadata that is persistently stored @@ -66,6 +67,7 @@ public class LedgerMetadata { private int ackQuorumSize; private long length; private long lastEntryId; + private long ctime; private LedgerMetadataFormat.State state; private SortedMap<Long, ArrayList<BookieSocketAddress>> ensembles = @@ -82,6 +84,7 @@ public class LedgerMetadata { this.ensembleSize = ensembleSize; this.writeQuorumSize = writeQuorumSize; this.ackQuorumSize = ackQuorumSize; + this.ctime = System.currentTimeMillis(); /* * It is set in PendingReadOp.readEntryComplete, and @@ -104,6 +107,7 @@ public class LedgerMetadata { LedgerMetadata(LedgerMetadata other) { this.ensembleSize = other.ensembleSize; this.writeQuorumSize = other.writeQuorumSize; + this.ctime = other.ctime; this.ackQuorumSize = other.ackQuorumSize; this.length = other.length; this.lastEntryId = other.lastEntryId; @@ -150,6 +154,14 @@ public class LedgerMetadata { return writeQuorumSize; } + /** + * Get the creation timestamp of the ledger + * @return + */ + public long getCtime() { + return ctime; + } + public int getAckQuorumSize() { return ackQuorumSize; } @@ -255,7 +267,7 @@ public class LedgerMetadata { LedgerMetadataFormat.Builder builder = LedgerMetadataFormat.newBuilder(); builder.setQuorumSize(writeQuorumSize).setAckQuorumSize(ackQuorumSize) .setEnsembleSize(ensembleSize).setLength(length) - .setState(state).setLastEntryId(lastEntryId); + .setState(state).setLastEntryId(lastEntryId).setCtime(ctime); if (hasPassword) { builder.setDigestType(digestType).setPassword(ByteString.copyFrom(password)); @@ -308,11 +320,13 @@ public class LedgerMetadata { * byte array to parse * @param version * version of the ledger metadata + * @param msCtime + * metadata store creation time, used for legacy ledgers * @return LedgerConfig * @throws IOException * if the given byte[] cannot be parsed */ - public static LedgerMetadata parseConfig(byte[] bytes, Version version) throws IOException { + public static LedgerMetadata parseConfig(byte[] bytes, Version version, Optional<Long> msCtime) throws IOException { LedgerMetadata lc = new LedgerMetadata(); lc.version = version; @@ -351,7 +365,12 @@ public class LedgerMetadata { LedgerMetadataFormat.Builder builder = LedgerMetadataFormat.newBuilder(); TextFormat.merge(reader, builder); LedgerMetadataFormat data = builder.build(); - lc.writeQuorumSize = data.getQuorumSize(); + lc.writeQuorumSize = data.getQuorumSize(); + if (data.hasCtime()) { + lc.ctime = data.getCtime(); + } else if (msCtime.isPresent()) { + lc.ctime = msCtime.get(); + } if (data.hasAckQuorumSize()) { lc.ackQuorumSize = data.getAckQuorumSize(); } else { @@ -463,6 +482,7 @@ public class LedgerMetadata { if (metadataFormatVersion != newMeta.metadataFormatVersion || ensembleSize != newMeta.ensembleSize || writeQuorumSize != newMeta.writeQuorumSize || + ctime != newMeta.ctime || ackQuorumSize != newMeta.ackQuorumSize || length != newMeta.length || state != newMeta.state || http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/bac3ed36/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java ---------------------------------------------------------------------- diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java index 1d7c591..fbf38ad 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java @@ -17,6 +17,7 @@ */ package org.apache.bookkeeper.meta; +import com.google.common.base.Optional; import java.io.IOException; import java.util.HashSet; import java.util.List; @@ -355,10 +356,14 @@ abstract class AbstractZkLedgerManager implements LedgerManager, Watcher { readCb.operationComplete(BKException.Code.ZKException, null); return; } - + if (stat == null) { + LOG.error("Could not parse ledger metadata for ledger: " + ledgerId+". Stat object is null"); + readCb.operationComplete(BKException.Code.ZKException, null); + return; + } LedgerMetadata metadata; try { - metadata = LedgerMetadata.parseConfig(data, new ZkVersion(stat.getVersion())); + metadata = LedgerMetadata.parseConfig(data, new ZkVersion(stat.getVersion()), Optional.of(stat.getCtime())); } catch (IOException e) { LOG.error("Could not parse ledger metadata for ledger: " + ledgerId, e); readCb.operationComplete(BKException.Code.ZKException, null); http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/bac3ed36/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java ---------------------------------------------------------------------- diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java index 9f7ef38..8a82d41 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java @@ -17,6 +17,7 @@ */ package org.apache.bookkeeper.meta; +import com.google.common.base.Optional; import static org.apache.bookkeeper.metastore.MetastoreTable.ALL_FIELDS; import static org.apache.bookkeeper.metastore.MetastoreTable.NON_FIELDS; @@ -408,7 +409,7 @@ public class MSLedgerManagerFactory extends LedgerManagerFactory { LedgerMetadata metadata; try { metadata = LedgerMetadata - .parseConfig(value.getValue().getField(META_FIELD), value.getVersion()); + .parseConfig(value.getValue().getField(META_FIELD), value.getVersion(), Optional.<Long>absent()); } catch (IOException e) { LOG.error("Could not parse ledger metadata for ledger " + ledgerId + " : ", e); readCb.operationComplete(BKException.Code.MetaStoreException, null); http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/bac3ed36/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/DataFormats.java ---------------------------------------------------------------------- diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/DataFormats.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/DataFormats.java index 65d5444..4552144 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/DataFormats.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/DataFormats.java @@ -52,6 +52,10 @@ public final class DataFormats { // optional int32 ackQuorumSize = 9; boolean hasAckQuorumSize(); int getAckQuorumSize(); + + // optional int64 ctime = 10; + boolean hasCtime(); + long getCtime(); } public static final class LedgerMetadataFormat extends com.google.protobuf.GeneratedMessage @@ -781,6 +785,16 @@ public final class DataFormats { return ackQuorumSize_; } + // optional int64 ctime = 10; + public static final int CTIME_FIELD_NUMBER = 10; + private long ctime_; + public boolean hasCtime() { + return ((bitField0_ & 0x00000100) == 0x00000100); + } + public long getCtime() { + return ctime_; + } + private void initFields() { quorumSize_ = 0; ensembleSize_ = 0; @@ -791,6 +805,7 @@ public final class DataFormats { digestType_ = org.apache.bookkeeper.proto.DataFormats.LedgerMetadataFormat.DigestType.CRC32; password_ = com.google.protobuf.ByteString.EMPTY; ackQuorumSize_ = 0; + ctime_ = 0L; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -853,6 +868,9 @@ public final class DataFormats { if (((bitField0_ & 0x00000080) == 0x00000080)) { output.writeInt32(9, ackQuorumSize_); } + if (((bitField0_ & 0x00000100) == 0x00000100)) { + output.writeInt64(10, ctime_); + } getUnknownFields().writeTo(output); } @@ -898,6 +916,10 @@ public final class DataFormats { size += com.google.protobuf.CodedOutputStream .computeInt32Size(9, ackQuorumSize_); } + if (((bitField0_ & 0x00000100) == 0x00000100)) { + size += com.google.protobuf.CodedOutputStream + .computeInt64Size(10, ctime_); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -1045,6 +1067,8 @@ public final class DataFormats { bitField0_ = (bitField0_ & ~0x00000080); ackQuorumSize_ = 0; bitField0_ = (bitField0_ & ~0x00000100); + ctime_ = 0L; + bitField0_ = (bitField0_ & ~0x00000200); return this; } @@ -1124,6 +1148,10 @@ public final class DataFormats { to_bitField0_ |= 0x00000080; } result.ackQuorumSize_ = ackQuorumSize_; + if (((from_bitField0_ & 0x00000200) == 0x00000200)) { + to_bitField0_ |= 0x00000100; + } + result.ctime_ = ctime_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -1190,6 +1218,9 @@ public final class DataFormats { if (other.hasAckQuorumSize()) { setAckQuorumSize(other.getAckQuorumSize()); } + if (other.hasCtime()) { + setCtime(other.getCtime()); + } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -1301,6 +1332,11 @@ public final class DataFormats { ackQuorumSize_ = input.readInt32(); break; } + case 80: { + bitField0_ |= 0x00000200; + ctime_ = input.readInt64(); + break; + } } } } @@ -1670,6 +1706,27 @@ public final class DataFormats { return this; } + // optional int64 ctime = 10; + private long ctime_ ; + public boolean hasCtime() { + return ((bitField0_ & 0x00000200) == 0x00000200); + } + public long getCtime() { + return ctime_; + } + public Builder setCtime(long value) { + bitField0_ |= 0x00000200; + ctime_ = value; + onChanged(); + return this; + } + public Builder clearCtime() { + bitField0_ = (bitField0_ & ~0x00000200); + ctime_ = 0L; + onChanged(); + return this; + } + // @@protoc_insertion_point(builder_scope:LedgerMetadataFormat) } @@ -3993,7 +4050,7 @@ public final class DataFormats { descriptor; static { java.lang.String[] descriptorData = { - "\n src/main/proto/DataFormats.proto\"\262\003\n\024L" + + "\n src/main/proto/DataFormats.proto\"\301\003\n\024L" + "edgerMetadataFormat\022\022\n\nquorumSize\030\001 \002(\005\022" + "\024\n\014ensembleSize\030\002 \002(\005\022\016\n\006length\030\003 \002(\003\022\023\n" + "\013lastEntryId\030\004 \001(\003\0220\n\005state\030\005 \002(\0162\033.Ledg" + @@ -4001,18 +4058,19 @@ public final class DataFormats { "\006 \003(\0132\035.LedgerMetadataFormat.Segment\0224\n\n" + "digestType\030\007 \001(\0162 .LedgerMetadataFormat." + "DigestType\022\020\n\010password\030\010 \001(\014\022\025\n\rackQuoru" + - "mSize\030\t \001(\005\0327\n\007Segment\022\026\n\016ensembleMember" + - "\030\001 \003(\t\022\024\n\014firstEntryId\030\002 \002(\003\".\n\005State\022\010\n", - "\004OPEN\020\001\022\017\n\013IN_RECOVERY\020\002\022\n\n\006CLOSED\020\003\"!\n\n" + - "DigestType\022\t\n\005CRC32\020\001\022\010\n\004HMAC\020\002\"@\n\037Ledge" + - "rRereplicationLayoutFormat\022\014\n\004type\030\001 \002(\t" + - "\022\017\n\007version\030\002 \002(\005\".\n\033UnderreplicatedLedg" + - "erFormat\022\017\n\007replica\030\001 \003(\t\"^\n\014CookieForma" + - "t\022\022\n\nbookieHost\030\001 \002(\t\022\022\n\njournalDir\030\002 \002(" + - "\t\022\022\n\nledgerDirs\030\003 \002(\t\022\022\n\ninstanceId\030\004 \001(" + - "\t\"\"\n\016LockDataFormat\022\020\n\010bookieId\030\001 \001(\t\"%\n" + - "\021AuditorVoteFormat\022\020\n\010bookieId\030\001 \001(\tB\037\n\033" + - "org.apache.bookkeeper.protoH\001" + "mSize\030\t \001(\005\022\r\n\005ctime\030\n \001(\003\0327\n\007Segment\022\026\n" + + "\016ensembleMember\030\001 \003(\t\022\024\n\014firstEntryId\030\002 ", + "\002(\003\".\n\005State\022\010\n\004OPEN\020\001\022\017\n\013IN_RECOVERY\020\002\022" + + "\n\n\006CLOSED\020\003\"!\n\nDigestType\022\t\n\005CRC32\020\001\022\010\n\004" + + "HMAC\020\002\"@\n\037LedgerRereplicationLayoutForma" + + "t\022\014\n\004type\030\001 \002(\t\022\017\n\007version\030\002 \002(\005\".\n\033Unde" + + "rreplicatedLedgerFormat\022\017\n\007replica\030\001 \003(\t" + + "\"^\n\014CookieFormat\022\022\n\nbookieHost\030\001 \002(\t\022\022\n\n" + + "journalDir\030\002 \002(\t\022\022\n\nledgerDirs\030\003 \002(\t\022\022\n\n" + + "instanceId\030\004 \001(\t\"\"\n\016LockDataFormat\022\020\n\010bo" + + "okieId\030\001 \001(\t\"%\n\021AuditorVoteFormat\022\020\n\010boo" + + "kieId\030\001 \001(\tB\037\n\033org.apache.bookkeeper.pro", + "toH\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -4024,7 +4082,7 @@ public final class DataFormats { internal_static_LedgerMetadataFormat_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_LedgerMetadataFormat_descriptor, - new java.lang.String[] { "QuorumSize", "EnsembleSize", "Length", "LastEntryId", "State", "Segment", "DigestType", "Password", "AckQuorumSize", }, + new java.lang.String[] { "QuorumSize", "EnsembleSize", "Length", "LastEntryId", "State", "Segment", "DigestType", "Password", "AckQuorumSize", "Ctime", }, org.apache.bookkeeper.proto.DataFormats.LedgerMetadataFormat.class, org.apache.bookkeeper.proto.DataFormats.LedgerMetadataFormat.Builder.class); internal_static_LedgerMetadataFormat_Segment_descriptor = http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/bac3ed36/bookkeeper-server/src/main/proto/DataFormats.proto ---------------------------------------------------------------------- diff --git a/bookkeeper-server/src/main/proto/DataFormats.proto b/bookkeeper-server/src/main/proto/DataFormats.proto index 67623ac..6d97c3a 100644 --- a/bookkeeper-server/src/main/proto/DataFormats.proto +++ b/bookkeeper-server/src/main/proto/DataFormats.proto @@ -48,6 +48,8 @@ message LedgerMetadataFormat { optional bytes password = 8; optional int32 ackQuorumSize = 9; + + optional int64 ctime = 10; } message LedgerRereplicationLayoutFormat { http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/bac3ed36/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java ---------------------------------------------------------------------- diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java index d34c064..70ff6e0 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java @@ -108,4 +108,29 @@ public class ListLedgersTest extends BaseTestCase { Assert.fail("Remove is not supported, we shouln't have reached this point"); } + + @Test(timeout = 60000) + public void testCtimeRecorded() + throws Exception { + + ClientConfiguration conf = new ClientConfiguration() + .setZkServers(zkUtil.getZooKeeperConnectString()); + + BookKeeper bkc = new BookKeeper(conf); + + bkc.createLedger(digestType, "testPasswd". + getBytes()).close(); + + BookKeeperAdmin admin = new BookKeeperAdmin(zkUtil. + getZooKeeperConnectString()); + Iterable<Long> iterable = admin.listLedgers(); + + for (Long lId : iterable) { + LedgerHandle ledger = bkc.openLedger(lId, digestType, "testPasswd".getBytes()); + LedgerMetadata metaData = ledger.getLedgerMetadata(); + Assert.assertTrue("ctime was not recorded", metaData.getCtime() > 0); + } + + } + }
