This is an automated email from the ASF dual-hosted git repository.
zhaijia pushed a commit to branch branch-4.6
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/branch-4.6 by this push:
new 4e74ee1 ISSUE #490: Add a flag to not serialize `ctime` field
4e74ee1 is described below
commit 4e74ee10c21940b3b3d5b64dae988bb6aa8e715c
Author: Sijie Guo <[email protected]>
AuthorDate: Sun Nov 12 15:02:25 2017 +0800
ISSUE #490: Add a flag to not serialize `ctime` field
Descriptions of the changes in this PR:
This request is for DL bookkeeper version upgrade. Because twitter's bk
version doesn't include this change, so any ledgers written by new bk version
will fail to be read by old version. To simplify the upgrade story, it is good
to allow not serialize ctime field.
Author: Sijie Guo <[email protected]>
Reviewers: Enrico Olivelli <[email protected]>, Jia Zhai <None>, Sijie
Guo <[email protected]>, Matteo Merli <[email protected]>
This closes #718 from sijie/flag_to_not_serialize_ctime, closes #490
(cherry picked from commit a9b64e61598a987ffa6249ab398215436a7928f2)
Signed-off-by: Jia Zhai <[email protected]>
---
.../apache/bookkeeper/client/LedgerCreateOp.java | 9 ++-
.../apache/bookkeeper/client/LedgerMetadata.java | 43 ++++++++-----
.../bookkeeper/conf/ClientConfiguration.java | 31 +++++++++-
.../bookkeeper/client/LedgerMetadataTest.java | 70 ++++++++++++++++++++++
.../apache/bookkeeper/client/ListLedgersTest.java | 22 -------
.../client/api/BookKeeperBuildersTest.java | 10 +++-
.../replication/AuditorLedgerCheckerTest.java | 2 +-
7 files changed, 146 insertions(+), 41 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerCreateOp.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerCreateOp.java
index 4cc62a0..fc37c04 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerCreateOp.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerCreateOp.java
@@ -93,7 +93,14 @@ class LedgerCreateOp implements GenericCallback<Void> {
LedgerCreateOp(BookKeeper bk, int ensembleSize, int writeQuorumSize, int
ackQuorumSize, DigestType digestType,
byte[] passwd, CreateCallback cb, Object ctx, final Map<String,
byte[]> customMetadata) {
this.bk = bk;
- this.metadata = new LedgerMetadata(ensembleSize, writeQuorumSize,
ackQuorumSize, digestType, passwd, customMetadata);
+ this.metadata = new LedgerMetadata(
+ ensembleSize,
+ writeQuorumSize,
+ ackQuorumSize,
+ digestType,
+ passwd,
+ customMetadata,
+ bk.getConf().getStoreSystemtimeAsLedgerCreationTime());
this.digestType = digestType;
this.passwd = passwd;
this.cb = cb;
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
index 5683105..0a0b87c 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
@@ -75,6 +75,7 @@ public class LedgerMetadata {
private long length;
private long lastEntryId;
private long ctime;
+ private boolean storeSystemtimeAsLedgerCreationTime;
private LedgerMetadataFormat.State state;
private SortedMap<Long, ArrayList<BookieSocketAddress>> ensembles =
@@ -88,12 +89,18 @@ public class LedgerMetadata {
private Map<String, byte[]> customMetadata = Maps.newHashMap();
- public LedgerMetadata(int ensembleSize, int writeQuorumSize, int
ackQuorumSize,
- BookKeeper.DigestType digestType, byte[] password,
Map<String, byte[]> customMetadata) {
+ public LedgerMetadata(int ensembleSize,
+ int writeQuorumSize,
+ int ackQuorumSize,
+ BookKeeper.DigestType digestType,
+ byte[] password,
+ Map<String, byte[]> customMetadata,
+ boolean storeSystemtimeAsLedgerCreationTime) {
this.ensembleSize = ensembleSize;
this.writeQuorumSize = writeQuorumSize;
this.ackQuorumSize = ackQuorumSize;
this.ctime = System.currentTimeMillis();
+ this.storeSystemtimeAsLedgerCreationTime =
storeSystemtimeAsLedgerCreationTime;
/*
* It is set in PendingReadOp.readEntryComplete, and
@@ -115,7 +122,7 @@ public class LedgerMetadata {
public LedgerMetadata(int ensembleSize, int writeQuorumSize, int
ackQuorumSize,
BookKeeper.DigestType digestType, byte[] password) {
- this(ensembleSize, writeQuorumSize, ackQuorumSize, digestType,
password, null);
+ this(ensembleSize, writeQuorumSize, ackQuorumSize, digestType,
password, null, false);
}
/**
@@ -282,19 +289,15 @@ public class LedgerMetadata {
this.customMetadata = customMetadata;
}
- /**
- * Generates a byte array of this object
- *
- * @return the metadata serialized into a byte array
- */
- public byte[] serialize() {
- if (metadataFormatVersion == 1) {
- return serializeVersion1();
- }
+ LedgerMetadataFormat buildProtoFormat() {
LedgerMetadataFormat.Builder builder =
LedgerMetadataFormat.newBuilder();
builder.setQuorumSize(writeQuorumSize).setAckQuorumSize(ackQuorumSize)
.setEnsembleSize(ensembleSize).setLength(length)
- .setState(state).setLastEntryId(lastEntryId).setCtime(ctime);
+ .setState(state).setLastEntryId(lastEntryId);
+
+ if (storeSystemtimeAsLedgerCreationTime) {
+ builder.setCtime(ctime);
+ }
if (hasPassword) {
builder.setDigestType(digestType).setPassword(ByteString.copyFrom(password));
@@ -316,10 +319,22 @@ public class LedgerMetadata {
}
builder.addSegment(segmentBuilder.build());
}
+ return builder.build();
+ }
+
+ /**
+ * Generates a byte array of this object
+ *
+ * @return the metadata serialized into a byte array
+ */
+ public byte[] serialize() {
+ if (metadataFormatVersion == 1) {
+ return serializeVersion1();
+ }
StringBuilder s = new StringBuilder();
s.append(VERSION_KEY).append(tSplitter).append(CURRENT_METADATA_FORMAT_VERSION).append(lSplitter);
- s.append(TextFormat.printToString(builder.build()));
+ s.append(TextFormat.printToString(buildProtoFormat()));
if (LOG.isDebugEnabled()) {
LOG.debug("Serialized config: {}", s);
}
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ClientConfiguration.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ClientConfiguration.java
index 8725a6c..f4ffd52 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ClientConfiguration.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ClientConfiguration.java
@@ -114,6 +114,9 @@ public class ClientConfiguration extends
AbstractConfiguration {
protected final static String ENSEMBLE_PLACEMENT_POLICY =
"ensemblePlacementPolicy";
protected final static String NETWORK_TOPOLOGY_STABILIZE_PERIOD_SECONDS =
"networkTopologyStabilizePeriodSeconds";
+ // Ledger Metadata Parameters
+ protected static final String STORE_SYSTEMTIME_AS_LEDGER_CREATION_TIME =
"storeSystemTimeAsLedgerCreationTime";
+
// Stats
protected final static String ENABLE_TASK_EXECUTION_STATS =
"enableTaskExecutionStats";
protected final static String TASK_EXECUTION_WARN_TIME_MICROS =
"taskExecutionWarnTimeMicros";
@@ -1632,9 +1635,10 @@ public class ClientConfiguration extends
AbstractConfiguration {
* @param regClientClass
* ClientClass
*/
- public void setRegistrationClientClass(
+ public ClientConfiguration setRegistrationClientClass(
Class<? extends RegistrationClient> regClientClass) {
setProperty(REGISTRATION_CLIENT_CLASS, regClientClass);
+ return this;
}
/**
@@ -1648,4 +1652,29 @@ public class ClientConfiguration extends
AbstractConfiguration {
ZKRegistrationClient.class, RegistrationClient.class,
defaultLoader);
}
+
+ /**
+ * Enable the client to use system time as the ledger creation time.
+ *
+ * <p>If this is enabled, the client will write a ctime field into the
ledger metadata.
+ * Otherwise, nothing will be written. The creation time of this ledger
will be the ctime
+ * of the metadata record in metadata store.
+ *
+ * @param enabled flag to enable/disable client using system time as the
ledger creation time.
+ */
+ public ClientConfiguration setStoreSystemtimeAsLedgerCreationTime(boolean
enabled) {
+ setProperty(STORE_SYSTEMTIME_AS_LEDGER_CREATION_TIME, enabled);
+ return this;
+ }
+
+ /**
+ * Return the flag that indicates whether client is using system time as
the ledger creation time when
+ * creating ledgers.
+ *
+ * @return the flag that indicates whether client is using system time as
the ledger creation time when
+ * creating ledgers.
+ */
+ public boolean getStoreSystemtimeAsLedgerCreationTime() {
+ return getBoolean(STORE_SYSTEMTIME_AS_LEDGER_CREATION_TIME, false);
+ }
}
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/LedgerMetadataTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/LedgerMetadataTest.java
new file mode 100644
index 0000000..5cb1468
--- /dev/null
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/LedgerMetadataTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bookkeeper.client;
+
+import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Collections;
+import org.apache.bookkeeper.client.BookKeeper.DigestType;
+import org.apache.bookkeeper.proto.DataFormats.LedgerMetadataFormat;
+import org.junit.Test;
+
+/**
+ * Unit test for ledger metadata
+ */
+public class LedgerMetadataTest {
+
+ @Test
+ public void testStoreSystemtimeAsLedgerCtimeEnabled()
+ throws Exception {
+ byte[] passwd = "testPasswd".getBytes(UTF_8);
+
+ LedgerMetadata lm = new LedgerMetadata(
+ 3,
+ 3,
+ 2,
+ DigestType.CRC32,
+ passwd,
+ Collections.emptyMap(),
+ true);
+ LedgerMetadataFormat format = lm.buildProtoFormat();
+ assertTrue(format.hasCtime());
+ }
+
+ @Test
+ public void testStoreSystemtimeAsLedgerCtimeDisabled()
+ throws Exception {
+ byte[] passwd = "testPasswd".getBytes(UTF_8);
+
+ LedgerMetadata lm = new LedgerMetadata(
+ 3,
+ 3,
+ 2,
+ DigestType.CRC32,
+ passwd,
+ Collections.emptyMap(),
+ false);
+ LedgerMetadataFormat format = lm.buildProtoFormat();
+ assertFalse(format.hasCtime());
+ }
+
+}
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java
index 12d108a..fa32ef6 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java
@@ -101,28 +101,6 @@ public class ListLedgersTest extends
BookKeeperClusterTestCase {
}
- @Test
- public void testCtimeRecorded()
- throws Exception {
-
- ClientConfiguration conf = new ClientConfiguration()
- .setZkServers(zkUtil.getZooKeeperConnectString());
- BookKeeper bkc = new BookKeeper(conf);
-
- bkc.createLedger(digestType, "testPasswd".
- getBytes()).close();
-
- BookKeeperAdmin admin = new BookKeeperAdmin(zkUtil.
- getZooKeeperConnectString());
- Iterable<Long> iterable = admin.listLedgers();
-
- for (Long lId : iterable) {
- LedgerHandle ledger = bkc.openLedger(lId, digestType,
"testPasswd".getBytes());
- LedgerMetadata metaData = ledger.getLedgerMetadata();
- Assert.assertTrue("ctime was not recorded", metaData.getCtime() >
0);
- }
-
- }
}
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/api/BookKeeperBuildersTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/api/BookKeeperBuildersTest.java
index 946ee4e..edda096 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/api/BookKeeperBuildersTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/api/BookKeeperBuildersTest.java
@@ -309,8 +309,14 @@ public class BookKeeperBuildersTest extends
MockBookKeeperTestCase {
protected LedgerMetadata generateLedgerMetadata(int ensembleSize,
int writeQuorumSize, int ackQuorumSize, byte[] password,
Map<String, byte[]> customMetadata) {
- LedgerMetadata ledgerMetadata = new LedgerMetadata(ensembleSize,
writeQuorumSize,
- ackQuorumSize, BookKeeper.DigestType.CRC32, password,
customMetadata);
+ LedgerMetadata ledgerMetadata = new LedgerMetadata(
+ ensembleSize,
+ writeQuorumSize,
+ ackQuorumSize,
+ BookKeeper.DigestType.CRC32,
+ password,
+ customMetadata,
+ true);
ledgerMetadata.addEnsemble(0, generateNewEnsemble(ensembleSize));
return ledgerMetadata;
}
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java
index 6627f82..7968850 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java
@@ -524,7 +524,7 @@ public class AuditorLedgerCheckerTest extends
BookKeeperClusterTestCase {
int numofledgers = 5;
Random rand = new Random();
for (int i = 0; i < numofledgers; i++) {
- LedgerMetadata metadata = new LedgerMetadata(3, 2, 2,
DigestType.CRC32, "passwd".getBytes(), null);
+ LedgerMetadata metadata = new LedgerMetadata(3, 2, 2,
DigestType.CRC32, "passwd".getBytes());
ArrayList<BookieSocketAddress> ensemble = new
ArrayList<BookieSocketAddress>();
ensemble.add(new BookieSocketAddress("99.99.99.99:9999"));
ensemble.add(new BookieSocketAddress("11.11.11.11:1111"));
--
To stop receiving notification emails like this one, please contact
['"[email protected]" <[email protected]>'].
