This is an automated email from the ASF dual-hosted git repository.
li4wang pushed a commit to branch branch-3.9
in repository https://gitbox.apache.org/repos/asf/zookeeper.git
The following commit(s) were added to refs/heads/branch-3.9 by this push:
new ebe136e5b ZOOKEEPER-4730: Incorrect datadir and logdir size reported
from admin and 4lw dirs command (#2134)
ebe136e5b is described below
commit ebe136e5bffa8d2d884bdbdb1940712088df627e
Author: li4wang <68786536+li4w...@users.noreply.github.com>
AuthorDate: Mon Feb 12 10:40:34 2024 -0800
ZOOKEEPER-4730: Incorrect datadir and logdir size reported from admin and
4lw dirs command (#2134)
Author: Li Wang <liw...@apple.com>
---
.../org/apache/zookeeper/server/PurgeTxnLog.java | 2 +-
.../apache/zookeeper/server/ZooKeeperServer.java | 16 +++++----
.../server/persistence/FileTxnSnapLog.java | 6 ++--
.../zookeeper/server/ZooKeeperServerMainTest.java | 2 +-
.../zookeeper/server/ZooKeeperServerTest.java | 40 ++++++++++++++++++++++
.../server/persistence/FileTxnSnapLogTest.java | 2 +-
.../server/quorum/QuorumPeerMainTest.java | 2 +-
.../server/quorum/QuorumPeerTestBase.java | 2 +-
.../zookeeper/server/quorum/WatchLeakTest.java | 2 +-
.../apache/zookeeper/server/quorum/Zab1_0Test.java | 6 ++--
.../zookeeper/test/NonRecoverableErrorTest.java | 2 +-
.../apache/zookeeper/test/RepeatStartupTest.java | 2 +-
.../org/apache/zookeeper/test/TruncateTest.java | 2 +-
13 files changed, 64 insertions(+), 22 deletions(-)
diff --git
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/PurgeTxnLog.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/PurgeTxnLog.java
index d2fa93f5a..299071348 100644
---
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/PurgeTxnLog.java
+++
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/PurgeTxnLog.java
@@ -135,7 +135,7 @@ public class PurgeTxnLog {
}
// add all non-excluded log files
- File[] logs = txnLog.getDataDir().listFiles(new
MyFileFilter(PREFIX_LOG));
+ File[] logs = txnLog.getDataLogDir().listFiles(new
MyFileFilter(PREFIX_LOG));
List<File> files = new ArrayList<>();
if (logs != null) {
files.addAll(Arrays.asList(logs));
diff --git
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java
index f460b2d5b..19f1fac45 100644
---
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java
+++
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java
@@ -388,13 +388,13 @@ public class ZooKeeperServer implements SessionExpirer,
ServerStats.Provider {
+ " minSessionTimeout {} ms"
+ " maxSessionTimeout {} ms"
+ " clientPortListenBacklog {}"
- + " datadir {}"
+ + " dataLogdir {}"
+ " snapdir {}",
tickTime,
getMinSessionTimeout(),
getMaxSessionTimeout(),
getClientPortListenBacklog(),
- txnLogFactory.getDataDir(),
+ txnLogFactory.getDataLogDir(),
txnLogFactory.getSnapDir());
}
@@ -447,7 +447,7 @@ public class ZooKeeperServer implements SessionExpirer,
ServerStats.Provider {
pwriter.print("dataDirSize=");
pwriter.println(getDataDirSize());
pwriter.print("dataLogDir=");
- pwriter.println(zkDb.snapLog.getDataDir().getAbsolutePath());
+ pwriter.println(zkDb.snapLog.getDataLogDir().getAbsolutePath());
pwriter.print("dataLogSize=");
pwriter.println(getLogDirSize());
pwriter.print("tickTime=");
@@ -469,7 +469,7 @@ public class ZooKeeperServer implements SessionExpirer,
ServerStats.Provider {
return new ZooKeeperServerConf(
getClientPort(),
zkDb.snapLog.getSnapDir().getAbsolutePath(),
- zkDb.snapLog.getDataDir().getAbsolutePath(),
+ zkDb.snapLog.getDataLogDir().getAbsolutePath(),
getTickTime(),
getMaxClientCnxnsPerHost(),
getMinSessionTimeout(),
@@ -655,7 +655,7 @@ public class ZooKeeperServer implements SessionExpirer,
ServerStats.Provider {
if (zkDb == null) {
return 0L;
}
- File path = zkDb.snapLog.getDataDir();
+ File path = zkDb.snapLog.getSnapDir();
return getDirSize(path);
}
@@ -664,7 +664,7 @@ public class ZooKeeperServer implements SessionExpirer,
ServerStats.Provider {
if (zkDb == null) {
return 0L;
}
- File path = zkDb.snapLog.getSnapDir();
+ File path = zkDb.snapLog.getDataLogDir();
return getDirSize(path);
}
@@ -1513,7 +1513,9 @@ public class ZooKeeperServer implements SessionExpirer,
ServerStats.Provider {
throw new CloseRequestException(msg,
ServerCnxn.DisconnectReason.NOT_READ_ONLY_CLIENT);
}
if (request.getLastZxidSeen() > zkDb.dataTree.lastProcessedZxid) {
- String msg = "Refusing session request for client "
+ String msg = "Refusing session(0x"
+ + Long.toHexString(sessionId)
+ + ") request for client "
+ cnxn.getRemoteSocketAddress()
+ " as it has seen zxid 0x"
+ Long.toHexString(request.getLastZxidSeen())
diff --git
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/FileTxnSnapLog.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/FileTxnSnapLog.java
index fa6b4099a..ead3cc03a 100644
---
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/FileTxnSnapLog.java
+++
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/FileTxnSnapLog.java
@@ -204,11 +204,11 @@ public class FileTxnSnapLog {
}
/**
- * get the datadir used by this filetxn
+ * get the data log dir used by this filetxn
* snap log
- * @return the data dir
+ * @return the data log dir
*/
- public File getDataDir() {
+ public File getDataLogDir() {
return this.dataDir;
}
diff --git
a/zookeeper-server/src/test/java/org/apache/zookeeper/server/ZooKeeperServerMainTest.java
b/zookeeper-server/src/test/java/org/apache/zookeeper/server/ZooKeeperServerMainTest.java
index 920fdf406..0a76cb363 100644
---
a/zookeeper-server/src/test/java/org/apache/zookeeper/server/ZooKeeperServerMainTest.java
+++
b/zookeeper-server/src/test/java/org/apache/zookeeper/server/ZooKeeperServerMainTest.java
@@ -198,7 +198,7 @@ public class ZooKeeperServerMainTest extends ZKTestCase
implements Watcher {
// inject problem in server
ZooKeeperServer zooKeeperServer =
main.getCnxnFactory().getZooKeeperServer();
FileTxnSnapLog snapLog = zooKeeperServer.getTxnLogFactory();
- FileTxnSnapLog fileTxnSnapLogWithError = new
FileTxnSnapLog(snapLog.getDataDir(), snapLog.getSnapDir()) {
+ FileTxnSnapLog fileTxnSnapLogWithError = new
FileTxnSnapLog(snapLog.getDataLogDir(), snapLog.getSnapDir()) {
@Override
public void commit() throws IOException {
throw new IOException("Input/output error");
diff --git
a/zookeeper-server/src/test/java/org/apache/zookeeper/server/ZooKeeperServerTest.java
b/zookeeper-server/src/test/java/org/apache/zookeeper/server/ZooKeeperServerTest.java
index e9f9b7db5..6a6cfd7ab 100644
---
a/zookeeper-server/src/test/java/org/apache/zookeeper/server/ZooKeeperServerTest.java
+++
b/zookeeper-server/src/test/java/org/apache/zookeeper/server/ZooKeeperServerTest.java
@@ -29,6 +29,7 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.UUID;
+import org.apache.zookeeper.PortAssignment;
import org.apache.zookeeper.ZKTestCase;
import org.apache.zookeeper.metrics.MetricsUtils;
import org.apache.zookeeper.proto.ConnectRequest;
@@ -42,6 +43,45 @@ import org.junit.jupiter.api.Test;
public class ZooKeeperServerTest extends ZKTestCase {
+ @Test
+ public void testDirSize() throws Exception {
+ ZooKeeperServer zks = null;
+ ServerCnxnFactory cnxnFactory = null;
+
+ try {
+ final File dataDir = ClientBase.createTmpDir();
+ final File logDir = ClientBase.createTmpDir();
+
+ zks = new ZooKeeperServer(dataDir, logDir, 3000);
+
+ // validate dir size before server starts
+ assertEquals(0, zks.getDataDirSize());
+ assertEquals(0, zks.getLogDirSize());
+
+ // start server
+ final String hostPort = "127.0.0.1:" + PortAssignment.unique();
+ final int port = Integer.parseInt(hostPort.split(":")[1]);
+ cnxnFactory = ServerCnxnFactory.createFactory(port, -1);
+ cnxnFactory.startup(zks);
+ assertTrue(ClientBase.waitForServerUp(hostPort, 120000));
+
+ // validate data size is greater than 0 as snapshot has been taken
when server starts
+ assertTrue(zks.getDataDirSize() > 0);
+
+ // validate log size is 0 as no txn yet
+ assertEquals(0, zks.getLogDirSize());
+ } finally {
+ if (cnxnFactory != null) {
+ cnxnFactory.shutdown();
+ }
+
+ if (zks != null) {
+ zks.shutdown();
+ }
+ }
+ }
+
+
@Test
public void testSortDataDirAscending() {
File[] files = new File[5];
diff --git
a/zookeeper-server/src/test/java/org/apache/zookeeper/server/persistence/FileTxnSnapLogTest.java
b/zookeeper-server/src/test/java/org/apache/zookeeper/server/persistence/FileTxnSnapLogTest.java
index 466cf3f4b..656eeb8a0 100644
---
a/zookeeper-server/src/test/java/org/apache/zookeeper/server/persistence/FileTxnSnapLogTest.java
+++
b/zookeeper-server/src/test/java/org/apache/zookeeper/server/persistence/FileTxnSnapLogTest.java
@@ -177,7 +177,7 @@ public class FileTxnSnapLogTest {
assertTrue(logDir.exists());
assertTrue(snapDir.exists());
- assertTrue(fileTxnSnapLog.getDataDir().exists());
+ assertTrue(fileTxnSnapLog.getDataLogDir().exists());
assertTrue(fileTxnSnapLog.getSnapDir().exists());
}
diff --git
a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java
b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java
index be81aa6c5..1cb2a2b8e 100644
---
a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java
+++
b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerMainTest.java
@@ -935,7 +935,7 @@ public class QuorumPeerMainTest extends QuorumPeerTestBase {
// Assert
FileTxnSnapLog txnFactory = qpMain.getQuorumPeer().getTxnFactory();
- assertEquals(Paths.get(dataLogDir.getAbsolutePath(),
"version-2").toString(), txnFactory.getDataDir().getAbsolutePath());
+ assertEquals(Paths.get(dataLogDir.getAbsolutePath(),
"version-2").toString(), txnFactory.getDataLogDir().getAbsolutePath());
assertEquals(Paths.get(dataDir.getAbsolutePath(),
"version-2").toString(), txnFactory.getSnapDir().getAbsolutePath());
} finally {
FileUtils.deleteDirectory(dataDir);
diff --git
a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerTestBase.java
b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerTestBase.java
index 2a7b69258..754dca656 100644
---
a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerTestBase.java
+++
b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerTestBase.java
@@ -356,7 +356,7 @@ public class QuorumPeerTestBase extends ZKTestCase
implements Watcher {
}
public void reinitialize() throws IOException {
- File dataDir = main.quorumPeer.getTxnFactory().getDataDir();
+ File dataDir = main.quorumPeer.getTxnFactory().getDataLogDir();
ClientBase.recursiveDelete(dataDir);
ClientBase.createInitializeFile(dataDir.getParentFile());
}
diff --git
a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/WatchLeakTest.java
b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/WatchLeakTest.java
index 93d3b845c..b51cfbe3b 100644
---
a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/WatchLeakTest.java
+++
b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/WatchLeakTest.java
@@ -102,7 +102,7 @@ public class WatchLeakTest {
QuorumPeer quorumPeer = mock(QuorumPeer.class);
FileTxnSnapLog logfactory = mock(FileTxnSnapLog.class);
// Directories are not used but we need it to avoid NPE
- when(logfactory.getDataDir()).thenReturn(new File(""));
+ when(logfactory.getDataLogDir()).thenReturn(new File(""));
when(logfactory.getSnapDir()).thenReturn(new File(""));
FollowerZooKeeperServer fzks = null;
diff --git
a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/Zab1_0Test.java
b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/Zab1_0Test.java
index dcf7819b1..f0db38e26 100644
---
a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/Zab1_0Test.java
+++
b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/Zab1_0Test.java
@@ -575,7 +575,7 @@ public class Zab1_0Test extends ZKTestCase {
File tmpDir = File.createTempFile("test", "dir", testData);
tmpDir.delete();
tmpDir.mkdir();
- File logDir =
f.fzk.getTxnLogFactory().getDataDir().getParentFile();
+ File logDir =
f.fzk.getTxnLogFactory().getDataLogDir().getParentFile();
File snapDir =
f.fzk.getTxnLogFactory().getSnapDir().getParentFile();
//Spy on ZK so we can check if a snapshot happened or not.
f.zk = spy(f.zk);
@@ -709,7 +709,7 @@ public class Zab1_0Test extends ZKTestCase {
File tmpDir = File.createTempFile("test", "dir", testData);
tmpDir.delete();
tmpDir.mkdir();
- File logDir =
f.fzk.getTxnLogFactory().getDataDir().getParentFile();
+ File logDir =
f.fzk.getTxnLogFactory().getDataLogDir().getParentFile();
File snapDir =
f.fzk.getTxnLogFactory().getSnapDir().getParentFile();
//Spy on ZK so we can check if a snapshot happened or not.
f.zk = spy(f.zk);
@@ -940,7 +940,7 @@ public class Zab1_0Test extends ZKTestCase {
File tmpDir = File.createTempFile("test", "dir", testData);
tmpDir.delete();
tmpDir.mkdir();
- File logDir =
o.zk.getTxnLogFactory().getDataDir().getParentFile();
+ File logDir =
o.zk.getTxnLogFactory().getDataLogDir().getParentFile();
File snapDir =
o.zk.getTxnLogFactory().getSnapDir().getParentFile();
try {
assertEquals(0, o.self.getAcceptedEpoch());
diff --git
a/zookeeper-server/src/test/java/org/apache/zookeeper/test/NonRecoverableErrorTest.java
b/zookeeper-server/src/test/java/org/apache/zookeeper/test/NonRecoverableErrorTest.java
index 3eaa1d3ff..ed592df6f 100644
---
a/zookeeper-server/src/test/java/org/apache/zookeeper/test/NonRecoverableErrorTest.java
+++
b/zookeeper-server/src/test/java/org/apache/zookeeper/test/NonRecoverableErrorTest.java
@@ -93,7 +93,7 @@ public class NonRecoverableErrorTest extends
QuorumPeerTestBase {
// inject problem in leader
FileTxnSnapLog snapLog = leader.getActiveServer().getTxnLogFactory();
- FileTxnSnapLog fileTxnSnapLogWithError = new
FileTxnSnapLog(snapLog.getDataDir(), snapLog.getSnapDir()) {
+ FileTxnSnapLog fileTxnSnapLogWithError = new
FileTxnSnapLog(snapLog.getDataLogDir(), snapLog.getSnapDir()) {
@Override
public void commit() throws IOException {
throw new IOException("Input/output error");
diff --git
a/zookeeper-server/src/test/java/org/apache/zookeeper/test/RepeatStartupTest.java
b/zookeeper-server/src/test/java/org/apache/zookeeper/test/RepeatStartupTest.java
index 4dff8e276..8c79af341 100644
---
a/zookeeper-server/src/test/java/org/apache/zookeeper/test/RepeatStartupTest.java
+++
b/zookeeper-server/src/test/java/org/apache/zookeeper/test/RepeatStartupTest.java
@@ -49,7 +49,7 @@ public class RepeatStartupTest extends ZKTestCase {
QuorumBase.shutdown(qb.s4);
QuorumBase.shutdown(qb.s5);
String hp = qb.hostPort.split(",")[0];
- ZooKeeperServer zks = new
ZooKeeperServer(qb.s1.getTxnFactory().getSnapDir(),
qb.s1.getTxnFactory().getDataDir(), 3000);
+ ZooKeeperServer zks = new
ZooKeeperServer(qb.s1.getTxnFactory().getSnapDir(),
qb.s1.getTxnFactory().getDataLogDir(), 3000);
final int PORT = Integer.parseInt(hp.split(":")[1]);
ServerCnxnFactory factory = ServerCnxnFactory.createFactory(PORT, -1);
diff --git
a/zookeeper-server/src/test/java/org/apache/zookeeper/test/TruncateTest.java
b/zookeeper-server/src/test/java/org/apache/zookeeper/test/TruncateTest.java
index f81afc867..c7654003f 100644
--- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/TruncateTest.java
+++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/TruncateTest.java
@@ -121,7 +121,7 @@ public class TruncateTest extends ZKTestCase {
append(zkdb, i);
}
zkdb.close();
- File[] logs = snaplog.getDataDir().listFiles();
+ File[] logs = snaplog.getDataLogDir().listFiles();
for (int i = 0; i < logs.length; i++) {
LOG.debug("Deleting: {}", logs[i].getName());
assertTrue(logs[i].delete(), "Failed to delete log file: " +
logs[i].getName());
