(bookkeeper) branch branch-4.16 updated: [fix] Fix ByteBuf release/retain in PerChannelBookClient (#4289)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch branch-4.16
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/branch-4.16 by this push:
new a77458d56c [fix] Fix ByteBuf release/retain in PerChannelBookClient
(#4289)
a77458d56c is described below
commit a77458d56cf22e3b79d5beded8dbc0beca3ff22c
Author: fengyubiao
AuthorDate: Thu Apr 18 06:37:29 2024 +0800
[fix] Fix ByteBuf release/retain in PerChannelBookClient (#4289)
* [fix] ByteBuf release/retain incorrect
* improve the code comment
* fix other cases
* modify the code comment
* improve the code
* improve the test
* add description
---
.../org/apache/bookkeeper/proto/PerChannelBookieClient.java | 11 ---
.../java/org/apache/bookkeeper/test/BookieClientTest.java | 3 ++-
2 files changed, 10 insertions(+), 4 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java
index b4cf194e24..3a7af5b99a 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java
@@ -841,10 +841,10 @@ public class PerChannelBookieClient extends
ChannelInboundHandlerAdapter {
cb, ctx, ledgerId,
entryId));
final Channel c = channel;
if (c == null) {
-// usually checked in writeAndFlush, but we have extra check
-// because we need to release toSend.
+// Manually release the binary data(variable "request") that we
manually created when it can not be sent out
+// because the channel is switching.
errorOut(completionKey);
-ReferenceCountUtil.release(toSend);
+ReferenceCountUtil.release(request);
return;
} else {
// addEntry times out on backpressure
@@ -1136,6 +1136,7 @@ public class PerChannelBookieClient extends
ChannelInboundHandlerAdapter {
if (channel == null) {
LOG.warn("Operation {} failed: channel == null",
StringUtils.requestToString(request));
errorOut(key);
+ReferenceCountUtil.release(request);
return;
}
@@ -1150,6 +1151,7 @@ public class PerChannelBookieClient extends
ChannelInboundHandlerAdapter {
StringUtils.requestToString(request));
errorOut(key, BKException.Code.TooManyRequestsException);
+ReferenceCountUtil.release(request);
return;
}
@@ -1171,6 +1173,9 @@ public class PerChannelBookieClient extends
ChannelInboundHandlerAdapter {
} catch (Throwable e) {
LOG.warn("Operation {} failed",
StringUtils.requestToString(request), e);
errorOut(key);
+// If the request goes into the writeAndFlush, it should be
handled well by Netty. So all the exceptions we
+// get here, we can release the request.
+ReferenceCountUtil.release(request);
}
}
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieClientTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieClientTest.java
index a110e833ac..f5202d8c0e 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieClientTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieClientTest.java
@@ -22,7 +22,6 @@ package org.apache.bookkeeper.test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import io.netty.buffer.UnpooledByteBufAllocator;
@@ -36,6 +35,7 @@ import java.util.Arrays;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
+import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.bookie.MockUncleanShutdownDetection;
import org.apache.bookkeeper.bookie.TestBookieImpl;
import org.apache.bookkeeper.client.BKException;
@@ -69,6 +69,7 @@ import org.junit.Test;
/**
* Test the bookie client.
*/
+@Slf4j
public class BookieClientTest {
BookieServer bs;
File tmpDir;
(bookkeeper) branch branch-4.17 updated: [fix] Fix ByteBuf release/retain in PerChannelBookClient (#4289)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch branch-4.17
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/branch-4.17 by this push:
new 02a06c4be2 [fix] Fix ByteBuf release/retain in PerChannelBookClient
(#4289)
02a06c4be2 is described below
commit 02a06c4be2cde4284e620fdb4c7800cb81d49cbf
Author: fengyubiao
AuthorDate: Thu Apr 18 06:37:29 2024 +0800
[fix] Fix ByteBuf release/retain in PerChannelBookClient (#4289)
* [fix] ByteBuf release/retain incorrect
* improve the code comment
* fix other cases
* modify the code comment
* improve the code
* improve the test
* add description
---
.../bookkeeper/proto/PerChannelBookieClient.java | 11 +-
.../apache/bookkeeper/test/BookieClientTest.java | 171 +
2 files changed, 179 insertions(+), 3 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java
index 5ebafe8eca..f1ff35f033 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java
@@ -842,10 +842,10 @@ public class PerChannelBookieClient extends
ChannelInboundHandlerAdapter {
cb, ctx, ledgerId,
entryId));
final Channel c = channel;
if (c == null) {
-// usually checked in writeAndFlush, but we have extra check
-// because we need to release toSend.
+// Manually release the binary data(variable "request") that we
manually created when it can not be sent out
+// because the channel is switching.
errorOut(completionKey);
-ReferenceCountUtil.release(toSend);
+ReferenceCountUtil.release(request);
return;
} else {
// addEntry times out on backpressure
@@ -1180,6 +1180,7 @@ public class PerChannelBookieClient extends
ChannelInboundHandlerAdapter {
if (channel == null) {
LOG.warn("Operation {} failed: channel == null",
StringUtils.requestToString(request));
errorOut(key);
+ReferenceCountUtil.release(request);
return;
}
@@ -1194,6 +1195,7 @@ public class PerChannelBookieClient extends
ChannelInboundHandlerAdapter {
StringUtils.requestToString(request));
errorOut(key, BKException.Code.TooManyRequestsException);
+ReferenceCountUtil.release(request);
return;
}
@@ -1215,6 +1217,9 @@ public class PerChannelBookieClient extends
ChannelInboundHandlerAdapter {
} catch (Throwable e) {
LOG.warn("Operation {} failed",
StringUtils.requestToString(request), e);
errorOut(key);
+// If the request goes into the writeAndFlush, it should be
handled well by Netty. So all the exceptions we
+// get here, we can release the request.
+ReferenceCountUtil.release(request);
}
}
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieClientTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieClientTest.java
index 60f89159a0..2fadbbd2c2 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieClientTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieClientTest.java
@@ -27,8 +27,10 @@ import static org.junit.Assert.assertTrue;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.ByteBufUtil;
+import io.netty.buffer.PooledByteBufAllocator;
import io.netty.buffer.Unpooled;
import io.netty.buffer.UnpooledByteBufAllocator;
+import io.netty.channel.Channel;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.util.ReferenceCounted;
@@ -36,16 +38,20 @@ import io.netty.util.concurrent.DefaultThreadFactory;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.time.Duration;
import java.util.Arrays;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.bookie.MockUncleanShutdownDetection;
import org.apache.bookkeeper.bookie.TestBookieImpl;
import org.apache.bookkeeper.client.BKException;
import org.apache.bookkeeper.client.BKException.Code;
+import org.apach
(bookkeeper) branch master updated: [fix] Fix ByteBuf release/retain in PerChannelBookClient (#4289)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 1694d67713 [fix] Fix ByteBuf release/retain in PerChannelBookClient
(#4289)
1694d67713 is described below
commit 1694d67713496a67efdf2e3e076cf798713346c8
Author: fengyubiao
AuthorDate: Thu Apr 18 06:37:29 2024 +0800
[fix] Fix ByteBuf release/retain in PerChannelBookClient (#4289)
* [fix] ByteBuf release/retain incorrect
* improve the code comment
* fix other cases
* modify the code comment
* improve the code
* improve the test
* add description
---
.../bookkeeper/proto/PerChannelBookieClient.java | 11 +-
.../apache/bookkeeper/test/BookieClientTest.java | 171 +
2 files changed, 179 insertions(+), 3 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java
index 95eef54111..8485d21b74 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java
@@ -842,10 +842,10 @@ public class PerChannelBookieClient extends
ChannelInboundHandlerAdapter {
cb, ctx, ledgerId,
entryId));
final Channel c = channel;
if (c == null) {
-// usually checked in writeAndFlush, but we have extra check
-// because we need to release toSend.
+// Manually release the binary data(variable "request") that we
manually created when it can not be sent out
+// because the channel is switching.
errorOut(completionKey);
-ReferenceCountUtil.release(toSend);
+ReferenceCountUtil.release(request);
return;
} else {
// addEntry times out on backpressure
@@ -1180,6 +1180,7 @@ public class PerChannelBookieClient extends
ChannelInboundHandlerAdapter {
if (channel == null) {
LOG.warn("Operation {} failed: channel == null",
StringUtils.requestToString(request));
errorOut(key);
+ReferenceCountUtil.release(request);
return;
}
@@ -1194,6 +1195,7 @@ public class PerChannelBookieClient extends
ChannelInboundHandlerAdapter {
StringUtils.requestToString(request));
errorOut(key, BKException.Code.TooManyRequestsException);
+ReferenceCountUtil.release(request);
return;
}
@@ -1215,6 +1217,9 @@ public class PerChannelBookieClient extends
ChannelInboundHandlerAdapter {
} catch (Throwable e) {
LOG.warn("Operation {} failed",
StringUtils.requestToString(request), e);
errorOut(key);
+// If the request goes into the writeAndFlush, it should be
handled well by Netty. So all the exceptions we
+// get here, we can release the request.
+ReferenceCountUtil.release(request);
}
}
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieClientTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieClientTest.java
index 60f89159a0..2fadbbd2c2 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieClientTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieClientTest.java
@@ -27,8 +27,10 @@ import static org.junit.Assert.assertTrue;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.ByteBufUtil;
+import io.netty.buffer.PooledByteBufAllocator;
import io.netty.buffer.Unpooled;
import io.netty.buffer.UnpooledByteBufAllocator;
+import io.netty.channel.Channel;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.util.ReferenceCounted;
@@ -36,16 +38,20 @@ import io.netty.util.concurrent.DefaultThreadFactory;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.time.Duration;
import java.util.Arrays;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.bookie.MockUncleanShutdownDetection;
import org.apache.bookkeeper.bookie.TestBookieImpl;
import org.apache.bookkeeper.client.BKException;
import org.apache.bookkeeper.client.BKException.Code;
+import org.apache.bookkee
(bookkeeper) branch branch-4.16 updated: Fixed creation of temporary dir in NativeUtils (#4262)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch branch-4.16
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/branch-4.16 by this push:
new a23b1a0294 Fixed creation of temporary dir in NativeUtils (#4262)
a23b1a0294 is described below
commit a23b1a02949f580f4e584dbff9131b26bd0712ff
Author: Matteo Merli
AuthorDate: Tue Apr 2 18:56:10 2024 -0700
Fixed creation of temporary dir in NativeUtils (#4262)
### Motivation
Creating the temp directory for unpacking the native library is failing for
the affinity library.
### Changes
Use `Files.createTempDirectory()` instead.
---
.../com/scurrilous/circe/utils/NativeUtils.java| 12 +-
.../common/util/affinity/impl/NativeUtils.java | 26 ++
2 files changed, 22 insertions(+), 16 deletions(-)
diff --git
a/circe-checksum/src/main/java/com/scurrilous/circe/utils/NativeUtils.java
b/circe-checksum/src/main/java/com/scurrilous/circe/utils/NativeUtils.java
index 2932c65000..988264c952 100644
--- a/circe-checksum/src/main/java/com/scurrilous/circe/utils/NativeUtils.java
+++ b/circe-checksum/src/main/java/com/scurrilous/circe/utils/NativeUtils.java
@@ -26,6 +26,8 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.Locale;
/**
@@ -50,13 +52,9 @@ public class NativeUtils {
String[] parts = path.split("/");
String filename = (parts.length > 0) ? parts[parts.length - 1] : null;
-File dir = File.createTempFile("native", "");
-dir.delete();
-if (!(dir.mkdir())) {
-throw new IOException("Failed to create temp directory " +
dir.getAbsolutePath());
-}
-dir.deleteOnExit();
-File temp = new File(dir, filename);
+Path dir = Files.createTempDirectory("native");
+dir.toFile().deleteOnExit();
+File temp = new File(dir.toString(), filename);
temp.deleteOnExit();
byte[] buffer = new byte[1024];
diff --git
a/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/NativeUtils.java
b/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/NativeUtils.java
index 2dbec828d3..fde69f9011 100644
---
a/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/NativeUtils.java
+++
b/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/NativeUtils.java
@@ -24,9 +24,11 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
-import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import lombok.NonNull;
import lombok.experimental.UtilityClass;
/**
@@ -46,17 +48,17 @@ public class NativeUtils {
value = "RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE",
justification = "work around for java 9:
https://github.com/spotbugs/spotbugs/issues/493;)
public static void loadLibraryFromJar(String path) throws Exception {
-
com.google.common.base.Preconditions.checkArgument(path.startsWith("/"),
"absolute path must start with /");
+checkArgument(path.startsWith("/"), "absolute path must start with /");
String[] parts = path.split("/");
-String filename = (parts.length > 0) ? parts[parts.length - 1] : null;
+checkArgument(parts.length > 0, "absolute path must contain file
name");
-File dir = File.createTempFile("native", "");
-if (!(dir.mkdir())) {
-throw new IOException("Failed to create temp directory " +
dir.getAbsolutePath());
-}
-dir.deleteOnExit();
-File temp = new File(dir, filename);
+String filename = parts[parts.length - 1];
+checkArgument(path.startsWith("/"), "absolute path must start with /");
+
+Path dir = Files.createTempDirectory("native");
+dir.toFile().deleteOnExit();
+File temp = new File(dir.toString(), filename);
temp.deleteOnExit();
byte[] buffer = new byte[1024];
@@ -79,4 +81,10 @@ public class NativeUtils {
System.load(temp.getAbsolutePath());
}
+
+private static void checkArgument(boolean expression, @NonNull Object
errorMessage) {
+if (!expression) {
+throw new IllegalArgumentException(String.valueOf(errorMessage));
+}
+}
}
(bookkeeper) branch branch-4.17 updated: Fixed creation of temporary dir in NativeUtils (#4262)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch branch-4.17
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/branch-4.17 by this push:
new 996b5d4dda Fixed creation of temporary dir in NativeUtils (#4262)
996b5d4dda is described below
commit 996b5d4dda522f7333133bca7150b1646ef494e2
Author: Matteo Merli
AuthorDate: Tue Apr 2 18:56:10 2024 -0700
Fixed creation of temporary dir in NativeUtils (#4262)
### Motivation
Creating the temp directory for unpacking the native library is failing for
the affinity library.
### Changes
Use `Files.createTempDirectory()` instead.
---
.../com/scurrilous/circe/utils/NativeUtils.java| 12 +-
.../common/util/affinity/impl/NativeUtils.java | 26 ++
2 files changed, 22 insertions(+), 16 deletions(-)
diff --git
a/circe-checksum/src/main/java/com/scurrilous/circe/utils/NativeUtils.java
b/circe-checksum/src/main/java/com/scurrilous/circe/utils/NativeUtils.java
index 2932c65000..988264c952 100644
--- a/circe-checksum/src/main/java/com/scurrilous/circe/utils/NativeUtils.java
+++ b/circe-checksum/src/main/java/com/scurrilous/circe/utils/NativeUtils.java
@@ -26,6 +26,8 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.Locale;
/**
@@ -50,13 +52,9 @@ public class NativeUtils {
String[] parts = path.split("/");
String filename = (parts.length > 0) ? parts[parts.length - 1] : null;
-File dir = File.createTempFile("native", "");
-dir.delete();
-if (!(dir.mkdir())) {
-throw new IOException("Failed to create temp directory " +
dir.getAbsolutePath());
-}
-dir.deleteOnExit();
-File temp = new File(dir, filename);
+Path dir = Files.createTempDirectory("native");
+dir.toFile().deleteOnExit();
+File temp = new File(dir.toString(), filename);
temp.deleteOnExit();
byte[] buffer = new byte[1024];
diff --git
a/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/NativeUtils.java
b/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/NativeUtils.java
index 2dbec828d3..fde69f9011 100644
---
a/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/NativeUtils.java
+++
b/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/NativeUtils.java
@@ -24,9 +24,11 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
-import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import lombok.NonNull;
import lombok.experimental.UtilityClass;
/**
@@ -46,17 +48,17 @@ public class NativeUtils {
value = "RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE",
justification = "work around for java 9:
https://github.com/spotbugs/spotbugs/issues/493;)
public static void loadLibraryFromJar(String path) throws Exception {
-
com.google.common.base.Preconditions.checkArgument(path.startsWith("/"),
"absolute path must start with /");
+checkArgument(path.startsWith("/"), "absolute path must start with /");
String[] parts = path.split("/");
-String filename = (parts.length > 0) ? parts[parts.length - 1] : null;
+checkArgument(parts.length > 0, "absolute path must contain file
name");
-File dir = File.createTempFile("native", "");
-if (!(dir.mkdir())) {
-throw new IOException("Failed to create temp directory " +
dir.getAbsolutePath());
-}
-dir.deleteOnExit();
-File temp = new File(dir, filename);
+String filename = parts[parts.length - 1];
+checkArgument(path.startsWith("/"), "absolute path must start with /");
+
+Path dir = Files.createTempDirectory("native");
+dir.toFile().deleteOnExit();
+File temp = new File(dir.toString(), filename);
temp.deleteOnExit();
byte[] buffer = new byte[1024];
@@ -79,4 +81,10 @@ public class NativeUtils {
System.load(temp.getAbsolutePath());
}
+
+private static void checkArgument(boolean expression, @NonNull Object
errorMessage) {
+if (!expression) {
+throw new IllegalArgumentException(String.valueOf(errorMessage));
+}
+}
}
(bookkeeper) branch master updated: Fix error stack track may expose to external user (#4223)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new b5b518182b Fix error stack track may expose to external user (#4223)
b5b518182b is described below
commit b5b518182b7cea448273dbfad1e307460d782423
Author: Hang Chen
AuthorDate: Tue Mar 5 00:13:05 2024 +0800
Fix error stack track may expose to external user (#4223)
* fix error stack track may expose to external user
* update code
* fix check style
---
.../apache/bookkeeper/http/servlet/BookieHttpServiceServlet.java | 9 ++---
1 file changed, 2 insertions(+), 7 deletions(-)
diff --git
a/bookkeeper-http/servlet-http-server/src/main/java/org/apache/bookkeeper/http/servlet/BookieHttpServiceServlet.java
b/bookkeeper-http/servlet-http-server/src/main/java/org/apache/bookkeeper/http/servlet/BookieHttpServiceServlet.java
index 10f7b11914..30fe063786 100644
---
a/bookkeeper-http/servlet-http-server/src/main/java/org/apache/bookkeeper/http/servlet/BookieHttpServiceServlet.java
+++
b/bookkeeper-http/servlet-http-server/src/main/java/org/apache/bookkeeper/http/servlet/BookieHttpServiceServlet.java
@@ -35,7 +35,6 @@ import org.apache.bookkeeper.http.HttpRouter;
import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.HttpServer.ApiType;
import org.apache.bookkeeper.http.HttpServiceProvider;
-import org.apache.bookkeeper.http.service.ErrorHttpService;
import org.apache.bookkeeper.http.service.HttpEndpointService;
import org.apache.bookkeeper.http.service.HttpServiceRequest;
import org.apache.bookkeeper.http.service.HttpServiceResponse;
@@ -91,16 +90,12 @@ public class BookieHttpServiceServlet extends HttpServlet {
return;
}
response = httpEndpointService.handle(request);
-} catch (Throwable e) {
- LOG.error("Error while service Bookie API request " + uri, e);
- response = new ErrorHttpService().handle(request);
-}
-if (response != null) {
httpResponse.setStatus(response.getStatusCode());
try (Writer out = httpResponse.getWriter()) {
out.write(response.getBody());
}
-} else {
+} catch (Throwable e) {
+ LOG.error("Error while service Bookie API request {}", uri, e);
httpResponse.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
}
}
(bookkeeper) branch master updated: Fix uncontrolled data used in path expression (#4221)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 34d85153f7 Fix uncontrolled data used in path expression (#4221)
34d85153f7 is described below
commit 34d85153f7387a5e1012f31d694ce8956eeed275
Author: Hang Chen
AuthorDate: Mon Mar 4 12:08:41 2024 +0800
Fix uncontrolled data used in path expression (#4221)
* Fix uncontrolled data used in path expression
* update code
* update code
---
.../src/main/java/org/apache/bookkeeper/util/LocalBookKeeper.java | 8
1 file changed, 8 insertions(+)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LocalBookKeeper.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LocalBookKeeper.java
index dff6d1b8ba..ca467ab297 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LocalBookKeeper.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LocalBookKeeper.java
@@ -58,6 +58,7 @@ import
org.apache.bookkeeper.shims.zk.ZooKeeperServerShimFactory;
import org.apache.bookkeeper.stats.NullStatsLogger;
import org.apache.bookkeeper.zookeeper.ZooKeeperClient;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.Op;
@@ -333,6 +334,13 @@ public class LocalBookKeeper implements AutoCloseable {
* @throws IOException
*/
private void serializeLocalBookieConfig(ServerConfiguration
localBookieConfig, String fileName) throws IOException {
+if (StringUtils.isBlank(fileName)
+|| fileName.contains("..")
+|| fileName.contains("/")
+|| fileName.contains("\\")) {
+throw new IllegalArgumentException("Invalid filename: " +
fileName);
+}
+
File localBookieConfFile = new File(localBookiesConfigDir, fileName);
if (localBookieConfFile.exists() && !localBookieConfFile.delete()) {
throw new IOException(
(bookkeeper) branch master updated: Add filename check for unTar (#4222)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 48b7d1ebb1 Add filename check for unTar (#4222)
48b7d1ebb1 is described below
commit 48b7d1ebb1138074356da9790fb20f3219c887a4
Author: Hang Chen
AuthorDate: Mon Mar 4 12:07:49 2024 +0800
Add filename check for unTar (#4222)
* add filename check for unTar
* update code
---
.../org/apache/bookkeeper/tests/integration/utils/DockerUtils.java | 6 +-
.../apache/bookkeeper/tests/integration/utils/MavenClassLoader.java | 4
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git
a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/DockerUtils.java
b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/DockerUtils.java
index bae9057fa1..a3b8778946 100644
---
a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/DockerUtils.java
+++
b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/DockerUtils.java
@@ -135,7 +135,11 @@ public class DockerUtils {
TarArchiveEntry entry = stream.getNextTarEntry();
while (entry != null) {
if (entry.isFile()) {
-File output = new File(getTargetDirectory(containerId),
entry.getName().replace("/", "-"));
+File targetDir = getTargetDirectory(containerId);
+File output = new File(targetDir,
entry.getName().replace("/", "-"));
+if
(!output.toPath().normalize().startsWith(targetDir.toPath())) {
+throw new IOException("Bad zip entry");
+}
try (FileOutputStream os = new FileOutputStream(output)) {
byte[] block = new byte[readBlockSize];
int read = stream.read(block, 0, readBlockSize);
diff --git
a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/MavenClassLoader.java
b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/MavenClassLoader.java
index 2b1fabf6be..d77c3920ac 100644
---
a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/MavenClassLoader.java
+++
b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/MavenClassLoader.java
@@ -367,6 +367,10 @@ public class MavenClassLoader implements AutoCloseable {
TarArchiveEntry entry;
while ((entry = (TarArchiveEntry) debInputStream.getNextEntry())
!= null) {
final File outputFile = new File(outputDir, entry.getName());
+if
(!outputFile.toPath().normalize().startsWith(outputDir.toPath())) {
+throw new IOException("Bad zip entry");
+}
+
if (!outputFile.getParentFile().exists()) {
outputFile.getParentFile().mkdirs();
}
(bookkeeper) branch master updated: Upgrade mockito from 3.12.4 to 4.11.0 (#4218)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 34bd4b38fa Upgrade mockito from 3.12.4 to 4.11.0 (#4218)
34bd4b38fa is described below
commit 34bd4b38fa13f105608ae4e7b4c749124c271e1e
Author: Zixuan Liu
AuthorDate: Wed Feb 21 23:39:28 2024 +0800
Upgrade mockito from 3.12.4 to 4.11.0 (#4218)
---
.../org/apache/bookkeeper/bookie/BookieImpl.java | 26 ---
.../java/org/apache/bookkeeper/bookie/Journal.java | 38 +++---
.../bookie/BookieMultipleJournalsTest.java | 4 +--
pom.xml| 2 +-
4 files changed, 50 insertions(+), 20 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java
index 654ace4547..a660a13ce8 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java
@@ -87,7 +87,7 @@ import org.slf4j.LoggerFactory;
/**
* Implements a bookie.
*/
-public class BookieImpl extends BookieCriticalThread implements Bookie {
+public class BookieImpl implements Bookie {
private static final Logger LOG = LoggerFactory.getLogger(Bookie.class);
@@ -119,6 +119,8 @@ public class BookieImpl extends BookieCriticalThread
implements Bookie {
protected StateManager stateManager;
+private BookieCriticalThread bookieThread;
+
// Expose Stats
final StatsLogger statsLogger;
private final BookieStats bookieStats;
@@ -390,7 +392,6 @@ public class BookieImpl extends BookieCriticalThread
implements Bookie {
ByteBufAllocator allocator,
Supplier bookieServiceInfoProvider)
throws IOException, InterruptedException, BookieException {
-super("Bookie-" + conf.getBookiePort());
this.bookieServiceInfoProvider = bookieServiceInfoProvider;
this.statsLogger = statsLogger;
this.conf = conf;
@@ -656,7 +657,9 @@ public class BookieImpl extends BookieCriticalThread
implements Bookie {
@Override
public synchronized void start() {
-setDaemon(true);
+bookieThread = new BookieCriticalThread(() -> run(), "Bookie-" +
conf.getBookiePort());
+bookieThread.setDaemon(true);
+
ThreadRegistry.register("BookieThread", 0);
if (LOG.isDebugEnabled()) {
LOG.debug("I'm starting a bookie with journal directories {}",
@@ -717,7 +720,7 @@ public class BookieImpl extends BookieCriticalThread
implements Bookie {
syncThread.start();
// start bookie thread
-super.start();
+bookieThread.start();
// After successful bookie startup, register listener for disk
// error/full notifications.
@@ -741,6 +744,20 @@ public class BookieImpl extends BookieCriticalThread
implements Bookie {
}
}
+@Override
+public void join() throws InterruptedException {
+if (bookieThread != null) {
+bookieThread.join();
+}
+}
+
+public boolean isAlive() {
+if (bookieThread == null) {
+return false;
+}
+return bookieThread.isAlive();
+}
+
/*
* Get the DiskFailure listener for the bookie
*/
@@ -824,7 +841,6 @@ public class BookieImpl extends BookieCriticalThread
implements Bookie {
return stateManager.isRunning();
}
-@Override
public void run() {
// start journals
for (Journal journal: journals) {
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
index 3b730cb476..df10dfe522 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
@@ -42,6 +42,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
import
org.apache.bookkeeper.bookie.LedgerDirsManager.NoWritableLedgerDirException;
import org.apache.bookkeeper.bookie.stats.JournalStats;
import org.apache.bookkeeper.common.collections.BatchedArrayBlockingQueue;
@@ -66,13 +67,15 @@ import org.slf4j.LoggerFactory;
/**
* Provide journal related management.
*/
-public class Journal extends BookieCriticalThread implements CheckpointSource {
+public class Journal implements CheckpointSource {
private static final Logger LOG = LoggerFactory.getLogger(Journal.class);
private static final RecyclableArrayList
(bookkeeper) branch master updated: [BP-62] Add more test for Bookkeeper. (#4210)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new eb27e6ed3c [BP-62] Add more test for Bookkeeper. (#4210)
eb27e6ed3c is described below
commit eb27e6ed3ccd1ee46f372ae6fd0a53ccea81908f
Author: Yan Zhao
AuthorDate: Wed Feb 21 02:14:12 2024 +0800
[BP-62] Add more test for Bookkeeper. (#4210)
* Add test to cover Bookkeeper test.
* fix ci.
* Fix ci.
---
.../benchmark/BenchReadThroughputLatency.java | 27 ++--
.../org/apache/bookkeeper/client/LedgerHandle.java | 4 +-
.../bookkeeper/conf/ClientConfiguration.java | 5 +
.../apache/bookkeeper/client/BookKeeperTest.java | 179 +
4 files changed, 204 insertions(+), 11 deletions(-)
diff --git
a/bookkeeper-benchmark/src/main/java/org/apache/bookkeeper/benchmark/BenchReadThroughputLatency.java
b/bookkeeper-benchmark/src/main/java/org/apache/bookkeeper/benchmark/BenchReadThroughputLatency.java
index f3247ab621..5c04558b84 100644
---
a/bookkeeper-benchmark/src/main/java/org/apache/bookkeeper/benchmark/BenchReadThroughputLatency.java
+++
b/bookkeeper-benchmark/src/main/java/org/apache/bookkeeper/benchmark/BenchReadThroughputLatency.java
@@ -73,7 +73,7 @@ public class BenchReadThroughputLatency {
}
};
-private static void readLedger(ClientConfiguration conf, long ledgerId,
byte[] passwd) {
+private static void readLedger(ClientConfiguration conf, long ledgerId,
byte[] passwd, int batchEntries) {
LOG.info("Reading ledger {}", ledgerId);
BookKeeper bk = null;
long time = 0;
@@ -102,17 +102,23 @@ public class BenchReadThroughputLatency {
}
long starttime = System.nanoTime();
-while (lastRead < lastConfirmed) {
+while (entriesRead <= lastConfirmed) {
long nextLimit = lastRead + 10;
-long readTo = Math.min(nextLimit, lastConfirmed);
-Enumeration entries = lh.readEntries(lastRead
+ 1, readTo);
-lastRead = readTo;
+Enumeration entries;
+if (batchEntries <= 0) {
+long readTo = Math.min(nextLimit, lastConfirmed);
+entries = lh.readEntries(lastRead + 1, readTo);
+} else {
+entries = lh.batchReadEntries(lastRead, batchEntries,
-1);
+}
while (entries.hasMoreElements()) {
LedgerEntry e = entries.nextElement();
entriesRead++;
+lastRead = e.getEntryId();
if ((entriesRead % 1) == 0) {
-LOG.info("{} entries read", entriesRead);
+LOG.info("{} entries read from ledger {}",
entriesRead, ledgerId);
}
+e.getEntryBuffer().release();
}
}
long endtime = System.nanoTime();
@@ -159,6 +165,8 @@ public class BenchReadThroughputLatency {
options.addOption("sockettimeout", true, "Socket timeout for
bookkeeper client. In seconds. Default 5");
options.addOption("useV2", false, "Whether use V2 protocol to read
ledgers from the bookie server.");
options.addOption("help", false, "This message");
+options.addOption("batchentries", true, "The batch read entries count.
"
++ "If the value is greater than 0, uses batch read. Or uses
the single read. Default 1000");
CommandLineParser parser = new PosixParser();
CommandLine cmd = parser.parse(options, args);
@@ -171,6 +179,7 @@ public class BenchReadThroughputLatency {
final String servers = cmd.getOptionValue("zookeeper",
"localhost:2181");
final byte[] passwd = cmd.getOptionValue("password",
"benchPasswd").getBytes(UTF_8);
final int sockTimeout =
Integer.parseInt(cmd.getOptionValue("sockettimeout", "5"));
+final int batchentries =
Integer.parseInt(cmd.getOptionValue("batchentries", "1000"));
if (cmd.hasOption("ledger") && cmd.hasOption("listen")) {
LOG.error("Cannot used -ledger and -listen together");
usage(options);
@@ -210,7 +219,7 @@ public class BenchReadThroughputLatency {
try {
if (event.getType() == Event.EventType.NodeCreated
&&
(bookkeeper) branch master updated (ee56a953fe -> 978414fdbf)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git from ee56a953fe Use mockito instead of powermock (#4212) add 978414fdbf Add `InetAddress.getLocalHost().getCanonicalHostName()` for MyRack. (#4213) No new revisions were added by this update. Summary of changes: .../apache/bookkeeper/client/TestRegionAwareEnsemblePlacementPolicy.java | 1 + 1 file changed, 1 insertion(+)
(bookkeeper) branch master updated: Use mockito instead of powermock (#4212)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new ee56a953fe Use mockito instead of powermock (#4212)
ee56a953fe is described below
commit ee56a953fe0f7ff5ea5ecc2c1afd57c5376ff780
Author: Zixuan Liu
AuthorDate: Wed Feb 21 02:13:16 2024 +0800
Use mockito instead of powermock (#4212)
---
.../org/apache/bookkeeper/bookie/BookieImpl.java | 17 +-
.../org/apache/bookkeeper/bookie/BookieShell.java | 16 +-
.../bookkeeper/bookie/GarbageCollectorThread.java | 10 +-
.../java/org/apache/bookkeeper/bookie/Journal.java | 37 +++-
.../bookkeeper/bookie/LegacyCookieValidation.java | 7 +
.../bookkeeper/bookie/SortedLedgerStorage.java | 11 +-
.../org/apache/bookkeeper/bookie/SyncThread.java | 7 +-
.../bookkeeper/bookie/stats/JournalStats.java | 8 +-
.../apache/bookkeeper/client/BookKeeperAdmin.java | 7 +
.../org/apache/bookkeeper/client/LedgerHandle.java | 4 +
.../bookkeeper/meta/zk/ZKMetadataBookieDriver.java | 8 +
.../bookkeeper/meta/zk/ZKMetadataClientDriver.java | 18 +-
.../org/apache/bookkeeper/proto/BookieServer.java | 11 ++
.../apache/bookkeeper/server/EmbeddedServer.java | 9 +-
.../bookkeeper/server/service/BookieService.java | 4 +-
.../tools/cli/commands/bookie/LastMarkCommand.java | 6 +
.../cli/commands/bookies/ClusterInfoCommand.java | 5 +
.../cli/commands/bookies/ListBookiesCommand.java | 13 +-
.../tools/cli/commands/bookies/RecoverCommand.java | 3 +-
.../cli/commands/client/SimpleTestCommand.java | 13 +-
.../bookkeeper/bookie/BookieJournalForceTest.java | 59 +++---
.../bookie/BookieJournalMaxMemoryTest.java | 21 +--
.../bookie/BookieJournalPageCacheFlushTest.java| 40 +++--
.../bookkeeper/bookie/BookieJournalTest.java | 36 ++--
.../apache/bookkeeper/bookie/BookieShellTest.java | 197 ++---
.../bookie/BookieWriteToJournalTest.java | 19 +-
.../bookie/GarbageCollectorThreadTest.java | 5 +-
.../bookkeeper/bookie/LedgerDirsManagerTest.java | 19 +-
.../bookie/LedgerStorageCheckpointTest.java| 41 +++--
.../storage/ldb/LedgersIndexRebuildTest.java | 14 +-
.../bookkeeper/client/BookieWriteLedgerTest.java | 3 +-
.../bookkeeper/client/api/WriteAdvHandleTest.java | 2 +-
.../discover/AbstractTestZkRegistrationClient.java | 13 +-
.../meta/AbstractZkLedgerManagerTest.java | 26 +--
.../meta/zk/ZKMetadataBookieDriverTest.java| 36 ++--
.../meta/zk/ZKMetadataClientDriverTest.java| 26 ++-
.../meta/zk/ZKMetadataDriverBaseTest.java | 39 ++--
.../meta/zk/ZKMetadataDriverTestBase.java | 14 +-
.../bookkeeper/server/TestEmbeddedServer.java | 154 ++--
.../bookkeeper/server/http/TestHttpService.java| 2 +-
.../zookeeper/MockZooKeeperTestCase.java | 52 +++---
.../powermock/extensions/configuration.properties | 22 ---
pom.xml| 23 +--
.../bookkeeper/clients/StorageClientImpl.java | 20 ++-
.../bookkeeper/clients/StorageClientImplTest.java | 49 ++---
.../clients/impl/kv/ByteBufTableImpl.java | 6 +
.../shaded/DistributedLogCoreShadedJarTest.java| 17 +-
47 files changed, 639 insertions(+), 530 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java
index 6aac4ecf68..654ace4547 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java
@@ -433,7 +433,7 @@ public class BookieImpl extends BookieCriticalThread
implements Bookie {
// instantiate the journals
journals = Lists.newArrayList();
for (int i = 0; i < journalDirectories.size(); i++) {
-journals.add(new Journal(i, journalDirectories.get(i),
+journals.add(Journal.newJournal(i, journalDirectories.get(i),
conf, ledgerDirsManager, statsLogger.scope(JOURNAL_SCOPE),
allocator, journalAliveListener));
}
@@ -496,6 +496,21 @@ public class BookieImpl extends BookieCriticalThread
implements Bookie {
this.bookieStats = new BookieStats(statsLogger,
journalDirectories.size(), conf.getJournalQueueSize());
}
+@VisibleForTesting
+public static BookieImpl newBookieImpl(ServerConfiguration conf,
+ RegistrationManager
registrationManager,
+ LedgerStorage storage,
+ DiskChecker diskChecker,
+ LedgerDirsManager ledgerDirsMana
(bookkeeper) branch master updated: AutoRecovery supports batch read (#4211)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new ef4ce1385f AutoRecovery supports batch read (#4211)
ef4ce1385f is described below
commit ef4ce1385f8f59fe81cb9f47d72777356a99969d
Author: Hang Chen
AuthorDate: Wed Feb 21 02:11:12 2024 +0800
AutoRecovery supports batch read (#4211)
* AutoRecovery support batch read
* Fix check style
* address comments
---
.../client/LedgerFragmentReplicator.java | 142 +++--
.../bookkeeper/conf/ClientConfiguration.java | 23
.../replication/TestReplicationWorker.java | 63 +
3 files changed, 216 insertions(+), 12 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerFragmentReplicator.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerFragmentReplicator.java
index 5cc22362ac..9f6c90d29e 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerFragmentReplicator.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerFragmentReplicator.java
@@ -171,29 +171,41 @@ public class LedgerFragmentReplicator {
return;
}
-/*
- * Add all the entries to entriesToReplicate list from
- * firstStoredEntryId to lastStoredEntryID.
- */
-List entriesToReplicate = new LinkedList();
-long lastStoredEntryId = lf.getLastStoredEntryId();
-for (long i = lf.getFirstStoredEntryId(); i <= lastStoredEntryId; i++)
{
-entriesToReplicate.add(i);
-}
/*
* Now asynchronously replicate all of the entries for the ledger
* fragment that were on the dead bookie.
*/
+int entriesToReplicateCnt = (int) (endEntryId - startEntryId + 1);
MultiCallback ledgerFragmentEntryMcb = new MultiCallback(
-entriesToReplicate.size(), ledgerFragmentMcb, null,
BKException.Code.OK,
+entriesToReplicateCnt, ledgerFragmentMcb, null,
BKException.Code.OK,
BKException.Code.LedgerRecoveryException);
if (this.replicationThrottle != null) {
this.replicationThrottle.resetRate(this.conf.getReplicationRateByBytes());
}
-for (final Long entryId : entriesToReplicate) {
-recoverLedgerFragmentEntry(entryId, lh, ledgerFragmentEntryMcb,
+
+if (conf.isRecoveryBatchReadEnabled()
+&& conf.getUseV2WireProtocol()
+&& conf.isBatchReadEnabled()
+&& lh.getLedgerMetadata().getEnsembleSize() ==
lh.getLedgerMetadata().getWriteQuorumSize()) {
+batchRecoverLedgerFragmentEntry(startEntryId, endEntryId, lh,
ledgerFragmentEntryMcb,
newBookies, onReadEntryFailureCallback);
+
+} else {
+/*
+ * Add all the entries to entriesToReplicate list from
+ * firstStoredEntryId to lastStoredEntryID.
+ */
+List entriesToReplicate = new LinkedList();
+long lastStoredEntryId = lf.getLastStoredEntryId();
+for (long i = lf.getFirstStoredEntryId(); i <= lastStoredEntryId;
i++) {
+entriesToReplicate.add(i);
+}
+for (final Long entryId : entriesToReplicate) {
+recoverLedgerFragmentEntry(entryId, lh, ledgerFragmentEntryMcb,
+newBookies, onReadEntryFailureCallback);
+}
}
+
}
/**
@@ -433,6 +445,112 @@ public class LedgerFragmentReplicator {
}, null);
}
+void batchRecoverLedgerFragmentEntry(final long startEntryId,
+ final long endEntryId,
+ final LedgerHandle lh,
+ final AsyncCallback.VoidCallback
ledgerFragmentMcb,
+ final Set newBookies,
+ final BiConsumer
onReadEntryFailureCallback)
+throws InterruptedException {
+int entriesToReplicateCnt = (int) (endEntryId - startEntryId + 1);
+int maxBytesToReplicate = conf.getReplicationRateByBytes();
+if (replicationThrottle != null) {
+if (maxBytesToReplicate != -1 && maxBytesToReplicate >
averageEntrySize.get() * entriesToReplicateCnt) {
+maxBytesToReplicate = averageEntrySize.get() *
entriesToReplicateCnt;
+}
+replicationThrottle.acquire(maxBytesToReplicate);
+}
+
+lh.asyncBatchReadEntries(startEntryId, entriesToReplicateCnt,
maxBytesToReplicate,
+new ReadCallback() {
+@Override
+public
(bookkeeper) branch master updated: Improve DefaultEntryLogger read performance. (#4038)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new f5e4a983e4 Improve DefaultEntryLogger read performance. (#4038)
f5e4a983e4 is described below
commit f5e4a983e48f90e40dad4115be43505d47434e0a
Author: Yan Zhao
AuthorDate: Mon Feb 12 09:20:14 2024 +0800
Improve DefaultEntryLogger read performance. (#4038)
* Avoid system call to improve read performance.
* Fix ci.
* Add comments for getCurrentWritingLogId
* Fix ci.
* Consider compacting log.
* Fix checkstyle.
* Address the comment.
* Address comment.
* Address the comments.
* Add tests.
* Fix checkstyle.
* address the comments.
* Fix concurrency problem.
---
.../bookkeeper/bookie/BufferedReadChannel.java | 27 +++-
.../bookkeeper/bookie/DefaultEntryLogger.java | 13 +-
.../bookkeeper/bookie/EntryLogManagerBase.java | 6 ++-
.../bookie/EntryLogManagerForSingleEntryLog.java | 5 ++-
.../bookkeeper/bookie/EntryLoggerAllocator.java| 27 +---
.../bookie/TransactionalEntryLogCompactor.java | 11 +
.../bookkeeper/bookie/DefaultEntryLogTest.java | 48 ++
7 files changed, 125 insertions(+), 12 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BufferedReadChannel.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BufferedReadChannel.java
index 22f5a81690..4de3890e08 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BufferedReadChannel.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BufferedReadChannel.java
@@ -30,7 +30,7 @@ import java.nio.channels.FileChannel;
/**
* A Buffered channel without a write buffer. Only reads are buffered.
*/
-public class BufferedReadChannel extends BufferedChannelBase {
+public class BufferedReadChannel extends BufferedChannelBase {
// The capacity of the read buffer.
protected final int readCapacity;
@@ -43,9 +43,16 @@ public class BufferedReadChannel extends BufferedChannelBase
{
long invocationCount = 0;
long cacheHitCount = 0;
+private volatile long fileSize = -1;
+final boolean sealed;
public BufferedReadChannel(FileChannel fileChannel, int readCapacity) {
+this(fileChannel, readCapacity, false);
+}
+
+public BufferedReadChannel(FileChannel fileChannel, int readCapacity,
boolean sealed) {
super(fileChannel);
+this.sealed = sealed;
this.readCapacity = readCapacity;
this.readBuffer = Unpooled.buffer(readCapacity);
}
@@ -64,10 +71,26 @@ public class BufferedReadChannel extends
BufferedChannelBase {
return read(dest, pos, dest.writableBytes());
}
+@Override
+public long size() throws IOException {
+if (sealed) {
+if (fileSize == -1) {
+synchronized (this) {
+if (fileSize == -1) {
+fileSize = validateAndGetFileChannel().size();
+}
+}
+}
+return fileSize;
+} else {
+return validateAndGetFileChannel().size();
+}
+}
+
public synchronized int read(ByteBuf dest, long pos, int length) throws
IOException {
invocationCount++;
long currentPosition = pos;
-long eof = validateAndGetFileChannel().size();
+long eof = size();
// return -1 if the given position is greater than or equal to the
file's current size.
if (pos >= eof) {
return -1;
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/DefaultEntryLogger.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/DefaultEntryLogger.java
index d02ede52fb..c47c0411c2 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/DefaultEntryLogger.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/DefaultEntryLogger.java
@@ -587,6 +587,10 @@ public class DefaultEntryLogger implements EntryLogger {
}
}
+void clearCompactingLogId() {
+entryLoggerAllocator.clearCompactingLogId();
+}
+
/**
* Flushes all rotated log channels. After log channels are flushed,
* move leastUnflushedLogId ptr to current logId.
@@ -894,7 +898,8 @@ public class DefaultEntryLogger implements EntryLogger {
}
}
-private BufferedReadChannel getChannelForLogId(long entryLogId) throws
IOException {
+@VisibleForTesting
+BufferedReadChannel getChannelForLogId(long entryLogId) throws IOException
{
BufferedReadChannel fc = getFromChannels(entryLogId);
if (fc != null) {
return fc;
@@ -91
(bookkeeper) branch branch-4.16 updated: Fix checksum calculation bug when the payload is a CompositeByteBuf with readerIndex > 0 (#4196) (#4205)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch branch-4.16 in repository https://gitbox.apache.org/repos/asf/bookkeeper.git The following commit(s) were added to refs/heads/branch-4.16 by this push: new 1e891515e6 Fix checksum calculation bug when the payload is a CompositeByteBuf with readerIndex > 0 (#4196) (#4205) 1e891515e6 is described below commit 1e891515e6cc1638bf93aa183936a61c281527c3 Author: Lari Hotari AuthorDate: Fri Feb 9 08:28:47 2024 -0800 Fix checksum calculation bug when the payload is a CompositeByteBuf with readerIndex > 0 (#4196) (#4205) * Add a test that reproduces a bug in checksum calculation * Revert "Fixed unnecessary copy to heap (#2701)" changes to ByteBufList This partially reverts commit 3c9c7102538909fd3764ea7314e7618d6d9458fd. * Remove CompositeBuffer unwrapping in DigestManager * Rename update -> internalUpdate so that unwrapping logic could be added to update * Remove unnecessary unwrapping logic in Java9IntHash * Add safe way to handle CompositeByteBuf * Add license header * Fix checkstyle * Refactor ByteBuf visitor solution * Fix checkstyle * Reformat * Refactor recursive visiting * Revisit equals, hashCode and toString * Refactor test case * Add support for UnpooledHeapByteBuf.getBytes which passes an array * Add support for visiting buffers backed by byte[] arrays - getBytes calls setBytes with a byte[] argument for heap ByteBufs * Move ByteBufVisitor to org.apache.bookkeeper.util package * Update javadoc * Refactor to use stateless visitor so that instance can be shared * Improve test so that a single scenario can be used for debugging * Fix bug in Java9IntHash calculation that assumed crc32c_update(x) == ~crc32c_update(~x) - Java9IntHash uses private methods from java.util.zip.CRC32C class, updateBytes and updateDirectByteBuffer. When inspecting the use and interface contract, it doesn't match how it is used in Java9IntHash. This PR addresses that by introducing a separate initial value for initializing the accumulated value so that the initial value could match the logic in java.util.zip.CRC32C.reset method. There's also a separate method for finalizing the accumulated value into a final checksum value. This is to match the java.util.zip.CRC32C.getValue method's logic (uses bitwise complement operator ~). - With a quick glance, it might appear that the previous logic is similar. However it isn't since I have a failing test which gets fixed with this change. I haven't yet added the Java9IntHash level unit test case to prove how it differs. It must be related to integer value overflow. For the CRC32C function, I believe it means that it cannot be assumed in all cases that func(x) == ~func(~x). That's the assumption that the previous code was making. It probably applies for many inputs, but not all. It would break in overflow cases. * Fix checkstyle * Fix checkstyle * Fix missing depth increment that prevents StackOverflowException * Properly handle the depth increase and decrease * Remove unnecessary condition * Use more efficient way to read bytes to the target array * Don't use ByteBufVisitor if it's not necessary * Revert "Fix bug in Java9IntHash calculation that assumed crc32c_update(x) == ~crc32c_update(~x)" This reverts commit 272e962930a31cbc237c5e7c0bd0c93213520ba4. * Fix issue in resume byte[] version that was added - input and output should be complemented. explanation has been added to the resume ByteBuf method * Polish ByteBufVisitor - reuse GetBytesCallbackByteBuf instance for handling the root ByteBuf instance * Use extracted method * Fix bug with array handling * Polish ByteBufVisitor * Optimize the buffer copying in the case where array or memory address cannot be accessed - read-only buffers will need to be copied before reading - use ByteBuf.copy for direct buffers with pooled allocator when the algorithm can accept a memory address buffer - use the 64kB threadlocal byte[] buffer for copying all other inputs * Check if memory address is accepted * Improve comments about complement (current = ~current) in resume * Print thread dump when build is cancelled * Filter empty buffers and arrays in ByteBufVisitor (cherry picked from commit 9c373f7e7b5b62d57dc5d295039229c2461ae518) --- .github/workflows/bk-ci.yml|4 + .../proto/checksum/CRC32CDigestManager.java
(bookkeeper) branch master updated: Use netty-bom for aligning netty library versions, add epoll for linux-aarch_64 (#4204)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git The following commit(s) were added to refs/heads/master by this push: new bb9da9b8e1 Use netty-bom for aligning netty library versions, add epoll for linux-aarch_64 (#4204) bb9da9b8e1 is described below commit bb9da9b8e1a215f2e7f575af68e867868d2eec88 Author: Lari Hotari AuthorDate: Thu Feb 8 11:42:32 2024 -0800 Use netty-bom for aligning netty library versions, add epoll for linux-aarch_64 (#4204) --- .../src/main/resources/LICENSE-all.bin.txt | 1 + .../src/main/resources/LICENSE-bkctl.bin.txt | 1 + .../src/main/resources/LICENSE-server.bin.txt | 1 + .../src/main/resources/NOTICE-all.bin.txt | 1 + .../src/main/resources/NOTICE-bkctl.bin.txt| 1 + .../src/main/resources/NOTICE-server.bin.txt | 1 + bookkeeper-server/pom.xml | 7 +- pom.xml| 82 +++--- 8 files changed, 20 insertions(+), 75 deletions(-) diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index 28d0baefaf..eac9503e6c 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -237,6 +237,7 @@ Apache Software License, Version 2. - lib/io.netty-netty-tcnative-classes-2.0.61.Final.jar [11] - lib/io.netty-netty-transport-4.1.104.Final.jar [11] - lib/io.netty-netty-transport-classes-epoll-4.1.104.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.104.Final-linux-aarch_64.jar [11] - lib/io.netty-netty-transport-native-epoll-4.1.104.Final-linux-x86_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.24.Final-linux-x86_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.24.Final-linux-aarch_64.jar [11] diff --git a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt index df93ad4db4..09864a06bb 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt @@ -235,6 +235,7 @@ Apache Software License, Version 2. - lib/io.netty-netty-tcnative-classes-2.0.61.Final.jar [11] - lib/io.netty-netty-transport-4.1.104.Final.jar [11] - lib/io.netty-netty-transport-classes-epoll-4.1.104.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.104.Final-linux-aarch_64.jar [11] - lib/io.netty-netty-transport-native-epoll-4.1.104.Final-linux-x86_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.24.Final-linux-x86_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.24.Final-linux-aarch_64.jar [11] diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index 3735796025..a9c51be92f 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -237,6 +237,7 @@ Apache Software License, Version 2. - lib/io.netty-netty-tcnative-classes-2.0.61.Final.jar [11] - lib/io.netty-netty-transport-4.1.104.Final.jar [11] - lib/io.netty-netty-transport-classes-epoll-4.1.104.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.104.Final-linux-aarch_64.jar [11] - lib/io.netty-netty-transport-native-epoll-4.1.104.Final-linux-x86_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.24.Final-linux-x86_64.jar [11] - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.24.Final-linux-aarch_64.jar [11] diff --git a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt index 95c83adba5..797b6ada0a 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt @@ -43,6 +43,7 @@ LongAdder), which was released with the following comments: - lib/io.netty-netty-tcnative-classes-2.0.61.Final.jar - lib/io.netty-netty-transport-4.1.104.Final.jar - lib/io.netty-netty-transport-classes-epoll-4.1.104.Final.jar +- lib/io.netty-netty-transport-native-epoll-4.1.104.Final-linux-aarch_64.jar - lib/io.netty-netty-transport-native-epoll-4.1.104.Final-linux-x86_64.jar - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.24.Final-linux-x86_64.jar - lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.24.Final-linux-aarch_64.jar diff --git a/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt index f9e530e4af..64573652ca 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt +++ b/bookkeeper-dist
(bookkeeper) branch dependabot/maven/org.jsoup-jsoup-1.15.3 deleted (was e50d5f232e)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch dependabot/maven/org.jsoup-jsoup-1.15.3 in repository https://gitbox.apache.org/repos/asf/bookkeeper.git was e50d5f232e Bump jsoup from 1.14.3 to 1.15.3 The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
(bookkeeper) branch master updated: Bump jsoup from 1.14.3 to 1.15.3 (#3465)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git The following commit(s) were added to refs/heads/master by this push: new 27377dcb53 Bump jsoup from 1.14.3 to 1.15.3 (#3465) 27377dcb53 is described below commit 27377dcb53cb9f224a729ecee8b552afd3bca73b Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Wed Feb 7 14:02:45 2024 -0800 Bump jsoup from 1.14.3 to 1.15.3 (#3465) Bumps [jsoup](https://github.com/jhy/jsoup) from 1.14.3 to 1.15.3. - [Release notes](https://github.com/jhy/jsoup/releases) - [Changelog](https://github.com/jhy/jsoup/blob/master/CHANGES) - [Commits](https://github.com/jhy/jsoup/compare/jsoup-1.14.3...jsoup-1.15.3) --- updated-dependencies: - dependency-name: org.jsoup:jsoup dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 038c13c410..a7394b4907 100644 --- a/pom.xml +++ b/pom.xml @@ -143,7 +143,7 @@ 9.4.53.v20231009 1.19 2.8.2 -1.14.3 +1.15.3 4.12 5.8.2
(bookkeeper) branch merlimat-patch-1 created (now cf06328a5f)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch merlimat-patch-1 in repository https://gitbox.apache.org/repos/asf/bookkeeper.git at cf06328a5f Enable CodeQL static code scanner This branch includes the following new commits: new cf06328a5f Enable CodeQL static code scanner The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(bookkeeper) 01/01: Enable CodeQL static code scanner
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch merlimat-patch-1
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit cf06328a5f8cc070f5d58ff32afd9a59c451d0d1
Author: Matteo Merli
AuthorDate: Wed Feb 7 11:07:04 2024 -0800
Enable CodeQL static code scanner
---
.github/workflows/codeql.yml | 101 +++
1 file changed, 101 insertions(+)
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
new file mode 100644
index 00..19b6393ee5
--- /dev/null
+++ b/.github/workflows/codeql.yml
@@ -0,0 +1,101 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: "CodeQL"
+
+on:
+ push:
+branches: [ "master" ]
+ pull_request:
+branches: [ "master" ]
+ schedule:
+- cron: '27 11 * * 6'
+
+jobs:
+ analyze:
+name: Analyze
+runs-on: 'ubuntu-latest'
+timeout-minutes: 360
+permissions:
+ # required for all workflows
+ security-events: write
+
+ # only required for workflows in private repositories
+ actions: read
+ contents: read
+
+strategy:
+ fail-fast: false
+ matrix:
+language: [ 'c-cpp', 'java-kotlin', 'python' ]
+
+steps:
+- name: Checkout repository
+ uses: actions/checkout@v4
+
+- name: Detect changed files
+ id: changes
+ uses: apache/pulsar-test-infra/paths-filter@master
+ with:
+filters: .github/changes-filter.yaml
+list-files: csv
+
+- name: Check changed files
+ id: check_changes
+ run: |
+echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) ==
fromJSON(steps.changes.outputs.docs_count) &&
fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT
+
+- name: Cache local Maven repository
+ if: steps.check_changes.outputs.docs_only != 'true'
+ id: cache
+ uses: actions/cache@v3
+ with:
+path: |
+ ~/.m2/repository/*/*/*
+ !~/.m2/repository/org/apache/bookkeeper
+ !~/.m2/repository/org/apache/distributedlog
+key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+
+# Initializes the CodeQL tools for scanning.
+- name: Initialize CodeQL
+ uses: github/codeql-action/init@v3
+ with:
+languages: ${{ matrix.language }}
+
+- name: Set up JDK 11
+ if: steps.check_changes.outputs.docs_only != 'true'
+ uses: actions/setup-java@v2
+ with:
+distribution: 'temurin'
+java-version: 11
+
+- name: Set up Maven
+ uses: apache/pulsar-test-infra/setup-maven@master
+ with:
+maven-version: 3.8.7
+
+- name: Validate pull request
+ if: steps.check_changes.outputs.docs_only != 'true'
+ run: |
+mvn -T 1C -B -nsu clean install -Ddistributedlog -DskipTests
-Dorg.slf4j.simpleLogger.defaultLogLevel=INFO
+
+- name: Perform CodeQL Analysis
+ if: steps.check_changes.outputs.docs_only != 'true'
+ uses: github/codeql-action/analyze@v3
+ with:
+category: "/language:${{matrix.language}}"
(bookkeeper) branch master updated: Fix checksum calculation bug when the payload is a CompositeByteBuf with readerIndex > 0 (#4196)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git The following commit(s) were added to refs/heads/master by this push: new 9c373f7e7b Fix checksum calculation bug when the payload is a CompositeByteBuf with readerIndex > 0 (#4196) 9c373f7e7b is described below commit 9c373f7e7b5b62d57dc5d295039229c2461ae518 Author: Lari Hotari AuthorDate: Wed Feb 7 10:33:00 2024 -0800 Fix checksum calculation bug when the payload is a CompositeByteBuf with readerIndex > 0 (#4196) * Add a test that reproduces a bug in checksum calculation * Revert "Fixed unnecessary copy to heap (#2701)" changes to ByteBufList This partially reverts commit 3c9c7102538909fd3764ea7314e7618d6d9458fd. * Remove CompositeBuffer unwrapping in DigestManager * Rename update -> internalUpdate so that unwrapping logic could be added to update * Remove unnecessary unwrapping logic in Java9IntHash * Add safe way to handle CompositeByteBuf * Add license header * Fix checkstyle * Refactor ByteBuf visitor solution * Fix checkstyle * Reformat * Refactor recursive visiting * Revisit equals, hashCode and toString * Refactor test case * Add support for UnpooledHeapByteBuf.getBytes which passes an array * Add support for visiting buffers backed by byte[] arrays - getBytes calls setBytes with a byte[] argument for heap ByteBufs * Move ByteBufVisitor to org.apache.bookkeeper.util package * Update javadoc * Refactor to use stateless visitor so that instance can be shared * Improve test so that a single scenario can be used for debugging * Fix bug in Java9IntHash calculation that assumed crc32c_update(x) == ~crc32c_update(~x) - Java9IntHash uses private methods from java.util.zip.CRC32C class, updateBytes and updateDirectByteBuffer. When inspecting the use and interface contract, it doesn't match how it is used in Java9IntHash. This PR addresses that by introducing a separate initial value for initializing the accumulated value so that the initial value could match the logic in java.util.zip.CRC32C.reset method. There's also a separate method for finalizing the accumulated value into a final checksum value. This is to match the java.util.zip.CRC32C.getValue method's logic (uses bitwise complement operator ~). - With a quick glance, it might appear that the previous logic is similar. However it isn't since I have a failing test which gets fixed with this change. I haven't yet added the Java9IntHash level unit test case to prove how it differs. It must be related to integer value overflow. For the CRC32C function, I believe it means that it cannot be assumed in all cases that func(x) == ~func(~x). That's the assumption that the previous code was making. It probably applies for many inputs, but not all. It would break in overflow cases. * Fix checkstyle * Fix checkstyle * Fix missing depth increment that prevents StackOverflowException * Properly handle the depth increase and decrease * Remove unnecessary condition * Use more efficient way to read bytes to the target array * Don't use ByteBufVisitor if it's not necessary * Revert "Fix bug in Java9IntHash calculation that assumed crc32c_update(x) == ~crc32c_update(~x)" This reverts commit 272e962930a31cbc237c5e7c0bd0c93213520ba4. * Fix issue in resume byte[] version that was added - input and output should be complemented. explanation has been added to the resume ByteBuf method * Polish ByteBufVisitor - reuse GetBytesCallbackByteBuf instance for handling the root ByteBuf instance * Use extracted method * Fix bug with array handling * Polish ByteBufVisitor * Optimize the buffer copying in the case where array or memory address cannot be accessed - read-only buffers will need to be copied before reading - use ByteBuf.copy for direct buffers with pooled allocator when the algorithm can accept a memory address buffer - use the 64kB threadlocal byte[] buffer for copying all other inputs * Check if memory address is accepted * Improve comments about complement (current = ~current) in resume * Print thread dump when build is cancelled * Filter empty buffers and arrays in ByteBufVisitor --- .github/workflows/bk-ci.yml|4 + .../proto/checksum/CRC32CDigestManager.java| 12 +- .../proto/checksum/CRC32DigestManager.java | 14 +- .../bookkeeper/proto/checksu
(bookkeeper) branch master updated: Change callbacks in CleanupLedgerManager as a Set (#4123)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 113d40ac50 Change callbacks in CleanupLedgerManager as a Set (#4123)
113d40ac50 is described below
commit 113d40ac5057709b3e44b9281231456b4ef81065
Author: houxiaoyu
AuthorDate: Fri Jan 12 01:21:41 2024 +0800
Change callbacks in CleanupLedgerManager as a Set (#4123)
---
.../apache/bookkeeper/meta/CleanupLedgerManager.java | 19 ---
1 file changed, 8 insertions(+), 11 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/CleanupLedgerManager.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/CleanupLedgerManager.java
index 669304c1c1..bc4cefe6fc 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/CleanupLedgerManager.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/CleanupLedgerManager.java
@@ -23,7 +23,6 @@ import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.client.BKException;
@@ -55,7 +54,7 @@ public class CleanupLedgerManager implements LedgerManager {
public void operationComplete(int rc, T result) {
closeLock.readLock().lock();
try {
-if (!closed && null != removeCallback(cb)) {
+if (!closed && removeCallback(cb)) {
cb.operationComplete(rc, result);
}
} finally {
@@ -78,8 +77,7 @@ public class CleanupLedgerManager implements LedgerManager {
}
private final LedgerManager underlying;
-private final ConcurrentMap callbacks =
-new ConcurrentHashMap();
+private final Set callbacks =
ConcurrentHashMap.newKeySet();
private boolean closed = false;
private final ReentrantReadWriteLock closeLock = new
ReentrantReadWriteLock();
private final Set> futures =
ConcurrentHashMap.newKeySet();
@@ -94,7 +92,7 @@ public class CleanupLedgerManager implements LedgerManager {
}
private void addCallback(GenericCallback callback) {
-callbacks.put(callback, callback);
+callbacks.add(callback);
}
@Override
@@ -107,7 +105,7 @@ public class CleanupLedgerManager implements LedgerManager {
underlying.unregisterLedgerMetadataListener(ledgerId, listener);
}
-private GenericCallback removeCallback(GenericCallback callback) {
+private boolean removeCallback(GenericCallback callback) {
return callbacks.remove(callback);
}
@@ -206,7 +204,7 @@ public class CleanupLedgerManager implements LedgerManager {
underlying.asyncProcessLedgers(processor, new
AsyncCallback.VoidCallback() {
@Override
public void processResult(int rc, String path, Object ctx) {
-if (null != removeCallback(stub)) {
+if (removeCallback(stub)) {
finalCb.processResult(rc, path, ctx);
}
}
@@ -239,14 +237,13 @@ public class CleanupLedgerManager implements
LedgerManager {
return;
}
closed = true;
-keys = new HashSet(callbacks.keySet());
+keys = new HashSet<>(callbacks);
} finally {
closeLock.writeLock().unlock();
}
for (GenericCallback key : keys) {
-GenericCallback callback = callbacks.remove(key);
-if (null != callback) {
-
callback.operationComplete(BKException.Code.ClientClosedException, null);
+if (callbacks.remove(key)) {
+key.operationComplete(BKException.Code.ClientClosedException,
null);
}
}
BKException exception = new BKException.BKClientClosedException();
(bookkeeper) branch master updated: Improve auto-recovery noise log when some bookie down. (#4118)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 54168b5d64 Improve auto-recovery noise log when some bookie down.
(#4118)
54168b5d64 is described below
commit 54168b5d6441f7cbe0b478867d8301a6eee0b891
Author: Yan Zhao
AuthorDate: Thu Jan 11 05:50:48 2024 +0800
Improve auto-recovery noise log when some bookie down. (#4118)
* Improve auto-recovery noise log when some bookie down.
* Address comment.
---
.../client/RackawareEnsemblePlacementPolicyImpl.java | 19 ---
.../client/TopologyAwareEnsemblePlacementPolicy.java | 5 ++---
2 files changed, 18 insertions(+), 6 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java
index 7f219854ed..3863a26a24 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java
@@ -507,13 +507,26 @@ public class RackawareEnsemblePlacementPolicyImpl extends
TopologyAwareEnsembleP
try {
excludeBookies =
addDefaultRackBookiesIfMinNumRacksIsEnforced(excludeBookies);
excludeBookies.addAll(currentEnsemble);
+
+Set ensembleNodes = new HashSet<>();
+Set excludeNodes = new HashSet<>();
BookieNode bn = knownBookies.get(bookieToReplace);
if (null == bn) {
bn = createBookieNode(bookieToReplace);
}
-
-Set ensembleNodes = convertBookiesToNodes(currentEnsemble);
-Set excludeNodes = convertBookiesToNodes(excludeBookies);
+for (BookieId bookieId : currentEnsemble) {
+if (bookieId.equals(bookieToReplace)) {
+continue;
+}
+ensembleNodes.add(convertBookieToNode(bookieId));
+}
+for (BookieId bookieId : excludeBookies) {
+if (bookieId.equals(bookieToReplace)) {
+excludeNodes.add(bn);
+continue;
+}
+excludeNodes.add(convertBookieToNode(bookieId));
+}
excludeNodes.addAll(ensembleNodes);
excludeNodes.add(bn);
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java
index 463d9599de..4976f96e8c 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/TopologyAwareEnsemblePlacementPolicy.java
@@ -824,9 +824,8 @@ abstract class TopologyAwareEnsemblePlacementPolicy
implements
if (null != historyBookie) {
return historyBookie.getNetworkLocation();
}
-
-LOG.error("Cannot resolve bookieId {} to a network address,
resolving as {}", addr,
- NetworkTopology.DEFAULT_REGION_AND_RACK, err);
+LOG.error("Cannot resolve bookieId {} to a network address,
resolving as {}. {}", addr,
+ NetworkTopology.DEFAULT_REGION_AND_RACK,
err.getMessage());
return NetworkTopology.DEFAULT_REGION_AND_RACK;
}
}
(bookkeeper) branch branch-4.16 updated: Chagne the method getUnderreplicatedFragments to the package private (#4174)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch branch-4.16
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/branch-4.16 by this push:
new 15c37b65f1 Chagne the method getUnderreplicatedFragments to the
package private (#4174)
15c37b65f1 is described below
commit 15c37b65f1c1c1c0c2d4c7ea00a90a6f7cdba1a4
Author: Yong Zhang
AuthorDate: Thu Jan 11 03:37:05 2024 +0800
Chagne the method getUnderreplicatedFragments to the package private (#4174)
---
###Motivation
Allow to override the getUnderreplicatedFragments to change
the customize the behaviour.
---
.../main/java/org/apache/bookkeeper/replication/ReplicationWorker.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java
index f22231c567..cf1f2f2f59 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java
@@ -600,7 +600,7 @@ public class ReplicationWorker implements Runnable {
/**
* Gets the under replicated fragments.
*/
-private Set getUnderreplicatedFragments(LedgerHandle lh,
Long ledgerVerificationPercentage)
+Set getUnderreplicatedFragments(LedgerHandle lh, Long
ledgerVerificationPercentage)
throws InterruptedException {
//The data loss fragments is first to repair. If a fragment is
data_loss and not_adhering_placement
//at the same time, we only fix data_loss in this time. After fix
data_loss, the fragment is still
(bookkeeper) branch master updated: Chagne the method getUnderreplicatedFragments to the package private (#4174)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new bdbb5ec4a6 Chagne the method getUnderreplicatedFragments to the
package private (#4174)
bdbb5ec4a6 is described below
commit bdbb5ec4a69ef18db871f9ab2eb82b91cbb66099
Author: Yong Zhang
AuthorDate: Thu Jan 11 03:37:05 2024 +0800
Chagne the method getUnderreplicatedFragments to the package private (#4174)
---
###Motivation
Allow to override the getUnderreplicatedFragments to change
the customize the behaviour.
---
.../main/java/org/apache/bookkeeper/replication/ReplicationWorker.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java
index f22231c567..cf1f2f2f59 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationWorker.java
@@ -600,7 +600,7 @@ public class ReplicationWorker implements Runnable {
/**
* Gets the under replicated fragments.
*/
-private Set getUnderreplicatedFragments(LedgerHandle lh,
Long ledgerVerificationPercentage)
+Set getUnderreplicatedFragments(LedgerHandle lh, Long
ledgerVerificationPercentage)
throws InterruptedException {
//The data loss fragments is first to repair. If a fragment is
data_loss and not_adhering_placement
//at the same time, we only fix data_loss in this time. After fix
data_loss, the fragment is still
(bookkeeper) branch master updated: rebase (#4146)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new c40f756d1a rebase (#4146)
c40f756d1a is described below
commit c40f756d1af87a5b4e7b20a9ea0f70c8d40b3f72
Author: houxiaoyu
AuthorDate: Tue Jan 9 13:50:53 2024 +0800
rebase (#4146)
---
.../java/org/apache/bookkeeper/bookie/DefaultEntryLogger.java | 11 ---
1 file changed, 11 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/DefaultEntryLogger.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/DefaultEntryLogger.java
index 4d43d2ebbb..d02ede52fb 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/DefaultEntryLogger.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/DefaultEntryLogger.java
@@ -255,8 +255,6 @@ public class DefaultEntryLogger implements EntryLogger {
*
*/
static final int LOGFILE_HEADER_SIZE = 1024;
-final ByteBuf logfileHeader = Unpooled.buffer(LOGFILE_HEADER_SIZE);
-
static final int HEADER_VERSION_POSITION = 4;
static final int LEDGERS_MAP_OFFSET_POSITION = HEADER_VERSION_POSITION + 4;
@@ -328,15 +326,6 @@ public class DefaultEntryLogger implements EntryLogger {
addListener(listener);
}
-// Initialize the entry log header buffer. This cannot be a static
object
-// since in our unit tests, we run multiple Bookies and thus
EntryLoggers
-// within the same JVM. All of these Bookie instances access this
header
-// so there can be race conditions when entry logs are rolled over and
-// this header buffer is cleared before writing it into the new
logChannel.
-logfileHeader.writeBytes("BKLO".getBytes(UTF_8));
-logfileHeader.writeInt(HEADER_CURRENT_VERSION);
-logfileHeader.writerIndex(LOGFILE_HEADER_SIZE);
-
// Find the largest logId
long logId = INVALID_LID;
for (File dir : ledgerDirsManager.getAllLedgerDirs()) {
(bookkeeper) branch master updated (031069cf31 -> 381510e459)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git from 031069cf31 Entry write support local node region aware placement policy (#4063) add 381510e459 Increase journal thread priority (#4120) No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/bookkeeper/bookie/Journal.java | 2 ++ 1 file changed, 2 insertions(+)
(bookkeeper) branch master updated (f88f437872 -> 031069cf31)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git from f88f437872 Update copy right year from 2023 to 2024 (#4167) add 031069cf31 Entry write support local node region aware placement policy (#4063) No new revisions were added by this update. Summary of changes: .../client/RegionAwareEnsemblePlacementPolicy.java | 15 ++- .../TestRegionAwareEnsemblePlacementPolicy.java| 115 + 2 files changed, 128 insertions(+), 2 deletions(-)
(bookkeeper) branch master updated (6f5d756550 -> bce832e389)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git from 6f5d756550 Upgrade Netty to 4.1.104.Final and io_uring to 0.0.24.Final (#4157) add bce832e389 [fix][conf-stats] Fix default configuration 'statsProviderClass' (#4156) No new revisions were added by this update. Summary of changes: conf/bk_server.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(bookkeeper) branch master updated: Upgrade Netty to 4.1.104.Final and io_uring to 0.0.24.Final (#4157)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git The following commit(s) were added to refs/heads/master by this push: new 6f5d756550 Upgrade Netty to 4.1.104.Final and io_uring to 0.0.24.Final (#4157) 6f5d756550 is described below commit 6f5d75655066537ff51f6bb0b56831242ffe7363 Author: Lari Hotari AuthorDate: Fri Dec 22 19:12:26 2023 +0200 Upgrade Netty to 4.1.104.Final and io_uring to 0.0.24.Final (#4157) --- .../src/main/resources/LICENSE-all.bin.txt | 94 +++--- .../src/main/resources/LICENSE-bkctl.bin.txt | 90 ++--- .../src/main/resources/LICENSE-server.bin.txt | 94 +++--- .../src/main/resources/NOTICE-all.bin.txt | 34 .../src/main/resources/NOTICE-bkctl.bin.txt| 30 +++ .../src/main/resources/NOTICE-server.bin.txt | 34 pom.xml| 4 +- 7 files changed, 190 insertions(+), 190 deletions(-) diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index 979ab63141..830e9a7275 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -217,17 +217,17 @@ Apache Software License, Version 2. - lib/commons-io-commons-io-2.7.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] -- lib/io.netty-netty-buffer-4.1.94.Final.jar [11] -- lib/io.netty-netty-codec-4.1.94.Final.jar [11] -- lib/io.netty-netty-codec-dns-4.1.94.Final.jar [11] -- lib/io.netty-netty-codec-http-4.1.94.Final.jar [11] -- lib/io.netty-netty-codec-http2-4.1.94.Final.jar [11] -- lib/io.netty-netty-codec-socks-4.1.94.Final.jar [11] -- lib/io.netty-netty-common-4.1.94.Final.jar [11] -- lib/io.netty-netty-handler-4.1.94.Final.jar [11] -- lib/io.netty-netty-handler-proxy-4.1.94.Final.jar [11] -- lib/io.netty-netty-resolver-4.1.94.Final.jar [11] -- lib/io.netty-netty-resolver-dns-4.1.94.Final.jar [11] +- lib/io.netty-netty-buffer-4.1.104.Final.jar [11] +- lib/io.netty-netty-codec-4.1.104.Final.jar [11] +- lib/io.netty-netty-codec-dns-4.1.104.Final.jar [11] +- lib/io.netty-netty-codec-http-4.1.104.Final.jar [11] +- lib/io.netty-netty-codec-http2-4.1.104.Final.jar [11] +- lib/io.netty-netty-codec-socks-4.1.104.Final.jar [11] +- lib/io.netty-netty-common-4.1.104.Final.jar [11] +- lib/io.netty-netty-handler-4.1.104.Final.jar [11] +- lib/io.netty-netty-handler-proxy-4.1.104.Final.jar [11] +- lib/io.netty-netty-resolver-4.1.104.Final.jar [11] +- lib/io.netty-netty-resolver-dns-4.1.104.Final.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.61.Final.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.61.Final-linux-aarch_64.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.61.Final-linux-x86_64.jar [11] @@ -235,13 +235,13 @@ Apache Software License, Version 2. - lib/io.netty-netty-tcnative-boringssl-static-2.0.61.Final-osx-x86_64.jar [11] - lib/io.netty-netty-tcnative-boringssl-static-2.0.61.Final-windows-x86_64.jar [11] - lib/io.netty-netty-tcnative-classes-2.0.61.Final.jar [11] -- lib/io.netty-netty-transport-4.1.94.Final.jar [11] -- lib/io.netty-netty-transport-classes-epoll-4.1.94.Final.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.94.Final-linux-x86_64.jar [11] -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.21.Final-linux-x86_64.jar [11] -- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.21.Final-linux-aarch_64.jar [11] -- lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.21.Final.jar [11] -- lib/io.netty-netty-transport-native-unix-common-4.1.94.Final.jar [11] +- lib/io.netty-netty-transport-4.1.104.Final.jar [11] +- lib/io.netty-netty-transport-classes-epoll-4.1.104.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.104.Final-linux-x86_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.24.Final-linux-x86_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.24.Final-linux-aarch_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.24.Final.jar [11] +- lib/io.netty-netty-transport-native-unix-common-4.1.104.Final.jar [11] - lib/io.prometheus-simpleclient-0.15.0.jar [12] - lib/io.prometheus-simpleclient_common-0.15.0.jar [12] - lib/io.prometheus-simpleclient_hotspot-0.15.0.jar [12] @@ -360,7 +360,7 @@ Apache Software License, Version 2. [8] Source available at https://github.com/apache/commons-io/tree/rel/commons-io-2.7 [9] Source available at https://github.com/apache/commons-lang/tree/LANG_2_6 [10] Source available at https://github.com/apache/commons-logging/tree/commons-logging-1.1.1 -[11] Source available at https
(bookkeeper) 01/02: cut branch-4.16
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch branch-4.16 in repository https://gitbox.apache.org/repos/asf/bookkeeper.git commit 35b02c38527c65e73e329a920eea19846d39cad8 Author: chenhang AuthorDate: Fri Oct 14 17:41:56 2022 +0800 cut branch-4.16 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4f88958e52..fd1fd3f1aa 100644 --- a/pom.xml +++ b/pom.xml @@ -40,7 +40,7 @@ scm:git:https://github.com/apache/bookkeeper.git scm:git:https://github.com/apache/bookkeeper.git https://github.com/apache/bookkeeper -branch-4.13 +branch-4.16 JIRA
(bookkeeper) branch branch-4.16 updated (bdefd9977a -> a9f882b872)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch branch-4.16 in repository https://gitbox.apache.org/repos/asf/bookkeeper.git from bdefd9977a Bump python to 4.16.4-alpha-0 new 35b02c3852 cut branch-4.16 new a9f882b872 fix calculate checkSum when using Java9IntHash (#4140) The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../scurrilous/circe/checksum/Java9IntHash.java| 12 +- .../circe/checksum/Java9IntHashTest.java | 121 + pom.xml| 2 +- 3 files changed, 133 insertions(+), 2 deletions(-) create mode 100644 circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java
(bookkeeper) 02/02: fix calculate checkSum when using Java9IntHash (#4140)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch branch-4.16
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
commit a9f882b87244c407cbf33a21c522edf74d070c08
Author: fengyubiao
AuthorDate: Thu Dec 7 13:49:32 2023 +0800
fix calculate checkSum when using Java9IntHash (#4140)
* fix calculate checkSum when using Java9IntHash
* -
* improve performence
* add test
* edit test
* rename test
* fix license
* address comments
---
.../scurrilous/circe/checksum/Java9IntHash.java| 12 +-
.../circe/checksum/Java9IntHashTest.java | 121 +
2 files changed, 132 insertions(+), 1 deletion(-)
diff --git
a/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Java9IntHash.java
b/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Java9IntHash.java
index 622c66d9ee..31af153666 100644
---
a/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Java9IntHash.java
+++
b/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Java9IntHash.java
@@ -19,6 +19,7 @@
package com.scurrilous.circe.checksum;
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.CompositeByteBuf;
import io.netty.util.concurrent.FastThreadLocal;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
@@ -106,14 +107,23 @@ public class Java9IntHash implements IntHash {
} else if (buffer.hasArray()) {
int arrayOffset = buffer.arrayOffset() + offset;
negCrc = resume(negCrc, buffer.array(), arrayOffset, len);
+} else if (buffer instanceof CompositeByteBuf) {
+ CompositeByteBuf compositeByteBuf = (CompositeByteBuf) buffer;
+ int loopedCurrent = current;
+ for (int i = 0; i < compositeByteBuf.numComponents(); i ++) {
+ loopedCurrent = resume(loopedCurrent,
compositeByteBuf.component(i));
+ }
+ return loopedCurrent;
} else {
byte[] b = TL_BUFFER.get();
int toRead = len;
+int loopOffset = offset;
while (toRead > 0) {
int length = Math.min(toRead, b.length);
-buffer.slice(offset, len).readBytes(b, 0, length);
+buffer.slice(loopOffset, length).readBytes(b, 0, length);
negCrc = resume(negCrc, b, 0, length);
toRead -= length;
+loopOffset += length;
}
}
diff --git
a/circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java
b/circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java
new file mode 100644
index 00..3fb57b4a1a
--- /dev/null
+++
b/circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.scurrilous.circe.checksum;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.CompositeByteBuf;
+import io.netty.buffer.DuplicatedByteBuf;
+import java.util.Random;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.Assert;
+import org.junit.Test;
+
+@Slf4j
+public class Java9IntHashTest {
+
+private ByteBuf[] generateByteBuffers() {
+Random random = new Random();
+int hugeDataLen = 4096 * 3;
+byte[] hugeData = new byte[hugeDataLen];
+for (int i = 0; i < hugeDataLen; i ++) {
+hugeData[i] = (byte) (random.nextInt() % 127);
+}
+
+// b_total = b1 + b2 + b3;
+ByteBuf bTotal = ByteBufAllocator.DEFAULT.heapBuffer(6 + hugeDataLen);
+bTotal.writeBytes(new byte[]{1,2,3,4,5,6});
+bTotal.writeBytes(hugeData);
+ByteBuf b1 = ByteBufAllocator.DEFAULT.heapBuffer(3);
+b1.writeBytes(new byte[]{1,2,3});
+ByteBuf b2 = ByteBufAllocator.DEFAULT.heapBuffer(3);
+b2.writeBytes(new byte[]{4,5,6});
+ByteBuf b3 = ByteBufAllocator.DEFAULT.heapBuffer(hugeDataLen);
+b3.writeBytes(hugeData);
+
+return new ByteBuf[]{bTot
(bookkeeper) branch master updated: fix calculate checkSum when using Java9IntHash (#4140)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new be1749c3db fix calculate checkSum when using Java9IntHash (#4140)
be1749c3db is described below
commit be1749c3db4374b5a222b6f8312813da8433dc40
Author: fengyubiao
AuthorDate: Thu Dec 7 13:49:32 2023 +0800
fix calculate checkSum when using Java9IntHash (#4140)
* fix calculate checkSum when using Java9IntHash
* -
* improve performence
* add test
* edit test
* rename test
* fix license
* address comments
---
.../scurrilous/circe/checksum/Java9IntHash.java| 12 +-
.../circe/checksum/Java9IntHashTest.java | 121 +
2 files changed, 132 insertions(+), 1 deletion(-)
diff --git
a/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Java9IntHash.java
b/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Java9IntHash.java
index 622c66d9ee..31af153666 100644
---
a/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Java9IntHash.java
+++
b/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Java9IntHash.java
@@ -19,6 +19,7 @@
package com.scurrilous.circe.checksum;
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.CompositeByteBuf;
import io.netty.util.concurrent.FastThreadLocal;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
@@ -106,14 +107,23 @@ public class Java9IntHash implements IntHash {
} else if (buffer.hasArray()) {
int arrayOffset = buffer.arrayOffset() + offset;
negCrc = resume(negCrc, buffer.array(), arrayOffset, len);
+} else if (buffer instanceof CompositeByteBuf) {
+ CompositeByteBuf compositeByteBuf = (CompositeByteBuf) buffer;
+ int loopedCurrent = current;
+ for (int i = 0; i < compositeByteBuf.numComponents(); i ++) {
+ loopedCurrent = resume(loopedCurrent,
compositeByteBuf.component(i));
+ }
+ return loopedCurrent;
} else {
byte[] b = TL_BUFFER.get();
int toRead = len;
+int loopOffset = offset;
while (toRead > 0) {
int length = Math.min(toRead, b.length);
-buffer.slice(offset, len).readBytes(b, 0, length);
+buffer.slice(loopOffset, length).readBytes(b, 0, length);
negCrc = resume(negCrc, b, 0, length);
toRead -= length;
+loopOffset += length;
}
}
diff --git
a/circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java
b/circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java
new file mode 100644
index 00..3fb57b4a1a
--- /dev/null
+++
b/circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.scurrilous.circe.checksum;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.CompositeByteBuf;
+import io.netty.buffer.DuplicatedByteBuf;
+import java.util.Random;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.Assert;
+import org.junit.Test;
+
+@Slf4j
+public class Java9IntHashTest {
+
+private ByteBuf[] generateByteBuffers() {
+Random random = new Random();
+int hugeDataLen = 4096 * 3;
+byte[] hugeData = new byte[hugeDataLen];
+for (int i = 0; i < hugeDataLen; i ++) {
+hugeData[i] = (byte) (random.nextInt() % 127);
+}
+
+// b_total = b1 + b2 + b3;
+ByteBuf bTotal = ByteBufAllocator.DEFAULT.heapBuffer(6 + hugeDataLen);
+bTotal.writeBytes(new byte[]{1,2,3,4,5,6});
+bTotal.writeBytes(hugeData);
+ByteBuf b1 = ByteBufAllocator.DEFAULT.heapBuffer(3);
+b1.writeBytes(new byte[]{1,2,3});
+ByteBuf b2 = ByteBufAllocator.DEFAULT.heapBuffer(3);
+b2.writeBytes(new byte[]
[bookkeeper] branch master updated: Optimize bookie decommission check wait interval (#4070)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 09aec9ca52 Optimize bookie decommission check wait interval (#4070)
09aec9ca52 is described below
commit 09aec9ca520af7164fe5de378413e63d0c3889f9
Author: Hang Chen
AuthorDate: Sat Sep 9 00:48:23 2023 +0800
Optimize bookie decommission check wait interval (#4070)
* Optimise bookie decommission check wait interval
* fix a bug
---
.../main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java | 9 ++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java
index f9ac460bde..ed2db8a032 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java
@@ -1631,14 +1631,17 @@ public class BookKeeperAdmin implements AutoCloseable {
private void waitForLedgersToBeReplicated(Collection ledgers,
BookieId thisBookieAddress,
LedgerManager ledgerManager) throws InterruptedException,
TimeoutException {
-int maxSleepTimeInBetweenChecks = 10 * 60 * 1000; // 10 minutes
-int sleepTimePerLedger = 10 * 1000; // 10 secs
+int maxSleepTimeInBetweenChecks = 5 * 60 * 1000; // 5 minutes
+int sleepTimePerLedger = 3 * 1000; // 3 secs
Predicate validateBookieIsNotPartOfEnsemble = ledgerId ->
!areEntriesOfLedgerStoredInTheBookie(ledgerId,
thisBookieAddress, ledgerManager);
+ledgers.removeIf(validateBookieIsNotPartOfEnsemble);
+
while (!ledgers.isEmpty()) {
-LOG.info("Count of Ledgers which need to be rereplicated: {}",
ledgers.size());
int sleepTimeForThisCheck = (long) ledgers.size() *
sleepTimePerLedger > maxSleepTimeInBetweenChecks
? maxSleepTimeInBetweenChecks : ledgers.size() *
sleepTimePerLedger;
+LOG.info("Count of Ledgers which need to be rereplicated: {},
waiting {} seconds for next check",
+ledgers.size(), sleepTimeForThisCheck / 1000);
Thread.sleep(sleepTimeForThisCheck);
if (LOG.isDebugEnabled()) {
LOG.debug("Making sure following ledgers replication to be
completed: {}", ledgers);
[bookkeeper] branch master updated: Add explanatory messages to the asserts (#4055)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 1666d820a3 Add explanatory messages to the asserts (#4055)
1666d820a3 is described below
commit 1666d820a3d98ee6702e39c7cb0ebe51b1fdfd32
Author: Taher Ghaleb
AuthorDate: Fri Sep 1 01:20:10 2023 -0400
Add explanatory messages to the asserts (#4055)
* Add explanatory messages to the asserts
* Fix assert exploratory message parameters
---
.../clients/impl/container/TestStorageContainerInfo.java | 8
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git
a/stream/clients/java/base/src/test/java/org/apache/bookkeeper/clients/impl/container/TestStorageContainerInfo.java
b/stream/clients/java/base/src/test/java/org/apache/bookkeeper/clients/impl/container/TestStorageContainerInfo.java
index 7603b5f3a8..ce0dc3650b 100644
---
a/stream/clients/java/base/src/test/java/org/apache/bookkeeper/clients/impl/container/TestStorageContainerInfo.java
+++
b/stream/clients/java/base/src/test/java/org/apache/bookkeeper/clients/impl/container/TestStorageContainerInfo.java
@@ -42,10 +42,10 @@ public class TestStorageContainerInfo {
revision,
endpoint,
Lists.newArrayList(endpoint));
-assertEquals(groupId, sc.getGroupId());
-assertEquals(revision, sc.getRevision());
-assertEquals(endpoint, sc.getWriteEndpoint());
-assertEquals(Lists.newArrayList(endpoint), sc.getReadEndpoints());
+assertEquals("Group ID mismatch", groupId, sc.getGroupId());
+assertEquals("Revision mismatch", revision, sc.getRevision());
+assertEquals("Write Endpoint mismatch", endpoint,
sc.getWriteEndpoint());
+assertEquals("Read Endpoint mismatch", Lists.newArrayList(endpoint),
sc.getReadEndpoints());
}
}
[bookkeeper] branch master updated (50bf016ecd -> 7f64246ad3)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git from 50bf016ecd Fix journal without flush (#3979) add 7f64246ad3 Format otel module depedencies (#3970) No new revisions were added by this update. Summary of changes: pom.xml | 9 - .../bookkeeper-stats-providers/otel-metrics-provider/pom.xml | 12 stats/bookkeeper-stats-providers/pom.xml | 1 + 3 files changed, 9 insertions(+), 13 deletions(-)
[bookkeeper] branch master updated (c924cfeb50 -> 575a81c79e)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
from c924cfeb50 Avoid compaction to trigger extra flushes DbLedgerStorage
(#3959)
add 575a81c79e Added support for OpenTelemetry metrics backend (#3968)
No new revisions were added by this update.
Summary of changes:
bookkeeper-dist/all/pom.xml| 5 +
bookkeeper-dist/server/pom.xml | 5 +
.../src/main/resources/LICENSE-all.bin.txt | 31 +
.../src/main/resources/LICENSE-server.bin.txt | 31 +
bookkeeper-server/pom.xml | 6 +
pom.xml| 2 +
.../pom.xml| 71 +-
.../apache/bookkeeper/stats/otel/OtelCounter.java} | 55 +++-
.../bookkeeper/stats/otel/OtelMetricsProvider.java | 150 +
.../bookkeeper/stats/otel/OtelOpStatsLogger.java | 69 ++
.../bookkeeper/stats/otel/OtelStatsLogger.java}| 62 +
.../apache/bookkeeper/stats/otel/ScopeContext.java | 20 ++-
.../bookkeeper/stats/otel}/package-info.java | 4 +-
13 files changed, 398 insertions(+), 113 deletions(-)
copy stats/bookkeeper-stats-providers/{prometheus-metrics-provider =>
otel-metrics-provider}/pom.xml (50%)
copy
stats/bookkeeper-stats-providers/{prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/LongAdderCounter.java
=>
otel-metrics-provider/src/main/java/org/apache/bookkeeper/stats/otel/OtelCounter.java}
(50%)
create mode 100644
stats/bookkeeper-stats-providers/otel-metrics-provider/src/main/java/org/apache/bookkeeper/stats/otel/OtelMetricsProvider.java
create mode 100644
stats/bookkeeper-stats-providers/otel-metrics-provider/src/main/java/org/apache/bookkeeper/stats/otel/OtelOpStatsLogger.java
copy
stats/bookkeeper-stats-providers/{prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/PrometheusStatsLogger.java
=>
otel-metrics-provider/src/main/java/org/apache/bookkeeper/stats/otel/OtelStatsLogger.java}
(58%)
copy
bookkeeper-server/src/main/java/org/apache/bookkeeper/versioning/Versioned.java
=>
stats/bookkeeper-stats-providers/otel-metrics-provider/src/main/java/org/apache/bookkeeper/stats/otel/ScopeContext.java
(73%)
copy {bookkeeper-http/http-server/src/main/java/org/apache/bookkeeper/http =>
stats/bookkeeper-stats-providers/otel-metrics-provider/src/main/java/org/apache/bookkeeper/stats/otel}/package-info.java
(88%)
[bookkeeper] branch master updated: Avoid compaction to trigger extra flushes DbLedgerStorage (#3959)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new c924cfeb50 Avoid compaction to trigger extra flushes DbLedgerStorage
(#3959)
c924cfeb50 is described below
commit c924cfeb509c4e65e33a82ae88ad423017edb669
Author: Matteo Merli
AuthorDate: Mon May 22 12:21:46 2023 -0700
Avoid compaction to trigger extra flushes DbLedgerStorage (#3959)
* Avoid compaction to trigger extra flushes DbLedgerStorage
* Expanded comment
* Fixed test
* Fixed direct io test
---
.../ldb/SingleDirectoryDbLedgerStorage.java| 25 --
.../bookie/storage/ldb/DbLedgerStorageTest.java| 2 ++
2 files changed, 25 insertions(+), 2 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java
index f9553c5fe6..f54ed28e2b 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java
@@ -943,9 +943,30 @@ public class SingleDirectoryDbLedgerStorage implements
CompactableLedgerStorage
@Override
public void updateEntriesLocations(Iterable locations)
throws IOException {
-// Trigger a flush to have all the entries being compacted in the db
storage
-flush();
+// Before updating the DB with the new location for the compacted
entries, we need to
+// make sure that there is no ongoing flush() operation.
+// If there were a flush, we could have the following situation, which
is highly
+// unlikely though possible:
+// 1. Flush operation has written the write-cache content into
entry-log files
+// 2. The DB location index is not yet updated
+// 3. Compaction is triggered and starts compacting some of the recent
files
+// 4. Compaction will write the "new location" into the DB
+// 5. The pending flush() will overwrite the DB with the "old
location", pointing
+//to a file that no longer exists
+//
+// To avoid this race condition, we need that all the entries that are
potentially
+// included in the compaction round to have all the indexes already
flushed into
+// the DB.
+// The easiest lightweight way to achieve this is to wait for any
pending
+// flush operation to be completed before updating the index with the
compacted
+// entries, by blocking on the flushMutex.
+flushMutex.lock();
+flushMutex.unlock();
+// We don't need to keep the flush mutex locked here while updating
the DB.
+// It's fine to have a concurrent flush operation at this point,
because we
+// know that none of the entries being flushed was included in the
compaction
+// round that we are dealing with.
entryLocationIndex.updateLocations(locations);
}
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java
index 4efdf06edb..2a7e8e2869 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java
@@ -225,6 +225,7 @@ public class DbLedgerStorageTest {
entry3.writeBytes("entry-3".getBytes());
storage.addEntry(entry3);
+
// Simulate bookie compaction
SingleDirectoryDbLedgerStorage singleDirStorage = ((DbLedgerStorage)
storage).getLedgerStorageList().get(0);
EntryLogger entryLogger = singleDirStorage.getEntryLogger();
@@ -236,6 +237,7 @@ public class DbLedgerStorageTest {
long location = entryLogger.addEntry(4L, newEntry3);
newEntry3.resetReaderIndex();
+storage.flush();
List locations = Lists.newArrayList(new
EntryLocation(4, 3, location));
singleDirStorage.updateEntriesLocations(locations);
[bookkeeper] branch branch-4.16 updated: Bump branch-4.16 development version to 4.16.1-SNAPSHOT (#3920)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch branch-4.16 in repository https://gitbox.apache.org/repos/asf/bookkeeper.git The following commit(s) were added to refs/heads/branch-4.16 by this push: new d1b3dbfa5c Bump branch-4.16 development version to 4.16.1-SNAPSHOT (#3920) d1b3dbfa5c is described below commit d1b3dbfa5c3848f7cb5cb62142ab7d9445b86f65 Author: Hang Chen AuthorDate: Sat Apr 15 01:06:51 2023 +0800 Bump branch-4.16 development version to 4.16.1-SNAPSHOT (#3920) --- bookkeeper-benchmark/pom.xml | 2 +- bookkeeper-common-allocator/pom.xml | 2 +- bookkeeper-common/pom.xml| 2 +- bookkeeper-dist/all/pom.xml | 2 +- bookkeeper-dist/bkctl/pom.xml| 2 +- bookkeeper-dist/pom.xml | 2 +- bookkeeper-dist/server/pom.xml | 2 +- bookkeeper-http/http-server/pom.xml | 2 +- bookkeeper-http/pom.xml | 2 +- bookkeeper-http/servlet-http-server/pom.xml | 2 +- bookkeeper-http/vertx-http-server/pom.xml| 2 +- bookkeeper-proto/pom.xml | 2 +- bookkeeper-server/pom.xml| 2 +- bookkeeper-slogger/api/pom.xml | 2 +- bookkeeper-slogger/pom.xml | 2 +- bookkeeper-slogger/slf4j/pom.xml | 2 +- buildtools/pom.xml | 4 ++-- circe-checksum/pom.xml | 2 +- cpu-affinity/pom.xml | 2 +- metadata-drivers/etcd/pom.xml| 2 +- metadata-drivers/pom.xml | 2 +- microbenchmarks/pom.xml | 2 +- native-io/pom.xml| 2 +- pom.xml | 2 +- shaded/bookkeeper-server-shaded/pom.xml | 2 +- shaded/bookkeeper-server-tests-shaded/pom.xml| 2 +- shaded/distributedlog-core-shaded/pom.xml| 2 +- shaded/pom.xml | 2 +- stats/bookkeeper-stats-api/pom.xml | 2 +- stats/bookkeeper-stats-providers/codahale-metrics-provider/pom.xml | 2 +- stats/bookkeeper-stats-providers/pom.xml | 2 +- stats/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml | 2 +- stats/pom.xml| 2 +- stats/utils/pom.xml | 2 +- stream/api/pom.xml | 2 +- stream/bk-grpc-name-resolver/pom.xml | 2 +- stream/clients/java/all/pom.xml | 2 +- stream/clients/java/base/pom.xml | 2 +- stream/clients/java/kv/pom.xml | 2 +- stream/clients/java/pom.xml | 2 +- stream/clients/pom.xml | 2 +- stream/common/pom.xml| 2 +- stream/distributedlog/common/pom.xml | 2 +- stream/distributedlog/core/pom.xml | 2 +- stream/distributedlog/io/dlfs/pom.xml| 2 +- stream/distributedlog/io/pom.xml | 2 +- stream/distributedlog/pom.xml| 2 +- stream/distributedlog/protocol/pom.xml | 2 +- stream/pom.xml | 2 +- stream/proto/pom.xml | 2 +- stream/server/pom.xml| 2 +- stream/statelib/pom.xml | 2 +- stream/storage/api/pom.xml | 2 +- stream/storage/impl/pom.xml | 2 +- stream/storage/pom.xml | 2 +- stream/tests-common/pom.xml | 2 +- tests/backward-compat/bc-non-fips/pom.xml| 2 +- tests/backward-compat/current-server-old-clients
[bookkeeper] branch master updated (f5455f0158 -> 912896deb2)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git from f5455f0158 [feature] [server] add dbStorage_readAheadCacheBatchBytesSize properties when read ahead entries (#3895) add 912896deb2 Cleanup CbThreadFactory (#3907) No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/bookkeeper/bookie/Journal.java | 13 - 1 file changed, 13 deletions(-)
[bookkeeper] branch master updated: fix BookieWriteLedgerTest parameterized failed (#3890)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new ee1424456b fix BookieWriteLedgerTest parameterized failed (#3890)
ee1424456b is described below
commit ee1424456bd8b9b2d70157ee4f2d518605482075
Author: Hang Chen
AuthorDate: Tue Mar 28 00:05:03 2023 +0800
fix BookieWriteLedgerTest parameterized failed (#3890)
---
.../java/org/apache/bookkeeper/client/BookieWriteLedgerTest.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieWriteLedgerTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieWriteLedgerTest.java
index 2a57f9952f..f05f8cda27 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieWriteLedgerTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieWriteLedgerTest.java
@@ -123,6 +123,9 @@ public class BookieWriteLedgerTest extends
@Override
@Before
public void setUp() throws Exception {
+baseConf.setJournalWriteData(writeJournal);
+baseClientConf.setUseV2WireProtocol(useV2);
+
super.setUp();
rng = new Random(0); // Initialize the Random
// Number Generator
@@ -136,14 +139,12 @@ public class BookieWriteLedgerTest extends
String ledgerManagerFactory =
"org.apache.bookkeeper.meta.HierarchicalLedgerManagerFactory";
// set ledger manager
baseConf.setLedgerManagerFactoryClassName(ledgerManagerFactory);
-baseConf.setJournalWriteData(writeJournal);
/*
* 'testLedgerCreateAdvWithLedgerIdInLoop2' testcase relies on
skipListSizeLimit,
* so setting it to some small value for making that testcase lite.
*/
baseConf.setSkipListSizeLimit(4 * 1024 * 1024);
baseClientConf.setLedgerManagerFactoryClassName(ledgerManagerFactory);
-baseClientConf.setUseV2WireProtocol(useV2);
}
/**
[bookkeeper] branch master updated: Use BatchedArrayBlockingQueue in Journal (#3843)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 97e3bb1962 Use BatchedArrayBlockingQueue in Journal (#3843)
97e3bb1962 is described below
commit 97e3bb1962ad253c683494a7f5c362a4f974511f
Author: Matteo Merli
AuthorDate: Tue Mar 21 08:03:28 2023 -0700
Use BatchedArrayBlockingQueue in Journal (#3843)
* Use BatchedArrayBlockingQueue in Journal
# Conflicts:
#
bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
* remove unnecessary imports.
* Address the comments.
* fix the ci problems.
-
Co-authored-by: horizonzy
---
.../java/org/apache/bookkeeper/bookie/Journal.java | 322 ++---
.../bookkeeper/bookie/BookieJournalForceTest.java | 21 +-
.../bookie/BookieJournalPageCacheFlushTest.java| 24 +-
3 files changed, 174 insertions(+), 193 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
index 50eb605414..4fd2c2f28d 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
@@ -38,16 +38,15 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
-import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import
org.apache.bookkeeper.bookie.LedgerDirsManager.NoWritableLedgerDirException;
import org.apache.bookkeeper.bookie.stats.JournalStats;
+import org.apache.bookkeeper.common.collections.BatchedArrayBlockingQueue;
+import org.apache.bookkeeper.common.collections.BatchedBlockingQueue;
import org.apache.bookkeeper.common.collections.BlockingMpscQueue;
import org.apache.bookkeeper.common.collections.RecyclableArrayList;
import org.apache.bookkeeper.common.util.MemoryLimitController;
@@ -489,36 +488,32 @@ public class Journal extends BookieCriticalThread
implements CheckpointSource {
}
}
-final List localRequests = new ArrayList<>();
final ObjectHashSet writeHandlers = new
ObjectHashSet<>();
+final ForceWriteRequest[] localRequests = new
ForceWriteRequest[conf.getJournalQueueSize()];
while (running) {
try {
-int numReqInLastForceWrite = 0;
+int numEntriesInLastForceWrite = 0;
-int requestsCount =
forceWriteRequests.drainTo(localRequests);
-if (requestsCount == 0) {
-ForceWriteRequest fwr = forceWriteRequests.take();
-localRequests.add(fwr);
-requestsCount = 1;
-}
+int requestsCount =
forceWriteRequests.takeAll(localRequests);
journalStats.getForceWriteQueueSize().addCount(-requestsCount);
// Sync and mark the journal up to the position of the
last entry in the batch
-ForceWriteRequest lastRequest =
localRequests.get(requestsCount - 1);
+ForceWriteRequest lastRequest =
localRequests[requestsCount - 1];
syncJournal(lastRequest);
// All the requests in the batch are now fully-synced. We
can trigger sending the
// responses
for (int i = 0; i < requestsCount; i++) {
-ForceWriteRequest req = localRequests.get(i);
-numReqInLastForceWrite += req.process(writeHandlers);
+ForceWriteRequest req = localRequests[i];
+numEntriesInLastForceWrite +=
req.process(writeHandlers);
+localRequests[i] = null;
req.recycle();
}
journalStats.getForceWriteGroupingCountStats()
-.registerSuccessfulValue(numReqInLastForceWrite);
+
.registerSuccessfulValue(numEntriesInLastForceWrite);
writeHandlers.forEach(
(ObjectProcedure)
BookieRequestHandler::flushPendingResponse);
@@ -529,16 +524,8 @@ public class Journal extends BookieCriticalThread
implements CheckpointSource {
} catch (InterruptedException e) {
Thread.currentThr
[bookkeeper] branch master updated: Add log for entry log file delete. (#3872)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new b3aca10dea Add log for entry log file delete. (#3872)
b3aca10dea is described below
commit b3aca10deacf71cdca49c9939d54e8efbdaa58ef
Author: Yan Zhao
AuthorDate: Tue Mar 21 01:59:18 2023 +0800
Add log for entry log file delete. (#3872)
* Add log for entry log file delete.
* add log info.
* Address the comment.
* Address the comment.
* revert the code.
---
.../main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java | 1 +
1 file changed, 1 insertion(+)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
index c06df7c228..5de2292263 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
@@ -751,6 +751,7 @@ public class GarbageCollectorThread implements Runnable {
EntryLogMetadata entryLogMeta =
entryLogger.getEntryLogMetadata(entryLogId, throttler);
removeIfLedgerNotExists(entryLogMeta);
if (entryLogMeta.isEmpty()) {
+LOG.info("Entry log file {} is empty, delete it from
disk.", Long.toHexString(entryLogId));
entryLogger.removeEntryLog(entryLogId);
// remove it from entrylogmetadata-map if it is present in
// the map
[bookkeeper] branch master updated: Single buffer for small add requests (#3783)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 234b817cdb Single buffer for small add requests (#3783)
234b817cdb is described below
commit 234b817cdb4e054887ffd5e42eaed25dc02daf63
Author: Matteo Merli
AuthorDate: Mon Mar 20 10:38:11 2023 -0700
Single buffer for small add requests (#3783)
* Single buffer for small add requests
* Fixed checkstyle
* Fixed treating of ComposityByteBuf
* Fixed merge issues
* Fixed merge issues
* WIP
* Fixed test and removed dead code
* Removed unused import
* Fixed BookieJournalTest
* removed unused import
* fix the checkstyle
* fix failed test
* fix failed test
-
Co-authored-by: chenhang
---
.../client/LedgerFragmentReplicator.java | 17 +++--
.../org/apache/bookkeeper/client/PendingAddOp.java | 7 +-
.../org/apache/bookkeeper/proto/AuthHandler.java | 6 +-
.../org/apache/bookkeeper/proto/BookieClient.java | 3 +-
.../apache/bookkeeper/proto/BookieClientImpl.java | 7 +-
.../bookkeeper/proto/BookieProtoEncoding.java | 22 ++-
.../apache/bookkeeper/proto/BookieProtocol.java| 53 ---
.../bookkeeper/proto/PerChannelBookieClient.java | 31 +
.../bookkeeper/proto/checksum/DigestManager.java | 75 --
.../org/apache/bookkeeper/client/ClientUtil.java | 6 +-
.../bookkeeper/client/LedgerHandleAdapter.java | 9 +--
.../bookkeeper/client/MockBookKeeperTestCase.java | 8 ++-
.../client/ParallelLedgerRecoveryTest.java | 7 +-
.../client/ReadLastConfirmedAndEntryOpTest.java| 14 ++--
.../bookkeeper/client/TestPendingReadLacOp.java| 11 +++-
.../proto/BookieBackpressureForV2Test.java | 4 ++
.../apache/bookkeeper/proto/MockBookieClient.java | 33 --
.../org/apache/bookkeeper/proto/MockBookies.java | 4 +-
.../apache/bookkeeper/proto/ProtocolBenchmark.java | 16 -
.../proto/checksum/DigestManagerBenchmark.java | 5 +-
20 files changed, 187 insertions(+), 151 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerFragmentReplicator.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerFragmentReplicator.java
index 877a3ac300..6b439b0960 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerFragmentReplicator.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerFragmentReplicator.java
@@ -29,7 +29,9 @@ import static
org.apache.bookkeeper.replication.ReplicationStats.REPLICATION_WOR
import static
org.apache.bookkeeper.replication.ReplicationStats.WRITE_DATA_LATENCY;
import com.google.common.util.concurrent.RateLimiter;
+import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
+import io.netty.util.ReferenceCounted;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.Iterator;
@@ -403,17 +405,24 @@ public class LedgerFragmentReplicator {
numEntriesRead.inc();
numBytesRead.registerSuccessfulValue(dataLength);
-ByteBufList toSend = lh.getDigestManager()
+ReferenceCounted toSend = lh.getDigestManager()
.computeDigestAndPackageForSending(entryId,
lh.getLastAddConfirmed(), entry.getLength(),
-Unpooled.wrappedBuffer(data, 0, data.length));
+Unpooled.wrappedBuffer(data, 0, data.length),
+lh.getLedgerKey(),
+0
+);
if (replicationThrottle != null) {
-updateAverageEntrySize(toSend.readableBytes());
+if (toSend instanceof ByteBuf) {
+updateAverageEntrySize(((ByteBuf)
toSend).readableBytes());
+} else if (toSend instanceof ByteBufList) {
+updateAverageEntrySize(((ByteBufList)
toSend).readableBytes());
+}
}
for (BookieId newBookie : newBookies) {
long startWriteEntryTime = MathUtils.nowInNano();
bkc.getBookieClient().addEntry(newBookie, lh.getId(),
-lh.getLedgerKey(), entryId,
ByteBufList.clone(toSend),
+lh.getLedgerKey(), entryId, toSend,
multiWriteCallback, dataLength,
BookieProtocol.FLAG_RECOVERY_ADD,
false, WriteFlag.NONE);
writeDataLatency.registerSuccessfulEvent(
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper
[bookkeeper] branch master updated: Fix compaction threshold default value precision problem. (#3871)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new e19cb9d822 Fix compaction threshold default value precision problem.
(#3871)
e19cb9d822 is described below
commit e19cb9d822c316a13cbf015fd4a627fce44172be
Author: Yan Zhao
AuthorDate: Tue Mar 21 00:42:57 2023 +0800
Fix compaction threshold default value precision problem. (#3871)
* Fix compaction threshold precision problem.
* Fix compaction threshold precision problem.
---
.../java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java | 8
.../main/java/org/apache/bookkeeper/conf/ServerConfiguration.java | 4 ++--
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
index bdffd689a5..c06df7c228 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
@@ -218,7 +218,7 @@ public class GarbageCollectorThread implements Runnable {
this.throttler = new AbstractLogCompactor.Throttler(conf);
if (minorCompactionInterval > 0 && minorCompactionThreshold > 0) {
-if (minorCompactionThreshold > 1.0f) {
+if (minorCompactionThreshold > 1.0d) {
throw new IOException("Invalid minor compaction threshold "
+ minorCompactionThreshold);
}
@@ -230,16 +230,16 @@ public class GarbageCollectorThread implements Runnable {
}
if (isForceAllowCompaction) {
-if (minorCompactionThreshold > 0 && minorCompactionThreshold <
1.0f) {
+if (minorCompactionThreshold > 0 && minorCompactionThreshold <
1.0d) {
isForceMinorCompactionAllow = true;
}
-if (majorCompactionThreshold > 0 && majorCompactionThreshold <
1.0f) {
+if (majorCompactionThreshold > 0 && majorCompactionThreshold <
1.0d) {
isForceMajorCompactionAllow = true;
}
}
if (majorCompactionInterval > 0 && majorCompactionThreshold > 0) {
-if (majorCompactionThreshold > 1.0f) {
+if (majorCompactionThreshold > 1.0d) {
throw new IOException("Invalid major compaction threshold "
+ majorCompactionThreshold);
}
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
index 4db536ab49..cf74f6af01 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
@@ -1676,7 +1676,7 @@ public class ServerConfiguration extends
AbstractConfiguration
[bookkeeper] branch master updated (c3e5bfe431 -> dda42a3a33)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
from c3e5bfe431 [Bugfix] make metadataDriver initialization more robust
(#3873)
add dda42a3a33 Enable CI for the streamstorage python client (#3875)
No new revisions were added by this update.
Summary of changes:
.github/workflows/bk-streamstorage-python.yml | 85 ++
stream/clients/pom.xml | 18 +
stream/clients/python/.flake8 |1 +
stream/clients/python/.gitignore |2 +
stream/clients/python/README.rst | 19 +
.../python/bookkeeper/common/protobuf_helpers.py | 22 +-
.../clients/python/bookkeeper/proto/cluster_pb2.py | 207 +--
.../clients/python/bookkeeper/proto/common_pb2.py | 70 +-
stream/clients/python/bookkeeper/proto/kv_pb2.py | 192 +--
.../clients/python/bookkeeper/proto/kv_rpc_pb2.py | 1302 ++
.../python/bookkeeper/proto/kv_store_pb2.py| 312 +
.../clients/python/bookkeeper/proto/storage_pb2.py | 1328 +++
.../clients/python/bookkeeper/proto/stream_pb2.py | 1401 +++-
.../clients/python/docker}/Dockerfile | 33 +-
.../test.sh => docker/build-local-image.sh}| 20 +-
.../python/scripts/docker_integration_tests.sh |7 +-
.../python/scripts/run_integration_tests.sh|3 +
stream/clients/python/scripts/test.sh |1 -
stream/clients/python/setup.py |2 +-
.../tests/integration/bookkeeper/kv/test_client.py |1 -
20 files changed, 759 insertions(+), 4267 deletions(-)
create mode 100644 .github/workflows/bk-streamstorage-python.yml
copy {docker => stream/clients/python/docker}/Dockerfile (58%)
copy stream/clients/python/{scripts/test.sh => docker/build-local-image.sh}
(62%)
[bookkeeper] branch master updated (caea6c9451 -> ebf3108d54)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git from caea6c9451 [Flaky] Fix flaky test in testRaceGuavaEvictAndReleaseBeforeRetain (#3857) add ebf3108d54 Fix NPE in BenchThroughputLatency (#3859) No new revisions were added by this update. Summary of changes: .../java/org/apache/bookkeeper/benchmark/BenchThroughputLatency.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
[bookkeeper] branch master updated (1a8d84251b -> caea6c9451)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git from 1a8d84251b Bump checkstyle-plugin from 3.1.2 to 3.2.1 (#3850) add caea6c9451 [Flaky] Fix flaky test in testRaceGuavaEvictAndReleaseBeforeRetain (#3857) No new revisions were added by this update. Summary of changes: .../java/org/apache/bookkeeper/bookie/FileInfoBackingCacheTest.java | 6 -- 1 file changed, 4 insertions(+), 2 deletions(-)
[bookkeeper] branch master updated: add V2 protocal and warmupMessages support for benchMark (#3856)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 3a488150e9 add V2 protocal and warmupMessages support for benchMark
(#3856)
3a488150e9 is described below
commit 3a488150e939b2980bbe2169a6c83f4f570d5e6a
Author: Hang Chen
AuthorDate: Sat Mar 11 00:30:04 2023 +0800
add V2 protocal and warmupMessages support for benchMark (#3856)
---
.../apache/bookkeeper/benchmark/BenchThroughputLatency.java | 13 ++---
1 file changed, 10 insertions(+), 3 deletions(-)
diff --git
a/bookkeeper-benchmark/src/main/java/org/apache/bookkeeper/benchmark/BenchThroughputLatency.java
b/bookkeeper-benchmark/src/main/java/org/apache/bookkeeper/benchmark/BenchThroughputLatency.java
index 84573eec7c..83ebe4229c 100644
---
a/bookkeeper-benchmark/src/main/java/org/apache/bookkeeper/benchmark/BenchThroughputLatency.java
+++
b/bookkeeper-benchmark/src/main/java/org/apache/bookkeeper/benchmark/BenchThroughputLatency.java
@@ -257,6 +257,8 @@ public class BenchThroughputLatency implements AddCallback,
Runnable {
options.addOption("skipwarmup", false, "Skip warm up, default false");
options.addOption("sendlimit", true, "Max number of entries to send.
Default 2000");
options.addOption("latencyFile", true, "File to dump latencies.
Default is latencyDump.dat");
+options.addOption("useV2", false, "Whether use V2 protocol to send
requests to the bookie server");
+options.addOption("warmupMessages", true, "Number of messages to warm
up. Default 1");
options.addOption("help", false, "This message");
CommandLineParser parser = new PosixParser();
@@ -281,6 +283,7 @@ public class BenchThroughputLatency implements AddCallback,
Runnable {
}
int throttle = Integer.parseInt(cmd.getOptionValue("throttle",
"1"));
int sendLimit = Integer.parseInt(cmd.getOptionValue("sendlimit",
"2000"));
+int warmupMessages =
Integer.parseInt(cmd.getOptionValue("warmupMessages", "1"));
final int sockTimeout =
Integer.parseInt(cmd.getOptionValue("sockettimeout", "5"));
@@ -321,11 +324,15 @@ public class BenchThroughputLatency implements
AddCallback, Runnable {
ClientConfiguration conf = new ClientConfiguration();
conf.setThrottleValue(throttle).setReadTimeout(sockTimeout).setZkServers(servers);
+if (cmd.hasOption("useV2")) {
+conf.setUseV2WireProtocol(true);
+}
+
if (!cmd.hasOption("skipwarmup")) {
long throughput;
LOG.info("Starting warmup");
-throughput = warmUp(data, ledgers, ensemble, quorum, passwd, conf);
+throughput = warmUp(data, ledgers, ensemble, quorum, passwd,
warmupMessages, conf);
LOG.info("Warmup tp: " + throughput);
LOG.info("Warmup phase finished");
}
@@ -438,7 +445,7 @@ public class BenchThroughputLatency implements AddCallback,
Runnable {
* TODO: update benchmark to use metadata service uri {@link
https://github.com/apache/bookkeeper/issues/1331}
*/
private static long warmUp(byte[] data, int ledgers, int ensemble, int
qSize,
- byte[] passwd, ClientConfiguration conf)
+ byte[] passwd, int warmupMessages,
ClientConfiguration conf)
throws KeeperException, IOException, InterruptedException,
BKException {
final CountDownLatch connectLatch = new CountDownLatch(1);
final int bookies;
@@ -465,7 +472,7 @@ public class BenchThroughputLatency implements AddCallback,
Runnable {
}
BenchThroughputLatency warmup = new BenchThroughputLatency(bookies,
bookies, bookies, passwd,
- ledgers,
1, conf);
+ ledgers,
warmupMessages, conf);
warmup.setEntryData(data);
Thread thread = new Thread(warmup);
thread.start();
[bookkeeper] branch master updated (3a488150e9 -> cd43c91add)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git from 3a488150e9 add V2 protocal and warmupMessages support for benchMark (#3856) add cd43c91add disable trimStackTrack for code-coverage profile (#3854) No new revisions were added by this update. Summary of changes: pom.xml | 1 + 1 file changed, 1 insertion(+)
[bookkeeper] branch master updated (caddb6ee5b -> 0a21dd5387)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git from caddb6ee5b Correct the running job name for the test group (#3851) add 0a21dd5387 add timeout for two flaky timeout tests (#3855) No new revisions were added by this update. Summary of changes: .../bookie/FileInfoBackingCacheTest.java | 10 - .../bookkeeper/client/HandleFailuresTest.java | 24 +++--- 2 files changed, 17 insertions(+), 17 deletions(-)
[bookkeeper] branch master updated: Make read entry request recyclable (#3842)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 09365dfdd8 Make read entry request recyclable (#3842)
09365dfdd8 is described below
commit 09365dfdd8aa20c603d7a134d7333b907ce50d24
Author: Penghui Li
AuthorDate: Wed Mar 8 01:50:00 2023 +0800
Make read entry request recyclable (#3842)
* Make read entry request recyclable
* Move recycle to finally block
* Fix test and comments
* Fix test
---
.../bookkeeper/proto/BookieProtoEncoding.java | 6 ++--
.../apache/bookkeeper/proto/BookieProtocol.java| 35 --
.../bookkeeper/proto/PerChannelBookieClient.java | 5 ++--
.../bookkeeper/proto/ReadEntryProcessor.java | 1 +
.../bookkeeper/proto/ReadEntryProcessorTest.java | 6 ++--
.../bookkeeper/proto/TestBackwardCompatCMS42.java | 5 ++--
6 files changed, 44 insertions(+), 14 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java
index 5f886dbe0d..8cf40d2b8e 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java
@@ -142,7 +142,7 @@ public class BookieProtoEncoding {
if (r.hasMasterKey()) {
buf.writeBytes(r.getMasterKey(), 0,
BookieProtocol.MASTER_KEY_LENGTH);
}
-
+r.recycle();
return buf;
} else if (r instanceof BookieProtocol.AuthRequest) {
BookkeeperProtocol.AuthMessage am =
((BookieProtocol.AuthRequest) r).getAuthMessage();
@@ -193,9 +193,9 @@ public class BookieProtoEncoding {
if ((flags & BookieProtocol.FLAG_DO_FENCING) ==
BookieProtocol.FLAG_DO_FENCING
&& version >= 2) {
byte[] masterKey = readMasterKey(packet);
-return new BookieProtocol.ReadRequest(version, ledgerId,
entryId, flags, masterKey);
+return BookieProtocol.ReadRequest.create(version,
ledgerId, entryId, flags, masterKey);
} else {
-return new BookieProtocol.ReadRequest(version, ledgerId,
entryId, flags, null);
+return BookieProtocol.ReadRequest.create(version,
ledgerId, entryId, flags, null);
}
case BookieProtocol.AUTH:
BookkeeperProtocol.AuthMessage.Builder builder =
BookkeeperProtocol.AuthMessage.newBuilder();
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtocol.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtocol.java
index 89654449aa..86c3ed5469 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtocol.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtocol.java
@@ -362,14 +362,43 @@ public interface BookieProtocol {
* A Request that reads data.
*/
class ReadRequest extends Request {
-ReadRequest(byte protocolVersion, long ledgerId, long entryId,
-short flags, byte[] masterKey) {
-init(protocolVersion, READENTRY, ledgerId, entryId, flags,
masterKey);
+
+static ReadRequest create(byte protocolVersion, long ledgerId, long
entryId,
+ short flags, byte[] masterKey) {
+ReadRequest read = RECYCLER.get();
+read.protocolVersion = protocolVersion;
+read.opCode = READENTRY;
+read.ledgerId = ledgerId;
+read.entryId = entryId;
+read.flags = flags;
+read.masterKey = masterKey;
+return read;
}
boolean isFencing() {
return (flags & FLAG_DO_FENCING) == FLAG_DO_FENCING;
}
+
+private final Handle recyclerHandle;
+
+private ReadRequest(Handle recyclerHandle) {
+this.recyclerHandle = recyclerHandle;
+}
+
+private static final Recycler RECYCLER = new
Recycler() {
+@Override
+protected ReadRequest newObject(Handle handle) {
+return new ReadRequest(handle);
+}
+};
+
+@Override
+public void recycle() {
+ledgerId = -1;
+entryId = -1;
+masterKey = null;
+recyclerHandle.recycle(this);
+}
}
/**
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java
index 47d5303bb3..4d08faad20 100644
---
a/bookkee
[bookkeeper] branch master updated (09365dfdd8 -> f9313195f0)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git from 09365dfdd8 Make read entry request recyclable (#3842) add f9313195f0 Avoid unnecessary force write. (#3847) No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/bookkeeper/bookie/Journal.java | 12 ++-- 1 file changed, 10 insertions(+), 2 deletions(-)
[bookkeeper] branch master updated: Fix memory leak issue of reading small entries (#3844)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 5703132316 Fix memory leak issue of reading small entries (#3844)
5703132316 is described below
commit 57031323166bd55004aaf50e3f46611dbb7eacf7
Author: Penghui Li
AuthorDate: Wed Mar 8 00:07:06 2023 +0800
Fix memory leak issue of reading small entries (#3844)
---
.../src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java | 1 +
1 file changed, 1 insertion(+)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java
index edbffa5f43..5f886dbe0d 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java
@@ -271,6 +271,7 @@ public class BookieProtoEncoding {
if (isSmallEntry) {
buf.writeBytes(rr.getData());
+rr.release();
return buf;
} else {
return ByteBufList.get(buf, rr.getData());
[bookkeeper] branch master updated: Added BatchedArrayBlockingQueue (#3838)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 73c5a0e011 Added BatchedArrayBlockingQueue (#3838)
73c5a0e011 is described below
commit 73c5a0e0111754690e5cd74fb030ee64a2f829d9
Author: Matteo Merli
AuthorDate: Mon Mar 6 06:35:11 2023 -0800
Added BatchedArrayBlockingQueue (#3838)
---
.../collections/BatchedArrayBlockingQueue.java | 409 +
.../common/collections/BatchedBlockingQueue.java | 55 +++
.../common/collections/BlockingMpscQueue.java | 41 ++-
.../collections/BatchedArrayBlockingQueueTest.java | 312
.../bookkeeper/common/MpScQueueBenchmark.java | 134 +++
5 files changed, 950 insertions(+), 1 deletion(-)
diff --git
a/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/collections/BatchedArrayBlockingQueue.java
b/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/collections/BatchedArrayBlockingQueue.java
new file mode 100644
index 00..646391b49e
--- /dev/null
+++
b/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/collections/BatchedArrayBlockingQueue.java
@@ -0,0 +1,409 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.bookkeeper.common.collections;
+
+import java.util.AbstractQueue;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * This implements a {@link BlockingQueue} backed by an array with fixed
capacity.
+ *
+ * This queue only allows 1 consumer thread to dequeue items and multiple
producer threads.
+ */
+public class BatchedArrayBlockingQueue
+extends AbstractQueue
+implements BlockingQueue, BatchedBlockingQueue {
+
+private final ReentrantLock lock = new ReentrantLock();
+
+private final Condition notEmpty = lock.newCondition();
+private final Condition notFull = lock.newCondition();
+
+private final int capacity;
+private final T[] data;
+
+private int size;
+
+private int consumerIdx;
+private int producerIdx;
+
+@SuppressWarnings("unchecked")
+public BatchedArrayBlockingQueue(int capacity) {
+this.capacity = capacity;
+this.data = (T[]) new Object[this.capacity];
+}
+
+private T dequeueOne() {
+T item = data[consumerIdx];
+data[consumerIdx] = null;
+if (++consumerIdx == capacity) {
+consumerIdx = 0;
+}
+
+if (size-- == capacity) {
+notFull.signalAll();
+}
+
+return item;
+}
+
+private void enqueueOne(T item) {
+data[producerIdx] = item;
+if (++producerIdx == capacity) {
+producerIdx = 0;
+}
+
+if (size++ == 0) {
+notEmpty.signalAll();
+}
+}
+
+@Override
+public T poll() {
+lock.lock();
+
+try {
+if (size == 0) {
+return null;
+}
+
+return dequeueOne();
+} finally {
+lock.unlock();
+}
+}
+
+@Override
+public T peek() {
+lock.lock();
+
+try {
+if (size == 0) {
+return null;
+}
+
+return data[consumerIdx];
+} finally {
+lock.unlock();
+}
+}
+
+@Override
+public boolean offer(T e) {
+lock.lock();
+
+try {
+if (size == capacity) {
+return false;
+}
+
+enqueueOne(e);
+
+return true;
+} finally {
+lock.unlock();
+}
+}
+
+@Override
+public void put(T e) throws InterruptedException {
+lock.lockInterruptibly();
+
+try {
+while (size == capacity) {
+notFull.await();
+}
+
+enq
[bookkeeper] branch master updated: Pass BookieRequestHandler instead of Channel to the request processors (#3835)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new dfde3d6836 Pass BookieRequestHandler instead of Channel to the request
processors (#3835)
dfde3d6836 is described below
commit dfde3d6836094cc39d8a3603cb268ca0f633350b
Author: Matteo Merli
AuthorDate: Sun Mar 5 09:43:37 2023 -0800
Pass BookieRequestHandler instead of Channel to the request processors
(#3835)
* Pass BookieRequestHandler instead of Channel to the request processors
* Fixed checkstyle
* Fixed ForceLedgerProcessorV3Test
* Fixed TestBookieRequestProcessor
* Fixed line length
---
.../bookkeeper/processor/RequestProcessor.java | 4 +-
.../bookkeeper/proto/BookieRequestHandler.java | 11 ++-
.../bookkeeper/proto/BookieRequestProcessor.java | 94 --
.../bookkeeper/proto/ForceLedgerProcessorV3.java | 7 +-
.../bookkeeper/proto/GetBookieInfoProcessorV3.java | 5 +-
.../proto/GetListOfEntriesOfLedgerProcessorV3.java | 5 +-
.../proto/LongPollReadEntryProcessorV3.java| 5 +-
.../bookkeeper/proto/PacketProcessorBase.java | 28 ---
.../bookkeeper/proto/PacketProcessorBaseV3.java| 7 +-
.../bookkeeper/proto/ReadEntryProcessor.java | 14 ++--
.../bookkeeper/proto/ReadEntryProcessorV3.java | 11 +--
.../bookkeeper/proto/ReadLacProcessorV3.java | 5 +-
.../bookkeeper/proto/WriteEntryProcessor.java | 12 +--
.../bookkeeper/proto/WriteEntryProcessorV3.java| 13 +--
.../bookkeeper/proto/WriteLacProcessorV3.java | 8 +-
.../proto/ForceLedgerProcessorV3Test.java | 19 -
.../proto/GetBookieInfoProcessorV3Test.java| 12 ++-
.../proto/LongPollReadEntryProcessorV3Test.java| 9 ++-
.../bookkeeper/proto/ReadEntryProcessorTest.java | 15 +++-
.../proto/TestBookieRequestProcessor.java | 14 +++-
.../bookkeeper/proto/WriteEntryProcessorTest.java | 45 ++-
.../proto/WriteEntryProcessorV3Test.java | 13 ++-
22 files changed, 223 insertions(+), 133 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/processor/RequestProcessor.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/processor/RequestProcessor.java
index 50b97e906b..5a4238e64d 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/processor/RequestProcessor.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/processor/RequestProcessor.java
@@ -20,7 +20,7 @@
*/
package org.apache.bookkeeper.processor;
-import io.netty.channel.Channel;
+import org.apache.bookkeeper.proto.BookieRequestHandler;
/**
* A request processor that is used for processing requests at bookie side.
@@ -41,5 +41,5 @@ public interface RequestProcessor extends AutoCloseable {
* @param channel
* channel received the given request r
*/
-void processRequest(Object r, Channel channel);
+void processRequest(Object r, BookieRequestHandler channel);
}
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieRequestHandler.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieRequestHandler.java
index 93be69cd37..c9d65a7317 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieRequestHandler.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieRequestHandler.java
@@ -32,20 +32,27 @@ import org.slf4j.LoggerFactory;
/**
* Serverside handler for bookkeeper requests.
*/
-class BookieRequestHandler extends ChannelInboundHandlerAdapter {
+public class BookieRequestHandler extends ChannelInboundHandlerAdapter {
private static final Logger LOG =
LoggerFactory.getLogger(BookieRequestHandler.class);
private final RequestProcessor requestProcessor;
private final ChannelGroup allChannels;
+private ChannelHandlerContext ctx;
+
BookieRequestHandler(ServerConfiguration conf, RequestProcessor processor,
ChannelGroup allChannels) {
this.requestProcessor = processor;
this.allChannels = allChannels;
}
+public ChannelHandlerContext ctx() {
+return ctx;
+}
+
@Override
public void channelActive(ChannelHandlerContext ctx) throws Exception {
LOG.info("Channel connected {}", ctx.channel());
+this.ctx = ctx;
super.channelActive(ctx);
}
@@ -75,6 +82,6 @@ class BookieRequestHandler extends
ChannelInboundHandlerAdapter {
ctx.fireChannelRead(msg);
return;
}
-requestProcessor.processRequest(msg, ctx.channel());
+requestProcessor.processRequest(msg, this);
}
}
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieRequestProcessor.java
b/bookkeeper-server/src/main/java/
[bookkeeper] branch master updated (20aad8006e -> b4112dfdbf)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git from 20aad8006e Avoid using thread-local WriteSet when possible (#3829) add b4112dfdbf Fix Spotbugs check failed (#3836) No new revisions were added by this update. Summary of changes: .github/workflows/bk-ci.yml| 44 +- .../org/apache/bookkeeper/client/PendingAddOp.java | 2 + .../org/apache/bookkeeper/util/PageCacheUtil.java | 2 +- .../apache/bookkeeper/bookie/LedgerCacheTest.java | 9 - .../main/resources/bookkeeper/findbugsExclude.xml | 5 +++ .../bookkeeper/bookie/GroupSortBenchmark.java | 3 +- pom.xml| 2 +- 7 files changed, 61 insertions(+), 6 deletions(-)
[bookkeeper] branch master updated: Avoid using thread-local WriteSet when possible (#3829)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 20aad8006e Avoid using thread-local WriteSet when possible (#3829)
20aad8006e is described below
commit 20aad8006e7bb21f034dfe61bc68a38968428d1e
Author: Matteo Merli
AuthorDate: Thu Mar 2 07:52:10 2023 -0800
Avoid using thread-local WriteSet when possible (#3829)
---
.../bookkeeper/client/DistributionSchedule.java | 12
.../org/apache/bookkeeper/client/LedgerChecker.java | 7 +++
.../org/apache/bookkeeper/client/PendingAddOp.java | 21 +
.../apache/bookkeeper/client/PendingWriteLacOp.java | 10 +++---
.../client/RoundRobinDistributionSchedule.java | 18 +-
5 files changed, 36 insertions(+), 32 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DistributionSchedule.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DistributionSchedule.java
index ea9017467f..2646d3abc5 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DistributionSchedule.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DistributionSchedule.java
@@ -136,11 +136,23 @@ public interface DistributionSchedule {
}
};
+int getWriteQuorumSize();
+
/**
* Return the set of bookie indices to send the message to.
*/
WriteSet getWriteSet(long entryId);
+/**
+ * Return the WriteSet bookie index for a given and index
+ * in the WriteSet.
+ *
+ * @param entryId
+ * @param writeSetIndex
+ * @return
+ */
+int getWriteSetBookieIndex(long entryId, int writeSetIndex);
+
/**
* Return the set of bookies indices to send the messages to the whole
ensemble.
*
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerChecker.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerChecker.java
index a5cf3eec9a..6bbdc098de 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerChecker.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerChecker.java
@@ -443,18 +443,17 @@ public class LedgerChecker {
}
});
-DistributionSchedule.WriteSet writeSet =
lh.getDistributionSchedule().getWriteSet(entryToRead);
-for (int i = 0; i < writeSet.size(); i++) {
+DistributionSchedule ds = lh.getDistributionSchedule();
+for (int i = 0; i < ds.getWriteQuorumSize(); i++) {
try {
acquirePermit();
-BookieId addr = curEnsemble.get(writeSet.get(i));
+BookieId addr =
curEnsemble.get(ds.getWriteSetBookieIndex(entryToRead, i));
bookieClient.readEntry(addr, lh.getId(), entryToRead,
eecb, null, BookieProtocol.FLAG_NONE);
} catch (InterruptedException e) {
LOG.error("InterruptedException when checking entry :
{}", entryToRead, e);
}
}
-writeSet.recycle();
return;
} else {
fragments.add(lastLedgerFragment);
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java
index 3a78696e88..05f740d33a 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java
@@ -198,14 +198,9 @@ class PendingAddOp implements WriteCallback {
// completes.
//
// We call sendAddSuccessCallback when unsetting t cover this case.
-DistributionSchedule.WriteSet writeSet =
lh.distributionSchedule.getWriteSet(entryId);
-try {
-if (!writeSet.contains(bookieIndex)) {
-lh.sendAddSuccessCallbacks();
-return;
-}
-} finally {
-writeSet.recycle();
+if (!lh.distributionSchedule.hasEntry(entryId, bookieIndex)) {
+lh.sendAddSuccessCallbacks();
+return;
}
if (callbackTriggered) {
@@ -256,14 +251,8 @@ class PendingAddOp implements WriteCallback {
lh.maybeHandleDelayedWriteBookieFailure();
// Iterate over set and trigger the sendWriteRequests
-DistributionSchedule.WriteSet writeSet =
lh.distributionSchedule.getWriteSet(entryId);
-
-try {
-for (int i = 0; i <
[bookkeeper] branch master updated (3dcd8d5869 -> 7852db6059)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git from 3dcd8d5869 Remove callback threadpool for sending add responses (#3825) add 7852db6059 remove unused 'close ledger manager' comment (#3827) No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java | 2 -- 1 file changed, 2 deletions(-)
[bookkeeper] branch master updated: Update testcontainers to last version to support m1 (#3819)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new a96317e353 Update testcontainers to last version to support m1 (#3819)
a96317e353 is described below
commit a96317e3537abff1d5759356ee27fc1044740f44
Author: ZhangJian He
AuthorDate: Wed Mar 1 09:54:01 2023 +0800
Update testcontainers to last version to support m1 (#3819)
---
.../org/apache/bookkeeper/metadata/etcd/helpers/KeySetReaderTest.java | 2 +-
.../org/apache/bookkeeper/metadata/etcd/testing/EtcdContainer.java| 4 ++--
pom.xml | 2 +-
.../java/org/apache/bookkeeper/tests/containers/BookieContainer.java | 2 +-
.../main/java/org/apache/bookkeeper/tests/containers/ZKContainer.java | 2 +-
.../org/apache/bookkeeper/tests/containers/wait/HttpWaitStrategy.java | 2 +-
6 files changed, 7 insertions(+), 7 deletions(-)
diff --git
a/metadata-drivers/etcd/src/test/java/org/apache/bookkeeper/metadata/etcd/helpers/KeySetReaderTest.java
b/metadata-drivers/etcd/src/test/java/org/apache/bookkeeper/metadata/etcd/helpers/KeySetReaderTest.java
index ad1084f066..b4bdf6ea79 100644
---
a/metadata-drivers/etcd/src/test/java/org/apache/bookkeeper/metadata/etcd/helpers/KeySetReaderTest.java
+++
b/metadata-drivers/etcd/src/test/java/org/apache/bookkeeper/metadata/etcd/helpers/KeySetReaderTest.java
@@ -41,7 +41,7 @@ import org.apache.bookkeeper.versioning.Version.Occurred;
import org.apache.bookkeeper.versioning.Versioned;
import org.apache.commons.compress.utils.Sets;
import org.junit.Test;
-import org.testcontainers.shaded.org.apache.commons.lang.RandomStringUtils;
+import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils;
/**
* Integration test {@link KeySetReader}.
diff --git
a/metadata-drivers/etcd/src/test/java/org/apache/bookkeeper/metadata/etcd/testing/EtcdContainer.java
b/metadata-drivers/etcd/src/test/java/org/apache/bookkeeper/metadata/etcd/testing/EtcdContainer.java
index 2ef547d1f3..0178e4d2ab 100644
---
a/metadata-drivers/etcd/src/test/java/org/apache/bookkeeper/metadata/etcd/testing/EtcdContainer.java
+++
b/metadata-drivers/etcd/src/test/java/org/apache/bookkeeper/metadata/etcd/testing/EtcdContainer.java
@@ -59,7 +59,7 @@ public class EtcdContainer extends
GenericContainer {
}
public String getExternalServiceUri() {
-return "etcd://" + getContainerIpAddress() + ":" + getEtcdClientPort()
+ "/clusters/" + clusterName;
+return "etcd://" + getHost() + ":" + getEtcdClientPort() +
"/clusters/" + clusterName;
}
public String getInternalServiceUri() {
@@ -113,7 +113,7 @@ public class EtcdContainer extends
GenericContainer {
}
public String getClientEndpoint() {
-return String.format("http://%s:%d;, getContainerIpAddress(),
getEtcdClientPort());
+return String.format("http://%s:%d;, getHost(), getEtcdClientPort());
}
private WaitStrategy waitStrategy() {
diff --git a/pom.xml b/pom.xml
index a46ffaeaf8..8efb42dcdd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -171,7 +171,7 @@
1.33
4.6.0
1.3.2
-1.15.1
+1.17.6
4.3.8
3.8.1
1.1.7.7
diff --git
a/tests/integration-tests-topologies/src/main/java/org/apache/bookkeeper/tests/containers/BookieContainer.java
b/tests/integration-tests-topologies/src/main/java/org/apache/bookkeeper/tests/containers/BookieContainer.java
index 9e430e0fa0..4ba5c987ba 100644
---
a/tests/integration-tests-topologies/src/main/java/org/apache/bookkeeper/tests/containers/BookieContainer.java
+++
b/tests/integration-tests-topologies/src/main/java/org/apache/bookkeeper/tests/containers/BookieContainer.java
@@ -61,7 +61,7 @@ public class BookieContainer> extends Chaos
}
public String getExternalGrpcEndpointStr() {
-return getContainerIpAddress() + ":" + getMappedPort(BOOKIE_GRPC_PORT);
+return getHost() + ":" + getMappedPort(BOOKIE_GRPC_PORT);
}
public String getInternalGrpcEndpointStr() {
diff --git
a/tests/integration-tests-topologies/src/main/java/org/apache/bookkeeper/tests/containers/ZKContainer.java
b/tests/integration-tests-topologies/src/main/java/org/apache/bookkeeper/tests/containers/ZKContainer.java
index c417b03199..08767fec82 100644
---
a/tests/integration-tests-topologies/src/main/java/org/apache/bookkeeper/tests/containers/ZKContainer.java
+++
b/tests/integration-tests-topologies/src/main/java/org/apache/bookkeeper/tests/containers/ZKContainer.java
@@ -43,7 +43,7 @@ public class ZKContainer>
extends MetadataStore
@Override
public String getExternalServiceUri() {
-return "zk://" + getContainerIpAddress() + ":" +
getMappedPort(ZK_PORT) +
[bookkeeper] branch master updated: Use JNI directly for posix_fadvise (#3824)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new fbd18aeeca Use JNI directly for posix_fadvise (#3824)
fbd18aeeca is described below
commit fbd18aeecaf8089c4a15676fea0dff3b32615293
Author: Matteo Merli
AuthorDate: Tue Feb 28 17:52:40 2023 -0800
Use JNI directly for posix_fadvise (#3824)
* Use JNI directly for posix_fadvise
* Fixed checkstyle
* fixed checkstyle
* Fixed jni function name
---
.../src/main/resources/LICENSE-all.bin.txt | 2 -
.../src/main/resources/LICENSE-bkctl.bin.txt | 2 -
.../src/main/resources/LICENSE-server.bin.txt | 2 -
bookkeeper-server/pom.xml | 4 --
.../apache/bookkeeper/bookie/JournalChannel.java | 6 +--
.../util/{NativeIO.java => PageCacheUtil.java} | 62 --
.../bookkeeper/common/util/nativeio/NativeIO.java | 6 +++
.../common/util/nativeio/NativeIOImpl.java | 5 ++
.../common/util/nativeio/NativeIOJni.java | 2 +
.../src/main/native-io-jni/cpp/native_io_jni.c | 21
pom.xml| 8 ---
shaded/distributedlog-core-shaded/pom.xml | 6 ---
12 files changed, 60 insertions(+), 66 deletions(-)
diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt
b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt
index 5e917f13bc..b66bc8b749 100644
--- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt
+++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt
@@ -258,7 +258,6 @@ Apache Software License, Version 2.
- lib/org.apache.logging.log4j-log4j-api-2.18.0.jar [17]
- lib/org.apache.logging.log4j-log4j-core-2.18.0.jar [17]
- lib/org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar [17]
-- lib/net.java.dev.jna-jna-5.12.1.jar [18]
- lib/org.apache.commons-commons-collections4-4.1.jar [19]
- lib/org.apache.commons-commons-lang3-3.6.jar [20]
- lib/org.apache.zookeeper-zookeeper-3.8.1.jar [21]
@@ -339,7 +338,6 @@ Apache Software License, Version 2.
[15] Source available at https://github.com/eclipse/vert.x/tree/4.3.2
[16] Source available at https://github.com/vert-x3/vertx-web/tree/4.3.2
[17] Source available at
https://github.com/apache/logging-log4j2/tree/rel/2.18.0
-[18] Source available at https://github.com/java-native-access/jna/tree/5.12.1
[19] Source available at
https://github.com/apache/commons-collections/tree/collections-4.1
[20] Source available at https://github.com/apache/commons-lang/tree/LANG_3_6
[21] Source available at https://github.com/apache/zookeeper/tree/release-3.8.0
diff --git a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt
b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt
index 102daaa7f7..013d46207a 100644
--- a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt
+++ b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt
@@ -243,7 +243,6 @@ Apache Software License, Version 2.
- lib/org.apache.logging.log4j-log4j-api-2.18.0.jar [16]
- lib/org.apache.logging.log4j-log4j-core-2.18.0.jar [16]
- lib/org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar [16]
-- lib/net.java.dev.jna-jna-5.12.1.jar [17]
- lib/org.apache.commons-commons-collections4-4.1.jar [18]
- lib/org.apache.commons-commons-lang3-3.6.jar [19]
- lib/org.apache.zookeeper-zookeeper-3.8.1.jar [20]
@@ -305,7 +304,6 @@ Apache Software License, Version 2.
[10] Source available at
https://github.com/apache/commons-logging/tree/commons-logging-1.1.1
[11] Source available at https://github.com/netty/netty/tree/netty-4.1.89.Final
[16] Source available at
https://github.com/apache/logging-log4j2/tree/rel/2.18.0
-[17] Source available at https://github.com/java-native-access/jna/tree/5.12.1
[18] Source available at
https://github.com/apache/commons-collections/tree/collections-4.1
[19] Source available at https://github.com/apache/commons-lang/tree/LANG_3_6
[20] Source available at https://github.com/apache/zookeeper/tree/release-3.8.0
diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt
b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt
index 23018b504b..cde305a40b 100644
--- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt
+++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt
@@ -258,7 +258,6 @@ Apache Software License, Version 2.
- lib/org.apache.logging.log4j-log4j-api-2.18.0.jar [17]
- lib/org.apache.logging.log4j-log4j-core-2.18.0.jar [17]
- lib/org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar [17]
-- lib/net.java.dev.jna-jna-5.12.1.jar [18]
- lib/org.apache.commons-commons-collections4-4.1.jar [19]
- lib/org.apache.commons-commons-lang3-3.6.jar [20]
- lib/org.apache.zookeeper-zookeeper-3.8.1.jar [21]
@@ -335,7 +334,6 @@ Apache Software License, Version 2.
[15] Sou
[bookkeeper] branch master updated (6d00336728 -> b70d128ad0)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
from 6d00336728 fix mailing-lists dead links (#3804)
add b70d128ad0 Upgrade Netty to 4.1.89.Final (#3816)
No new revisions were added by this update.
Summary of changes:
bookkeeper-dist/src/assemble/bin-all.xml | 2 +-
bookkeeper-dist/src/assemble/bin-server.xml| 2 +-
bookkeeper-dist/src/assemble/bkctl.xml | 2 +-
.../src/main/resources/LICENSE-all.bin.txt | 150 ++---
.../src/main/resources/LICENSE-bkctl.bin.txt | 146 ++--
.../src/main/resources/LICENSE-server.bin.txt | 150 ++---
.../src/main/resources/NOTICE-all.bin.txt | 34 ++---
.../src/main/resources/NOTICE-bkctl.bin.txt| 30 ++---
.../src/main/resources/NOTICE-server.bin.txt | 34 ++---
.../LICENSE.aalto-xml.txt | 0
.../LICENSE.base64.txt | 0
.../LICENSE.bouncycastle.txt | 0
.../LICENSE.caliper.txt| 0
.../LICENSE.commons-lang.txt | 0
.../LICENSE.commons-logging.txt| 0
.../LICENSE.compress-lzf.txt | 0
.../LICENSE.dnsinfo.txt| 0
.../LICENSE.harmony.txt| 0
.../LICENSE.hpack.txt | 0
.../LICENSE.hyper-hpack.txt| 0
.../LICENSE.jboss-marshalling.txt | 0
.../LICENSE.jbzip2.txt | 0
.../LICENSE.jctools.txt| 0
.../LICENSE.jfastlz.txt| 0
.../LICENSE.jsr166y.txt| 0
.../LICENSE.jzlib.txt | 0
.../LICENSE.libdivsufsort.txt | 0
.../LICENSE.log4j.txt | 0
.../LICENSE.lz4.txt| 0
.../LICENSE.lzma-java.txt | 0
.../LICENSE.mvn-wrapper.txt| 0
.../LICENSE.nghttp2-hpack.txt | 0
.../LICENSE.protobuf.txt | 0
.../LICENSE.slf4j.txt | 0
.../LICENSE.snappy.txt | 0
.../LICENSE.webbit.txt | 0
.../NOTICE.harmony.txt | 0
pom.xml| 4 +-
38 files changed, 277 insertions(+), 277 deletions(-)
rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.87.Final =>
netty-4.1.89.Final}/LICENSE.aalto-xml.txt (100%)
rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.87.Final =>
netty-4.1.89.Final}/LICENSE.base64.txt (100%)
rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.87.Final =>
netty-4.1.89.Final}/LICENSE.bouncycastle.txt (100%)
rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.87.Final =>
netty-4.1.89.Final}/LICENSE.caliper.txt (100%)
rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.87.Final =>
netty-4.1.89.Final}/LICENSE.commons-lang.txt (100%)
rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.87.Final =>
netty-4.1.89.Final}/LICENSE.commons-logging.txt (100%)
rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.87.Final =>
netty-4.1.89.Final}/LICENSE.compress-lzf.txt (100%)
rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.87.Final =>
netty-4.1.89.Final}/LICENSE.dnsinfo.txt (100%)
rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.87.Final =>
netty-4.1.89.Final}/LICENSE.harmony.txt (100%)
rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.87.Final =>
netty-4.1.89.Final}/LICENSE.hpack.txt (100%)
rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.87.Final =>
netty-4.1.89.Final}/LICENSE.hyper-hpack.txt (100%)
rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.87.Final =>
netty-4.1.89.Final}/LICENSE.jboss-marshalling.txt (100%)
rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.87.Final =>
netty-4.1.89.Final}/LICENSE.jbzip2.txt (100%)
rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.87.Final =>
netty-4.1.89.Final}/LICENSE.jctools.txt (100%)
rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.87.Final =>
netty-4.1.89.Final}/LICENSE.jfastlz.txt (100%)
rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.87.Final =>
netty-4.1.89.Final}/LICENSE.jsr166y.txt (100%)
rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.87.Final =>
netty-4.1.89.Final}/LICENSE.jzlib.txt (100%)
rename bookkeeper-dist/src/main/resources/deps/{netty-4.1.87.Final =>
netty-4.1.89.Final}/LICENSE.libdivsufsort.txt (100%)
rename bookkeeper-dist/src/main/res
[bookkeeper] branch master updated: Fix: typo for ledger-metadata param: restorefromfile (#3823)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 0ca4fe2697 Fix: typo for ledger-metadata param: restorefromfile (#3823)
0ca4fe2697 is described below
commit 0ca4fe26978c231c81796f4cfd52ae8fa3b2fc29
Author: Rajan Dhabalia
AuthorDate: Mon Feb 27 16:47:19 2023 -0800
Fix: typo for ledger-metadata param: restorefromfile (#3823)
---
.../src/main/java/org/apache/bookkeeper/bookie/BookieShell.java | 2 +-
.../bookkeeper/tools/cli/commands/client/LedgerMetaDataCommand.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
index b1ab726f0f..944b97e79d 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
@@ -815,7 +815,7 @@ public class BookieShell implements Tool {
System.err.println("Must specify a ledger id");
return -1;
}
-if (cmdLine.hasOption("dumptofile") &&
cmdLine.hasOption("restorefromefile")) {
+if (cmdLine.hasOption("dumptofile") &&
cmdLine.hasOption("restorefromfile")) {
System.err.println("Only one of --dumptofile and
--restorefromfile can be specified");
return -2;
}
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/client/LedgerMetaDataCommand.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/client/LedgerMetaDataCommand.java
index ae06a67fd1..4bbb50e7cc 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/client/LedgerMetaDataCommand.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/client/LedgerMetaDataCommand.java
@@ -89,7 +89,7 @@ public class LedgerMetaDataCommand extends
BookieCommand
[bookkeeper] branch master updated: Support update ledger metadata option bk-cli (#3821)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new e0ee708269 Support update ledger metadata option bk-cli (#3821)
e0ee708269 is described below
commit e0ee70826974dd02c24133565bb1d81c30a753f1
Author: Rajan Dhabalia
AuthorDate: Mon Feb 27 16:46:36 2023 -0800
Support update ledger metadata option bk-cli (#3821)
---
.../java/org/apache/bookkeeper/bookie/BookieShell.java | 2 ++
.../tools/cli/commands/client/LedgerMetaDataCommand.java | 16 +++-
.../cli/commands/client/LedgerMetaDataCommandTest.java | 13 +
3 files changed, 30 insertions(+), 1 deletion(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
index f3aaa13027..15e1fc6f05 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
@@ -805,6 +805,7 @@ public class BookieShell implements Tool {
opts.addOption("l", "ledgerid", true, "Ledger ID");
opts.addOption("dumptofile", true, "Dump metadata for ledger, to a
file");
opts.addOption("restorefromfile", true, "Restore metadata for
ledger, from a file");
+opts.addOption("update", false, "Update metadata if ledger already
exist");
}
@Override
@@ -827,6 +828,7 @@ public class BookieShell implements Tool {
if (cmdLine.hasOption("restorefromfile")) {
flag.restoreFromFile(cmdLine.getOptionValue("restorefromfile"));
}
+flag.update(cmdLine.hasOption("update"));
LedgerMetaDataCommand cmd = new
LedgerMetaDataCommand(ledgerIdFormatter);
cmd.apply(bkConf, flag);
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/client/LedgerMetaDataCommand.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/client/LedgerMetaDataCommand.java
index 324bc1d4dd..ae06a67fd1 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/client/LedgerMetaDataCommand.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/client/LedgerMetaDataCommand.java
@@ -29,6 +29,7 @@ import java.util.Optional;
import java.util.concurrent.ExecutionException;
import lombok.Setter;
import lombok.experimental.Accessors;
+import org.apache.bookkeeper.client.BKException.BKLedgerExistException;
import org.apache.bookkeeper.client.api.LedgerMetadata;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.meta.LedgerManager;
@@ -38,6 +39,7 @@ import org.apache.bookkeeper.tools.cli.helpers.BookieCommand;
import org.apache.bookkeeper.tools.framework.CliFlags;
import org.apache.bookkeeper.tools.framework.CliSpec;
import org.apache.bookkeeper.util.LedgerIdFormatter;
+import org.apache.bookkeeper.versioning.LongVersion;
import org.apache.bookkeeper.versioning.Versioned;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -92,6 +94,10 @@ public class LedgerMetaDataCommand extends
BookieCommand
[bookkeeper] branch master updated (e0ee708269 -> ade30bda19)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git from e0ee708269 Support update ledger metadata option bk-cli (#3821) add ade30bda19 Fix: bookie-shell ledger-metadata usage with correct param (#3822) No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/bookkeeper/bookie/BookieShell.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[bookkeeper] branch master updated (c389b96f12 -> 284cf9611a)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git from c389b96f12 Speed up shutdown in test cases (#3809) add 284cf9611a Remove compiling with `-Werror` to support Java 19 (#3820) No new revisions were added by this update. Summary of changes: pom.xml | 1 - 1 file changed, 1 deletion(-)
[bookkeeper] branch master updated: Speed up shutdown in test cases (#3809)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new c389b96f12 Speed up shutdown in test cases (#3809)
c389b96f12 is described below
commit c389b96f12cd8dbcb654acc84234afe3f981f698
Author: Hang Chen
AuthorDate: Tue Feb 28 03:02:17 2023 +0800
Speed up shutdown in test cases (#3809)
* speed up shutdown in test cases
* fix failed test
---
.../java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java | 3 ++-
.../java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java | 5 -
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java
index ff390e3b27..a0e795034e 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java
@@ -26,6 +26,7 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.util.concurrent.TimeUnit;
import org.apache.bookkeeper.bookie.BookieImpl;
import org.apache.bookkeeper.meta.zk.ZKMetadataClientDriver;
import org.apache.bookkeeper.net.BookieId;
@@ -118,7 +119,7 @@ public class AutoRecoveryMainTest extends
BookKeeperClusterTestCase {
assertNotNull(currentAuditor);
Auditor auditor1 = main1.auditorElector.getAuditor();
assertEquals("Current Auditor should be AR1", currentAuditor,
BookieImpl.getBookieId(confByIndex(0)));
-Awaitility.await().untilAsserted(() -> {
+Awaitility.waitAtMost(30, TimeUnit.SECONDS).untilAsserted(() -> {
assertNotNull(auditor1);
assertTrue("Auditor of AR1 should be running",
auditor1.isRunning());
});
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java
index 8f8596b5e2..76b21ecea2 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java
@@ -275,6 +275,7 @@ public abstract class BookKeeperClusterTestCase {
bkc.close();
}
+stopReplicationService();
for (ServerTester t : servers) {
t.shutdown();
}
@@ -316,6 +317,7 @@ public abstract class BookKeeperClusterTestCase {
}
protected void stopAllBookies(boolean shutdownClient) throws Exception {
+stopReplicationService();
for (ServerTester t : servers) {
t.shutdown();
}
@@ -607,6 +609,7 @@ public abstract class BookKeeperClusterTestCase {
throws Exception {
// shut down bookie server
List confs = new ArrayList<>();
+stopReplicationService();
for (ServerTester server : servers) {
server.shutdown();
confs.add(server.getConfiguration());
@@ -964,7 +967,7 @@ public abstract class BookKeeperClusterTestCase {
if (autoRecovery != null) {
if (LOG.isDebugEnabled()) {
-LOG.debug("Shutdown auto recovery for bookieserver: {}",
address);
+LOG.debug("Shutdown auto recovery for bookie server: {}",
address);
}
autoRecovery.shutdown();
}
[bookkeeper] branch master updated: Improved efficiency in DigestManager.verify() (#3810)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 1f8de8ffa2 Improved efficiency in DigestManager.verify() (#3810)
1f8de8ffa2 is described below
commit 1f8de8ffa2e63bb748b096c1427f58e6b07e5c5c
Author: Matteo Merli
AuthorDate: Sun Feb 26 20:44:34 2023 -0800
Improved efficiency in DigestManager.verify() (#3810)
---
.../proto/checksum/CRC32CDigestManager.java| 4 +-
.../proto/checksum/CRC32DigestManager.java | 6 +-
.../bookkeeper/proto/checksum/DigestManager.java | 63 +++--
.../proto/checksum/DirectMemoryCRC32Digest.java| 5 +-
.../proto/checksum/DummyDigestManager.java | 2 +-
.../proto/checksum/MacDigestManager.java | 4 +-
.../proto/checksum/StandardCRC32Digest.java| 4 +-
.../circe/checksum/Crc32cIntChecksum.java | 4 +-
.../com/scurrilous/circe/checksum/IntHash.java | 5 +
.../scurrilous/circe/checksum/Java8IntHash.java| 16 +++-
.../scurrilous/circe/checksum/Java9IntHash.java| 20 +++-
.../com/scurrilous/circe/checksum/JniIntHash.java | 20 ++--
.../scurrilous/circe/checksum/ChecksumTest.java| 10 +-
.../proto/checksum/DigestManagerBenchmark.java | 102 +
.../proto/checksum/DigestTypeBenchmark.java| 2 +-
15 files changed, 204 insertions(+), 63 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/CRC32CDigestManager.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/CRC32CDigestManager.java
index c3be132c39..446e0b35ab 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/CRC32CDigestManager.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/CRC32CDigestManager.java
@@ -52,9 +52,9 @@ class CRC32CDigestManager extends DigestManager {
}
@Override
-void update(ByteBuf data) {
+void update(ByteBuf data, int offset, int len) {
MutableInt current = currentCrc.get();
final int lastCrc = current.intValue();
-current.setValue(Crc32cIntChecksum.resumeChecksum(lastCrc, data));
+current.setValue(Crc32cIntChecksum.resumeChecksum(lastCrc, data,
offset, len));
}
}
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/CRC32DigestManager.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/CRC32DigestManager.java
index d06bc8030c..ea34f13069 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/CRC32DigestManager.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/CRC32DigestManager.java
@@ -33,7 +33,7 @@ class CRC32DigestManager extends DigestManager {
interface CRC32Digest {
long getValueAndReset();
-void update(ByteBuf buf);
+void update(ByteBuf buf, int offset, int len);
}
private static final FastThreadLocal crc = new
FastThreadLocal() {
@@ -62,7 +62,7 @@ class CRC32DigestManager extends DigestManager {
}
@Override
-void update(ByteBuf data) {
-crc.get().update(data);
+void update(ByteBuf data, int offset, int len) {
+crc.get().update(data, offset, len);
}
}
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/DigestManager.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/DigestManager.java
index 7b0f57a945..b142448aa4 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/DigestManager.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/DigestManager.java
@@ -19,9 +19,12 @@ package org.apache.bookkeeper.proto.checksum;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.ByteBufUtil;
import io.netty.buffer.CompositeByteBuf;
+import io.netty.buffer.PooledByteBufAllocator;
import io.netty.buffer.Unpooled;
import io.netty.util.ReferenceCountUtil;
+import io.netty.util.concurrent.FastThreadLocal;
import java.security.GeneralSecurityException;
import java.security.NoSuchAlgorithmException;
import org.apache.bookkeeper.client.BKException.BKDigestMatchException;
@@ -51,10 +54,10 @@ public abstract class DigestManager {
abstract int getMacCodeLength();
void update(byte[] data) {
-update(Unpooled.wrappedBuffer(data, 0, data.length));
+update(Unpooled.wrappedBuffer(data), 0, data.length);
}
-abstract void update(ByteBuf buffer);
+abstract void update(ByteBuf buffer, int offset, int len);
abstract void populateValueAndReset(ByteBuf buffer);
@@ -109,7 +112,7 @@ public abstract class DigestManager {
headersBuffer.writeLong(lastAddConfirmed
[bookkeeper] branch master updated: Use mixed quick/insertion sort (#3807)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new f65b72d54b Use mixed quick/insertion sort (#3807)
f65b72d54b is described below
commit f65b72d54bff677adb273c509d84ada7f93727a9
Author: Matteo Merli
AuthorDate: Sun Feb 26 20:41:28 2023 -0800
Use mixed quick/insertion sort (#3807)
---
.../bookie/storage/ldb/ArrayGroupSort.java | 96 +-
.../bookkeeper/bookie/storage/ldb/WriteCache.java | 4 +-
.../bookie/storage/ldb/ArraySortGroupTest.java | 36 ++--
.../bookkeeper/bookie/GroupSortBenchmark.java | 11 +--
4 files changed, 70 insertions(+), 77 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/ArrayGroupSort.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/ArrayGroupSort.java
index 78cf3396ec..af4f3ef931 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/ArrayGroupSort.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/ArrayGroupSort.java
@@ -22,58 +22,61 @@ package org.apache.bookkeeper.bookie.storage.ldb;
import static com.google.common.base.Preconditions.checkArgument;
+import lombok.experimental.UtilityClass;
+
/**
* Sort an array of longs, grouping the items in tuples.
*
* Group size decides how many longs are included in the tuples and key
size controls how many items to use for
* comparison.
*/
+@UtilityClass
public class ArrayGroupSort {
-private final int keySize;
-private final int groupSize;
+private static final int INSERTION_SORT_THRESHOLD = 100;
-public ArrayGroupSort(int keySize, int groupSize) {
-checkArgument(keySize > 0);
-checkArgument(groupSize > 0);
-checkArgument(keySize <= groupSize, "keySize need to be less or equal
the groupSize");
-this.keySize = keySize;
-this.groupSize = groupSize;
-}
+private static final int GROUP_SIZE = 4;
public void sort(long[] array) {
sort(array, 0, array.length);
}
-public void sort(long[] array, int offset, int length) {
-checkArgument(length % groupSize == 0, "Array length must be multiple
of groupSize");
-quickSort(array, offset, (length + offset - groupSize));
+public static void sort(long[] array, int offset, int length) {
+checkArgument(length % GROUP_SIZE == 0, "Array length must be multiple
of 4");
+quickSort(array, offset, (length + offset - GROUP_SIZE));
}
// Private
-private void quickSort(long[] array, int low, int high) {
-if (low < high) {
-int pivotIdx = partition(array, low, high);
-quickSort(array, low, pivotIdx - groupSize);
-quickSort(array, pivotIdx + groupSize, high);
+private static void quickSort(long[] array, int low, int high) {
+if (low >= high) {
+return;
}
+
+if (high - low < INSERTION_SORT_THRESHOLD) {
+insertionSort(array, low, high);
+return;
+}
+
+int pivotIdx = partition(array, low, high);
+quickSort(array, low, pivotIdx - GROUP_SIZE);
+quickSort(array, pivotIdx + GROUP_SIZE, high);
}
-private int alignGroup(int count) {
-return count - (count % groupSize);
+private static int alignGroup(int count) {
+return count - (count % GROUP_SIZE);
}
-private int partition(long[] array, int low, int high) {
+private static int partition(long[] array, int low, int high) {
int mid = low + alignGroup((high - low) / 2);
swap(array, mid, high);
int i = low;
-for (int j = low; j < high; j += groupSize) {
+for (int j = low; j < high; j += GROUP_SIZE) {
if (isLess(array, j, high)) {
swap(array, j, i);
-i += groupSize;
+i += GROUP_SIZE;
}
}
@@ -81,26 +84,43 @@ public class ArrayGroupSort {
return i;
}
-private void swap(long[] array, int a, int b) {
-long tmp;
-for (int k = 0; k < groupSize; k++) {
-tmp = array[a + k];
-array[a + k] = array[b + k];
-array[b + k] = tmp;
+private static void swap(long[] array, int a, int b) {
+long tmp0 = array[a];
+long tmp1 = array[a + 1];
+long tmp2 = array[a + 2];
+long tmp3 = array[a + 3];
+
+array[a] = array[b];
+array[a + 1] = array[b + 1];
+array[a + 2] = array[b + 2];
+array[a + 3] = array[b + 3];
+
+array[b] = tmp0;
+array[b + 1] = tmp1;
+array[b + 2] = tmp2;
+array[b + 3] = tmp3;
+}
+
+
[bookkeeper] branch master updated: Fixed the pivot selection in the group quick-sort (#3800)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 7f263fa8f4 Fixed the pivot selection in the group quick-sort (#3800)
7f263fa8f4 is described below
commit 7f263fa8f41924ef0996f84beffdf008480075d8
Author: Matteo Merli
AuthorDate: Sun Feb 26 17:00:02 2023 -0800
Fixed the pivot selection in the group quick-sort (#3800)
* Fixed the pivot selection in the group quick-sort
* Fixed formatting
* Fixed alignment
---
.../bookie/storage/ldb/ArrayGroupSort.java | 10 +-
.../bookkeeper/bookie/GroupSortBenchmark.java | 121 +
2 files changed, 129 insertions(+), 2 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/ArrayGroupSort.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/ArrayGroupSort.java
index 27874e1c8c..78cf3396ec 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/ArrayGroupSort.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/ArrayGroupSort.java
@@ -60,12 +60,18 @@ public class ArrayGroupSort {
}
}
+private int alignGroup(int count) {
+return count - (count % groupSize);
+}
+
private int partition(long[] array, int low, int high) {
-int pivotIdx = high;
+int mid = low + alignGroup((high - low) / 2);
+swap(array, mid, high);
+
int i = low;
for (int j = low; j < high; j += groupSize) {
-if (isLess(array, j, pivotIdx)) {
+if (isLess(array, j, high)) {
swap(array, j, i);
i += groupSize;
}
diff --git
a/microbenchmarks/src/main/java/org/apache/bookkeeper/bookie/GroupSortBenchmark.java
b/microbenchmarks/src/main/java/org/apache/bookkeeper/bookie/GroupSortBenchmark.java
new file mode 100644
index 00..8290e54fb4
--- /dev/null
+++
b/microbenchmarks/src/main/java/org/apache/bookkeeper/bookie/GroupSortBenchmark.java
@@ -0,0 +1,121 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.bookkeeper.bookie;
+
+import java.util.Arrays;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import org.apache.bookkeeper.bookie.storage.ldb.ArrayGroupSort;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+@OutputTimeUnit(TimeUnit.SECONDS)
+@Fork(1)
+@Warmup(iterations = 1, time = 10)
+@Measurement(iterations = 3, time = 10)
+public class GroupSortBenchmark {
+
+private static final int N = 10_000;
+
+@State(Scope.Benchmark)
+public static class TestState {
+
+private final long[] randomItems = new long[N * 4];
+private final long[] sortedItems;
+private final long[] reverseSortedItems = new long[N * 4];
+private final long[] groupSortedItems;
+private final long[] reverseGroupSortedItems = new long[N * 4];
+
+private long[] items;
+
+
+private final ArrayGroupSort sorter = new ArrayGroupSort(2, 4);
+
+public TestState() {
+Random r = new Random();
+for (int i = 0; i < (N * 4); i++) {
+randomItems[i] = r.nextLong();
+}
+
+groupSortedItems = Arrays.copyOf(randomItems, randomItems.length);
+sorter.sort(groupSortedItems);
+for (int i = 0; i < (N * 4); i += 4) {
+reverseGroupSortedItems[i] = groupSortedItems[(N - 1) * 4 - i];
+reverseGroupSortedItems[i + 1] = groupSortedItems[(N - 1) * 4
- i + 1];
+reverseGroupSortedItems[i + 2] = groupSor
[bookkeeper] branch master updated: Fixed locking of PendingAddOp (#3806)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 8d922634c7 Fixed locking of PendingAddOp (#3806)
8d922634c7 is described below
commit 8d922634c71ac96daeb2fa7db7692424895df722
Author: Matteo Merli
AuthorDate: Sun Feb 26 11:10:52 2023 -0800
Fixed locking of PendingAddOp (#3806)
* Fixed locking of PendingAddOp
* Fixed stack overflow
* Fixed stack overflow in different point
---
.../src/main/java/org/apache/bookkeeper/client/LedgerHandle.java| 3 ++-
.../src/main/java/org/apache/bookkeeper/client/PendingAddOp.java| 6 +++---
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandle.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandle.java
index 04270c72f5..ea5295dc7a 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandle.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandle.java
@@ -1870,7 +1870,8 @@ public class LedgerHandle implements WriteHandle {
LOG.debug("Ensemble change is disabled. Retry sending to
failed bookies {} for ledger {}.",
failedBookies, ledgerId);
}
-unsetSuccessAndSendWriteRequest(getCurrentEnsemble(),
failedBookies.keySet());
+executeOrdered(() ->
+unsetSuccessAndSendWriteRequest(getCurrentEnsemble(),
failedBookies.keySet()));
return;
}
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java
index 8dc5037019..3a78696e88 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java
@@ -231,7 +231,7 @@ class PendingAddOp implements WriteCallback {
/**
* Initiate the add operation.
*/
-public void initiate() {
+public synchronized void initiate() {
hasRun = true;
if (callbackTriggered) {
// this should only be true if the request was failed due
@@ -452,7 +452,7 @@ class PendingAddOp implements WriteCallback {
}
-private void maybeRecycle() {
+private synchronized void maybeRecycle() {
/**
* We have opportunity to recycle two objects here.
* PendingAddOp#toSend and LedgerHandle#pendingAddOp
@@ -482,7 +482,7 @@ class PendingAddOp implements WriteCallback {
}
}
-public void recyclePendAddOpObject() {
+public synchronized void recyclePendAddOpObject() {
entryId = LedgerHandle.INVALID_ENTRY_ID;
currentLedgerLength = -1;
if (payload != null) {
[bookkeeper] branch master updated: Made PendingAddOp thread safe (#3784)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new c295f1880f Made PendingAddOp thread safe (#3784)
c295f1880f is described below
commit c295f1880f6c9c18ceb9bc985db9c88e636b9171
Author: Matteo Merli
AuthorDate: Tue Feb 21 12:52:03 2023 -0800
Made PendingAddOp thread safe (#3784)
---
.../org/apache/bookkeeper/client/LedgerHandle.java | 11 ++---
.../apache/bookkeeper/client/LedgerHandleAdv.java | 8 +---
.../org/apache/bookkeeper/client/PendingAddOp.java | 50 --
.../apache/bookkeeper/proto/BookieClientImpl.java | 15 +--
.../bookkeeper/proto/PerChannelBookieClient.java | 4 +-
.../apache/bookkeeper/client/PendingAddOpTest.java | 2 +-
6 files changed, 26 insertions(+), 64 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandle.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandle.java
index 99936705be..04270c72f5 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandle.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandle.java
@@ -1337,7 +1337,7 @@ public class LedgerHandle implements WriteHandle {
});
} catch (RejectedExecutionException e) {
op.cb.addCompleteWithLatency(BookKeeper.getReturnRc(clientCtx.getBookieClient(),
-
BKException.Code.InterruptedException),
+BKException.Code.InterruptedException),
LedgerHandle.this, INVALID_ENTRY_ID, 0, op.ctx);
op.recyclePendAddOpObject();
}
@@ -1355,13 +1355,8 @@ public class LedgerHandle implements WriteHandle {
}
}
-try {
-executeOrdered(op);
-} catch (RejectedExecutionException e) {
-op.cb.addCompleteWithLatency(
-BookKeeper.getReturnRc(clientCtx.getBookieClient(),
BKException.Code.InterruptedException),
-LedgerHandle.this, INVALID_ENTRY_ID, 0, op.ctx);
-}
+op.initiate();
+
}
synchronized void updateLastConfirmed(long lac, long len) {
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandleAdv.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandleAdv.java
index 92bddc9cc4..c94a9154f5 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandleAdv.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerHandleAdv.java
@@ -275,13 +275,7 @@ public class LedgerHandleAdv extends LedgerHandle
implements WriteAdvHandle {
}
}
-try {
-clientCtx.getMainWorkerPool().executeOrdered(ledgerId, op);
-} catch (RejectedExecutionException e) {
-
op.cb.addCompleteWithLatency(BookKeeper.getReturnRc(clientCtx.getBookieClient(),
-
BKException.Code.InterruptedException),
- LedgerHandleAdv.this, op.getEntryId(), 0,
op.ctx);
-}
+op.initiate();
}
@Override
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java
index d0ff59e45c..8dc5037019 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java
@@ -32,7 +32,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
import org.apache.bookkeeper.client.AsyncCallback.AddCallbackWithLatency;
import org.apache.bookkeeper.client.api.WriteFlag;
@@ -52,7 +51,7 @@ import org.slf4j.LoggerFactory;
*
*
*/
-class PendingAddOp implements Runnable, WriteCallback {
+class PendingAddOp implements WriteCallback {
private static final Logger LOG =
LoggerFactory.getLogger(PendingAddOp.class);
ByteBuf payload;
@@ -68,7 +67,7 @@ class PendingAddOp implements Runnable, WriteCallback {
LedgerHandle lh;
ClientContext clientCtx;
boolean isRecoveryAdd = false;
-long requestTimeNanos;
+volatile long requestTimeNanos;
long qwcLatency; // Quorum Write Completion Latency after response from
quorum bookies.
Set addEntrySuccessBookies;
long writeDelayedStartTime; // min fault domains completion latency after
response from ack quorum bookies
@@ -143,7 +142,7 @@ class PendingAddOp implements Runnable, WriteCallback {
return this.entryId
[bookkeeper] branch master updated: change directIO configuration names to avoid confuse (#3791)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 5fced0d434 change directIO configuration names to avoid confuse (#3791)
5fced0d434 is described below
commit 5fced0d4346f851b5698f1bdd3f7a06ee62a7fcb
Author: Hang Chen
AuthorDate: Wed Feb 22 01:53:40 2023 +0800
change directIO configuration names to avoid confuse (#3791)
---
.../bookie/storage/ldb/DbLedgerStorage.java| 6 ++---
conf/bk_server.conf| 26 ++
2 files changed, 29 insertions(+), 3 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java
index 81eab673dd..e02615df63 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java
@@ -81,11 +81,11 @@ public class DbLedgerStorage implements LedgerStorage {
public static final String READ_AHEAD_CACHE_MAX_SIZE_MB =
"dbStorage_readAheadCacheMaxSizeMb";
public static final String DIRECT_IO_ENTRYLOGGER =
"dbStorage_directIOEntryLogger";
public static final String DIRECT_IO_ENTRYLOGGER_TOTAL_WRITEBUFFER_SIZE_MB
=
-"dbStorage_directIOEntryLoggerTotalWriteBufferSizeMb";
+"dbStorage_directIOEntryLoggerTotalWriteBufferSizeMB";
public static final String DIRECT_IO_ENTRYLOGGER_TOTAL_READBUFFER_SIZE_MB =
-"dbStorage_directIOEntryLoggerTotalReadBufferSizeMb";
+"dbStorage_directIOEntryLoggerTotalReadBufferSizeMB";
public static final String DIRECT_IO_ENTRYLOGGER_READBUFFER_SIZE_MB =
-"dbStorage_directIOEntryLoggerReadBufferSizeMb";
+"dbStorage_directIOEntryLoggerReadBufferSizeMB";
public static final String DIRECT_IO_ENTRYLOGGER_MAX_FD_CACHE_TIME_SECONDS
=
"dbStorage_directIOEntryLoggerMaxFdCacheTimeSeconds";
diff --git a/conf/bk_server.conf b/conf/bk_server.conf
index 6304579a39..5017175219 100755
--- a/conf/bk_server.conf
+++ b/conf/bk_server.conf
@@ -768,6 +768,32 @@ gcEntryLogMetadataCacheEnabled=false
# dbStorage_rocksDB_logPath=
# dbStorage_rocksDB_format_version=2
+#
+## DirectIO entry logger configuration
+#
+# DirectIO entry logger only support DbLedgerStorage
+
+# Enable/Disable directIO entry logger.
+# dbStorage_directIOEntryLogger = false
+
+# Total write buffer size in megabytes for all the entry directories.
+# The write buffer size of each entry directory needs to be divided by the
number of entry directories.
+# By default it will be allocated to 12.5% of the available direct memory.
+# dbStorage_directIOEntryLoggerTotalWriteBufferSizeMB=
+
+# Total read buffer size in megabytes for all the entry directories.
+# The read buffer size of each entry directory needs to be divided by the
number of entry directories.
+# By default it will be allocated to 12.5% of the available direct memory.
+# dbStorage_directIOEntryLoggerTotalReadBufferSizeMB=
+
+# The buffer size, in megabytes, for each direct reader to read data from the
entry log file.
+# An entry log file will have only one direct reader.
+# By default it will be set to 8MB
+# dbStorage_directIOEntryLoggerReadBufferSizeMB=8
+
+# Maximum cache time after a direct reader is accessed.
+# dbStorage_directIOEntryLoggerMaxFdCacheTimeSeconds=300
+
## Metadata Services
##
[bookkeeper] branch master updated: Fix CI is not failed when test have errors (#3790)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 6b6ea76201 Fix CI is not failed when test have errors (#3790)
6b6ea76201 is described below
commit 6b6ea7620157b600ded505b38dec472da824fe17
Author: Yong Zhang
AuthorDate: Wed Feb 22 01:52:54 2023 +0800
Fix CI is not failed when test have errors (#3790)
* Fix CI is not failed when test have errors
---
### Motivation
We added the test coverage to report the tests status.
But when we use the code-coverage profile, it will ignore
test failure because we set `true`.
We should do the test and test coverage in different steps
and fail the test if there have errors.
* Fix the test errors
---
.github/workflows/bk-ci.yml| 2 +-
.../test/java/org/apache/bookkeeper/bookie/BookieJournalForceTest.java | 2 ++
.../java/org/apache/bookkeeper/bookie/BookieJournalMaxMemoryTest.java | 2 ++
.../org/apache/bookkeeper/bookie/BookieJournalPageCacheFlushTest.java | 2 ++
.../src/test/java/org/apache/bookkeeper/bookie/BookieJournalTest.java | 3 ++-
.../src/test/java/org/apache/bookkeeper/bookie/BookieShellTest.java| 2 ++
.../java/org/apache/bookkeeper/bookie/BookieWriteToJournalTest.java| 3 ++-
.../apache/bookkeeper/discover/AbstractTestZkRegistrationClient.java | 2 ++
.../java/org/apache/bookkeeper/meta/AbstractZkLedgerManagerTest.java | 2 +-
.../java/org/apache/bookkeeper/meta/zk/ZKMetadataBookieDriverTest.java | 2 ++
.../java/org/apache/bookkeeper/meta/zk/ZKMetadataClientDriverTest.java | 2 ++
.../java/org/apache/bookkeeper/meta/zk/ZKMetadataDriverBaseTest.java | 2 ++
.../src/test/java/org/apache/bookkeeper/server/TestEmbeddedServer.java | 2 ++
pom.xml| 3 +--
14 files changed, 25 insertions(+), 6 deletions(-)
diff --git a/.github/workflows/bk-ci.yml b/.github/workflows/bk-ci.yml
index 493c4dc200..c38a2680ac 100644
--- a/.github/workflows/bk-ci.yml
+++ b/.github/workflows/bk-ci.yml
@@ -434,4 +434,4 @@ jobs:
with:
name: dependency report
path: target/dependency-check-report.html
- retention-days: 7
\ No newline at end of file
+ retention-days: 7
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalForceTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalForceTest.java
index 6d0aababef..2ed3a48efb 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalForceTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalForceTest.java
@@ -55,6 +55,7 @@ import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.powermock.reflect.Whitebox;
@@ -63,6 +64,7 @@ import org.powermock.reflect.Whitebox;
* Test the bookie journal.
*/
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore({"javax.xml.*", "org.xml.*", "org.w3c.*", "javax.xml.*",
"com.sun.org.apache.xerces.*"})
@PrepareForTest({JournalChannel.class, Journal.class,
DefaultFileChannel.class})
@Slf4j
public class BookieJournalForceTest {
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalMaxMemoryTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalMaxMemoryTest.java
index 804a7f3fb1..cd9aac2b8b 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalMaxMemoryTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalMaxMemoryTest.java
@@ -38,6 +38,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.powermock.reflect.Whitebox;
@@ -46,6 +47,7 @@ import org.powermock.reflect.Whitebox;
* Test the bookie journal max memory controller.
*/
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore({"javax.xml.*", "org.xml.*", "org.w3c.*",
"com.sun.org.apache.xerces.*"})
@PrepareForTest({JournalChannel.class, Journal.class})
@Slf4j
public class BookieJournalMaxMemoryTest {
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJourn
[bookkeeper] branch master updated: Optimize ReadResponse for small entry sizes (#3597)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 396ec12dc7 Optimize ReadResponse for small entry sizes (#3597)
396ec12dc7 is described below
commit 396ec12dc747fc21105d65f13d2dedace247d1a2
Author: Matteo Merli
AuthorDate: Wed Nov 2 10:42:51 2022 -0700
Optimize ReadResponse for small entry sizes (#3597)
* Optimize ReadResponse for small entry sizes
* Fixed checkstyle
---
.../bookkeeper/proto/BookieProtoEncoding.java | 49 ++
1 file changed, 32 insertions(+), 17 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java
index 70962dfd63..d804a41b0e 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java
@@ -28,7 +28,6 @@ import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.ByteBufInputStream;
import io.netty.buffer.ByteBufOutputStream;
-import io.netty.buffer.Unpooled;
import io.netty.channel.ChannelHandler.Sharable;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
@@ -51,6 +50,13 @@ import org.slf4j.LoggerFactory;
public class BookieProtoEncoding {
private static final Logger LOG =
LoggerFactory.getLogger(BookieProtoEncoding.class);
+/**
+ * Threshold under which an entry is considered to be "small".
+ *
+ * Small entries payloads are copied instead of being passed around as
references.
+ */
+public static final int SMALL_ENTRY_SIZE_THRESHOLD = 16 * 1024;
+
/**
* An encoder/decoder interface for the Bookkeeper protocol.
*/
@@ -246,38 +252,47 @@ public class BookieProtoEncoding {
return msg;
}
BookieProtocol.Response r = (BookieProtocol.Response) msg;
-ByteBuf buf = allocator.buffer(RESPONSE_HEADERS_SIZE + 4 /* frame
size */);
-buf.writerIndex(4); // Leave the placeholder for the frame size
-buf.writeInt(PacketHeader.toInt(r.getProtocolVersion(),
r.getOpCode(), (short) 0));
try {
if (msg instanceof BookieProtocol.ReadResponse) {
+BookieProtocol.ReadResponse rr =
(BookieProtocol.ReadResponse) r;
+int payloadSize = rr.getData().readableBytes();
+int responseSize = RESPONSE_HEADERS_SIZE + payloadSize;
+boolean isSmallEntry = payloadSize <
SMALL_ENTRY_SIZE_THRESHOLD;
+
+int bufferSize = 4 /* frame size */ + RESPONSE_HEADERS_SIZE
++ (isSmallEntry ? payloadSize : 0);
+ByteBuf buf = allocator.buffer(bufferSize);
+buf.writeInt(responseSize);
+buf.writeInt(PacketHeader.toInt(r.getProtocolVersion(),
r.getOpCode(), (short) 0));
buf.writeInt(r.getErrorCode());
buf.writeLong(r.getLedgerId());
buf.writeLong(r.getEntryId());
-BookieProtocol.ReadResponse rr =
(BookieProtocol.ReadResponse) r;
-if (rr.hasData()) {
-int frameSize = RESPONSE_HEADERS_SIZE +
rr.getData().readableBytes();
-buf.setInt(0, frameSize);
-return ByteBufList.get(buf, rr.getData());
-} else {
-buf.setInt(0, RESPONSE_HEADERS_SIZE); // Frame size
+if (isSmallEntry) {
+buf.writeBytes(rr.getData());
return buf;
+} else {
+return ByteBufList.get(buf, rr.getData());
}
} else if (msg instanceof BookieProtocol.AddResponse) {
+ByteBuf buf = allocator.buffer(RESPONSE_HEADERS_SIZE + 4
/* frame size */);
+buf.writeInt(RESPONSE_HEADERS_SIZE);
+buf.writeInt(PacketHeader.toInt(r.getProtocolVersion(),
r.getOpCode(), (short) 0));
buf.writeInt(r.getErrorCode());
buf.writeLong(r.getLedgerId());
buf.writeLong(r.getEntryId());
-buf.setInt(0, RESPONSE_HEADERS_SIZE); // Frame size
-
return buf;
} else if (msg instanceof BookieProtocol.AuthResponse) {
BookkeeperProtocol.AuthMessage am =
((BookieProtocol.AuthResponse) r).getAuthMessage();
-ByteBuf payload = Unpooled.wrappedBuffer(a
[bookkeeper] branch master updated: Allow to use IO uring instead of epoll (#3595)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git The following commit(s) were added to refs/heads/master by this push: new b82e2247e6 Allow to use IO uring instead of epoll (#3595) b82e2247e6 is described below commit b82e2247e67d6a2b3f2f8cb76bc2b0a7c733b5c5 Author: Matteo Merli AuthorDate: Wed Nov 2 10:31:02 2022 -0700 Allow to use IO uring instead of epoll (#3595) * Allow to use IO uring instead of epoll * Removed unused import * Fixed license file * Fixed license file --- bookkeeper-common/pom.xml | 10 .../src/main/resources/LICENSE-all.bin.txt | 3 + .../src/main/resources/LICENSE-bkctl.bin.txt | 3 + .../src/main/resources/LICENSE-server.bin.txt | 3 + .../src/main/resources/NOTICE-all.bin.txt | 2 + .../src/main/resources/NOTICE-bkctl.bin.txt| 2 + .../src/main/resources/NOTICE-server.bin.txt | 2 + .../apache/bookkeeper/proto/BookieNettyServer.java | 8 ++- .../bookkeeper/proto/PerChannelBookieClient.java | 12 +++- .../org/apache/bookkeeper/util/EventLoopUtil.java | 64 ++ pom.xml| 18 ++ 11 files changed, 101 insertions(+), 26 deletions(-) diff --git a/bookkeeper-common/pom.xml b/bookkeeper-common/pom.xml index 29d77bccde..7526bf02e3 100644 --- a/bookkeeper-common/pom.xml +++ b/bookkeeper-common/pom.xml @@ -59,6 +59,16 @@ org.jctools jctools-core + + io.netty.incubator + netty-incubator-transport-native-io_uring + linux-x86_64 + + + io.netty.incubator + netty-incubator-transport-native-io_uring + linux-aarch_64 + com.google.code.findbugs jsr305 diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index 1e59674231..2ab4db7601 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -238,6 +238,9 @@ Apache Software License, Version 2. - lib/io.netty-netty-transport-4.1.81.Final.jar [11] - lib/io.netty-netty-transport-classes-epoll-4.1.81.Final.jar [11] - lib/io.netty-netty-transport-native-epoll-4.1.81.Final-linux-x86_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.15.Final-linux-x86_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.15.Final-linux-aarch_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.15.Final.jar [11] - lib/io.netty-netty-transport-native-unix-common-4.1.81.Final.jar [11] - lib/io.prometheus-simpleclient-0.15.0.jar [12] - lib/io.prometheus-simpleclient_common-0.15.0.jar [12] diff --git a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt index f2a563691f..439115c3dd 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt @@ -236,6 +236,9 @@ Apache Software License, Version 2. - lib/io.netty-netty-transport-4.1.81.Final.jar [11] - lib/io.netty-netty-transport-classes-epoll-4.1.81.Final.jar [11] - lib/io.netty-netty-transport-native-epoll-4.1.81.Final-linux-x86_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.15.Final-linux-x86_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.15.Final-linux-aarch_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.15.Final.jar [11] - lib/io.netty-netty-transport-native-unix-common-4.1.81.Final.jar [11] - lib/org.apache.logging.log4j-log4j-api-2.18.0.jar [16] - lib/org.apache.logging.log4j-log4j-core-2.18.0.jar [16] diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index b8ffed69e0..5d69c3c2df 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -238,6 +238,9 @@ Apache Software License, Version 2. - lib/io.netty-netty-transport-4.1.81.Final.jar [11] - lib/io.netty-netty-transport-classes-epoll-4.1.81.Final.jar [11] - lib/io.netty-netty-transport-native-epoll-4.1.81.Final-linux-x86_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.15.Final-linux-x86_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.15.Final-linux-aarch_64.jar [11] +- lib/io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.15.Final.jar [11] - lib/io.netty-netty-transport-native-unix-common-4.1.81.Final.jar [11] - lib/io.prometheus-simpleclient-0.15.0.jar [12] - lib/io.prometheus-simpleclient_common-0.15.0.jar
[bookkeeper] branch master updated: Add ByteBuf refCnt test for AddEntry and use pooled ByteBuf reduce heap usage (#3598)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 9843ad62c9 Add ByteBuf refCnt test for AddEntry and use pooled ByteBuf
reduce heap usage (#3598)
9843ad62c9 is described below
commit 9843ad62c95cc6d975da8a0be9bccc8e2850bf3f
Author: wenbingshen
AuthorDate: Wed Nov 2 23:40:58 2022 +0800
Add ByteBuf refCnt test for AddEntry and use pooled ByteBuf reduce heap
usage (#3598)
* Fix ByteBuf memory leak problem when addEntryInternal
* fix checkstyle
* use pooled ByteBuf
---
.../org/apache/bookkeeper/bookie/BookieImpl.java | 31 +---
.../java/org/apache/bookkeeper/bookie/Journal.java | 5 --
.../apache/bookkeeper/bookie/BookieImplTest.java | 83 ++
3 files changed, 103 insertions(+), 16 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java
index 6b8e482256..5822fecc10 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java
@@ -931,6 +931,17 @@ public class BookieImpl extends BookieCriticalThread
implements Bookie {
return journals.get(MathUtils.signSafeMod(ledgerId, journals.size()));
}
+@VisibleForTesting
+public ByteBuf createMasterKeyEntry(long ledgerId, byte[] masterKey) {
+// new handle, we should add the key to journal ensure we can rebuild
+ByteBuf bb = allocator.directBuffer(8 + 8 + 4 + masterKey.length);
+bb.writeLong(ledgerId);
+bb.writeLong(METAENTRY_ID_LEDGER_KEY);
+bb.writeInt(masterKey.length);
+bb.writeBytes(masterKey);
+return bb;
+}
+
/**
* Add an entry to a ledger as specified by handle.
*/
@@ -948,15 +959,13 @@ public class BookieImpl extends BookieCriticalThread
implements Bookie {
// Force the load into masterKey cache
byte[] oldValue = masterKeyCache.putIfAbsent(ledgerId, masterKey);
if (oldValue == null) {
-// new handle, we should add the key to journal ensure we can
rebuild
-ByteBuffer bb = ByteBuffer.allocate(8 + 8 + 4 +
masterKey.length);
-bb.putLong(ledgerId);
-bb.putLong(METAENTRY_ID_LEDGER_KEY);
-bb.putInt(masterKey.length);
-bb.put(masterKey);
-bb.flip();
-
-getJournal(ledgerId).logAddEntry(bb, false /* ackBeforeSync
*/, new NopWriteCallback(), null);
+ByteBuf masterKeyEntry = createMasterKeyEntry(ledgerId,
masterKey);
+try {
+getJournal(ledgerId).logAddEntry(
+masterKeyEntry, false /* ackBeforeSync */, new
NopWriteCallback(), null);
+} finally {
+ReferenceCountUtil.safeRelease(masterKeyEntry);
+}
}
}
@@ -1002,7 +1011,7 @@ public class BookieImpl extends BookieCriticalThread
implements Bookie {
bookieStats.getAddBytesStats().registerFailedValue(entrySize);
}
-entry.release();
+ReferenceCountUtil.safeRelease(entry);
}
}
@@ -1096,7 +1105,7 @@ public class BookieImpl extends BookieCriticalThread
implements Bookie {
bookieStats.getAddBytesStats().registerFailedValue(entrySize);
}
-entry.release();
+ReferenceCountUtil.safeRelease(entry);
}
}
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
index 93c58277b9..1a986707a8 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
@@ -960,11 +960,6 @@ public class Journal extends BookieCriticalThread
implements CheckpointSource {
}
}
-public void logAddEntry(ByteBuffer entry, boolean ackBeforeSync,
WriteCallback cb, Object ctx)
-throws InterruptedException {
-logAddEntry(Unpooled.wrappedBuffer(entry), ackBeforeSync, cb, ctx);
-}
-
/**
* record an add entry operation in journal.
*/
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieImplTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieImplTest.java
index 5cd1f49c42..a93b5ac3a0 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieImplTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieImplTest.java
@@ -21,6 +21,7
[bookkeeper] branch master updated (96b2446b35 -> 02be9d2ee2)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git from 96b2446b35 fix memtable getEntryStats typo (#3592) add 02be9d2ee2 Avoid extra buffer to prepend frame size (#3560) No new revisions were added by this update. Summary of changes: .../apache/bookkeeper/proto/BookieNettyServer.java | 6 +--- .../bookkeeper/proto/BookieProtoEncoding.java | 42 -- .../apache/bookkeeper/proto/BookieProtocol.java| 9 +++-- .../bookkeeper/proto/PerChannelBookieClient.java | 6 ++-- .../org/apache/bookkeeper/util/ByteBufList.java| 24 + .../bookkeeper/proto/BookieProtoEncodingTest.java | 6 ++-- 6 files changed, 44 insertions(+), 49 deletions(-)
[bookkeeper] branch master updated: fix memtable getEntryStats typo (#3592)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 96b2446b35 fix memtable getEntryStats typo (#3592)
96b2446b35 is described below
commit 96b2446b3521f83932239df64d546819519df566
Author: Hang Chen
AuthorDate: Tue Nov 1 05:42:30 2022 +0800
fix memtable getEntryStats typo (#3592)
---
.../java/org/apache/bookkeeper/bookie/stats/EntryMemTableStats.java| 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/EntryMemTableStats.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/EntryMemTableStats.java
index 62840f9c13..35419ae9f3 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/EntryMemTableStats.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/EntryMemTableStats.java
@@ -20,6 +20,7 @@
package org.apache.bookkeeper.bookie.stats;
import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.BOOKIE_ADD_ENTRY;
+import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.BOOKIE_READ_ENTRY;
import static org.apache.bookkeeper.bookie.BookKeeperServerStats.BOOKIE_SCOPE;
import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.CATEGORY_SERVER;
import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.SKIP_LIST_FLUSH_BYTES;
@@ -60,7 +61,7 @@ public class EntryMemTableStats {
@StatsDoc(
name = SKIP_LIST_GET_ENTRY,
help = "operation stats of getting entries from memtable",
-parent = BOOKIE_ADD_ENTRY
+parent = BOOKIE_READ_ENTRY
)
private final OpStatsLogger getEntryStats;
@StatsDoc(
[bookkeeper] branch master updated: Correct link class name (#3594)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 9efd606d1c Correct link class name (#3594)
9efd606d1c is described below
commit 9efd606d1cf1a2f4573606335490548f59b083c2
Author: gaozhangmin
AuthorDate: Tue Nov 1 05:01:37 2022 +0800
Correct link class name (#3594)
Co-authored-by: gavingaozhangmin
---
.../org/apache/bookkeeper/meta/LedgerManager.java | 27 +-
1 file changed, 16 insertions(+), 11 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/LedgerManager.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/LedgerManager.java
index a76e244997..20d61066cd 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/LedgerManager.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/LedgerManager.java
@@ -48,9 +48,10 @@ public interface LedgerManager extends Closeable {
*Metadata provided when creating the new ledger
* @return Future which, when completed returns the metadata of the newly
created ledger.
* Completed with an exception:
- *{@link BKException.BKLedgerExistException} if given
ledger id exist
- *{@link BKException.BKZKException}/{@link
BKException.BKMetaStoreException} for other issues
- *
+ *{@link
org.apache.bookkeeper.client.BKException.BKLedgerExistException} if given
ledger id exist
+ *{@link org.apache.bookkeeper.client.BKException.ZKException}
+ */{@link
org.apache.bookkeeper.client.BKException.BKMetadataSerializationException} for
other issues
+ *
*/
CompletableFuture> createLedgerMetadata(long
ledgerId, LedgerMetadata metadata);
@@ -63,10 +64,12 @@ public interface LedgerManager extends Closeable {
* Ledger metadata version
* @return Future which, when completed, denotes that the ledger metadata
has been removed.
* Completed with an exception:
- * {@link BKException.BKMetadataVersionException} if version
doesn't match
- * {@link
BKException.BKNoSuchLedgerExistsOnMetadataServerException} if ledger not
exist
- * {@link BKException.ZKException} for other issues
- *
+ * {@link
org.apache.bookkeeper.client.BKException.BKMetadataVersionException}
+ * if version doesn't match
+ * {@link
org.apache.bookkeeper.client.BKException.BKNoSuchLedgerExistsOnMetadataServerException}
+ * if ledger not exist
+ * {@link org.apache.bookkeeper.client.BKException.ZKException}
for other issues
+ *
*/
CompletableFuture removeLedgerMetadata(long ledgerId, Version
version);
@@ -77,8 +80,9 @@ public interface LedgerManager extends Closeable {
* Ledger Id
* @return Future which, when completed, contains the requested versioned
metadata.
* Completed with an exception::
- * {@link
BKException.BKNoSuchLedgerExistsOnMetadataServerException} if ledger not
exist
- * {@link BKException.ZKException} for other issues
+ * {@link
org.apache.bookkeeper.client.BKException.BKNoSuchLedgerExistsOnMetadataServerException}
+ * if ledger not exist
+ * {@link
org.apache.bookkeeper.client.BKException.ZKException} for other issues
*
*/
CompletableFuture> readLedgerMetadata(long
ledgerId);
@@ -94,8 +98,9 @@ public interface LedgerManager extends Closeable {
* The version of the metadata we expect to be overwriting.
* @return Future which, when completed, contains the newly written
metadata.
* Comleted with an exceptione:
- * {@link BKException.BKMetadataVersionException} if version
in metadata doesn't match
- * {@link BKException.ZKException} for other issue
+ * {@link
org.apache.bookkeeper.client.BKException.BKMetadataVersionException}
+ * if version in metadata doesn't match
+ * {@link
org.apache.bookkeeper.client.BKException.ZKException} for other issue
*
*/
CompletableFuture> writeLedgerMetadata(long
ledgerId, LedgerMetadata metadata,
[bookkeeper] branch master updated: add javadoc comments for test classes (#3587)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 75a293b77c add javadoc comments for test classes (#3587)
75a293b77c is described below
commit 75a293b77ccc878036318a8a2d50ac8f9ed9dcc5
Author: Hang Chen
AuthorDate: Tue Nov 1 05:00:10 2022 +0800
add javadoc comments for test classes (#3587)
---
.../org/apache/bookkeeper/client/BookieRecoveryUseIOThreadTest.java| 3 +++
.../apache/bookkeeper/client/api/BookKeeperBuildersOpenLedgerTest.java | 3 +++
2 files changed, 6 insertions(+)
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieRecoveryUseIOThreadTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieRecoveryUseIOThreadTest.java
index fa1dba0cda..a9d7a0ef50 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieRecoveryUseIOThreadTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieRecoveryUseIOThreadTest.java
@@ -26,6 +26,9 @@ import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
import org.junit.Assert;
import org.junit.Test;
+/**
+ * Tests for Bookie recovery use IO threads.
+ */
public class BookieRecoveryUseIOThreadTest extends BookKeeperClusterTestCase {
public BookieRecoveryUseIOThreadTest() {
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/api/BookKeeperBuildersOpenLedgerTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/api/BookKeeperBuildersOpenLedgerTest.java
index 46821cc353..00e14fe2c1 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/api/BookKeeperBuildersOpenLedgerTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/api/BookKeeperBuildersOpenLedgerTest.java
@@ -42,6 +42,9 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+/**
+ * Tests for BookKeeper open ledger operations.
+ */
@RunWith(Parameterized.class)
public class BookKeeperBuildersOpenLedgerTest extends MockBookKeeperTestCase {
[bookkeeper] branch master updated (a58f11ff13 -> ec79daa6c7)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
from a58f11ff13 Change masterKeyCache to dynamic size (#3522)
add ec79daa6c7 Single thread executor should not have max capacity by
default (#3575)
No new revisions were added by this update.
Summary of changes:
...=> GrowableMpScArrayConsumerBlockingQueue.java} | 208 +
.../common/util/SingleThreadExecutor.java | 13 +-
.../GrowableArrayBlockingQueueTest.java| 76 +++-
.../common/util/TestSingleThreadExecutor.java | 29 +++
4 files changed, 201 insertions(+), 125 deletions(-)
rename
bookkeeper-common/src/main/java/org/apache/bookkeeper/common/collections/{GrowableArrayBlockingQueue.java
=> GrowableMpScArrayConsumerBlockingQueue.java} (57%)
[bookkeeper] branch master updated: Use SingleThreadExecutor for OrderedExecutor and drainTo() tasks into local array (#3546)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new dff6d03a59 Use SingleThreadExecutor for OrderedExecutor and drainTo()
tasks into local array (#3546)
dff6d03a59 is described below
commit dff6d03a59b293019a3c1144515e6f931986f82f
Author: Matteo Merli
AuthorDate: Thu Oct 20 13:24:21 2022 -0700
Use SingleThreadExecutor for OrderedExecutor and drainTo() tasks into local
array (#3546)
* Use SingleThreadExecutor for OrderedExecutor and drainTo() tasks into
local array
* Added metrics to executor
* Fixed checkstyle
* Made the test more resilient
* Made the tests not relying on thread.sleep()
* Fixed testRejectWhenQueueIsFull
* Ignore spotbugs warning
* Fixed annotation formatting
* Removed test assertion that had already been changed
---
bookkeeper-common/pom.xml | 5 +
.../bookkeeper/common/util/OrderedExecutor.java| 107
.../bookkeeper/common/util/OrderedScheduler.java | 10 +-
.../common/util/SingleThreadExecutor.java | 296 +
.../common/util/TestSingleThreadExecutor.java | 290
5 files changed, 652 insertions(+), 56 deletions(-)
diff --git a/bookkeeper-common/pom.xml b/bookkeeper-common/pom.xml
index 62aad8735e..29d77bccde 100644
--- a/bookkeeper-common/pom.xml
+++ b/bookkeeper-common/pom.xml
@@ -85,6 +85,11 @@
commons-lang3
test
+
+ org.awaitility
+ awaitility
+ test
+
diff --git
a/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/util/OrderedExecutor.java
b/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/util/OrderedExecutor.java
index 9ee84ba9ed..40c3fb0283 100644
---
a/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/util/OrderedExecutor.java
+++
b/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/util/OrderedExecutor.java
@@ -29,19 +29,16 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Random;
-import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
-import org.apache.bookkeeper.common.collections.BlockingMpscQueue;
import org.apache.bookkeeper.common.util.affinity.CpuAffinity;
import org.apache.bookkeeper.stats.Gauge;
import org.apache.bookkeeper.stats.NullStatsLogger;
@@ -294,20 +291,17 @@ public class OrderedExecutor implements ExecutorService {
}
}
-protected ThreadPoolExecutor createSingleThreadExecutor(ThreadFactory
factory) {
-BlockingQueue queue;
-if (enableBusyWait) {
-// Use queue with busy-wait polling strategy
-queue = new BlockingMpscQueue<>(maxTasksInQueue > 0 ?
maxTasksInQueue : DEFAULT_MAX_ARRAY_QUEUE_SIZE);
+protected ExecutorService createSingleThreadExecutor(ThreadFactory
factory) {
+if (maxTasksInQueue > 0) {
+return new SingleThreadExecutor(factory, maxTasksInQueue, true);
} else {
-// By default, use regular JDK LinkedBlockingQueue
-queue = new LinkedBlockingQueue<>();
+return new SingleThreadExecutor(factory);
}
-return new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, queue,
factory);
}
-protected ExecutorService getBoundedExecutor(ThreadPoolExecutor executor) {
-return new BoundedExecutorService(executor, this.maxTasksInQueue);
+protected ExecutorService getBoundedExecutor(ExecutorService executor) {
+checkArgument(executor instanceof ThreadPoolExecutor);
+return new BoundedExecutorService((ThreadPoolExecutor) executor,
this.maxTasksInQueue);
}
protected ExecutorService addExecutorDecorators(ExecutorService executor) {
@@ -400,11 +394,14 @@ public class OrderedExecutor implements ExecutorService {
threads = new ExecutorService[numThreads];
threadIds = new long[numThreads];
for (int i = 0; i < numThreads; i++) {
-ThreadPoolExecutor thread = createSingleThreadExecutor(
+ExecutorService thread = createSingleThreadExecutor(
new ThreadFactoryBuilder().setNameFormat(name + "-" +
getClass().getSimpleName() + "-" + i + "-%d")
.setThreadFactor
[bookkeeper] branch branch-4.16 updated: Use BlockingQueue.drainTo() in JournalForceWrite thread (#3545)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch branch-4.16
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/branch-4.16 by this push:
new 2a7fae2090 Use BlockingQueue.drainTo() in JournalForceWrite thread
(#3545)
2a7fae2090 is described below
commit 2a7fae20900626a5f841239d95f5ad1035e7cd8b
Author: Matteo Merli
AuthorDate: Tue Oct 18 17:03:26 2022 -0700
Use BlockingQueue.drainTo() in JournalForceWrite thread (#3545)
---
.../java/org/apache/bookkeeper/bookie/Journal.java | 149 -
1 file changed, 86 insertions(+), 63 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
index 69a0ac8566..a3e0086b69 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
@@ -487,6 +487,11 @@ public class Journal extends BookieCriticalThread
implements CheckpointSource {
// should we group force writes
private final boolean enableGroupForceWrites;
private final Counter forceWriteThreadTime;
+
+boolean shouldForceWrite = true;
+int numReqInLastForceWrite = 0;
+boolean forceWriteMarkerSent = false;
+
public ForceWriteThread(Thread threadToNotifyOnEx,
boolean enableGroupForceWrites,
StatsLogger statsLogger) {
@@ -508,90 +513,108 @@ public class Journal extends BookieCriticalThread
implements CheckpointSource {
}
}
-boolean shouldForceWrite = true;
-int numReqInLastForceWrite = 0;
long busyStartTime = System.nanoTime();
-boolean forceWriteMarkerSent = false;
+
+List localRequests = new ArrayList<>();
+
while (running) {
-ForceWriteRequest req = null;
try {
-
forceWriteThreadTime.addLatency(MathUtils.elapsedNanos(busyStartTime),
TimeUnit.NANOSECONDS);
-req = forceWriteRequests.take();
-busyStartTime = System.nanoTime();
-// Force write the file and then notify the write
completions
-//
-if (!req.isMarker) {
-if (shouldForceWrite) {
-// if we are going to force write, any request
that is already in the
-// queue will benefit from this force write - post
a marker prior to issuing
-// the flush so until this marker is encountered
we can skip the force write
-if (enableGroupForceWrites) {
-ForceWriteRequest marker =
-createForceWriteRequest(req.logFile, 0, 0,
null, false, true);
-forceWriteMarkerSent =
forceWriteRequests.offer(marker);
-if (!forceWriteMarkerSent) {
-marker.recycle();
-Counter failures =
journalStats.getForceWriteGroupingFailures();
-failures.inc();
-LOG.error(
-"Fail to send force write grouping
marker,"
-+ " Journal.forceWriteRequests
queue(capacity {}) is full,"
-+ " current failure counter is {}.",
-conf.getJournalQueueSize(),
failures.get());
-}
-}
-// If we are about to issue a write, record the
number of requests in
-// the last force write and then reset the counter
so we can accumulate
-// requests in the write we are about to issue
-if (numReqInLastForceWrite > 0) {
-journalStats.getForceWriteGroupingCountStats()
-
.registerSuccessfulValue(numReqInLastForceWrite);
-numReqInLastForceWrite = 0;
-}
-}
+int requestsCount =
forceWriteRequests.drainTo(localRequests);
+if (requestsCount == 0) {
+ForceWriteRequest fwr = forceWriteRequests.take();
+localRequests.add(fwr);
+requestsCount = 1;
}
-numReqInLastForceWrite += req.process(shouldForceWrite);
-
-
[bookkeeper] branch master updated: Use BlockingQueue.drainTo() in JournalForceWrite thread (#3545)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new b63cca8bbb Use BlockingQueue.drainTo() in JournalForceWrite thread
(#3545)
b63cca8bbb is described below
commit b63cca8bbb0208afa235abfdfc5f8dafd9eda2dd
Author: Matteo Merli
AuthorDate: Tue Oct 18 17:03:26 2022 -0700
Use BlockingQueue.drainTo() in JournalForceWrite thread (#3545)
---
.../java/org/apache/bookkeeper/bookie/Journal.java | 149 -
1 file changed, 86 insertions(+), 63 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
index 69a0ac8566..a3e0086b69 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java
@@ -487,6 +487,11 @@ public class Journal extends BookieCriticalThread
implements CheckpointSource {
// should we group force writes
private final boolean enableGroupForceWrites;
private final Counter forceWriteThreadTime;
+
+boolean shouldForceWrite = true;
+int numReqInLastForceWrite = 0;
+boolean forceWriteMarkerSent = false;
+
public ForceWriteThread(Thread threadToNotifyOnEx,
boolean enableGroupForceWrites,
StatsLogger statsLogger) {
@@ -508,90 +513,108 @@ public class Journal extends BookieCriticalThread
implements CheckpointSource {
}
}
-boolean shouldForceWrite = true;
-int numReqInLastForceWrite = 0;
long busyStartTime = System.nanoTime();
-boolean forceWriteMarkerSent = false;
+
+List localRequests = new ArrayList<>();
+
while (running) {
-ForceWriteRequest req = null;
try {
-
forceWriteThreadTime.addLatency(MathUtils.elapsedNanos(busyStartTime),
TimeUnit.NANOSECONDS);
-req = forceWriteRequests.take();
-busyStartTime = System.nanoTime();
-// Force write the file and then notify the write
completions
-//
-if (!req.isMarker) {
-if (shouldForceWrite) {
-// if we are going to force write, any request
that is already in the
-// queue will benefit from this force write - post
a marker prior to issuing
-// the flush so until this marker is encountered
we can skip the force write
-if (enableGroupForceWrites) {
-ForceWriteRequest marker =
-createForceWriteRequest(req.logFile, 0, 0,
null, false, true);
-forceWriteMarkerSent =
forceWriteRequests.offer(marker);
-if (!forceWriteMarkerSent) {
-marker.recycle();
-Counter failures =
journalStats.getForceWriteGroupingFailures();
-failures.inc();
-LOG.error(
-"Fail to send force write grouping
marker,"
-+ " Journal.forceWriteRequests
queue(capacity {}) is full,"
-+ " current failure counter is {}.",
-conf.getJournalQueueSize(),
failures.get());
-}
-}
-// If we are about to issue a write, record the
number of requests in
-// the last force write and then reset the counter
so we can accumulate
-// requests in the write we are about to issue
-if (numReqInLastForceWrite > 0) {
-journalStats.getForceWriteGroupingCountStats()
-
.registerSuccessfulValue(numReqInLastForceWrite);
-numReqInLastForceWrite = 0;
-}
-}
+int requestsCount =
forceWriteRequests.drainTo(localRequests);
+if (requestsCount == 0) {
+ForceWriteRequest fwr = forceWriteRequests.take();
+localRequests.add(fwr);
+requestsCount = 1;
}
-numReqInLastForceWrite += req.process(shouldForceWrite);
-
-
[bookkeeper] branch master updated: Speed up the rebuildinding of RocksDB index (#3458)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 7004d99493 Speed up the rebuildinding of RocksDB index (#3458)
7004d99493 is described below
commit 7004d994938e914561d8343a96d9d75ce98b2837
Author: Matteo Merli
AuthorDate: Mon Sep 19 08:59:57 2022 -0700
Speed up the rebuildinding of RocksDB index (#3458)
* Speed up the rebuildinding of RocksDB index
* fix check style
Co-authored-by: chenhang
---
.../bookie/storage/ldb/LocationsIndexRebuildOp.java | 19 ++-
1 file changed, 18 insertions(+), 1 deletion(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LocationsIndexRebuildOp.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LocationsIndexRebuildOp.java
index 4edbc2c3e9..37de152fb6 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LocationsIndexRebuildOp.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LocationsIndexRebuildOp.java
@@ -31,6 +31,8 @@ import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
import org.apache.bookkeeper.bookie.BookieImpl;
import org.apache.bookkeeper.bookie.DefaultEntryLogger;
import org.apache.bookkeeper.bookie.LedgerDirsManager;
@@ -53,6 +55,8 @@ public class LocationsIndexRebuildOp {
this.conf = conf;
}
+private static final int BATCH_COMMIT_SIZE = 10_000;
+
public void initiate() throws IOException {
LOG.info("Starting locations index rebuilding");
File[] indexDirs = conf.getIndexDirs();
@@ -90,6 +94,8 @@ public class LocationsIndexRebuildOp {
int totalEntryLogs = entryLogs.size();
int completedEntryLogs = 0;
LOG.info("Scanning {} entry logs", totalEntryLogs);
+AtomicReference batch = new
AtomicReference<>(newIndex.newBatch());
+AtomicInteger count = new AtomicInteger();
for (long entryLogId : entryLogs) {
entryLogger.scanEntryLog(entryLogId, new EntryLogScanner() {
@@ -108,7 +114,15 @@ public class LocationsIndexRebuildOp {
// Update the ledger index page
LongPairWrapper key = LongPairWrapper.get(ledgerId,
entryId);
LongWrapper value = LongWrapper.get(location);
-newIndex.put(key.array, value.array);
+batch.get().put(key.array, value.array);
+
+if (count.incrementAndGet() > BATCH_COMMIT_SIZE) {
+batch.get().flush();
+batch.get().close();
+
+batch.set(newIndex.newBatch());
+count.set(0);
+}
}
@Override
@@ -122,6 +136,9 @@ public class LocationsIndexRebuildOp {
completedEntryLogs, totalEntryLogs);
}
+batch.get().flush();
+batch.get().close();
+
newIndex.sync();
newIndex.close();
}
[bookkeeper] branch master updated (aba568087c -> 2b1a24543b)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git from aba568087c upgrade hadoop version to 3.2.4 (#3485) add 2b1a24543b Add 4.15.1 to versions file (#3475) No new revisions were added by this update. Summary of changes: .../version-4.15.0/admin/autorecovery.md | 130 .../versioned_docs/version-4.15.0/admin/bookies.md | 174 - .../version-4.15.0/admin/decomission.md| 42 - .../version-4.15.0/admin/geo-replication.md| 22 - .../versioned_docs/version-4.15.0/admin/http.md| 510 - .../versioned_docs/version-4.15.0/admin/metrics.md | 37 - .../versioned_docs/version-4.15.0/admin/perf.md| 4 - .../version-4.15.0/admin/placement.md | 4 - .../versioned_docs/version-4.15.0/admin/upgrade.md | 176 - .../version-4.15.0/api/distributedlog-api.md | 86 --- .../version-4.15.0/api/ledger-adv-api.md | 112 --- .../version-4.15.0/api/ledger-api.md | 841 - .../versioned_docs/version-4.15.0/api/overview.md | 18 - .../version-4.15.0/deployment/kubernetes.md| 180 - .../version-4.15.0/deployment/manual.md| 65 -- .../version-4.15.0/development/codebase.md | 4 - .../version-4.15.0/development/protocol.md | 149 .../version-4.15.0/getting-started/concepts.md | 205 - .../version-4.15.0/getting-started/installation.md | 50 -- .../version-4.15.0/getting-started/run-locally.md | 14 - .../version-4.15.0/overview/overview.md| 61 -- .../versioned_docs/version-4.15.0/reference/cli.md | 618 --- .../version-4.15.0/reference/config.md | 357 - .../version-4.15.0/security/overview.md| 21 - .../versioned_docs/version-4.15.0/security/sasl.md | 201 - .../versioned_docs/version-4.15.0/security/tls.md | 211 -- .../version-4.15.0/security/zookeeper.md | 41 - .../versioned_docs/version-4.15.1/admin/bookies.md | 2 +- .../version-4.15.1/api/distributedlog-api.md | 2 +- .../version-4.15.1/getting-started/concepts.md | 2 +- .../version-4.15.1/getting-started/installation.md | 2 +- .../versioned_docs/version-4.15.1/security/tls.md | 2 +- .../version-4.15.0-sidebars.json | 161 site3/website/versions.json| 2 +- 34 files changed, 6 insertions(+), 4500 deletions(-) delete mode 100644 site3/website/versioned_docs/version-4.15.0/admin/autorecovery.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/admin/bookies.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/admin/decomission.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/admin/geo-replication.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/admin/http.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/admin/metrics.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/admin/perf.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/admin/placement.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/admin/upgrade.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/api/distributedlog-api.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/api/ledger-adv-api.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/api/ledger-api.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/api/overview.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/deployment/kubernetes.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/deployment/manual.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/development/codebase.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/development/protocol.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/getting-started/concepts.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/getting-started/installation.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/getting-started/run-locally.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/overview/overview.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/reference/cli.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/reference/config.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/security/overview.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/security/sasl.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/security/tls.md delete mode 100644 site3/website/versioned_docs/version-4.15.0/security/zookeeper.md delete mode 100644 site3/website/versioned_sidebars/version-4.15.0-sidebars.json
[bookkeeper] branch master updated: upgrade hadoop version to 3.2.4 (#3485)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git The following commit(s) were added to refs/heads/master by this push: new aba568087c upgrade hadoop version to 3.2.4 (#3485) aba568087c is described below commit aba568087c6dff99a343346107153ed1d979677b Author: Hang Chen AuthorDate: Tue Sep 13 11:47:37 2022 +0800 upgrade hadoop version to 3.2.4 (#3485) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 17a82c0dc6..135152e1da 100644 --- a/pom.xml +++ b/pom.xml @@ -135,7 +135,7 @@ 1.47.0 31.0.1-jre 1.1.1 -2.10.0 +3.2.4 1.3 2.1.10 2.13.2.20220328
[bookkeeper] branch master updated: use LinkedList to take place of ArrayList. (#3330)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 93a6f67029 use LinkedList to take place of ArrayList. (#3330)
93a6f67029 is described below
commit 93a6f670296ebe5e86af22bfb632eeb50825c5e7
Author: Yan Zhao
AuthorDate: Sat Jul 23 01:01:40 2022 +0800
use LinkedList to take place of ArrayList. (#3330)
---
.../org/apache/bookkeeper/client/ListenerBasedPendingReadOp.java| 4 ++--
.../src/main/java/org/apache/bookkeeper/client/PendingReadOp.java | 6 +++---
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/ListenerBasedPendingReadOp.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/ListenerBasedPendingReadOp.java
index 108a805ca9..6733b2e9ea 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/ListenerBasedPendingReadOp.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/ListenerBasedPendingReadOp.java
@@ -46,11 +46,11 @@ class ListenerBasedPendingReadOp extends PendingReadOp {
@Override
protected void submitCallback(int code) {
LedgerEntryRequest request;
-while (!seq.isEmpty() && (request = seq.get(0)) != null) {
+while (!seq.isEmpty() && (request = seq.getFirst()) != null) {
if (!request.isComplete()) {
return;
}
-seq.remove(0);
+seq.removeFirst();
long latencyNanos = MathUtils.elapsedNanos(requestTimeNanos);
LedgerEntry entry;
if (BKException.Code.OK == request.getRc()) {
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingReadOp.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingReadOp.java
index 60f76f8eed..476c44e72f 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingReadOp.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingReadOp.java
@@ -23,9 +23,9 @@ package org.apache.bookkeeper.client;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.ListenableFuture;
import io.netty.buffer.ByteBuf;
-import java.util.ArrayList;
import java.util.BitSet;
import java.util.HashSet;
+import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
@@ -59,7 +59,7 @@ class PendingReadOp implements ReadEntryCallback,
SafeRunnable {
private static final Logger LOG =
LoggerFactory.getLogger(PendingReadOp.class);
private ScheduledFuture speculativeTask = null;
-protected final List seq;
+protected final LinkedList seq;
private final CompletableFuture future;
private final Set heardFromHosts;
private final BitSet heardFromHostsBitSet;
@@ -463,7 +463,7 @@ class PendingReadOp implements ReadEntryCallback,
SafeRunnable {
long startEntryId,
long endEntryId,
boolean isRecoveryRead) {
-this.seq = new ArrayList<>((int) ((endEntryId + 1) - startEntryId));
+this.seq = new LinkedList<>();
this.future = new CompletableFuture<>();
this.lh = lh;
this.clientCtx = clientCtx;
[bookkeeper] branch master updated: Added flag to control whether to transition to read-only mode when any disks full (#3212)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 34746858b4 Added flag to control whether to transition to read-only
mode when any disks full (#3212)
34746858b4 is described below
commit 34746858b47e3962914f646803e930afe77859a3
Author: Hang Chen
AuthorDate: Sat Jul 23 00:59:42 2022 +0800
Added flag to control whether to transition to read-only mode when any
disks full (#3212)
* Added flag to control whether to transition to read-only mode when any
disk is full
* add unit test
* address comments
* add docs and config in conf/bk_server.conf
---
.../org/apache/bookkeeper/bookie/BookieImpl.java | 21 +
.../bookkeeper/bookie/LedgerDirsManager.java | 13
.../bookkeeper/bookie/LedgerDirsMonitor.java | 26 ++-
.../bookkeeper/conf/ServerConfiguration.java | 22 ++
.../bookkeeper/bookie/LedgerDirsManagerTest.java | 90 ++
conf/bk_server.conf| 9 ++-
site3/website/docs/reference/config.md | 6 +-
7 files changed, 181 insertions(+), 6 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java
index 9bc084ee92..4dc717e9bb 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java
@@ -741,6 +741,9 @@ public class BookieImpl extends BookieCriticalThread
implements Bookie {
@Override
public void diskWritable(File disk) {
+if (conf.isReadOnlyModeOnAnyDiskFullEnabled()) {
+return;
+}
// Transition to writable mode when a disk becomes writable
again.
stateManager.setHighPriorityWritesAvailability(true);
stateManager.transitionToWritableMode();
@@ -748,6 +751,24 @@ public class BookieImpl extends BookieCriticalThread
implements Bookie {
@Override
public void diskJustWritable(File disk) {
+if (conf.isReadOnlyModeOnAnyDiskFullEnabled()) {
+return;
+}
+// Transition to writable mode when a disk becomes writable
again.
+stateManager.setHighPriorityWritesAvailability(true);
+stateManager.transitionToWritableMode();
+}
+
+@Override
+public void anyDiskFull(boolean highPriorityWritesAllowed) {
+if (conf.isReadOnlyModeOnAnyDiskFullEnabled()) {
+
stateManager.setHighPriorityWritesAvailability(highPriorityWritesAllowed);
+stateManager.transitionToReadOnlyMode();
+}
+}
+
+@Override
+public void allDisksWritable() {
// Transition to writable mode when a disk becomes writable
again.
stateManager.setHighPriorityWritesAvailability(true);
stateManager.transitionToWritableMode();
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java
index a9d438e4b7..b9934a2ba7 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java
@@ -434,6 +434,19 @@ public class LedgerDirsManager {
*/
default void allDisksFull(boolean highPriorityWritesAllowed) {}
+/**
+ * This will be notified whenever all disks are detected as not full.
+ *
+ */
+default void allDisksWritable() {}
+
+/**
+ * This will be notified whenever any disks are detected as full.
+ *
+ * @param highPriorityWritesAllowed the parameter indicates we are
still have disk spaces for high priority
+ * * writes even disks are
detected as "full"
+ */
+default void anyDiskFull(boolean highPriorityWritesAllowed) {}
/**
* This will notify the fatal errors.
*/
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java
index 2b7c90152d..f7c2dc31ba 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java
@@ -52,7 +52,7 @@ class LedgerDirsMonitor {
pri
[bookkeeper] branch branch-4.15 updated: Switch back ordered executor to LinkedBlockingQueue (#3384)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/branch-4.15 by this push:
new e536d23947 Switch back ordered executor to LinkedBlockingQueue (#3384)
e536d23947 is described below
commit e536d23947dfa6a34f976561f19e9c22b2e0410d
Author: Matteo Merli
AuthorDate: Fri Jul 22 09:50:56 2022 -0700
Switch back ordered executor to LinkedBlockingQueue (#3384)
---
.../main/java/org/apache/bookkeeper/common/util/OrderedExecutor.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/util/OrderedExecutor.java
b/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/util/OrderedExecutor.java
index c58f794f02..ded0f36a2f 100644
---
a/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/util/OrderedExecutor.java
+++
b/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/util/OrderedExecutor.java
@@ -36,6 +36,7 @@ import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@@ -45,7 +46,6 @@ import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.common.collections.BlockingMpscQueue;
-import org.apache.bookkeeper.common.collections.GrowableArrayBlockingQueue;
import org.apache.bookkeeper.common.util.affinity.CpuAffinity;
import org.apache.bookkeeper.stats.Gauge;
import org.apache.bookkeeper.stats.NullStatsLogger;
@@ -305,7 +305,7 @@ public class OrderedExecutor implements ExecutorService {
queue = new BlockingMpscQueue<>(maxTasksInQueue > 0 ?
maxTasksInQueue : DEFAULT_MAX_ARRAY_QUEUE_SIZE);
} else {
// By default, use regular JDK LinkedBlockingQueue
-queue = new GrowableArrayBlockingQueue<>();
+queue = new LinkedBlockingQueue<>();
}
return new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, queue,
factory);
}
[bookkeeper] branch master updated: Switch back ordered executor to LinkedBlockingQueue (#3384)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 92cdf5bfd2 Switch back ordered executor to LinkedBlockingQueue (#3384)
92cdf5bfd2 is described below
commit 92cdf5bfd262be7e1d2e2eef17610e37fcc7a44b
Author: Matteo Merli
AuthorDate: Fri Jul 22 09:50:56 2022 -0700
Switch back ordered executor to LinkedBlockingQueue (#3384)
---
.../main/java/org/apache/bookkeeper/common/util/OrderedExecutor.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/util/OrderedExecutor.java
b/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/util/OrderedExecutor.java
index c58f794f02..ded0f36a2f 100644
---
a/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/util/OrderedExecutor.java
+++
b/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/util/OrderedExecutor.java
@@ -36,6 +36,7 @@ import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@@ -45,7 +46,6 @@ import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.common.collections.BlockingMpscQueue;
-import org.apache.bookkeeper.common.collections.GrowableArrayBlockingQueue;
import org.apache.bookkeeper.common.util.affinity.CpuAffinity;
import org.apache.bookkeeper.stats.Gauge;
import org.apache.bookkeeper.stats.NullStatsLogger;
@@ -305,7 +305,7 @@ public class OrderedExecutor implements ExecutorService {
queue = new BlockingMpscQueue<>(maxTasksInQueue > 0 ?
maxTasksInQueue : DEFAULT_MAX_ARRAY_QUEUE_SIZE);
} else {
// By default, use regular JDK LinkedBlockingQueue
-queue = new GrowableArrayBlockingQueue<>();
+queue = new LinkedBlockingQueue<>();
}
return new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, queue,
factory);
}
[bookkeeper] branch master updated (8d4b9e2ce4 -> 677ccec3eb)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
from 8d4b9e2ce4 upgrade groovy from 2.5.17 to 3.0.11 to fix
CVE-2019-11358(7.5) (#3346)
add 677ccec3eb BP-47 (task6): Direct I/O entrylogger support (#3263)
No new revisions were added by this update.
Summary of changes:
.../bookkeeper/bookie/AbstractLogCompactor.java| 7 +-
.../bookkeeper/bookie/DefaultEntryLogger.java | 17 +-
.../apache/bookkeeper/bookie/EntryLocation.java| 10 +
.../{package-info.java => EntryLogIds.java}| 17 +-
.../bookkeeper/bookie/storage/EntryLogIdsImpl.java | 158 +
.../DirectCompactionEntryLog.java | 285 +
.../directentrylogger/DirectEntryLogger.java | 513
.../directentrylogger/DirectEntryLoggerStats.java | 177 ++
.../storage/directentrylogger/DirectReader.java| 343 +++
.../storage/directentrylogger/DirectWriter.java| 315 ++
.../bookie/storage/directentrylogger/Events.java | 145 +
.../bookie/storage/directentrylogger/Header.java | 105
.../storage/directentrylogger/LogMetadata.java | 191 ++
.../storage/directentrylogger/LogReader.java | 80 +++
.../storage/directentrylogger/LogReaderScan.java | 61 ++
.../storage/directentrylogger/LogWriter.java | 81 +++
.../directentrylogger/WriterWithMetadata.java | 79 +++
.../bookie/GarbageCollectorThreadTest.java | 9 +
.../bookkeeper/bookie/MockLedgerStorage.java | 18 +-
.../bookie/storage/EntryLogTestUtils.java | 42 +-
.../bookie/storage/MockEntryLogIds.java} | 18 +-
.../bookkeeper/bookie/storage/TestEntryLogIds.java | 244
.../{BufferTest.java => TestBuffer.java} | 2 +-
.../directentrylogger/TestDirectEntryLogger.java | 517
.../TestDirectEntryLoggerCompat.java | 651 +
.../directentrylogger/TestDirectReader.java| 510
.../directentrylogger/TestDirectWriter.java| 333 +++
.../TestTransactionalEntryLogCompactor.java| 615 +++
native-io/pom.xml | 1 +
.../common/util/nativeio/NativeIOJni.java | 4 +-
pom.xml| 6 +
31 files changed, 5528 insertions(+), 26 deletions(-)
copy
bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/{package-info.java
=> EntryLogIds.java} (80%)
create mode 100644
bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/EntryLogIdsImpl.java
create mode 100644
bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/directentrylogger/DirectCompactionEntryLog.java
create mode 100644
bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/directentrylogger/DirectEntryLogger.java
create mode 100644
bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/directentrylogger/DirectEntryLoggerStats.java
create mode 100644
bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/directentrylogger/DirectReader.java
create mode 100644
bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/directentrylogger/DirectWriter.java
create mode 100644
bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/directentrylogger/Events.java
create mode 100644
bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/directentrylogger/Header.java
create mode 100644
bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/directentrylogger/LogMetadata.java
create mode 100644
bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/directentrylogger/LogReader.java
create mode 100644
bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/directentrylogger/LogReaderScan.java
create mode 100644
bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/directentrylogger/LogWriter.java
create mode 100644
bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/directentrylogger/WriterWithMetadata.java
copy
bookkeeper-server/src/{main/java/org/apache/bookkeeper/bookie/storage/package-info.java
=> test/java/org/apache/bookkeeper/bookie/storage/MockEntryLogIds.java} (74%)
create mode 100644
bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/TestEntryLogIds.java
rename
bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/directentrylogger/{BufferTest.java
=> TestBuffer.java} (99%)
create mode 100644
bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/directentrylogger/TestDirectEntryLogger.java
create mode 100644
bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/directentrylogger/TestDirectEntryLoggerCompat.java
create mode 100644
bookkeeper-
[bookkeeper] branch branch-4.15 updated: Try to use Java9 CRC32C when JNI based CRC is not available (#3309)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch branch-4.15
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/branch-4.15 by this push:
new dba6a493c1 Try to use Java9 CRC32C when JNI based CRC is not available
(#3309)
dba6a493c1 is described below
commit dba6a493c1b92f1b848ea7c3d23cdd764f6bbe0e
Author: Matteo Merli
AuthorDate: Tue Jun 7 18:26:50 2022 -0700
Try to use Java9 CRC32C when JNI based CRC is not available (#3309)
* Try to use Java9 CRC32C when JNI based CRC is not available
* Fixed shading test dependency
* Fixed dlog shading deps
---
.../proto/checksum/CRC32CDigestManager.java| 11 ---
.../circe/checksum/Crc32cIntChecksum.java | 40 ++--
.../circe/checksum/Crc32cLongChecksum.java | 92 -
.../com/scurrilous/circe/checksum/IntHash.java | 26 +
.../scurrilous/circe/checksum/Java8IntHash.java| 46 +
.../scurrilous/circe/checksum/Java9IntHash.java| 110 +
.../com/scurrilous/circe/checksum/JniIntHash.java | 46 +
.../scurrilous/circe/checksum/ChecksumTest.java| 32 +-
microbenchmarks/run.sh | 4 +-
.../proto/checksum/DigestTypeBenchmark.java| 28 +++---
shaded/distributedlog-core-shaded/pom.xml | 4 -
11 files changed, 257 insertions(+), 182 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/CRC32CDigestManager.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/CRC32CDigestManager.java
index f12b5479f8..c3be132c39 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/CRC32CDigestManager.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/CRC32CDigestManager.java
@@ -19,21 +19,15 @@ package org.apache.bookkeeper.proto.checksum;
*/
import com.scurrilous.circe.checksum.Crc32cIntChecksum;
-import com.scurrilous.circe.crc.Sse42Crc32C;
-
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.util.concurrent.FastThreadLocal;
-
import lombok.extern.slf4j.Slf4j;
-
import org.apache.commons.lang3.mutable.MutableInt;
@Slf4j
class CRC32CDigestManager extends DigestManager {
-private static boolean nonSupportedMessagePrinted = false;
-
private static final FastThreadLocal currentCrc = new
FastThreadLocal() {
@Override
protected MutableInt initialValue() throws Exception {
@@ -43,11 +37,6 @@ class CRC32CDigestManager extends DigestManager {
public CRC32CDigestManager(long ledgerId, boolean useV2Protocol,
ByteBufAllocator allocator) {
super(ledgerId, useV2Protocol, allocator);
-
-if (!Sse42Crc32C.isSupported() && !nonSupportedMessagePrinted) {
-log.warn("Sse42Crc32C is not supported, will use a slower CRC32C
implementation.");
-nonSupportedMessagePrinted = true;
-}
}
@Override
diff --git
a/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Crc32cIntChecksum.java
b/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Crc32cIntChecksum.java
index 528244b12b..c286b6d51b 100644
---
a/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Crc32cIntChecksum.java
+++
b/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Crc32cIntChecksum.java
@@ -18,30 +18,20 @@
*/
package com.scurrilous.circe.checksum;
-import static com.scurrilous.circe.params.CrcParameters.CRC32C;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.scurrilous.circe.IncrementalIntHash;
import com.scurrilous.circe.crc.Sse42Crc32C;
-import com.scurrilous.circe.crc.StandardCrcProvider;
import io.netty.buffer.ByteBuf;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
public class Crc32cIntChecksum {
-private static final Logger log =
LoggerFactory.getLogger(Crc32cIntChecksum.class);
-
-@VisibleForTesting
-final static IncrementalIntHash CRC32C_HASH;
+private final static IntHash CRC32C_HASH;
static {
if (Sse42Crc32C.isSupported()) {
-CRC32C_HASH = new Crc32cSse42Provider().getIncrementalInt(CRC32C);
-log.info("SSE4.2 CRC32C provider initialized");
+CRC32C_HASH = new JniIntHash();
+} else if (Java9IntHash.HAS_JAVA9_CRC32C) {
+CRC32C_HASH = new Java9IntHash();
} else {
-CRC32C_HASH = new StandardCrcProvider().getIncrementalInt(CRC32C);
-log.warn("Failed to load Circe JNI library. Falling back to Java
based CRC32c provider");
+CRC32C_HASH = new Java8IntHash();
}
}
@@ -53,17 +43,9 @@ public class Crc32cIntChecksum {
* @return
*/
public static int computeChecksum(ByteBuf payload) {
-if (payl
[bookkeeper] branch master updated: Try to use Java9 CRC32C when JNI based CRC is not available (#3309)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 168e4c6bcb Try to use Java9 CRC32C when JNI based CRC is not available
(#3309)
168e4c6bcb is described below
commit 168e4c6bcbf5c54cb9f14b7a4ccdff2a5e485ee2
Author: Matteo Merli
AuthorDate: Tue Jun 7 18:26:50 2022 -0700
Try to use Java9 CRC32C when JNI based CRC is not available (#3309)
* Try to use Java9 CRC32C when JNI based CRC is not available
* Fixed shading test dependency
* Fixed dlog shading deps
---
.../proto/checksum/CRC32CDigestManager.java| 11 ---
.../circe/checksum/Crc32cIntChecksum.java | 40 ++--
.../circe/checksum/Crc32cLongChecksum.java | 92 -
.../com/scurrilous/circe/checksum/IntHash.java | 26 +
.../scurrilous/circe/checksum/Java8IntHash.java| 46 +
.../scurrilous/circe/checksum/Java9IntHash.java| 110 +
.../com/scurrilous/circe/checksum/JniIntHash.java | 46 +
.../scurrilous/circe/checksum/ChecksumTest.java| 32 +-
microbenchmarks/run.sh | 4 +-
.../proto/checksum/DigestTypeBenchmark.java| 28 +++---
shaded/distributedlog-core-shaded/pom.xml | 4 -
11 files changed, 257 insertions(+), 182 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/CRC32CDigestManager.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/CRC32CDigestManager.java
index f12b5479f8..c3be132c39 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/CRC32CDigestManager.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/checksum/CRC32CDigestManager.java
@@ -19,21 +19,15 @@ package org.apache.bookkeeper.proto.checksum;
*/
import com.scurrilous.circe.checksum.Crc32cIntChecksum;
-import com.scurrilous.circe.crc.Sse42Crc32C;
-
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.util.concurrent.FastThreadLocal;
-
import lombok.extern.slf4j.Slf4j;
-
import org.apache.commons.lang3.mutable.MutableInt;
@Slf4j
class CRC32CDigestManager extends DigestManager {
-private static boolean nonSupportedMessagePrinted = false;
-
private static final FastThreadLocal currentCrc = new
FastThreadLocal() {
@Override
protected MutableInt initialValue() throws Exception {
@@ -43,11 +37,6 @@ class CRC32CDigestManager extends DigestManager {
public CRC32CDigestManager(long ledgerId, boolean useV2Protocol,
ByteBufAllocator allocator) {
super(ledgerId, useV2Protocol, allocator);
-
-if (!Sse42Crc32C.isSupported() && !nonSupportedMessagePrinted) {
-log.warn("Sse42Crc32C is not supported, will use a slower CRC32C
implementation.");
-nonSupportedMessagePrinted = true;
-}
}
@Override
diff --git
a/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Crc32cIntChecksum.java
b/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Crc32cIntChecksum.java
index 528244b12b..c286b6d51b 100644
---
a/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Crc32cIntChecksum.java
+++
b/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Crc32cIntChecksum.java
@@ -18,30 +18,20 @@
*/
package com.scurrilous.circe.checksum;
-import static com.scurrilous.circe.params.CrcParameters.CRC32C;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.scurrilous.circe.IncrementalIntHash;
import com.scurrilous.circe.crc.Sse42Crc32C;
-import com.scurrilous.circe.crc.StandardCrcProvider;
import io.netty.buffer.ByteBuf;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
public class Crc32cIntChecksum {
-private static final Logger log =
LoggerFactory.getLogger(Crc32cIntChecksum.class);
-
-@VisibleForTesting
-final static IncrementalIntHash CRC32C_HASH;
+private final static IntHash CRC32C_HASH;
static {
if (Sse42Crc32C.isSupported()) {
-CRC32C_HASH = new Crc32cSse42Provider().getIncrementalInt(CRC32C);
-log.info("SSE4.2 CRC32C provider initialized");
+CRC32C_HASH = new JniIntHash();
+} else if (Java9IntHash.HAS_JAVA9_CRC32C) {
+CRC32C_HASH = new Java9IntHash();
} else {
-CRC32C_HASH = new StandardCrcProvider().getIncrementalInt(CRC32C);
-log.warn("Failed to load Circe JNI library. Falling back to Java
based CRC32c provider");
+CRC32C_HASH = new Java8IntHash();
}
}
@@ -53,17 +43,9 @@ public class Crc32cIntChecksum {
* @return
*/
public static int computeChecksum(ByteBuf payload) {
-if (payload.hasMemo
[bookkeeper] branch master updated (b4b3ca3b6b -> 4b852808b7)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git from b4b3ca3b6b 4.14.5 release notes and doc (#3199) add 4b852808b7 Updated organization to Streamlio former members (#3280) No new revisions were added by this update. Summary of changes: site3/website/src/pages/project/who.md | 12 ++-- 1 file changed, 6 insertions(+), 6 deletions(-)
[bookkeeper] branch master updated (873602965d -> 78662ac165)
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git from 873602965d Fix link Apache Pulsar#persistent-storage (#3271) add 78662ac165 Enable Netty and BookKeeper IO optimizations on jdk17 (#3234) No new revisions were added by this update. Summary of changes: bin/common.sh| 20 ++-- bin/common_gradle.sh | 20 ++-- pom.xml | 1 + tests/scripts/src/test/bash/bk_test_bin_common.sh| 16 ++-- .../src/test/bash/gradle/bk_test_bin_common.sh | 14 -- 5 files changed, 63 insertions(+), 8 deletions(-)
[bookkeeper] branch master updated: [minor] [optimize] Remove redundant toString call (#3254)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 90d55017d [minor] [optimize] Remove redundant toString call (#3254)
90d55017d is described below
commit 90d55017da68111ffc578b611de998269f701f82
Author: ZhangJian He
AuthorDate: Thu May 5 08:26:43 2022 +0800
[minor] [optimize] Remove redundant toString call (#3254)
---
.../benchmark/BenchThroughputLatency.java | 2 +-
.../common/conf/ComponentConfiguration.java| 2 +-
.../java/org/apache/bookkeeper/bookie/Cookie.java | 2 +-
.../java/org/apache/bookkeeper/bookie/Journal.java | 3 +--
.../apache/bookkeeper/bookie/JournalChannel.java | 15 ++
.../bookkeeper/conf/AbstractConfiguration.java | 2 +-
.../bookkeeper/net/AbstractDNSToSwitchMapping.java | 2 +-
.../apache/bookkeeper/net/NetworkTopologyImpl.java | 6 +++---
.../apache/bookkeeper/net/ScriptBasedMapping.java | 8
.../apache/bookkeeper/sasl/SaslServerState.java| 3 +--
.../apache/bookkeeper/sasl/TGTRefreshThread.java | 2 +-
.../server/http/service/DecommissionService.java | 2 +-
.../server/http/service/ListBookiesService.java| 2 +-
.../LostBookieRecoveryDelayCommand.java| 4 ++--
.../org/apache/bookkeeper/util/DiskChecker.java| 8
.../org/apache/bookkeeper/util/StringUtils.java| 16 +++
.../bookkeeper/verifier/BookkeeperVerifier.java| 6 +++---
.../bookkeeper/bookie/SkipListArenaTest.java | 2 +-
.../ldb/PersistentEntryLogMetadataMapTest.java | 4 ++--
...KeeperDiskSpaceWeightedLedgerPlacementTest.java | 2 +-
.../discover/AbstractTestZkRegistrationClient.java | 4 ++--
.../java/org/apache/bookkeeper/tls/TestTLS.java| 24 +++---
.../apache/bookkeeper/slogger/ConsoleSlogger.java | 2 +-
.../apache/bookkeeper/metadata/etcd/EtcdUtils.java | 2 +-
.../distributedlog/admin/DistributedLogAdmin.java | 16 +++
.../distributedlog/tools/DistributedLogTool.java | 10 -
.../java/org/apache/distributedlog/tools/Tool.java | 4 ++--
.../distributedlog/TestAppendOnlyStreamReader.java | 4 ++--
.../tests/containers/wait/HttpWaitStrategy.java| 2 +-
.../QueryAutoRecoveryStatusCommandTest.java| 1 -
.../autorecovery/WhoIsAuditorCommandTest.java | 1 -
.../cli/commands/bookie/FormatCommandTest.java | 2 --
.../cli/commands/bookie/LedgerCommandTest.java | 2 --
.../cli/commands/bookie/SanityTestCommandTest.java | 1 -
.../cli/commands/bookies/InitCommandTest.java | 1 -
.../cli/commands/cookie/GetCookieCommandTest.java | 2 +-
.../cli/commands/cluster/InitClusterCommand.java | 4 ++--
37 files changed, 77 insertions(+), 98 deletions(-)
diff --git
a/bookkeeper-benchmark/src/main/java/org/apache/bookkeeper/benchmark/BenchThroughputLatency.java
b/bookkeeper-benchmark/src/main/java/org/apache/bookkeeper/benchmark/BenchThroughputLatency.java
index 08c08e57c..23cac6449 100644
---
a/bookkeeper-benchmark/src/main/java/org/apache/bookkeeper/benchmark/BenchThroughputLatency.java
+++
b/bookkeeper-benchmark/src/main/java/org/apache/bookkeeper/benchmark/BenchThroughputLatency.java
@@ -404,7 +404,7 @@ public class BenchThroughputLatency implements AddCallback,
Runnable {
OutputStream fos = new BufferedOutputStream(new
FileOutputStream(latencyFile));
for (Long l: latency) {
-fos.write((Long.toString(l) + "\t" + (l / 100) +
"ms\n").getBytes(UTF_8));
+fos.write((l + "\t" + (l / 100) + "ms\n").getBytes(UTF_8));
}
fos.flush();
fos.close();
diff --git
a/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/conf/ComponentConfiguration.java
b/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/conf/ComponentConfiguration.java
index 8b2759709..64e4a9356 100644
---
a/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/conf/ComponentConfiguration.java
+++
b/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/conf/ComponentConfiguration.java
@@ -310,7 +310,7 @@ public abstract class ComponentConfiguration implements
Configuration {
Map configMap = new HashMap<>();
Iterator iterator = this.getKeys();
while (iterator.hasNext()) {
-String key = iterator.next().toString();
+String key = iterator.next();
Object property = this.getProperty(key);
if (property != null) {
configMap.put(key, property.toString());
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Cookie.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Cookie.java
index d285a8920..1889cd302 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/
[bookkeeper] branch master updated: Allow running units test on jdk17 (#3195)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new b989fbcda Allow running units test on jdk17 (#3195)
b989fbcda is described below
commit b989fbcdae74db6e26a815af605634747f431fb9
Author: ZhangJian He
AuthorDate: Mon Apr 11 09:03:22 2022 +0800
Allow running units test on jdk17 (#3195)
* Allow running units test on jdk17
* Allow running units test on jdk17
---
pom.xml | 16 +++-
stream/distributedlog/common/pom.xml| 2 +-
stream/distributedlog/core/pom.xml | 2 +-
stream/distributedlog/pom.xml | 2 +-
stream/pom.xml | 2 +-
tests/integration-tests-base-groovy/pom.xml | 2 +-
6 files changed, 20 insertions(+), 6 deletions(-)
diff --git a/pom.xml b/pom.xml
index dd9cb22b0..b20224670 100644
--- a/pom.xml
+++ b/pom.xml
@@ -112,6 +112,7 @@
1.8
true
2
+
1.18.2
@@ -907,7 +908,7 @@
maven-surefire-plugin
${maven-surefire-plugin.version}
- -Xmx2G -Djava.net.preferIPv4Stack=true
-Dio.netty.leakDetection.level=paranoid
+ -Xmx2G -Djava.net.preferIPv4Stack=true
-Dio.netty.leakDetection.level=paranoid ${test.additional.args}
${redirectTestOutputToFile}
${forkCount.variable}
false
@@ -1211,6 +1212,19 @@
src/apple_m1_aol.properties
+
+ jdk11
+
+[11,)
+
+
+
+ --add-opens java.base/jdk.internal.loader=ALL-UNNAMED
+ --add-opens java.base/java.lang=ALL-UNNAMED
+ --add-opens java.base/java.io=ALL-UNNAMED
+
+
+
apache-release
diff --git a/stream/distributedlog/common/pom.xml
b/stream/distributedlog/common/pom.xml
index abf09b9cb..8c6d4febf 100644
--- a/stream/distributedlog/common/pom.xml
+++ b/stream/distributedlog/common/pom.xml
@@ -99,7 +99,7 @@
${maven-surefire-plugin.version}
${redirectTestOutputToFile}
- -Xmx3G -Djava.net.preferIPv4Stack=true
-XX:MaxDirectMemorySize=2G
+ -Xmx3G -Djava.net.preferIPv4Stack=true
-XX:MaxDirectMemorySize=2G ${test.additional.args}
always
1800
diff --git a/stream/distributedlog/core/pom.xml
b/stream/distributedlog/core/pom.xml
index bef0e4c1c..67ef4b422 100644
--- a/stream/distributedlog/core/pom.xml
+++ b/stream/distributedlog/core/pom.xml
@@ -106,7 +106,7 @@
false
${redirectTestOutputToFile}
- -Xmx3G -Djava.net.preferIPv4Stack=true
-XX:MaxDirectMemorySize=2G
+ -Xmx3G -Djava.net.preferIPv4Stack=true
-XX:MaxDirectMemorySize=2G ${test.additional.args}
always
1800
diff --git a/stream/distributedlog/pom.xml b/stream/distributedlog/pom.xml
index 01385778f..25bcd66dd 100644
--- a/stream/distributedlog/pom.xml
+++ b/stream/distributedlog/pom.xml
@@ -73,7 +73,7 @@
${maven-surefire-plugin.version}
${redirectTestOutputToFile}
- -Xmx3G -Djava.net.preferIPv4Stack=true
-XX:MaxDirectMemorySize=2G -Dio.netty.leakDetection.level=PARANOID
+ -Xmx3G -Djava.net.preferIPv4Stack=true
-XX:MaxDirectMemorySize=2G -Dio.netty.leakDetection.level=PARANOID
${test.additional.args}
always
1800
diff --git a/stream/pom.xml b/stream/pom.xml
index ed863740b..f311ed213 100644
--- a/stream/pom.xml
+++ b/stream/pom.xml
@@ -58,7 +58,7 @@
true
${redirectTestOutputToFile}
- -Xmx3G -Djava.net.preferIPv4Stack=true
-XX:MaxDirectMemorySize=2G -Dio.netty.leakDetection.level=PARANOID
+ -Xmx3G -Djava.net.preferIPv4Stack=true
-XX:MaxDirectMemorySize=2G -Dio.netty.leakDetection.level=PARANOID
${test.additional.args}
always
1800
diff --git a/tests/integration-tests-base-groovy/pom.xml
b/tests/integration-tests-base-groovy/pom.xml
index 97209f3d9..1f2d737df 100644
--- a/tests/integration-tests-base-groovy/pom.xml
+++ b/tests/integration-tests-base-groovy/pom.xml
@@ -85,7 +85,7 @@
https://issues.apache.org/jira/browse/SUREFIRE-1476 //-->
2.8.1
- -Xmx4G -Djava.net.preferIPv4Stack=true
+ -Xmx4G -Djava.net.preferIPv4Stack=true
${test.additional.args}
1
false
