This is an automated email from the ASF dual-hosted git repository.
merlimat pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new ad8f4fc333 Migrate from Yahoo DataSketches to Apache DataSketches
7.0.1 (KLL) (#4774)
ad8f4fc333 is described below
commit ad8f4fc3338456e687abab2fc61f00a9108fabf0
Author: Lari Hotari <[email protected]>
AuthorDate: Wed May 6 02:01:46 2026 +0300
Migrate from Yahoo DataSketches to Apache DataSketches 7.0.1 (KLL) (#4774)
The com.yahoo.datasketches:sketches-core:0.8.3 dependency has been
unmaintained since the project moved to Apache. Switch to the current
org.apache.datasketches:datasketches-java:7.0.1 and adopt the KLL
quantiles sketch, which the upstream maintainers recommend over the
classic DoublesSketch for new code (faster, smaller, comparable
accuracy at default K=200).
DataSketchesOpStatsLogger changes:
- On-heap throughout, matching the previous code (no off-heap usage).
- KLL has no reset(); per-thread sketches are reassigned to a fresh
newHeapInstance() under the existing StampedLock write lock.
- KLL has no DoublesUnion; aggregation uses merge() directly into a
freshly allocated KllDoublesSketch each rotation.
- KllDoublesSketch.getQuantile() throws SketchesArgumentException on
an empty sketch, so getQuantileValue() now guards with isEmpty() to
preserve the prior NaN return.
LICENSE bundles list datasketches-java-7.0.1.jar and the transitively
required datasketches-memory-4.1.0.jar separately, since the two jars
come from distinct upstream repositories.
---
.../src/main/resources/LICENSE-all.bin.txt | 7 +++--
.../src/main/resources/LICENSE-server.bin.txt | 7 +++--
pom.xml | 6 ++--
.../prometheus-metrics-provider/pom.xml | 4 +--
.../prometheus/DataSketchesOpStatsLogger.java | 33 ++++++++++------------
5 files changed, 28 insertions(+), 29 deletions(-)
diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt
b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt
index 01c890aae0..983c78f154 100644
--- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt
+++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt
@@ -279,8 +279,8 @@ Apache Software License, Version 2.
- lib/org.eclipse.jetty.toolchain-jetty-servlet-api-4.0.9.jar [22]
- lib/org.rocksdb-rocksdbjni-9.9.3.jar [23]
- lib/com.beust-jcommander-1.82.jar [24]
-- lib/com.yahoo.datasketches-memory-0.8.3.jar [25]
-- lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25]
+- lib/org.apache.datasketches-datasketches-java-7.0.1.jar [25]
+- lib/org.apache.datasketches-datasketches-memory-4.1.0.jar [27]
- lib/at.yawk.lz4-lz4-java-1.10.2.jar [26]
- lib/com.google.api-api-common-2.53.0.jar [63]
- lib/com.google.api.grpc-proto-google-common-protos-2.63.2.jar [28]
@@ -388,8 +388,9 @@ Apache Software License, Version 2.
[22] Source available at
https://github.com/jetty/jetty.project/tree/jetty-12.1.7
[23] Source available at https://github.com/facebook/rocksdb/tree/v9.9.3
[24] Source available at https://github.com/cbeust/jcommander/tree/1.82
-[25] Source available at
https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3
+[25] Source available at https://github.com/apache/datasketches-java/tree/7.0.1
[26] Source available at https://github.com/yawkat/lz4-java/tree/v1.10.2
+[27] Source available at
https://github.com/apache/datasketches-memory/tree/4.1.0
[28] Source available at
https://github.com/googleapis/sdk-platform-java/tree/v2.63.2/java-common-protos
[29] Source available at https://github.com/google/gson/tree/gson-parent-2.12.1
[30] Source available at
https://github.com/census-instrumentation/opencensus-java/tree/v0.31.1
diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt
b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt
index 869a3818c6..de6aeb6c96 100644
--- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt
+++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt
@@ -279,8 +279,8 @@ Apache Software License, Version 2.
- lib/org.eclipse.jetty.toolchain-jetty-servlet-api-4.0.9.jar [22]
- lib/org.rocksdb-rocksdbjni-9.9.3.jar [23]
- lib/com.beust-jcommander-1.82.jar [24]
-- lib/com.yahoo.datasketches-memory-0.8.3.jar [25]
-- lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25]
+- lib/org.apache.datasketches-datasketches-java-7.0.1.jar [25]
+- lib/org.apache.datasketches-datasketches-memory-4.1.0.jar [27]
- lib/at.yawk.lz4-lz4-java-1.10.2.jar [26]
- lib/com.google.api-api-common-2.53.0.jar [62]
- lib/com.google.api.grpc-proto-google-common-protos-2.63.2.jar [28]
@@ -384,8 +384,9 @@ Apache Software License, Version 2.
[22] Source available at
https://github.com/jetty/jetty.project/tree/jetty-12.1.7
[23] Source available at https://github.com/facebook/rocksdb/tree/v9.9.3
[24] Source available at https://github.com/cbeust/jcommander/tree/1.82
-[25] Source available at
https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3
+[25] Source available at https://github.com/apache/datasketches-java/tree/7.0.1
[26] Source available at https://github.com/yawkat/lz4-java/tree/v1.10.2
+[27] Source available at
https://github.com/apache/datasketches-memory/tree/4.1.0
[28] Source available at
https://github.com/googleapis/sdk-platform-java/tree/v2.63.2/java-common-protos
[29] Source available at https://github.com/google/gson/tree/gson-parent-2.12.1
[30] Source available at
https://github.com/census-instrumentation/opencensus-java/tree/v0.31.1
diff --git a/pom.xml b/pom.xml
index 4a1ef17ff8..c7ae8ee878 100644
--- a/pom.xml
+++ b/pom.xml
@@ -181,7 +181,7 @@
<mockito.version>4.11.0</mockito.version>
<netty.version>4.2.13.Final</netty.version>
<prometheus.version>0.15.0</prometheus.version>
- <datasketches.version>0.8.3</datasketches.version>
+ <datasketches.version>7.0.1</datasketches.version>
<httpclient.version>4.5.13</httpclient.version>
<httpcore.version>4.4.15</httpcore.version>
<protobuf.version>4.34.0</protobuf.version>
@@ -670,8 +670,8 @@
</dependency>
<!-- data-sketches -->
<dependency>
- <groupId>com.yahoo.datasketches</groupId>
- <artifactId>sketches-core</artifactId>
+ <groupId>org.apache.datasketches</groupId>
+ <artifactId>datasketches-java</artifactId>
<version>${datasketches.version}</version>
</dependency>
<!-- opentelemetry -->
diff --git
a/stats/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml
b/stats/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml
index c36284757e..57af90075c 100644
--- a/stats/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml
+++ b/stats/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml
@@ -70,8 +70,8 @@
</dependency>
<dependency>
- <groupId>com.yahoo.datasketches</groupId>
- <artifactId>sketches-core</artifactId>
+ <groupId>org.apache.datasketches</groupId>
+ <artifactId>datasketches-java</artifactId>
</dependency>
<dependency>
diff --git
a/stats/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/DataSketchesOpStatsLogger.java
b/stats/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/DataSketchesOpStatsLogger.java
index 1abe85088c..f9a069d283 100644
---
a/stats/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/DataSketchesOpStatsLogger.java
+++
b/stats/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/DataSketchesOpStatsLogger.java
@@ -16,10 +16,6 @@
*/
package org.apache.bookkeeper.stats.prometheus;
-import com.yahoo.sketches.quantiles.DoublesSketch;
-import com.yahoo.sketches.quantiles.DoublesSketchBuilder;
-import com.yahoo.sketches.quantiles.DoublesUnion;
-import com.yahoo.sketches.quantiles.DoublesUnionBuilder;
import io.netty.util.concurrent.FastThreadLocal;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -28,6 +24,7 @@ import java.util.concurrent.atomic.LongAdder;
import java.util.concurrent.locks.StampedLock;
import org.apache.bookkeeper.stats.OpStatsData;
import org.apache.bookkeeper.stats.OpStatsLogger;
+import org.apache.datasketches.kll.KllDoublesSketch;
/**
* OpStatsLogger implementation that uses DataSketches library to calculate
the approximated latency quantiles.
@@ -43,8 +40,8 @@ public class DataSketchesOpStatsLogger implements
OpStatsLogger {
/*
* These are the sketches where all the aggregated results are published.
*/
- private volatile DoublesSketch successResult;
- private volatile DoublesSketch failResult;
+ private volatile KllDoublesSketch successResult;
+ private volatile KllDoublesSketch failResult;
private final LongAdder successCountAdder = new LongAdder();
private final LongAdder failCountAdder = new LongAdder();
@@ -145,22 +142,22 @@ public class DataSketchesOpStatsLogger implements
OpStatsLogger {
current = replacement;
replacement = local;
- final DoublesUnion aggregateSuccess = new
DoublesUnionBuilder().build();
- final DoublesUnion aggregateFail = new DoublesUnionBuilder().build();
+ final KllDoublesSketch aggregateSuccess =
KllDoublesSketch.newHeapInstance();
+ final KllDoublesSketch aggregateFail =
KllDoublesSketch.newHeapInstance();
local.map.forEach((localData, b) -> {
long stamp = localData.lock.writeLock();
try {
- aggregateSuccess.update(localData.successSketch);
- localData.successSketch.reset();
- aggregateFail.update(localData.failSketch);
- localData.failSketch.reset();
+ aggregateSuccess.merge(localData.successSketch);
+ aggregateFail.merge(localData.failSketch);
+ localData.successSketch = KllDoublesSketch.newHeapInstance();
+ localData.failSketch = KllDoublesSketch.newHeapInstance();
} finally {
localData.lock.unlockWrite(stamp);
}
});
- successResult = aggregateSuccess.getResultAndReset();
- failResult = aggregateFail.getResultAndReset();
+ successResult = aggregateSuccess;
+ failResult = aggregateFail;
}
public long getCount(boolean success) {
@@ -172,8 +169,8 @@ public class DataSketchesOpStatsLogger implements
OpStatsLogger {
}
public double getQuantileValue(boolean success, double quantile) {
- DoublesSketch s = success ? successResult : failResult;
- return s != null ? s.getQuantile(quantile) : Double.NaN;
+ KllDoublesSketch s = success ? successResult : failResult;
+ return (s != null && !s.isEmpty()) ? s.getQuantile(quantile) :
Double.NaN;
}
public Map<String, String> getLabels() {
@@ -190,8 +187,8 @@ public class DataSketchesOpStatsLogger implements
OpStatsLogger {
}
private static class LocalData {
- private final DoublesSketch successSketch = new
DoublesSketchBuilder().build();
- private final DoublesSketch failSketch = new
DoublesSketchBuilder().build();
+ private KllDoublesSketch successSketch =
KllDoublesSketch.newHeapInstance();
+ private KllDoublesSketch failSketch =
KllDoublesSketch.newHeapInstance();
private final StampedLock lock = new StampedLock();
}
