This is an automated email from the ASF dual-hosted git repository.
kfaraz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 91cfdde679b Add log and metric for overlord segment cache (#17728)
91cfdde679b is described below
commit 91cfdde679bd6e27578c92fa8b21a99c1405d908
Author: Kashif Faraz <[email protected]>
AuthorDate: Fri Feb 14 23:38:52 2025 +0530
Add log and metric for overlord segment cache (#17728)
Add metric `segment/metadataCache/transactions`
---
docs/operations/metrics.md | 1 +
.../segment/cache/HeapMemorySegmentMetadataCache.java | 8 +++++++-
.../org/apache/druid/metadata/segment/cache/Metric.java | 5 +++++
.../segment/cache/HeapMemorySegmentMetadataCacheTest.java | 15 +++++++++++++++
4 files changed, 28 insertions(+), 1 deletion(-)
diff --git a/docs/operations/metrics.md b/docs/operations/metrics.md
index 0d68b930fe0..7e033b1933a 100644
--- a/docs/operations/metrics.md
+++ b/docs/operations/metrics.md
@@ -322,6 +322,7 @@ The following metrics are emitted only when [segment
metadata caching](../config
|`segment/used/count`|Number of used segments currently present in the
metadata store.|`dataSource`|
|`segment/unused/count`|Number of unused segments currently present in the
metadata store.|`dataSource`|
|`segment/pending/count`|Number of pending segments currently present in the
metadata store.|`dataSource`|
+|`segment/metadataCache/transactions`|Number of read or write transactions
performed on the cache for a single datasource.|`dataSource`|
|`segment/metadataCache/sync/time`|Number of milliseconds taken for the cache
to sync with the metadata store.||
|`segment/metadataCache/deleted`|Total number of segments deleted from the
cache during the latest sync.||
|`segment/metadataCache/skipped`|Total number of unparseable segment records
that were skipped in the latest sync.||
diff --git
a/server/src/main/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCache.java
b/server/src/main/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCache.java
index f4808ee47b1..4a451d162e6 100644
---
a/server/src/main/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCache.java
+++
b/server/src/main/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCache.java
@@ -138,8 +138,13 @@ public class HeapMemorySegmentMetadataCache implements
SegmentMetadataCache
@LifecycleStart
public void start()
{
+ if (!isCacheEnabled) {
+ log.info("Segment metadata cache is not enabled.");
+ return;
+ }
+
synchronized (cacheStateLock) {
- if (isCacheEnabled && currentCacheState == CacheState.STOPPED) {
+ if (currentCacheState == CacheState.STOPPED) {
updateCacheState(CacheState.FOLLOWER, "Scheduling sync with metadata
store");
scheduleSyncWithMetadataStore(pollDuration.getMillis());
}
@@ -200,6 +205,7 @@ public class HeapMemorySegmentMetadataCache implements
SegmentMetadataCache
public DatasourceSegmentCache getDatasource(String dataSource)
{
verifyCacheIsUsableAndAwaitSync();
+ emitMetric(dataSource, Metric.TRANSACTION_COUNT, 1);
return getCacheForDatasource(dataSource);
}
diff --git
a/server/src/main/java/org/apache/druid/metadata/segment/cache/Metric.java
b/server/src/main/java/org/apache/druid/metadata/segment/cache/Metric.java
index de9e0da0994..476b6b1f6aa 100644
--- a/server/src/main/java/org/apache/druid/metadata/segment/cache/Metric.java
+++ b/server/src/main/java/org/apache/druid/metadata/segment/cache/Metric.java
@@ -47,6 +47,11 @@ public class Metric
// CACHE METRICS
private static final String METRIC_NAME_PREFIX = "segment/metadataCache/";
+ /**
+ * Number of transactions performed on the cache for a datasource.
+ */
+ public static final String TRANSACTION_COUNT = "transactions";
+
/**
* Time taken in milliseconds for the latest sync with metadata store.
*/
diff --git
a/server/src/test/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCacheTest.java
b/server/src/test/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCacheTest.java
index 9b0d205af05..0b6cf606f44 100644
---
a/server/src/test/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCacheTest.java
+++
b/server/src/test/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCacheTest.java
@@ -32,6 +32,7 @@ import
org.apache.druid.metadata.IndexerSqlMetadataStorageCoordinatorTestBase;
import org.apache.druid.metadata.PendingSegmentRecord;
import org.apache.druid.metadata.SegmentsMetadataManagerConfig;
import org.apache.druid.metadata.TestDerbyConnector;
+import org.apache.druid.query.DruidMetrics;
import org.apache.druid.segment.TestDataSource;
import org.apache.druid.segment.TestHelper;
import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec;
@@ -51,6 +52,7 @@ import org.junit.Test;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.Set;
public class HeapMemorySegmentMetadataCacheTest
@@ -629,6 +631,19 @@ public class HeapMemorySegmentMetadataCacheTest
);
}
+ @Test
+ public void testGetDatasource_increasesTransactionCount()
+ {
+ setupAndSyncCache();
+ cache.getDatasource(TestDataSource.WIKI);
+ cache.getDatasource(TestDataSource.WIKI);
+ serviceEmitter.verifyEmitted(
+ Metric.TRANSACTION_COUNT,
+ Map.of(DruidMetrics.DATASOURCE, TestDataSource.WIKI),
+ 2
+ );
+ }
+
private void insertSegmentsInMetadataStore(Set<DataSegmentPlus> segments)
{
final String table =
derbyConnectorRule.metadataTablesConfigSupplier().get().getSegmentsTable();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
