This is an automated email from the ASF dual-hosted git repository. rcordier pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
commit b80b772ead580304a12de6c644e819dd6db22f0d Author: duc91 <duc91....@gmail.com> AuthorDate: Thu Jun 4 15:56:06 2020 +0700 JAMES-3170 Add Grafana board for CachedLatencyMetric, CachedHitMetric,CachedMissMetric. --- .../CacheBlobStore-15911761170000-dashboard.json | 346 +++++++++++++++++++++ .../blob/cassandra/cache/CachedBlobStore.java | 56 ++-- .../cassandra/cache/CassandraBlobStoreCache.java | 9 +- .../blob/cassandra/cache/CachedBlobStoreTest.java | 296 +++++++++++++----- .../cache/CassandraBlobStoreCacheTest.java | 2 +- 5 files changed, 606 insertions(+), 103 deletions(-) diff --git a/grafana-reporting/CacheBlobStore-15911761170000-dashboard.json b/grafana-reporting/CacheBlobStore-15911761170000-dashboard.json new file mode 100644 index 0000000..c51b0bb --- /dev/null +++ b/grafana-reporting/CacheBlobStore-15911761170000-dashboard.json @@ -0,0 +1,346 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 4, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 4, + "panels": [], + "title": "CachedBlobStore_Metric", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": 0, + "trimEdges": 0 + }, + "type": "date_histogram" + } + ], + "metrics": [ + { + "field": "p50", + "id": "1", + "meta": {}, + "settings": {}, + "type": "avg" + }, + { + "field": "p75", + "id": "3", + "meta": {}, + "settings": {}, + "type": "avg" + }, + { + "field": "p95", + "id": "4", + "meta": {}, + "settings": {}, + "type": "avg" + }, + { + "field": "p99", + "id": "5", + "meta": {}, + "settings": {}, + "type": "avg" + } + ], + "query": "name:\"blobstoreCacheLatency\"", + "refId": "A", + "timeField": "@timestamp" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "miss", + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": 0, + "trimEdges": 0 + }, + "type": "date_histogram" + } + ], + "metrics": [ + { + "field": "1", + "id": "3", + "meta": {}, + "pipelineAgg": "1", + "settings": {}, + "type": "derivative" + } + ], + "query": "name:\"blobstoreCacheMisses\"", + "refId": "A", + "timeField": "@timestamp" + }, + { + "alias": "hit", + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": 0, + "trimEdges": 0 + }, + "type": "date_histogram" + } + ], + "metrics": [ + { + "field": "1", + "id": "3", + "meta": {}, + "pipelineAgg": "1", + "settings": {}, + "type": "derivative" + } + ], + "query": "name:\"blobstoreCacheHits\"", + "refId": "B", + "timeField": "@timestamp" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CachedHits/CachedMisses", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "locale", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "decimals": 0, + "format": "locale", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 25, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "CachedBlobstore", + "uid": "XbLcT6zMz", + "version": 5 +} \ No newline at end of file diff --git a/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/cache/CachedBlobStore.java b/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/cache/CachedBlobStore.java index d33c52f..9acbbed 100644 --- a/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/cache/CachedBlobStore.java +++ b/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/cache/CachedBlobStore.java @@ -98,20 +98,22 @@ public class CachedBlobStore implements BlobStore { this.firstBytes = firstBytes; this.hasMore = hasMore; } + } public static final String BACKEND = "blobStoreBackend"; - public static final String BLOBSTORE_CACHED_LATENCY_METRIC_NAME = "blobstoreCachedLatency"; - public static final String BLOBSTORE_CACHED_HIT_COUNT_METRIC_NAME = "blobstoreCachedHit"; - public static final String BLOBSTORE_CACHED_MISS_COUNT_METRIC_NAME = "blobstoreCachedMiss"; + public static final String BLOBSTORE_CACHED_MISS_COUNT_METRIC_NAME = "blobStoreCacheMisses"; + public static final String BLOBSTORE_CACHED_LATENCY_METRIC_NAME = "blobStoreCacheLatency"; + public static final String BLOBSTORE_CACHED_HIT_COUNT_METRIC_NAME = "blobStoreCacheHits"; + + private final MetricFactory metricFactory; private final Metric metricRetrieveHitCount; private final Metric metricRetrieveMissCount; private final BlobStoreCache cache; private final BlobStore backend; private final Integer sizeThresholdInBytes; - private final MetricFactory metricFactory; @Inject public CachedBlobStore(BlobStoreCache cache, @@ -121,37 +123,48 @@ public class CachedBlobStore implements BlobStore { this.cache = cache; this.backend = backend; this.sizeThresholdInBytes = cacheConfiguration.getSizeThresholdInBytes(); + this.metricFactory = metricFactory; - metricRetrieveHitCount = metricFactory.generate(BLOBSTORE_CACHED_HIT_COUNT_METRIC_NAME); - metricRetrieveMissCount = metricFactory.generate(BLOBSTORE_CACHED_MISS_COUNT_METRIC_NAME); + this.metricRetrieveMissCount = metricFactory.generate(BLOBSTORE_CACHED_MISS_COUNT_METRIC_NAME); + this.metricRetrieveHitCount = metricFactory.generate(BLOBSTORE_CACHED_HIT_COUNT_METRIC_NAME); } @Override public InputStream read(BucketName bucketName, BlobId blobId) throws ObjectStoreIOException, ObjectNotFoundException { return Mono.just(bucketName) .filter(getDefaultBucketName()::equals) - .flatMap(ignored -> readFromCache(blobId) - .flatMap(this::toInputStream)) + .flatMap(defaultBucket -> readInDefaultBucket(bucketName, blobId)) + .switchIfEmpty(readFromBackend(bucketName, blobId)) + .blockOptional() + .orElseThrow(() -> new ObjectNotFoundException(String.format("Could not retrieve blob metadata for %s", blobId.asString()))); + } + + private Mono<InputStream> readInDefaultBucket(BucketName bucketName, BlobId blobId) { + return readFromCache(blobId) + .flatMap(this::toInputStream) .switchIfEmpty(readFromBackend(bucketName, blobId) .flatMap(inputStream -> Mono.fromCallable(() -> ReadAheadInputStream.eager().of(inputStream).length(sizeThresholdInBytes)) .flatMap(readAheadInputStream -> putInCacheIfNeeded(bucketName, readAheadInputStream, blobId) - .thenReturn(readAheadInputStream.in)))) - .blockOptional() - .orElseThrow(() -> new ObjectNotFoundException(String.format("Could not retrieve blob metadata for %s", blobId))); + .thenReturn(readAheadInputStream.in)))); } @Override public Mono<byte[]> readBytes(BucketName bucketName, BlobId blobId) { return Mono.just(bucketName) .filter(getDefaultBucketName()::equals) - .flatMap(ignored -> readFromCache(blobId) - .switchIfEmpty(readBytesFromBackend(bucketName, blobId) - .filter(this::isAbleToCache) - .flatMap(bytes -> saveInCache(blobId, bytes).then(Mono.just(bytes))))) + .flatMap(deleteBucket -> readBytesInDefaultBucket(bucketName, blobId)) .switchIfEmpty(readBytesFromBackend(bucketName, blobId)); } + private Mono<byte[]> readBytesInDefaultBucket(BucketName bucketName, BlobId blobId) { + return readFromCache(blobId) + .switchIfEmpty(readBytesFromBackend(bucketName, blobId) + .filter(this::isAbleToCache) + .doOnNext(any -> metricRetrieveMissCount.increment()) + .flatMap(bytes -> saveInCache(blobId, bytes).then(Mono.just(bytes)))); + } + @Override public Mono<BlobId> save(BucketName bucketName, byte[] bytes, StoragePolicy storagePolicy) { return Mono.from(backend.save(bucketName, bytes, storagePolicy)) @@ -213,6 +226,7 @@ public class CachedBlobStore implements BlobStore { private Mono<Void> putInCacheIfNeeded(BucketName bucketName, ReadAheadInputStream readAhead, BlobId blobId) { return Mono.justOrEmpty(readAhead.firstBytes) .filter(bytes -> isAbleToCache(readAhead, bucketName)) + .doOnNext(any -> metricRetrieveMissCount.increment()) .flatMap(bytes -> Mono.from(cache.cache(blobId, bytes))); } @@ -244,16 +258,16 @@ public class CachedBlobStore implements BlobStore { return Mono.fromCallable(() -> new ByteArrayInputStream(bytes)); } - private Mono<InputStream> readFromBackend(BucketName bucketName, BlobId blobId) { - return Mono.fromCallable(() -> backend.read(bucketName, blobId)); - } - private Mono<byte[]> readFromCache(BlobId blobId) { return Mono.from(metricFactory.runPublishingTimerMetric(BLOBSTORE_CACHED_LATENCY_METRIC_NAME, cache.read(blobId))) - .doOnNext(bytes -> metricRetrieveHitCount.increment()); + .doOnNext(any -> metricRetrieveHitCount.increment()); + } + + private Mono<InputStream> readFromBackend(BucketName bucketName, BlobId blobId) { + return Mono.fromCallable(() -> backend.read(bucketName, blobId)); } private Mono<byte[]> readBytesFromBackend(BucketName bucketName, BlobId blobId) { - return Mono.from(backend.readBytes(bucketName, blobId)).doOnNext(bytes -> metricRetrieveMissCount.increment()); + return Mono.from(backend.readBytes(bucketName, blobId)); } } diff --git a/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/cache/CassandraBlobStoreCache.java b/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/cache/CassandraBlobStoreCache.java index f2211a5..5584a6c 100644 --- a/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/cache/CassandraBlobStoreCache.java +++ b/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/cache/CassandraBlobStoreCache.java @@ -64,7 +64,8 @@ public class CassandraBlobStoreCache implements BlobStoreCache { @Inject @VisibleForTesting - CassandraBlobStoreCache(@Named(InjectionNames.CACHE) Session session, CassandraCacheConfiguration cacheConfiguration) { + CassandraBlobStoreCache(@Named(InjectionNames.CACHE) Session session, + CassandraCacheConfiguration cacheConfiguration) { this.cassandraAsyncExecutor = new CassandraAsyncExecutor(session); this.insertStatement = prepareInsert(session); this.selectStatement = prepareSelect(session); @@ -81,13 +82,11 @@ public class CassandraBlobStoreCache implements BlobStoreCache { @Override public Mono<byte[]> read(BlobId blobId) { - return cassandraAsyncExecutor - .executeSingleRow( + return cassandraAsyncExecutor.executeSingleRow( selectStatement.bind() .setString(ID, blobId.asString()) .setConsistencyLevel(ONE) - .setReadTimeoutMillis(readTimeOutFromDataBase) - ) + .setReadTimeoutMillis(readTimeOutFromDataBase)) .map(this::toByteArray); } diff --git a/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/cache/CachedBlobStoreTest.java b/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/cache/CachedBlobStoreTest.java index 884f14d..ad6c3d3 100644 --- a/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/cache/CachedBlobStoreTest.java +++ b/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/cache/CachedBlobStoreTest.java @@ -25,9 +25,8 @@ import static org.apache.james.blob.api.BucketName.DEFAULT; import static org.apache.james.blob.cassandra.cache.BlobStoreCacheContract.EIGHT_KILOBYTES; import static org.apache.james.blob.cassandra.cache.CachedBlobStore.BLOBSTORE_CACHED_HIT_COUNT_METRIC_NAME; import static org.apache.james.blob.cassandra.cache.CachedBlobStore.BLOBSTORE_CACHED_LATENCY_METRIC_NAME; +import static org.apache.james.blob.cassandra.cache.CachedBlobStore.BLOBSTORE_CACHED_MISS_COUNT_METRIC_NAME; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatCode; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; @@ -42,11 +41,13 @@ import org.apache.james.blob.api.BlobStoreContract; import org.apache.james.blob.api.BucketName; import org.apache.james.blob.api.HashBlobId; import org.apache.james.blob.api.ObjectNotFoundException; +import org.apache.james.blob.api.TestBlobId; import org.apache.james.blob.cassandra.CassandraBlobModule; import org.apache.james.blob.cassandra.CassandraBlobStore; import org.apache.james.metrics.tests.RecordingMetricFactory; import org.assertj.core.api.SoftAssertions; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -76,8 +77,8 @@ public class CachedBlobStoreTest implements BlobStoreContract { .sizeThresholdInBytes(EIGHT_KILOBYTES.length + 1) .timeOut(Duration.ofSeconds(60)) .build(); - cache = new CassandraBlobStoreCache(cassandra.getConf(), cacheConfig); metricFactory = new RecordingMetricFactory(); + cache = new CassandraBlobStoreCache(cassandra.getConf(), cacheConfig); testee = new CachedBlobStore(cache, backend, cacheConfig, metricFactory); } @@ -103,9 +104,9 @@ public class CachedBlobStoreTest implements BlobStoreContract { public void shouldNotCacheWhenNotDefaultBucketName() { BlobId blobId = Mono.from(testee().save(TEST_BUCKETNAME, EIGHT_KILOBYTES, SIZE_BASED)).block(); - SoftAssertions.assertSoftly(ignored -> { - assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); - assertThat(Mono.from(backend.readBytes(TEST_BUCKETNAME, blobId)).block()).containsExactly(EIGHT_KILOBYTES); + SoftAssertions.assertSoftly(soflty -> { + soflty.assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); + soflty.assertThat(Mono.from(backend.readBytes(TEST_BUCKETNAME, blobId)).block()).containsExactly(EIGHT_KILOBYTES); }); } @@ -113,9 +114,9 @@ public class CachedBlobStoreTest implements BlobStoreContract { public void shouldNotCacheWhenDefaultBucketNameAndBigByteDataAndSizeBase() { BlobId blobId = Mono.from(testee().save(DEFAULT_BUCKETNAME, TWELVE_MEGABYTES, SIZE_BASED)).block(); - SoftAssertions.assertSoftly(ignored -> { - assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); - assertThat(Mono.from(backend.readBytes(DEFAULT_BUCKETNAME, blobId)).block()).containsExactly(TWELVE_MEGABYTES); + SoftAssertions.assertSoftly(soflty -> { + soflty.assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); + soflty.assertThat(Mono.from(backend.readBytes(DEFAULT_BUCKETNAME, blobId)).block()).containsExactly(TWELVE_MEGABYTES); }); } @@ -124,8 +125,8 @@ public class CachedBlobStoreTest implements BlobStoreContract { BlobId blobId = Mono.from(testee().save(DEFAULT_BUCKETNAME, EIGHT_KILOBYTES, SIZE_BASED)).block(); SoftAssertions.assertSoftly(soflty -> { - assertThat(Mono.from(cache.read(blobId)).block()).containsExactly(EIGHT_KILOBYTES); - assertThat(Mono.from(backend.readBytes(DEFAULT_BUCKETNAME, blobId)).block()).containsExactly(EIGHT_KILOBYTES); + soflty.assertThat(Mono.from(cache.read(blobId)).block()).containsExactly(EIGHT_KILOBYTES); + soflty.assertThat(Mono.from(backend.readBytes(DEFAULT_BUCKETNAME, blobId)).block()).containsExactly(EIGHT_KILOBYTES); }); } @@ -134,8 +135,8 @@ public class CachedBlobStoreTest implements BlobStoreContract { BlobId blobId = Mono.from(testee().save(DEFAULT_BUCKETNAME, EIGHT_KILOBYTES, HIGH_PERFORMANCE)).block(); SoftAssertions.assertSoftly(soflty -> { - assertThat(Mono.from(cache.read(blobId)).block()).containsExactly(EIGHT_KILOBYTES); - assertThat(Mono.from(backend.readBytes(DEFAULT_BUCKETNAME, blobId)).block()).containsExactly(EIGHT_KILOBYTES); + soflty.assertThat(Mono.from(cache.read(blobId)).block()).containsExactly(EIGHT_KILOBYTES); + soflty.assertThat(Mono.from(backend.readBytes(DEFAULT_BUCKETNAME, blobId)).block()).containsExactly(EIGHT_KILOBYTES); }); } @@ -144,8 +145,8 @@ public class CachedBlobStoreTest implements BlobStoreContract { BlobId blobId = Mono.from(testee().save(DEFAULT_BUCKETNAME, EIGHT_KILOBYTES, LOW_COST)).block(); SoftAssertions.assertSoftly(soflty -> { - assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); - assertThat(Mono.from(backend.readBytes(DEFAULT_BUCKETNAME, blobId)).block()).containsExactly(EIGHT_KILOBYTES); + soflty.assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); + soflty.assertThat(Mono.from(backend.readBytes(DEFAULT_BUCKETNAME, blobId)).block()).containsExactly(EIGHT_KILOBYTES); }); } @@ -154,8 +155,8 @@ public class CachedBlobStoreTest implements BlobStoreContract { BlobId blobId = Mono.from(testee().save(DEFAULT_BUCKETNAME, new ByteArrayInputStream(EMPTY_BYTEARRAY), SIZE_BASED)).block(); SoftAssertions.assertSoftly(soflty -> { - assertThat(new ByteArrayInputStream(Mono.from(cache.read(blobId)).block())).hasSameContentAs(new ByteArrayInputStream(EMPTY_BYTEARRAY)); - assertThat(Mono.from(backend.readBytes(DEFAULT_BUCKETNAME, blobId)).block()).containsExactly(EMPTY_BYTEARRAY); + soflty.assertThat(new ByteArrayInputStream(Mono.from(cache.read(blobId)).block())).hasSameContentAs(new ByteArrayInputStream(EMPTY_BYTEARRAY)); + soflty.assertThat(Mono.from(backend.readBytes(DEFAULT_BUCKETNAME, blobId)).block()).containsExactly(EMPTY_BYTEARRAY); }); } @@ -164,8 +165,8 @@ public class CachedBlobStoreTest implements BlobStoreContract { BlobId blobId = Mono.from(testee().save(DEFAULT_BUCKETNAME, EMPTY_BYTEARRAY, SIZE_BASED)).block(); SoftAssertions.assertSoftly(soflty -> { - assertThat(new ByteArrayInputStream(Mono.from(cache.read(blobId)).block())).hasSameContentAs(new ByteArrayInputStream(EMPTY_BYTEARRAY)); - assertThat(Mono.from(backend.readBytes(DEFAULT_BUCKETNAME, blobId)).block()).containsExactly(EMPTY_BYTEARRAY); + soflty.assertThat(new ByteArrayInputStream(Mono.from(cache.read(blobId)).block())).hasSameContentAs(new ByteArrayInputStream(EMPTY_BYTEARRAY)); + soflty.assertThat(Mono.from(backend.readBytes(DEFAULT_BUCKETNAME, blobId)).block()).containsExactly(EMPTY_BYTEARRAY); }); } @@ -174,9 +175,9 @@ public class CachedBlobStoreTest implements BlobStoreContract { BlobId blobId = Mono.from(testee().save(DEFAULT_BUCKETNAME, new ByteArrayInputStream(APPROXIMATELY_FIVE_KILOBYTES), SIZE_BASED)).block(); SoftAssertions.assertSoftly(soflty -> { - assertThat(new ByteArrayInputStream(Mono.from(cache.read(blobId)).block())) + soflty.assertThat(new ByteArrayInputStream(Mono.from(cache.read(blobId)).block())) .hasSameContentAs(new ByteArrayInputStream(APPROXIMATELY_FIVE_KILOBYTES)); - assertThat(new ByteArrayInputStream(Mono.from(backend.readBytes(DEFAULT_BUCKETNAME, blobId)).block())) + soflty.assertThat(new ByteArrayInputStream(Mono.from(backend.readBytes(DEFAULT_BUCKETNAME, blobId)).block())) .hasSameContentAs(new ByteArrayInputStream(APPROXIMATELY_FIVE_KILOBYTES)); }); } @@ -186,9 +187,9 @@ public class CachedBlobStoreTest implements BlobStoreContract { BlobId blobId = Mono.from(testee().save(DEFAULT_BUCKETNAME, APPROXIMATELY_FIVE_KILOBYTES, SIZE_BASED)).block(); SoftAssertions.assertSoftly(soflty -> { - assertThat(new ByteArrayInputStream(Mono.from(cache.read(blobId)).block())) + soflty.assertThat(new ByteArrayInputStream(Mono.from(cache.read(blobId)).block())) .hasSameContentAs(new ByteArrayInputStream(APPROXIMATELY_FIVE_KILOBYTES)); - assertThat(new ByteArrayInputStream(Mono.from(backend.readBytes(DEFAULT_BUCKETNAME, blobId)).block())) + soflty.assertThat(new ByteArrayInputStream(Mono.from(backend.readBytes(DEFAULT_BUCKETNAME, blobId)).block())) .hasSameContentAs(new ByteArrayInputStream(APPROXIMATELY_FIVE_KILOBYTES)); }); } @@ -197,11 +198,11 @@ public class CachedBlobStoreTest implements BlobStoreContract { public void shouldRemoveBothInCacheAndBackendWhenDefaultBucketName() { BlobId blobId = Mono.from(testee().save(DEFAULT_BUCKETNAME, EIGHT_KILOBYTES, SIZE_BASED)).block(); - SoftAssertions.assertSoftly(ignored -> { - assertThatCode(Mono.from(testee().delete(DEFAULT_BUCKETNAME, blobId))::block) + SoftAssertions.assertSoftly(soflty -> { + soflty.assertThatCode(Mono.from(testee().delete(DEFAULT_BUCKETNAME, blobId))::block) .doesNotThrowAnyException(); - assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); - assertThatThrownBy(() -> Mono.from(backend.readBytes(DEFAULT_BUCKETNAME, blobId)).block()) + soflty.assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); + soflty.assertThatThrownBy(() -> Mono.from(backend.readBytes(DEFAULT_BUCKETNAME, blobId)).block()) .isInstanceOf(ObjectNotFoundException.class); }); } @@ -211,10 +212,10 @@ public class CachedBlobStoreTest implements BlobStoreContract { BlobId blobId = Mono.from(backend.save(DEFAULT_BUCKETNAME, APPROXIMATELY_FIVE_KILOBYTES, SIZE_BASED)).block(); SoftAssertions.assertSoftly(soflty -> { - assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); - assertThat(new ByteArrayInputStream(Mono.from(testee().readBytes(DEFAULT_BUCKETNAME, blobId)).block())) + soflty.assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); + soflty.assertThat(new ByteArrayInputStream(Mono.from(testee().readBytes(DEFAULT_BUCKETNAME, blobId)).block())) .hasSameContentAs(new ByteArrayInputStream(APPROXIMATELY_FIVE_KILOBYTES)); - assertThat(new ByteArrayInputStream(Mono.from(cache.read(blobId)).block())) + soflty.assertThat(new ByteArrayInputStream(Mono.from(cache.read(blobId)).block())) .hasSameContentAs(new ByteArrayInputStream(APPROXIMATELY_FIVE_KILOBYTES)); }); } @@ -224,10 +225,10 @@ public class CachedBlobStoreTest implements BlobStoreContract { BlobId blobId = Mono.from(backend.save(DEFAULT_BUCKETNAME, APPROXIMATELY_FIVE_KILOBYTES, SIZE_BASED)).block(); SoftAssertions.assertSoftly(soflty -> { - assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); - assertThat(testee().read(DEFAULT_BUCKETNAME, blobId)) + soflty.assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); + soflty.assertThat(testee().read(DEFAULT_BUCKETNAME, blobId)) .hasSameContentAs(new ByteArrayInputStream(APPROXIMATELY_FIVE_KILOBYTES)); - assertThat(new ByteArrayInputStream(Mono.from(cache.read(blobId)).block())) + soflty.assertThat(new ByteArrayInputStream(Mono.from(cache.read(blobId)).block())) .hasSameContentAs(new ByteArrayInputStream(APPROXIMATELY_FIVE_KILOBYTES)); }); } @@ -237,10 +238,10 @@ public class CachedBlobStoreTest implements BlobStoreContract { BlobId blobId = Mono.from(backend.save(TEST_BUCKETNAME, APPROXIMATELY_FIVE_KILOBYTES, SIZE_BASED)).block(); SoftAssertions.assertSoftly(soflty -> { - assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); - assertThat(new ByteArrayInputStream(Mono.from(testee().readBytes(TEST_BUCKETNAME, blobId)).block())) + soflty.assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); + soflty.assertThat(new ByteArrayInputStream(Mono.from(testee().readBytes(TEST_BUCKETNAME, blobId)).block())) .hasSameContentAs(new ByteArrayInputStream(APPROXIMATELY_FIVE_KILOBYTES)); - assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); + soflty.assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); }); } @@ -249,10 +250,10 @@ public class CachedBlobStoreTest implements BlobStoreContract { BlobId blobId = Mono.from(backend.save(TEST_BUCKETNAME, new ByteArrayInputStream(APPROXIMATELY_FIVE_KILOBYTES), SIZE_BASED)).block(); SoftAssertions.assertSoftly(soflty -> { - assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); - assertThat(testee().read(TEST_BUCKETNAME, blobId)) + soflty.assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); + soflty.assertThat(testee().read(TEST_BUCKETNAME, blobId)) .hasSameContentAs(new ByteArrayInputStream(APPROXIMATELY_FIVE_KILOBYTES)); - assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); + soflty.assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); }); } @@ -261,10 +262,10 @@ public class CachedBlobStoreTest implements BlobStoreContract { BlobId blobId = Mono.from(backend.save(DEFAULT_BUCKETNAME, new ByteArrayInputStream(TWELVE_MEGABYTES), SIZE_BASED)).block(); SoftAssertions.assertSoftly(soflty -> { - assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); - assertThat(new ByteArrayInputStream(Mono.from(testee().readBytes(DEFAULT_BUCKETNAME, blobId)).block())) + soflty.assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); + soflty.assertThat(new ByteArrayInputStream(Mono.from(testee().readBytes(DEFAULT_BUCKETNAME, blobId)).block())) .hasSameContentAs(new ByteArrayInputStream(TWELVE_MEGABYTES)); - assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); + soflty.assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); }); } @@ -273,43 +274,186 @@ public class CachedBlobStoreTest implements BlobStoreContract { BlobId blobId = Mono.from(backend.save(DEFAULT_BUCKETNAME, new ByteArrayInputStream(TWELVE_MEGABYTES), SIZE_BASED)).block(); SoftAssertions.assertSoftly(soflty -> { - assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); - assertThat(testee().read(DEFAULT_BUCKETNAME, blobId)) + soflty.assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); + soflty.assertThat(testee().read(DEFAULT_BUCKETNAME, blobId)) .hasSameContentAs(new ByteArrayInputStream(TWELVE_MEGABYTES)); - assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); + soflty.assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty(); }); } - @Test - public void readBlobstoreCachedShouldPublishTimerMetrics() { - BlobId blobId = Mono.from(backend.save(DEFAULT_BUCKETNAME, APPROXIMATELY_FIVE_KILOBYTES, SIZE_BASED)).block(); - - testee.read(DEFAULT_BUCKETNAME, blobId); - testee.read(DEFAULT_BUCKETNAME, blobId); - - assertThat(metricFactory.executionTimesFor(BLOBSTORE_CACHED_LATENCY_METRIC_NAME)) - .hasSize(2); - } - - @Test - public void readBlobstoreCachedShouldCountWhenHit() { - BlobId blobId = Mono.from(testee.save(DEFAULT_BUCKETNAME, APPROXIMATELY_FIVE_KILOBYTES, SIZE_BASED)).block(); - - testee.read(DEFAULT_BUCKETNAME, blobId); - testee.read(DEFAULT_BUCKETNAME, blobId); - - assertThat(metricFactory.countFor(BLOBSTORE_CACHED_HIT_COUNT_METRIC_NAME)) - .isEqualTo(2); - } - - @Test - public void readBlobstoreCachedShouldCountWhenMissed() { - BlobId blobId = Mono.from(testee.save(DEFAULT_BUCKETNAME, APPROXIMATELY_FIVE_KILOBYTES, SIZE_BASED)).block(); - - testee.read(DEFAULT_BUCKETNAME, blobId); - testee.read(DEFAULT_BUCKETNAME, blobId); - - assertThat(metricFactory.countFor(BLOBSTORE_CACHED_HIT_COUNT_METRIC_NAME)) - .isEqualTo(2); + @Nested + class MetricsTest { + @Test + public void readBlobStoreCacheWithNoneDefaultBucketNameShouldNotImpact() { + BlobId blobId = Mono.from(backend.save(TEST_BUCKETNAME, APPROXIMATELY_FIVE_KILOBYTES, SIZE_BASED)).block(); + + testee.read(TEST_BUCKETNAME, blobId); + testee.read(TEST_BUCKETNAME, blobId); + + SoftAssertions.assertSoftly(soflty -> { + assertThat(metricFactory.countFor(BLOBSTORE_CACHED_MISS_COUNT_METRIC_NAME)) + .describedAs(BLOBSTORE_CACHED_MISS_COUNT_METRIC_NAME) + .isEqualTo(0); + soflty.assertThat(metricFactory.countFor(BLOBSTORE_CACHED_HIT_COUNT_METRIC_NAME)) + .describedAs(BLOBSTORE_CACHED_HIT_COUNT_METRIC_NAME) + .isEqualTo(0); + assertThat(metricFactory.executionTimesFor(BLOBSTORE_CACHED_LATENCY_METRIC_NAME)) + .describedAs(BLOBSTORE_CACHED_LATENCY_METRIC_NAME) + .hasSize(0); + }); + } + + @Test + public void readBytesWithNoneDefaultBucketNameShouldNotImpact() { + BlobId blobId = Mono.from(backend.save(TEST_BUCKETNAME, APPROXIMATELY_FIVE_KILOBYTES, SIZE_BASED)).block(); + + Mono.from(testee.readBytes(TEST_BUCKETNAME, blobId)).block(); + Mono.from(testee.readBytes(TEST_BUCKETNAME, blobId)).block(); + + + SoftAssertions.assertSoftly(soflty -> { + assertThat(metricFactory.countFor(BLOBSTORE_CACHED_MISS_COUNT_METRIC_NAME)) + .describedAs(BLOBSTORE_CACHED_MISS_COUNT_METRIC_NAME) + .isEqualTo(0); + soflty.assertThat(metricFactory.countFor(BLOBSTORE_CACHED_HIT_COUNT_METRIC_NAME)) + .describedAs(BLOBSTORE_CACHED_HIT_COUNT_METRIC_NAME) + .isEqualTo(0); + assertThat(metricFactory.executionTimesFor(BLOBSTORE_CACHED_LATENCY_METRIC_NAME)) + .describedAs(BLOBSTORE_CACHED_LATENCY_METRIC_NAME) + .hasSize(0); + }); + } + + @Test + public void readBlobStoreCacheShouldPublishTimerMetrics() { + BlobId blobId = Mono.from(backend.save(DEFAULT_BUCKETNAME, APPROXIMATELY_FIVE_KILOBYTES, SIZE_BASED)).block(); + + testee.read(DEFAULT_BUCKETNAME, blobId); + testee.read(DEFAULT_BUCKETNAME, blobId); + + assertThat(metricFactory.executionTimesFor(BLOBSTORE_CACHED_LATENCY_METRIC_NAME)).hasSize(2); + } + + @Test + public void readBytesCacheShouldPublishTimerMetrics() { + BlobId blobId = Mono.from(backend.save(DEFAULT_BUCKETNAME, APPROXIMATELY_FIVE_KILOBYTES, SIZE_BASED)).block(); + + Mono.from(testee.readBytes(DEFAULT_BUCKETNAME, blobId)).block(); + Mono.from(testee.readBytes(DEFAULT_BUCKETNAME, blobId)).block(); + + assertThat(metricFactory.executionTimesFor(BLOBSTORE_CACHED_LATENCY_METRIC_NAME)).hasSize(2); + } + + @Test + public void readBytesShouldNotIncreaseCacheCounterForBigBlobs() { + BlobId blobId = Mono.from(backend.save(DEFAULT_BUCKETNAME, ELEVEN_KILOBYTES, SIZE_BASED)).block(); + + Mono.from(testee.readBytes(DEFAULT_BUCKETNAME, blobId)).block(); + Mono.from(testee.readBytes(DEFAULT_BUCKETNAME, blobId)).block(); + + SoftAssertions.assertSoftly(soflty -> { + assertThat(metricFactory.countFor(BLOBSTORE_CACHED_MISS_COUNT_METRIC_NAME)) + .describedAs(BLOBSTORE_CACHED_MISS_COUNT_METRIC_NAME) + .isEqualTo(0); + soflty.assertThat(metricFactory.countFor(BLOBSTORE_CACHED_HIT_COUNT_METRIC_NAME)) + .describedAs(BLOBSTORE_CACHED_HIT_COUNT_METRIC_NAME) + .isEqualTo(0); + }); + } + + @Test + public void readInputStreamShouldNotIncreaseCacheCounterForBigBlobs() { + BlobId blobId = Mono.from(backend.save(DEFAULT_BUCKETNAME, ELEVEN_KILOBYTES, SIZE_BASED)).block(); + + testee.read(DEFAULT_BUCKETNAME, blobId); + testee.read(DEFAULT_BUCKETNAME, blobId); + + SoftAssertions.assertSoftly(soflty -> { + assertThat(metricFactory.countFor(BLOBSTORE_CACHED_MISS_COUNT_METRIC_NAME)) + .describedAs(BLOBSTORE_CACHED_MISS_COUNT_METRIC_NAME) + .isEqualTo(0); + soflty.assertThat(metricFactory.countFor(BLOBSTORE_CACHED_HIT_COUNT_METRIC_NAME)) + .describedAs(BLOBSTORE_CACHED_HIT_COUNT_METRIC_NAME) + .isEqualTo(0); + }); + } + + + @Test + public void readBlobStoreCacheShouldCountWhenHit() { + BlobId blobId = Mono.from(testee.save(DEFAULT_BUCKETNAME, APPROXIMATELY_FIVE_KILOBYTES, SIZE_BASED)).block(); + + testee.read(DEFAULT_BUCKETNAME, blobId); + testee.read(DEFAULT_BUCKETNAME, blobId); + + assertThat(metricFactory.countFor(BLOBSTORE_CACHED_HIT_COUNT_METRIC_NAME)).isEqualTo(2); + } + + @Test + public void readBytesCacheShouldCountWhenHit() { + BlobId blobId = Mono.from(testee.save(DEFAULT_BUCKETNAME, APPROXIMATELY_FIVE_KILOBYTES, SIZE_BASED)).block(); + + Mono.from(testee.readBytes(DEFAULT_BUCKETNAME, blobId)).block(); + Mono.from(testee.readBytes(DEFAULT_BUCKETNAME, blobId)).block(); + + assertThat(metricFactory.countFor(BLOBSTORE_CACHED_HIT_COUNT_METRIC_NAME)).isEqualTo(2); + } + + + @Test + public void readBlobStoreCacheShouldCountWhenMissed() { + BlobId blobId = Mono.from(backend.save(DEFAULT_BUCKETNAME, APPROXIMATELY_FIVE_KILOBYTES, SIZE_BASED)).block(); + Mono.from(cache.remove(blobId)).block(); + + testee.read(DEFAULT_BUCKETNAME, blobId); + + assertThat(metricFactory.countFor(BLOBSTORE_CACHED_MISS_COUNT_METRIC_NAME)).isEqualTo(1); + } + + @Test + public void readBytesCacheShouldCountWhenMissed() { + BlobId blobId = Mono.from(backend.save(DEFAULT_BUCKETNAME, APPROXIMATELY_FIVE_KILOBYTES, SIZE_BASED)).block(); + Mono.from(cache.remove(blobId)).block(); + + Mono.from(testee.readBytes(DEFAULT_BUCKETNAME, blobId)).block(); + + assertThat(metricFactory.countFor(BLOBSTORE_CACHED_MISS_COUNT_METRIC_NAME)).isEqualTo(1); + } + + @Test + public void metricsShouldNotWorkExceptLatencyWhenReadNonExistingBlob() { + SoftAssertions.assertSoftly(soflty -> { + soflty.assertThatThrownBy(() -> testee.read(DEFAULT_BUCKETNAME, new TestBlobId.Factory().randomId())) + .isInstanceOf(ObjectNotFoundException.class); + + soflty.assertThat(metricFactory.countFor(BLOBSTORE_CACHED_MISS_COUNT_METRIC_NAME)) + .describedAs(BLOBSTORE_CACHED_MISS_COUNT_METRIC_NAME) + .isEqualTo(0); + soflty.assertThat(metricFactory.countFor(BLOBSTORE_CACHED_HIT_COUNT_METRIC_NAME)) + .describedAs(BLOBSTORE_CACHED_HIT_COUNT_METRIC_NAME) + .isEqualTo(0); + soflty.assertThat(metricFactory.executionTimesFor(BLOBSTORE_CACHED_LATENCY_METRIC_NAME)) + .describedAs(BLOBSTORE_CACHED_LATENCY_METRIC_NAME) + .hasSize(1); + }); + } + + @Test + public void metricsShouldNotWorkExceptLatencyWhenReadNonExistingBlobAsBytes() { + SoftAssertions.assertSoftly(soflty -> { + soflty.assertThatThrownBy(() -> Mono.from(testee.readBytes(DEFAULT_BUCKETNAME, new TestBlobId.Factory().randomId())).blockOptional()) + .isInstanceOf(ObjectNotFoundException.class); + + soflty.assertThat(metricFactory.countFor(BLOBSTORE_CACHED_MISS_COUNT_METRIC_NAME)) + .describedAs(BLOBSTORE_CACHED_MISS_COUNT_METRIC_NAME) + .isEqualTo(0); + soflty.assertThat(metricFactory.countFor(BLOBSTORE_CACHED_HIT_COUNT_METRIC_NAME)) + .describedAs(BLOBSTORE_CACHED_HIT_COUNT_METRIC_NAME) + .isEqualTo(0); + soflty.assertThat(metricFactory.executionTimesFor(BLOBSTORE_CACHED_LATENCY_METRIC_NAME)) + .describedAs(BLOBSTORE_CACHED_LATENCY_METRIC_NAME) + .hasSize(1); + }); + } } } diff --git a/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/cache/CassandraBlobStoreCacheTest.java b/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/cache/CassandraBlobStoreCacheTest.java index 10c1feb..9841853 100644 --- a/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/cache/CassandraBlobStoreCacheTest.java +++ b/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/cache/CassandraBlobStoreCacheTest.java @@ -27,8 +27,8 @@ import org.apache.james.backends.cassandra.CassandraCluster; import org.apache.james.backends.cassandra.CassandraClusterExtension; import org.apache.james.blob.api.BlobId; import org.apache.james.blob.api.HashBlobId; -import org.junit.jupiter.api.Test; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import reactor.core.publisher.Mono; --------------------------------------------------------------------- To unsubscribe, e-mail: server-dev-unsubscr...@james.apache.org For additional commands, e-mail: server-dev-h...@james.apache.org