This is an automated email from the ASF dual-hosted git repository.
rcordier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
The following commit(s) were added to refs/heads/master by this push:
new ebded1d42c JAMES-3925 Rework JMAP quota cleanup
ebded1d42c is described below
commit ebded1d42c91af4ccd20179f92302c34cdbd8383
Author: Tung Tran <[email protected]>
AuthorDate: Fri Aug 25 15:16:20 2023 +0700
JAMES-3925 Rework JMAP quota cleanup
---
.../james/modules/data/CassandraJmapModule.java | 2 -
.../jmap/cassandra/upload/BucketNameGenerator.java | 54 ---------------
.../upload/CassandraUploadRepository.java | 36 +++++-----
.../jmap/cassandra/upload/UploadConfiguration.java | 61 -----------------
.../james/jmap/cassandra/upload/UploadDAO.java | 33 ++++-----
.../james/jmap/cassandra/upload/UploadModule.java | 13 +---
.../cassandra/upload/BucketNameGeneratorTest.java | 67 ------------------
.../upload/CassandraUploadRepositoryTest.java | 24 +------
.../webadmin/data/jmap/JmapUploadRoutesTest.java | 80 ++++++++++++----------
9 files changed, 80 insertions(+), 290 deletions(-)
diff --git
a/server/container/guice/cassandra/src/main/java/org/apache/james/modules/data/CassandraJmapModule.java
b/server/container/guice/cassandra/src/main/java/org/apache/james/modules/data/CassandraJmapModule.java
index a6c0884c27..f6a1310753 100644
---
a/server/container/guice/cassandra/src/main/java/org/apache/james/modules/data/CassandraJmapModule.java
+++
b/server/container/guice/cassandra/src/main/java/org/apache/james/modules/data/CassandraJmapModule.java
@@ -59,7 +59,6 @@ import
org.apache.james.jmap.cassandra.pushsubscription.CassandraPushSubscriptio
import
org.apache.james.jmap.cassandra.pushsubscription.CassandraPushSubscriptionRepository;
import org.apache.james.jmap.cassandra.upload.CassandraUploadRepository;
import org.apache.james.jmap.cassandra.upload.CassandraUploadUsageRepository;
-import org.apache.james.jmap.cassandra.upload.UploadConfiguration;
import org.apache.james.jmap.cassandra.upload.UploadDAO;
import org.apache.james.jmap.cassandra.upload.UploadModule;
import org.apache.james.user.api.DeleteUserDataTaskStep;
@@ -83,7 +82,6 @@ public class CassandraJmapModule extends AbstractModule {
bind(UploadDAO.class).in(Scopes.SINGLETON);
bind(UploadRepository.class).to(CassandraUploadRepository.class);
bind(UploadUsageRepository.class).to(CassandraUploadUsageRepository.class);
-
bind(UploadConfiguration.class).toInstance(UploadConfiguration.SINGLETON);
bind(CassandraCustomIdentityDAO.class).in(Scopes.SINGLETON);
bind(CustomIdentityDAO.class).to(CassandraCustomIdentityDAO.class);
diff --git
a/server/data/data-jmap-cassandra/src/main/java/org/apache/james/jmap/cassandra/upload/BucketNameGenerator.java
b/server/data/data-jmap-cassandra/src/main/java/org/apache/james/jmap/cassandra/upload/BucketNameGenerator.java
deleted file mode 100644
index b96dc458a6..0000000000
---
a/server/data/data-jmap-cassandra/src/main/java/org/apache/james/jmap/cassandra/upload/BucketNameGenerator.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/****************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one *
- * or more contributor license agreements. See the NOTICE file *
- * distributed with this work for additional information *
- * regarding copyright ownership. The ASF licenses this file *
- * to you under the Apache License, Version 2.0 (the *
- * "License"); you may not use this file except in compliance *
- * with the License. You may obtain a copy of the License at *
- * *
- * http://www.apache.org/licenses/LICENSE-2.0 *
- * *
- * Unless required by applicable law or agreed to in writing, *
- * software distributed under the License is distributed on an *
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
- * KIND, either express or implied. See the License for the *
- * specific language governing permissions and limitations *
- * under the License. *
- ****************************************************************/
-
-package org.apache.james.jmap.cassandra.upload;
-
-import java.time.Clock;
-import java.time.LocalDate;
-import java.time.ZoneOffset;
-import java.time.temporal.ChronoUnit;
-import java.util.function.Predicate;
-
-import javax.inject.Inject;
-
-public class BucketNameGenerator {
- private final Clock clock;
-
- @Inject
- public BucketNameGenerator(Clock clock) {
- this.clock = clock;
- }
-
- public UploadBucketName current() {
- int weekCount = currentWeekCount();
- return new UploadBucketName(weekCount);
- }
-
- public Predicate<UploadBucketName> evictionPredicate() {
- final int currentWeekCount = currentWeekCount();
-
- return uploadBucketName -> uploadBucketName.getWeekNumber() <
currentWeekCount - 1;
- }
-
- private int currentWeekCount() {
- return Math.toIntExact(ChronoUnit.WEEKS.between(
- LocalDate.ofEpochDay(0),
- LocalDate.ofInstant(clock.instant(), ZoneOffset.UTC)));
- }
-}
diff --git
a/server/data/data-jmap-cassandra/src/main/java/org/apache/james/jmap/cassandra/upload/CassandraUploadRepository.java
b/server/data/data-jmap-cassandra/src/main/java/org/apache/james/jmap/cassandra/upload/CassandraUploadRepository.java
index 5d525d89b2..d890284126 100644
---
a/server/data/data-jmap-cassandra/src/main/java/org/apache/james/jmap/cassandra/upload/CassandraUploadRepository.java
+++
b/server/data/data-jmap-cassandra/src/main/java/org/apache/james/jmap/cassandra/upload/CassandraUploadRepository.java
@@ -19,9 +19,12 @@
package org.apache.james.jmap.cassandra.upload;
import static org.apache.james.blob.api.BlobStore.StoragePolicy.LOW_COST;
+import static org.apache.james.util.ReactorUtils.DEFAULT_CONCURRENCY;
import java.io.InputStream;
import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
import javax.inject.Inject;
@@ -34,7 +37,6 @@ import org.apache.james.jmap.api.model.UploadMetaData;
import org.apache.james.jmap.api.model.UploadNotFoundException;
import org.apache.james.jmap.api.upload.UploadRepository;
import org.apache.james.mailbox.model.ContentType;
-import org.reactivestreams.Publisher;
import com.datastax.oss.driver.api.core.uuid.Uuids;
import com.google.common.io.CountingInputStream;
@@ -43,56 +45,56 @@ import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
public class CassandraUploadRepository implements UploadRepository {
+
+ public static final BucketName UPLOAD_BUCKET =
BucketName.of("jmap-uploads");
+ public static final Duration EXPIRE_DURATION = Duration.ofDays(7);
private final UploadDAO uploadDAO;
private final BlobStore blobStore;
- private final BucketNameGenerator bucketNameGenerator;
private final Clock clock;
@Inject
- public CassandraUploadRepository(UploadDAO uploadDAO, BlobStore blobStore,
BucketNameGenerator bucketNameGenerator, Clock clock) {
+ public CassandraUploadRepository(UploadDAO uploadDAO, BlobStore blobStore,
Clock clock) {
this.uploadDAO = uploadDAO;
this.blobStore = blobStore;
- this.bucketNameGenerator = bucketNameGenerator;
this.clock = clock;
}
@Override
- public Publisher<UploadMetaData> upload(InputStream data, ContentType
contentType, Username user) {
+ public Mono<UploadMetaData> upload(InputStream data, ContentType
contentType, Username user) {
UploadId uploadId = generateId();
- UploadBucketName uploadBucketName = bucketNameGenerator.current();
- BucketName bucketName = uploadBucketName.asBucketName();
return Mono.fromCallable(() -> new CountingInputStream(data))
- .flatMap(countingInputStream ->
Mono.from(blobStore.save(bucketName, countingInputStream, LOW_COST))
- .map(blobId -> new UploadDAO.UploadRepresentation(uploadId,
bucketName, blobId, contentType, countingInputStream.getCount(), user,
clock.instant()))
+ .flatMap(countingInputStream ->
Mono.from(blobStore.save(UPLOAD_BUCKET, countingInputStream, LOW_COST))
+ .map(blobId -> new UploadDAO.UploadRepresentation(uploadId,
blobId, contentType, countingInputStream.getCount(), user, clock.instant()))
.flatMap(upload -> uploadDAO.save(upload)
.thenReturn(upload.toUploadMetaData())));
}
@Override
- public Publisher<Upload> retrieve(UploadId id, Username user) {
+ public Mono<Upload> retrieve(UploadId id, Username user) {
return uploadDAO.retrieve(user, id)
.map(upload -> Upload.from(upload.toUploadMetaData(),
- () -> blobStore.read(upload.getBucketName(),
upload.getBlobId(), LOW_COST)))
+ () -> blobStore.read(UPLOAD_BUCKET, upload.getBlobId(),
LOW_COST)))
.switchIfEmpty(Mono.error(() -> new UploadNotFoundException(id)));
}
@Override
- public Publisher<Void> delete(UploadId id, Username user) {
+ public Mono<Void> delete(UploadId id, Username user) {
return uploadDAO.delete(user, id);
}
@Override
- public Publisher<UploadMetaData> listUploads(Username user) {
+ public Flux<UploadMetaData> listUploads(Username user) {
return uploadDAO.list(user)
.map(UploadDAO.UploadRepresentation::toUploadMetaData);
}
public Mono<Void> purge() {
- return Flux.from(blobStore.listBuckets())
- .<UploadBucketName>handle((bucketName, sink) ->
UploadBucketName.ofBucket(bucketName).ifPresentOrElse(sink::next,
sink::complete))
- .filter(bucketNameGenerator.evictionPredicate())
- .concatMap(bucket -> blobStore.deleteBucket(bucket.asBucketName()))
+ Instant sevenDaysAgo = clock.instant().minus(EXPIRE_DURATION);
+ return Flux.from(uploadDAO.all())
+ .filter(upload -> upload.getUploadDate().isBefore(sevenDaysAgo))
+ .flatMap(upload -> Mono.from(blobStore.delete(UPLOAD_BUCKET,
upload.getBlobId()))
+ .then(uploadDAO.delete(upload.getUser(), upload.getId())),
DEFAULT_CONCURRENCY)
.then();
}
diff --git
a/server/data/data-jmap-cassandra/src/main/java/org/apache/james/jmap/cassandra/upload/UploadConfiguration.java
b/server/data/data-jmap-cassandra/src/main/java/org/apache/james/jmap/cassandra/upload/UploadConfiguration.java
deleted file mode 100644
index f54d24ddf7..0000000000
---
a/server/data/data-jmap-cassandra/src/main/java/org/apache/james/jmap/cassandra/upload/UploadConfiguration.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/****************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one *
- * or more contributor license agreements. See the NOTICE file *
- * distributed with this work for additional information *
- * regarding copyright ownership. The ASF licenses this file *
- * to you under the Apache License, Version 2.0 (the *
- * "License"); you may not use this file except in compliance *
- * with the License. You may obtain a copy of the License at *
- * *
- * http://www.apache.org/licenses/LICENSE-2.0 *
- * *
- * Unless required by applicable law or agreed to in writing, *
- * software distributed under the License is distributed on an *
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
- * KIND, either express or implied. See the License for the *
- * specific language governing permissions and limitations *
- * under the License. *
- ****************************************************************/
-
-package org.apache.james.jmap.cassandra.upload;
-
-import java.time.Duration;
-
-import com.google.common.base.MoreObjects;
-import com.google.common.base.Objects;
-
-public class UploadConfiguration {
- public static final UploadConfiguration SINGLETON = new
UploadConfiguration(Duration.ofDays(7));
-
- private final Duration uploadTtlDuration;
-
- public UploadConfiguration(Duration uploadTtlDuration) {
- this.uploadTtlDuration = uploadTtlDuration;
- }
-
- public Duration getUploadTtlDuration() {
- return uploadTtlDuration;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof UploadConfiguration) {
- UploadConfiguration other = (UploadConfiguration) obj;
- return Objects.equal(uploadTtlDuration, other.uploadTtlDuration);
- }
- return false;
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(uploadTtlDuration);
- }
-
- @Override
- public String toString() {
- return MoreObjects
- .toStringHelper(this)
- .add("uploadTtlDuration", uploadTtlDuration)
- .toString();
- }
-}
diff --git
a/server/data/data-jmap-cassandra/src/main/java/org/apache/james/jmap/cassandra/upload/UploadDAO.java
b/server/data/data-jmap-cassandra/src/main/java/org/apache/james/jmap/cassandra/upload/UploadDAO.java
index 8b7c04133f..2f74a21b15 100644
---
a/server/data/data-jmap-cassandra/src/main/java/org/apache/james/jmap/cassandra/upload/UploadDAO.java
+++
b/server/data/data-jmap-cassandra/src/main/java/org/apache/james/jmap/cassandra/upload/UploadDAO.java
@@ -24,7 +24,6 @@ import static
com.datastax.oss.driver.api.querybuilder.QueryBuilder.deleteFrom;
import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.insertInto;
import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.selectFrom;
import static org.apache.james.jmap.cassandra.upload.UploadModule.BLOB_ID;
-import static org.apache.james.jmap.cassandra.upload.UploadModule.BUCKET_ID;
import static org.apache.james.jmap.cassandra.upload.UploadModule.CONTENT_TYPE;
import static org.apache.james.jmap.cassandra.upload.UploadModule.ID;
import static org.apache.james.jmap.cassandra.upload.UploadModule.SIZE;
@@ -40,7 +39,6 @@ import javax.inject.Inject;
import org.apache.james.backends.cassandra.utils.CassandraAsyncExecutor;
import org.apache.james.blob.api.BlobId;
-import org.apache.james.blob.api.BucketName;
import org.apache.james.core.Username;
import org.apache.james.jmap.api.model.UploadId;
import org.apache.james.jmap.api.model.UploadMetaData;
@@ -59,18 +57,16 @@ import reactor.core.publisher.Mono;
public class UploadDAO {
public static class UploadRepresentation {
private final UploadId id;
- private final BucketName bucketName;
private final BlobId blobId;
private final ContentType contentType;
private final long size;
private final Username user;
private final Instant uploadDate;
- public UploadRepresentation(UploadId id, BucketName bucketName, BlobId
blobId, ContentType contentType, long size, Username user, Instant uploadDate) {
+ public UploadRepresentation(UploadId id, BlobId blobId, ContentType
contentType, long size, Username user, Instant uploadDate) {
this.user = user;
Preconditions.checkArgument(size >= 0, "Size must be strictly
positive");
this.id = id;
- this.bucketName = bucketName;
this.blobId = blobId;
this.contentType = contentType;
this.size = size;
@@ -81,10 +77,6 @@ public class UploadDAO {
return id;
}
- public BucketName getBucketName() {
- return bucketName;
- }
-
public BlobId getBlobId() {
return blobId;
}
@@ -114,7 +106,6 @@ public class UploadDAO {
if (obj instanceof UploadRepresentation) {
UploadRepresentation other = (UploadRepresentation) obj;
return Objects.equal(id, other.id)
- && Objects.equal(bucketName, other.bucketName)
&& Objects.equal(user, other.user)
&& Objects.equal(blobId, other.blobId)
&& Objects.equal(contentType, other.contentType)
@@ -126,7 +117,7 @@ public class UploadDAO {
@Override
public int hashCode() {
- return Objects.hashCode(id, bucketName, blobId, contentType, size,
user, uploadDate);
+ return Objects.hashCode(id, blobId, contentType, size, user,
uploadDate);
}
@Override
@@ -134,7 +125,6 @@ public class UploadDAO {
return MoreObjects
.toStringHelper(this)
.add("id", id)
- .add("bucketName", bucketName)
.add("blobId", blobId)
.add("contentType", contentType)
.add("user", user)
@@ -150,22 +140,21 @@ public class UploadDAO {
private final PreparedStatement insert;
private final PreparedStatement selectOne;
private final PreparedStatement delete;
-
private final PreparedStatement list;
+ private final PreparedStatement all;
+
@Inject
- public UploadDAO(CqlSession session, BlobId.Factory blobIdFactory,
UploadConfiguration configuration) {
+ public UploadDAO(CqlSession session, BlobId.Factory blobIdFactory) {
this.executor = new CassandraAsyncExecutor(session);
this.blobIdFactory = blobIdFactory;
this.insert = session.prepare(insertInto(TABLE_NAME)
.value(ID, bindMarker(ID))
- .value(BUCKET_ID, bindMarker(BUCKET_ID))
.value(BLOB_ID, bindMarker(BLOB_ID))
.value(SIZE, bindMarker(SIZE))
.value(USER, bindMarker(USER))
.value(CONTENT_TYPE, bindMarker(CONTENT_TYPE))
.value(UPLOAD_DATE, bindMarker(UPLOAD_DATE))
- .usingTtl((int) configuration.getUploadTtlDuration().getSeconds())
.build());
this.list = session.prepare(selectFrom(TABLE_NAME)
@@ -183,13 +172,17 @@ public class UploadDAO {
.whereColumn(USER).isEqualTo(bindMarker(USER))
.whereColumn(ID).isEqualTo(bindMarker(ID))
.build());
+
+ this.all = session.prepare(selectFrom(TABLE_NAME)
+ .all()
+ .allowFiltering()
+ .build());
}
public Mono<Void> save(UploadRepresentation uploadRepresentation) {
return executor.executeVoid(insert.bind()
.setString(USER, uploadRepresentation.getUser().asString())
.setUuid(ID, uploadRepresentation.getId().getId())
- .setString(BUCKET_ID,
uploadRepresentation.getBucketName().asString())
.setString(BLOB_ID, uploadRepresentation.getBlobId().asString())
.setLong(SIZE, uploadRepresentation.getSize())
.setInstant(UPLOAD_DATE, uploadRepresentation.getUploadDate())
@@ -215,9 +208,13 @@ public class UploadDAO {
.setUuid(ID, uploadId.getId()));
}
+ public Flux<UploadRepresentation> all() {
+ return Flux.from(executor.executeRows(all.bind()))
+ .map(rowToUploadRepresentation());
+ }
+
private Function<Row, UploadRepresentation> rowToUploadRepresentation() {
return row -> new UploadRepresentation(UploadId.from(row.getUuid(ID)),
- BucketName.of(row.getString(BUCKET_ID)),
blobIdFactory.from(row.getString(BLOB_ID)),
ContentType.of(row.getString(CONTENT_TYPE)),
row.getLong(SIZE),
diff --git
a/server/data/data-jmap-cassandra/src/main/java/org/apache/james/jmap/cassandra/upload/UploadModule.java
b/server/data/data-jmap-cassandra/src/main/java/org/apache/james/jmap/cassandra/upload/UploadModule.java
index eeb166a712..8711a7a608 100644
---
a/server/data/data-jmap-cassandra/src/main/java/org/apache/james/jmap/cassandra/upload/UploadModule.java
+++
b/server/data/data-jmap-cassandra/src/main/java/org/apache/james/jmap/cassandra/upload/UploadModule.java
@@ -19,40 +19,29 @@
package org.apache.james.jmap.cassandra.upload;
-import static
com.datastax.oss.driver.api.querybuilder.SchemaBuilder.RowsPerPartition.rows;
-import static
com.datastax.oss.driver.api.querybuilder.schema.compaction.TimeWindowCompactionStrategy.CompactionWindowUnit.DAYS;
-import static
org.apache.james.backends.cassandra.utils.CassandraConstants.DEFAULT_CACHED_ROW_PER_PARTITION;
-
import org.apache.james.backends.cassandra.components.CassandraModule;
import com.datastax.oss.driver.api.core.CqlIdentifier;
import com.datastax.oss.driver.api.core.type.DataTypes;
-import com.datastax.oss.driver.api.querybuilder.SchemaBuilder;
public interface UploadModule {
- String TABLE_NAME = "uploads_2";
+ String TABLE_NAME = "uploadsV2";
CqlIdentifier ID = CqlIdentifier.fromCql("id");
CqlIdentifier CONTENT_TYPE = CqlIdentifier.fromCql("content_type");
CqlIdentifier SIZE = CqlIdentifier.fromCql("size");
- CqlIdentifier BUCKET_ID = CqlIdentifier.fromCql("bucket_id");
CqlIdentifier BLOB_ID = CqlIdentifier.fromCql("blob_id");
CqlIdentifier USER = CqlIdentifier.fromCql("user");
CqlIdentifier UPLOAD_DATE = CqlIdentifier.fromCql("upload_date");
CassandraModule MODULE = CassandraModule.table(TABLE_NAME)
.comment("Holds JMAP uploads")
- .options(options -> options
- .withCompaction(SchemaBuilder.timeWindowCompactionStrategy()
- .withCompactionWindow(7, DAYS))
- .withCaching(true, rows(DEFAULT_CACHED_ROW_PER_PARTITION)))
.statement(statement -> types -> statement
.withPartitionKey(USER, DataTypes.TEXT)
.withClusteringColumn(ID, DataTypes.TIMEUUID)
.withColumn(CONTENT_TYPE, DataTypes.TEXT)
.withColumn(SIZE, DataTypes.BIGINT)
- .withColumn(BUCKET_ID, DataTypes.TEXT)
.withColumn(BLOB_ID, DataTypes.TEXT)
.withColumn(UPLOAD_DATE, DataTypes.TIMESTAMP))
.build();
diff --git
a/server/data/data-jmap-cassandra/src/test/java/org/apache/james/jmap/cassandra/upload/BucketNameGeneratorTest.java
b/server/data/data-jmap-cassandra/src/test/java/org/apache/james/jmap/cassandra/upload/BucketNameGeneratorTest.java
deleted file mode 100644
index ee3d6388d0..0000000000
---
a/server/data/data-jmap-cassandra/src/test/java/org/apache/james/jmap/cassandra/upload/BucketNameGeneratorTest.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/****************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one *
- * or more contributor license agreements. See the NOTICE file *
- * distributed with this work for additional information *
- * regarding copyright ownership. The ASF licenses this file *
- * to you under the Apache License, Version 2.0 (the *
- * "License"); you may not use this file except in compliance *
- * with the License. You may obtain a copy of the License at *
- * *
- * http://www.apache.org/licenses/LICENSE-2.0 *
- * *
- * Unless required by applicable law or agreed to in writing, *
- * software distributed under the License is distributed on an *
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
- * KIND, either express or implied. See the License for the *
- * specific language governing permissions and limitations *
- * under the License. *
- ****************************************************************/
-
-package org.apache.james.jmap.cassandra.upload;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-import java.time.Clock;
-import java.time.ZoneOffset;
-import java.time.ZonedDateTime;
-
-import org.junit.jupiter.api.Test;
-
-class BucketNameGeneratorTest {
- ZonedDateTime NOW = ZonedDateTime.parse("2015-10-30T14:12:00Z");
-
- @Test
- void currentShouldReturnCorrectValue() {
- BucketNameGenerator generator = new
BucketNameGenerator(Clock.fixed(NOW.toInstant(), ZoneOffset.UTC));
-
- assertThat(generator.current()).isEqualTo(new UploadBucketName(2391));
- }
-
- @Test
- void evictionPredicateShouldKeepPresentBucket() {
- BucketNameGenerator generator = new
BucketNameGenerator(Clock.fixed(NOW.toInstant(), ZoneOffset.UTC));
-
- assertThat(generator.evictionPredicate().test(new
UploadBucketName(2391))).isFalse();
- }
-
- @Test
- void evictionPredicateShouldKeepFutureBuckets() {
- BucketNameGenerator generator = new
BucketNameGenerator(Clock.fixed(NOW.toInstant(), ZoneOffset.UTC));
-
- assertThat(generator.evictionPredicate().test(new
UploadBucketName(2392))).isFalse();
- }
-
- @Test
- void evictionPredicateShouldKeepRecentBuckets() {
- BucketNameGenerator generator = new
BucketNameGenerator(Clock.fixed(NOW.toInstant(), ZoneOffset.UTC));
-
- assertThat(generator.evictionPredicate().test(new
UploadBucketName(2390))).isFalse();
- }
-
- @Test
- void evictionPredicateShouldMatchOldBuckets() {
- BucketNameGenerator generator = new
BucketNameGenerator(Clock.fixed(NOW.toInstant(), ZoneOffset.UTC));
-
- assertThat(generator.evictionPredicate().test(new
UploadBucketName(2389))).isTrue();
- }
-}
\ No newline at end of file
diff --git
a/server/data/data-jmap-cassandra/src/test/java/org/apache/james/jmap/cassandra/upload/CassandraUploadRepositoryTest.java
b/server/data/data-jmap-cassandra/src/test/java/org/apache/james/jmap/cassandra/upload/CassandraUploadRepositoryTest.java
index bcab9871dd..e5cf9403be 100644
---
a/server/data/data-jmap-cassandra/src/test/java/org/apache/james/jmap/cassandra/upload/CassandraUploadRepositoryTest.java
+++
b/server/data/data-jmap-cassandra/src/test/java/org/apache/james/jmap/cassandra/upload/CassandraUploadRepositoryTest.java
@@ -19,30 +19,21 @@
package org.apache.james.jmap.cassandra.upload;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-
import java.time.Clock;
-import java.time.Duration;
import org.apache.james.backends.cassandra.CassandraClusterExtension;
import org.apache.james.blob.api.BucketName;
import org.apache.james.blob.api.HashBlobId;
import org.apache.james.blob.memory.MemoryBlobStoreDAO;
-import org.apache.james.core.Username;
import org.apache.james.jmap.api.model.UploadId;
-import org.apache.james.jmap.api.model.UploadNotFoundException;
import org.apache.james.jmap.api.upload.UploadRepository;
import org.apache.james.jmap.api.upload.UploadRepositoryContract;
-import org.apache.james.mailbox.model.ContentType;
import org.apache.james.server.blob.deduplication.DeDuplicationBlobStore;
import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import com.datastax.oss.driver.api.core.uuid.Uuids;
-import reactor.core.publisher.Mono;
-
class CassandraUploadRepositoryTest implements UploadRepositoryContract {
@RegisterExtension
static CassandraClusterExtension cassandra = new
CassandraClusterExtension(UploadModule.MODULE);
@@ -52,9 +43,8 @@ class CassandraUploadRepositoryTest implements
UploadRepositoryContract {
void setUp() {
Clock clock = Clock.systemUTC();
testee = new CassandraUploadRepository(new
UploadDAO(cassandra.getCassandraCluster().getConf(),
- new HashBlobId.Factory(),
- new UploadConfiguration(Duration.ofSeconds(5))), new
DeDuplicationBlobStore(new MemoryBlobStoreDAO(), BucketName.of("default"), new
HashBlobId.Factory()),
- new BucketNameGenerator(clock), clock);
+ new HashBlobId.Factory()), new DeDuplicationBlobStore(new
MemoryBlobStoreDAO(), BucketName.of("default"), new HashBlobId.Factory()),
+ clock);
}
@Override
@@ -67,14 +57,4 @@ class CassandraUploadRepositoryTest implements
UploadRepositoryContract {
return testee;
}
- @Test
- void uploadShouldExpire() throws Exception {
- Username bob = Username.of("bob");
- UploadId id = Mono.from(testee.upload(data(),
ContentType.of("text/plain"), bob)).block().uploadId();
-
- Thread.sleep(6000);
-
- assertThatThrownBy(() -> Mono.from(testee.retrieve(id,
bob)).blockOptional())
- .isInstanceOf(UploadNotFoundException.class);
- }
}
\ No newline at end of file
diff --git
a/server/protocols/webadmin/webadmin-jmap/src/test/java/org/apache/james/webadmin/data/jmap/JmapUploadRoutesTest.java
b/server/protocols/webadmin/webadmin-jmap/src/test/java/org/apache/james/webadmin/data/jmap/JmapUploadRoutesTest.java
index 173864f655..267528ae2c 100644
---
a/server/protocols/webadmin/webadmin-jmap/src/test/java/org/apache/james/webadmin/data/jmap/JmapUploadRoutesTest.java
+++
b/server/protocols/webadmin/webadmin-jmap/src/test/java/org/apache/james/webadmin/data/jmap/JmapUploadRoutesTest.java
@@ -21,6 +21,7 @@ package org.apache.james.webadmin.data.jmap;
import static io.restassured.RestAssured.given;
import static io.restassured.http.ContentType.JSON;
+import static
org.apache.james.jmap.cassandra.upload.CassandraUploadRepository.UPLOAD_BUCKET;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy;
import static org.awaitility.Durations.ONE_HUNDRED_MILLISECONDS;
@@ -29,11 +30,10 @@ import static org.hamcrest.Matchers.notNullValue;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
-import java.time.Duration;
import java.time.ZonedDateTime;
-import java.util.List;
import java.util.Map;
+import org.apache.commons.io.IOUtils;
import org.apache.james.backends.cassandra.CassandraClusterExtension;
import org.apache.james.backends.cassandra.components.CassandraModule;
import org.apache.james.blob.api.BlobStore;
@@ -42,15 +42,14 @@ import org.apache.james.blob.api.HashBlobId;
import org.apache.james.blob.memory.MemoryBlobStoreDAO;
import org.apache.james.core.Username;
import org.apache.james.jmap.api.model.UploadId;
+import org.apache.james.jmap.api.model.UploadMetaData;
import org.apache.james.jmap.api.model.UploadNotFoundException;
-import org.apache.james.jmap.cassandra.upload.BucketNameGenerator;
import org.apache.james.jmap.cassandra.upload.CassandraUploadRepository;
-import org.apache.james.jmap.cassandra.upload.UploadConfiguration;
import org.apache.james.jmap.cassandra.upload.UploadDAO;
import org.apache.james.jmap.cassandra.upload.UploadModule;
import org.apache.james.json.DTOConverter;
import org.apache.james.mailbox.model.ContentType;
-import org.apache.james.server.blob.deduplication.DeDuplicationBlobStore;
+import org.apache.james.server.blob.deduplication.PassThroughBlobStore;
import org.apache.james.task.Hostname;
import org.apache.james.task.MemoryTaskManager;
import org.apache.james.utils.UpdatableTickingClock;
@@ -67,7 +66,6 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
-import org.testcontainers.shaded.org.apache.commons.io.IOUtils;
import io.restassured.RestAssured;
import reactor.core.publisher.Flux;
@@ -88,7 +86,6 @@ class JmapUploadRoutesTest {
private WebAdminServer webAdminServer;
private MemoryTaskManager taskManager;
private BlobStore blobStore;
- private BucketNameGenerator bucketNameGenerator;
private CassandraUploadRepository cassandraUploadRepository;
private UpdatableTickingClock clock;
@@ -100,16 +97,12 @@ class JmapUploadRoutesTest {
void setUp() {
taskManager = new MemoryTaskManager(new Hostname("foo"));
clock = new UpdatableTickingClock(TIMESTAMP.toInstant());
- bucketNameGenerator = new BucketNameGenerator(clock);
- blobStore = new DeDuplicationBlobStore(new MemoryBlobStoreDAO(),
+ blobStore = new PassThroughBlobStore(new MemoryBlobStoreDAO(),
BucketName.of("default"),
new HashBlobId.Factory());
cassandraUploadRepository = new CassandraUploadRepository(new
UploadDAO(cassandraCluster.getCassandraCluster().getConf(),
- new HashBlobId.Factory(),
- new UploadConfiguration(Duration.ofSeconds(5))),
- blobStore,
- bucketNameGenerator, clock);
+ new HashBlobId.Factory()), blobStore, clock);
JsonTransformer jsonTransformer = new JsonTransformer();
TasksRoutes tasksRoutes = new TasksRoutes(taskManager,
jsonTransformer,
DTOConverter.of(UploadCleanupTaskAdditionalInformationDTO.SERIALIZATION_MODULE));
@@ -198,10 +191,8 @@ class JmapUploadRoutesTest {
}
@Test
- void cleanUploadTaskShouldRemoveExpiredBucket() {
- BucketName expiredBucket =
bucketNameGenerator.current().asBucketName();
-
- Mono.from(cassandraUploadRepository.upload(DATA, CONTENT_TYPE,
USERNAME)).block();
+ void cleanUploadTaskShouldRemoveExpiredBlob() {
+ UploadMetaData uploadMetaData =
Mono.from(cassandraUploadRepository.upload(DATA, CONTENT_TYPE,
USERNAME)).block();
clock.setInstant(TIMESTAMP.plusWeeks(3).toInstant());
@@ -216,15 +207,13 @@ class JmapUploadRoutesTest {
.when()
.get(taskId + "/await");
- assertThat(Flux.from(blobStore.listBuckets()).collectList().block())
- .doesNotContain(expiredBucket);
+
assertThat(Flux.from(blobStore.listBlobs(UPLOAD_BUCKET)).collectList().block())
+ .doesNotContain(uploadMetaData.blobId());
}
@Test
- void cleanUploadTaskShouldNotRemoveUnExpiredBucket() {
- BucketName unExpiredBucket =
bucketNameGenerator.current().asBucketName();
-
- Mono.from(cassandraUploadRepository.upload(DATA, CONTENT_TYPE,
USERNAME)).block();
+ void cleanUploadTaskShouldNotRemoveUnExpiredBlob() {
+ UploadMetaData upload =
Mono.from(cassandraUploadRepository.upload(DATA, CONTENT_TYPE,
USERNAME)).block();
String taskId = given()
.queryParam("scope", "expired")
@@ -237,26 +226,41 @@ class JmapUploadRoutesTest {
.when()
.get(taskId + "/await");
- assertThat(Flux.from(blobStore.listBuckets()).collectList().block())
- .contains(unExpiredBucket);
+
assertThat(Flux.from(blobStore.listBlobs(UPLOAD_BUCKET)).collectList().block())
+ .containsOnly(upload.blobId());
+ }
+
+ @Test
+ void cleanUploadTaskShouldNotRemoveUnExpiredUpload() {
+ UploadMetaData upload =
Mono.from(cassandraUploadRepository.upload(DATA, CONTENT_TYPE,
USERNAME)).block();
+
+ String taskId = given()
+ .queryParam("scope", "expired")
+ .delete()
+ .jsonPath()
+ .get("taskId");
+
+ given()
+ .basePath(TasksRoutes.BASE)
+ .when()
+ .get(taskId + "/await");
+
+
assertThat(cassandraUploadRepository.listUploads(USERNAME).collectList().block())
+ .containsOnly(upload);
}
@Test
void cleanUploadTaskShouldSuccessWhenMixCase() {
- BucketName expiredBucketName1 =
bucketNameGenerator.current().asBucketName();
- Mono.from(cassandraUploadRepository.upload(DATA, CONTENT_TYPE,
USERNAME)).block();
+ UploadMetaData upload1 =
Mono.from(cassandraUploadRepository.upload(IOUtils.toInputStream("DATA 1",
StandardCharsets.UTF_8), CONTENT_TYPE, USERNAME)).block();
clock.setInstant(TIMESTAMP.plusWeeks(1).toInstant());
- BucketName expiredBucketName2 =
bucketNameGenerator.current().asBucketName();
- Mono.from(cassandraUploadRepository.upload(DATA, CONTENT_TYPE,
USERNAME)).block();
+ UploadMetaData upload2 =
Mono.from(cassandraUploadRepository.upload(IOUtils.toInputStream("DATA 2",
StandardCharsets.UTF_8), CONTENT_TYPE, USERNAME)).block();
clock.setInstant(TIMESTAMP.plusWeeks(3).toInstant());
- BucketName unExpiredBucketName1 =
bucketNameGenerator.current().asBucketName();
- Mono.from(cassandraUploadRepository.upload(DATA, CONTENT_TYPE,
USERNAME)).block();
+ UploadMetaData upload3 =
Mono.from(cassandraUploadRepository.upload(IOUtils.toInputStream("DATA 3",
StandardCharsets.UTF_8), CONTENT_TYPE, USERNAME)).block();
clock.setInstant(TIMESTAMP.plusWeeks(4).toInstant());
- BucketName unExpiredBucketName2 =
bucketNameGenerator.current().asBucketName();
- Mono.from(cassandraUploadRepository.upload(DATA, CONTENT_TYPE,
USERNAME)).block();
+ UploadMetaData upload4 =
Mono.from(cassandraUploadRepository.upload(IOUtils.toInputStream("DATA 4",
StandardCharsets.UTF_8), CONTENT_TYPE, USERNAME)).block();
String taskId = given()
.queryParam("scope", "expired")
@@ -269,11 +273,13 @@ class JmapUploadRoutesTest {
.when()
.get(taskId + "/await");
- List<BucketName> bucketNameList =
Flux.from(blobStore.listBuckets()).collectList().block();
+
assertThat(cassandraUploadRepository.listUploads(USERNAME).collectList().block())
+ .doesNotContain(upload1, upload2)
+ .contains(upload3, upload4);
- assertThat(bucketNameList)
- .contains(unExpiredBucketName1, unExpiredBucketName2)
- .doesNotContain(expiredBucketName1, expiredBucketName2);
+
assertThat(Flux.from(blobStore.listBlobs(UPLOAD_BUCKET)).collectList().block())
+ .doesNotContain(upload1.blobId(), upload2.blobId())
+ .contains(upload3.blobId(), upload4.blobId());
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
