sijie closed pull request #1965: Allow ML offload immediately after ledger
closed
URL: https://github.com/apache/incubator-pulsar/pull/1965
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java
index 13d259c177..698d2459ae 100644
---
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java
+++
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java
@@ -412,7 +412,7 @@ public long getOffloadLedgerDeletionLagMillis() {
/**
* Size, in bytes, at which the managed ledger will start to automatically
offload ledgers to longterm storage.
- * A negative value disables autotriggering.
+ * A negative value disables autotriggering. A threshold of 0 offloads
data as soon as possible.
* Offloading will not occur if no offloader has been set {@link
#setLedgerOffloader(LedgerOffloader)}.
* Automatical offloading occurs when the ledger is rolled, and the
ledgers up to that point exceed the threshold.
*
diff --git
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
index 3e24eb30e8..ac546cefdb 100644
---
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
+++
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
@@ -1582,7 +1582,7 @@ private void
scheduleDeferredTrimming(CompletableFuture<?> promise) {
}
private void maybeOffloadInBackground(CompletableFuture<PositionImpl>
promise) {
- if (config.getOffloadAutoTriggerSizeThresholdBytes() > 0) {
+ if (config.getOffloadAutoTriggerSizeThresholdBytes() >= 0) {
executor.executeOrdered(name, safeRun(() ->
maybeOffload(promise)));
}
}
diff --git
a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java
b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java
index 27fe14bf68..6d21ee2a7f 100644
---
a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java
+++
b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java
@@ -865,6 +865,37 @@ public void multipleAutoTriggers() throws Exception {
ledger.getLedgersInfoAsList().get(2).getLedgerId()));
}
+ @Test
+ public void offloadAsSoonAsClosed() throws Exception {
+
+ MockLedgerOffloader offloader = new MockLedgerOffloader();
+ ManagedLedgerConfig config = new ManagedLedgerConfig();
+ config.setMaxEntriesPerLedger(10);
+ config.setOffloadAutoTriggerSizeThresholdBytes(0);
+ config.setRetentionTime(10, TimeUnit.MINUTES);
+ config.setLedgerOffloader(offloader);
+
+ ManagedLedgerImpl ledger =
(ManagedLedgerImpl)factory.open("my_test_ledger", config);
+
+ for (int i = 0; i < 11; i++) {
+ ledger.addEntry(buildEntry(10, "entry-" + i));
+ }
+
+ assertEventuallyTrue(() -> offloader.offloadedLedgers().size() == 1);
+ Assert.assertEquals(offloader.offloadedLedgers(),
+
ImmutableSet.of(ledger.getLedgersInfoAsList().get(0).getLedgerId()));
+
+ for (int i = 0; i < 10; i++) {
+ ledger.addEntry(buildEntry(10, "entry-" + i));
+ }
+
+ assertEventuallyTrue(() -> offloader.offloadedLedgers().size() == 2);
+ Assert.assertEquals(offloader.offloadedLedgers(),
+
ImmutableSet.of(ledger.getLedgersInfoAsList().get(0).getLedgerId(),
+
ledger.getLedgersInfoAsList().get(1).getLedgerId()));
+ }
+
+
static void assertEventuallyTrue(BooleanSupplier predicate) throws
Exception {
// wait up to 3 seconds
for (int i = 0; i < 30 && !predicate.getAsBoolean(); i++) {
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
