zymap commented on code in PR #17398:
URL: https://github.com/apache/pulsar/pull/17398#discussion_r964596306
##########
managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java:
##########
@@ -2358,60 +2359,82 @@ private void
maybeOffloadInBackground(CompletableFuture<PositionImpl> promise) {
private void maybeOffload(CompletableFuture<PositionImpl> finalPromise) {
if (!offloadMutex.tryLock()) {
scheduledExecutor.schedule(safeRun(() ->
maybeOffloadInBackground(finalPromise)),
- 100, TimeUnit.MILLISECONDS);
- } else {
- CompletableFuture<PositionImpl> unlockingPromise = new
CompletableFuture<>();
- unlockingPromise.whenComplete((res, ex) -> {
- offloadMutex.unlock();
- if (ex != null) {
- finalPromise.completeExceptionally(ex);
- } else {
- finalPromise.complete(res);
- }
- });
+ 100, TimeUnit.MILLISECONDS);
+ return;
+ }
- if (config.getLedgerOffloader() != null
- && config.getLedgerOffloader() !=
NullLedgerOffloader.INSTANCE
- && config.getLedgerOffloader().getOffloadPolicies() != null
- &&
config.getLedgerOffloader().getOffloadPolicies().getManagedLedgerOffloadThresholdInBytes()
- != null) {
- long threshold =
config.getLedgerOffloader().getOffloadPolicies()
- .getManagedLedgerOffloadThresholdInBytes();
-
- long sizeSummed = 0;
- long alreadyOffloadedSize = 0;
- long toOffloadSize = 0;
-
- ConcurrentLinkedDeque<LedgerInfo> toOffload = new
ConcurrentLinkedDeque<>();
-
- // go through ledger list from newest to oldest and build a
list to offload in oldest to newest order
- for (Map.Entry<Long, LedgerInfo> e :
ledgers.descendingMap().entrySet()) {
- long size = e.getValue().getSize();
- sizeSummed += size;
- boolean alreadyOffloaded = e.getValue().hasOffloadContext()
- && e.getValue().getOffloadContext().getComplete();
- if (alreadyOffloaded) {
- alreadyOffloadedSize += size;
- } else if (sizeSummed > threshold) {
- toOffloadSize += size;
- toOffload.addFirst(e.getValue());
- }
- }
+ CompletableFuture<PositionImpl> unlockingPromise = new
CompletableFuture<>();
+ unlockingPromise.whenComplete((res, ex) -> {
+ offloadMutex.unlock();
+ if (ex != null) {
+ finalPromise.completeExceptionally(ex);
+ } else {
+ finalPromise.complete(res);
+ }
+ });
- if (toOffload.size() > 0) {
- log.info("[{}] Going to automatically offload ledgers {}"
- + ", total size = {}, already offloaded =
{}, to offload = {}",
- name,
toOffload.stream().map(LedgerInfo::getLedgerId).collect(Collectors.toList()),
- sizeSummed, alreadyOffloadedSize, toOffloadSize);
- offloadLoop(unlockingPromise, toOffload,
PositionImpl.LATEST, Optional.empty());
- } else {
- // offloadLoop will complete immediately with an empty
list to offload
- log.debug("[{}] Nothing to offload, total size = {},
already offloaded = {}, threshold = {}",
- name, sizeSummed, alreadyOffloadedSize, threshold);
- unlockingPromise.complete(PositionImpl.LATEST);
+ if (config.getLedgerOffloader() == null || config.getLedgerOffloader()
== NullLedgerOffloader.INSTANCE
+ || config.getLedgerOffloader().getOffloadPolicies() == null) {
+ return;
Review Comment:
you need to complete the future to release the lock.
And it's better to get the debug log back.
##########
managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java:
##########
@@ -2358,60 +2359,82 @@ private void
maybeOffloadInBackground(CompletableFuture<PositionImpl> promise) {
private void maybeOffload(CompletableFuture<PositionImpl> finalPromise) {
if (!offloadMutex.tryLock()) {
scheduledExecutor.schedule(safeRun(() ->
maybeOffloadInBackground(finalPromise)),
- 100, TimeUnit.MILLISECONDS);
- } else {
- CompletableFuture<PositionImpl> unlockingPromise = new
CompletableFuture<>();
- unlockingPromise.whenComplete((res, ex) -> {
- offloadMutex.unlock();
- if (ex != null) {
- finalPromise.completeExceptionally(ex);
- } else {
- finalPromise.complete(res);
- }
- });
+ 100, TimeUnit.MILLISECONDS);
+ return;
+ }
- if (config.getLedgerOffloader() != null
- && config.getLedgerOffloader() !=
NullLedgerOffloader.INSTANCE
- && config.getLedgerOffloader().getOffloadPolicies() != null
- &&
config.getLedgerOffloader().getOffloadPolicies().getManagedLedgerOffloadThresholdInBytes()
- != null) {
- long threshold =
config.getLedgerOffloader().getOffloadPolicies()
- .getManagedLedgerOffloadThresholdInBytes();
-
- long sizeSummed = 0;
- long alreadyOffloadedSize = 0;
- long toOffloadSize = 0;
-
- ConcurrentLinkedDeque<LedgerInfo> toOffload = new
ConcurrentLinkedDeque<>();
-
- // go through ledger list from newest to oldest and build a
list to offload in oldest to newest order
- for (Map.Entry<Long, LedgerInfo> e :
ledgers.descendingMap().entrySet()) {
- long size = e.getValue().getSize();
- sizeSummed += size;
- boolean alreadyOffloaded = e.getValue().hasOffloadContext()
- && e.getValue().getOffloadContext().getComplete();
- if (alreadyOffloaded) {
- alreadyOffloadedSize += size;
- } else if (sizeSummed > threshold) {
- toOffloadSize += size;
- toOffload.addFirst(e.getValue());
- }
- }
+ CompletableFuture<PositionImpl> unlockingPromise = new
CompletableFuture<>();
+ unlockingPromise.whenComplete((res, ex) -> {
+ offloadMutex.unlock();
+ if (ex != null) {
+ finalPromise.completeExceptionally(ex);
+ } else {
+ finalPromise.complete(res);
+ }
+ });
- if (toOffload.size() > 0) {
- log.info("[{}] Going to automatically offload ledgers {}"
- + ", total size = {}, already offloaded =
{}, to offload = {}",
- name,
toOffload.stream().map(LedgerInfo::getLedgerId).collect(Collectors.toList()),
- sizeSummed, alreadyOffloadedSize, toOffloadSize);
- offloadLoop(unlockingPromise, toOffload,
PositionImpl.LATEST, Optional.empty());
- } else {
- // offloadLoop will complete immediately with an empty
list to offload
- log.debug("[{}] Nothing to offload, total size = {},
already offloaded = {}, threshold = {}",
- name, sizeSummed, alreadyOffloadedSize, threshold);
- unlockingPromise.complete(PositionImpl.LATEST);
+ if (config.getLedgerOffloader() == null || config.getLedgerOffloader()
== NullLedgerOffloader.INSTANCE
+ || config.getLedgerOffloader().getOffloadPolicies() == null) {
+ return;
+ }
+
+ String threshold = null;
+ long sizeSummed = 0;
+ long toOffloadSize = 0;
+ long alreadyOffloadedSize = 0;
+ ConcurrentLinkedDeque<LedgerInfo> toOffload = new
ConcurrentLinkedDeque<>();
+ if
(config.getLedgerOffloader().getOffloadPolicies().getManagedLedgerOffloadThresholdInBytes()
!= null) {
Review Comment:
Shouldn't it be more like the retention policy? We can offload them by time
and size at the same time I think
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
