sijie closed pull request #1542: LedgerOffloader interface for ManagedLedger URL: https://github.com/apache/incubator-pulsar/pull/1542
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloader.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloader.java new file mode 100644 index 0000000000..b31d7e5390 --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloader.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger; + +import com.google.common.annotations.Beta; + +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +import org.apache.bookkeeper.client.api.ReadHandle; + +/** + * Interface for offloading ledgers to longterm storage + */ +@Beta +public interface LedgerOffloader { + /** + * Offload the passed in ledger to longterm storage. + * Metadata passed in is for inspection purposes only and should be stored + * alongside the ledger data. + * + * When the returned future completes, the ledger has been persisted to the + * loadterm storage, so it is safe to delete the original copy in bookkeeper. + * + * The returned futures completes with a opaque byte[] which contains context + * information required to identify the ledger in the longterm storage. This + * context is stored alongside the ledger info in the managed ledger metadata. + * It is passed to #readOffloaded(byte[]) to read back the ledger. + * + * @param ledger the ledger to offload + * @param extraMetadata metadata to be stored with the ledger for informational + * purposes + * @return a future, which when completed, returns a context byte[] to identify + * the stored ledger + */ + CompletableFuture<byte[]> offload(ReadHandle ledger, + Map<String, String> extraMetadata); + + /** + * Create a ReadHandle which can be used to read a ledger back from longterm + * storage. + * + * The passed offloadContext should be a byte[] that has previously been received + * from a call to #offload(ReadHandle,Map). + * + * @param ledgerId the ID of the ledger to load from longterm storage + * @param offloadContext a context that identifies the ledger in longterm storage + * @return a future, which when completed, returns a ReadHandle + */ + CompletableFuture<ReadHandle> readOffloaded(long ledgerId, byte[] offloadContext); + + /** + * Delete a ledger from long term storage. + * + * The passed offloadContext should be a byte[] that has previously been received + * from a call to #offload(ReadHandle,Map). + * + * @param ledgerId the ID of the ledger to delete from longterm storage + * @param offloadContext a context that identifies the ledger in longterm storage + * @return a future, which when completed, signifies that the ledger has + * been deleted + */ + CompletableFuture<Void> deleteOffloaded(long ledgerId, byte[] offloadContext); +} + diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java index 9aa5e1debf..3cb3fa9209 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java @@ -26,6 +26,8 @@ import java.util.concurrent.TimeUnit; import org.apache.bookkeeper.client.api.DigestType; +import org.apache.bookkeeper.mledger.impl.NullLedgerOffloader; + /** * Configuration class for a ManagedLedger. */ @@ -54,6 +56,7 @@ private DigestType digestType = DigestType.CRC32C; private byte[] password = "".getBytes(Charsets.UTF_8); + private LedgerOffloader ledgerOffloader = NullLedgerOffloader.INSTANCE; public boolean isCreateIfMissing() { return createIfMissing; @@ -421,4 +424,25 @@ public int getMaxUnackedRangesToPersistInZk() { public void setMaxUnackedRangesToPersistInZk(int maxUnackedRangesToPersistInZk) { this.maxUnackedRangesToPersistInZk = maxUnackedRangesToPersistInZk; } + + /** + * Get ledger offloader which will be used to offload ledgers to longterm storage. + * + * The default offloader throws an exception on any attempt to offload. + * + * @return a ledger offloader + */ + public LedgerOffloader getLedgerOffloader() { + return ledgerOffloader; + } + + /** + * Set ledger offloader to use for offloading ledgers to longterm storage. + * + * @param offloader the ledger offloader to use + */ + public ManagedLedgerConfig setLedgerOffloader(LedgerOffloader offloader) { + this.ledgerOffloader = offloader; + return this; + } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NullLedgerOffloader.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NullLedgerOffloader.java new file mode 100644 index 0000000000..5f92653335 --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NullLedgerOffloader.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl; + +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +import org.apache.bookkeeper.client.api.ReadHandle; +import org.apache.bookkeeper.mledger.LedgerOffloader; + +/** + * Null implementation that throws an error on any invokation. + */ +public class NullLedgerOffloader implements LedgerOffloader { + public static NullLedgerOffloader INSTANCE = new NullLedgerOffloader(); + + @Override + public CompletableFuture<byte[]> offload(ReadHandle ledger, + Map<String, String> extraMetadata) { + CompletableFuture<byte[]> promise = new CompletableFuture<>(); + promise.completeExceptionally(new UnsupportedOperationException()); + return promise; + } + + @Override + public CompletableFuture<ReadHandle> readOffloaded(long ledgerId, byte[] offloadContext) { + CompletableFuture<ReadHandle> promise = new CompletableFuture<>(); + promise.completeExceptionally(new UnsupportedOperationException()); + return promise; + } + + @Override + public CompletableFuture<Void> deleteOffloaded(long ledgerId, byte[] offloadContext) { + CompletableFuture<Void> promise = new CompletableFuture<>(); + promise.completeExceptionally(new UnsupportedOperationException()); + return promise; + } +} ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
