This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push: new 16bfc44 LedgerOffloader interface for ManagedLedger (#1542) 16bfc44 is described below commit 16bfc441bcaee5073b70b8a97614071647b3f60e Author: Ivan Kelly <iv...@apache.org> AuthorDate: Wed Apr 11 12:16:09 2018 +0200 LedgerOffloader interface for ManagedLedger (#1542) An interface which can be used for offloading ledgers to longterm storage. Master issue #1511 --- .../apache/bookkeeper/mledger/LedgerOffloader.java | 81 ++++++++++++++++++++++ .../bookkeeper/mledger/ManagedLedgerConfig.java | 24 +++++++ .../mledger/impl/NullLedgerOffloader.java | 54 +++++++++++++++ 3 files changed, 159 insertions(+) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloader.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloader.java new file mode 100644 index 0000000..b31d7e5 --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloader.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger; + +import com.google.common.annotations.Beta; + +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +import org.apache.bookkeeper.client.api.ReadHandle; + +/** + * Interface for offloading ledgers to longterm storage + */ +@Beta +public interface LedgerOffloader { + /** + * Offload the passed in ledger to longterm storage. + * Metadata passed in is for inspection purposes only and should be stored + * alongside the ledger data. + * + * When the returned future completes, the ledger has been persisted to the + * loadterm storage, so it is safe to delete the original copy in bookkeeper. + * + * The returned futures completes with a opaque byte[] which contains context + * information required to identify the ledger in the longterm storage. This + * context is stored alongside the ledger info in the managed ledger metadata. + * It is passed to #readOffloaded(byte[]) to read back the ledger. + * + * @param ledger the ledger to offload + * @param extraMetadata metadata to be stored with the ledger for informational + * purposes + * @return a future, which when completed, returns a context byte[] to identify + * the stored ledger + */ + CompletableFuture<byte[]> offload(ReadHandle ledger, + Map<String, String> extraMetadata); + + /** + * Create a ReadHandle which can be used to read a ledger back from longterm + * storage. + * + * The passed offloadContext should be a byte[] that has previously been received + * from a call to #offload(ReadHandle,Map). + * + * @param ledgerId the ID of the ledger to load from longterm storage + * @param offloadContext a context that identifies the ledger in longterm storage + * @return a future, which when completed, returns a ReadHandle + */ + CompletableFuture<ReadHandle> readOffloaded(long ledgerId, byte[] offloadContext); + + /** + * Delete a ledger from long term storage. + * + * The passed offloadContext should be a byte[] that has previously been received + * from a call to #offload(ReadHandle,Map). + * + * @param ledgerId the ID of the ledger to delete from longterm storage + * @param offloadContext a context that identifies the ledger in longterm storage + * @return a future, which when completed, signifies that the ledger has + * been deleted + */ + CompletableFuture<Void> deleteOffloaded(long ledgerId, byte[] offloadContext); +} + diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java index 9aa5e1d..3cb3fa9 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java @@ -26,6 +26,8 @@ import java.util.Arrays; import java.util.concurrent.TimeUnit; import org.apache.bookkeeper.client.api.DigestType; +import org.apache.bookkeeper.mledger.impl.NullLedgerOffloader; + /** * Configuration class for a ManagedLedger. */ @@ -54,6 +56,7 @@ public class ManagedLedgerConfig { private DigestType digestType = DigestType.CRC32C; private byte[] password = "".getBytes(Charsets.UTF_8); + private LedgerOffloader ledgerOffloader = NullLedgerOffloader.INSTANCE; public boolean isCreateIfMissing() { return createIfMissing; @@ -421,4 +424,25 @@ public class ManagedLedgerConfig { public void setMaxUnackedRangesToPersistInZk(int maxUnackedRangesToPersistInZk) { this.maxUnackedRangesToPersistInZk = maxUnackedRangesToPersistInZk; } + + /** + * Get ledger offloader which will be used to offload ledgers to longterm storage. + * + * The default offloader throws an exception on any attempt to offload. + * + * @return a ledger offloader + */ + public LedgerOffloader getLedgerOffloader() { + return ledgerOffloader; + } + + /** + * Set ledger offloader to use for offloading ledgers to longterm storage. + * + * @param offloader the ledger offloader to use + */ + public ManagedLedgerConfig setLedgerOffloader(LedgerOffloader offloader) { + this.ledgerOffloader = offloader; + return this; + } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NullLedgerOffloader.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NullLedgerOffloader.java new file mode 100644 index 0000000..5f92653 --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NullLedgerOffloader.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger.impl; + +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +import org.apache.bookkeeper.client.api.ReadHandle; +import org.apache.bookkeeper.mledger.LedgerOffloader; + +/** + * Null implementation that throws an error on any invokation. + */ +public class NullLedgerOffloader implements LedgerOffloader { + public static NullLedgerOffloader INSTANCE = new NullLedgerOffloader(); + + @Override + public CompletableFuture<byte[]> offload(ReadHandle ledger, + Map<String, String> extraMetadata) { + CompletableFuture<byte[]> promise = new CompletableFuture<>(); + promise.completeExceptionally(new UnsupportedOperationException()); + return promise; + } + + @Override + public CompletableFuture<ReadHandle> readOffloaded(long ledgerId, byte[] offloadContext) { + CompletableFuture<ReadHandle> promise = new CompletableFuture<>(); + promise.completeExceptionally(new UnsupportedOperationException()); + return promise; + } + + @Override + public CompletableFuture<Void> deleteOffloaded(long ledgerId, byte[] offloadContext) { + CompletableFuture<Void> promise = new CompletableFuture<>(); + promise.completeExceptionally(new UnsupportedOperationException()); + return promise; + } +} -- To stop receiving notification emails like this one, please contact si...@apache.org.