This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 15d583c Issue #1304: in listunderreplicated cmd provide option to
print missing replicalist
15d583c is described below
commit 15d583cb6162cf6beb10591ad3d0b6f436c81ea0
Author: cguttapalem <[email protected]>
AuthorDate: Tue Apr 3 13:48:23 2018 -0700
Issue #1304: in listunderreplicated cmd provide option to print missing
replicalist
Descriptions of the changes in this PR:
in listunderreplicated bookieshell command,
provide option to print missing replicalist
Master Issue: #1304
Author: cguttapalem <[email protected]>
Reviewers: Enrico Olivelli <[email protected]>, Sijie Guo
<[email protected]>
This closes #1305 from reddycharan/urlbookies, closes #1304
---
.../org/apache/bookkeeper/bookie/BookieShell.java | 14 +++++++--
.../apache/bookkeeper/client/BookKeeperAdmin.java | 7 +++--
.../meta/LedgerUnderreplicationManager.java | 5 +++-
.../meta/ZkLedgerUnderreplicationManager.java | 17 +++++++----
.../service/ListUnderReplicatedLedgerService.java | 34 ++++++++++++++++++----
.../bookkeeper/client/BookKeeperAdminTest.java | 12 ++++++--
.../bookkeeper/client/BookieDecommissionTest.java | 17 +++++++----
7 files changed, 81 insertions(+), 25 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
index d5f6c8d..d01a17c 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
@@ -874,6 +874,7 @@ public class BookieShell implements Tool {
super(CMD_LISTUNDERREPLICATED);
opts.addOption("missingreplica", true, "Bookie Id of missing
replica");
opts.addOption("excludingmissingreplica", true, "Bookie Id of
missing replica to ignore");
+ opts.addOption("printmissingreplica", false, "Whether to print
missingreplicas list?");
}
@Override
@@ -890,7 +891,7 @@ public class BookieShell implements Tool {
@Override
String getUsage() {
return "listunderreplicated [[-missingreplica <bookieaddress>]"
- + " [-excludingmissingreplica <bookieaddress>]]";
+ + " [-excludingmissingreplica <bookieaddress>]]
[-printmissingreplica]";
}
@Override
@@ -898,6 +899,7 @@ public class BookieShell implements Tool {
final String includingBookieId =
cmdLine.getOptionValue("missingreplica");
final String excludingBookieId =
cmdLine.getOptionValue("excludingmissingreplica");
+ final boolean printMissingReplica =
cmdLine.hasOption("printmissingreplica");
final Predicate<List<String>> predicate;
if (!StringUtils.isBlank(includingBookieId) &&
!StringUtils.isBlank(excludingBookieId)) {
@@ -921,9 +923,15 @@ public class BookieShell implements Tool {
Thread.currentThread().interrupt();
throw new UncheckedExecutionException("Interrupted on
newing ledger underreplicated manager", e);
}
- Iterator<Long> iter =
underreplicationManager.listLedgersToRereplicate(predicate);
+ Iterator<Map.Entry<Long, List<String>>> iter =
underreplicationManager
+ .listLedgersToRereplicate(predicate,
printMissingReplica);
while (iter.hasNext()) {
-
System.out.println(ledgerIdFormatter.formatLedgerId(iter.next()));
+
System.out.println(ledgerIdFormatter.formatLedgerId(iter.next().getKey()));
+ if (printMissingReplica) {
+ iter.next().getValue().forEach((missingReplica) -> {
+ System.out.println("\t" + missingReplica);
+ });
+ }
}
return null;
});
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java
index 56865db..50f2a04 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperAdmin.java
@@ -1489,13 +1489,16 @@ public class BookKeeperAdmin implements AutoCloseable {
// for double-checking, check if any ledgers are listed as
underreplicated because of this bookie
Predicate<List<String>> predicate = replicasList ->
replicasList.contains(bookieAddress.toString());
- Iterator<Long> urLedgerIterator =
underreplicationManager.listLedgersToRereplicate(predicate);
+ Iterator<Map.Entry<Long, List<String>>> urLedgerIterator =
underreplicationManager
+ .listLedgersToRereplicate(predicate, false);
if (urLedgerIterator.hasNext()) {
//if there are any then wait and make sure those ledgers are
replicated properly
LOG.info("Still in some underreplicated ledgers metadata, this
bookie is part of its ensemble. "
+ "Have to make sure that those ledger fragments are
rereplicated");
List<Long> urLedgers = new ArrayList<>();
- urLedgerIterator.forEachRemaining(urLedgers::add);
+ urLedgerIterator.forEachRemaining((urLedger) -> {
+ urLedgers.add(urLedger.getKey());
+ });
waitForLedgersToBeReplicated(urLedgers, bookieAddress,
bkc.ledgerManager);
}
}
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/LedgerUnderreplicationManager.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/LedgerUnderreplicationManager.java
index 88204e6..21c6a4f 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/LedgerUnderreplicationManager.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/LedgerUnderreplicationManager.java
@@ -19,6 +19,7 @@ package org.apache.bookkeeper.meta;
import java.util.Iterator;
import java.util.List;
+import java.util.Map.Entry;
import java.util.function.Predicate;
import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.GenericCallback;
@@ -53,9 +54,11 @@ public interface LedgerUnderreplicationManager extends
AutoCloseable {
* otherwise it will read the content of the ZNode to decide on filtering.
*
* @param predicate filter to use while listing under replicated ledgers.
'null' if filtering is not required
+ * @param includeReplicaList whether to include missing replicalist in the
output
* @return an iterator which returns ledger ids
*/
- Iterator<Long> listLedgersToRereplicate(Predicate<List<String>> predicate);
+ Iterator<Entry<Long, List<String>>>
listLedgersToRereplicate(Predicate<List<String>> predicate,
+ boolean includeReplicaList);
/**
* Acquire a underreplicated ledger for rereplication. The ledger
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/ZkLedgerUnderreplicationManager.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/ZkLedgerUnderreplicationManager.java
index 3fa0daf..406645b 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/ZkLedgerUnderreplicationManager.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/ZkLedgerUnderreplicationManager.java
@@ -25,6 +25,7 @@ import com.google.common.base.Joiner;
import com.google.protobuf.TextFormat;
import java.net.UnknownHostException;
+import java.util.AbstractMap;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
@@ -359,15 +360,17 @@ public class ZkLedgerUnderreplicationManager implements
LedgerUnderreplicationMa
* otherwise it will read the content of the ZNode to decide on filtering.
*
* @param predicate filter to use while listing under replicated ledgers.
'null' if filtering is not required.
+ * @param includeReplicaList whether to include missing replicalist in the
output.
* @return an iterator which returns ledger ids
*/
@Override
- public Iterator<Long> listLedgersToRereplicate(final
Predicate<List<String>> predicate) {
+ public Iterator<Map.Entry<Long, List<String>>>
listLedgersToRereplicate(final Predicate<List<String>> predicate,
+ boolean includeReplicaList) {
final Queue<String> queue = new LinkedList<String>();
queue.add(urLedgerPath);
- return new Iterator<Long>() {
- final Queue<Long> curBatch = new LinkedList<Long>();
+ return new Iterator<Map.Entry<Long, List<String>>>() {
+ final Queue<Map.Entry<Long, List<String>>> curBatch = new
LinkedList<Map.Entry<Long, List<String>>>();
@Override
public void remove() {
@@ -388,9 +391,11 @@ public class ZkLedgerUnderreplicationManager implements
LedgerUnderreplicationMa
String child = parent + "/" + c;
if (c.startsWith("urL")) {
long ledgerId = getLedgerId(child);
+ List<String> replicaList =
getLedgerUnreplicationInfo(ledgerId).getReplicaList();
if ((predicate == null)
- ||
predicate.test(getLedgerUnreplicationInfo(ledgerId).getReplicaList())) {
- curBatch.add(ledgerId);
+ || predicate.test(replicaList)) {
+ curBatch.add(new
AbstractMap.SimpleImmutableEntry<Long, List<String>>(ledgerId,
+ ((includeReplicaList) ?
replicaList : null)));
}
} else {
queue.add(child);
@@ -412,7 +417,7 @@ public class ZkLedgerUnderreplicationManager implements
LedgerUnderreplicationMa
}
@Override
- public Long next() {
+ public Map.Entry<Long, List<String>> next() {
assert curBatch.size() > 0;
return curBatch.remove();
}
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListUnderReplicatedLedgerService.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListUnderReplicatedLedgerService.java
index 3a07ef6..237760c 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListUnderReplicatedLedgerService.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListUnderReplicatedLedgerService.java
@@ -22,6 +22,7 @@ import static
com.google.common.base.Preconditions.checkNotNull;
import com.google.common.collect.Lists;
import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Predicate;
@@ -69,6 +70,8 @@ public class ListUnderReplicatedLedgerService implements
HttpEndpointService {
if (HttpServer.Method.GET == request.getMethod()) {
final String includingBookieId;
final String excludingBookieId;
+ boolean printMissingReplica = false;
+
if (params != null && params.containsKey("missingreplica")) {
includingBookieId = params.get("missingreplica");
} else {
@@ -79,6 +82,9 @@ public class ListUnderReplicatedLedgerService implements
HttpEndpointService {
} else {
excludingBookieId = null;
}
+ if (params != null && params.containsKey("printmissingreplica")) {
+ printMissingReplica = true;
+ }
Predicate<List<String>> predicate = null;
if (!StringUtils.isBlank(includingBookieId) &&
!StringUtils.isBlank(excludingBookieId)) {
predicate = replicasList ->
(replicasList.contains(includingBookieId)
@@ -90,21 +96,39 @@ public class ListUnderReplicatedLedgerService implements
HttpEndpointService {
}
try {
- List<Long> outputLedgers = Lists.newArrayList();
+ boolean hasURLedgers = false;
+ List<Long> outputLedgers = null;
+ Map<Long, List<String>> outputLedgersWithMissingReplica = null;
LedgerManagerFactory mFactory =
bookieServer.getBookie().getLedgerManagerFactory();
LedgerUnderreplicationManager underreplicationManager =
mFactory.newLedgerUnderreplicationManager();
- Iterator<Long> iter =
underreplicationManager.listLedgersToRereplicate(predicate);
+ Iterator<Map.Entry<Long, List<String>>> iter =
underreplicationManager
+ .listLedgersToRereplicate(predicate,
printMissingReplica);
+ hasURLedgers = iter.hasNext();
+ if (hasURLedgers) {
+ if (printMissingReplica) {
+ outputLedgersWithMissingReplica = new
LinkedHashMap<Long, List<String>>();
+ } else {
+ outputLedgers = Lists.newArrayList();
+ }
+ }
while (iter.hasNext()) {
- outputLedgers.add(iter.next());
+ if (printMissingReplica) {
+ Map.Entry<Long, List<String>> urlWithMissingReplica =
iter.next();
+
outputLedgersWithMissingReplica.put(urlWithMissingReplica.getKey(),
+ urlWithMissingReplica.getValue());
+ } else {
+ outputLedgers.add(iter.next().getKey());
+ }
}
- if (outputLedgers.isEmpty()) {
+ if (!hasURLedgers) {
response.setCode(HttpServer.StatusCode.NOT_FOUND);
response.setBody("No under replicated ledgers found");
return response;
} else {
response.setCode(HttpServer.StatusCode.OK);
- String jsonResponse = JsonUtil.toJson(outputLedgers);
+ String jsonResponse = JsonUtil
+ .toJson(printMissingReplica ?
outputLedgersWithMissingReplica : outputLedgers);
LOG.debug("output body: " + jsonResponse);
response.setBody(jsonResponse);
return response;
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperAdminTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperAdminTest.java
index a6a8676..4908a2f 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperAdminTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookKeeperAdminTest.java
@@ -31,12 +31,15 @@ import java.io.File;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
import java.util.Random;
import org.apache.bookkeeper.bookie.Bookie;
import org.apache.bookkeeper.client.BookKeeper.DigestType;
import org.apache.bookkeeper.conf.ClientConfiguration;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager;
+import org.apache.bookkeeper.proto.BookieServer;
import
org.apache.bookkeeper.replication.ReplicationException.UnavailableException;
import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
import org.apache.bookkeeper.util.BookKeeperConstants;
@@ -105,6 +108,7 @@ public class BookKeeperAdminTest extends
BookKeeperClusterTestCase {
ledgerHandle.addEntry(0, "data".getBytes());
ledgerHandle.close();
+ BookieServer bookieToKill = bs.get(1);
killBookie(1);
/*
* since lostBookieRecoveryDelay is set, when a bookie is died, it will
@@ -113,9 +117,13 @@ public class BookKeeperAdminTest extends
BookKeeperClusterTestCase {
*/
bkAdmin.triggerAudit();
Thread.sleep(500);
- Iterator<Long> ledgersToRereplicate =
urLedgerMgr.listLedgersToRereplicate(null);
+ Iterator<Map.Entry<Long, List<String>>> ledgersToRereplicate =
urLedgerMgr.listLedgersToRereplicate(null,
+ true);
assertTrue("There are supposed to be underreplicatedledgers",
ledgersToRereplicate.hasNext());
- assertEquals("Underreplicated ledgerId", ledgerId,
ledgersToRereplicate.next().longValue());
+ Entry<Long, List<String>> urlWithReplicaList =
ledgersToRereplicate.next();
+ assertEquals("Underreplicated ledgerId", ledgerId,
urlWithReplicaList.getKey().longValue());
+ assertTrue("Missingreplica of Underreplicated ledgerId should contain
" + bookieToKill.getLocalAddress(),
+
urlWithReplicaList.getValue().contains(bookieToKill.getLocalAddress().toString()));
bkAdmin.close();
}
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieDecommissionTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieDecommissionTest.java
index 04b66a4..0e4928e 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieDecommissionTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieDecommissionTest.java
@@ -22,6 +22,9 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.bookie.Bookie;
import org.apache.bookkeeper.client.BKException.BKIllegalOpException;
@@ -89,10 +92,11 @@ public class BookieDecommissionTest extends
BookKeeperClusterTestCase {
bkAdmin.decommissionBookie(Bookie.getBookieAddress(killedBookieConf));
bkAdmin.triggerAudit();
Thread.sleep(500);
- Iterator<Long> ledgersToRereplicate =
urLedgerMgr.listLedgersToRereplicate(null);
+ Iterator<Map.Entry<Long, List<String>>> ledgersToRereplicate =
urLedgerMgr.listLedgersToRereplicate(null,
+ false);
if (ledgersToRereplicate.hasNext()) {
while (ledgersToRereplicate.hasNext()) {
- Long ledgerId = ledgersToRereplicate.next();
+ Long ledgerId = ledgersToRereplicate.next().getKey();
log.error("Ledger: {} is underreplicated which is not
expected", ledgerId);
}
fail("There are not supposed to be any underreplicatedledgers");
@@ -102,10 +106,10 @@ public class BookieDecommissionTest extends
BookKeeperClusterTestCase {
bkAdmin.decommissionBookie(Bookie.getBookieAddress(killedBookieConf));
bkAdmin.triggerAudit();
Thread.sleep(500);
- ledgersToRereplicate = urLedgerMgr.listLedgersToRereplicate(null);
+ ledgersToRereplicate = urLedgerMgr.listLedgersToRereplicate(null,
false);
if (ledgersToRereplicate.hasNext()) {
while (ledgersToRereplicate.hasNext()) {
- Long ledgerId = ledgersToRereplicate.next();
+ Long ledgerId = ledgersToRereplicate.next().getKey();
log.error("Ledger: {} is underreplicated which is not
expected", ledgerId);
}
fail("There are not supposed to be any underreplicatedledgers");
@@ -162,10 +166,11 @@ public class BookieDecommissionTest extends
BookKeeperClusterTestCase {
bkAdmin.decommissionBookie(Bookie.getBookieAddress(killedBookieConf));
bkAdmin.triggerAudit();
Thread.sleep(500);
- Iterator<Long> ledgersToRereplicate =
urLedgerMgr.listLedgersToRereplicate(null);
+ Iterator<Map.Entry<Long, List<String>>> ledgersToRereplicate =
urLedgerMgr.listLedgersToRereplicate(null,
+ false);
if (ledgersToRereplicate.hasNext()) {
while (ledgersToRereplicate.hasNext()) {
- Long ledgerId = ledgersToRereplicate.next();
+ Long ledgerId = ledgersToRereplicate.next().getKey();
log.error("Ledger: {} is underreplicated which is not
expected", ledgerId);
}
fail("There are not supposed to be any underreplicatedledgers");
--
To stop receiving notification emails like this one, please contact
[email protected].
