jiazhai closed pull request #905: ISSUE #904: short hostname for bookieid
URL: https://github.com/apache/bookkeeper/pull/905
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/bookkeeper-server/conf/bk_server.conf
b/bookkeeper-server/conf/bk_server.conf
index 5e2b58e16..ebc0ec7ca 100755
--- a/bookkeeper-server/conf/bk_server.conf
+++ b/bookkeeper-server/conf/bk_server.conf
@@ -437,6 +437,10 @@
ledgerManagerFactoryClass=org.apache.bookkeeper.meta.HierarchicalLedgerManagerFa
# Set the rate at which compaction will readd entries. The unit is adds per
second.
# compactionRate=1000
+# If bookie is using hostname for registration and in ledger metadata then
+# whether to use short hostname or FQDN hostname. Defaults to false.
+# useShortHostName=false
+
# Threshold of minor compaction
# For those entry log files whose remaining size percentage reaches below
# this threshold will be compacted in a minor compaction.
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
index 90b193922..aaabdb5f1 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
@@ -43,6 +43,7 @@
import java.io.FileNotFoundException;
import java.io.FilenameFilter;
import java.io.IOException;
+import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
@@ -559,15 +560,26 @@ public static BookieSocketAddress
getBookieAddress(ServerConfiguration conf)
if (iface == null) {
iface = "default";
}
+
String hostName = DNS.getDefaultHost(iface);
InetSocketAddress inetAddr = new InetSocketAddress(hostName,
conf.getBookiePort());
if (inetAddr.isUnresolved()) {
throw new UnknownHostException("Unable to resolve default
hostname: "
+ hostName + " for interface: " + iface);
}
- String hostAddress = inetAddr.getAddress().getHostAddress();
+ String hostAddress = null;
+ InetAddress iAddress = inetAddr.getAddress();
if (conf.getUseHostNameAsBookieID()) {
- hostAddress = inetAddr.getAddress().getCanonicalHostName();
+ hostAddress = iAddress.getCanonicalHostName();
+ if (conf.getUseShortHostName()) {
+ /*
+ * if short hostname is used, then FQDN is not used. Short
+ * hostname is the hostname cut at the first dot.
+ */
+ hostAddress = hostAddress.split("\\.", 2)[0];
+ }
+ } else {
+ hostAddress = iAddress.getHostAddress();
}
BookieSocketAddress addr =
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
index 2e1dc1797..fc707d1b7 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
@@ -23,6 +23,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.common.net.InetAddresses;
import com.google.common.util.concurrent.AbstractFuture;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
@@ -111,7 +112,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
/**
* Bookie Shell is to provide utilities for users to administer a bookkeeper
cluster.
*/
@@ -1244,8 +1244,8 @@ Options getOptions() {
super(CMD_LISTBOOKIES);
opts.addOption("rw", "readwrite", false, "Print readwrite
bookies");
opts.addOption("ro", "readonly", false, "Print readonly bookies");
- opts.addOption("h", "hostnames", false,
- "Also print hostname of the bookie");
+ // @deprecated 'rw'/'ro' option print both hostname and ip, so
this option is not needed anymore
+ opts.addOption("h", "hostnames", false, "Also print hostname of
the bookie");
}
@Override
@@ -1274,11 +1274,7 @@ public int runCmd(CommandLine cmdLine) throws Exception {
bookies.addAll(roBookies);
}
for (BookieSocketAddress b : bookies) {
- System.out.print(b);
- if (cmdLine.hasOption("h")) {
- System.out.print("\t" +
b.getSocketAddress().getHostName());
- }
- System.out.println("");
+ System.out.println(getBookieSocketAddrStringRepresentation(b));
count++;
}
if (count == 0) {
@@ -1586,10 +1582,7 @@ int runCmd(CommandLine cmdLine) throws Exception {
LOG.info("No auditor elected");
return -1;
}
- LOG.info("Auditor: {}/{}:{}",
-
bookieId.getSocketAddress().getAddress().getCanonicalHostName(),
-
bookieId.getSocketAddress().getAddress().getHostAddress(),
- bookieId.getSocketAddress().getPort());
+ LOG.info("Auditor: " +
getBookieSocketAddrStringRepresentation(bookieId));
} finally {
if (zk != null) {
zk.close();
@@ -2020,9 +2013,10 @@ public int runCmd(CommandLine cmdLine) throws Exception {
long totalFree = 0, total = 0;
for (Map.Entry<BookieSocketAddress, BookieInfo> e :
map.entrySet()) {
BookieInfo bInfo = e.getValue();
- System.out.println(e.getKey() + ":\tFree: " +
bInfo.getFreeDiskSpace()
- + getReadable(bInfo.getFreeDiskSpace()) + "\tTotal: "
+ bInfo.getTotalDiskSpace()
- + getReadable(bInfo.getTotalDiskSpace()));
+ BookieSocketAddress bookieId = e.getKey();
+
System.out.println(getBookieSocketAddrStringRepresentation(bookieId) +
":\tFree: "
+ + bInfo.getFreeDiskSpace() +
getReadable(bInfo.getFreeDiskSpace()) + "\tTotal: "
+ + bInfo.getTotalDiskSpace() +
getReadable(bInfo.getTotalDiskSpace()));
totalFree += bInfo.getFreeDiskSpace();
total += bInfo.getTotalDiskSpace();
}
@@ -2447,6 +2441,27 @@ public int run(String[] args) throws Exception {
return cmd.runCmd(newArgs);
}
+ /*
+ * The string returned is of the form:
+ * 'hostname'('otherformofhostname'):'port number'
+ *
+ * where hostname and otherformofhostname are ipaddress and
+ * canonicalhostname or viceversa
+ */
+ private static String
getBookieSocketAddrStringRepresentation(BookieSocketAddress bookieId) {
+ String hostname = bookieId.getHostName();
+ boolean isHostNameIpAddress = InetAddresses.isInetAddress(hostname);
+ String otherFormOfHostname = null;
+ if (isHostNameIpAddress) {
+ otherFormOfHostname =
bookieId.getSocketAddress().getAddress().getCanonicalHostName();
+ } else {
+ otherFormOfHostname =
bookieId.getSocketAddress().getAddress().getHostAddress();
+ }
+ String bookieSocketAddrStringRepresentation = hostname + "(" +
otherFormOfHostname + ")" + ":"
+ + bookieId.getSocketAddress().getPort();
+ return bookieSocketAddrStringRepresentation;
+ }
+
/**
* Returns the sorted list of the files in the given folders with the
given file extensions.
* Sorting is done on the basis of CreationTime if the CreationTime is not
available or if they are equal
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
index 38eccbae3..3990e74de 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
@@ -135,6 +135,7 @@
// Whether the bookie should use its hostname or ipaddress for the
// registration.
protected static final String USE_HOST_NAME_AS_BOOKIE_ID =
"useHostNameAsBookieID";
+ protected static final String USE_SHORT_HOST_NAME = "useShortHostName";
protected static final String ENABLE_LOCAL_TRANSPORT =
"enableLocalTransport";
protected static final String DISABLE_SERVER_SOCKET_BIND =
"disableServerSocketBind";
@@ -2068,6 +2069,34 @@ public ServerConfiguration
setUseHostNameAsBookieID(boolean useHostName) {
return this;
}
+ /**
+ * If bookie is using hostname for registration and in ledger metadata then
+ * whether to use short hostname or FQDN hostname. Defaults to false.
+ *
+ * @return true, then bookie will be registered with its short hostname and
+ * short hostname will be used in ledger metadata. Otherwise bookie
+ * will use its FQDN hostname
+ */
+ public boolean getUseShortHostName() {
+ return getBoolean(USE_SHORT_HOST_NAME, false);
+ }
+
+ /**
+ * Configure the bookie to use its short hostname or FQDN hostname to
+ * register with the co-ordination service(eg: zookeeper) and in ledger
+ * metadata.
+ *
+ * @see #getUseShortHostName
+ * @param useShortHostName
+ * whether to use short hostname for registration and in
+ * ledgermetadata
+ * @return server configuration
+ */
+ public ServerConfiguration setUseShortHostName(boolean useShortHostName) {
+ setProperty(USE_SHORT_HOST_NAME, useShortHostName);
+ return this;
+ }
+
/**
* Get whether to listen for local JVM clients. Defaults to false.
*
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java
index a1fa29039..0c240da6b 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java
@@ -62,6 +62,7 @@
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.data.Stat;
+import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
@@ -243,6 +244,48 @@ public void testBookieRegistration() throws Exception {
}
}
+ @Test(timeout = 20000)
+ public void testBookieRegistrationWithFQDNHostNameAsBookieID() throws
Exception {
+ File tmpDir = createTempDir("bookie", "test");
+
+ final ServerConfiguration conf =
TestBKConfiguration.newServerConfiguration().setZkServers(null)
+ .setJournalDirName(tmpDir.getPath()).setLedgerDirNames(new
String[] { tmpDir.getPath() })
+ .setUseHostNameAsBookieID(true);
+
+ final String bkRegPath = conf.getZkAvailableBookiesPath() + "/"
+ + InetAddress.getLocalHost().getCanonicalHostName() + ":" +
conf.getBookiePort();
+
+ MockBookie bWithFQDNHostname = new MockBookie(conf);
+ conf.setZkServers(zkUtil.getZooKeeperConnectString());
+ rm.initialize(conf, () -> {}, NullStatsLogger.INSTANCE);
+ bWithFQDNHostname.registrationManager = rm;
+
+ bWithFQDNHostname.testRegisterBookie(conf);
+ Stat bkRegNode1 = zkc.exists(bkRegPath, false);
+ Assert.assertNotNull("Bookie registration node doesn't exists!",
bkRegNode1);
+ }
+
+ @Test(timeout = 20000)
+ public void testBookieRegistrationWithShortHostNameAsBookieID() throws
Exception {
+ File tmpDir = createTempDir("bookie", "test");
+
+ final ServerConfiguration conf =
TestBKConfiguration.newServerConfiguration().setZkServers(null)
+ .setJournalDirName(tmpDir.getPath()).setLedgerDirNames(new
String[] { tmpDir.getPath() })
+ .setUseHostNameAsBookieID(true).setUseShortHostName(true);
+
+ final String bkRegPath = conf.getZkAvailableBookiesPath() + "/"
+ +
(InetAddress.getLocalHost().getCanonicalHostName().split("\\.", 2)[0]) + ":" +
conf.getBookiePort();
+
+ MockBookie bWithShortHostname = new MockBookie(conf);
+ conf.setZkServers(zkUtil.getZooKeeperConnectString());
+ rm.initialize(conf, () -> {}, NullStatsLogger.INSTANCE);
+ bWithShortHostname.registrationManager = rm;
+
+ bWithShortHostname.testRegisterBookie(conf);
+ Stat bkRegNode1 = zkc.exists(bkRegPath, false);
+ Assert.assertNotNull("Bookie registration node doesn't exists!",
bkRegNode1);
+ }
+
/**
* Verify the bookie registration, it should throw
* KeeperException.NodeExistsException if the znode still exists even after
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/UpdateCookieCmdTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/UpdateCookieCmdTest.java
index faa357f70..0d9f7ebfa 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/UpdateCookieCmdTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/UpdateCookieCmdTest.java
@@ -77,6 +77,14 @@ public void testUpdateCookieIpAddressToHostname() throws
Exception {
updateCookie("-bookieId", "hostname", true);
}
+ /**
+ * updatecookie to short hostname.
+ */
+ @Test
+ public void testUpdateCookieIpAddressToShortHostname() throws Exception {
+ updateCookie("-bookieId", "hostname", true, true);
+ }
+
/**
* updatecookie to ipaddress.
*/
@@ -190,6 +198,11 @@ private void verifyCookieInZooKeeper(ServerConfiguration
conf, int expectedCount
}
private void updateCookie(String option, String optionVal, boolean
useHostNameAsBookieID) throws Exception {
+ updateCookie(option, optionVal, useHostNameAsBookieID, false);
+ }
+
+ private void updateCookie(String option, String optionVal, boolean
useHostNameAsBookieID, boolean useShortHostName)
+ throws Exception {
ServerConfiguration conf = new ServerConfiguration(bsConfs.get(0));
BookieServer bks = bs.get(0);
bks.shutdown();
@@ -202,12 +215,14 @@ private void updateCookie(String option, String
optionVal, boolean useHostNameAs
LOG.info("Perform updatecookie command");
ServerConfiguration newconf = new ServerConfiguration(conf);
newconf.setUseHostNameAsBookieID(useHostNameAsBookieID);
+ newconf.setUseShortHostName(useShortHostName);
BookieShell bkShell = new BookieShell();
bkShell.setConf(newconf);
String[] argv = new String[] { "updatecookie", option, optionVal };
Assert.assertEquals("Failed to return exit code!", 0,
bkShell.run(argv));
newconf.setUseHostNameAsBookieID(useHostNameAsBookieID);
+ newconf.setUseShortHostName(useShortHostName);
cookie = Cookie.readFromRegistrationManager(rm, newconf).getValue();
Assert.assertEquals("Wrongly updated cookie!", previousBookieID,
!cookie.isBookieHostCreatedFromIp());
Assert.assertEquals("Wrongly updated cookie!", useHostNameAsBookieID,
!cookie.isBookieHostCreatedFromIp());
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/UpdateLedgerOpTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/UpdateLedgerOpTest.java
index 42e744f18..1e58747fa 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/UpdateLedgerOpTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/UpdateLedgerOpTest.java
@@ -72,10 +72,22 @@ public void progress(long updated, long issued) {
};
/**
- * Tests verifies update bookie id when there are many ledgers.
+ * Tests verifies update bookie id to FQDN hostname when there are many
ledgers.
*/
@Test
- public void testManyLedgers() throws Exception {
+ public void testManyLedgersWithFQDNHostname() throws Exception {
+ testManyLedgers(false);
+ }
+
+ /**
+ * Tests verifies update bookie id to short hostname when there are many
ledgers.
+ */
+ @Test(timeout = 120000)
+ public void testManyLedgersWithShortHostname() throws Exception {
+ testManyLedgers(true);
+ }
+
+ public void testManyLedgers(boolean useShortHostName) throws Exception {
BookKeeper bk = new BookKeeper(baseClientConf, zkc);
BookKeeperAdmin bkadmin = new BookKeeperAdmin(bk);
@@ -91,6 +103,7 @@ public void testManyLedgers() throws Exception {
BookieSocketAddress curBookieAddr = ensemble.get(0);
baseConf.setUseHostNameAsBookieID(true);
+ baseConf.setUseShortHostName(useShortHostName);
BookieSocketAddress curBookieId = Bookie.getBookieAddress(baseConf);
BookieSocketAddress toBookieAddr = new
BookieSocketAddress(curBookieId.getHostName() + ":"
+ curBookieAddr.getPort());
@@ -156,11 +169,24 @@ public void testLimitLessThanTotalLedgers() throws
Exception {
}
/**
- * Tests verifies the ensemble reformation after updating the bookie id in
- * the existing ensemble.
+ * Tests verifies the ensemble reformation after updating the bookie id to
+ * FQDN hostname in the existing ensemble.
*/
@Test
- public void testChangeEnsembleAfterRenaming() throws Exception {
+ public void testChangeEnsembleAfterRenamingToFQDNHostname() throws
Exception {
+ testChangeEnsembleAfterRenaming(false);
+ }
+
+ /**
+ * Tests verifies the ensemble reformation after updating the bookie id to
+ * short hostname in the existing ensemble.
+ */
+ @Test(timeout = 120000)
+ public void testChangeEnsembleAfterRenamingToShortHostname() throws
Exception {
+ testChangeEnsembleAfterRenaming(true);
+ }
+
+ public void testChangeEnsembleAfterRenaming(boolean useShortHostName)
throws Exception {
BookKeeper bk = new BookKeeper(baseClientConf, zkc);
BookKeeperAdmin bkadmin = new BookKeeperAdmin(bk);
@@ -178,6 +204,7 @@ public void testChangeEnsembleAfterRenaming() throws
Exception {
}
assertNotNull("Couldn't find the bookie in ledger metadata!",
curBookieAddr);
baseConf.setUseHostNameAsBookieID(true);
+ baseConf.setUseShortHostName(useShortHostName);
BookieSocketAddress toBookieId = Bookie.getBookieAddress(baseConf);
BookieSocketAddress toBookieAddr = new
BookieSocketAddress(toBookieId.getHostName() + ":"
+ curBookieAddr.getPort());
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
