This is an automated email from the ASF dual-hosted git repository.
zhaijia pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new de9fe40 ISSUE #904: short hostname for bookieid
de9fe40 is described below
commit de9fe40fd5b2189d89f1f51c62201aff08718064
Author: Charan Reddy Guttapalem <[email protected]>
AuthorDate: Mon Dec 25 20:43:23 2017 +0800
ISSUE #904: short hostname for bookieid
Descriptions of the changes in this PR:
- provide useShortHostName config to use short hostName for bookieId
- corresponding testcases
- For BKShell commands print appropriate string representation (hostname
and ipaddress)
and port number of bookies
Master Issue: #904
Author: Charan Reddy Guttapalem <[email protected]>
Author: cguttapalem <[email protected]>
Reviewers: Jia Zhai <None>, Sijie Guo <[email protected]>
This closes #905 from reddycharan/shorthostname, closes #904
---
bookkeeper-server/conf/bk_server.conf | 4 ++
.../java/org/apache/bookkeeper/bookie/Bookie.java | 16 +++++++-
.../org/apache/bookkeeper/bookie/BookieShell.java | 45 ++++++++++++++--------
.../bookkeeper/conf/ServerConfiguration.java | 29 ++++++++++++++
.../bookie/BookieInitializationTest.java | 43 +++++++++++++++++++++
.../bookkeeper/bookie/UpdateCookieCmdTest.java | 15 ++++++++
.../bookkeeper/client/UpdateLedgerOpTest.java | 37 +++++++++++++++---
7 files changed, 167 insertions(+), 22 deletions(-)
diff --git a/bookkeeper-server/conf/bk_server.conf
b/bookkeeper-server/conf/bk_server.conf
index 5e2b58e..ebc0ec7 100755
--- a/bookkeeper-server/conf/bk_server.conf
+++ b/bookkeeper-server/conf/bk_server.conf
@@ -437,6 +437,10 @@
ledgerManagerFactoryClass=org.apache.bookkeeper.meta.HierarchicalLedgerManagerFa
# Set the rate at which compaction will readd entries. The unit is adds per
second.
# compactionRate=1000
+# If bookie is using hostname for registration and in ledger metadata then
+# whether to use short hostname or FQDN hostname. Defaults to false.
+# useShortHostName=false
+
# Threshold of minor compaction
# For those entry log files whose remaining size percentage reaches below
# this threshold will be compacted in a minor compaction.
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
index 90b1939..aaabdb5 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
@@ -43,6 +43,7 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.FilenameFilter;
import java.io.IOException;
+import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
@@ -559,15 +560,26 @@ public class Bookie extends BookieCriticalThread {
if (iface == null) {
iface = "default";
}
+
String hostName = DNS.getDefaultHost(iface);
InetSocketAddress inetAddr = new InetSocketAddress(hostName,
conf.getBookiePort());
if (inetAddr.isUnresolved()) {
throw new UnknownHostException("Unable to resolve default
hostname: "
+ hostName + " for interface: " + iface);
}
- String hostAddress = inetAddr.getAddress().getHostAddress();
+ String hostAddress = null;
+ InetAddress iAddress = inetAddr.getAddress();
if (conf.getUseHostNameAsBookieID()) {
- hostAddress = inetAddr.getAddress().getCanonicalHostName();
+ hostAddress = iAddress.getCanonicalHostName();
+ if (conf.getUseShortHostName()) {
+ /*
+ * if short hostname is used, then FQDN is not used. Short
+ * hostname is the hostname cut at the first dot.
+ */
+ hostAddress = hostAddress.split("\\.", 2)[0];
+ }
+ } else {
+ hostAddress = iAddress.getHostAddress();
}
BookieSocketAddress addr =
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
index 2e1dc17..fc707d1 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
@@ -23,6 +23,7 @@ import static com.google.common.base.Charsets.UTF_8;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.common.net.InetAddresses;
import com.google.common.util.concurrent.AbstractFuture;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
@@ -111,7 +112,6 @@ import org.apache.zookeeper.ZooKeeper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
/**
* Bookie Shell is to provide utilities for users to administer a bookkeeper
cluster.
*/
@@ -1244,8 +1244,8 @@ public class BookieShell implements Tool {
super(CMD_LISTBOOKIES);
opts.addOption("rw", "readwrite", false, "Print readwrite
bookies");
opts.addOption("ro", "readonly", false, "Print readonly bookies");
- opts.addOption("h", "hostnames", false,
- "Also print hostname of the bookie");
+ // @deprecated 'rw'/'ro' option print both hostname and ip, so
this option is not needed anymore
+ opts.addOption("h", "hostnames", false, "Also print hostname of
the bookie");
}
@Override
@@ -1274,11 +1274,7 @@ public class BookieShell implements Tool {
bookies.addAll(roBookies);
}
for (BookieSocketAddress b : bookies) {
- System.out.print(b);
- if (cmdLine.hasOption("h")) {
- System.out.print("\t" +
b.getSocketAddress().getHostName());
- }
- System.out.println("");
+ System.out.println(getBookieSocketAddrStringRepresentation(b));
count++;
}
if (count == 0) {
@@ -1586,10 +1582,7 @@ public class BookieShell implements Tool {
LOG.info("No auditor elected");
return -1;
}
- LOG.info("Auditor: {}/{}:{}",
-
bookieId.getSocketAddress().getAddress().getCanonicalHostName(),
-
bookieId.getSocketAddress().getAddress().getHostAddress(),
- bookieId.getSocketAddress().getPort());
+ LOG.info("Auditor: " +
getBookieSocketAddrStringRepresentation(bookieId));
} finally {
if (zk != null) {
zk.close();
@@ -2020,9 +2013,10 @@ public class BookieShell implements Tool {
long totalFree = 0, total = 0;
for (Map.Entry<BookieSocketAddress, BookieInfo> e :
map.entrySet()) {
BookieInfo bInfo = e.getValue();
- System.out.println(e.getKey() + ":\tFree: " +
bInfo.getFreeDiskSpace()
- + getReadable(bInfo.getFreeDiskSpace()) + "\tTotal: "
+ bInfo.getTotalDiskSpace()
- + getReadable(bInfo.getTotalDiskSpace()));
+ BookieSocketAddress bookieId = e.getKey();
+
System.out.println(getBookieSocketAddrStringRepresentation(bookieId) +
":\tFree: "
+ + bInfo.getFreeDiskSpace() +
getReadable(bInfo.getFreeDiskSpace()) + "\tTotal: "
+ + bInfo.getTotalDiskSpace() +
getReadable(bInfo.getTotalDiskSpace()));
totalFree += bInfo.getFreeDiskSpace();
total += bInfo.getTotalDiskSpace();
}
@@ -2447,6 +2441,27 @@ public class BookieShell implements Tool {
return cmd.runCmd(newArgs);
}
+ /*
+ * The string returned is of the form:
+ * 'hostname'('otherformofhostname'):'port number'
+ *
+ * where hostname and otherformofhostname are ipaddress and
+ * canonicalhostname or viceversa
+ */
+ private static String
getBookieSocketAddrStringRepresentation(BookieSocketAddress bookieId) {
+ String hostname = bookieId.getHostName();
+ boolean isHostNameIpAddress = InetAddresses.isInetAddress(hostname);
+ String otherFormOfHostname = null;
+ if (isHostNameIpAddress) {
+ otherFormOfHostname =
bookieId.getSocketAddress().getAddress().getCanonicalHostName();
+ } else {
+ otherFormOfHostname =
bookieId.getSocketAddress().getAddress().getHostAddress();
+ }
+ String bookieSocketAddrStringRepresentation = hostname + "(" +
otherFormOfHostname + ")" + ":"
+ + bookieId.getSocketAddress().getPort();
+ return bookieSocketAddrStringRepresentation;
+ }
+
/**
* Returns the sorted list of the files in the given folders with the
given file extensions.
* Sorting is done on the basis of CreationTime if the CreationTime is not
available or if they are equal
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
index 38eccba..3990e74 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
@@ -135,6 +135,7 @@ public class ServerConfiguration extends
AbstractConfiguration<ServerConfigurati
// Whether the bookie should use its hostname or ipaddress for the
// registration.
protected static final String USE_HOST_NAME_AS_BOOKIE_ID =
"useHostNameAsBookieID";
+ protected static final String USE_SHORT_HOST_NAME = "useShortHostName";
protected static final String ENABLE_LOCAL_TRANSPORT =
"enableLocalTransport";
protected static final String DISABLE_SERVER_SOCKET_BIND =
"disableServerSocketBind";
@@ -2069,6 +2070,34 @@ public class ServerConfiguration extends
AbstractConfiguration<ServerConfigurati
}
/**
+ * If bookie is using hostname for registration and in ledger metadata then
+ * whether to use short hostname or FQDN hostname. Defaults to false.
+ *
+ * @return true, then bookie will be registered with its short hostname and
+ * short hostname will be used in ledger metadata. Otherwise bookie
+ * will use its FQDN hostname
+ */
+ public boolean getUseShortHostName() {
+ return getBoolean(USE_SHORT_HOST_NAME, false);
+ }
+
+ /**
+ * Configure the bookie to use its short hostname or FQDN hostname to
+ * register with the co-ordination service(eg: zookeeper) and in ledger
+ * metadata.
+ *
+ * @see #getUseShortHostName
+ * @param useShortHostName
+ * whether to use short hostname for registration and in
+ * ledgermetadata
+ * @return server configuration
+ */
+ public ServerConfiguration setUseShortHostName(boolean useShortHostName) {
+ setProperty(USE_SHORT_HOST_NAME, useShortHostName);
+ return this;
+ }
+
+ /**
* Get whether to listen for local JVM clients. Defaults to false.
*
* @return true, then bookie will be listen for local JVM clients
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java
index a1fa290..0c240da 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java
@@ -62,6 +62,7 @@ import org.apache.bookkeeper.zookeeper.ZooKeeperClient;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.data.Stat;
+import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
@@ -243,6 +244,48 @@ public class BookieInitializationTest extends
BookKeeperClusterTestCase {
}
}
+ @Test(timeout = 20000)
+ public void testBookieRegistrationWithFQDNHostNameAsBookieID() throws
Exception {
+ File tmpDir = createTempDir("bookie", "test");
+
+ final ServerConfiguration conf =
TestBKConfiguration.newServerConfiguration().setZkServers(null)
+ .setJournalDirName(tmpDir.getPath()).setLedgerDirNames(new
String[] { tmpDir.getPath() })
+ .setUseHostNameAsBookieID(true);
+
+ final String bkRegPath = conf.getZkAvailableBookiesPath() + "/"
+ + InetAddress.getLocalHost().getCanonicalHostName() + ":" +
conf.getBookiePort();
+
+ MockBookie bWithFQDNHostname = new MockBookie(conf);
+ conf.setZkServers(zkUtil.getZooKeeperConnectString());
+ rm.initialize(conf, () -> {}, NullStatsLogger.INSTANCE);
+ bWithFQDNHostname.registrationManager = rm;
+
+ bWithFQDNHostname.testRegisterBookie(conf);
+ Stat bkRegNode1 = zkc.exists(bkRegPath, false);
+ Assert.assertNotNull("Bookie registration node doesn't exists!",
bkRegNode1);
+ }
+
+ @Test(timeout = 20000)
+ public void testBookieRegistrationWithShortHostNameAsBookieID() throws
Exception {
+ File tmpDir = createTempDir("bookie", "test");
+
+ final ServerConfiguration conf =
TestBKConfiguration.newServerConfiguration().setZkServers(null)
+ .setJournalDirName(tmpDir.getPath()).setLedgerDirNames(new
String[] { tmpDir.getPath() })
+ .setUseHostNameAsBookieID(true).setUseShortHostName(true);
+
+ final String bkRegPath = conf.getZkAvailableBookiesPath() + "/"
+ +
(InetAddress.getLocalHost().getCanonicalHostName().split("\\.", 2)[0]) + ":" +
conf.getBookiePort();
+
+ MockBookie bWithShortHostname = new MockBookie(conf);
+ conf.setZkServers(zkUtil.getZooKeeperConnectString());
+ rm.initialize(conf, () -> {}, NullStatsLogger.INSTANCE);
+ bWithShortHostname.registrationManager = rm;
+
+ bWithShortHostname.testRegisterBookie(conf);
+ Stat bkRegNode1 = zkc.exists(bkRegPath, false);
+ Assert.assertNotNull("Bookie registration node doesn't exists!",
bkRegNode1);
+ }
+
/**
* Verify the bookie registration, it should throw
* KeeperException.NodeExistsException if the znode still exists even after
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/UpdateCookieCmdTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/UpdateCookieCmdTest.java
index faa357f..0d9f7eb 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/UpdateCookieCmdTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/UpdateCookieCmdTest.java
@@ -78,6 +78,14 @@ public class UpdateCookieCmdTest extends
BookKeeperClusterTestCase {
}
/**
+ * updatecookie to short hostname.
+ */
+ @Test
+ public void testUpdateCookieIpAddressToShortHostname() throws Exception {
+ updateCookie("-bookieId", "hostname", true, true);
+ }
+
+ /**
* updatecookie to ipaddress.
*/
@Test
@@ -190,6 +198,11 @@ public class UpdateCookieCmdTest extends
BookKeeperClusterTestCase {
}
private void updateCookie(String option, String optionVal, boolean
useHostNameAsBookieID) throws Exception {
+ updateCookie(option, optionVal, useHostNameAsBookieID, false);
+ }
+
+ private void updateCookie(String option, String optionVal, boolean
useHostNameAsBookieID, boolean useShortHostName)
+ throws Exception {
ServerConfiguration conf = new ServerConfiguration(bsConfs.get(0));
BookieServer bks = bs.get(0);
bks.shutdown();
@@ -202,12 +215,14 @@ public class UpdateCookieCmdTest extends
BookKeeperClusterTestCase {
LOG.info("Perform updatecookie command");
ServerConfiguration newconf = new ServerConfiguration(conf);
newconf.setUseHostNameAsBookieID(useHostNameAsBookieID);
+ newconf.setUseShortHostName(useShortHostName);
BookieShell bkShell = new BookieShell();
bkShell.setConf(newconf);
String[] argv = new String[] { "updatecookie", option, optionVal };
Assert.assertEquals("Failed to return exit code!", 0,
bkShell.run(argv));
newconf.setUseHostNameAsBookieID(useHostNameAsBookieID);
+ newconf.setUseShortHostName(useShortHostName);
cookie = Cookie.readFromRegistrationManager(rm, newconf).getValue();
Assert.assertEquals("Wrongly updated cookie!", previousBookieID,
!cookie.isBookieHostCreatedFromIp());
Assert.assertEquals("Wrongly updated cookie!", useHostNameAsBookieID,
!cookie.isBookieHostCreatedFromIp());
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/UpdateLedgerOpTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/UpdateLedgerOpTest.java
index 42e744f..1e58747 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/UpdateLedgerOpTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/UpdateLedgerOpTest.java
@@ -72,10 +72,22 @@ public class UpdateLedgerOpTest extends
BookKeeperClusterTestCase {
};
/**
- * Tests verifies update bookie id when there are many ledgers.
+ * Tests verifies update bookie id to FQDN hostname when there are many
ledgers.
*/
@Test
- public void testManyLedgers() throws Exception {
+ public void testManyLedgersWithFQDNHostname() throws Exception {
+ testManyLedgers(false);
+ }
+
+ /**
+ * Tests verifies update bookie id to short hostname when there are many
ledgers.
+ */
+ @Test(timeout = 120000)
+ public void testManyLedgersWithShortHostname() throws Exception {
+ testManyLedgers(true);
+ }
+
+ public void testManyLedgers(boolean useShortHostName) throws Exception {
BookKeeper bk = new BookKeeper(baseClientConf, zkc);
BookKeeperAdmin bkadmin = new BookKeeperAdmin(bk);
@@ -91,6 +103,7 @@ public class UpdateLedgerOpTest extends
BookKeeperClusterTestCase {
BookieSocketAddress curBookieAddr = ensemble.get(0);
baseConf.setUseHostNameAsBookieID(true);
+ baseConf.setUseShortHostName(useShortHostName);
BookieSocketAddress curBookieId = Bookie.getBookieAddress(baseConf);
BookieSocketAddress toBookieAddr = new
BookieSocketAddress(curBookieId.getHostName() + ":"
+ curBookieAddr.getPort());
@@ -156,11 +169,24 @@ public class UpdateLedgerOpTest extends
BookKeeperClusterTestCase {
}
/**
- * Tests verifies the ensemble reformation after updating the bookie id in
- * the existing ensemble.
+ * Tests verifies the ensemble reformation after updating the bookie id to
+ * FQDN hostname in the existing ensemble.
*/
@Test
- public void testChangeEnsembleAfterRenaming() throws Exception {
+ public void testChangeEnsembleAfterRenamingToFQDNHostname() throws
Exception {
+ testChangeEnsembleAfterRenaming(false);
+ }
+
+ /**
+ * Tests verifies the ensemble reformation after updating the bookie id to
+ * short hostname in the existing ensemble.
+ */
+ @Test(timeout = 120000)
+ public void testChangeEnsembleAfterRenamingToShortHostname() throws
Exception {
+ testChangeEnsembleAfterRenaming(true);
+ }
+
+ public void testChangeEnsembleAfterRenaming(boolean useShortHostName)
throws Exception {
BookKeeper bk = new BookKeeper(baseClientConf, zkc);
BookKeeperAdmin bkadmin = new BookKeeperAdmin(bk);
@@ -178,6 +204,7 @@ public class UpdateLedgerOpTest extends
BookKeeperClusterTestCase {
}
assertNotNull("Couldn't find the bookie in ledger metadata!",
curBookieAddr);
baseConf.setUseHostNameAsBookieID(true);
+ baseConf.setUseShortHostName(useShortHostName);
BookieSocketAddress toBookieId = Bookie.getBookieAddress(baseConf);
BookieSocketAddress toBookieAddr = new
BookieSocketAddress(toBookieId.getHostName() + ":"
+ curBookieAddr.getPort());
--
To stop receiving notification emails like this one, please contact
['"[email protected]" <[email protected]>'].
