This is an automated email from the ASF dual-hosted git repository.
eolivelli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 1d8366a [TABLE SERVICE] Improve bin/bookkeeper standalone to start
all available service components
1d8366a is described below
commit 1d8366a0a2468856d3a5671d36b17f542cb0c75c
Author: Sijie Guo <[email protected]>
AuthorDate: Mon Jul 23 16:36:20 2018 +0200
[TABLE SERVICE] Improve bin/bookkeeper standalone to start all available
service components
Descriptions of the changes in this PR:
*Motivation*
Currently `bin/bookkeeper standalone` simply starts local bookies. It is
not very useful when using
standalone for local development for other service components.
*Changes*
- change `bin/bookkeeper standalone` to start a cluster of zookeeper,
bookkeeper and table service containers
- add `bin/standalone.process` to start a cluster of zookeeper, bookkeeper
and table service containers in one process
- change `bin/standalone` to support both process and docker-compose
Author: Sijie Guo <[email protected]>
Reviewers: Enrico Olivelli <[email protected]>, Jia Zhai <None>
This closes #1553 from sijie/improve_bk_standalone
---
bin/bookkeeper | 16 ++-
bin/bookkeeper-daemon.sh | 5 +-
bin/standalone | 44 ++++++-
bin/standalone.docker-compose | 2 +-
bin/standalone.process | 83 ++++++++++++
bookkeeper-common/pom.xml | 12 ++
.../common/conf/ComponentConfiguration.java | 31 +++++
.../apache/bookkeeper/common}/util/JsonUtil.java | 19 ++-
.../bookkeeper/conf/AbstractConfiguration.java | 4 +-
.../org/apache/bookkeeper/proto/BookieServer.java | 2 +-
.../server/http/service/ConfigurationService.java | 2 +-
.../server/http/service/DecommissionService.java | 2 +-
.../server/http/service/DeleteLedgerService.java | 2 +-
.../server/http/service/GetLastLogMarkService.java | 2 +-
.../server/http/service/GetLedgerMetaService.java | 2 +-
.../server/http/service/ListBookieInfoService.java | 2 +-
.../server/http/service/ListBookiesService.java | 2 +-
.../server/http/service/ListDiskFilesService.java | 2 +-
.../server/http/service/ListLedgerService.java | 2 +-
.../service/ListUnderReplicatedLedgerService.java | 2 +-
.../service/LostBookieRecoveryDelayService.java | 2 +-
.../http/service/ReadLedgerEntryService.java | 2 +-
.../server/http/service/RecoveryBookieService.java | 2 +-
.../bookkeeper/server/http/TestHttpService.java | 2 +-
bin/standalone => conf/standalone.conf | 23 ++--
.../stream/cluster/StandaloneStarter.java | 143 +++++++++++++++++++--
.../bookkeeper/stream/cluster/StreamCluster.java | 88 +++++++------
.../stream/cluster/StreamClusterSpec.java | 10 +-
.../bookkeeper/stream/server/StorageServer.java | 6 +
.../stream/server/service/BookieWatchService.java | 2 +-
.../tests/containers/BKStandaloneContainer.java | 16 ++-
.../integration/standalone/StandaloneTest.java | 43 +++++++
32 files changed, 460 insertions(+), 117 deletions(-)
diff --git a/bin/bookkeeper b/bin/bookkeeper
index ceb0470..07e7db3 100755
--- a/bin/bookkeeper
+++ b/bin/bookkeeper
@@ -23,9 +23,13 @@ BK_HOME=`cd ${BINDIR}/..;pwd`
source ${BK_HOME}/bin/common.sh
-BOOKIE_MODULE_PATH=bookkeeper-server
-BOOKIE_MODULE_NAME="(org.apache.bookkeeper-)?bookkeeper-server"
-BOOKIE_MODULE_HOME=${BK_HOME}/${BOOKIE_MODULE_PATH}
+if [ "x$1" == "xstandalone" ]; then
+ BOOKIE_MODULE_PATH=stream/server
+ BOOKIE_MODULE_NAME="(org.apache.bookkeeper-)?stream-storage-server"
+else
+ BOOKIE_MODULE_PATH=bookkeeper-server
+ BOOKIE_MODULE_NAME="(org.apache.bookkeeper-)?bookkeeper-server"
+fi
# find the module jar
BOOKIE_JAR=$(find_module_jar ${BOOKIE_MODULE_PATH} ${BOOKIE_MODULE_NAME})
@@ -44,7 +48,7 @@ where command is one of:
bookie Run a bookie server
autorecovery Run AutoRecovery service daemon
localbookie <n> Run a test ensemble of <n> bookies locally
- standalone <n> Run a standalone cluster of <n> bookies locally
+ standalone Run a standalone cluster (with all service components)
locally
upgrade Upgrade bookie filesystem
shell Run shell for admin commands
zookeeper Run zookeeper server
@@ -130,10 +134,12 @@ if [ ${COMMAND} == "bookie" ]; then
exec ${JAVA} ${OPTS} ${JMX_ARGS} org.apache.bookkeeper.server.Main --conf
${BOOKIE_CONF} $@
elif [ ${COMMAND} == "autorecovery" ]; then
exec ${JAVA} ${OPTS} ${JMX_ARGS}
org.apache.bookkeeper.replication.AutoRecoveryMain --conf ${BOOKIE_CONF} $@
-elif [ ${COMMAND} == "localbookie" -o ${COMMAND} == "standalone" ]; then
+elif [ ${COMMAND} == "localbookie" ]; then
NUMBER=$1
shift
exec ${JAVA} ${OPTS} ${JMX_ARGS} -Dzookeeper.4lw.commands.whitelist='*'
org.apache.bookkeeper.util.LocalBookKeeper ${NUMBER} ${BOOKIE_CONF} $@
+elif [ ${COMMAND} == "standalone" ]; then
+ exec ${JAVA} ${OPTS} ${JMX_ARGS} -Dzookeeper.4lw.commands.whitelist='*'
org.apache.bookkeeper.stream.cluster.StandaloneStarter --conf
${BK_HOME}/conf/standalone.conf $@
elif [ ${COMMAND} == "upgrade" ]; then
exec ${JAVA} ${OPTS} org.apache.bookkeeper.bookie.FileSystemUpgrade --conf
${BOOKIE_CONF} $@
elif [ $COMMAND == "zookeeper" ]; then
diff --git a/bin/bookkeeper-daemon.sh b/bin/bookkeeper-daemon.sh
index 3a8bbdc..d64a4489 100755
--- a/bin/bookkeeper-daemon.sh
+++ b/bin/bookkeeper-daemon.sh
@@ -69,6 +69,9 @@ case $command in
(autorecovery)
echo "doing $startStop $command ..."
;;
+ (standalone)
+ echo "doing $startStop $command ..."
+ ;;
(*)
echo "Error: unknown service name $command"
usage
@@ -134,7 +137,7 @@ case $startStop in
count=0
location=$BOOKIE_LOG_DIR
- while kill -0 $TARGET_PID > /dev/null;
+ while kill -0 $TARGET_PID > /dev/null 2>&1;
do
echo "Shutdown is in progress... Please wait..."
sleep 1
diff --git a/bin/standalone b/bin/standalone
index d4bc0f5..6ed5b09 100755
--- a/bin/standalone
+++ b/bin/standalone
@@ -19,13 +19,47 @@
# * See the License for the specific language governing permissions and
# * limitations under the License.
# */
+usage() {
+ cat <<EOF
+Usage: standalone <runtime> (up|down)
+where runtime is one of:
+ process Run a standalone cluster in one process
+ docker-compose Run a standalone cluster in a docker-composed cluster
+EOF
+}
BINDIR=${BK_BINDIR:-"`dirname "$0"`"}
-DOCKER_COMPOSE=$(which docker-compose)
-if [ $? != 0 ]; then
- echo "Error: docker-compose is not found in ${PATH}." 1>&2
+if [ $# -lt 2 ]
+then
+ echo "Error: no enough arguments provided."
+ usage
exit 1
-else
- ${BINDIR}/standalone.docker-compose $@
fi
+
+runtime=$1
+shift
+action=$1
+shift
+
+case $runtime in
+ (process)
+ ${BINDIR}/standalone.process $action $@
+ ;;
+ (docker-compose)
+ DOCKER_COMPOSE=$(which docker-compose)
+ if [ $? != 0 ]; then
+ echo "Error: docker-compose is not found in ${PATH}." 1>&2
+ usage
+ exit 1
+ else
+ echo ${BINDIR}/standalone.docker-compose $action $@
+ ${BINDIR}/standalone.docker-compose $action $@
+ fi
+ ;;
+ (*)
+ echo "Error: unknown runtime $command"
+ usage
+ exit 1
+ ;;
+esac
diff --git a/bin/standalone.docker-compose b/bin/standalone.docker-compose
index 4cf1ffd..05808a7 100755
--- a/bin/standalone.docker-compose
+++ b/bin/standalone.docker-compose
@@ -111,7 +111,7 @@ function generate_docker_compose_file() {
function show_help() {
cat <<EOF
-Usage: standalone [-c <cluster_name>] [-h] <action:[up|down]>
+Usage: standalone.docker-compose [-c <cluster_name>] [-h] <action:[up|down]>
EOF
}
diff --git a/bin/standalone.process b/bin/standalone.process
new file mode 100755
index 0000000..661995d
--- /dev/null
+++ b/bin/standalone.process
@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+#
+# vim:et:ft=sh:sts=2:sw=2
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+function show_help() {
+ cat <<EOF
+Usage: standalone.process [-c <cluster_name>] [-h] <action:[up|down]>
+EOF
+}
+
+BINDIR=${BK_BINDIR:-"`dirname "$0"`"}
+
+source ${BINDIR}/common.sh
+
+DATA_ROOT_DIR=${BK_DATA_DIR:-"${BK_HOME}/data"}
+mkdir -p ${DATA_ROOT_DIR}
+
+# main entrypoint
+
+CLUSTER_NAME="bk-standalone"
+OPTIND=1
+
+while getopts "h:c:" opt; do
+ case "${opt}" in
+ c )
+ CLUSTER_NAME=${OPTARG}
+ echo "use cluster = '${CLUSTER_NAME}'."
+ ;;
+ h|\? )
+ show_help
+ exit 1
+ ;;
+ esac
+done
+
+shift $((OPTIND-1))
+
+[ "${1:-}" = "--" ] && shift
+
+if [ $# -le 0 ]; then
+ show_help
+ exit 1
+fi
+
+ACTION=$1
+DAEMON_ACTION=""
+case "${ACTION}" in
+ up)
+ DAEMON_ACTION="start"
+ ;;
+ down)
+ DAEMON_ACTION="stop"
+ ;;
+ *)
+ echo "Unknown action : ${ACTION}"
+ show_help
+ exit 1
+ ;;
+esac
+
+
+CLUSTER_DATA_ROOT="${DATA_ROOT_DIR}/${CLUSTER_NAME}"
+mkdir -p ${CLUSTER_DATA_ROOT}
+
+${BINDIR}/bookkeeper-daemon.sh ${DAEMON_ACTION} standalone --data-dir
${CLUSTER_DATA_ROOT}
diff --git a/bookkeeper-common/pom.xml b/bookkeeper-common/pom.xml
index 740f16a..ea07705 100644
--- a/bookkeeper-common/pom.xml
+++ b/bookkeeper-common/pom.xml
@@ -39,6 +39,18 @@
<artifactId>netty-common</artifactId>
</dependency>
<dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
+ <dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
<scope>provided</scope>
diff --git
a/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/conf/ComponentConfiguration.java
b/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/conf/ComponentConfiguration.java
index 7d22443..8b27597 100644
---
a/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/conf/ComponentConfiguration.java
+++
b/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/conf/ComponentConfiguration.java
@@ -21,9 +21,13 @@ package org.apache.bookkeeper.common.conf;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.net.URL;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
+import org.apache.bookkeeper.common.util.JsonUtil;
+import org.apache.bookkeeper.common.util.JsonUtil.ParseJsonException;
import org.apache.commons.configuration.CompositeConfiguration;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
@@ -287,4 +291,31 @@ public abstract class ComponentConfiguration implements
Configuration {
public List<Object> getList(String key, List<?> defaultValue) {
return conf.getList(getKeyName(key), defaultValue);
}
+
+ /**
+ * returns the string representation of json format of this config.
+ *
+ * @return
+ * @throws ParseJsonException
+ */
+ public String asJson() {
+ try {
+ return JsonUtil.toJson(toMap());
+ } catch (ParseJsonException e) {
+ throw new RuntimeException("Failed to serialize the configuration
as json", e);
+ }
+ }
+
+ private Map<String, Object> toMap() {
+ Map<String, Object> configMap = new HashMap<>();
+ Iterator<String> iterator = this.getKeys();
+ while (iterator.hasNext()) {
+ String key = iterator.next().toString();
+ Object property = this.getProperty(key);
+ if (property != null) {
+ configMap.put(key, property.toString());
+ }
+ }
+ return configMap;
+ }
}
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/JsonUtil.java
b/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/util/JsonUtil.java
similarity index 77%
rename from
bookkeeper-server/src/main/java/org/apache/bookkeeper/util/JsonUtil.java
rename to
bookkeeper-common/src/main/java/org/apache/bookkeeper/common/util/JsonUtil.java
index e5e7142..afc90a4 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/JsonUtil.java
+++
b/bookkeeper-common/src/main/java/org/apache/bookkeeper/common/util/JsonUtil.java
@@ -1,5 +1,4 @@
-/**
- *
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -8,17 +7,15 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * http://www.apache.org/licenses/LICENSE-2.0
*
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
-package org.apache.bookkeeper.util;
+package org.apache.bookkeeper.common.util;
import com.fasterxml.jackson.databind.ObjectMapper;
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java
index 2c80f43..481c5a5 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/AbstractConfiguration.java
@@ -28,14 +28,14 @@ import java.util.Map;
import javax.net.ssl.SSLEngine;
import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.common.util.JsonUtil;
+import org.apache.bookkeeper.common.util.JsonUtil.ParseJsonException;
import org.apache.bookkeeper.feature.Feature;
import org.apache.bookkeeper.meta.AbstractZkLedgerManagerFactory;
import org.apache.bookkeeper.meta.HierarchicalLedgerManagerFactory;
import org.apache.bookkeeper.meta.LedgerManagerFactory;
import org.apache.bookkeeper.meta.LongHierarchicalLedgerManagerFactory;
import org.apache.bookkeeper.util.EntryFormatter;
-import org.apache.bookkeeper.util.JsonUtil;
-import org.apache.bookkeeper.util.JsonUtil.ParseJsonException;
import org.apache.bookkeeper.util.LedgerIdFormatter;
import org.apache.bookkeeper.util.ReflectionUtils;
import org.apache.bookkeeper.util.StringEntryFormatter;
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java
index d386b91..711c232 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java
@@ -37,6 +37,7 @@ import org.apache.bookkeeper.bookie.BookieException;
import org.apache.bookkeeper.bookie.ExitCode;
import org.apache.bookkeeper.bookie.ReadOnlyBookie;
import org.apache.bookkeeper.client.BKException;
+import org.apache.bookkeeper.common.util.JsonUtil.ParseJsonException;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.net.BookieSocketAddress;
import org.apache.bookkeeper.processor.RequestProcessor;
@@ -48,7 +49,6 @@ import org.apache.bookkeeper.stats.StatsLogger;
import org.apache.bookkeeper.tls.SecurityException;
import org.apache.bookkeeper.tls.SecurityHandlerFactory;
import org.apache.bookkeeper.tls.SecurityProviderFactoryFactory;
-import org.apache.bookkeeper.util.JsonUtil.ParseJsonException;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ConfigurationService.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ConfigurationService.java
index 2ac9e8e..bcf5578 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ConfigurationService.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ConfigurationService.java
@@ -23,12 +23,12 @@ import static
com.google.common.base.Preconditions.checkNotNull;
import java.util.HashMap;
import java.util.Map;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.service.HttpEndpointService;
import org.apache.bookkeeper.http.service.HttpServiceRequest;
import org.apache.bookkeeper.http.service.HttpServiceResponse;
-import org.apache.bookkeeper.util.JsonUtil;
/**
* HttpEndpointService that handle Bookkeeper Configuration related http
request.
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/DecommissionService.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/DecommissionService.java
index ea05f6e..0091ce9 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/DecommissionService.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/DecommissionService.java
@@ -24,13 +24,13 @@ import java.util.HashMap;
import java.util.concurrent.ExecutorService;
import org.apache.bookkeeper.client.BookKeeperAdmin;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.service.HttpEndpointService;
import org.apache.bookkeeper.http.service.HttpServiceRequest;
import org.apache.bookkeeper.http.service.HttpServiceResponse;
import org.apache.bookkeeper.net.BookieSocketAddress;
-import org.apache.bookkeeper.util.JsonUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/DeleteLedgerService.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/DeleteLedgerService.java
index 2593e94..626b89f 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/DeleteLedgerService.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/DeleteLedgerService.java
@@ -22,13 +22,13 @@ import static
com.google.common.base.Preconditions.checkNotNull;
import java.util.Map;
import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ClientConfiguration;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.service.HttpEndpointService;
import org.apache.bookkeeper.http.service.HttpServiceRequest;
import org.apache.bookkeeper.http.service.HttpServiceResponse;
-import org.apache.bookkeeper.util.JsonUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/GetLastLogMarkService.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/GetLastLogMarkService.java
index 4cbd1dc..0ffa9bb 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/GetLastLogMarkService.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/GetLastLogMarkService.java
@@ -30,13 +30,13 @@ import java.util.Map;
import org.apache.bookkeeper.bookie.Journal;
import org.apache.bookkeeper.bookie.LedgerDirsManager;
import org.apache.bookkeeper.bookie.LogMark;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.service.HttpEndpointService;
import org.apache.bookkeeper.http.service.HttpServiceRequest;
import org.apache.bookkeeper.http.service.HttpServiceResponse;
import org.apache.bookkeeper.util.DiskChecker;
-import org.apache.bookkeeper.util.JsonUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/GetLedgerMetaService.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/GetLedgerMetaService.java
index 5a06faf..43b4994 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/GetLedgerMetaService.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/GetLedgerMetaService.java
@@ -24,6 +24,7 @@ import static
com.google.common.base.Preconditions.checkNotNull;
import com.google.common.collect.Maps;
import java.util.Map;
import org.apache.bookkeeper.client.LedgerMetadata;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.service.HttpEndpointService;
@@ -32,7 +33,6 @@ import org.apache.bookkeeper.http.service.HttpServiceResponse;
import org.apache.bookkeeper.meta.LedgerManager;
import org.apache.bookkeeper.meta.LedgerManagerFactory;
import org.apache.bookkeeper.proto.BookieServer;
-import org.apache.bookkeeper.util.JsonUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListBookieInfoService.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListBookieInfoService.java
index 36afa92..14439be 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListBookieInfoService.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListBookieInfoService.java
@@ -29,6 +29,7 @@ import java.util.Map;
import org.apache.bookkeeper.client.BookKeeper;
import org.apache.bookkeeper.client.BookieInfoReader;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ClientConfiguration;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
@@ -36,7 +37,6 @@ import org.apache.bookkeeper.http.service.HttpEndpointService;
import org.apache.bookkeeper.http.service.HttpServiceRequest;
import org.apache.bookkeeper.http.service.HttpServiceResponse;
import org.apache.bookkeeper.net.BookieSocketAddress;
-import org.apache.bookkeeper.util.JsonUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListBookiesService.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListBookiesService.java
index b28c715..f969ff7 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListBookiesService.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListBookiesService.java
@@ -27,13 +27,13 @@ import java.util.Collection;
import java.util.Map;
import org.apache.bookkeeper.client.BookKeeperAdmin;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.service.HttpEndpointService;
import org.apache.bookkeeper.http.service.HttpServiceRequest;
import org.apache.bookkeeper.http.service.HttpServiceResponse;
import org.apache.bookkeeper.net.BookieSocketAddress;
-import org.apache.bookkeeper.util.JsonUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListDiskFilesService.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListDiskFilesService.java
index 260b82d..330ea7f 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListDiskFilesService.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListDiskFilesService.java
@@ -27,12 +27,12 @@ import java.io.File;
import java.util.List;
import java.util.Map;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.service.HttpEndpointService;
import org.apache.bookkeeper.http.service.HttpServiceRequest;
import org.apache.bookkeeper.http.service.HttpServiceResponse;
-import org.apache.bookkeeper.util.JsonUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListLedgerService.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListLedgerService.java
index 758af70..efd00e7 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListLedgerService.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListLedgerService.java
@@ -29,6 +29,7 @@ import java.util.List;
import java.util.Map;
import org.apache.bookkeeper.client.BKException;
import org.apache.bookkeeper.client.LedgerMetadata;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.service.HttpEndpointService;
@@ -38,7 +39,6 @@ import org.apache.bookkeeper.meta.LedgerManager;
import org.apache.bookkeeper.meta.LedgerManagerFactory;
import org.apache.bookkeeper.proto.BookieServer;
import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks;
-import org.apache.bookkeeper.util.JsonUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListUnderReplicatedLedgerService.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListUnderReplicatedLedgerService.java
index 237760c..7303be0 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListUnderReplicatedLedgerService.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListUnderReplicatedLedgerService.java
@@ -26,6 +26,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Predicate;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.service.HttpEndpointService;
@@ -34,7 +35,6 @@ import org.apache.bookkeeper.http.service.HttpServiceResponse;
import org.apache.bookkeeper.meta.LedgerManagerFactory;
import org.apache.bookkeeper.meta.LedgerUnderreplicationManager;
import org.apache.bookkeeper.proto.BookieServer;
-import org.apache.bookkeeper.util.JsonUtil;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/LostBookieRecoveryDelayService.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/LostBookieRecoveryDelayService.java
index fcc9eff..6442daa 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/LostBookieRecoveryDelayService.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/LostBookieRecoveryDelayService.java
@@ -23,12 +23,12 @@ import static
com.google.common.base.Preconditions.checkNotNull;
import java.util.HashMap;
import org.apache.bookkeeper.client.BookKeeperAdmin;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.service.HttpEndpointService;
import org.apache.bookkeeper.http.service.HttpServiceRequest;
import org.apache.bookkeeper.http.service.HttpServiceResponse;
-import org.apache.bookkeeper.util.JsonUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ReadLedgerEntryService.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ReadLedgerEntryService.java
index 9bed45d..4c2141d 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ReadLedgerEntryService.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ReadLedgerEntryService.java
@@ -27,12 +27,12 @@ import java.util.Iterator;
import java.util.Map;
import org.apache.bookkeeper.client.BookKeeperAdmin;
import org.apache.bookkeeper.client.LedgerEntry;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.service.HttpEndpointService;
import org.apache.bookkeeper.http.service.HttpServiceRequest;
import org.apache.bookkeeper.http.service.HttpServiceResponse;
-import org.apache.bookkeeper.util.JsonUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/RecoveryBookieService.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/RecoveryBookieService.java
index afc5bcd..3a08baf 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/RecoveryBookieService.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/RecoveryBookieService.java
@@ -28,13 +28,13 @@ import java.util.concurrent.ExecutorService;
import org.apache.bookkeeper.bookie.Cookie;
import org.apache.bookkeeper.client.BookKeeperAdmin;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.http.HttpServer;
import org.apache.bookkeeper.http.service.HttpEndpointService;
import org.apache.bookkeeper.http.service.HttpServiceRequest;
import org.apache.bookkeeper.http.service.HttpServiceResponse;
import org.apache.bookkeeper.net.BookieSocketAddress;
-import org.apache.bookkeeper.util.JsonUtil;
import org.apache.bookkeeper.versioning.Versioned;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/server/http/TestHttpService.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/server/http/TestHttpService.java
index 1ed1a22..a2a7728 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/server/http/TestHttpService.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/server/http/TestHttpService.java
@@ -35,6 +35,7 @@ import org.apache.bookkeeper.client.BookKeeper;
import org.apache.bookkeeper.client.LedgerHandle;
import org.apache.bookkeeper.client.LedgerHandleAdapter;
import org.apache.bookkeeper.client.LedgerMetadata;
+import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.conf.TestBKConfiguration;
import org.apache.bookkeeper.http.HttpServer;
@@ -48,7 +49,6 @@ import org.apache.bookkeeper.net.BookieSocketAddress;
import org.apache.bookkeeper.replication.AuditorElector;
import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
import org.apache.bookkeeper.test.TestCallbacks;
-import org.apache.bookkeeper.util.JsonUtil;
import org.apache.bookkeeper.zookeeper.ZooKeeperClient;
import org.apache.zookeeper.ZooKeeper;
import org.junit.Before;
diff --git a/bin/standalone b/conf/standalone.conf
similarity index 63%
copy from bin/standalone
copy to conf/standalone.conf
index d4bc0f5..4b52f2b 100755
--- a/bin/standalone
+++ b/conf/standalone.conf
@@ -1,7 +1,3 @@
-#!/usr/bin/env bash
-#
-# vim:et:ft=sh:sts=2:sw=2
-#
#/**
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements. See the NOTICE file
@@ -20,12 +16,15 @@
# * limitations under the License.
# */
-BINDIR=${BK_BINDIR:-"`dirname "$0"`"}
+# Standalone configuration
-DOCKER_COMPOSE=$(which docker-compose)
-if [ $? != 0 ]; then
- echo "Error: docker-compose is not found in ${PATH}." 1>&2
- exit 1
-else
- ${BINDIR}/standalone.docker-compose $@
-fi
+##################################################################
+##################################################################
+# stream/table service
+##################################################################
+##################################################################
+
+### Storage ###
+
+# the cluster controller schedule interval, in milliseconds. default is 30
seconds.
+storage.cluster.controller.schedule.interval.ms=30000
diff --git
a/stream/server/src/main/java/org/apache/bookkeeper/stream/cluster/StandaloneStarter.java
b/stream/server/src/main/java/org/apache/bookkeeper/stream/cluster/StandaloneStarter.java
index 16d73d2..a8c94ea 100644
---
a/stream/server/src/main/java/org/apache/bookkeeper/stream/cluster/StandaloneStarter.java
+++
b/stream/server/src/main/java/org/apache/bookkeeper/stream/cluster/StandaloneStarter.java
@@ -17,10 +17,16 @@
*/
package org.apache.bookkeeper.stream.cluster;
-import java.nio.file.Files;
-import java.nio.file.Path;
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import java.io.File;
import java.util.concurrent.CountDownLatch;
import lombok.extern.slf4j.Slf4j;
+import org.apache.bookkeeper.common.net.ServiceURI;
+import org.apache.commons.configuration.CompositeConfiguration;
+import org.apache.commons.configuration.PropertiesConfiguration;
/**
* A starter to start a standalone cluster.
@@ -28,6 +34,78 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class StandaloneStarter {
+ /**
+ * Args for standalone starter.
+ */
+ private static class StarterArgs {
+
+ @Parameter(
+ names = {
+ "-c", "--conf"
+ },
+ description = "Configuration file path")
+ String configFile = null;
+
+ @Parameter(
+ names = {
+ "--num-bookies"
+ },
+ description = "Num of bookies")
+ int numBookies = 1;
+
+ @Parameter(
+ names = {
+ "--zk-port"
+ },
+ description = "ZooKeeper port")
+ int zkPort = 2181;
+
+ @Parameter(
+ names = {
+ "-u", "--metadata-service-uri"
+ },
+ description = "Uri of the external metadata service."
+ + " If set, the standalone will not start any metadata service
itself.")
+ String metadataServiceUri = null;
+
+ @Parameter(
+ names = {
+ "--initial-bookie-port"
+ },
+ description = "Initial bookie port")
+ int initialBookiePort = 3181;
+
+ @Parameter(
+ names = {
+ "--initial-bookie-grpc-port"
+ },
+ description = "Initial bookie grpc port")
+ int initialBookieGrpcPort = 4181;
+
+ @Parameter(
+ names = {
+ "--data-dir"
+ },
+ description = "Location to store standalone data"
+ )
+ String dataDir = "data";
+
+ @Parameter(
+ names = {
+ "--wipe-data"
+ },
+ description = "Clean up previous standalone data")
+ boolean wipeData = false;
+
+ @Parameter(
+ names = {
+ "-h", "--help"
+ },
+ description = "Show this help message")
+ boolean help = false;
+
+ }
+
public static void main(String[] args) throws Exception {
int retCode = doMain(args);
@@ -35,16 +113,57 @@ public class StandaloneStarter {
}
static int doMain(String[] args) throws Exception {
- Path rootDir = Files.createTempDirectory("data");
- rootDir.toFile().deleteOnExit();
-
- StreamClusterSpec spec = StreamClusterSpec.builder()
- .numServers(3)
- .initialBookiePort(3181)
- .initialGrpcPort(4181)
- .serveReadOnlyTable(true)
- .shouldStartZooKeeper(true)
- .storageRootDir(rootDir.toFile())
+ StarterArgs starterArgs = new StarterArgs();
+
+ JCommander commander = new JCommander();
+ try {
+ commander.setProgramName("standalone-starter");
+ commander.addObject(starterArgs);
+ commander.parse(args);
+ if (starterArgs.help) {
+ commander.usage();
+ return 0;
+ }
+ } catch (Exception e) {
+ commander.usage();
+ return -1;
+ }
+
+ StreamClusterSpec.StreamClusterSpecBuilder specBuilder =
StreamClusterSpec.builder();
+ if (starterArgs.metadataServiceUri == null) {
+ specBuilder = specBuilder
+ .zkPort(starterArgs.zkPort)
+ .shouldStartZooKeeper(true);
+ } else {
+ ServiceURI serviceURI =
ServiceURI.create(starterArgs.metadataServiceUri);
+ specBuilder = specBuilder
+ .metadataServiceUri(serviceURI)
+ .shouldStartZooKeeper(false);
+ }
+
+ CompositeConfiguration conf = new CompositeConfiguration();
+ if (null != starterArgs.configFile) {
+ PropertiesConfiguration propsConf = new
PropertiesConfiguration(starterArgs.configFile);
+ conf.addConfiguration(propsConf);
+ }
+
+ checkArgument(starterArgs.numBookies > 0, "Invalid number of bookies :
" + starterArgs.numBookies);
+ if (starterArgs.numBookies == 1) {
+ conf.setProperty("dlog.bkcEnsembleSize", 1);
+ conf.setProperty("dlog.bkcWriteQuorumSize", 1);
+ conf.setProperty("dlog.bkcAckQuorumSize", 1);
+ } else {
+ conf.setProperty("dlog.bkcEnsembleSize", starterArgs.numBookies);
+ conf.setProperty("dlog.bkcWriteQuorumSize",
starterArgs.numBookies);
+ conf.setProperty("dlog.bkcAckQuorumSize", starterArgs.numBookies -
1);
+ }
+
+ StreamClusterSpec spec = specBuilder
+ .baseConf(conf)
+ .numServers(starterArgs.numBookies)
+ .initialBookiePort(starterArgs.initialBookiePort)
+ .initialGrpcPort(starterArgs.initialBookieGrpcPort)
+ .storageRootDir(new File(starterArgs.dataDir))
.build();
CountDownLatch liveLatch = new CountDownLatch(1);
diff --git
a/stream/server/src/main/java/org/apache/bookkeeper/stream/cluster/StreamCluster.java
b/stream/server/src/main/java/org/apache/bookkeeper/stream/cluster/StreamCluster.java
index e5a6df4..407b27f 100644
---
a/stream/server/src/main/java/org/apache/bookkeeper/stream/cluster/StreamCluster.java
+++
b/stream/server/src/main/java/org/apache/bookkeeper/stream/cluster/StreamCluster.java
@@ -14,17 +14,15 @@
package org.apache.bookkeeper.stream.cluster;
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
import static org.apache.bookkeeper.common.concurrent.FutureUtils.result;
import static
org.apache.bookkeeper.stream.protocol.ProtocolConstants.DEFAULT_STREAM_CONF;
-import static
org.apache.bookkeeper.stream.storage.StorageConstants.ZK_METADATA_ROOT_PATH;
-import static
org.apache.bookkeeper.stream.storage.StorageConstants.getSegmentsRootPath;
-import static
org.apache.bookkeeper.stream.storage.StorageConstants.getStoragePath;
import com.google.common.collect.Lists;
import java.io.File;
import java.io.IOException;
import java.net.BindException;
-import java.net.URI;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -34,10 +32,12 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.clients.StorageClientBuilder;
import org.apache.bookkeeper.clients.admin.StorageAdminClient;
import org.apache.bookkeeper.clients.config.StorageClientSettings;
-import org.apache.bookkeeper.clients.exceptions.NamespaceExistsException;
+import org.apache.bookkeeper.clients.exceptions.ClientException;
+import org.apache.bookkeeper.clients.exceptions.NamespaceNotFoundException;
import org.apache.bookkeeper.clients.utils.NetUtils;
import org.apache.bookkeeper.common.component.AbstractLifecycleComponent;
import org.apache.bookkeeper.common.component.LifecycleComponent;
+import org.apache.bookkeeper.common.net.ServiceURI;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.meta.MetadataDrivers;
import org.apache.bookkeeper.shims.zk.ZooKeeperServerShim;
@@ -46,12 +46,11 @@ import
org.apache.bookkeeper.stream.proto.NamespaceConfiguration;
import org.apache.bookkeeper.stream.proto.NamespaceProperties;
import org.apache.bookkeeper.stream.proto.common.Endpoint;
import org.apache.bookkeeper.stream.server.StorageServer;
-import org.apache.bookkeeper.stream.storage.StorageConstants;
import org.apache.bookkeeper.stream.storage.conf.StorageConfiguration;
import org.apache.bookkeeper.stream.storage.exceptions.StorageRuntimeException;
import org.apache.bookkeeper.stream.storage.impl.cluster.ZkClusterInitializer;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
-import org.apache.distributedlog.DistributedLogConfiguration;
import org.apache.distributedlog.LocalDLMEmulator;
/**
@@ -73,10 +72,9 @@ public class StreamCluster
return new StreamCluster(spec);
}
- private static ServerConfiguration newBookieConfiguration(String
zkEnsemble) {
+ private static ServerConfiguration newBookieConfiguration(ServiceURI
serviceURI) {
ServerConfiguration serverConf = new ServerConfiguration();
- serverConf.setMetadataServiceUri(
- "zk://" + zkEnsemble +
getSegmentsRootPath(StorageConstants.ZK_METADATA_ROOT_PATH));
+ serverConf.setMetadataServiceUri(serviceURI.getUri().toString());
serverConf.setAllowLoopback(true);
serverConf.setGcWaitTime(300000);
serverConf.setDiskUsageWarnThreshold(0.9999f);
@@ -86,7 +84,7 @@ public class StreamCluster
private final StreamClusterSpec spec;
private final List<Endpoint> rpcEndpoints;
- private String zkEnsemble;
+ private ServiceURI metadataServiceUri;
private int zkPort;
private ZooKeeperServerShim zks;
private List<LifecycleComponent> servers;
@@ -109,24 +107,21 @@ public class StreamCluster
return rpcEndpoints;
}
- public URI getDefaultBackendUri() {
- return URI.create("distributedlog://" + zkEnsemble +
getStoragePath(ZK_METADATA_ROOT_PATH));
- }
-
private void startZooKeeper() throws Exception {
if (!spec.shouldStartZooKeeper()) {
- zkPort = spec.zkPort();
- zkEnsemble = spec.zkServers() + ":" + zkPort;
+ metadataServiceUri = checkNotNull(spec.metadataServiceUri,
+ "No metadata service uri is configured while configuring not
to start zookeeper");
return;
}
File zkDir = new File(spec.storageRootDir(), "zookeeper");
Pair<ZooKeeperServerShim, Integer> zkServerAndPort =
- LocalDLMEmulator.runZookeeperOnAnyPort(zkDir);
+ LocalDLMEmulator.runZookeeperOnAnyPort(spec.zkPort(), zkDir);
zks = zkServerAndPort.getLeft();
zkPort = zkServerAndPort.getRight();
log.info("Started zookeeper at port {}.", zkPort);
- zkEnsemble = "127.0.0.1:" + zkPort;
+ metadataServiceUri = ServiceURI.create(
+ "zk://127.0.0.1:" + zkPort + "/ledgers");
}
private void stopZooKeeper() {
@@ -137,12 +132,17 @@ public class StreamCluster
}
private void initializeCluster() throws Exception {
- new ZkClusterInitializer(zkEnsemble).initializeCluster(
- URI.create("zk://" + zkEnsemble),
+
checkArgument(ServiceURI.SERVICE_ZK.equals(metadataServiceUri.getServiceName()),
+ "Only support zookeeper based metadata service now");
+ String[] serviceHosts = metadataServiceUri.getServiceHosts();
+ String metadataServers = StringUtils.join(serviceHosts, ',');
+
+ new ZkClusterInitializer(metadataServers).initializeCluster(
+ metadataServiceUri.getUri(),
spec.numServers() * 2);
// format the bookkeeper cluster
-
MetadataDrivers.runFunctionWithMetadataBookieDriver(newBookieConfiguration(zkEnsemble),
driver -> {
+
MetadataDrivers.runFunctionWithMetadataBookieDriver(newBookieConfiguration(metadataServiceUri),
driver -> {
try {
boolean initialized =
driver.getRegistrationManager().initNewCluster();
if (initialized) {
@@ -170,23 +170,20 @@ public class StreamCluster
}
LifecycleComponent server = null;
try {
- ServerConfiguration serverConf =
newBookieConfiguration(zkEnsemble);
+ ServerConfiguration serverConf =
newBookieConfiguration(metadataServiceUri);
+ serverConf.loadConf(spec.baseConf());
serverConf.setBookiePort(bookiePort);
File bkDir = new File(spec.storageRootDir(), "bookie_" +
bookiePort);
serverConf.setJournalDirName(bkDir.getPath());
serverConf.setLedgerDirNames(new String[]{bkDir.getPath()});
- DistributedLogConfiguration dlConf = new
DistributedLogConfiguration();
- dlConf.loadConf(serverConf);
-
File rangesStoreDir = new File(spec.storageRootDir(),
"ranges_" + grpcPort);
StorageConfiguration storageConf = new
StorageConfiguration(serverConf);
storageConf.setRangeStoreDirNames(new
String[]{rangesStoreDir.getPath()});
- storageConf.setServeReadOnlyTables(spec.serveReadOnlyTable);
log.info("Attempting to start storage server at (bookie port =
{}, grpc port = {})"
- + " : bkDir = {}, rangesStoreDir = {},
serveReadOnlyTables = {}",
- bookiePort, grpcPort, bkDir, rangesStoreDir,
spec.serveReadOnlyTable);
+ + " : bkDir = {}, rangesStoreDir = {}",
+ bookiePort, grpcPort, bkDir, rangesStoreDir);
server = StorageServer.buildStorageServer(
serverConf,
grpcPort);
@@ -245,18 +242,27 @@ public class StreamCluster
.withSettings(settings)
.buildAdmin()) {
- System.out.println("Creating namespace '" + namespaceName + "'
...");
- try {
- NamespaceProperties nsProps = result(
- admin.createNamespace(
- namespaceName,
- NamespaceConfiguration.newBuilder()
- .setDefaultStreamConf(DEFAULT_STREAM_CONF)
- .build()));
- System.out.println("Successfully created namespace '" +
namespaceName + "':");
- System.out.println(nsProps);
- } catch (NamespaceExistsException nee) {
- System.out.println("Namespace '" + namespaceName + "' already
exists.");
+ boolean created = false;
+ while (!created) {
+ try {
+ result(admin.getNamespace(namespaceName));
+ created = true;
+ } catch (NamespaceNotFoundException nnfe) {
+ log.info("Namespace '{}' is not found.");
+ log.info("Creating namespace '{}' ...", namespaceName);
+ try {
+ NamespaceProperties nsProps = result(
+ admin.createNamespace(
+ namespaceName,
+ NamespaceConfiguration.newBuilder()
+ .setDefaultStreamConf(DEFAULT_STREAM_CONF)
+ .build()));
+ log.info("Successfully created namespace '{}':",
namespaceName);
+ log.info("{}", nsProps);
+ } catch (ClientException ce) {
+ // encountered exception, try to fetch the namespace
again
+ }
+ }
}
}
}
diff --git
a/stream/server/src/main/java/org/apache/bookkeeper/stream/cluster/StreamClusterSpec.java
b/stream/server/src/main/java/org/apache/bookkeeper/stream/cluster/StreamClusterSpec.java
index 41608bc..76fd97d 100644
---
a/stream/server/src/main/java/org/apache/bookkeeper/stream/cluster/StreamClusterSpec.java
+++
b/stream/server/src/main/java/org/apache/bookkeeper/stream/cluster/StreamClusterSpec.java
@@ -20,6 +20,7 @@ import lombok.Builder.Default;
import lombok.Getter;
import lombok.Setter;
import lombok.experimental.Accessors;
+import org.apache.bookkeeper.common.net.ServiceURI;
import org.apache.commons.configuration.CompositeConfiguration;
/**
@@ -47,12 +48,12 @@ public class StreamClusterSpec {
CompositeConfiguration baseConf;
/**
- * Returns the zookeeper servers used in this cluster.
+ * Returns the metadata service uri that the servers will connect to.
*
- * @return the zookeeper servers used in this cluster.
+ * @return the metadata service uri that the servers will connect to.
*/
@Default
- String zkServers = "127.0.0.1";
+ ServiceURI metadataServiceUri = null;
/**
* Returns if should start zookeeper.
@@ -94,7 +95,4 @@ public class StreamClusterSpec {
@Default
File storageRootDir = new File("data/bookkeeper");
- @Default
- boolean serveReadOnlyTable = false;
-
}
diff --git
a/stream/server/src/main/java/org/apache/bookkeeper/stream/server/StorageServer.java
b/stream/server/src/main/java/org/apache/bookkeeper/stream/server/StorageServer.java
index ad8491e..97ff66a 100644
---
a/stream/server/src/main/java/org/apache/bookkeeper/stream/server/StorageServer.java
+++
b/stream/server/src/main/java/org/apache/bookkeeper/stream/server/StorageServer.java
@@ -206,11 +206,17 @@ public class StorageServer {
StatsProviderService statsProviderService = new
StatsProviderService(bkConf);
rootStatsLogger =
statsProviderService.getStatsProvider().getStatsLogger("");
serverBuilder.addComponent(statsProviderService);
+ log.info("Bookie configuration : {}", bkConf.asJson());
} else {
rootStatsLogger = checkNotNull(externalStatsLogger,
"External stats logger is not provided while not starting
stats provider");
}
+ // dump configurations
+ log.info("Dlog configuration : {}", dlConf.asJson());
+ log.info("Storage configuration : {}", storageConf.asJson());
+ log.info("Server configuration : {}", serverConf.asJson());
+
// Create the bookie service
ServerConfiguration bkServerConf;
if (startBookieAndStartProvider) {
diff --git
a/stream/server/src/main/java/org/apache/bookkeeper/stream/server/service/BookieWatchService.java
b/stream/server/src/main/java/org/apache/bookkeeper/stream/server/service/BookieWatchService.java
index 5a645d3..4a3741f 100644
---
a/stream/server/src/main/java/org/apache/bookkeeper/stream/server/service/BookieWatchService.java
+++
b/stream/server/src/main/java/org/apache/bookkeeper/stream/server/service/BookieWatchService.java
@@ -83,7 +83,7 @@ public class BookieWatchService
bookies =
FutureUtils.result(client.getWritableBookies()).getValue();
log.info("Only {} bookies are live since {} seconds elapsed, "
+ "wait for another {} bookies for another 1 second",
- bookies.size(), minNumBookies - bookies.size(),
stopwatch.elapsed(TimeUnit.SECONDS));
+ bookies.size(), stopwatch.elapsed(TimeUnit.SECONDS),
minNumBookies - bookies.size());
}
}
diff --git
a/tests/integration-tests-topologies/src/main/java/org/apache/bookkeeper/tests/containers/BKStandaloneContainer.java
b/tests/integration-tests-topologies/src/main/java/org/apache/bookkeeper/tests/containers/BKStandaloneContainer.java
index 5f1e438..7beb291 100644
---
a/tests/integration-tests-topologies/src/main/java/org/apache/bookkeeper/tests/containers/BKStandaloneContainer.java
+++
b/tests/integration-tests-topologies/src/main/java/org/apache/bookkeeper/tests/containers/BKStandaloneContainer.java
@@ -23,7 +23,7 @@ import static java.time.temporal.ChronoUnit.SECONDS;
import java.time.Duration;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
-import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy;
+import org.testcontainers.containers.wait.strategy.HostPortWaitStrategy;
/**
* Test Container for Bookies.
@@ -33,6 +33,7 @@ public class BKStandaloneContainer<SelfT extends
BKStandaloneContainer<SelfT>> e
private static final int ZK_PORT = 2181;
private static final int BOOKIE_BASE_PORT = 3181;
+ private static final int BOOKIE_GRPC_BASE_PORT = 4181;
private static final String IMAGE_NAME =
"apachebookkeeper/bookkeeper-current:latest";
@@ -60,14 +61,19 @@ public class BKStandaloneContainer<SelfT extends
BKStandaloneContainer<SelfT>> e
}
setCommand(
"standalone",
- "" + numBookies);
+ "--num-bookies",
+ String.valueOf(numBookies),
+ "--zk-port",
+ String.valueOf(ZK_PORT),
+ "--initial-bookie-port",
+ String.valueOf(BOOKIE_BASE_PORT),
+ "--initial-bookie-grpc-port",
+ String.valueOf(BOOKIE_GRPC_BASE_PORT));
}
@Override
public void start() {
- this.waitStrategy = new LogMessageWaitStrategy()
- .withRegEx(".*ForceWrite Thread started.*\\s")
- .withTimes(numBookies)
+ this.waitStrategy = new HostPortWaitStrategy()
.withStartupTimeout(Duration.of(60, SECONDS));
this.withCreateContainerCmdModifier(createContainerCmd -> {
createContainerCmd.withHostName(STANDALONE_HOST_NAME);
diff --git
a/tests/integration/standalone/src/test/java/org/apache/bookkeeper/tests/integration/standalone/StandaloneTest.java
b/tests/integration/standalone/src/test/java/org/apache/bookkeeper/tests/integration/standalone/StandaloneTest.java
index df082d4..f7b7077 100644
---
a/tests/integration/standalone/src/test/java/org/apache/bookkeeper/tests/integration/standalone/StandaloneTest.java
+++
b/tests/integration/standalone/src/test/java/org/apache/bookkeeper/tests/integration/standalone/StandaloneTest.java
@@ -23,7 +23,9 @@ import static org.junit.Assert.assertTrue;
import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.tests.containers.BKStandaloneContainer;
import org.junit.ClassRule;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TestName;
import org.testcontainers.containers.Container.ExecResult;
/**
@@ -32,6 +34,9 @@ import org.testcontainers.containers.Container.ExecResult;
@Slf4j
public class StandaloneTest {
+ @Rule
+ public final TestName testName = new TestName();
+
@ClassRule
public static BKStandaloneContainer bkContainer = new
BKStandaloneContainer<>("integrationtest", 3);
@@ -50,4 +55,42 @@ public class StandaloneTest {
result.getStdout().contains("100 entries written to ledger"));
}
+ //
+ // `namespace` commands
+ //
+
+ @Test
+ public void createNamespace() throws Exception {
+ String nsName = testName.getMethodName();
+ ExecResult result = bkContainer.execCmd(
+ "/opt/bookkeeper/bin/bkctl",
+ "-u bk://localhost:4181",
+ "namespace",
+ "create",
+ nsName
+ );
+ assertTrue(
+ result.getStdout(),
+ result.getStdout().contains("Successfully created namespace '" +
nsName + "'"));
+ }
+
+ //
+ // `tables` commands
+ //
+
+ @Test
+ public void createTable() throws Exception {
+ String tableName = testName.getMethodName();
+ ExecResult result = bkContainer.execCmd(
+ "/opt/bookkeeper/bin/bkctl",
+ "-u bk://localhost:4181",
+ "tables",
+ "create",
+ tableName
+ );
+ assertTrue(
+ result.getStdout(),
+ result.getStdout().contains("Successfully created stream '" +
tableName + "'"));
+ }
+
}
