This is an automated email from the ASF dual-hosted git repository. massakam pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pulsar-client-node.git
commit 37b447b2eb0531fa2e592ff02ded8821ae970367 Author: hrsakai <[email protected]> AuthorDate: Mon Apr 22 13:57:38 2019 +0900 Add scripts for tests in docker container --- docker-tests.sh | 37 ++++++ pulsar-test-service-start.sh | 45 ++++++++ pulsar-test-service-stop.sh | 26 +++++ run-unit-tests.sh | 37 ++++++ tests/conf/standalone.conf | 265 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 410 insertions(+) diff --git a/docker-tests.sh b/docker-tests.sh new file mode 100755 index 0000000..fab1042 --- /dev/null +++ b/docker-tests.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +ROOT_DIR=$(git rev-parse --show-toplevel) +cd $ROOT_DIR + +BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-apachepulsar/pulsar-build}" +BUILD_IMAGE_VERSION="${BUILD_IMAGE_VERSION:-ubuntu-16.04}" + +IMAGE="$BUILD_IMAGE_NAME:$BUILD_IMAGE_VERSION" + +echo "---- Testing Pulsar node client using image $IMAGE" + +docker pull $IMAGE + +DOCKER_CMD="docker run -i -v $ROOT_DIR:/pulsar-client-node $IMAGE" + +# Start Pulsar standalone instance +# and execute the tests +$DOCKER_CMD bash -c "cd /pulsar-client-node && ./run-unit-tests.sh" diff --git a/pulsar-test-service-start.sh b/pulsar-test-service-start.sh new file mode 100755 index 0000000..af9ad03 --- /dev/null +++ b/pulsar-test-service-start.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e + +ROOT_DIR=$(git rev-parse --show-toplevel) +cd $ROOT_DIR + +VERSION="${VERSION:-2.3.1}" +PULSAR_DIR="${PULSAR_DIR:-/tmp/pulsar-test-dist}" +PKG=apache-pulsar-${VERSION}-bin.tar.gz + +rm -rf $PULSAR_DIR +curl -L --create-dir "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=pulsar/pulsar-${VERSION}/${PKG}"; -o $PULSAR_DIR/$PKG +tar xfz $PULSAR_DIR/$PKG -C $PULSAR_DIR --strip-components 1 + +DATA_DIR=/tmp/pulsar-test-data +rm -rf $DATA_DIR +mkdir -p $DATA_DIR + +export PULSAR_STANDALONE_CONF=$ROOT_DIR/tests/conf/standalone.conf +$PULSAR_DIR/bin/pulsar-daemon start standalone \ + --no-functions-worker --no-stream-storage \ + --zookeeper-dir $DATA_DIR/zookeeper \ + --bookkeeper-dir $DATA_DIR/bookkeeper + +echo "-- Wait for Pulsar service to be ready" +until curl http://localhost:8080/metrics > /dev/null 2>&1 ; do sleep 1; done diff --git a/pulsar-test-service-stop.sh b/pulsar-test-service-stop.sh new file mode 100755 index 0000000..f162426 --- /dev/null +++ b/pulsar-test-service-stop.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e + +PULSAR_DIR="${PULSAR_DIR:-/tmp/pulsar-test-dist}" +cd $PULSAR_DIR + +bin/pulsar-daemon stop standalone diff --git a/run-unit-tests.sh b/run-unit-tests.sh new file mode 100755 index 0000000..c8358a8 --- /dev/null +++ b/run-unit-tests.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e + +# install pulsar cpp client pkg +VERSION="${VERSION:-2.3.1}" +PULSAR_PKG_DIR="/tmp/pulsar-test-pkg" +rm -rf $PULSAR_PKG_DIR +for pkg in apache-pulsar-client-dev.deb apache-pulsar-client.deb;do + curl -L --create-dir "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=pulsar/pulsar-${VERSION}/DEB/${pkg}"; -o $PULSAR_PKG_DIR/$pkg +done; +apt install $PULSAR_PKG_DIR/apache-pulsar-client*.deb + +./pulsar-test-service-start.sh +npm install && npm run build && npm run test +RES=$? +./pulsar-test-service-stop.sh + +exit $RES diff --git a/tests/conf/standalone.conf b/tests/conf/standalone.conf new file mode 100755 index 0000000..18d4c42 --- /dev/null +++ b/tests/conf/standalone.conf @@ -0,0 +1,265 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +### --- General broker settings --- ### + +# Zookeeper quorum connection string +zookeeperServers= + +# Global Zookeeper quorum connection string +globalZookeeperServers= + +brokerServicePort=6650 + +# Port to use to server HTTP request +webServicePort=8080 + +# Hostname or IP address the service binds on, default is 0.0.0.0. +bindAddress=0.0.0.0 + +# Hostname or IP address the service advertises to the outside world. If not set, the value of InetAddress.getLocalHost().getHostName() is used. +advertisedAddress=localhost + +# Name of the cluster to which this broker belongs to +clusterName=standalone + +# Zookeeper session timeout in milliseconds +zooKeeperSessionTimeoutMillis=30000 + +# Time to wait for broker graceful shutdown. After this time elapses, the process will be killed +brokerShutdownTimeoutMs=3000 + +# Enable backlog quota check. Enforces action on topic when the quota is reached +backlogQuotaCheckEnabled=true + +# How often to check for topics that have reached the quota +backlogQuotaCheckIntervalInSeconds=60 + +# Default per-topic backlog quota limit +backlogQuotaDefaultLimitGB=10 + +# Enable the deletion of inactive topics +brokerDeleteInactiveTopicsEnabled=true + +# How often to check for inactive topics +brokerDeleteInactiveTopicsFrequencySeconds=60 + +# How frequently to proactively check and purge expired messages +messageExpiryCheckIntervalInMinutes=5 + +# Enable check for minimum allowed client library version +clientLibraryVersionCheckEnabled=false + +# Allow client libraries with no version information +clientLibraryVersionCheckAllowUnversioned=true + +# Path for the file used to determine the rotation status for the broker when responding +# to service discovery health checks +statusFilePath=/usr/local/apache/htdocs + +# Max number of unacknowledged messages allowed to receive messages by a consumer on a shared subscription. Broker will stop sending +# messages to consumer once, this limit reaches until consumer starts acknowledging messages back +# Using a value of 0, is disabling unackeMessage limit check and consumer can receive messages without any restriction +maxUnackedMessagesPerConsumer=50000 + +### --- Authentication --- ### + +# Enable authentication +authenticationEnabled=false + +# Autentication provider name list, which is comma separated list of class names +authenticationProviders=false + +# Enforce authorization +authorizationEnabled=false + +# Role names that are treated as "super-user", meaning they will be able to do all admin +# operations and publish/consume from all topics +superUserRoles= + +# Authentication settings of the broker itself. Used when the broker connects to other brokers, +# either in same or other clusters +brokerClientAuthenticationPlugin= +brokerClientAuthenticationParameters= + + +### --- BookKeeper Client --- ### + +# Authentication plugin to use when connecting to bookies +bookkeeperClientAuthenticationPlugin= + +# BookKeeper auth plugin implementatation specifics parameters name and values +bookkeeperClientAuthenticationParametersName= +bookkeeperClientAuthenticationParameters= + +# Timeout for BK add / read operations +bookkeeperClientTimeoutInSeconds=30 + +# Speculative reads are initiated if a read request doesn't complete within a certain time +# Using a value of 0, is disabling the speculative reads +bookkeeperClientSpeculativeReadTimeoutInMillis=0 + +# Enable bookies health check. Bookies that have more than the configured number of failure within +# the interval will be quarantined for some time. During this period, new ledgers won't be created +# on these bookies +bookkeeperClientHealthCheckEnabled=true +bookkeeperClientHealthCheckIntervalSeconds=60 +bookkeeperClientHealthCheckErrorThresholdPerInterval=5 +bookkeeperClientHealthCheckQuarantineTimeInSeconds=1800 + +# Enable rack-aware bookie selection policy. BK will chose bookies from different racks when +# forming a new bookie ensemble +bookkeeperClientRackawarePolicyEnabled=true + +# Enable region-aware bookie selection policy. BK will chose bookies from +# different regions and racks when forming a new bookie ensemble +# If enabled, the value of bookkeeperClientRackawarePolicyEnabled is ignored +bookkeeperClientRegionawarePolicyEnabled=false + +# Enable/disable reordering read sequence on reading entries. +bookkeeperClientReorderReadSequenceEnabled=false + +# Enable bookie isolation by specifying a list of bookie groups to choose from. Any bookie +# outside the specified groups will not be used by the broker +bookkeeperClientIsolationGroups= + +### --- Managed Ledger --- ### + +# Number of bookies to use when creating a ledger +managedLedgerDefaultEnsembleSize=1 + +# Number of copies to store for each message +managedLedgerDefaultWriteQuorum=1 + +# Number of guaranteed copies (acks to wait before write is complete) +managedLedgerDefaultAckQuorum=1 + +# Amount of memory to use for caching data payload in managed ledger. This memory +# is allocated from JVM direct memory and it's shared across all the topics +# running in the same broker +managedLedgerCacheSizeMB=1024 + +# Threshold to which bring down the cache level when eviction is triggered +managedLedgerCacheEvictionWatermark=0.9 + +# Rate limit the amount of writes generated by consumer acking the messages +managedLedgerDefaultMarkDeleteRateLimit=0.1 + +# Max number of entries to append to a ledger before triggering a rollover +# A ledger rollover is triggered on these conditions +# * Either the max rollover time has been reached +# * or max entries have been written to the ledged and at least min-time +# has passed +managedLedgerMaxEntriesPerLedger=50000 + +# Minimum time between ledger rollover for a topic +managedLedgerMinLedgerRolloverTimeMinutes=10 + +# Maximum time before forcing a ledger rollover for a topic +managedLedgerMaxLedgerRolloverTimeMinutes=240 + +# Max number of entries to append to a cursor ledger +managedLedgerCursorMaxEntriesPerLedger=50000 + +# Max time before triggering a rollover on a cursor ledger +managedLedgerCursorRolloverTimeInSeconds=14400 + + + +### --- Load balancer --- ### + +# Enable load balancer +loadBalancerEnabled=false + +# Strategy to assign a new bundle +loadBalancerPlacementStrategy=weightedRandomSelection + +# Percentage of change to trigger load report update +loadBalancerReportUpdateThresholdPercentage=10 + +# maximum interval to update load report +loadBalancerReportUpdateMaxIntervalMinutes=15 + +# Frequency of report to collect +loadBalancerHostUsageCheckIntervalMinutes=1 + +# Load shedding interval. Broker periodically checks whether some traffic should be offload from +# some over-loaded broker to other under-loaded brokers +loadBalancerSheddingIntervalMinutes=30 + +# Prevent the same topics to be shed and moved to other broker more that once within this timeframe +loadBalancerSheddingGracePeriodMinutes=30 + +# Usage threshold to determine a broker as under-loaded +loadBalancerBrokerUnderloadedThresholdPercentage=1 + +# Usage threshold to determine a broker as over-loaded +loadBalancerBrokerOverloadedThresholdPercentage=85 + +# Interval to update namespace bundle resource quotat +loadBalancerResourceQuotaUpdateIntervalMinutes=15 + +# Usage threshold to determine a broker is having just right level of load +loadBalancerBrokerComfortLoadLevelPercentage=65 + +# enable/disable namespace bundle auto split +loadBalancerAutoBundleSplitEnabled=false + +# interval to detect & split hot namespace bundle +loadBalancerNamespaceBundleSplitIntervalMinutes=15 + +# maximum topics in a bundle, otherwise bundle split will be triggered +loadBalancerNamespaceBundleMaxTopics=1000 + +# maximum sessions (producers + consumers) in a bundle, otherwise bundle split will be triggered +loadBalancerNamespaceBundleMaxSessions=1000 + +# maximum msgRate (in + out) in a bundle, otherwise bundle split will be triggered +loadBalancerNamespaceBundleMaxMsgRate=1000 + +# maximum bandwidth (in + out) in a bundle, otherwise bundle split will be triggered +loadBalancerNamespaceBundleMaxBandwidthMbytes=100 + +# maximum number of bundles in a namespace +loadBalancerNamespaceMaximumBundles=128 + +### --- Replication --- ### + +# Enable replication metrics +replicationMetricsEnabled=true + +# Max number of connections to open for each broker in a remote cluster +# More connections host-to-host lead to better throughput over high-latency +# links. +replicationConnectionsPerBroker=16 + +# Replicator producer queue size +replicationProducerQueueSize=1000 + +# Default message retention time +defaultRetentionTimeInMinutes=0 + +# Default retention size +defaultRetentionSizeInMB=0 + +# How often to check whether the connections are still alive +keepAliveIntervalSeconds=30 + +# How often broker checks for inactive topics to be deleted (topics with no subscriptions and no one connected) +brokerServicePurgeInactiveFrequencyInSeconds=60
