This is an automated email from the ASF dual-hosted git repository.
xyz pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/pulsar-client-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new 2f61c62 Speed up the process to start standalone for tests (#236)
2f61c62 is described below
commit 2f61c6284b978d03e2e76db065485e7373fff0b3
Author: Yunze Xu <xyzinfern...@163.com>
AuthorDate: Mon Apr 10 11:28:39 2023 +0800
Speed up the process to start standalone for tests (#236)
### Motivation
It took too much time to start Pulsar standalone for tests because of
the slow JVM startup when running `pulsar-admin` commands.
### Modifications
Use `curl` to send REST requests directly. After this change, the
startup time reduced from 1m39.597s to 32.451s in my local env.
---
.../start-mim-test-service-inside-container.sh | 49 +++++---
.../start-test-service-inside-container.sh | 128 ++++++++++++++-------
2 files changed, 119 insertions(+), 58 deletions(-)
diff --git a/build-support/start-mim-test-service-inside-container.sh
b/build-support/start-mim-test-service-inside-container.sh
index d265191..e7b307d 100755
--- a/build-support/start-mim-test-service-inside-container.sh
+++ b/build-support/start-mim-test-service-inside-container.sh
@@ -31,8 +31,24 @@ bin/pulsar tokens create \
--secret-key file:///pulsar/data/tokens/secret.key \
> /pulsar/data/tokens/token.txt
+# Unset the HTTP proxy to avoid the REST requests being affected
+export http_proxy=
+TOKEN=$(bin/pulsar tokens create \
+ --subject superUser \
+ --secret-key file:///pulsar/data/tokens/secret.key)
+
+# Create "standalone" cluster if it does not exist
+put() {
+ curl -H "Authorization: Bearer $TOKEN" \
+ -L http://localhost:8081/admin/v2/$1 \
+ -H 'Content-Type: application/json' \
+ -X PUT \
+ -d $(echo $2 | sed 's/ //g')
+}
+
export PULSAR_STANDALONE_CONF=test-conf/standalone-ssl-mim.conf
export PULSAR_PID_DIR=/tmp
+sed -i 's/immediateFlush: false/immediateFlush: true/' conf/log4j2.yaml
bin/pulsar-daemon start standalone \
--no-functions-worker --no-stream-storage \
--bookkeeper-dir data/bookkeeper
@@ -42,25 +58,28 @@ until curl http://localhost:8081/metrics > /dev/null 2>&1 ;
do sleep 1; done
echo "-- Pulsar service is ready -- Configure permissions"
-export PULSAR_CLIENT_CONF=test-conf/client-ssl-mim.conf
-
# Create "standalone" cluster if it does not exist
-bin/pulsar-admin clusters list | grep -q '^standalone$' ||
- bin/pulsar-admin clusters create \
- standalone \
- --url http://localhost:8081/ \
- --url-secure https://localhost:8444/ \
- --broker-url pulsar://localhost:6652/ \
- --broker-url-secure pulsar+ssl://localhost:6653/
+put clusters/standalone '{
+ "serviceUrl": "http://localhost:8081/";,
+ "serviceUrlTls": "https://localhost:8444/";,
+ "brokerServiceUrl": "pulsar://localhost:6652/",
+ "brokerServiceUrlTls": "pulsar+ssl://localhost:6653/"
+}'
# Create "private" tenant
-bin/pulsar-admin tenants create private -r "" -c "standalone"
+put tenants/private '{
+ "adminRoles": [],
+ "allowedClusters": ["standalone"]
+}'
# Create "private/auth" with required authentication
-bin/pulsar-admin namespaces create private/auth --clusters standalone
-
-bin/pulsar-admin namespaces grant-permission private/auth \
- --actions produce,consume \
- --role "token-principal"
+put namespaces/private/auth '{
+ "auth_policies": {
+ "namespace_auth": {
+ "token-principal": ["produce", "consume"]
+ }
+ },
+ "replication_clusters": ["standalone"]
+}'
echo "-- Ready to start tests"
diff --git a/build-support/start-test-service-inside-container.sh
b/build-support/start-test-service-inside-container.sh
index 0517f77..678341a 100755
--- a/build-support/start-test-service-inside-container.sh
+++ b/build-support/start-test-service-inside-container.sh
@@ -31,8 +31,26 @@ bin/pulsar tokens create \
--secret-key file:///pulsar/data/tokens/secret.key \
> /pulsar/data/tokens/token.txt
+# Unset the HTTP proxy to avoid the REST requests being affected
+export http_proxy=
+TOKEN=$(bin/pulsar tokens create \
+ --subject superUser \
+ --secret-key file:///pulsar/data/tokens/secret.key)
+
+# Create "standalone" cluster if it does not exist
+put() {
+ curl -H "Authorization: Bearer $TOKEN" \
+ -L http://localhost:8080/admin/v2/$1 \
+ -H 'Content-Type: application/json' \
+ -X PUT \
+ -d $(echo $2 | sed 's/ //g')
+}
+
export PULSAR_STANDALONE_CONF=test-conf/standalone-ssl.conf
export PULSAR_PID_DIR=/tmp
+export PULSAR_STANDALONE_USE_ZOOKEEPER=1
+sed -i 's/immediateFlush: false/immediateFlush: true/' conf/log4j2.yaml
+
bin/pulsar-daemon start standalone \
--no-functions-worker --no-stream-storage \
--bookkeeper-dir data/bookkeeper
@@ -42,60 +60,84 @@ until curl http://localhost:8080/metrics > /dev/null 2>&1 ;
do sleep 1; done
echo "-- Pulsar service is ready -- Configure permissions"
-export PULSAR_CLIENT_CONF=test-conf/client-ssl.conf
-
-# Create "standalone" cluster if it does not exist
-bin/pulsar-admin clusters list | grep -q '^standalone$' ||
- bin/pulsar-admin clusters create \
- standalone \
- --url http://localhost:8080/ \
- --url-secure https://localhost:8443/ \
- --broker-url pulsar://localhost:6650/ \
- --broker-url-secure pulsar+ssl://localhost:6651/
-
-# Update "public" tenant
-bin/pulsar-admin tenants create public -r "anonymous" -c "standalone"
-
-# Update "public/default" with no auth required
-bin/pulsar-admin namespaces create public/default -c standalone
-bin/pulsar-admin namespaces grant-permission public/default \
- --actions produce,consume \
- --role "anonymous"
+put clusters/standalone '{
+ "serviceUrl": "http://localhost:8080/";,
+ "serviceUrlTls": "https://localhost:8443/";,
+ "brokerServiceUrl": "pulsar://localhost:6650/",
+ "brokerServiceUrlTls": "pulsar+ssl://localhost:6651/"
+}'
+
+# Create "public" tenant
+put tenants/public '{
+ "adminRoles": ["anonymous"],
+ "allowedClusters": ["standalone"]
+}'
+
+# Create "public/default" with no auth required
+put namespaces/public/default '{
+ "auth_policies": {
+ "namespace_auth": {
+ "anonymous": ["produce", "consume"]
+ }
+ },
+ "replication_clusters": ["standalone"]
+}'
# Create "public/default-2" with no auth required
-bin/pulsar-admin namespaces create public/default-2 \
- --clusters standalone
-bin/pulsar-admin namespaces grant-permission public/default-2 \
- --actions produce,consume \
- --role "anonymous"
+put namespaces/public/default-2 '{
+ "auth_policies": {
+ "namespace_auth": {
+ "anonymous": ["produce", "consume"]
+ }
+ },
+ "replication_clusters": ["standalone"]
+}'
# Create "public/default-3" with no auth required
-bin/pulsar-admin namespaces create public/default-3 \
- --clusters standalone
-bin/pulsar-admin namespaces grant-permission public/default-3 \
- --actions produce,consume \
- --role "anonymous"
+put namespaces/public/default-3 '{
+ "auth_policies": {
+ "namespace_auth": {
+ "anonymous": ["produce", "consume"]
+ }
+ },
+ "replication_clusters": ["standalone"]
+}'
# Create "public/default-4" with encryption required
-bin/pulsar-admin namespaces create public/default-4 \
- --clusters standalone
-bin/pulsar-admin namespaces grant-permission public/default-4 \
- --actions produce,consume \
- --role "anonymous"
-bin/pulsar-admin namespaces set-encryption-required public/default-4 -e
+put namespaces/public/default-4 '{
+ "auth_policies": {
+ "namespace_auth": {
+ "anonymous": ["produce", "consume"]
+ }
+ },
+ "encryption_required": true,
+ "replication_clusters": ["standalone"]
+}'
# Create "public/test-backlog-quotas" to test backlog quotas policy
-bin/pulsar-admin namespaces create public/test-backlog-quotas \
- --clusters standalone
+put namespaces/public/test-backlog-quotas '{
+ "auth_policies": {
+ "namespace_auth": {
+ "anonymous": ["produce", "consume"]
+ }
+ },
+ "replication_clusters": ["standalone"]
+}'
# Create "private" tenant
-bin/pulsar-admin tenants create private -r "" -c "standalone"
+put tenants/private '{
+ "adminRoles": [],
+ "allowedClusters": ["standalone"]
+}'
# Create "private/auth" with required authentication
-bin/pulsar-admin namespaces create private/auth --clusters standalone
-
-bin/pulsar-admin namespaces grant-permission private/auth \
- --actions produce,consume \
- --role "token-principal"
+put namespaces/private/auth '{
+ "auth_policies": {
+ "namespace_auth": {
+ "token-principal": ["produce", "consume"]
+ }
+ },
+ "replication_clusters": ["standalone"]
+}'
echo "-- Ready to start tests"
