This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-inlong.git
The following commit(s) were added to refs/heads/master by this push:
new be2b977 [INLONG-1938][DataProxy] add multi-pulsar cluster config demo
(#1940)
be2b977 is described below
commit be2b977cbe38c2d075175d5e5559ff6aa1eb6162
Author: baomingyu <[email protected]>
AuthorDate: Thu Dec 9 20:38:25 2021 +0800
[INLONG-1938][DataProxy] add multi-pulsar cluster config demo (#1940)
---
inlong-dataproxy/README.md | 16 +++
inlong-dataproxy/conf/flume-mulit-pulsar-demo.conf | 120 +++++++++++++++++++++
2 files changed, 136 insertions(+)
diff --git a/inlong-dataproxy/README.md b/inlong-dataproxy/README.md
new file mode 100644
index 0000000..adc9d29
--- /dev/null
+++ b/inlong-dataproxy/README.md
@@ -0,0 +1,16 @@
+# Send Message To Multi-pulsar Cluster
+
+# flum.conf
+conf for use multi-pulsar cluster demo is like flume-mulit-pulsar-demo.conf
+when use pulsar ,you can config these parameters to flume.conf like
flume-mulit-pulsar-demo.conf:
+
+ 1. type (*): value must be 'org.apache.inlong.dataproxy.sink.PulsarSink'
+ 2. pulsar_server_url_list (*): value is pulsar broker url , like this
'pulsar://127.0.0.1:6650', multi-pulsar cluster use '|' as seperator
+ 3. send_timeout_MILL: send message timeout, unit is millisecond, default
value is 30000 (mean 30s)
+ 4. stat_interval_sec: stat info will be made period time , unit is second,
default value is 60s
+ 5. thread-num: sink thread num. default value is 8
+ 6. client-id-cache: whether use cache in client, default value is true
+ 7. max_pending_messages: default value is 10000
+ 8. max_batching_messages: default value is 1000
+ 9. enable_batch: default is true
+ 10. block_if_queue_full: default is true
\ No newline at end of file
diff --git a/inlong-dataproxy/conf/flume-mulit-pulsar-demo.conf
b/inlong-dataproxy/conf/flume-mulit-pulsar-demo.conf
new file mode 100644
index 0000000..f28c274
--- /dev/null
+++ b/inlong-dataproxy/conf/flume-mulit-pulsar-demo.conf
@@ -0,0 +1,120 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+agent1.sources = tcp-source
+agent1.channels = ch-msg1 ch-msg2 ch-msg5 ch-msg6
+agent1.sinks = pulsar-sink-msg1 pulsar-sink-msg2 pulsar-sink-msg5
pulsar-sink-msg6
+
+
+agent1.sources.tcp-source.channels = ch-msg1 ch-msg2 ch-msg5 ch-msg6
+agent1.sources.tcp-source.type = org.apache.flume.source.SimpleTcpSource
+agent1.sources.tcp-source.msg-factory-name =
org.apache.flume.source.ServerMessageFactory
+agent1.sources.tcp-source.host = 127.0.0.1
+agent1.sources.tcp-source.port = 46801
+agent1.sources.tcp-source.max-msg-length = 524288
+agent1.sources.tcp-source.topic = test1
+agent1.sources.tcp-source.attr = m=9
+agent1.sources.tcp-source.connections = 30000
+agent1.sources.tcp-source.max-threads = 64
+agent1.sources.tcp-source.receiveBufferSize = 1048576
+agent1.sources.tcp-source.sendBufferSize = 1048576
+agent1.sources.tcp-source.custom-cp = true
+agent1.sources.tcp-source.selector.type =
org.apache.flume.channel.FailoverChannelSelector
+agent1.sources.tcp-source.selector.master = ch-msg1 ch-msg2
+agent1.sources.tcp-source.metric-recovery-path=/data/DataProxy/file/recovery
+agent1.sources.tcp-source.metric-agent-port=8003
+agent1.sources.tcp-source.metric-cache-size=1000000
+agent1.sources.tcp-source.set=10
+
+agent1.channels.ch-msg1.type = memory
+agent1.channels.ch-msg1.capacity = 10000
+agent1.channels.ch-msg1.keep-alive = 0
+agent1.channels.ch-msg1.transactionCapacity = 200
+
+agent1.channels.ch-msg2.type = memory
+agent1.channels.ch-msg2.capacity = 10000
+agent1.channels.ch-msg2.keep-alive = 0
+agent1.channels.ch-msg2.transactionCapacity = 200
+
+agent1.channels.ch-msg5.type = file
+agent1.channels.ch-msg5.capacity = 100000000
+agent1.channels.ch-msg5.maxFileSize = 1073741824
+agent1.channels.ch-msg5.minimumRequiredSpace = 1073741824
+agent1.channels.ch-msg5.checkpointDir = /data/DataProxy/file/ch-msg5/check
+agent1.channels.ch-msg5.dataDirs = /data/DataProxy/file/ch-msg5/data
+agent1.channels.ch-msg5.fsyncPerTransaction = false
+agent1.channels.ch-msg5.fsyncInterval = 10
+
+agent1.channels.ch-msg6.type = file
+agent1.channels.ch-msg6.capacity = 100000000
+agent1.channels.ch-msg6.maxFileSize = 1073741824
+agent1.channels.ch-msg6.minimumRequiredSpace = 1073741824
+agent1.channels.ch-msg6.checkpointDir = /data/DataProxy/file/ch-msg6/check
+agent1.channels.ch-msg6.dataDirs = /data/DataProxy/file/ch-msg6/data
+agent1.channels.ch-msg6.fsyncPerTransaction = false
+agent1.channels.ch-msg6.fsyncInterval = 10
+
+
+agent1.sinks.pulsar-sink-msg1.channel = ch-msg1
+agent1.sinks.pulsar-sink-msg1.type = org.apache.flume.sink.PulsarSink
+agent1.sinks.pulsar-sink-msg1.pulsar_server_url_list =
pulsar://127.0.0.1:6650,127.0.0.2:6650|pulsar://127.0.0.3:6650
+agent1.sinks.pulsar-sink-msg1.send_timeout_MILL = 30000
+agent1.sinks.pulsar-sink-msg1.stat_interval_sec = 60
+agent1.sinks.pulsar-sink-msg1.thread-num = 8
+agent1.sinks.pulsar-sink-msg1.client-id-cache = true
+agent1.sinks.pulsar-sink-msg1.max_survived_time = 300000
+agent1.sinks.pulsar-sink-msg1.max_survived_size = 3000000
+agent1.sinks.pulsar-sink-msg1.netty_write_buffer_high_water_mark = 20971520
+agent1.sinks.pulsar-sink-msg1.disk-io-rate-per-sec=20000000
+
+agent1.sinks.pulsar-sink-msg2.channel = ch-msg2
+agent1.sinks.pulsar-sink-msg2.type = org.apache.flume.sink.PulsarSink
+agent1.sinks.pulsar-sink-msg2.pulsar_server_url_list =
pulsar://127.0.0.1:6650,127.0.0.2:6650|pulsar://127.0.0.3:6650
+agent1.sinks.pulsar-sink-msg2.send_timeout_MILL = 30000
+agent1.sinks.pulsar-sink-msg2.stat_interval_sec = 60
+agent1.sinks.pulsar-sink-msg2.thread-num = 8
+agent1.sinks.pulsar-sink-msg2.client-id-cache = true
+agent1.sinks.pulsar-sink-msg2.max_survived_time = 300000
+agent1.sinks.pulsar-sink-msg2.max_survived_size = 3000000
+agent1.sinks.pulsar-sink-msg2.netty_write_buffer_high_water_mark = 20971520
+agent1.sinks.pulsar-sink-msg2.disk-io-rate-per-sec=20000000
+
+agent1.sinks.pulsar-sink-msg5.channel = ch-msg5
+agent1.sinks.pulsar-sink-msg5.type = org.apache.flume.sink.PulsarSink
+agent1.sinks.pulsar-sink-msg5.pulsar_server_url_list =
pulsar://127.0.0.1:6650,127.0.0.2:6650|pulsar://127.0.0.3:6650
+agent1.sinks.pulsar-sink-msg5.send_timeout_MILL = 30000
+agent1.sinks.pulsar-sink-msg5.stat_interval_sec = 60
+agent1.sinks.pulsar-sink-msg5.thread-num = 8
+agent1.sinks.pulsar-sink-msg5.client-id-cache = true
+agent1.sinks.pulsar-sink-msg5.max_survived_time = 300000
+agent1.sinks.pulsar-sink-msg5.max_survived_size = 3000000
+agent1.sinks.pulsar-sink-msg5.netty_write_buffer_high_water_mark = 20971520
+agent1.sinks.pulsar-sink-msg5.disk-io-rate-per-sec=20000000
+
+agent1.sinks.pulsar-sink-msg6.channel = ch-msg6
+agent1.sinks.pulsar-sink-msg6.type = org.apache.flume.sink.PulsarSink
+agent1.sinks.pulsar-sink-msg6.pulsar_server_url_list =
pulsar://127.0.0.1:6650,127.0.0.2:6650|pulsar://127.0.0.3:6650
+agent1.sinks.pulsar-sink-msg6.send_timeout_MILL = 30000
+agent1.sinks.pulsar-sink-msg6.stat_interval_sec = 60
+agent1.sinks.pulsar-sink-msg6.thread-num = 8
+agent1.sinks.pulsar-sink-msg6.client-id-cache = true
+agent1.sinks.pulsar-sink-msg6.max_survived_time = 300000
+agent1.sinks.pulsar-sink-msg6.max_survived_size = 3000000
+agent1.sinks.pulsar-sink-msg6.netty_write_buffer_high_water_mark = 20971520
+agent1.sinks.pulsar-sink-msg6.disk-io-rate-per-sec=20000000
\ No newline at end of file
