Repository: incubator-distributedlog Updated Branches: refs/heads/master 67b8b6541 -> e9c429ed5
DL-152: removed symlink distributedlog-service/conf and copied files from distributedlog-core/conf to distributed-service/conf â¦tributedlog-core/conf to distributedlog-service/conf Author: adamtracymartin <atmar...@yahoo.com> Reviewers: Sijie Guo <si...@apache.org> Closes #105 from adamtracymartin/DL-152 Project: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/commit/e9c429ed Tree: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/tree/e9c429ed Diff: http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/diff/e9c429ed Branch: refs/heads/master Commit: e9c429ed5e32ab8ebea280217401519ba51cc803 Parents: 67b8b65 Author: adamtracymartin <atmar...@yahoo.com> Authored: Sun Jan 8 14:06:38 2017 -0800 Committer: Sijie Guo <si...@apache.org> Committed: Sun Jan 8 14:06:38 2017 -0800 ---------------------------------------------------------------------- distributedlog-service/conf | 1 - .../conf/bookie.conf.template | 185 +++++++++++++++++++ distributedlog-service/conf/distributedlog.conf | 127 +++++++++++++ distributedlog-service/conf/dlogenv.sh | 77 ++++++++ distributedlog-service/conf/log4j.properties | 62 +++++++ distributedlog-service/conf/write_proxy.conf | 145 +++++++++++++++ .../conf/zookeeper.conf.dynamic.template | 1 + .../conf/zookeeper.conf.template | 82 ++++++++ 8 files changed, 679 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/e9c429ed/distributedlog-service/conf ---------------------------------------------------------------------- diff --git a/distributedlog-service/conf b/distributedlog-service/conf deleted file mode 120000 index 451c305..0000000 --- a/distributedlog-service/conf +++ /dev/null @@ -1 +0,0 @@ -../distributedlog-core/conf \ No newline at end of file diff --git a/distributedlog-service/conf/bookie.conf.template b/distributedlog-service/conf/bookie.conf.template new file mode 100644 index 0000000..3dacb94 --- /dev/null +++ b/distributedlog-service/conf/bookie.conf.template @@ -0,0 +1,185 @@ +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +## Bookie settings + +# Port that bookie server listen on +bookiePort=3181 + +# TODO: change the journal directory +# Directory Bookkeeper outputs its write ahead log +journalDirectory=/tmp/data/bk/journal + +# TODO: change the ledgers directory +# Directory Bookkeeper outputs ledger snapshots +ledgerDirectories=/tmp/data/bk/ledgers + +# TODO: change the index directory +# Directory in which index files will be stored. +indexDirectories=/tmp/data/bk/ledgers + +# Ledger Manager Class +# What kind of ledger manager is used to manage how ledgers are stored, managed +# and garbage collected. Try to read 'BookKeeper Internals' for detail info. +ledgerManagerType=hierarchical + +# Root zookeeper path to store ledger metadata +# This parameter is used by zookeeper-based ledger manager as a root znode to +# store all ledgers. +zkLedgersRootPath=/messaging/bookkeeper/ledgers + +# Max file size of entry logger, in bytes +# A new entry log file will be created when the old one reaches the file size limitation +logSizeLimit=1073741823 + +# Max file size of journal file, in mega bytes +# A new journal file will be created when the old one reaches the file size limitation +# +journalMaxSizeMB=2048 + +# Max number of old journal file to kept +# Keep a number of old journal files would help data recovery in specia case +# +journalMaxBackups=5 + +# How long the interval to trigger next garbage collection, in milliseconds +# Since garbage collection is running in background, too frequent gc +# will heart performance. It is better to give a higher number of gc +# interval if there is enough disk capacity. +# gc per 1 hour (aligning with most DL rolling interval) +gcInitialWaitTime=600000 +gcWaitTime=3600000 +# do minor compaction per 2 hours +minorCompactionInterval=7200 +minorCompactionThreshold=0.2 +# disable major compaction +majorCompactionInterval=0 +# reduce major compaction threshold to a low value to prevent bad force compaction behavior +majorCompactionThreshold=0.3 +# Compaction Rate & Max Outstanding +compactionRate=10737418 +compactionMaxOutstandingRequests=10737418 + +# How long the interval to flush ledger index pages to disk, in milliseconds +# Flushing index files will introduce much random disk I/O. +# If separating journal dir and ledger dirs each on different devices, +# flushing would not affect performance. But if putting journal dir +# and ledger dirs on same device, performance degrade significantly +# on too frequent flushing. You can consider increment flush interval +# to get better performance, but you need to pay more time on bookie +# server restart after failure. +# +flushInterval=1000 + +# Interval to watch whether bookie is dead or not, in milliseconds +# +# bookieDeathWatchInterval=1000 + +## zookeeper client settings + +# A list of one of more servers on which zookeeper is running. +# The server list can be comma separated values, for example: +# zkServers=zk1:2181,zk2:2181,zk3:2181 +zkServers=localhost:2181 + +# ZooKeeper client session timeout in milliseconds +# Bookie server will exit if it received SESSION_EXPIRED because it +# was partitioned off from ZooKeeper for more than the session timeout +# JVM garbage collection, disk I/O will cause SESSION_EXPIRED. +# Increment this value could help avoiding this issue +zkTimeout=30000 + +## NIO Server settings + +# This settings is used to enabled/disabled Nagle's algorithm, which is a means of +# improving the efficiency of TCP/IP networks by reducing the number of packets +# that need to be sent over the network. +# If you are sending many small messages, such that more than one can fit in +# a single IP packet, setting server.tcpnodelay to false to enable Nagle algorithm +# can provide better performance. +# Default value is true. +# +serverTcpNoDelay=true + +## ledger cache settings + +# Max number of ledger index files could be opened in bookie server +# If number of ledger index files reaches this limitation, bookie +# server started to swap some ledgers from memory to disk. +# Too frequent swap will affect performance. You can tune this number +# to gain performance according your requirements. +openFileLimit=20000 + +# Size of a index page in ledger cache, in bytes +# A larger index page can improve performance writing page to disk, +# which is efficent when you have small number of ledgers and these +# ledgers have similar number of entries. +# If you have large number of ledgers and each ledger has fewer entries, +# smaller index page would improve memory usage. +pageSize=8192 + +# How many index pages provided in ledger cache +# If number of index pages reaches this limitation, bookie server +# starts to swap some ledgers from memory to disk. You can increment +# this value when you found swap became more frequent. But make sure +# pageLimit*pageSize should not more than JVM max memory limitation, +# otherwise you would got OutOfMemoryException. +# In general, incrementing pageLimit, using smaller index page would +# gain bettern performance in lager number of ledgers with fewer entries case +# If pageLimit is -1, bookie server will use 1/3 of JVM memory to compute +# the limitation of number of index pages. +pageLimit=131072 + +#If all ledger directories configured are full, then support only read requests for clients. +#If "readOnlyModeEnabled=true" then on all ledger disks full, bookie will be converted +#to read-only mode and serve only read requests. Otherwise the bookie will be shutdown. +readOnlyModeEnabled=true + +# Bookie Journal Settings +writeBufferSizeBytes=262144 +journalFlushWhenQueueEmpty=false +journalRemoveFromPageCache=true +journalAdaptiveGroupWrites=true +journalMaxGroupWaitMSec=4 +journalBufferedEntriesThreshold=180 +journalBufferedWritesThreshold=131072 +journalMaxGroupedEntriesToCommit=200 +journalPreAllocSizeMB=4 + +# Sorted Ledger Storage Settings +sortedLedgerStorageEnabled=true +skipListSizeLimit=67108864 +skipListArenaChunkSize=2097152 +skipListArenaMaxAllocSize=131072 +fileInfoCacheInitialCapacity=10000 +fileInfoMaxIdleTime=3600 + +# Bookie Threads Settings (NOTE: change this to align the cpu cores) +numAddWorkerThreads=4 +numJournalCallbackThreads=4 +numReadWorkerThreads=4 +numLongPollWorkerThreads=4 + +# stats +statsProviderClass=org.apache.bookkeeper.stats.CodahaleMetricsServletProvider +# Exporting codahale stats +codahaleStatsHttpPort=9001 +useHostNameAsBookieID=true +allowLoopback=true diff --git a/distributedlog-service/conf/distributedlog.conf b/distributedlog-service/conf/distributedlog.conf new file mode 100644 index 0000000..f2854c8 --- /dev/null +++ b/distributedlog-service/conf/distributedlog.conf @@ -0,0 +1,127 @@ +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +######################## +# ZooKeeper Client Settings +######################## + +# zookeeper settings +zkSessionTimeoutSeconds=30 +zkNumRetries=0 +zkRetryStartBackoffMillis=100 +zkRetryMaxBackoffMillis=200 +# bkc zookeeper settings +bkcZKSessionTimeoutSeconds=60 +bkcZKNumRetries=20 +bkcZKRetryStartBackoffMillis=100 +bkcZKRetryMaxBackoffMillis=200 + +######################## +# BookKeeper Client Settings +######################## + +# bookkeeper client timeouts +bkcWriteTimeoutSeconds=10 +bkcReadTimeoutSeconds=1 +bkcNumWorkerThreads=16 +# bkcNumIOThreads=16 +bkc.numChannelsPerBookie=1 +bkc.enableTaskExecutionStats=true +bkc.connectTimeoutMillis=1000 +bkc.enablePerHostStats=true + +######################## +# DL Settings +######################## + +# lock timeout +lockTimeoutSeconds=0 +# dl worker threads +numWorkerThreads=16 + +### Recovery Related Settings + +# recover log segments in background +recoverLogSegmentsInBackground=true +# disable max id in proxy +maxIdSanityCheck=true +# use allocator pool for proxy +enableLedgerAllocatorPool=false +# ledger allocator pool size +ledgerAllocatorPoolCoreSize=20 +# check stream exists or not +createStreamIfNotExists=true +# encode dc id in version +encodeDCIDInVersion=true +# logSegmentNameVersion +logSegmentNameVersion=1 + +### Write Performance Related Settings + +# ensemble size +ensemble-size=3 +write-quorum-size=3 +ack-quorum-size=2 +bkc.ensemblePlacementPolicy=org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy +bkc.delayEnsembleChange=true + +# sync settings +# buffer size is large because when we rewrite we perform a very large write to persist +# all queue state at once (up to max queue memory size, ex. 16MB). the write will be +# throttled if it takes too long, which can hurt performance, so important to optimize +# for this case. +output-buffer-size=512000 +enableImmediateFlush=false +periodicFlushFrequencyMilliSeconds=6 +logFlushTimeoutSeconds=120 + +### Ledger Rolling Related Settings + +# retention policy +retention-size=0 +# rolling ledgers (disable time rolling/enable size rolling) +rolling-interval=0 + +# max logsegment bytes=2GB +# much larger than max journal size, effectively never roll and let drpc do it +maxLogSegmentBytes=2147483648 + +# rolling concurrency +logSegmentRollingConcurrency=1 +# disable sanityCheckDelete +sanityCheckDelete=false +ledgerAllocatorPoolName=drpc-alloc-pool + +### Readahead settings + +enableReadAhead=true +ReadAheadBatchSize=10 +ReadAheadMaxEntries=100 +ReadAheadWaitTime=10 + +### Rate limit + +rpsSoftWriteLimit=1 +rpsHardWriteLimit=5 +rpsHardServiceLimit=15 + +### Config + +dynamicConfigReloadIntervalSec=5 diff --git a/distributedlog-service/conf/dlogenv.sh b/distributedlog-service/conf/dlogenv.sh new file mode 100644 index 0000000..ae6c459 --- /dev/null +++ b/distributedlog-service/conf/dlogenv.sh @@ -0,0 +1,77 @@ +#!/bin/sh +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +################## +# General +################## + +# Log4j configuration file +# DLOG_LOG_CONF= + +# Extra options to be passed to the jvm +# DLOG_EXTRA_OPTS= + +# Add extra paths to the dlog classpath +# DLOG_EXTRA_CLASSPATH= + +# Configure the root logger +# DLOG_ROOT_LOGGER= + +# Configure the log dir +# DLOG_LOG_DIR= + +# Configure the log file +# DLOG_LOG_FILE= + +################# +# ZooKeeper +################# + +# Configure zookeeper root logger +# ZK_ROOT_LOGGER= + +################# +# Bookie +################# + +# Configure bookie root logger +# BK_ROOT_LOGGER= + +################# +# Write Proxy +################# + +# Configure write proxy root logger +# WP_ROOT_LOGGER= + +# write proxy configuration file +# WP_CONF_FILE=${DL_HOME}/conf/write_proxy.conf + +# port and stats port +# WP_SERVICE_PORT=4181 +# WP_STATS_PORT=9000 + +# shard id +# WP_SHARD_ID=0 + +# write proxy namespace +# WP_NAMESPACE=distributedlog://127.0.0.1:2181/messaging/distributedlog/mynamespace diff --git a/distributedlog-service/conf/log4j.properties b/distributedlog-service/conf/log4j.properties new file mode 100644 index 0000000..cafc888 --- /dev/null +++ b/distributedlog-service/conf/log4j.properties @@ -0,0 +1,62 @@ +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# +# DistributedLog Logging Configuration +# + +# Default values +dlog.root.logger=INFO, R +dlog.log.dir=logs +dlog.log.file=dlog.log + +log4j.rootLogger=${dlog.root.logger} +log4j.logger.org.apache.zookeeper=INFO +log4j.logger.org.apache.bookkeeper=INFO + +# redirect executor output to executors.log since slow op warnings can be quite verbose +log4j.logger.com.twitter.distributedlog.util.MonitoredFuturePool=INFO, Executors +log4j.logger.com.twitter.distributedlog.util.MonitoredScheduledThreadPoolExecutor=INFO, Executors +log4j.logger.org.apache.bookkeeper.util.SafeRunnable=INFO, Executors +log4j.additivity.com.twitter.distributedlog.util.MonitoredFuturePool=false +log4j.additivity.com.twitter.distributedlog.util.MonitoredScheduledThreadPoolExecutor=false +log4j.additivity.org.apache.bookkeeper.util.SafeRunnable=false + +log4j.appender.Executors=org.apache.log4j.RollingFileAppender +log4j.appender.Executors.Threshold=INFO +log4j.appender.Executors.File=${dlog.log.dir}/executors.log +log4j.appender.Executors.MaxFileSize=20MB +log4j.appender.Executors.MaxBackupIndex=5 +log4j.appender.Executors.layout=org.apache.log4j.PatternLayout +log4j.appender.Executors.layout.ConversionPattern=%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m%n + +log4j.appender.R=org.apache.log4j.RollingFileAppender +log4j.appender.R.Threshold=INFO +log4j.appender.R.File=${dlog.log.dir}/${dlog.log.file} +log4j.appender.R.MaxFileSize=20MB +log4j.appender.R.MaxBackupIndex=50 +log4j.appender.R.layout=org.apache.log4j.PatternLayout +log4j.appender.R.layout.ConversionPattern=%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m%n + +log4j.appender.stderr=org.apache.log4j.ConsoleAppender +log4j.appender.stderr.Target=System.err +log4j.appender.stderr.Threshold=INFO +log4j.appender.stderr.layout=org.apache.log4j.PatternLayout +log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m%n diff --git a/distributedlog-service/conf/write_proxy.conf b/distributedlog-service/conf/write_proxy.conf new file mode 100644 index 0000000..271f177 --- /dev/null +++ b/distributedlog-service/conf/write_proxy.conf @@ -0,0 +1,145 @@ +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +######################## +# ZooKeeper Client Settings +######################## + +# zookeeper settings +zkSessionTimeoutSeconds=1 +zkNumRetries=0 +zkRetryStartBackoffMillis=100 +zkRetryMaxBackoffMillis=200 +# bkc zookeeper settings +bkcZKSessionTimeoutSeconds=60 +bkcZKNumRetries=20 +bkcZKRetryStartBackoffMillis=100 +bkcZKRetryMaxBackoffMillis=200 + +######################## +# BookKeeper Client Settings +######################## + +# bookkeeper client timeouts +bkcWriteTimeoutSeconds=2 +bkcReadTimeoutSeconds=2 +bkcNumWorkerThreads=32 +bkc.numChannelsPerBookie=1 +bkc.enableTaskExecutionStats=true +bkc.connectTimeoutMillis=200 +bkc.enableParallelRecoveryRead=true +bkc.recoveryReadBatchSize=5 +bkc.enablePerHostStats=true + +######################## +# DL Settings +######################## + +# Metadata Settings + +# ledger metadata version that supports sequence id +ledger-metadata-layout=5 + +# lock timeout +lockTimeoutSeconds=0 +# dl worker threads +numWorkerThreads=32 + +### Recovery Related Settings + +# recover log segments in background +recoverLogSegmentsInBackground=false +# disable max id in proxy +maxIdSanityCheck=false +# use allocator pool for proxy +enableLedgerAllocatorPool=true +# ledger allocator pool path +ledgerAllocatorPoolPath=.write_proxy_allocation_pool +# ledger allocator pool size +ledgerAllocatorPoolCoreSize=40 +# check stream exists or not +createStreamIfNotExists=true +# encode dc id in version +encodeDCIDInVersion=true +# logSegmentNameVersion +logSegmentNameVersion=1 + +### Write Performance Related Settings + +# ensemble size +ensemble-size=3 +write-quorum-size=3 +ack-quorum-size=2 +bkc.ensemblePlacementPolicy=org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy +bkc.delayEnsembleChange=true +bkc.writeRequestToChannelAsync=true + +# enable immediate flush +enableImmediateFlush=true +# 0k output buffer +output-buffer-size=0 +# disable periodical flush +periodicFlushFrequencyMilliSeconds=0 +enableTaskExecutionStats=true +taskExecutionWarnTimeMicros=100000 + +### Ledger Rolling Related Settings + +# retention policy +retention-size=4 +# rolling ledgers (enable time rolling): 120 minutes = 2 hours +rolling-interval=120 +# max logsegment bytes : 2GB +maxLogSegmentBytes=2147483648 +# rolling concurrency +logSegmentRollingConcurrency=1 +# disable sanityCheckDelete +sanityCheckDelete=false +# compression codec +compressionType=lz4 + +### Per Stream Stats +enablePerStreamStat=true + +######################## +# DL Settings +######################## + +# proxy server settings +server_mode=DURABLE +serviceTimeoutMs=60000 +streamProbationTimeoutMs=120000 +server_threads=16 +server_dlsn_version=1 +server_enable_perstream_stat=true +server_graceful_shutdown_period_ms=35000 + +# write limits +perWriterOutstandingWriteLimit=-1 +globalOutstandingWriteLimit=-1 +outstandingWriteLimitDarkmode=false + +# bytes per second limit applied at the host level (50MBps on 1Gib machines) +bpsHardServiceLimit=52428800 +# bytes per second limit after which no new streams may be acquired (65MBps on 1Gib machines) +bpsStreamAcquireServiceLimit=47185920 + +# limit the maximum number of streams +maxAcquiredPartitionsPerProxy=-1 diff --git a/distributedlog-service/conf/zookeeper.conf.dynamic.template b/distributedlog-service/conf/zookeeper.conf.dynamic.template new file mode 100644 index 0000000..4bda9f1 --- /dev/null +++ b/distributedlog-service/conf/zookeeper.conf.dynamic.template @@ -0,0 +1 @@ +#/**# * Copyright 2007 The Apache Software Foundation# *# * Licensed to the Apache Software Foundation (ASF) under one# * or more contributor license agreements. See the NOTICE file# * distributed with this work for additional information# * regarding copyright ownership. The ASF licenses this file# * to you under the Apache License, Version 2.0 (the# * "License"); you may not use this file except in compliance# * with the License. You may obtain a copy of the License at# *# * http://www.apache.org/licenses/LICENSE-2.0# *# * Unless required by applicable law or agreed to in writing, software# * distributed under the License is distributed on an "AS IS" BASIS,# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# * See the License for the specific language governing permissions and# * limitations under the License.# */server.1=127.0.0.1:2710:3710:participant;0.0.0.0:2181 \ No newline at end of file diff --git a/distributedlog-service/conf/zookeeper.conf.template b/distributedlog-service/conf/zookeeper.conf.template new file mode 100644 index 0000000..3c0546e --- /dev/null +++ b/distributedlog-service/conf/zookeeper.conf.template @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The number of milliseconds of each tick +tickTime=2000 + +# the port at which the clients will connect +clientPort=2181 + +# The number of ticks that the initial +# synchronization phase can take +initLimit=10 + +# The number of ticks that can pass between +# sending a request and getting an acknowledgement +syncLimit=30 + +# the directory where the snapshot is stored. +dataDir=/tmp/data/zookeeper + +# where txlog are written +dataLogDir=/tmp/data/zookeeper/txlog + +# the port at which the admin will listen +adminPort=9990 +zookeeper.admin.enableServer=true + +# limit on queued clients - default: 1000 +globalOutstandingLimit=1000 + +# number of transactions before snapshots are taken - default: 100000 +snapCount=100000 + +# max # of clients - 0==unlimited +maxClientCnxns=25 + +# Election implementation to use. A value of "0" corresponds to the original +# UDP-based version, "1" corresponds to the non-authenticated UDP-based +# version of fast leader election, "2" corresponds to the authenticated +# UDP-based version of fast leader election, and "3" corresponds to TCP-based +# version of fast leader election. Currently, only 0 and 3 are supported, +# 3 being the default +electionAlg=3 + +# Leader accepts client connections. Default value is "yes". The leader +# machine coordinates updates. For higher update throughput at thes slight +# expense of read throughput the leader can be configured to not accept +# clients and focus on coordination. +leaderServes=yes + +# Skips ACL checks. This results in a boost in throughput, but opens up full +# access to the data tree to everyone. +skipACL=no + +# Purge txn logs every hour. Before 3.4.x this was done with an external cron +# job, now we can do it internally. +autopurge.purgeInterval=1 + +# Prior to version 3.4 ZooKeeper has always used NIO directly, however in +# versions 3.4 and later Netty is supported as an option to NIO (replaces). +# serverCnxnFactory=org.apache.zookeeper.server.NIOServerCnxnFactory + +standaloneEnabled=false +# ZooKeeper Dynamic Reconfiguration +# See: https://zookeeper.apache.org/doc/trunk/zookeeperReconfig.html +# +# standaloneEnabled=false +# dynamicConfigFile=/path/to/zoo.cfg.dynamic +# +server.1=127.0.0.1:2710:3710:participant;0.0.0.0:2181 http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/e9c429ed/distributedlog-service/conf/bookie.conf.template ---------------------------------------------------------------------- diff --git a/distributedlog-service/conf/bookie.conf.template b/distributedlog-service/conf/bookie.conf.template new file mode 100644 index 0000000..3dacb94 --- /dev/null +++ b/distributedlog-service/conf/bookie.conf.template @@ -0,0 +1,185 @@ +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +## Bookie settings + +# Port that bookie server listen on +bookiePort=3181 + +# TODO: change the journal directory +# Directory Bookkeeper outputs its write ahead log +journalDirectory=/tmp/data/bk/journal + +# TODO: change the ledgers directory +# Directory Bookkeeper outputs ledger snapshots +ledgerDirectories=/tmp/data/bk/ledgers + +# TODO: change the index directory +# Directory in which index files will be stored. +indexDirectories=/tmp/data/bk/ledgers + +# Ledger Manager Class +# What kind of ledger manager is used to manage how ledgers are stored, managed +# and garbage collected. Try to read 'BookKeeper Internals' for detail info. +ledgerManagerType=hierarchical + +# Root zookeeper path to store ledger metadata +# This parameter is used by zookeeper-based ledger manager as a root znode to +# store all ledgers. +zkLedgersRootPath=/messaging/bookkeeper/ledgers + +# Max file size of entry logger, in bytes +# A new entry log file will be created when the old one reaches the file size limitation +logSizeLimit=1073741823 + +# Max file size of journal file, in mega bytes +# A new journal file will be created when the old one reaches the file size limitation +# +journalMaxSizeMB=2048 + +# Max number of old journal file to kept +# Keep a number of old journal files would help data recovery in specia case +# +journalMaxBackups=5 + +# How long the interval to trigger next garbage collection, in milliseconds +# Since garbage collection is running in background, too frequent gc +# will heart performance. It is better to give a higher number of gc +# interval if there is enough disk capacity. +# gc per 1 hour (aligning with most DL rolling interval) +gcInitialWaitTime=600000 +gcWaitTime=3600000 +# do minor compaction per 2 hours +minorCompactionInterval=7200 +minorCompactionThreshold=0.2 +# disable major compaction +majorCompactionInterval=0 +# reduce major compaction threshold to a low value to prevent bad force compaction behavior +majorCompactionThreshold=0.3 +# Compaction Rate & Max Outstanding +compactionRate=10737418 +compactionMaxOutstandingRequests=10737418 + +# How long the interval to flush ledger index pages to disk, in milliseconds +# Flushing index files will introduce much random disk I/O. +# If separating journal dir and ledger dirs each on different devices, +# flushing would not affect performance. But if putting journal dir +# and ledger dirs on same device, performance degrade significantly +# on too frequent flushing. You can consider increment flush interval +# to get better performance, but you need to pay more time on bookie +# server restart after failure. +# +flushInterval=1000 + +# Interval to watch whether bookie is dead or not, in milliseconds +# +# bookieDeathWatchInterval=1000 + +## zookeeper client settings + +# A list of one of more servers on which zookeeper is running. +# The server list can be comma separated values, for example: +# zkServers=zk1:2181,zk2:2181,zk3:2181 +zkServers=localhost:2181 + +# ZooKeeper client session timeout in milliseconds +# Bookie server will exit if it received SESSION_EXPIRED because it +# was partitioned off from ZooKeeper for more than the session timeout +# JVM garbage collection, disk I/O will cause SESSION_EXPIRED. +# Increment this value could help avoiding this issue +zkTimeout=30000 + +## NIO Server settings + +# This settings is used to enabled/disabled Nagle's algorithm, which is a means of +# improving the efficiency of TCP/IP networks by reducing the number of packets +# that need to be sent over the network. +# If you are sending many small messages, such that more than one can fit in +# a single IP packet, setting server.tcpnodelay to false to enable Nagle algorithm +# can provide better performance. +# Default value is true. +# +serverTcpNoDelay=true + +## ledger cache settings + +# Max number of ledger index files could be opened in bookie server +# If number of ledger index files reaches this limitation, bookie +# server started to swap some ledgers from memory to disk. +# Too frequent swap will affect performance. You can tune this number +# to gain performance according your requirements. +openFileLimit=20000 + +# Size of a index page in ledger cache, in bytes +# A larger index page can improve performance writing page to disk, +# which is efficent when you have small number of ledgers and these +# ledgers have similar number of entries. +# If you have large number of ledgers and each ledger has fewer entries, +# smaller index page would improve memory usage. +pageSize=8192 + +# How many index pages provided in ledger cache +# If number of index pages reaches this limitation, bookie server +# starts to swap some ledgers from memory to disk. You can increment +# this value when you found swap became more frequent. But make sure +# pageLimit*pageSize should not more than JVM max memory limitation, +# otherwise you would got OutOfMemoryException. +# In general, incrementing pageLimit, using smaller index page would +# gain bettern performance in lager number of ledgers with fewer entries case +# If pageLimit is -1, bookie server will use 1/3 of JVM memory to compute +# the limitation of number of index pages. +pageLimit=131072 + +#If all ledger directories configured are full, then support only read requests for clients. +#If "readOnlyModeEnabled=true" then on all ledger disks full, bookie will be converted +#to read-only mode and serve only read requests. Otherwise the bookie will be shutdown. +readOnlyModeEnabled=true + +# Bookie Journal Settings +writeBufferSizeBytes=262144 +journalFlushWhenQueueEmpty=false +journalRemoveFromPageCache=true +journalAdaptiveGroupWrites=true +journalMaxGroupWaitMSec=4 +journalBufferedEntriesThreshold=180 +journalBufferedWritesThreshold=131072 +journalMaxGroupedEntriesToCommit=200 +journalPreAllocSizeMB=4 + +# Sorted Ledger Storage Settings +sortedLedgerStorageEnabled=true +skipListSizeLimit=67108864 +skipListArenaChunkSize=2097152 +skipListArenaMaxAllocSize=131072 +fileInfoCacheInitialCapacity=10000 +fileInfoMaxIdleTime=3600 + +# Bookie Threads Settings (NOTE: change this to align the cpu cores) +numAddWorkerThreads=4 +numJournalCallbackThreads=4 +numReadWorkerThreads=4 +numLongPollWorkerThreads=4 + +# stats +statsProviderClass=org.apache.bookkeeper.stats.CodahaleMetricsServletProvider +# Exporting codahale stats +codahaleStatsHttpPort=9001 +useHostNameAsBookieID=true +allowLoopback=true http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/e9c429ed/distributedlog-service/conf/distributedlog.conf ---------------------------------------------------------------------- diff --git a/distributedlog-service/conf/distributedlog.conf b/distributedlog-service/conf/distributedlog.conf new file mode 100644 index 0000000..f2854c8 --- /dev/null +++ b/distributedlog-service/conf/distributedlog.conf @@ -0,0 +1,127 @@ +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +######################## +# ZooKeeper Client Settings +######################## + +# zookeeper settings +zkSessionTimeoutSeconds=30 +zkNumRetries=0 +zkRetryStartBackoffMillis=100 +zkRetryMaxBackoffMillis=200 +# bkc zookeeper settings +bkcZKSessionTimeoutSeconds=60 +bkcZKNumRetries=20 +bkcZKRetryStartBackoffMillis=100 +bkcZKRetryMaxBackoffMillis=200 + +######################## +# BookKeeper Client Settings +######################## + +# bookkeeper client timeouts +bkcWriteTimeoutSeconds=10 +bkcReadTimeoutSeconds=1 +bkcNumWorkerThreads=16 +# bkcNumIOThreads=16 +bkc.numChannelsPerBookie=1 +bkc.enableTaskExecutionStats=true +bkc.connectTimeoutMillis=1000 +bkc.enablePerHostStats=true + +######################## +# DL Settings +######################## + +# lock timeout +lockTimeoutSeconds=0 +# dl worker threads +numWorkerThreads=16 + +### Recovery Related Settings + +# recover log segments in background +recoverLogSegmentsInBackground=true +# disable max id in proxy +maxIdSanityCheck=true +# use allocator pool for proxy +enableLedgerAllocatorPool=false +# ledger allocator pool size +ledgerAllocatorPoolCoreSize=20 +# check stream exists or not +createStreamIfNotExists=true +# encode dc id in version +encodeDCIDInVersion=true +# logSegmentNameVersion +logSegmentNameVersion=1 + +### Write Performance Related Settings + +# ensemble size +ensemble-size=3 +write-quorum-size=3 +ack-quorum-size=2 +bkc.ensemblePlacementPolicy=org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy +bkc.delayEnsembleChange=true + +# sync settings +# buffer size is large because when we rewrite we perform a very large write to persist +# all queue state at once (up to max queue memory size, ex. 16MB). the write will be +# throttled if it takes too long, which can hurt performance, so important to optimize +# for this case. +output-buffer-size=512000 +enableImmediateFlush=false +periodicFlushFrequencyMilliSeconds=6 +logFlushTimeoutSeconds=120 + +### Ledger Rolling Related Settings + +# retention policy +retention-size=0 +# rolling ledgers (disable time rolling/enable size rolling) +rolling-interval=0 + +# max logsegment bytes=2GB +# much larger than max journal size, effectively never roll and let drpc do it +maxLogSegmentBytes=2147483648 + +# rolling concurrency +logSegmentRollingConcurrency=1 +# disable sanityCheckDelete +sanityCheckDelete=false +ledgerAllocatorPoolName=drpc-alloc-pool + +### Readahead settings + +enableReadAhead=true +ReadAheadBatchSize=10 +ReadAheadMaxEntries=100 +ReadAheadWaitTime=10 + +### Rate limit + +rpsSoftWriteLimit=1 +rpsHardWriteLimit=5 +rpsHardServiceLimit=15 + +### Config + +dynamicConfigReloadIntervalSec=5 http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/e9c429ed/distributedlog-service/conf/dlogenv.sh ---------------------------------------------------------------------- diff --git a/distributedlog-service/conf/dlogenv.sh b/distributedlog-service/conf/dlogenv.sh new file mode 100644 index 0000000..ae6c459 --- /dev/null +++ b/distributedlog-service/conf/dlogenv.sh @@ -0,0 +1,77 @@ +#!/bin/sh +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +################## +# General +################## + +# Log4j configuration file +# DLOG_LOG_CONF= + +# Extra options to be passed to the jvm +# DLOG_EXTRA_OPTS= + +# Add extra paths to the dlog classpath +# DLOG_EXTRA_CLASSPATH= + +# Configure the root logger +# DLOG_ROOT_LOGGER= + +# Configure the log dir +# DLOG_LOG_DIR= + +# Configure the log file +# DLOG_LOG_FILE= + +################# +# ZooKeeper +################# + +# Configure zookeeper root logger +# ZK_ROOT_LOGGER= + +################# +# Bookie +################# + +# Configure bookie root logger +# BK_ROOT_LOGGER= + +################# +# Write Proxy +################# + +# Configure write proxy root logger +# WP_ROOT_LOGGER= + +# write proxy configuration file +# WP_CONF_FILE=${DL_HOME}/conf/write_proxy.conf + +# port and stats port +# WP_SERVICE_PORT=4181 +# WP_STATS_PORT=9000 + +# shard id +# WP_SHARD_ID=0 + +# write proxy namespace +# WP_NAMESPACE=distributedlog://127.0.0.1:2181/messaging/distributedlog/mynamespace http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/e9c429ed/distributedlog-service/conf/log4j.properties ---------------------------------------------------------------------- diff --git a/distributedlog-service/conf/log4j.properties b/distributedlog-service/conf/log4j.properties new file mode 100644 index 0000000..cafc888 --- /dev/null +++ b/distributedlog-service/conf/log4j.properties @@ -0,0 +1,62 @@ +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# +# DistributedLog Logging Configuration +# + +# Default values +dlog.root.logger=INFO, R +dlog.log.dir=logs +dlog.log.file=dlog.log + +log4j.rootLogger=${dlog.root.logger} +log4j.logger.org.apache.zookeeper=INFO +log4j.logger.org.apache.bookkeeper=INFO + +# redirect executor output to executors.log since slow op warnings can be quite verbose +log4j.logger.com.twitter.distributedlog.util.MonitoredFuturePool=INFO, Executors +log4j.logger.com.twitter.distributedlog.util.MonitoredScheduledThreadPoolExecutor=INFO, Executors +log4j.logger.org.apache.bookkeeper.util.SafeRunnable=INFO, Executors +log4j.additivity.com.twitter.distributedlog.util.MonitoredFuturePool=false +log4j.additivity.com.twitter.distributedlog.util.MonitoredScheduledThreadPoolExecutor=false +log4j.additivity.org.apache.bookkeeper.util.SafeRunnable=false + +log4j.appender.Executors=org.apache.log4j.RollingFileAppender +log4j.appender.Executors.Threshold=INFO +log4j.appender.Executors.File=${dlog.log.dir}/executors.log +log4j.appender.Executors.MaxFileSize=20MB +log4j.appender.Executors.MaxBackupIndex=5 +log4j.appender.Executors.layout=org.apache.log4j.PatternLayout +log4j.appender.Executors.layout.ConversionPattern=%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m%n + +log4j.appender.R=org.apache.log4j.RollingFileAppender +log4j.appender.R.Threshold=INFO +log4j.appender.R.File=${dlog.log.dir}/${dlog.log.file} +log4j.appender.R.MaxFileSize=20MB +log4j.appender.R.MaxBackupIndex=50 +log4j.appender.R.layout=org.apache.log4j.PatternLayout +log4j.appender.R.layout.ConversionPattern=%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m%n + +log4j.appender.stderr=org.apache.log4j.ConsoleAppender +log4j.appender.stderr.Target=System.err +log4j.appender.stderr.Threshold=INFO +log4j.appender.stderr.layout=org.apache.log4j.PatternLayout +log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m%n http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/e9c429ed/distributedlog-service/conf/write_proxy.conf ---------------------------------------------------------------------- diff --git a/distributedlog-service/conf/write_proxy.conf b/distributedlog-service/conf/write_proxy.conf new file mode 100644 index 0000000..271f177 --- /dev/null +++ b/distributedlog-service/conf/write_proxy.conf @@ -0,0 +1,145 @@ +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +######################## +# ZooKeeper Client Settings +######################## + +# zookeeper settings +zkSessionTimeoutSeconds=1 +zkNumRetries=0 +zkRetryStartBackoffMillis=100 +zkRetryMaxBackoffMillis=200 +# bkc zookeeper settings +bkcZKSessionTimeoutSeconds=60 +bkcZKNumRetries=20 +bkcZKRetryStartBackoffMillis=100 +bkcZKRetryMaxBackoffMillis=200 + +######################## +# BookKeeper Client Settings +######################## + +# bookkeeper client timeouts +bkcWriteTimeoutSeconds=2 +bkcReadTimeoutSeconds=2 +bkcNumWorkerThreads=32 +bkc.numChannelsPerBookie=1 +bkc.enableTaskExecutionStats=true +bkc.connectTimeoutMillis=200 +bkc.enableParallelRecoveryRead=true +bkc.recoveryReadBatchSize=5 +bkc.enablePerHostStats=true + +######################## +# DL Settings +######################## + +# Metadata Settings + +# ledger metadata version that supports sequence id +ledger-metadata-layout=5 + +# lock timeout +lockTimeoutSeconds=0 +# dl worker threads +numWorkerThreads=32 + +### Recovery Related Settings + +# recover log segments in background +recoverLogSegmentsInBackground=false +# disable max id in proxy +maxIdSanityCheck=false +# use allocator pool for proxy +enableLedgerAllocatorPool=true +# ledger allocator pool path +ledgerAllocatorPoolPath=.write_proxy_allocation_pool +# ledger allocator pool size +ledgerAllocatorPoolCoreSize=40 +# check stream exists or not +createStreamIfNotExists=true +# encode dc id in version +encodeDCIDInVersion=true +# logSegmentNameVersion +logSegmentNameVersion=1 + +### Write Performance Related Settings + +# ensemble size +ensemble-size=3 +write-quorum-size=3 +ack-quorum-size=2 +bkc.ensemblePlacementPolicy=org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy +bkc.delayEnsembleChange=true +bkc.writeRequestToChannelAsync=true + +# enable immediate flush +enableImmediateFlush=true +# 0k output buffer +output-buffer-size=0 +# disable periodical flush +periodicFlushFrequencyMilliSeconds=0 +enableTaskExecutionStats=true +taskExecutionWarnTimeMicros=100000 + +### Ledger Rolling Related Settings + +# retention policy +retention-size=4 +# rolling ledgers (enable time rolling): 120 minutes = 2 hours +rolling-interval=120 +# max logsegment bytes : 2GB +maxLogSegmentBytes=2147483648 +# rolling concurrency +logSegmentRollingConcurrency=1 +# disable sanityCheckDelete +sanityCheckDelete=false +# compression codec +compressionType=lz4 + +### Per Stream Stats +enablePerStreamStat=true + +######################## +# DL Settings +######################## + +# proxy server settings +server_mode=DURABLE +serviceTimeoutMs=60000 +streamProbationTimeoutMs=120000 +server_threads=16 +server_dlsn_version=1 +server_enable_perstream_stat=true +server_graceful_shutdown_period_ms=35000 + +# write limits +perWriterOutstandingWriteLimit=-1 +globalOutstandingWriteLimit=-1 +outstandingWriteLimitDarkmode=false + +# bytes per second limit applied at the host level (50MBps on 1Gib machines) +bpsHardServiceLimit=52428800 +# bytes per second limit after which no new streams may be acquired (65MBps on 1Gib machines) +bpsStreamAcquireServiceLimit=47185920 + +# limit the maximum number of streams +maxAcquiredPartitionsPerProxy=-1 http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/e9c429ed/distributedlog-service/conf/zookeeper.conf.dynamic.template ---------------------------------------------------------------------- diff --git a/distributedlog-service/conf/zookeeper.conf.dynamic.template b/distributedlog-service/conf/zookeeper.conf.dynamic.template new file mode 100644 index 0000000..4bda9f1 --- /dev/null +++ b/distributedlog-service/conf/zookeeper.conf.dynamic.template @@ -0,0 +1 @@ +#/**# * Copyright 2007 The Apache Software Foundation# *# * Licensed to the Apache Software Foundation (ASF) under one# * or more contributor license agreements. See the NOTICE file# * distributed with this work for additional information# * regarding copyright ownership. The ASF licenses this file# * to you under the Apache License, Version 2.0 (the# * "License"); you may not use this file except in compliance# * with the License. You may obtain a copy of the License at# *# * http://www.apache.org/licenses/LICENSE-2.0# *# * Unless required by applicable law or agreed to in writing, software# * distributed under the License is distributed on an "AS IS" BASIS,# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# * See the License for the specific language governing permissions and# * limitations under the License.# */server.1=127.0.0.1:2710:3710:participant;0.0.0.0:2181 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/e9c429ed/distributedlog-service/conf/zookeeper.conf.template ---------------------------------------------------------------------- diff --git a/distributedlog-service/conf/zookeeper.conf.template b/distributedlog-service/conf/zookeeper.conf.template new file mode 100644 index 0000000..3c0546e --- /dev/null +++ b/distributedlog-service/conf/zookeeper.conf.template @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The number of milliseconds of each tick +tickTime=2000 + +# the port at which the clients will connect +clientPort=2181 + +# The number of ticks that the initial +# synchronization phase can take +initLimit=10 + +# The number of ticks that can pass between +# sending a request and getting an acknowledgement +syncLimit=30 + +# the directory where the snapshot is stored. +dataDir=/tmp/data/zookeeper + +# where txlog are written +dataLogDir=/tmp/data/zookeeper/txlog + +# the port at which the admin will listen +adminPort=9990 +zookeeper.admin.enableServer=true + +# limit on queued clients - default: 1000 +globalOutstandingLimit=1000 + +# number of transactions before snapshots are taken - default: 100000 +snapCount=100000 + +# max # of clients - 0==unlimited +maxClientCnxns=25 + +# Election implementation to use. A value of "0" corresponds to the original +# UDP-based version, "1" corresponds to the non-authenticated UDP-based +# version of fast leader election, "2" corresponds to the authenticated +# UDP-based version of fast leader election, and "3" corresponds to TCP-based +# version of fast leader election. Currently, only 0 and 3 are supported, +# 3 being the default +electionAlg=3 + +# Leader accepts client connections. Default value is "yes". The leader +# machine coordinates updates. For higher update throughput at thes slight +# expense of read throughput the leader can be configured to not accept +# clients and focus on coordination. +leaderServes=yes + +# Skips ACL checks. This results in a boost in throughput, but opens up full +# access to the data tree to everyone. +skipACL=no + +# Purge txn logs every hour. Before 3.4.x this was done with an external cron +# job, now we can do it internally. +autopurge.purgeInterval=1 + +# Prior to version 3.4 ZooKeeper has always used NIO directly, however in +# versions 3.4 and later Netty is supported as an option to NIO (replaces). +# serverCnxnFactory=org.apache.zookeeper.server.NIOServerCnxnFactory + +standaloneEnabled=false +# ZooKeeper Dynamic Reconfiguration +# See: https://zookeeper.apache.org/doc/trunk/zookeeperReconfig.html +# +# standaloneEnabled=false +# dynamicConfigFile=/path/to/zoo.cfg.dynamic +# +server.1=127.0.0.1:2710:3710:participant;0.0.0.0:2181