Repository: incubator-htrace Updated Branches: refs/heads/master 94bc87844 -> 3b10cfb53
HTRACE-309. htraced: improve leveldb configuration (Colin Patrick McCabe via iwasakims) Project: http://git-wip-us.apache.org/repos/asf/incubator-htrace/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-htrace/commit/3b10cfb5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-htrace/tree/3b10cfb5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-htrace/diff/3b10cfb5 Branch: refs/heads/master Commit: 3b10cfb534724eba81f6ef17657d32c42cf9a3cc Parents: 94bc878 Author: Masatake Iwasaki <[email protected]> Authored: Sat Nov 28 09:54:06 2015 +0900 Committer: Masatake Iwasaki <[email protected]> Committed: Sat Nov 28 09:54:06 2015 +0900 ---------------------------------------------------------------------- .../src/org/apache/htrace/conf/config_keys.go | 5 ++ .../src/org/apache/htrace/htraced/datastore.go | 60 +++++++++++++++++++- 2 files changed, 63 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/3b10cfb5/htrace-htraced/go/src/org/apache/htrace/conf/config_keys.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/conf/config_keys.go b/htrace-htraced/go/src/org/apache/htrace/conf/config_keys.go index 10204cd..6f6e8cf 100644 --- a/htrace-htraced/go/src/org/apache/htrace/conf/config_keys.go +++ b/htrace-htraced/go/src/org/apache/htrace/conf/config_keys.go @@ -96,6 +96,10 @@ const HTRACE_NUM_HRPC_HANDLERS = "num.hrpc.handlers" // this to read or write a message, we will abort the connection. const HTRACE_HRPC_IO_TIMEOUT_MS = "hrpc.io.timeout.ms" +// The leveldb write buffer size, or 0 to use the library default, which is 4 +// MB in leveldb 1.16. See leveldb's options.h for more details. +const HTRACE_LEVELDB_WRITE_BUFFER_SIZE = "leveldb.write.buffer.size" + // Default values for HTrace configuration keys. var DEFAULTS = map[string]string{ HTRACE_WEB_ADDRESS: fmt.Sprintf("0.0.0.0:%d", HTRACE_WEB_ADDRESS_DEFAULT_PORT), @@ -112,6 +116,7 @@ var DEFAULTS = map[string]string{ HTRACE_REAPER_HEARTBEAT_PERIOD_MS: fmt.Sprintf("%d", 90*1000), HTRACE_NUM_HRPC_HANDLERS: "20", HTRACE_HRPC_IO_TIMEOUT_MS: "60000", + HTRACE_LEVELDB_WRITE_BUFFER_SIZE: "0", } // Values to be used when creating test configurations http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/3b10cfb5/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go ---------------------------------------------------------------------- diff --git a/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go b/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go index 2a3d65c..828a6af 100644 --- a/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go +++ b/htrace-htraced/go/src/org/apache/htrace/htraced/datastore.go @@ -27,6 +27,7 @@ import ( "fmt" "github.com/jmhodges/levigo" "github.com/ugorji/go/codec" + "math" "org/apache/htrace/common" "org/apache/htrace/conf" "os" @@ -34,6 +35,7 @@ import ( "strings" "sync" "sync/atomic" + "syscall" "time" ) @@ -460,6 +462,9 @@ type dataStore struct { // When this datastore was started (in UTC milliseconds since the epoch) startMs int64 + + // The maximum number of open files to allow per shard. + maxFdPerShard int } func CreateDataStore(cnf *conf.Config, writtenSpans *common.Semaphore) (*dataStore, error) { @@ -482,6 +487,7 @@ func CreateDataStore(cnf *conf.Config, writtenSpans *common.Semaphore) (*dataSto store.readOpts = levigo.NewReadOptions() store.readOpts.SetFillCache(true) + store.readOpts.SetVerifyChecksums(false) store.writeOpts = levigo.NewWriteOptions() store.writeOpts.SetSync(false) @@ -514,9 +520,49 @@ func CreateDataStore(cnf *conf.Config, writtenSpans *common.Semaphore) (*dataSto }) } store.startMs = common.TimeToUnixMs(time.Now().UTC()) + err = store.calculateMaxOpenFilesPerShard() + if err != nil { + lg.Warnf("Unable to calculate maximum open files per shard: %s\n", + err.Error()) + } return store, nil } +// The maximum number of file descriptors we'll use on non-datastore things. +const NON_DATASTORE_FD_MAX = 300 + +// The minimum number of file descriptors per shard we will set. Setting fewer +// than this number could trigger a bug in some early versions of leveldb. +const MIN_FDS_PER_SHARD = 80 + +func (store *dataStore) calculateMaxOpenFilesPerShard() error { + var rlim syscall.Rlimit + err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlim) + if err != nil { + return err + } + // I think RLIMIT_NOFILE fits in 32 bits on all known operating systems, + // but there's no harm in being careful. 'int' in golang always holds at + // least 32 bits. + var maxFd int + if rlim.Cur > uint64(math.MaxInt32) { + maxFd = math.MaxInt32 + } else { + maxFd = int(rlim.Cur) + } + fdsPerShard := (maxFd - NON_DATASTORE_FD_MAX) / len(store.shards) + if fdsPerShard < MIN_FDS_PER_SHARD { + return errors.New(fmt.Sprintf("Expected to be able to use at least %d " + + "fds per shard, but we have %d shards and %d total fds to allocate, " + + "giving us only %d FDs per shard.", MIN_FDS_PER_SHARD, + len(store.shards), maxFd - NON_DATASTORE_FD_MAX, fdsPerShard)) + } + store.lg.Infof("maxFd = %d. Setting maxFdPerShard = %d\n", + maxFd, fdsPerShard) + store.maxFdPerShard = fdsPerShard + return nil +} + func CreateShard(store *dataStore, cnf *conf.Config, path string, clearStored bool) (*shard, error) { lg := store.lg @@ -543,6 +589,14 @@ func CreateShard(store *dataStore, cnf *conf.Config, path string, } var shd *shard openOpts := levigo.NewOptions() + openOpts.SetParanoidChecks(false) + writeBufferSize := cnf.GetInt(conf.HTRACE_LEVELDB_WRITE_BUFFER_SIZE) + if writeBufferSize > 0 { + openOpts.SetWriteBufferSize(writeBufferSize) + } + if store.maxFdPerShard > 0 { + openOpts.SetMaxOpenFiles(store.maxFdPerShard) + } defer openOpts.Close() newlyCreated := false ldb, err := levigo.Open(path, openOpts) @@ -636,8 +690,10 @@ func (store *dataStore) Close() { store.hb = nil } for idx := range store.shards { - store.shards[idx].Close() - store.shards[idx] = nil + if store.shards[idx] != nil { + store.shards[idx].Close() + store.shards[idx] = nil + } } if store.rpr != nil { store.rpr.Shutdown()
