commit missing changes on s2core.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/da668ace Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/da668ace Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/da668ace Branch: refs/heads/master Commit: da668ace749adf243d43c95413b460bba218731d Parents: f90a7f5 Author: DO YUNG YOON <[email protected]> Authored: Fri Jun 15 10:09:53 2018 +0900 Committer: DO YUNG YOON <[email protected]> Committed: Fri Jun 15 10:12:12 2018 +0900 ---------------------------------------------------------------------- .../scala/org/apache/s2graph/core/S2Graph.scala | 44 +++------------ .../apache/s2graph/core/S2GraphConfigs.scala | 57 ++++++++++++++++++++ 2 files changed, 63 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/da668ace/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala index dabbf80..95880f7 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/S2Graph.scala @@ -51,40 +51,8 @@ object S2Graph { val DefaultScore = 1.0 val FetchAllLimit = 10000000 val DefaultFetchLimit = 1000 - import S2GraphConfigs._ - - private val DefaultConfigs: Map[String, AnyRef] = Map( - S2GRAPH_STORE_BACKEND -> DEFAULT_S2GRAPH_STORE_BACKEND, - PHASE -> DEFAULT_PHASE, - HBaseConfigs.HBASE_ZOOKEEPER_QUORUM -> HBaseConfigs.DEFAULT_HBASE_ZOOKEEPER_QUORUM, - HBaseConfigs.HBASE_ZOOKEEPER_ZNODE_PARENT -> HBaseConfigs.DEFAULT_HBASE_ZOOKEEPER_ZNODE_PARENT, - HBaseConfigs.HBASE_TABLE_NAME -> HBaseConfigs.DEFAULT_HBASE_TABLE_NAME, - HBaseConfigs.HBASE_TABLE_COMPRESSION_ALGORITHM -> HBaseConfigs.DEFAULT_HBASE_TABLE_COMPRESSION_ALGORITHM, - HBaseConfigs.HBASE_CLIENT_RETRIES_NUMBER -> HBaseConfigs.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER, - HBaseConfigs.HBASE_RPCS_BUFFERED_FLUSH_INTERVAL -> HBaseConfigs.DEFAULT_HBASE_RPCS_BUFFERED_FLUSH_INTERVAL, - HBaseConfigs.HBASE_RPC_TIMEOUT -> HBaseConfigs.DEFAULT_HBASE_RPC_TIMEOUT, - DBConfigs.DB_DEFAULT_DRIVER -> DBConfigs.DEFAULT_DB_DEFAULT_DRIVER, - DBConfigs.DB_DEFAULT_URL -> DBConfigs.DEFAULT_DB_DEFAULT_URL, - DBConfigs.DB_DEFAULT_PASSWORD -> DBConfigs.DEFAULT_DB_DEFAULT_PASSWORD, - DBConfigs.DB_DEFAULT_USER -> DBConfigs.DEFAULT_DB_DEFAULT_USER, - CacheConfigs.CACHE_MAX_SIZE -> CacheConfigs.DEFAULT_CACHE_MAX_SIZE, - CacheConfigs.CACHE_TTL_SECONDS -> CacheConfigs.DEFAULT_CACHE_TTL_SECONDS, - ResourceCacheConfigs.RESOURCE_CACHE_MAX_SIZE -> ResourceCacheConfigs.DEFAULT_RESOURCE_CACHE_MAX_SIZE, - ResourceCacheConfigs.RESOURCE_CACHE_TTL_SECONDS -> ResourceCacheConfigs.DEFAULT_RESOURCE_CACHE_TTL_SECONDS, - MutatorConfigs.MAX_RETRY_NUMBER -> MutatorConfigs.DEFAULT_MAX_RETRY_NUMBER, - MutatorConfigs.LOCK_EXPIRE_TIME -> MutatorConfigs.DEFAULT_LOCK_EXPIRE_TIME, - MutatorConfigs.MAX_BACK_OFF -> MutatorConfigs.DEFAULT_MAX_BACK_OFF, - MutatorConfigs.BACK_OFF_TIMEOUT -> MutatorConfigs.DEFAULT_BACK_OFF_TIMEOUT, - MutatorConfigs.HBASE_FAIL_PROB -> MutatorConfigs.DEFAULT_HBASE_FAIL_PROB, - MutatorConfigs.DELETE_ALL_FETCH_SIZE -> MutatorConfigs.DEFAULT_DELETE_ALL_FETCH_SIZE, - MutatorConfigs.DELETE_ALL_FETCH_COUNT -> MutatorConfigs.DEFAULT_DELETE_ALL_FETCH_COUNT, - FutureCacheConfigs.FUTURE_CACHE_MAX_SIZE -> FutureCacheConfigs.DEFAULT_FUTURE_CACHE_MAX_SIZE, - FutureCacheConfigs.FUTURE_CACHE_EXPIRE_AFTER_WRITE -> FutureCacheConfigs.DEFAULT_FUTURE_CACHE_EXPIRE_AFTER_WRITE, - FutureCacheConfigs.FUTURE_CACHE_EXPIRE_AFTER_ACCESS -> FutureCacheConfigs.DEFAULT_FUTURE_CACHE_EXPIRE_AFTER_ACCESS, - FutureCacheConfigs.FUTURE_CACHE_METRIC_INTERVAL -> FutureCacheConfigs.DEFAULT_FUTURE_CACHE_METRIC_INTERVAL, - QueryConfigs.QUERY_HARDLIMIT -> QueryConfigs.DEFAULT_QUERY_HARDLIMIT, - LogConfigs.QUERY_LOG_SAMPLE_RATE -> LogConfigs.DEFAULT_QUERY_LOG_SAMPLE_RATE - ) + + private val DefaultConfigs = S2GraphConfigs.DEFAULT_CONFIGS var DefaultConfig: Config = ConfigFactory.parseMap(DefaultConfigs) val numOfThread = Runtime.getRuntime.availableProcessors() @@ -128,13 +96,13 @@ object S2Graph { } def initStorage(graph: S2GraphLike, config: Config)(ec: ExecutionContext): Storage = { - val storageBackend = config.getString("s2graph.storage.backend") + val storageBackend = config.getString(S2GraphConfigs.S2GRAPH_STORE_BACKEND) logger.info(s"[InitStorage]: $storageBackend") storageBackend match { case "hbase" => hbaseExecutor = - if (config.getString("hbase.zookeeper.quorum") == "localhost") + if (config.getString(S2GraphConfigs.HBaseConfigs.HBASE_ZOOKEEPER_QUORUM) == "localhost") AsynchbaseStorage.initLocalHBase(config) else null @@ -205,8 +173,8 @@ class S2Graph(_config: Config)(implicit val ec: ExecutionContext) extends S2Grap override val config = _config.withFallback(S2Graph.DefaultConfig) val storageBackend = Try { - config.getString("s2graph.storage.backend") - }.getOrElse("hbase") + config.getString(S2GraphConfigs.S2GRAPH_STORE_BACKEND) + }.getOrElse(S2GraphConfigs.DEFAULT_S2GRAPH_STORE_BACKEND) Schema.apply(config) Schema.loadCache() http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/da668ace/s2core/src/main/scala/org/apache/s2graph/core/S2GraphConfigs.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphConfigs.scala b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphConfigs.scala index 445b9f6..f262ea3 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/S2GraphConfigs.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/S2GraphConfigs.scala @@ -1,6 +1,19 @@ package org.apache.s2graph.core object S2GraphConfigs { + lazy val DEFAULTS: Map[String, AnyRef] = Map( + S2GRAPH_STORE_BACKEND -> DEFAULT_S2GRAPH_STORE_BACKEND, + PHASE -> DEFAULT_PHASE + ) + lazy val DEFAULT_CONFIGS = DEFAULTS ++ + S2GraphConfigs.HBaseConfigs.DEFAULTS ++ + S2GraphConfigs.DBConfigs.DEFAULTS ++ + S2GraphConfigs.CacheConfigs.DEFAULTS ++ + S2GraphConfigs.ResourceCacheConfigs.DEFAULTS ++ + S2GraphConfigs.MutatorConfigs.DEFAULTS ++ + S2GraphConfigs.QueryConfigs.DEFAULTS ++ + S2GraphConfigs.FutureCacheConfigs.DEFAULTS ++ + S2GraphConfigs.LogConfigs.DEFAULTS val S2GRAPH_STORE_BACKEND = "s2graph.storage.backend" val DEFAULT_S2GRAPH_STORE_BACKEND = "hbase" @@ -9,6 +22,15 @@ object S2GraphConfigs { val DEFAULT_PHASE = "dev" object HBaseConfigs { + lazy val DEFAULTS: Map[String, AnyRef] = Map( + HBASE_ZOOKEEPER_QUORUM -> DEFAULT_HBASE_ZOOKEEPER_QUORUM, + HBASE_ZOOKEEPER_ZNODE_PARENT -> DEFAULT_HBASE_ZOOKEEPER_ZNODE_PARENT, + HBASE_TABLE_NAME -> DEFAULT_HBASE_TABLE_NAME, + HBASE_TABLE_COMPRESSION_ALGORITHM -> DEFAULT_HBASE_TABLE_COMPRESSION_ALGORITHM, + HBASE_CLIENT_RETRIES_NUMBER -> DEFAULT_HBASE_CLIENT_RETRIES_NUMBER, + HBASE_RPCS_BUFFERED_FLUSH_INTERVAL -> DEFAULT_HBASE_RPCS_BUFFERED_FLUSH_INTERVAL, + HBASE_RPC_TIMEOUT -> DEFAULT_HBASE_RPC_TIMEOUT + ) val HBASE_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum" val DEFAULT_HBASE_ZOOKEEPER_QUORUM = "localhost" @@ -31,6 +53,12 @@ object S2GraphConfigs { val DEFAULT_HBASE_RPC_TIMEOUT = java.lang.Integer.valueOf(600000) } object DBConfigs { + lazy val DEFAULTS: Map[String, AnyRef] = Map( + DB_DEFAULT_DRIVER -> DEFAULT_DB_DEFAULT_DRIVER, + DB_DEFAULT_URL -> DEFAULT_DB_DEFAULT_URL, + DB_DEFAULT_PASSWORD -> DEFAULT_DB_DEFAULT_PASSWORD, + DB_DEFAULT_USER -> DEFAULT_DB_DEFAULT_USER + ) val DB_DEFAULT_DRIVER = "db.default.driver" val DEFAULT_DB_DEFAULT_DRIVER = "org.h2.Driver" @@ -44,6 +72,10 @@ object S2GraphConfigs { val DEFAULT_DB_DEFAULT_USER = "graph" } object CacheConfigs { + lazy val DEFAULTS: Map[String, AnyRef] = Map( + CACHE_MAX_SIZE -> DEFAULT_CACHE_MAX_SIZE, + CACHE_TTL_SECONDS -> DEFAULT_CACHE_TTL_SECONDS + ) val CACHE_MAX_SIZE = "cache.max.size" val DEFAULT_CACHE_MAX_SIZE = java.lang.Integer.valueOf(0) @@ -51,6 +83,10 @@ object S2GraphConfigs { val DEFAULT_CACHE_TTL_SECONDS = java.lang.Integer.valueOf(-1) } object ResourceCacheConfigs { + lazy val DEFAULTS: Map[String, AnyRef] = Map( + RESOURCE_CACHE_MAX_SIZE -> DEFAULT_RESOURCE_CACHE_MAX_SIZE, + RESOURCE_CACHE_TTL_SECONDS -> DEFAULT_RESOURCE_CACHE_TTL_SECONDS + ) val RESOURCE_CACHE_MAX_SIZE = "resource.cache.max.size" val DEFAULT_RESOURCE_CACHE_MAX_SIZE = java.lang.Integer.valueOf(1000) @@ -58,6 +94,15 @@ object S2GraphConfigs { val DEFAULT_RESOURCE_CACHE_TTL_SECONDS = java.lang.Integer.valueOf(-1) } object MutatorConfigs { + lazy val DEFAULTS: Map[String, AnyRef] = Map( + MAX_RETRY_NUMBER -> DEFAULT_MAX_RETRY_NUMBER, + LOCK_EXPIRE_TIME -> DEFAULT_LOCK_EXPIRE_TIME, + MAX_BACK_OFF -> DEFAULT_MAX_BACK_OFF, + BACK_OFF_TIMEOUT -> DEFAULT_BACK_OFF_TIMEOUT, + HBASE_FAIL_PROB -> DEFAULT_HBASE_FAIL_PROB, + DELETE_ALL_FETCH_SIZE -> DEFAULT_DELETE_ALL_FETCH_SIZE, + DELETE_ALL_FETCH_COUNT -> DEFAULT_DELETE_ALL_FETCH_COUNT + ) val MAX_RETRY_NUMBER = "max.retry.number" val DEFAULT_MAX_RETRY_NUMBER = java.lang.Integer.valueOf(100) @@ -80,10 +125,19 @@ object S2GraphConfigs { val DEFAULT_DELETE_ALL_FETCH_COUNT = java.lang.Integer.valueOf(200) } object QueryConfigs { + lazy val DEFAULTS: Map[String, AnyRef] = Map( + QUERY_HARDLIMIT -> DEFAULT_QUERY_HARDLIMIT + ) val QUERY_HARDLIMIT = "query.hardlimit" val DEFAULT_QUERY_HARDLIMIT = java.lang.Integer.valueOf(100000) } object FutureCacheConfigs { + lazy val DEFAULTS: Map[String, AnyRef] = Map( + FUTURE_CACHE_MAX_SIZE -> DEFAULT_FUTURE_CACHE_MAX_SIZE, + FUTURE_CACHE_EXPIRE_AFTER_WRITE -> DEFAULT_FUTURE_CACHE_EXPIRE_AFTER_WRITE, + FUTURE_CACHE_EXPIRE_AFTER_ACCESS -> DEFAULT_FUTURE_CACHE_EXPIRE_AFTER_ACCESS, + FUTURE_CACHE_METRIC_INTERVAL -> DEFAULT_FUTURE_CACHE_METRIC_INTERVAL + ) val FUTURE_CACHE_MAX_SIZE = "future.cache.max.size" val DEFAULT_FUTURE_CACHE_MAX_SIZE = java.lang.Integer.valueOf(100000) @@ -97,6 +151,9 @@ object S2GraphConfigs { val DEFAULT_FUTURE_CACHE_METRIC_INTERVAL = java.lang.Integer.valueOf(60000) } object LogConfigs { + lazy val DEFAULTS: Map[String, AnyRef] = Map( + QUERY_LOG_SAMPLE_RATE -> DEFAULT_QUERY_LOG_SAMPLE_RATE + ) val QUERY_LOG_SAMPLE_RATE = "query.log.sample.rate" val DEFAULT_QUERY_LOG_SAMPLE_RATE = Double.box(0.05) }
