Repository: incubator-s2graph Updated Branches: refs/heads/master 94862407f -> 69273249a
[S2GRAPH-102]: Add more configuration optoins in application.conf. JIRA: [S2GRAPH-102] https://issues.apache.org/jira/browse/S2GRAPH-102 Pull Request: Closes #72 Authors: DO YUNG YOON: steams...@apache.org Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/69273249 Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/69273249 Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/69273249 Branch: refs/heads/master Commit: 69273249a745c51bb612e79eea5d9c380c3c3f80 Parents: 9486240 Author: DO YUNG YOON <steams...@apache.org> Authored: Thu Sep 1 05:56:29 2016 +0900 Committer: DO YUNG YOON <steams...@apache.org> Committed: Thu Sep 1 05:56:29 2016 +0900 ---------------------------------------------------------------------- CHANGES | 2 + README.md | 3 +- build.sbt | 1 + conf/application.conf | 55 ++++++++++++++++++++ dev_support/README.md | 6 +-- dev_support/docker-compose.yml | 10 ++-- project/Common.scala | 4 +- .../counter/helper/DistributedScanner.scala | 4 +- .../apache/s2graph/rest/play/Bootstrap.scala | 4 +- .../s2graph/rest/play/config/Config.scala | 2 +- 10 files changed, 78 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/69273249/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index f55b3b2..28cfbcd 100644 --- a/CHANGES +++ b/CHANGES @@ -171,6 +171,8 @@ Release 0.12.1 - unreleased S2GRAPH-103: Remove dependencies on custom fork of asynchbase (Committed by DOYUNG YOON). + S2GRAPH-102: Add more configuration optoins in application.conf (Committed by DOYUNG YOON). + TEST S2GRAPH-21: Change PostProcessBenchmarkSpec not to store and fetch test data from storage. (Committed by DOYUNG YOON). http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/69273249/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md index 82fff4c..73a683f 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,8 @@ Once built from source, the following resources are expected under `target/apach `sh bin/start-s2graph.sh` will launch the S2Graph server along with a standalone [HBase server](https://hbase.apache.org/) as the data storage and [H2](http://www.h2database.com/html/main.html) as the metastore -Finally, join the mailing list by sending a message to [users-subscr...@s2graph.incubator.apache.org](mailto:users-subscr...@s2graph.incubator.apache.org?subject=send this email to subscribe) or [dev-subscr...@s2graph.incubator.apache.org](mailto:dev-subscr...@s2graph.incubator.apache.org?subject=send this email to subscribe)! +To see how to connect remote HBase and S2Graph configurations, check out conf/application.conf. +Currently we have tested with these HBase version(0.98, 1.0, 1.1, 1.2)(https://hub.docker.com/r/harisekhon/hbase/tags/). Project Layout ============== http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/69273249/build.sbt ---------------------------------------------------------------------- diff --git a/build.sbt b/build.sbt index c315771..94be364 100755 --- a/build.sbt +++ b/build.sbt @@ -25,6 +25,7 @@ name := "s2graph" lazy val commonSettings = Seq( organization := "org.apache.s2graph", scalaVersion := "2.11.7", + version := "0.1.0", scalacOptions := Seq("-language:postfixOps", "-unchecked", "-deprecation", "-feature", "-Xlint"), javaOptions ++= collection.JavaConversions.propertiesAsScalaMap(System.getProperties).map { case (key, value) => "-D" + key + "=" + value }.toSeq, testOptions in Test += Tests.Argument("-oDF"), http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/69273249/conf/application.conf ---------------------------------------------------------------------- diff --git a/conf/application.conf b/conf/application.conf index 39b5f68..4920380 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -17,6 +17,61 @@ # under the License. # +# HBase +# place your local/remote hbase cluster's zookeeper quorum, +# as default, s2graph server will start standalone hbase. +#hbase.zookeeper.quorum=localhost:2181 + +# Secure HBase +# currently only support kerberos. +#hbase.security.auth.enable = true +#hbase.security.authentication = kerberos +#hbase.kerberos.regionserver.principal = "your principal" +#hbase.sasl.clientconfig = Client +#java.security.krb5.conf="your krb config file" +#java.security.auth.login.config="jass config file" +#realm= +#principal= +#keytab="your keytab file location" + +# Asynchbase +# full list can be found at http://opentsdb.github.io/asynchbase/docs/build/html/configuration.html +#hbase.client.retries.number=10 +#hbase.rpcs.buffered_flush_interval=100 +#hbase.rpc.timeout=0 +#hbase.zookeeper.znode.parent="/hbase" + +# local retry number +# when write-write conflict occur, then S2Graph will retry as much as max.retry.number times with +# exponential backoff. max.back.off will be used maximum backoff time. +#max.retry.number=100 +#max.back.off=50 + +# Future cache. +# when 'cacheTTL' hint is given on query, then S2Graph will squash all requests on same vertex for +# `cacheTTL` time. this reduce requests on same key in backend storage significantly. +# to squash all requests on same vertex, S2Graph use simple local LRU cache(guava) and these configs are for +# setting LRU cache on local memory. +#future.cache.max.size=10000 +#future.cache.expire.after.write=60000 +#future.cache.expire.after.access=60000 +#future.cache.metric.interval=60000 + +# Local Cache +# S2Graph store metadata from metastore in local LRU cache(guava) to reduce requests into metastore. +#cache.ttl.seconds=60 +#cache.max.size=100000 + +# Kafka +# if kafka.metadata.broker.list is set, then S2Graph publish all incomming requests into kafka +# so that OLAP query can be run on HDFS file. +#kafka.metadata.broker.list="localhost:9092" + +# Meta Store +# S2Graph store metadata in JDBC-Compliant Databases(default H2, others may include MYSQL, Postgresql, etc). +# ex) to use mysql as metastore, change db.default.driver = "com.mysql.jdbc.Driver" +# and db.default.url to point to jdbc connection. +db.default.driver = "org.h2.Driver" db.default.url="jdbc:h2:file:./var/metastore;MODE=MYSQL" db.default.user="sa" db.default.password="sa" http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/69273249/dev_support/README.md ---------------------------------------------------------------------- diff --git a/dev_support/README.md b/dev_support/README.md index 50b04ee..2da1b55 100644 --- a/dev_support/README.md +++ b/dev_support/README.md @@ -22,11 +22,11 @@ # Run S2Graph using Docker 1. Build a docker image of the s2graph in the project's root directory - - `sbt "project s2rest_play" docker:publishLocal` + - `sbt "project s2rest_play" 'set version := "latest"' docker:publishLocal` + - find local image is created correctly by using `docker images` 2. Run MySQL and HBase container first. - - change directory to dev-support. `cd dev-support` + - change directory to dev-support. `cd dev_support` - `docker-compose build` - - `docker-compose up -d graph_mysql` will run MySQL and HBase at same time. 3. Run graph container - `docker-compose up -d` http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/69273249/dev_support/docker-compose.yml ---------------------------------------------------------------------- diff --git a/dev_support/docker-compose.yml b/dev_support/docker-compose.yml index c757e1d..5531fde 100644 --- a/dev_support/docker-compose.yml +++ b/dev_support/docker-compose.yml @@ -14,7 +14,7 @@ # limitations under the License. graph: - image: s2rest_play:0.1.0 + image: s2rest_play:latest container_name: graph net: container:graph_hbase @@ -26,9 +26,9 @@ graph_mysql: net: container:graph_hbase graph_hbase: - image: nerdammer/hbase:0.98.10.1 + image: harisekhon/hbase:1.2 container_name: graph_hbase - hostname: "${DOCKER_MACHINE_NAME}" + hostname: default ports: - "3306:3306" - "2181:2181" @@ -37,6 +37,10 @@ graph_hbase: - "60020:60020" - "60030:60030" - "9000:9000" + - "16000:16000" + - "16010:16010" + - "16201:16201" + - "16301:16301" expose: - "3306" - "9000" http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/69273249/project/Common.scala ---------------------------------------------------------------------- diff --git a/project/Common.scala b/project/Common.scala index 0f1fefb..ef793f1 100644 --- a/project/Common.scala +++ b/project/Common.scala @@ -22,8 +22,8 @@ object Common { lazy val sparkVersion = "1.4.1" lazy val playVersion = "2.3.10" - lazy val hbaseVersion = "1.0.1" + lazy val hbaseVersion = "1.2.2" // lazy val hbaseVersion = "1.0.0-cdh5.4.5" - lazy val hadoopVersion = "2.7.0" + lazy val hadoopVersion = "2.7.3" // lazy val hadoopVersion = "2.6.0-cdh5.4.5" } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/69273249/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/DistributedScanner.scala ---------------------------------------------------------------------- diff --git a/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/DistributedScanner.scala b/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/DistributedScanner.scala index 5f3c741..d78326d 100644 --- a/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/DistributedScanner.scala +++ b/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/DistributedScanner.scala @@ -86,4 +86,6 @@ class DistributedScanner(table: Table, scan: Scan) extends AbstractClientScanner scanner.close() } } - } + + override def renewLease(): Boolean = true +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/69273249/s2rest_play/app/org/apache/s2graph/rest/play/Bootstrap.scala ---------------------------------------------------------------------- diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/Bootstrap.scala b/s2rest_play/app/org/apache/s2graph/rest/play/Bootstrap.scala index a8acedf..686f6e9 100644 --- a/s2rest_play/app/org/apache/s2graph/rest/play/Bootstrap.scala +++ b/s2rest_play/app/org/apache/s2graph/rest/play/Bootstrap.scala @@ -59,7 +59,7 @@ object Global extends WithFilters(new GzipFilter()) { QueueActor.init(s2graph) - if (Config.IS_WRITE_SERVER && Config.KAFKA_PRODUCER_POOL_SIZE > 0) { + if (Config.IS_WRITE_SERVER) { ExceptionHandler.apply(config) } @@ -73,7 +73,7 @@ object Global extends WithFilters(new GzipFilter()) { override def onStop(app: Application) { QueueActor.shutdown() - if (Config.IS_WRITE_SERVER && Config.KAFKA_PRODUCER_POOL_SIZE > 0) { + if (Config.IS_WRITE_SERVER) { ExceptionHandler.shutdown() } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/69273249/s2rest_play/app/org/apache/s2graph/rest/play/config/Config.scala ---------------------------------------------------------------------- diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/config/Config.scala b/s2rest_play/app/org/apache/s2graph/rest/play/config/Config.scala index 2983cb7..3c488fe 100644 --- a/s2rest_play/app/org/apache/s2graph/rest/play/config/Config.scala +++ b/s2rest_play/app/org/apache/s2graph/rest/play/config/Config.scala @@ -41,7 +41,7 @@ object Config { //KAFKA lazy val KAFKA_METADATA_BROKER_LIST = conf.getString("kafka.metadata.broker.list").getOrElse("localhost") - lazy val KAFKA_PRODUCER_POOL_SIZE = conf.getInt("kafka.producer.pool.size").getOrElse(0) + lazy val KAFKA_LOG_TOPIC = s"s2graphIn${PHASE}" lazy val KAFKA_LOG_TOPIC_ASYNC = s"s2graphIn${PHASE}Async" lazy val KAFKA_FAIL_TOPIC = s"s2graphIn${PHASE}Failed"