Repository: incubator-s2graph
Updated Branches:
refs/heads/master 94862407f -> 69273249a
[S2GRAPH-102]: Add more configuration optoins in application.conf.
JIRA:
[S2GRAPH-102] https://issues.apache.org/jira/browse/S2GRAPH-102
Pull Request:
Closes #72
Authors:
DO YUNG YOON: [email protected]
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/69273249
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/69273249
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/69273249
Branch: refs/heads/master
Commit: 69273249a745c51bb612e79eea5d9c380c3c3f80
Parents: 9486240
Author: DO YUNG YOON <[email protected]>
Authored: Thu Sep 1 05:56:29 2016 +0900
Committer: DO YUNG YOON <[email protected]>
Committed: Thu Sep 1 05:56:29 2016 +0900
----------------------------------------------------------------------
CHANGES | 2 +
README.md | 3 +-
build.sbt | 1 +
conf/application.conf | 55 ++++++++++++++++++++
dev_support/README.md | 6 +--
dev_support/docker-compose.yml | 10 ++--
project/Common.scala | 4 +-
.../counter/helper/DistributedScanner.scala | 4 +-
.../apache/s2graph/rest/play/Bootstrap.scala | 4 +-
.../s2graph/rest/play/config/Config.scala | 2 +-
10 files changed, 78 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/69273249/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index f55b3b2..28cfbcd 100644
--- a/CHANGES
+++ b/CHANGES
@@ -171,6 +171,8 @@ Release 0.12.1 - unreleased
S2GRAPH-103: Remove dependencies on custom fork of asynchbase (Committed
by DOYUNG YOON).
+ S2GRAPH-102: Add more configuration optoins in application.conf (Committed
by DOYUNG YOON).
+
TEST
S2GRAPH-21: Change PostProcessBenchmarkSpec not to store and fetch test
data from storage. (Committed by DOYUNG YOON).
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/69273249/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 82fff4c..73a683f 100644
--- a/README.md
+++ b/README.md
@@ -50,7 +50,8 @@ Once built from source, the following resources are expected
under `target/apach
`sh bin/start-s2graph.sh` will launch the S2Graph server along with a
standalone [HBase server](https://hbase.apache.org/) as the data storage and
[H2](http://www.h2database.com/html/main.html) as the metastore
-Finally, join the mailing list by sending a message to
[[email protected]](mailto:[email protected]?subject=send
this email to subscribe) or
[[email protected]](mailto:[email protected]?subject=send
this email to subscribe)!
+To see how to connect remote HBase and S2Graph configurations, check out
conf/application.conf.
+Currently we have tested with these HBase version(0.98, 1.0, 1.1,
1.2)(https://hub.docker.com/r/harisekhon/hbase/tags/).
Project Layout
==============
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/69273249/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index c315771..94be364 100755
--- a/build.sbt
+++ b/build.sbt
@@ -25,6 +25,7 @@ name := "s2graph"
lazy val commonSettings = Seq(
organization := "org.apache.s2graph",
scalaVersion := "2.11.7",
+ version := "0.1.0",
scalacOptions := Seq("-language:postfixOps", "-unchecked", "-deprecation",
"-feature", "-Xlint"),
javaOptions ++=
collection.JavaConversions.propertiesAsScalaMap(System.getProperties).map {
case (key, value) => "-D" + key + "=" + value }.toSeq,
testOptions in Test += Tests.Argument("-oDF"),
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/69273249/conf/application.conf
----------------------------------------------------------------------
diff --git a/conf/application.conf b/conf/application.conf
index 39b5f68..4920380 100644
--- a/conf/application.conf
+++ b/conf/application.conf
@@ -17,6 +17,61 @@
# under the License.
#
+# HBase
+# place your local/remote hbase cluster's zookeeper quorum,
+# as default, s2graph server will start standalone hbase.
+#hbase.zookeeper.quorum=localhost:2181
+
+# Secure HBase
+# currently only support kerberos.
+#hbase.security.auth.enable = true
+#hbase.security.authentication = kerberos
+#hbase.kerberos.regionserver.principal = "your principal"
+#hbase.sasl.clientconfig = Client
+#java.security.krb5.conf="your krb config file"
+#java.security.auth.login.config="jass config file"
+#realm=
+#principal=
+#keytab="your keytab file location"
+
+# Asynchbase
+# full list can be found at
http://opentsdb.github.io/asynchbase/docs/build/html/configuration.html
+#hbase.client.retries.number=10
+#hbase.rpcs.buffered_flush_interval=100
+#hbase.rpc.timeout=0
+#hbase.zookeeper.znode.parent="/hbase"
+
+# local retry number
+# when write-write conflict occur, then S2Graph will retry as much as
max.retry.number times with
+# exponential backoff. max.back.off will be used maximum backoff time.
+#max.retry.number=100
+#max.back.off=50
+
+# Future cache.
+# when 'cacheTTL' hint is given on query, then S2Graph will squash all
requests on same vertex for
+# `cacheTTL` time. this reduce requests on same key in backend storage
significantly.
+# to squash all requests on same vertex, S2Graph use simple local LRU
cache(guava) and these configs are for
+# setting LRU cache on local memory.
+#future.cache.max.size=10000
+#future.cache.expire.after.write=60000
+#future.cache.expire.after.access=60000
+#future.cache.metric.interval=60000
+
+# Local Cache
+# S2Graph store metadata from metastore in local LRU cache(guava) to reduce
requests into metastore.
+#cache.ttl.seconds=60
+#cache.max.size=100000
+
+# Kafka
+# if kafka.metadata.broker.list is set, then S2Graph publish all incomming
requests into kafka
+# so that OLAP query can be run on HDFS file.
+#kafka.metadata.broker.list="localhost:9092"
+
+# Meta Store
+# S2Graph store metadata in JDBC-Compliant Databases(default H2, others may
include MYSQL, Postgresql, etc).
+# ex) to use mysql as metastore, change db.default.driver =
"com.mysql.jdbc.Driver"
+# and db.default.url to point to jdbc connection.
+db.default.driver = "org.h2.Driver"
db.default.url="jdbc:h2:file:./var/metastore;MODE=MYSQL"
db.default.user="sa"
db.default.password="sa"
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/69273249/dev_support/README.md
----------------------------------------------------------------------
diff --git a/dev_support/README.md b/dev_support/README.md
index 50b04ee..2da1b55 100644
--- a/dev_support/README.md
+++ b/dev_support/README.md
@@ -22,11 +22,11 @@
# Run S2Graph using Docker
1. Build a docker image of the s2graph in the project's root directory
- - `sbt "project s2rest_play" docker:publishLocal`
+ - `sbt "project s2rest_play" 'set version := "latest"'
docker:publishLocal`
+ - find local image is created correctly by using `docker images`
2. Run MySQL and HBase container first.
- - change directory to dev-support. `cd dev-support`
+ - change directory to dev-support. `cd dev_support`
- `docker-compose build`
- - `docker-compose up -d graph_mysql` will run MySQL and HBase at same
time.
3. Run graph container
- `docker-compose up -d`
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/69273249/dev_support/docker-compose.yml
----------------------------------------------------------------------
diff --git a/dev_support/docker-compose.yml b/dev_support/docker-compose.yml
index c757e1d..5531fde 100644
--- a/dev_support/docker-compose.yml
+++ b/dev_support/docker-compose.yml
@@ -14,7 +14,7 @@
# limitations under the License.
graph:
- image: s2rest_play:0.1.0
+ image: s2rest_play:latest
container_name: graph
net: container:graph_hbase
@@ -26,9 +26,9 @@ graph_mysql:
net: container:graph_hbase
graph_hbase:
- image: nerdammer/hbase:0.98.10.1
+ image: harisekhon/hbase:1.2
container_name: graph_hbase
- hostname: "${DOCKER_MACHINE_NAME}"
+ hostname: default
ports:
- "3306:3306"
- "2181:2181"
@@ -37,6 +37,10 @@ graph_hbase:
- "60020:60020"
- "60030:60030"
- "9000:9000"
+ - "16000:16000"
+ - "16010:16010"
+ - "16201:16201"
+ - "16301:16301"
expose:
- "3306"
- "9000"
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/69273249/project/Common.scala
----------------------------------------------------------------------
diff --git a/project/Common.scala b/project/Common.scala
index 0f1fefb..ef793f1 100644
--- a/project/Common.scala
+++ b/project/Common.scala
@@ -22,8 +22,8 @@ object Common {
lazy val sparkVersion = "1.4.1"
lazy val playVersion = "2.3.10"
- lazy val hbaseVersion = "1.0.1"
+ lazy val hbaseVersion = "1.2.2"
// lazy val hbaseVersion = "1.0.0-cdh5.4.5"
- lazy val hadoopVersion = "2.7.0"
+ lazy val hadoopVersion = "2.7.3"
// lazy val hadoopVersion = "2.6.0-cdh5.4.5"
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/69273249/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/DistributedScanner.scala
----------------------------------------------------------------------
diff --git
a/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/DistributedScanner.scala
b/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/DistributedScanner.scala
index 5f3c741..d78326d 100644
---
a/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/DistributedScanner.scala
+++
b/s2counter_core/src/main/scala/org/apache/s2graph/counter/helper/DistributedScanner.scala
@@ -86,4 +86,6 @@ class DistributedScanner(table: Table, scan: Scan) extends
AbstractClientScanner
scanner.close()
}
}
- }
+
+ override def renewLease(): Boolean = true
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/69273249/s2rest_play/app/org/apache/s2graph/rest/play/Bootstrap.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/Bootstrap.scala
b/s2rest_play/app/org/apache/s2graph/rest/play/Bootstrap.scala
index a8acedf..686f6e9 100644
--- a/s2rest_play/app/org/apache/s2graph/rest/play/Bootstrap.scala
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/Bootstrap.scala
@@ -59,7 +59,7 @@ object Global extends WithFilters(new GzipFilter()) {
QueueActor.init(s2graph)
- if (Config.IS_WRITE_SERVER && Config.KAFKA_PRODUCER_POOL_SIZE > 0) {
+ if (Config.IS_WRITE_SERVER) {
ExceptionHandler.apply(config)
}
@@ -73,7 +73,7 @@ object Global extends WithFilters(new GzipFilter()) {
override def onStop(app: Application) {
QueueActor.shutdown()
- if (Config.IS_WRITE_SERVER && Config.KAFKA_PRODUCER_POOL_SIZE > 0) {
+ if (Config.IS_WRITE_SERVER) {
ExceptionHandler.shutdown()
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/69273249/s2rest_play/app/org/apache/s2graph/rest/play/config/Config.scala
----------------------------------------------------------------------
diff --git a/s2rest_play/app/org/apache/s2graph/rest/play/config/Config.scala
b/s2rest_play/app/org/apache/s2graph/rest/play/config/Config.scala
index 2983cb7..3c488fe 100644
--- a/s2rest_play/app/org/apache/s2graph/rest/play/config/Config.scala
+++ b/s2rest_play/app/org/apache/s2graph/rest/play/config/Config.scala
@@ -41,7 +41,7 @@ object Config {
//KAFKA
lazy val KAFKA_METADATA_BROKER_LIST =
conf.getString("kafka.metadata.broker.list").getOrElse("localhost")
- lazy val KAFKA_PRODUCER_POOL_SIZE =
conf.getInt("kafka.producer.pool.size").getOrElse(0)
+
lazy val KAFKA_LOG_TOPIC = s"s2graphIn${PHASE}"
lazy val KAFKA_LOG_TOPIC_ASYNC = s"s2graphIn${PHASE}Async"
lazy val KAFKA_FAIL_TOPIC = s"s2graphIn${PHASE}Failed"
