Repository: incubator-s2graph Updated Branches: refs/heads/master ed6d36696 -> fabb212dd
[S2GRAPH-70]: Automate the process of building a distribution package include following subtasks. - [S2GRAPH-75]: Use an embedded database as the default metadata storage - [S2GRAPH-74]: Add a script that starts/stops a local hbase server - [S2GRAPH-73]: Add an SBT task and relevant scripts to construct a package for distribution JIRA: [S2GRAPH-70] https://issues.apache.org/jira/browse/S2GRAPH-70 Pull Request: Closes #57 Authors: Jong Wook Kim: [email protected] Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/fabb212d Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/fabb212d Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/fabb212d Branch: refs/heads/master Commit: fabb212dd69d47977987dc2b8ab43e2ae31808d4 Parents: ed6d366 Author: DO YUNG YOON <[email protected]> Authored: Sat Jun 11 23:03:59 2016 +0900 Committer: DO YUNG YOON <[email protected]> Committed: Sat Jun 11 23:03:59 2016 +0900 ---------------------------------------------------------------------- .gitignore | 1 - CHANGES | 12 + bin/hbase-standalone.sh | 41 ++++ bin/run-rat.sh | 31 --- bin/s2graph-common.sh | 42 ++++ bin/s2graph-daemon.sh | 169 +++++++++++++ bin/s2graph-env.sh | 60 +++++ bin/s2graph.sh | 32 +++ bin/start-s2graph.sh | 27 +++ bin/stop-s2graph.sh | 27 +++ build.sbt | 5 +- conf/application.conf | 22 ++ conf/hbase-site.xml | 26 ++ conf/logback.xml | 32 +++ dev/run-rat.sh | 31 +++ loader/build.sbt | 2 +- project/Packager.scala | 98 ++++++++ s2core/build.sbt | 13 +- s2core/lib/async-1.4.0.jar | Bin 18031 -> 0 bytes s2core/lib/asynchbase-1.7.1-SNAPSHOT.jar | Bin 1302410 -> 0 bytes s2core/lib/asynchbase-1.7.2-SNAPSHOT.jar | Bin 0 -> 5594923 bytes s2core/lib/netty-3.9.4.Final.jar | Bin 1310154 -> 0 bytes s2core/migrate/mysql/schema.sql | 238 ------------------- .../org/apache/s2graph/core/mysqls/schema.sql | 227 ++++++++++++++++++ .../org/apache/s2graph/core/mysqls/setup.sql | 28 +++ .../org/apache/s2graph/core/mysqls/Model.scala | 38 ++- s2counter_core/build.sbt | 4 + s2counter_loader/build.sbt | 1 + s2rest_netty/build.sbt | 1 + s2rest_netty/conf/reference.conf | 2 +- s2rest_netty/src/main/resources/reference.conf | 2 +- s2rest_play/build.sbt | 1 + spark/build.sbt | 1 + 33 files changed, 931 insertions(+), 283 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 1e21302..4b2e63d 100644 --- a/.gitignore +++ b/.gitignore @@ -90,7 +90,6 @@ project/boot/ ### PlayFramework ### # Ignore Play! working directory # -bin/ /db .eclipse /lib/ http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 3aa2e7b..a7f7512 100644 --- a/CHANGES +++ b/CHANGES @@ -68,6 +68,9 @@ Release 0.12.1 - unreleased S2GRAPH-22: Add missing shebang line to file. (Contributed by Injun Song<[email protected]>, committed by DOYUNG YOON) + + S2GRAPH-70: Automate the process of building a distribution package + (Contributed by Jong Wook Kim<[email protected]>, committed by DOYUNG YOON) BUG FIXES @@ -118,6 +121,15 @@ Release 0.12.1 - unreleased S2GRAPH-31: Remove playframework dependencies on s2core/build.sbt. (Committed by DOYUNG YOON). S2GRAPH-87: LICENSE file should be changed. (Committed by Sergio Fernández). + + S2GRAPH-73: Add an SBT task and relevant scripts to construct a package for distribution. + (Contributed by Jong Wook Kim<[email protected]>, committed by DOYUNG YOON) + + S2GRAPH-74: Add a script that starts/stops a local hbase server. + (Contributed by Jong Wook Kim<[email protected]>, committed by DOYUNG YOON) + + S2GRAPH-75: Use an embedded database as the default metadata storage. + (Contributed by Jong Wook Kim<[email protected]>, committed by DOYUNG YOON) TEST http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/bin/hbase-standalone.sh ---------------------------------------------------------------------- diff --git a/bin/hbase-standalone.sh b/bin/hbase-standalone.sh new file mode 100755 index 0000000..9d9ef97 --- /dev/null +++ b/bin/hbase-standalone.sh @@ -0,0 +1,41 @@ +#! /usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# starts/stops/restarts an HBase server running in standalone mode. + +usage="Usage: hbase-standalone.sh (start|stop|restart|run|status)" + +bin=$(cd "$(dirname "${BASH_SOURCE-$0}")">/dev/null; pwd) + +# load environment variables +. $bin/s2graph-env.sh +. $bin/s2graph-common.sh + +# show usage when executed without enough arguments +if [ $# -lt 1 ]; then + panic $usage +fi + +S2GRAPH_OPTS="$S2GRAPH_OPTS -Dproc_master" +S2GRAPH_OPTS="$S2GRAPH_OPTS -Dhbase.log.dir=$S2GRAPH_LOG_DIR" +S2GRAPH_OPTS="$S2GRAPH_OPTS -Dhbase.log.file=hbase.logger.log" +S2GRAPH_OPTS="$S2GRAPH_OPTS -Dhbase.home.dir=$S2GRAPH_HOME" +S2GRAPH_OPTS="$S2GRAPH_OPTS -Dhbase.id.str=s2graph" +S2GRAPH_OPTS="$S2GRAPH_OPTS -Dhbase.root.logger=INFO,RFA" + +export S2GRAPH_OPTS + +$bin/s2graph-daemon.sh $1 hbase start http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/bin/run-rat.sh ---------------------------------------------------------------------- diff --git a/bin/run-rat.sh b/bin/run-rat.sh deleted file mode 100644 index c22c938..0000000 --- a/bin/run-rat.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -base_dir=$(dirname $0)/.. -rat_excludes_file=$base_dir/.rat-excludes - -echo "Base Directory = $base_dir" - -if [ -z "$JAVA_HOME" ]; then - JAVA="java" -else - JAVA="$JAVA_HOME/bin/java" -fi - -rat_command="$JAVA -jar $base_dir/lib/apache-rat-0.11.jar --dir $base_dir --exclude-file ${rat_excludes_file} -f -a" - -echo "Running " $rat_command -$rat_command > $base_dir/rat.out http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/bin/s2graph-common.sh ---------------------------------------------------------------------- diff --git a/bin/s2graph-common.sh b/bin/s2graph-common.sh new file mode 100644 index 0000000..d03b22c --- /dev/null +++ b/bin/s2graph-common.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# print an error message to STDERR and quit +panic() { + echo "$@" 1>&2 + exit -1 +} + +# stop a process with specified pid, first by SIGTERM and then by SIGKILL +terminate() { + for pid in "$@"; do + started=$(date +%s) + kill $pid > /dev/null 2>&1 + printf "Waiting process $pid to finish" 1>&2 + while kill -0 $pid > /dev/null 2>&1; do + sleep 0.5 + printf "." 1>&2 + if [ $(( $(date +%s) - $started )) -ge $S2GRAPH_STOP_TIMEOUT ]; then + break + fi + done + printf "\n" 1>&2 + if kill -0 $pid > /dev/null 2>&1; then + echo "Force stopping $pid with a SIGKILL" 1>&2 + kill -9 $pid > /dev/null 2>&1 + fi + done +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/bin/s2graph-daemon.sh ---------------------------------------------------------------------- diff --git a/bin/s2graph-daemon.sh b/bin/s2graph-daemon.sh new file mode 100755 index 0000000..18c327b --- /dev/null +++ b/bin/s2graph-daemon.sh @@ -0,0 +1,169 @@ +#! /usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# starts/stops/restarts a java application as a daemon process +# also able to run the application directly using run + +usage="This script is intended to be used by other scripts in this directory." +usage=$"$usage\n Please refer to start-s2graph.sh and stop-s2graph.sh" +usage=$"$usage\n Usage: s2graph-daemon.sh (start|stop|restart|run|status)" +usage="$usage (s2rest_play|s2rest_netty|...) <args...>" + +bin=$(cd "$(dirname "${BASH_SOURCE-$0}")">/dev/null; pwd) + +# load environment variables +. $bin/s2graph-env.sh +. $bin/s2graph-common.sh + +# show usage when executed without enough arguments +if [ $# -le 1 ]; then + panic "$usage" +fi + +# to enable the JVM application to use relative paths as well +cd "$S2GRAPH_HOME" + +# arguments +action=$1 +shift +service=$1 +shift +this="$bin/$(basename ${BASH_SOURCE-$0})" +args="$@" + +# locate java +if [ -z $JAVA_HOME ]; then + JAVA=$(which java) +else + JAVA=$JAVA_HOME/bin/java +fi + +# require JVM 8 +JAVA_VERSION=$($JAVA -version 2>&1 | sed 's/.*version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q') +if [ $JAVA_VERSION -lt 18 ]; then + panic "Java 8 or later is required" +fi + +# construct the classpath +classpath="$S2GRAPH_CONF_DIR:$S2GRAPH_LIB_DIR/*" + +# determine the main class +case $service in +s2rest_play) + main="play.core.server.NettyServer" + ;; +hbase) + main="org.apache.hadoop.hbase.master.HMaster" + ;; +*) + panic "Unknown service: $service" +esac + +# file that contains the service's pid +pidfile="$S2GRAPH_PID_DIR/$service.pid" + +# file that contains the console output of the JVM +# this file should only contain abnormal logs, while +# the normal logs should go to the log files as configured separately, +# ideally with daily log-rotation. +log="$S2GRAPH_LOG_DIR/$service.log" + +start() { + if [ -f "$pidfile" ]; then + if kill -0 $(cat "$pidfile") > /dev/null 2>&1; then + panic "$service is already running as process $(cat "$pidfile")" + fi + rm -r "$pidfile" + fi + echo "Starting $service..." 1>&2 + + "$this" run "$service" $args < /dev/null >> "$log" 2>&1 & + disown -h -r + + # sleep enough, to check if there was any error during startup + sleep 3 + if [ -f "$pidfile" ]; then + echo "Service $service is running as process $(cat "$pidfile")" + else + echo "Service $service might have failed to start; see $log" + fi +} + +stop() { + if [ -f "$pidfile" ]; then + if kill -0 $(cat "$pidfile") > /dev/null 2>&1; then + echo "Stopping $service" 1>&2 + terminate $(cat "$pidfile") + else + echo "Process $(cat "$pidfile") is not running" 1>&2 + fi + else + echo "PID file does not exist at $pidfile" 1>&2 + fi + rm -f "$pidfile" +} + +cleanup() { + if [ -f "$pidfile" ]; then + if kill -0 $(cat "$pidfile") > /dev/null 2>&1; then + terminate $(cat "$pidfile") + fi + rm -f "$pidfile" + fi +} + +status() { + if [ -f "$pidfile" ]; then + if kill -0 $(cat "$pidfile") > /dev/null 2>&1; then + echo "Service $service is running as process $(cat "$pidfile")" 1>&2 + else + echo "PID file for service $service exists at $pidfile, " \ + "but process $(cat "$pidfile") is not running" 1>&2 + fi + else + echo "Service $service is not running" 1>&2 + fi +} + +run() { + trap cleanup SIGHUP SIGINT SIGTERM EXIT + echo "$JAVA" -cp "$classpath" -Ds2graph.home="$S2GRAPH_HOME" $S2GRAPH_OPTS "$main" $args 2>&1 & + "$JAVA" -cp "$classpath" -Ds2graph.home="$S2GRAPH_HOME" $S2GRAPH_OPTS "$main" $args 2>&1 & + pid=$! + echo "$pid" > "$pidfile" + wait "$pid" +} + +case $action in +start) + start + ;; +stop) + stop + ;; +restart) + stop + start + ;; +status) + status + ;; +run) + run + ;; +*) + panic "$usage" +esac http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/bin/s2graph-env.sh ---------------------------------------------------------------------- diff --git a/bin/s2graph-env.sh b/bin/s2graph-env.sh new file mode 100644 index 0000000..0145e1d --- /dev/null +++ b/bin/s2graph-env.sh @@ -0,0 +1,60 @@ +#! /usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file sets appropriate environment variables for running S2Graph. +# Should be sourced and not be directly executed. +# The executable flag for this file should not be set, e.g. 644. + +# Find the current directory +SOURCE="${BASH_SOURCE[0]}" +# resolve $SOURCE until the file is no longer a symlink +while [ -h "$SOURCE" ]; do + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" +done +DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + +# set S2GRAPH_HOME +if [ -z $S2GRAPH_HOME ]; then + export S2GRAPH_HOME="$( cd -P "$DIR/.." && pwd )" +fi + +# set other environment variables for directories, relative to $S2GRAPH_HOME +# not intended to be individually configurable, +# configurations can be added or overridden in conf/application.conf +export S2GRAPH_BIN_DIR=$S2GRAPH_HOME/bin +export S2GRAPH_LIB_DIR=$S2GRAPH_HOME/lib +export S2GRAPH_LOG_DIR=$S2GRAPH_HOME/logs +export S2GRAPH_CONF_DIR=$S2GRAPH_HOME/conf +export S2GRAPH_HBASE_DIR=$S2GRAPH_HOME/var/hbase +export S2GRAPH_METASTORE_DIR=$S2GRAPH_HOME/var/metastore +export S2GRAPH_PID_DIR=$S2GRAPH_HOME/var/pid + +mkdir -p "$S2GRAPH_LOG_DIR" "$S2GRAPH_CONF_DIR" "$S2GRAPH_HBASE_DIR" \ + "$S2GRAPH_METASTORE_DIR" "$S2GRAPH_PID_DIR" + +# apply the minimal JVM settings if none was given; +# in production, this variable is expected to be given by a deployment system. +if [ -z "$S2GRAPH_OPTS" ]; then + export S2GRAPH_OPTS="-Xms1g -Xmx1g" +fi + +# the maximum wait time for a process to finish, before sending SIGKILL +if [ -z "$S2GRAPH_STOP_TIMEOUT" ]; then + export S2GRAPH_STOP_TIMEOUT=60 # in seconds +fi http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/bin/s2graph.sh ---------------------------------------------------------------------- diff --git a/bin/s2graph.sh b/bin/s2graph.sh new file mode 100755 index 0000000..58ddc28 --- /dev/null +++ b/bin/s2graph.sh @@ -0,0 +1,32 @@ +#! /usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# starts/stops/restarts an S2Graph server + +usage="Usage: s2graph.sh (start|stop|restart|run|status)" + +bin=$(cd "$(dirname "${BASH_SOURCE-$0}")">/dev/null; pwd) + +# load environment variables +. $bin/s2graph-env.sh +. $bin/s2graph-common.sh + +# show usage when executed without enough arguments +if [ $# -lt 1 ]; then + panic $usage +fi + +$bin/s2graph-daemon.sh $1 s2rest_play http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/bin/start-s2graph.sh ---------------------------------------------------------------------- diff --git a/bin/start-s2graph.sh b/bin/start-s2graph.sh new file mode 100755 index 0000000..23ab1ab --- /dev/null +++ b/bin/start-s2graph.sh @@ -0,0 +1,27 @@ +#! /usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Starts the S2Graph server, with an hbase server as required + +bin=$(cd "$(dirname "${BASH_SOURCE-$0}")">/dev/null; pwd) + +# load environment variables +. $bin/s2graph-env.sh +. $bin/s2graph-common.sh + +$bin/hbase-standalone.sh start +$bin/s2graph.sh start http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/bin/stop-s2graph.sh ---------------------------------------------------------------------- diff --git a/bin/stop-s2graph.sh b/bin/stop-s2graph.sh new file mode 100755 index 0000000..124c192 --- /dev/null +++ b/bin/stop-s2graph.sh @@ -0,0 +1,27 @@ +#! /usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Stops the S2Graph server, along with an hbase server as required + +bin=$(cd "$(dirname "${BASH_SOURCE-$0}")">/dev/null; pwd) + +# load environment variables +. $bin/s2graph-env.sh +. $bin/s2graph-common.sh + +$bin/s2graph.sh stop +$bin/hbase-standalone.sh stop http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/build.sbt ---------------------------------------------------------------------- diff --git a/build.sbt b/build.sbt index 8571163..d3c6692 100755 --- a/build.sbt +++ b/build.sbt @@ -64,10 +64,13 @@ lazy val s2counter_loader = project.dependsOn(s2counter_core, spark) lazy val root = (project in file(".")) .aggregate(s2core, s2rest_play) + .dependsOn(s2rest_play, s2rest_netty, loader, s2counter_loader) // this enables packaging on the root project .settings(commonSettings: _*) lazy val runRatTask = taskKey[Unit]("Runs Apache rat on S2Graph") runRatTask := { - "sh bin/run-rat.sh" ! + "sh dev/run-rat.sh" ! } + +Packager.defaultSettings http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/conf/application.conf ---------------------------------------------------------------------- diff --git a/conf/application.conf b/conf/application.conf new file mode 100644 index 0000000..39b5f68 --- /dev/null +++ b/conf/application.conf @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +db.default.url="jdbc:h2:file:./var/metastore;MODE=MYSQL" +db.default.user="sa" +db.default.password="sa" http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/conf/hbase-site.xml ---------------------------------------------------------------------- diff --git a/conf/hbase-site.xml b/conf/hbase-site.xml new file mode 100644 index 0000000..adf29db --- /dev/null +++ b/conf/hbase-site.xml @@ -0,0 +1,26 @@ +<?xml version="1.0"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <property> + <name>hbase.tmp.dir</name> + <value>${s2graph.home}/var/hbase</value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/conf/logback.xml ---------------------------------------------------------------------- diff --git a/conf/logback.xml b/conf/logback.xml new file mode 100644 index 0000000..d3f09bc --- /dev/null +++ b/conf/logback.xml @@ -0,0 +1,32 @@ +<?xml version="1.0"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> + <layout class="ch.qos.logback.classic.PatternLayout"> + <Pattern> + %d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n + </Pattern> + </layout> + </appender> + <root level="INFO"> + <appender-ref ref="STDOUT" /> + </root> +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/dev/run-rat.sh ---------------------------------------------------------------------- diff --git a/dev/run-rat.sh b/dev/run-rat.sh new file mode 100644 index 0000000..c22c938 --- /dev/null +++ b/dev/run-rat.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +base_dir=$(dirname $0)/.. +rat_excludes_file=$base_dir/.rat-excludes + +echo "Base Directory = $base_dir" + +if [ -z "$JAVA_HOME" ]; then + JAVA="java" +else + JAVA="$JAVA_HOME/bin/java" +fi + +rat_command="$JAVA -jar $base_dir/lib/apache-rat-0.11.jar --dir $base_dir --exclude-file ${rat_excludes_file} -f -a" + +echo "Running " $rat_command +$rat_command > $base_dir/rat.out http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/loader/build.sbt ---------------------------------------------------------------------- diff --git a/loader/build.sbt b/loader/build.sbt index 2929da8..b74f305 100644 --- a/loader/build.sbt +++ b/loader/build.sbt @@ -28,6 +28,7 @@ projectDependencies := Seq( ) libraryDependencies ++= Seq( + "com.google.guava" % "guava" % "12.0.1" force(), // use this old version of guava to avoid incompatibility "org.apache.spark" %% "spark-core" % Common.sparkVersion % "provided", "org.apache.spark" %% "spark-streaming" % Common.sparkVersion % "provided", "org.apache.spark" %% "spark-hive" % Common.sparkVersion % "provided", @@ -35,7 +36,6 @@ libraryDependencies ++= Seq( "org.apache.httpcomponents" % "fluent-hc" % "4.2.5", "org.specs2" %% "specs2-core" % "2.4.11" % "test", "org.scalatest" %% "scalatest" % "2.2.1" % "test", - "org.apache.hbase" % "hbase-hadoop-compat" % "0.98.14-hadoop2", "org.apache.hadoop" % "hadoop-distcp" % Common.hadoopVersion ) http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/project/Packager.scala ---------------------------------------------------------------------- diff --git a/project/Packager.scala b/project/Packager.scala new file mode 100644 index 0000000..cff65fd --- /dev/null +++ b/project/Packager.scala @@ -0,0 +1,98 @@ +import java.nio.file.attribute.BasicFileAttributes +import java.nio.file.{FileVisitResult, Files, Path, Paths, SimpleFileVisitor, StandardCopyOption} + +import sbt.Keys._ +import sbt._ + +object Packager { + + val packagedJars = TaskKey[Seq[File]]("packaged-jars", "list of jar files for packaging") + val packager = TaskKey[File]("package", s"packages the project for deployment") + + val packagedJarsTask = (state, thisProjectRef).flatMap { (state, project) => + + /** return all tasks of given TaskKey's dependencies */ + def taskInAllDependencies[T](taskKey: TaskKey[T]): Task[Seq[T]] = { + val structure = Project.structure(state) + + Dag.topologicalSort(project) { ref => + Project.getProject(ref, structure).toList.flatMap { p => + p.dependencies.map(_.project) ++ p.aggregate + } + }.flatMap { p => + taskKey in p get structure.data + }.join + } + + for { + packaged <- taskInAllDependencies(packagedArtifacts) + srcs <- taskInAllDependencies(packageSrc in Compile) + docs <- taskInAllDependencies(packageDoc in Compile) + } yield { + packaged.flatten + .collect { case (artifact, path) if artifact.extension == "jar" => path } + .diff(srcs ++ docs) //remove srcs & docs since we do not need them in the dist + .distinct + } + } + + val packagerTask = (baseDirectory, packagedJars, dependencyClasspath in Runtime, target, streams).map { + (root, packaged, dependencies, target, streams) => + val base = target / "deploy" + + IO.delete(base) + IO.createDirectory(base) + + val subdirectories = Seq("bin", "conf", "lib") + + for (dir <- subdirectories) { + IO.createDirectory(base / dir) + } + + // copy jars to /lib + val libs = dependencies.filter(_.data.ext == "jar").map(_.data) ++ packaged + libs.foreach { jar => + val source = jar.toPath + val target = Paths.get(base.getPath, "lib", jar.getName) + + try { + // try to create hard links for speed and to save disk space + Files.createLink(target, source) + streams.log.debug(s"created link: $source -> $target") + } catch { + case e: Any => + Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING) + streams.log.debug(s"copied: $source -> $target") + } + } + + // copy other files recursively + for (subdir <- subdirectories if subdir != "lib") { + val source = (root / subdir).toPath + val target = (base / subdir).toPath + Files.walkFileTree(source, new SimpleFileVisitor[Path] { + override def preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult = { + val dest = target.resolve(source.relativize(dir)) + Files.createDirectories(dest) + streams.log.debug(s"created directory: $dest") + FileVisitResult.CONTINUE + } + + override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = { + val dest = target.resolve(source.relativize(file)) + Files.copy(file, dest) + streams.log.debug(s"copied: $file -> $dest") + FileVisitResult.CONTINUE + } + }) + } + + streams.log.info(s"package for distribution located at $base") + base + } + + val defaultSettings = Seq( + packagedJars <<= packagedJarsTask, + packager <<= packagerTask + ) +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2core/build.sbt ---------------------------------------------------------------------- diff --git a/s2core/build.sbt b/s2core/build.sbt index 6774c60..46c2795 100644 --- a/s2core/build.sbt +++ b/s2core/build.sbt @@ -26,15 +26,18 @@ libraryDependencies ++= Seq( "com.typesafe" % "config" % "1.2.1", "com.typesafe.play" %% "play-json" % Common.playVersion, "com.typesafe.akka" %% "akka-actor" % "2.3.4", - "org.apache.hbase" % "hbase-client" % Common.hbaseVersion excludeAll ExclusionRule(organization = "org.slf4j"), - "org.apache.hbase" % "hbase-common" % Common.hbaseVersion excludeAll ExclusionRule(organization = "org.slf4j"), - "org.apache.hbase" % "hbase-server" % Common.hbaseVersion excludeAll(ExclusionRule(organization = "org.slf4j"), ExclusionRule(organization = "com.google.protobuf")), - "org.apache.hadoop" % "hadoop-common" % Common.hadoopVersion excludeAll ExclusionRule(organization = "org.slf4j"), + "com.google.guava" % "guava" % "12.0.1" force(), // use this old version of guava to avoid incompatibility + "org.apache.hbase" % "hbase-client" % Common.hbaseVersion exclude("org.slf4j", "*"), + "org.apache.hbase" % "hbase-common" % Common.hbaseVersion exclude("org.slf4j", "*"), + "org.apache.hbase" % "hbase-server" % Common.hbaseVersion exclude("org.slf4j", "*") exclude("com.google.protobuf", "*"), + "org.apache.hbase" % "hbase-hadoop-compat" % Common.hbaseVersion exclude("org.slf4j", "*"), + "org.apache.hbase" % "hbase-hadoop2-compat" % Common.hbaseVersion exclude("org.slf4j", "*"), + "org.apache.kafka" % "kafka-clients" % "0.8.2.0" exclude("org.slf4j", "*") exclude("com.sun.jdmk", "*") exclude("com.sun.jmx", "*") exclude("javax.jms", "*"), "commons-pool" % "commons-pool" % "1.6", "org.scalatest" %% "scalatest" % "2.2.4" % "test", "org.scalikejdbc" %% "scalikejdbc" % "2.1.+", "mysql" % "mysql-connector-java" % "5.1.28", - "org.apache.kafka" % "kafka-clients" % "0.8.2.0" excludeAll(ExclusionRule(organization = "org.slf4j"), ExclusionRule(organization = "com.sun.jdmk"), ExclusionRule(organization = "com.sun.jmx"), ExclusionRule(organization = "javax.jms")), + "com.h2database" % "h2" % "1.4.192", "com.github.danielwegener" % "logback-kafka-appender" % "0.0.4" ) http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2core/lib/async-1.4.0.jar ---------------------------------------------------------------------- diff --git a/s2core/lib/async-1.4.0.jar b/s2core/lib/async-1.4.0.jar deleted file mode 100644 index 7fa1692..0000000 Binary files a/s2core/lib/async-1.4.0.jar and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2core/lib/asynchbase-1.7.1-SNAPSHOT.jar ---------------------------------------------------------------------- diff --git a/s2core/lib/asynchbase-1.7.1-SNAPSHOT.jar b/s2core/lib/asynchbase-1.7.1-SNAPSHOT.jar deleted file mode 100644 index 801d87d..0000000 Binary files a/s2core/lib/asynchbase-1.7.1-SNAPSHOT.jar and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2core/lib/asynchbase-1.7.2-SNAPSHOT.jar ---------------------------------------------------------------------- diff --git a/s2core/lib/asynchbase-1.7.2-SNAPSHOT.jar b/s2core/lib/asynchbase-1.7.2-SNAPSHOT.jar new file mode 100644 index 0000000..ade9f82 Binary files /dev/null and b/s2core/lib/asynchbase-1.7.2-SNAPSHOT.jar differ http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2core/lib/netty-3.9.4.Final.jar ---------------------------------------------------------------------- diff --git a/s2core/lib/netty-3.9.4.Final.jar b/s2core/lib/netty-3.9.4.Final.jar deleted file mode 100644 index 7ee6ba4..0000000 Binary files a/s2core/lib/netty-3.9.4.Final.jar and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2core/migrate/mysql/schema.sql ---------------------------------------------------------------------- diff --git a/s2core/migrate/mysql/schema.sql b/s2core/migrate/mysql/schema.sql deleted file mode 100644 index 9ce746c..0000000 --- a/s2core/migrate/mysql/schema.sql +++ /dev/null @@ -1,238 +0,0 @@ --- --- Licensed to the Apache Software Foundation (ASF) under one --- or more contributor license agreements. See the NOTICE file --- distributed with this work for additional information --- regarding copyright ownership. The ASF licenses this file --- to you under the Apache License, Version 2.0 (the --- "License"); you may not use this file except in compliance --- with the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, --- software distributed under the License is distributed on an --- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY --- KIND, either express or implied. See the License for the --- specific language governing permissions and limitations --- under the License. --- - -CREATE DATABASE IF NOT EXISTS graph_dev; - -CREATE USER 'graph'@'%' IDENTIFIED BY 'graph'; - -GRANT ALL PRIVILEGES ON graph_dev.* TO 'graph'@'%' identified by 'graph'; - -flush privileges; - -use graph_dev; - - -SET FOREIGN_KEY_CHECKS = 0; - --- ---------------------------- --- Table structure for `services` --- ---------------------------- -DROP TABLE IF EXISTS `services`; -CREATE TABLE `services` ( - `id` integer NOT NULL AUTO_INCREMENT, - `service_name` varchar(64) NOT NULL, - `access_token` varchar(64) NOT NULL, - `cluster` varchar(255) NOT NULL, - `hbase_table_name` varchar(255) NOT NULL, - `pre_split_size` integer NOT NULL default 0, - `hbase_table_ttl` integer, - PRIMARY KEY (`id`), - UNIQUE KEY `ux_service_name` (`service_name`), - INDEX `idx_access_token` (`access_token`), - INDEX `idx_cluster` (cluster(75)) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - --- ---------------------------- --- Table structure for `services_columns` --- ---------------------------- -DROP TABLE IF EXISTS `service_columns`; -CREATE TABLE `service_columns` ( - `id` integer NOT NULL AUTO_INCREMENT, - `service_id` integer NOT NULL, - `column_name` varchar(64) NOT NULL, - `column_type` varchar(8) NOT NULL, - `schema_version` varchar(8) NOT NULL default 'v2', - PRIMARY KEY (`id`), - UNIQUE KEY `ux_service_id_column_name` (`service_id`, `column_name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -ALTER TABLE service_columns add FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE; - - --- ---------------------------- --- Table structure for `column_metas` --- ---------------------------- -DROP TABLE IF EXISTS `column_metas`; -CREATE TABLE `column_metas` ( - `id` integer NOT NULL AUTO_INCREMENT, - `column_id` integer NOT NULL, - `name` varchar(64) NOT NULL, - `seq` tinyint NOT NULL, - `data_type` varchar(8) NOT NULL DEFAULT 'string', - PRIMARY KEY (`id`), - UNIQUE KEY `ux_column_id_name` (`column_id`, `name`), - INDEX `idx_column_id_seq` (`column_id`, `seq`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -ALTER TABLE column_metas ADD FOREIGN KEY(column_id) REFERENCES service_columns(id) ON DELETE CASCADE; - --- ---------------------------- --- Table structure for `labels` --- ---------------------------- - -DROP TABLE IF EXISTS `labels`; -CREATE TABLE `labels` ( - `id` integer NOT NULL AUTO_INCREMENT, - `label` varchar(64) NOT NULL, - `src_service_id` integer NOT NULL, - `src_column_name` varchar(64) NOT NULL, - `src_column_type` varchar(8) NOT NULL, - `tgt_service_id` integer NOT NULL, - `tgt_column_name` varchar(64) NOT NULL, - `tgt_column_type` varchar(8) NOT NULL, - `is_directed` tinyint NOT NULL DEFAULT 1, - `service_name` varchar(64), - `service_id` integer NOT NULL, - `consistency_level` varchar(8) NOT NULL DEFAULT 'weak', - `hbase_table_name` varchar(255) NOT NULL DEFAULT 's2graph', - `hbase_table_ttl` integer, - `schema_version` varchar(8) NOT NULL default 'v2', - `is_async` tinyint(4) NOT NULL default '0', - `compressionAlgorithm` varchar(64) NOT NULL DEFAULT 'lz4', - `options` text, - PRIMARY KEY (`id`), - UNIQUE KEY `ux_label` (`label`), - INDEX `idx_src_column_name` (`src_column_name`), - INDEX `idx_tgt_column_name` (`tgt_column_name`), - INDEX `idx_src_service_id` (`src_service_id`), - INDEX `idx_tgt_service_id` (`tgt_service_id`), - INDEX `idx_service_name` (`service_name`), - INDEX `idx_service_id` (`service_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -ALTER TABLE labels add FOREIGN KEY(service_id) REFERENCES services(id); - - - --- ---------------------------- --- Table structure for `label_metas` --- ---------------------------- -DROP TABLE IF EXISTS `label_metas`; -CREATE TABLE `label_metas` ( - `id` integer NOT NULL AUTO_INCREMENT, - `label_id` integer NOT NULL, - `name` varchar(64) NOT NULL, - `seq` tinyint NOT NULL, - `default_value` varchar(64) NOT NULL, - `data_type` varchar(8) NOT NULL DEFAULT 'long', - `used_in_index` tinyint NOT NULL DEFAULT 0, - PRIMARY KEY (`id`), - UNIQUE KEY `ux_label_id_name` (`label_id`, `name`), - INDEX `idx_label_id_seq` (`label_id`, `seq`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -ALTER TABLE label_metas ADD FOREIGN KEY(label_id) REFERENCES labels(id) ON DELETE CASCADE; - - --- ---------------------------- --- Table structure for `label_indices` --- ---------------------------- -DROP TABLE IF EXISTS `label_indices`; -CREATE TABLE `label_indices` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `label_id` int(11) NOT NULL, - `name` varchar(64) NOT NULL DEFAULT '_PK', - `seq` tinyint(4) NOT NULL, - `meta_seqs` varchar(64) NOT NULL, - `formulars` varchar(255) DEFAULT NULL, - PRIMARY KEY (`id`), - UNIQUE KEY `ux_label_id_seq` (`label_id`,`meta_seqs`), - UNIQUE KEY `ux_label_id_name` (`label_id`,`name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -ALTER TABLE label_indices ADD FOREIGN KEY(label_id) REFERENCES labels(id) ON DELETE CASCADE; - - --- ---------------------------- --- Table structure for `experiments` --- ---------------------------- -DROP TABLE IF EXISTS `experiments`; -CREATE TABLE `experiments` ( - `id` integer NOT NULL AUTO_INCREMENT, - `service_id` integer NOT NULL, - `service_name` varchar(128) NOT NULL, - `name` varchar(64) NOT NULL, - `description` varchar(255) NOT NULL, - `experiment_type` varchar(8) NOT NULL DEFAULT 'u', - `total_modular` int NOT NULL DEFAULT 100, - PRIMARY KEY (`id`), - UNIQUE KEY `ux_service_id_name` (`service_id`, `name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - --- ALTER TABLE experiments ADD FOREIGN KEY(service_id) REFERENCES service(id) ON DELETE CASCADE; - - --- ---------------------------- --- Table structure for `buckets` --- ---------------------------- -DROP TABLE IF EXISTS `buckets`; -CREATE TABLE `buckets` ( - `id` integer NOT NULL AUTO_INCREMENT, - `experiment_id` integer NOT NULL, - `modular` varchar(64) NOT NULL, - `http_verb` varchar(8) NOT NULL, - `api_path` text NOT NULL, - `uuid_key` varchar(128), - `uuid_placeholder` varchar(64), - `request_body` text NOT NULL, - `timeout` int NOT NULL DEFAULT 1000, - `impression_id` varchar(64) NOT NULL, - `is_graph_query` tinyint NOT NULL DEFAULT 1, - `is_empty` tinyint NOT NULL DEFAULT 0, - PRIMARY KEY (`id`), - UNIQUE KEY `ux_impression_id` (`impression_id`), - INDEX `idx_experiment_id` (`experiment_id`), - INDEX `idx_impression_id` (`impression_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - -SET FOREIGN_KEY_CHECKS = 1; - - --- ---------------------------- --- Table structure for `counter` --- ---------------------------- -DROP TABLE IF EXISTS `counter`; -CREATE TABLE `counter` ( - `id` int(11) unsigned NOT NULL AUTO_INCREMENT, - `use_flag` tinyint(1) NOT NULL DEFAULT '0', - `version` smallint(1) NOT NULL DEFAULT '1', - `service` varchar(64) NOT NULL DEFAULT '', - `action` varchar(64) NOT NULL DEFAULT '', - `item_type` int(11) NOT NULL DEFAULT '0', - `auto_comb` tinyint(1) NOT NULL DEFAULT '1', - `dimension` varchar(1024) NOT NULL, - `use_profile` tinyint(1) NOT NULL DEFAULT '0', - `bucket_imp_id` varchar(64) DEFAULT NULL, - `use_exact` tinyint(1) NOT NULL DEFAULT '1', - `use_rank` tinyint(1) NOT NULL DEFAULT '1', - `ttl` int(11) NOT NULL DEFAULT '172800', - `daily_ttl` int(11) DEFAULT NULL, - `hbase_table` varchar(1024) DEFAULT NULL, - `interval_unit` varchar(1024) DEFAULT NULL, - `rate_action_id` int(11) unsigned DEFAULT NULL, - `rate_base_id` int(11) unsigned DEFAULT NULL, - `rate_threshold` int(11) DEFAULT NULL, - PRIMARY KEY (`id`), - UNIQUE KEY `svc` (`service`,`action`), - KEY `rate_action_id` (`rate_action_id`), - KEY `rate_base_id` (`rate_base_id`), - CONSTRAINT `rate_action_id` FOREIGN KEY (`rate_action_id`) REFERENCES `counter` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION, - CONSTRAINT `rate_base_id` FOREIGN KEY (`rate_base_id`) REFERENCES `counter` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql ---------------------------------------------------------------------- diff --git a/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql b/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql new file mode 100644 index 0000000..d8ee5bc --- /dev/null +++ b/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql @@ -0,0 +1,227 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- + +SET FOREIGN_KEY_CHECKS = 0; + +-- ---------------------------- +-- Table structure for `services` +-- ---------------------------- +DROP TABLE IF EXISTS `services`; +CREATE TABLE `services` ( + `id` integer NOT NULL AUTO_INCREMENT, + `service_name` varchar(64) NOT NULL, + `access_token` varchar(64) NOT NULL, + `cluster` varchar(255) NOT NULL, + `hbase_table_name` varchar(255) NOT NULL, + `pre_split_size` integer NOT NULL default 0, + `hbase_table_ttl` integer, + PRIMARY KEY (`id`), + UNIQUE KEY `ux_services_service_name` (`service_name`), + INDEX `idx_services_access_token` (`access_token`), + INDEX `idx_services_cluster` (`cluster`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + +-- ---------------------------- +-- Table structure for `services_columns` +-- ---------------------------- +DROP TABLE IF EXISTS `service_columns`; +CREATE TABLE `service_columns` ( + `id` integer NOT NULL AUTO_INCREMENT, + `service_id` integer NOT NULL, + `column_name` varchar(64) NOT NULL, + `column_type` varchar(8) NOT NULL, + `schema_version` varchar(8) NOT NULL default 'v2', + PRIMARY KEY (`id`), + UNIQUE KEY `ux_service_id_column_name` (`service_id`, `column_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +ALTER TABLE service_columns add FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE; + + +-- ---------------------------- +-- Table structure for `column_metas` +-- ---------------------------- +DROP TABLE IF EXISTS `column_metas`; +CREATE TABLE `column_metas` ( + `id` integer NOT NULL AUTO_INCREMENT, + `column_id` integer NOT NULL, + `name` varchar(64) NOT NULL, + `seq` tinyint NOT NULL, + `data_type` varchar(8) NOT NULL DEFAULT 'string', + PRIMARY KEY (`id`), + UNIQUE KEY `ux_column_id_name` (`column_id`, `name`), + INDEX `idx_column_id_seq` (`column_id`, `seq`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +ALTER TABLE column_metas ADD FOREIGN KEY(column_id) REFERENCES service_columns(id) ON DELETE CASCADE; + +-- ---------------------------- +-- Table structure for `labels` +-- ---------------------------- + +DROP TABLE IF EXISTS `labels`; +CREATE TABLE `labels` ( + `id` integer NOT NULL AUTO_INCREMENT, + `label` varchar(64) NOT NULL, + `src_service_id` integer NOT NULL, + `src_column_name` varchar(64) NOT NULL, + `src_column_type` varchar(8) NOT NULL, + `tgt_service_id` integer NOT NULL, + `tgt_column_name` varchar(64) NOT NULL, + `tgt_column_type` varchar(8) NOT NULL, + `is_directed` tinyint NOT NULL DEFAULT 1, + `service_name` varchar(64), + `service_id` integer NOT NULL, + `consistency_level` varchar(8) NOT NULL DEFAULT 'weak', + `hbase_table_name` varchar(255) NOT NULL DEFAULT 's2graph', + `hbase_table_ttl` integer, + `schema_version` varchar(8) NOT NULL default 'v2', + `is_async` tinyint(4) NOT NULL default '0', + `compressionAlgorithm` varchar(64) NOT NULL DEFAULT 'lz4', + `options` text, + PRIMARY KEY (`id`), + UNIQUE KEY `ux_label` (`label`), + INDEX `idx_labels_src_column_name` (`src_column_name`), + INDEX `idx_labels_tgt_column_name` (`tgt_column_name`), + INDEX `idx_labels_src_service_id` (`src_service_id`), + INDEX `idx_labels_tgt_service_id` (`tgt_service_id`), + INDEX `idx_labels_service_name` (`service_name`), + INDEX `idx_labels_service_id` (`service_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +ALTER TABLE labels add FOREIGN KEY(service_id) REFERENCES services(id); + + + +-- ---------------------------- +-- Table structure for `label_metas` +-- ---------------------------- +DROP TABLE IF EXISTS `label_metas`; +CREATE TABLE `label_metas` ( + `id` integer NOT NULL AUTO_INCREMENT, + `label_id` integer NOT NULL, + `name` varchar(64) NOT NULL, + `seq` tinyint NOT NULL, + `default_value` varchar(64) NOT NULL, + `data_type` varchar(8) NOT NULL DEFAULT 'long', + `used_in_index` tinyint NOT NULL DEFAULT 0, + PRIMARY KEY (`id`), + UNIQUE KEY `ux_label_metas_label_id_name` (`label_id`, `name`), + INDEX `idx_label_metas_label_id_seq` (`label_id`, `seq`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +ALTER TABLE label_metas ADD FOREIGN KEY(label_id) REFERENCES labels(id) ON DELETE CASCADE; + + +-- ---------------------------- +-- Table structure for `label_indices` +-- ---------------------------- +DROP TABLE IF EXISTS `label_indices`; +CREATE TABLE `label_indices` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `label_id` int(11) NOT NULL, + `name` varchar(64) NOT NULL DEFAULT '_PK', + `seq` tinyint(4) NOT NULL, + `meta_seqs` varchar(64) NOT NULL, + `formulars` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `ux_label_indices_label_id_seq` (`label_id`,`meta_seqs`), + UNIQUE KEY `ux_label_indices_label_id_name` (`label_id`,`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +ALTER TABLE label_indices ADD FOREIGN KEY(label_id) REFERENCES labels(id) ON DELETE CASCADE; + + +-- ---------------------------- +-- Table structure for `experiments` +-- ---------------------------- +DROP TABLE IF EXISTS `experiments`; +CREATE TABLE `experiments` ( + `id` integer NOT NULL AUTO_INCREMENT, + `service_id` integer NOT NULL, + `service_name` varchar(128) NOT NULL, + `name` varchar(64) NOT NULL, + `description` varchar(255) NOT NULL, + `experiment_type` varchar(8) NOT NULL DEFAULT 'u', + `total_modular` int NOT NULL DEFAULT 100, + PRIMARY KEY (`id`), + UNIQUE KEY `ux_experiments_service_id_name` (`service_id`, `name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- ALTER TABLE experiments ADD FOREIGN KEY(service_id) REFERENCES service(id) ON DELETE CASCADE; + + +-- ---------------------------- +-- Table structure for `buckets` +-- ---------------------------- +DROP TABLE IF EXISTS `buckets`; +CREATE TABLE `buckets` ( + `id` integer NOT NULL AUTO_INCREMENT, + `experiment_id` integer NOT NULL, + `modular` varchar(64) NOT NULL, + `http_verb` varchar(8) NOT NULL, + `api_path` text NOT NULL, + `uuid_key` varchar(128), + `uuid_placeholder` varchar(64), + `request_body` text NOT NULL, + `timeout` int NOT NULL DEFAULT 1000, + `impression_id` varchar(64) NOT NULL, + `is_graph_query` tinyint NOT NULL DEFAULT 1, + `is_empty` tinyint NOT NULL DEFAULT 0, + PRIMARY KEY (`id`), + UNIQUE KEY `ux_buckets_impression_id` (`impression_id`), + INDEX `idx_buckets_experiment_id` (`experiment_id`), + INDEX `idx_buckets_impression_id` (`impression_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +SET FOREIGN_KEY_CHECKS = 1; + + +-- ---------------------------- +-- Table structure for `counter` +-- ---------------------------- +DROP TABLE IF EXISTS `counter`; +CREATE TABLE `counter` ( + `id` int(11) unsigned NOT NULL AUTO_INCREMENT, + `use_flag` tinyint(1) NOT NULL DEFAULT '0', + `version` smallint(1) NOT NULL DEFAULT '1', + `service` varchar(64) NOT NULL DEFAULT '', + `action` varchar(64) NOT NULL DEFAULT '', + `item_type` int(11) NOT NULL DEFAULT '0', + `auto_comb` tinyint(1) NOT NULL DEFAULT '1', + `dimension` varchar(1024) NOT NULL, + `use_profile` tinyint(1) NOT NULL DEFAULT '0', + `bucket_imp_id` varchar(64) DEFAULT NULL, + `use_exact` tinyint(1) NOT NULL DEFAULT '1', + `use_rank` tinyint(1) NOT NULL DEFAULT '1', + `ttl` int(11) NOT NULL DEFAULT '172800', + `daily_ttl` int(11) DEFAULT NULL, + `hbase_table` varchar(1024) DEFAULT NULL, + `interval_unit` varchar(1024) DEFAULT NULL, + `rate_action_id` int(11) unsigned DEFAULT NULL, + `rate_base_id` int(11) unsigned DEFAULT NULL, + `rate_threshold` int(11) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `counter_svc` (`service`,`action`), + KEY `counter_rate_action_id` (`rate_action_id`), + KEY `counter_rate_base_id` (`rate_base_id`), + CONSTRAINT `counter_rate_action_id` FOREIGN KEY (`rate_action_id`) REFERENCES `counter` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION, + CONSTRAINT `counter_rate_base_id` FOREIGN KEY (`rate_base_id`) REFERENCES `counter` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2core/src/main/resources/org/apache/s2graph/core/mysqls/setup.sql ---------------------------------------------------------------------- diff --git a/s2core/src/main/resources/org/apache/s2graph/core/mysqls/setup.sql b/s2core/src/main/resources/org/apache/s2graph/core/mysqls/setup.sql new file mode 100644 index 0000000..1bda27c --- /dev/null +++ b/s2core/src/main/resources/org/apache/s2graph/core/mysqls/setup.sql @@ -0,0 +1,28 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- + +CREATE DATABASE IF NOT EXISTS graph_dev; + +CREATE USER 'graph'@'%' IDENTIFIED BY 'graph'; + +GRANT ALL PRIVILEGES ON graph_dev.* TO 'graph'@'%' identified by 'graph'; + +flush privileges; + +use graph_dev; http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Model.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Model.scala b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Model.scala index 7439939..bf6dfb7 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Model.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Model.scala @@ -26,8 +26,9 @@ import org.apache.s2graph.core.utils.{SafeUpdateCache, logger} import scalikejdbc._ import scala.concurrent.ExecutionContext +import scala.io.Source import scala.language.{higherKinds, implicitConversions} -import scala.util.{Failure, Try} +import scala.util.{Success, Failure, Try} object Model { var maxSize = 10000 @@ -52,11 +53,42 @@ object Model { config.getString("db.default.user"), config.getString("db.default.password"), settings) + + checkSchema() + } + + def checkSchema(): Unit = { + withTx { implicit session => + sql"""show tables""".map(rs => rs.string(1)).list.apply() + } match { + case Success(tables) => + if (tables.isEmpty) { + // this is a very simple migration tool that only supports creating + // appropriate tables when there are no tables in the database at all. + // Ideally, it should be improved to a sophisticated migration tool + // that supports versioning, etc. + logger.info("Creating tables ...") + val schema = getClass.getResourceAsStream("schema.sql") + val lines = Source.fromInputStream(schema, "UTF-8").getLines + val sources = lines.map(_.split("-- ").head.trim).mkString("\n") + val statements = sources.split(";\n") + withTx { implicit session => + statements.foreach(sql => session.execute(sql)) + } match { + case Success(_) => + logger.info("Successfully imported schema") + case Failure(e) => + throw new RuntimeException("Error while importing schema", e) + } + } + case Failure(e) => + throw new RuntimeException("Could not list tables in the database", e) + } } def withTx[T](block: DBSession => T): Try[T] = { using(DB(ConnectionPool.borrow())) { conn => - val res = Try { + Try { conn.begin() val session = conn.withinTxSession() val result = block(session) @@ -69,8 +101,6 @@ object Model { conn.rollbackIfActive() Failure(e) } - - res } } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2counter_core/build.sbt ---------------------------------------------------------------------- diff --git a/s2counter_core/build.sbt b/s2counter_core/build.sbt index 4a49bd3..d4c0bdf 100644 --- a/s2counter_core/build.sbt +++ b/s2counter_core/build.sbt @@ -29,9 +29,13 @@ libraryDependencies ++= Seq( "com.typesafe.play" %% "play-json" % Common.playVersion, "com.typesafe.play" %% "play-ws" % Common.playVersion, "com.typesafe.akka" %% "akka-actor" % "2.3.4", + "com.google.guava" % "guava" % "12.0.1" force(), // use this old version of guava to avoid incompatibility "org.apache.hbase" % "hbase-client" % Common.hbaseVersion, "org.apache.hbase" % "hbase-common" % Common.hbaseVersion, + "org.apache.hbase" % "hbase-hadoop-compat" % Common.hbaseVersion, + "org.apache.hbase" % "hbase-hadoop2-compat" % Common.hbaseVersion, "org.apache.hadoop" % "hadoop-common" % Common.hadoopVersion, + "org.apache.hadoop" % "hadoop-hdfs" % Common.hadoopVersion, "redis.clients" % "jedis" % "2.6.0", "org.apache.kafka" % "kafka-clients" % "0.8.2.0", "mysql" % "mysql-connector-java" % "5.1.28", http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2counter_loader/build.sbt ---------------------------------------------------------------------- diff --git a/s2counter_loader/build.sbt b/s2counter_loader/build.sbt index 879f2c1..27f08f5 100644 --- a/s2counter_loader/build.sbt +++ b/s2counter_loader/build.sbt @@ -26,6 +26,7 @@ version := "0.12.1-SNAPSHOT" scalacOptions in Test ++= Seq("-Yrangepos") libraryDependencies ++= Seq( + "com.google.guava" % "guava" % "12.0.1" force(), // use this old version of guava to avoid incompatibility "org.apache.spark" %% "spark-core" % Common.sparkVersion % "provided", "org.apache.spark" %% "spark-streaming" % Common.sparkVersion % "provided", "org.apache.spark" %% "spark-streaming-kafka" % Common.sparkVersion, http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2rest_netty/build.sbt ---------------------------------------------------------------------- diff --git a/s2rest_netty/build.sbt b/s2rest_netty/build.sbt index c810482..b7c5484 100644 --- a/s2rest_netty/build.sbt +++ b/s2rest_netty/build.sbt @@ -22,5 +22,6 @@ name := "s2rest_netty" enablePlugins(JavaAppPackaging) libraryDependencies ++= Seq( + "com.google.guava" % "guava" % "12.0.1" force(), // use this old version of guava to avoid incompatibility "io.netty" % "netty-all" % "4.0.33.Final" ) http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2rest_netty/conf/reference.conf ---------------------------------------------------------------------- diff --git a/s2rest_netty/conf/reference.conf b/s2rest_netty/conf/reference.conf index a50e7b2..9b37eda 100644 --- a/s2rest_netty/conf/reference.conf +++ b/s2rest_netty/conf/reference.conf @@ -131,7 +131,7 @@ hbase.fail.prob=-1.0 # max allowd edges for deleteAll is multiply of above two configuration. # set global obejct package, TODO: remove global -application.global=com.kakao.s2graph.rest.Global +#application.global=com.kakao.s2graph.rest.Global akka { loggers = ["akka.event.slf4j.Slf4jLogger"] http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2rest_netty/src/main/resources/reference.conf ---------------------------------------------------------------------- diff --git a/s2rest_netty/src/main/resources/reference.conf b/s2rest_netty/src/main/resources/reference.conf index a50e7b2..9b37eda 100644 --- a/s2rest_netty/src/main/resources/reference.conf +++ b/s2rest_netty/src/main/resources/reference.conf @@ -131,7 +131,7 @@ hbase.fail.prob=-1.0 # max allowd edges for deleteAll is multiply of above two configuration. # set global obejct package, TODO: remove global -application.global=com.kakao.s2graph.rest.Global +#application.global=com.kakao.s2graph.rest.Global akka { loggers = ["akka.event.slf4j.Slf4jLogger"] http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2rest_play/build.sbt ---------------------------------------------------------------------- diff --git a/s2rest_play/build.sbt b/s2rest_play/build.sbt index 27922f0..6edb8e8 100644 --- a/s2rest_play/build.sbt +++ b/s2rest_play/build.sbt @@ -24,6 +24,7 @@ scalacOptions in Test ++= Seq("-Yrangepos") libraryDependencies ++= Seq( ws, filters, + "com.google.guava" % "guava" % "12.0.1" force(), // use this old version of guava to avoid incompatibility "com.github.danielwegener" % "logback-kafka-appender" % "0.0.3" ) http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/spark/build.sbt ---------------------------------------------------------------------- diff --git a/spark/build.sbt b/spark/build.sbt index 2c2a389..f5412d8 100644 --- a/spark/build.sbt +++ b/spark/build.sbt @@ -22,6 +22,7 @@ name := "s2spark" scalacOptions ++= Seq("-deprecation", "-feature") libraryDependencies ++= Seq( + "com.google.guava" % "guava" % "12.0.1" force(), // use this old version of guava to avoid incompatibility "org.apache.spark" %% "spark-core" % Common.sparkVersion % "provided", "org.apache.spark" %% "spark-streaming" % Common.sparkVersion % "provided", "org.apache.spark" %% "spark-streaming-kafka" % Common.sparkVersion,
