add example
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/c97986f3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/c97986f3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/c97986f3 Branch: refs/heads/master Commit: c97986f32dbbe05e6487a328ce766228ed5f33ea Parents: cbc23ec Author: Chul Kang <[email protected]> Authored: Wed Apr 11 02:52:40 2018 +0900 Committer: Chul Kang <[email protected]> Committed: Wed Apr 11 02:58:15 2018 +0900 ---------------------------------------------------------------------- example/common.sh | 54 +++++++++++ example/create_schema.sh | 33 +++++++ example/import_data.sh | 24 +++++ example/movielens/desc.txt | 8 ++ example/movielens/generate_input.sh | 11 +++ example/movielens/jobdesc.template | 106 +++++++++++++++++++++ example/movielens/schema/edge.rated.graphql | 42 ++++++++ example/movielens/schema/edge.related.graphql | 40 ++++++++ example/movielens/schema/edge.tagged.graphql | 37 +++++++ example/movielens/schema/service.graphql | 14 +++ example/movielens/schema/vertex.movie.graphql | 30 ++++++ example/movielens/schema/vertex.tag.graphql | 16 ++++ example/movielens/schema/vertex.user.graphql | 16 ++++ example/prepare.sh | 41 ++++++++ example/run.sh | 27 ++++++ 15 files changed, 499 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c97986f3/example/common.sh ---------------------------------------------------------------------- diff --git a/example/common.sh b/example/common.sh new file mode 100644 index 0000000..6b5b755 --- /dev/null +++ b/example/common.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +ROOTDIR=`pwd`/.. +WORKDIR=`pwd` +PREFIX="[s2graph] " +REST="localhost:8000" + +msg() { + LEVEL=$1 + MSG=$2 + TIME=`date +"%Y-%m-%d %H:%M:%S"` + echo "${PREFIX} $LEVEL $MSG" +} + +q() { + MSG=$1 + TIME=`date +"%Y-%m-%d %H:%M:%S"` + echo "" + read -r -p "${PREFIX} >>> $MSG " var +} + +info() { MSG=$1; msg "" "$MSG";} +warn() { MSG=$1; msg "[WARN] " "$MSG";} +error() { MSG=$1; msg "[ERROR] " "$MSG"; exit -1;} + +usage() { + echo "Usage: $0 [SERVICE_NAME]" + exit -1 +} + +graphql_rest() { + file=$1 + value=`cat ${file} | sed 's/\"/\\\"/g'` + query=$(echo $value|tr -d '\n') + + echo $query + curl -i -XPOST $REST/graphql -H 'content-type: application/json' -d " + { + \"query\": \"$query\", + \"variables\": null + }" + sleep 5 +} +get_services() { + curl -i -XPOST $REST/graphql -H 'content-type: application/json' -d ' + { + "query": "query{Management{Services{id name }}}" + }' +} +get_labels() { + curl -i -XPOST $REST/graphql -H 'content-type: application/json' -d ' + { + "query": "query{Management{Labels {id name}}}" + }' +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c97986f3/example/create_schema.sh ---------------------------------------------------------------------- diff --git a/example/create_schema.sh b/example/create_schema.sh new file mode 100755 index 0000000..9f511cb --- /dev/null +++ b/example/create_schema.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +source common.sh + +[ $# -ne 1 ] && { usage; } + +SERVICE=$1 +SERVICE_HOME=${WORKDIR}/${SERVICE} +SCHEMA_HOME=${SERVICE_HOME}/schema + +info "schema dir : $SCHEMA_HOME" + +q "generate input >>> " +cd ${SERVICE_HOME} +./generate_input.sh + +q "create service >>> " +graphql_rest ${SCHEMA_HOME}/service.graphql +get_services + +q "create vertices >>>" +for file in `ls ${SCHEMA_HOME}/vertex.*`; do + info "" + info "file: $file" + graphql_rest $file +done + +q "create edges >>> " +for file in `ls ${SCHEMA_HOME}/edge.*`; do + info "" + info "file: $file" + graphql_rest $file +done +get_labels http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c97986f3/example/import_data.sh ---------------------------------------------------------------------- diff --git a/example/import_data.sh b/example/import_data.sh new file mode 100755 index 0000000..6822c00 --- /dev/null +++ b/example/import_data.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +source common.sh + +[ $# -ne 1 ] && { usage; } + +SERVICE=$1 +JOBDESC_TPL=${WORKDIR}/${SERVICE}/jobdesc.template +JOBDESC=${WORKDIR}/${SERVICE}/jobdesc.json +WORKING_DIR=`pwd` +JAR=`ls ${ROOTDIR}/s2jobs/target/scala-2.11/s2jobs-assembly*.jar` + +info "WORKING_DIR : $WORKING_DIR" +info "JAR : $JAR" + +sed -e "s/\[=WORKING_DIR\]/${WORKING_DIR//\//\\/}/g" $JOBDESC_TPL > $JOBDESC + +unset HADOOP_CONF_DIR +info "spark submit.." +${SPARK_HOME}/bin/spark-submit \ + --class org.apache.s2graph.s2jobs.JobLauncher \ + --master local[2] \ + --driver-class-path h2-1.4.192.jar \ + $JAR file -f $JOBDESC -n SAMPLEJOB:$SERVICE + http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c97986f3/example/movielens/desc.txt ---------------------------------------------------------------------- diff --git a/example/movielens/desc.txt b/example/movielens/desc.txt new file mode 100644 index 0000000..0da9a86 --- /dev/null +++ b/example/movielens/desc.txt @@ -0,0 +1,8 @@ +MovieLens +GroupLens Research has collected and made available rating data sets from the MovieLens web site (http://movielens.org). +The data sets were collected over various periods of time, depending on the size of the set. + +This dataset (ml-latest-small) describes 5-star rating and free-text tagging activity from MovieLens, a movie recommendation service. +It contains 100004 ratings and 1296 tag applications across 9125 movies. +These data were created by 671 users between January 09, 1995 and October 16, 2016. +This dataset was generated on October 17, 2016. http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c97986f3/example/movielens/generate_input.sh ---------------------------------------------------------------------- diff --git a/example/movielens/generate_input.sh b/example/movielens/generate_input.sh new file mode 100755 index 0000000..b28640e --- /dev/null +++ b/example/movielens/generate_input.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +INPUT=input +mkdir -p ${INPUT} +rm -rf ${INPUT}/* +cd ${INPUT} +wget http://files.grouplens.org/datasets/movielens/ml-latest-small.zip +unzip ml-latest-small.zip +rm -f ml-latest-small.zip +mv ml-latest-small/* . +rmdir ml-latest-small http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c97986f3/example/movielens/jobdesc.template ---------------------------------------------------------------------- diff --git a/example/movielens/jobdesc.template b/example/movielens/jobdesc.template new file mode 100644 index 0000000..ed62383 --- /dev/null +++ b/example/movielens/jobdesc.template @@ -0,0 +1,106 @@ +{ + "name": "import_movielens", + "source": [ + { + "name": "movies", + "inputs": [], + "type": "file", + "options": { + "paths": "file://[=WORKING_DIR]/movielens/input/movies.csv", + "format": "csv", + "columns": "movieId,title,genres" + } + }, + { + "name": "ratings", + "inputs": [], + "type": "file", + "options": { + "paths": "file://[=WORKING_DIR]/movielens/input/ratings.csv", + "format": "csv", + "columns": "userId,movieId,rating,timestamp" + } + }, + { + "name": "tags", + "inputs": [], + "type": "file", + "options": { + "paths": "file://[=WORKING_DIR]/movielens/input/tags.csv", + "format": "csv", + "columns": "userId,movieId,tag,timestamp" + } + } + ], + "process": [ + { + "name": "vertex_movie", + "inputs": [ + "movies" + ], + "type": "sql", + "options": { + "sql": "SELECT \n(unix_timestamp() * 1000) as timestamp, \n'v' as elem, \nCAST(movieId AS LONG) AS id, \n'movielens' as service, \n'Movie' as column, \nto_json(\nnamed_struct(\n 'title', title, \n 'genres', genres\n)\n) as props \nFROM movies \nWHERE movieId != 'movieId'" + } + }, + { + "name": "edge_rated", + "inputs": [ + "ratings" + ], + "type": "sql", + "options": { + "sql": "SELECT \nCAST(timestamp AS LONG) * 1000 AS timestamp, \n'e' as elem, \nCAST(userId AS LONG) as `from`, \nCAST(movieId AS LONG) as to, \n'rated' as label, \nto_json(\nnamed_struct(\n 'score', CAST(rating as float)\n)\n) as props \nFROM ratings \nWHERE userId != 'userId'" + } + }, + { + "name": "edge_tagged", + "inputs": [ + "tags" + ], + "type": "sql", + "options": { + "sql": "SELECT \nCAST(timestamp AS LONG) * 1000 AS timestamp, \n'e' as elem, \nCAST(userId AS LONG) as `from`, \nCAST(movieId AS LONG) as to, \n'tagged' as label, \nto_json(\nnamed_struct('tag', tag)\n) as props \nFROM tags \nWHERE userId != 'userId'" + } + }, + { + "name": "edges", + "inputs": [ + "edge_rated", + "edge_tagged" + ], + "type": "sql", + "options": { + "sql": "SELECT * FROM edge_rated UNION SELECT * FROM edge_tagged" + } + } + ], + "sink": [ + { + "name": "vertex_sink", + "inputs": [ + "vertex_movie" + ], + "type": "s2graph", + "options": { + "db.default.driver":"org.h2.Driver", + "db.default.url": "jdbc:h2:tcp://localhost/./var/metastore;MODE=MYSQL", + "s2.spark.sql.streaming.sink.grouped.size": "10", + "s2.spark.sql.streaming.sink.wait.time": "10" + } + }, + { + "name": "edge_sink", + "inputs": [ + "edges" + ], + "type": "s2graph", + "options": { + "db.default.driver":"org.h2.Driver", + "db.default.url": "jdbc:h2:tcp://localhost/./var/metastore;MODE=MYSQL", + "s2.spark.sql.streaming.sink.grouped.size": "10", + "s2.spark.sql.streaming.sink.wait.time": "10" + } + } + ] +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c97986f3/example/movielens/schema/edge.rated.graphql ---------------------------------------------------------------------- diff --git a/example/movielens/schema/edge.rated.graphql b/example/movielens/schema/edge.rated.graphql new file mode 100644 index 0000000..6e3c40f --- /dev/null +++ b/example/movielens/schema/edge.rated.graphql @@ -0,0 +1,42 @@ +mutation{ + Management{ + createLabel( + name:"rated" + sourceService: { + movielens: { + columnName: User + } + } + targetService: { + movielens: { + columnName: Movie + } + } + serviceName: movielens + consistencyLevel: strong + props:[ + { + name: "score" + dataType: double + defaultValue: "0.0" + storeInGlobalIndex: true + } + ] + indices:{ + name:"_PK" + propNames:["score"] + } + ) { + isSuccess + message + object{ + id + name + props{ + name + } + } + } + } +} + http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c97986f3/example/movielens/schema/edge.related.graphql ---------------------------------------------------------------------- diff --git a/example/movielens/schema/edge.related.graphql b/example/movielens/schema/edge.related.graphql new file mode 100644 index 0000000..baf6e54 --- /dev/null +++ b/example/movielens/schema/edge.related.graphql @@ -0,0 +1,40 @@ +mutation{ + Management{ + createLabel( + name:"related" + sourceService: { + movielens: { + columnName: Movie + } + } + targetService: { + movielens: { + columnName: Tag + } + } + serviceName: movielens + consistencyLevel: weak + props:{ + name: "relevance" + dataType: double + defaultValue: "0.0" + storeInGlobalIndex: true + } + indices:{ + name:"_PK" + propNames:["relevance"] + } + + ) { + isSuccess + message + object{ + id + name + props{ + name + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c97986f3/example/movielens/schema/edge.tagged.graphql ---------------------------------------------------------------------- diff --git a/example/movielens/schema/edge.tagged.graphql b/example/movielens/schema/edge.tagged.graphql new file mode 100644 index 0000000..da5e997 --- /dev/null +++ b/example/movielens/schema/edge.tagged.graphql @@ -0,0 +1,37 @@ +mutation{ + Management{ + createLabel( + name:"tagged" + sourceService: { + movielens: { + columnName: User + } + } + targetService: { + movielens: { + columnName: Movie + } + } + serviceName: movielens + consistencyLevel: weak + props:[ + { + name: "tag" + dataType: string + defaultValue: "" + storeInGlobalIndex: true + } + ] + ) { + isSuccess + message + object{ + id + name + props{ + name + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c97986f3/example/movielens/schema/service.graphql ---------------------------------------------------------------------- diff --git a/example/movielens/schema/service.graphql b/example/movielens/schema/service.graphql new file mode 100644 index 0000000..ed7e157 --- /dev/null +++ b/example/movielens/schema/service.graphql @@ -0,0 +1,14 @@ +mutation{ + Management{ + createService( + name:"movielens" + ){ + isSuccess + message + object{ + id + name + } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c97986f3/example/movielens/schema/vertex.movie.graphql ---------------------------------------------------------------------- diff --git a/example/movielens/schema/vertex.movie.graphql b/example/movielens/schema/vertex.movie.graphql new file mode 100644 index 0000000..3702a1e --- /dev/null +++ b/example/movielens/schema/vertex.movie.graphql @@ -0,0 +1,30 @@ +mutation{ + Management{ + createServiceColumn( + serviceName:movielens + columnName:"Movie" + columnType: long + props: [ + { + name: "title" + dataType: string + defaultValue: "" + storeInGlobalIndex: true + }, + { + name: "genres" + dataType: string + defaultValue: "" + storeInGlobalIndex: true + } + ] + ){ + isSuccess + message + object{ + id + name + } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c97986f3/example/movielens/schema/vertex.tag.graphql ---------------------------------------------------------------------- diff --git a/example/movielens/schema/vertex.tag.graphql b/example/movielens/schema/vertex.tag.graphql new file mode 100644 index 0000000..05bd11c --- /dev/null +++ b/example/movielens/schema/vertex.tag.graphql @@ -0,0 +1,16 @@ +mutation{ + Management{ + createServiceColumn( + serviceName:movielens + columnName:"Tag" + columnType: string + ){ + isSuccess + message + object{ + id + name + } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c97986f3/example/movielens/schema/vertex.user.graphql ---------------------------------------------------------------------- diff --git a/example/movielens/schema/vertex.user.graphql b/example/movielens/schema/vertex.user.graphql new file mode 100644 index 0000000..bba8a16 --- /dev/null +++ b/example/movielens/schema/vertex.user.graphql @@ -0,0 +1,16 @@ +mutation{ + Management{ + createServiceColumn( + serviceName:movielens + columnName:"User" + columnType: long + ){ + isSuccess + message + object{ + id + name + } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c97986f3/example/prepare.sh ---------------------------------------------------------------------- diff --git a/example/prepare.sh b/example/prepare.sh new file mode 100755 index 0000000..f35c94c --- /dev/null +++ b/example/prepare.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +source common.sh + +q "1. s2graphql is running?" +status=`curl -s -o /dev/null -w "%{http_code}" "$REST"` +if [ $status != '200' ]; then + warn "s2graphql not running.. " + + cd $ROOTDIR + output="$(ls target/apache-s2graph-*-incubating-bin)" + if [ -z "$output" ]; then + info "build package..." + sbt clean package + fi + + info "now we will launch s2graphql using build scripts" + cd target/apache-s2graph-*-incubating-bin + ./bin/start-s2graph.sh s2graphql +else + info "s2graphql is running!!" +fi + +q "2. s2jobs assembly jar exists?" +jar=`ls $ROOTDIR/s2jobs/target/scala-2.11/s2jobs-assembly*.jar` +if [ -z $jar ]; then + warn "s2jobs assembly not exists.." + info "start s2jobs assembly now" + cd $ROOTDIR + sbt 'project s2jobs' clean assembly +else + info "s2jobs assembly exists!!" +fi + +q "3. SPARK_HOME is set?" +if [ -z $SPARK_HOME ]; then + error "it must be setting SPARK_HOME environment variable" +else + info "SPARK_HOME exists!! (${SPARK_HOME})" +fi + +info "prepare finished.." http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/c97986f3/example/run.sh ---------------------------------------------------------------------- diff --git a/example/run.sh b/example/run.sh new file mode 100755 index 0000000..08f2989 --- /dev/null +++ b/example/run.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +source common.sh + +SERVICE="movielens" +[ $# -gt 0 ] && { SERVICE=$1; } +DESC=`cat $SERVICE/desc.txt` + +info "" +info "" +info "Let's try to create the toy project '$SERVICE' using s2graphql and s2jobs." +info "" +while IFS='' read -r line || [[ -n "$line" ]]; do + info "$line" +done < "$SERVICE/desc.txt" + +q "First of all, we will check prerequisites" +./prepare.sh $SERVICE +[ $? -ne 0 ] && { exit -1; } + +q "And now, we create vertex and edge schema using graphql" +./create_schema.sh $SERVICE +[ $? -ne 0 ] && { exit -1; } + +q "Finally, we import example data to service" +./import_data.sh $SERVICE +[ $? -ne 0 ] && { exit -1; } +
