Repository: samza-hello-samza Updated Branches: refs/heads/latest 95933714b -> 28d1ca175
SAMZA-604: Provide build.gradle script for hello-samza project Project: http://git-wip-us.apache.org/repos/asf/samza-hello-samza/repo Commit: http://git-wip-us.apache.org/repos/asf/samza-hello-samza/commit/3be8dbd2 Tree: http://git-wip-us.apache.org/repos/asf/samza-hello-samza/tree/3be8dbd2 Diff: http://git-wip-us.apache.org/repos/asf/samza-hello-samza/diff/3be8dbd2 Branch: refs/heads/latest Commit: 3be8dbd2f5996a5ce6c31fb8422715e56488ac2e Parents: 9593371 Author: Ken Gidley <[email protected]> Authored: Fri Apr 24 16:45:46 2015 -0700 Committer: Yi Pan (Data Infrastructure) <[email protected]> Committed: Mon Dec 21 16:59:44 2015 -0800 ---------------------------------------------------------------------- .gitignore | 2 + README-gradle.txt | 51 +++++++ build.gradle | 207 ++++++++++++++++++++++++++ gradle.properties | 25 ++++ gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 52141 bytes gradle/wrapper/gradle-wrapper.properties | 6 + gradlew | 164 ++++++++++++++++++++ gradlew.bat | 90 +++++++++++ pom.xml | 4 + 9 files changed, 549 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/samza-hello-samza/blob/3be8dbd2/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 0435c14..849ce6a 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,5 @@ target/ */.cache deploy *.swp +build/ +.gradle/ http://git-wip-us.apache.org/repos/asf/samza-hello-samza/blob/3be8dbd2/README-gradle.txt ---------------------------------------------------------------------- diff --git a/README-gradle.txt b/README-gradle.txt new file mode 100644 index 0000000..a564427 --- /dev/null +++ b/README-gradle.txt @@ -0,0 +1,51 @@ + +To use gradle to build/run the hello-samza project: + +1) the project is configured to download and use gradle version 2.3 - on first task execution, it will download the required gradle jars. + +2) download/install yarn/kafka/zookeeper: + + $ ./gradlew installGrid + +3) build hello-samza job package: + + $ ./gradlew distTar + +4) deploy hello-samza project to grid: + + $ ./gradlew deployHelloSamza + +5) start the grid (starts up yarn/kafka/zookeeper): + + $ ./gradlew startGrid + +6) run the various Samza tasks that are part of hello-samza project: + + $ ./gradlew runWikiFeed + $ ./gradlew runWikiParser + $ ./gradlew runWikiStats + +7) view all the current Kafka topics: + + $ ./gradlew listKafkaTopics + +8) view the Kafka topics output by the various Samza tasks: + + $ ./gradlew dumpWikiRaw + ( output of Kafka topic scrolls by) + CTRL-c + + $ ./gradlew dumpWikiEdits + ( output of Kafka topic scrolls by) + CTRL-c + + $ ./gradlew dumpWikiStats + ( output of Kafka topic scrolls by) + CTRL-c + +9) stop all the components: + + $ ./gradlew stopGrid + +Shortcut: using the 'runWiki*' tasks directly will do steps 3-6 automatically. + http://git-wip-us.apache.org/repos/asf/samza-hello-samza/blob/3be8dbd2/build.gradle ---------------------------------------------------------------------- diff --git a/build.gradle b/build.gradle new file mode 100644 index 0000000..cbbd2b3 --- /dev/null +++ b/build.gradle @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +apply plugin: 'eclipse' +apply plugin: 'java' + +defaultTasks 'distTar' + +task wrapper(type: Wrapper) { + description = "Updates gradlew and supporting files." + gradleVersion = '2.3' +} + +version = "$SAMZA_VERSION" + +repositories { + mavenCentral() + maven { url "https://repository.apache.org/content/groups/public"; } +} + +// a configuration for dependencies that need exploding into package +configurations { + explode +} + +dependencies { + compile(group: 'org.codehaus.jackson', name: 'jackson-jaxrs', version: '1.8.5') + compile(group: 'org.slf4j', name: 'slf4j-api', version: "$SLF4J_VERSION") + compile(group: 'org.slf4j', name: 'slf4j-log4j12', version: "$SLF4J_VERSION") + compile(group: 'org.schwering', name: 'irclib', version: '1.10') + compile(group: 'org.apache.samza', name: 'samza-api', version: "$SAMZA_VERSION") + compile(group: 'org.apache.samza', name: 'samza-kv_2.10', version: "$SAMZA_VERSION") + + explode (group: 'org.apache.samza', name: 'samza-shell', ext: 'tgz', classifier: 'dist', version: "$SAMZA_VERSION") + + runtime(group: 'org.apache.samza', name: 'samza-core_2.10', version: "$SAMZA_VERSION") + runtime(group: 'org.apache.samza', name: 'samza-log4j', version: "$SAMZA_VERSION") + runtime(group: 'org.apache.samza', name: 'samza-shell', version: "$SAMZA_VERSION") + runtime(group: 'org.apache.samza', name: 'samza-yarn_2.10', version: "$SAMZA_VERSION") + runtime(group: 'org.apache.samza', name: 'samza-kv-rocksdb_2.10', version: "$SAMZA_VERSION") + runtime(group: 'org.apache.samza', name: 'samza-kafka_2.10', version: "$SAMZA_VERSION") + runtime(group: 'org.apache.kafka', name: 'kafka_2.10', version: "$KAFKA_VERSION") + runtime(group: 'org.apache.hadoop', name: 'hadoop-hdfs', version: "$HADOOP_VERSION") +} + +// make the samza distribution .tgz file +task distTar(dependsOn: build, type: Tar) { + destinationDir(new File(project.buildDir, "/distributions")) + compression(Compression.GZIP) + classifier('dist') + extension('tar.gz') + into("config") { + from("src/main/config") { + include "wikipedia-feed.properties" + include "wikipedia-parser.properties" + include "wikipedia-stats.properties" + + // expand the Maven tokens with Gradle equivalents. Also change 'target' (Maven) to 'build/distributions' (Gradle) + filter { String line -> + line.replaceAll('[\$][{]basedir[}]', project.projectDir.toString()).replaceAll('[\$][{]project.artifactId[}]', project.name.toString()).replaceAll('/target/', '/build/distributions/').replaceAll('[\$][{]pom.version[}]', version) + } + } + } + + into("bin") { + from { + configurations.explode.collect { tarTree(it) } + } + } + + into("lib") { + from configurations.runtime + from configurations.runtime.artifacts.files + from("src/main/resources/") { + include "log4j.xml" + } + } +} + +// install everything +task installGrid(type: Exec) { + workingDir(project.projectDir) + commandLine("bin/grid", "install", "all") + outputs.upToDateWhen { + ["kafka", "zookeeper", "yarn"].every { + (new File(project.projectDir, "deploy/" + it)).exists() + } + } +} + +// update the Samza job +task deployHelloSamza(dependsOn: [distTar, installGrid], type: Sync) { + into(new File(project.projectDir, "/deploy/samza")) + from(tarTree(distTar.archivePath)) +} + + +// run everything +task startGrid(type: Exec) { + workingDir(project.projectDir) + commandLine("bin/grid", "start", "all") + outputs.upToDateWhen { + // use running zookeeper as proxy + File zookeeperPidFile = new File("/tmp/zookeeper/zookeeper_server.pid") + zookeeperPidFile.exists() && + "kill -0 ${zookeeperPidFile.text}".execute().waitFor() == 0 + } +} + +// stop everything +task stopGrid(type: Exec) { + workingDir(project.projectDir) + commandLine("bin/grid", "stop", "all") +} + +// +// Samza helpers +// + +// helper task to run Samza jobs +class SamzaTask extends DefaultTask { + String configFile; + + @TaskAction + def startSamza() { + project.exec { + workingDir(project.projectDir) + commandLine("deploy/samza/bin/run-job.sh", + "--config-factory=org.apache.samza.config.factories.PropertiesConfigFactory", + "--config-path=file://${project.projectDir}/deploy/samza/config/${configFile}") + } + } +} + +// helper to run the Samza job to retrieve Wikipedia data +task runWikiFeed(dependsOn: [startGrid, deployHelloSamza], type: SamzaTask) { + configFile("wikipedia-feed.properties") +} + +// helper to run the Samza job process Wikipedia data +task runWikiParser(dependsOn: [startGrid, deployHelloSamza], type: SamzaTask) { + configFile("wikipedia-parser.properties") +} + +// helper to run the Samza job to summarize stats on Wikipedia edits +task runWikiStats(dependsOn: [startGrid, deployHelloSamza], type: SamzaTask) { + configFile("wikipedia-stats.properties") +} + + +// +// Kafka helpers +// + +// show all Kafka topics +task listKafkaTopics(type: Exec) { + workingDir(project.projectDir) + commandLine("deploy/kafka/bin/kafka-topics.sh", + "--zookeeper", "localhost:2181", + "--list") +} + +// helper task to monitor a Kafka topic +class KafkaDumpTask extends DefaultTask { + String topic; + + @TaskAction + def dumpTopic() { + project.exec { + workingDir(project.projectDir) + commandLine("deploy/kafka/bin/kafka-console-consumer.sh", + "--zookeeper", "localhost:2181", + "--topic", "${topic}") + } + } +} + +// helper to dump the wikipedia-raw topic +task dumpWikiRaw(dependsOn: startGrid, type: KafkaDumpTask) { + topic("wikipedia-raw") +} + +// helper to dump the wikipedia-edits topic +task dumpWikiEdits(dependsOn: startGrid, type: KafkaDumpTask) { + topic("wikipedia-edits") +} + +// helper to dump the wikipedia-stats topic +task dumpWikiStats(dependsOn: startGrid, type: KafkaDumpTask) { + topic("wikipedia-stats") +} http://git-wip-us.apache.org/repos/asf/samza-hello-samza/blob/3be8dbd2/gradle.properties ---------------------------------------------------------------------- diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 0000000..83b150f --- /dev/null +++ b/gradle.properties @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +SAMZA_VERSION=0.9.0 +KAFKA_VERSION=0.8.2.1 +HADOOP_VERSION=2.4.0 + +SLF4J_VERSION = 1.7.7 + http://git-wip-us.apache.org/repos/asf/samza-hello-samza/blob/3be8dbd2/gradle/wrapper/gradle-wrapper.jar ---------------------------------------------------------------------- diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..085a1cd Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ http://git-wip-us.apache.org/repos/asf/samza-hello-samza/blob/3be8dbd2/gradle/wrapper/gradle-wrapper.properties ---------------------------------------------------------------------- diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..3584fa0 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Mon Mar 23 14:55:28 PDT 2015 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-2.3-bin.zip http://git-wip-us.apache.org/repos/asf/samza-hello-samza/blob/3be8dbd2/gradlew ---------------------------------------------------------------------- diff --git a/gradlew b/gradlew new file mode 100755 index 0000000..91a7e26 --- /dev/null +++ b/gradlew @@ -0,0 +1,164 @@ +#!/usr/bin/env bash + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn ( ) { + echo "$*" +} + +die ( ) { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; +esac + +# For Cygwin, ensure paths are in UNIX format before anything is touched. +if $cygwin ; then + [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"` +fi + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >&- +APP_HOME="`pwd -P`" +cd "$SAVED" >&- + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules +function splitJvmOpts() { + JVM_OPTS=("$@") +} +eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS +JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" + +exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" http://git-wip-us.apache.org/repos/asf/samza-hello-samza/blob/3be8dbd2/gradlew.bat ---------------------------------------------------------------------- diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000..aec9973 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,90 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windowz variants + +if not "%OS%" == "Windows_NT" goto win9xME_args +if "%@eval[2+2]" == "4" goto 4NT_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* +goto execute + +:4NT_args +@rem Get arguments from the 4NT Shell from JP Software +set CMD_LINE_ARGS=%$ + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega http://git-wip-us.apache.org/repos/asf/samza-hello-samza/blob/3be8dbd2/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index c1d552f..f8cd7f7 100644 --- a/pom.xml +++ b/pom.xml @@ -189,6 +189,10 @@ under the License. <exclude>**/.cache/**</exclude> <exclude>deploy/**</exclude> <exclude>**/.project</exclude> + <exclude>**/.gradle/**</exclude> + <exclude>**/gradle/**</exclude> + <exclude>**/gradlew*</exclude> + <exclude>**/build/**</exclude> </excludes> </configuration> </plugin>
