Repository: incubator-samza Updated Branches: refs/heads/0.8.0 263d92178 -> 8f65fa428
SAMZA-450; allow log4j.xml to be overridden Project: http://git-wip-us.apache.org/repos/asf/incubator-samza/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-samza/commit/8f65fa42 Tree: http://git-wip-us.apache.org/repos/asf/incubator-samza/tree/8f65fa42 Diff: http://git-wip-us.apache.org/repos/asf/incubator-samza/diff/8f65fa42 Branch: refs/heads/0.8.0 Commit: 8f65fa428a85e0a5a23054012ba1ecd785c7de03 Parents: 263d921 Author: Chris Riccomini <[email protected]> Authored: Thu Oct 30 12:16:08 2014 -0700 Committer: Chris Riccomini <[email protected]> Committed: Thu Oct 30 12:16:08 2014 -0700 ---------------------------------------------------------------------- .../samza/job/local/ThreadJobFactory.scala | 2 +- samza-shell/src/main/bash/checkpoint-tool.sh | 5 ++--- samza-shell/src/main/bash/kill-yarn-job.sh | 5 ++--- samza-shell/src/main/bash/run-am.sh | 2 +- samza-shell/src/main/bash/run-class.sh | 20 +++++++++++++------- samza-shell/src/main/bash/run-container.sh | 2 +- samza-shell/src/main/bash/run-job.sh | 5 ++--- samza-shell/src/main/bash/run-migration-tool.sh | 5 ++--- 8 files changed, 24 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/8f65fa42/samza-core/src/main/scala/org/apache/samza/job/local/ThreadJobFactory.scala ---------------------------------------------------------------------- diff --git a/samza-core/src/main/scala/org/apache/samza/job/local/ThreadJobFactory.scala b/samza-core/src/main/scala/org/apache/samza/job/local/ThreadJobFactory.scala index 766df76..cc46d38 100644 --- a/samza-core/src/main/scala/org/apache/samza/job/local/ThreadJobFactory.scala +++ b/samza-core/src/main/scala/org/apache/samza/job/local/ThreadJobFactory.scala @@ -54,7 +54,7 @@ class ThreadJobFactory extends StreamJobFactory with Logging { // Give developers a nice friendly warning if they've specified task.opts and are using a threaded job. config.getTaskOpts match { - case Some(taskOpts) => warn("%s was specified in config, but is not being used because job is being executed with ThreadJob. You probably want to run %s=%s." format (TASK_JVM_OPTS, STREAM_JOB_FACTORY_CLASS, classOf[ThreadJobFactory].getName)) + case Some(taskOpts) => warn("%s was specified in config, but is not being used because job is being executed with ThreadJob. You probably want to run %s=%s." format (TASK_JVM_OPTS, STREAM_JOB_FACTORY_CLASS, classOf[ProcessJobFactory].getName)) case _ => None } http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/8f65fa42/samza-shell/src/main/bash/checkpoint-tool.sh ---------------------------------------------------------------------- diff --git a/samza-shell/src/main/bash/checkpoint-tool.sh b/samza-shell/src/main/bash/checkpoint-tool.sh index 5688b59..ccf7db2 100755 --- a/samza-shell/src/main/bash/checkpoint-tool.sh +++ b/samza-shell/src/main/bash/checkpoint-tool.sh @@ -16,8 +16,7 @@ # specific language governing permissions and limitations # under the License. -if [ -z "$SAMZA_LOG4J_CONFIG" ]; then - export SAMZA_LOG4J_CONFIG=file:$(dirname $0)/log4j-console.xml -fi +# Override log4j.xml to point to log4j-console.xml +[[ $JAVA_OPTS != *-Dlog4j.configuration* ]] && export JAVA_OPTS="$JAVA_OPTS -Dlog4j.configuration=file:$(dirname $0)/log4j-console.xml" exec $(dirname $0)/run-class.sh org.apache.samza.checkpoint.CheckpointTool $@ http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/8f65fa42/samza-shell/src/main/bash/kill-yarn-job.sh ---------------------------------------------------------------------- diff --git a/samza-shell/src/main/bash/kill-yarn-job.sh b/samza-shell/src/main/bash/kill-yarn-job.sh index d42e086..d2bc37f 100755 --- a/samza-shell/src/main/bash/kill-yarn-job.sh +++ b/samza-shell/src/main/bash/kill-yarn-job.sh @@ -16,8 +16,7 @@ # specific language governing permissions and limitations # under the License. -if [ -z "$SAMZA_LOG4J_CONFIG" ]; then - export SAMZA_LOG4J_CONFIG=file:$(dirname $0)/log4j-console.xml -fi +# Override log4j.xml to point to log4j-console.xml +[[ $JAVA_OPTS != *-Dlog4j.configuration* ]] && export JAVA_OPTS="$JAVA_OPTS -Dlog4j.configuration=file:$(dirname $0)/log4j-console.xml" exec $(dirname $0)/run-class.sh org.apache.hadoop.yarn.client.cli.ApplicationCLI application -kill $@ http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/8f65fa42/samza-shell/src/main/bash/run-am.sh ---------------------------------------------------------------------- diff --git a/samza-shell/src/main/bash/run-am.sh b/samza-shell/src/main/bash/run-am.sh index 3a6d40d..22b0e73 100755 --- a/samza-shell/src/main/bash/run-am.sh +++ b/samza-shell/src/main/bash/run-am.sh @@ -17,6 +17,6 @@ # under the License. # Check if server is set. If not - set server optimization -[[ $JAVA_OPTS != *-server* ]] && JAVA_OPTS="$JAVA_OPTS -server" +[[ $JAVA_OPTS != *-server* ]] && export JAVA_OPTS="$JAVA_OPTS -server" exec $(dirname $0)/run-class.sh org.apache.samza.job.yarn.SamzaAppMaster $@ http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/8f65fa42/samza-shell/src/main/bash/run-class.sh ---------------------------------------------------------------------- diff --git a/samza-shell/src/main/bash/run-class.sh b/samza-shell/src/main/bash/run-class.sh index 2395b59..3f7c49a 100755 --- a/samza-shell/src/main/bash/run-class.sh +++ b/samza-shell/src/main/bash/run-class.sh @@ -37,6 +37,7 @@ HADOOP_YARN_HOME="${HADOOP_YARN_HOME:-$HOME/.samza}" HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_YARN_HOME/conf}" CLASSPATH=$HADOOP_CONF_DIR GC_LOG_ROTATION_OPTS="-XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10241024" +DEFAULT_LOG4J_FILE=$base_dir/lib/log4j.xml for file in $base_dir/lib/*.[jw]ar; do @@ -77,13 +78,20 @@ function check_and_enable_64_bit_mode { fi } -# Initialize SAMZA_LOG4J_CONFIG if currently undefined -if [ -z "$SAMZA_LOG4J_CONFIG" ]; then - export SAMZA_LOG4J_CONFIG=file:$base_dir/lib/log4j.xml -fi - ### Inherit JVM_OPTS from task.opts configuration, and initialize defaults ### +# Check if log4j configuration is specified. If not - set to lib/log4j.xml +[[ $JAVA_OPTS != *-Dlog4j.configuration* && -f $DEFAULT_LOG4J_FILE ]] && JAVA_OPTS="$JAVA_OPTS -Dlog4j.configuration=file:$DEFAULT_LOG4J_FILE" + +# Check if samza.log.dir is specified. If not - set to environment variable if it is set +[[ $JAVA_OPTS != *-Dsamza.log.dir* && ! -z "$SAMZA_LOG_DIR" ]] && JAVA_OPTS="$JAVA_OPTS -Dsamza.log.dir=$SAMZA_LOG_DIR" + +# Check if samza.container.name is specified. If not - set to environment variable if it is set +[[ $JAVA_OPTS != *-Dsamza.container.name* && ! -z "$SAMZA_CONTAINER_NAME" ]] && JAVA_OPTS="$JAVA_OPTS -Dsamza.container.name=$SAMZA_CONTAINER_NAME" + +# Check if java.io.tmpdir is specified. If not - set to tmp in the base_dir +[[ $JAVA_OPTS != *-Djava.io.tmpdir* ]] && JAVA_OPTS="$JAVA_OPTS -Djava.io.tmpdir=$JAVA_TEMP_DIR" + # Check if a max-heap size is specified. If not - set a 768M heap [[ $JAVA_OPTS != *-Xmx* ]] && JAVA_OPTS="$JAVA_OPTS -Xmx768M" @@ -96,7 +104,5 @@ fi # Check if 64 bit is set. If not - try and set it if it's supported [[ $JAVA_OPTS != *-d64* ]] && check_and_enable_64_bit_mode -JAVA_OPTS="$JAVA_OPTS -Dlog4j.configuration=$SAMZA_LOG4J_CONFIG -Dsamza.log.dir=$SAMZA_LOG_DIR -Dsamza.container.name=$SAMZA_CONTAINER_NAME -Djava.io.tmpdir=$JAVA_TEMP_DIR" - echo $JAVA $JAVA_OPTS -cp $CLASSPATH $@ exec $JAVA $JAVA_OPTS -cp $CLASSPATH $@ http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/8f65fa42/samza-shell/src/main/bash/run-container.sh ---------------------------------------------------------------------- diff --git a/samza-shell/src/main/bash/run-container.sh b/samza-shell/src/main/bash/run-container.sh index af00eba..366bd79 100755 --- a/samza-shell/src/main/bash/run-container.sh +++ b/samza-shell/src/main/bash/run-container.sh @@ -17,6 +17,6 @@ # under the License. # Check if server is set. If not - set server optimization -[[ $JAVA_OPTS != *-server* ]] && JAVA_OPTS="$JAVA_OPTS -server" +[[ $JAVA_OPTS != *-server* ]] && export JAVA_OPTS="$JAVA_OPTS -server" exec $(dirname $0)/run-class.sh org.apache.samza.container.SamzaContainer $@ http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/8f65fa42/samza-shell/src/main/bash/run-job.sh ---------------------------------------------------------------------- diff --git a/samza-shell/src/main/bash/run-job.sh b/samza-shell/src/main/bash/run-job.sh index d43b556..346731e 100755 --- a/samza-shell/src/main/bash/run-job.sh +++ b/samza-shell/src/main/bash/run-job.sh @@ -16,8 +16,7 @@ # specific language governing permissions and limitations # under the License. -if [ -z "$SAMZA_LOG4J_CONFIG" ]; then - export SAMZA_LOG4J_CONFIG=file:$(dirname $0)/log4j-console.xml -fi +# Override log4j.xml to point to log4j-console.xml +[[ $JAVA_OPTS != *-Dlog4j.configuration* ]] && export JAVA_OPTS="$JAVA_OPTS -Dlog4j.configuration=file:$(dirname $0)/log4j-console.xml" exec $(dirname $0)/run-class.sh org.apache.samza.job.JobRunner $@ http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/8f65fa42/samza-shell/src/main/bash/run-migration-tool.sh ---------------------------------------------------------------------- diff --git a/samza-shell/src/main/bash/run-migration-tool.sh b/samza-shell/src/main/bash/run-migration-tool.sh index 7ac7f27..ae7042b 100644 --- a/samza-shell/src/main/bash/run-migration-tool.sh +++ b/samza-shell/src/main/bash/run-migration-tool.sh @@ -16,8 +16,7 @@ # specific language governing permissions and limitations # under the License. -if [ -z "$SAMZA_LOG4J_CONFIG" ]; then - export SAMZA_LOG4J_CONFIG=file:$(dirname $0)/log4j-console.xml -fi +# Override log4j.xml to point to log4j-console.xml +[[ $JAVA_OPTS != *-Dlog4j.configuration* ]] && export JAVA_OPTS="$JAVA_OPTS -Dlog4j.configuration=file:$(dirname $0)/log4j-console.xml" exec $(dirname $0)/run-class.sh org.apache.samza.util.CheckpointMigrationTool $@
