Updated Branches: refs/heads/master 8028d8deb -> 65fbc6866
SAMZA-76; switch SAMZA_OPTS to JAVA_OPTS, and make default container size 1G with an Xmx of 768. Project: http://git-wip-us.apache.org/repos/asf/incubator-samza/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-samza/commit/65fbc686 Tree: http://git-wip-us.apache.org/repos/asf/incubator-samza/tree/65fbc686 Diff: http://git-wip-us.apache.org/repos/asf/incubator-samza/diff/65fbc686 Branch: refs/heads/master Commit: 65fbc6866e7fd1403a64d3048ac2f0e628f4d0b7 Parents: 8028d8d Author: Chris Riccomini <[email protected]> Authored: Tue Nov 26 08:16:29 2013 -0800 Committer: Chris Riccomini <[email protected]> Committed: Tue Nov 26 08:16:29 2013 -0800 ---------------------------------------------------------------------- .../documentation/0.7.0/jobs/configuration-table.html | 7 ++++++- .../org/apache/samza/config/ShellCommandConfig.scala | 2 +- .../org/apache/samza/job/ShellCommandBuilder.scala | 2 +- samza-shell/src/main/bash/run-class.sh | 10 +++++----- .../samza/job/yarn/SamzaAppMasterTaskManager.scala | 2 +- .../main/scala/org/apache/samza/job/yarn/YarnJob.scala | 13 ++++++++++--- 6 files changed, 24 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/65fbc686/docs/learn/documentation/0.7.0/jobs/configuration-table.html ---------------------------------------------------------------------- diff --git a/docs/learn/documentation/0.7.0/jobs/configuration-table.html b/docs/learn/documentation/0.7.0/jobs/configuration-table.html index 41353b2..7232e02 100644 --- a/docs/learn/documentation/0.7.0/jobs/configuration-table.html +++ b/docs/learn/documentation/0.7.0/jobs/configuration-table.html @@ -185,10 +185,15 @@ </tr> <tr> <td>yarn.container.memory.mb</td> - <td>768</td> + <td>1024</td> <td>How much memory to ask for (per-container), when Samza is starting a YARN container.</td> </tr> <tr> + <td>yarn.am.container.memory.mb</td> + <td>1024</td> + <td>How much memory to ask for (per-application-master), when Samza is starting a YARN container.</td> + </tr> + <tr> <td>yarn.container.count</td> <td>1</td> <td>How many containers to start when a Samza job is started in YARN. Partitions are divided evenly among the containers.</td> http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/65fbc686/samza-core/src/main/scala/org/apache/samza/config/ShellCommandConfig.scala ---------------------------------------------------------------------- diff --git a/samza-core/src/main/scala/org/apache/samza/config/ShellCommandConfig.scala b/samza-core/src/main/scala/org/apache/samza/config/ShellCommandConfig.scala index 27ba5e4..b6efe46 100644 --- a/samza-core/src/main/scala/org/apache/samza/config/ShellCommandConfig.scala +++ b/samza-core/src/main/scala/org/apache/samza/config/ShellCommandConfig.scala @@ -38,7 +38,7 @@ object ShellCommandConfig { /** * Arguments to be passed to the processing running the TaskRunner (or equivalent, for non JVM languages). */ - val ENV_SAMZA_OPTS = "SAMZA_OPTS" + val ENV_JAVA_OPTS = "JAVA_OPTS" val COMMAND_SHELL_EXECUTE = "task.execute" val TASK_JVM_OPTS = "task.opts" http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/65fbc686/samza-core/src/main/scala/org/apache/samza/job/ShellCommandBuilder.scala ---------------------------------------------------------------------- diff --git a/samza-core/src/main/scala/org/apache/samza/job/ShellCommandBuilder.scala b/samza-core/src/main/scala/org/apache/samza/job/ShellCommandBuilder.scala index 09b2d22..c67e46d 100644 --- a/samza-core/src/main/scala/org/apache/samza/job/ShellCommandBuilder.scala +++ b/samza-core/src/main/scala/org/apache/samza/job/ShellCommandBuilder.scala @@ -34,6 +34,6 @@ class ShellCommandBuilder extends CommandBuilder { ShellCommandConfig.ENV_CONTAINER_NAME -> name, ShellCommandConfig.ENV_PARTITION_IDS -> parts, ShellCommandConfig.ENV_CONFIG -> JsonConfigSerializer.toJson(config), - ShellCommandConfig.ENV_SAMZA_OPTS -> config.getTaskOpts.getOrElse("")) + ShellCommandConfig.ENV_JAVA_OPTS -> config.getTaskOpts.getOrElse("")) } } http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/65fbc686/samza-shell/src/main/bash/run-class.sh ---------------------------------------------------------------------- diff --git a/samza-shell/src/main/bash/run-class.sh b/samza-shell/src/main/bash/run-class.sh index b593765..f4124e5 100755 --- a/samza-shell/src/main/bash/run-class.sh +++ b/samza-shell/src/main/bash/run-class.sh @@ -54,12 +54,12 @@ if [ -z "$SAMZA_CONTAINER_NAME" ]; then SAMZA_CONTAINER_NAME="undefined-samza-container-name" fi -if [ -z "$SAMZA_OPTS" ]; then - SAMZA_OPTS="-Xmx160M -XX:+PrintGCDateStamps -Xloggc:$SAMZA_LOG_DIR/gc.log -Dsamza.log.dir=$SAMZA_LOG_DIR -Dsamza.container.name=$SAMZA_CONTAINER_NAME" +if [ -z "$JAVA_OPTS" ]; then + JAVA_OPTS="-Xmx768M -XX:+PrintGCDateStamps -Xloggc:$SAMZA_LOG_DIR/gc.log -Dsamza.log.dir=$SAMZA_LOG_DIR -Dsamza.container.name=$SAMZA_CONTAINER_NAME" if [ -f $base_dir/lib/log4j.xml ]; then - SAMZA_OPTS="$SAMZA_OPTS -Dlog4j.configuration=file:$base_dir/lib/log4j.xml" + JAVA_OPTS="$JAVA_OPTS -Dlog4j.configuration=file:$base_dir/lib/log4j.xml" fi fi -echo $JAVA $SAMZA_OPTS -cp $CLASSPATH $@ -exec $JAVA $SAMZA_OPTS -cp $CLASSPATH $@ +echo $JAVA $JAVA_OPTS -cp $CLASSPATH $@ +exec $JAVA $JAVA_OPTS -cp $CLASSPATH $@ http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/65fbc686/samza-yarn/src/main/scala/org/apache/samza/job/yarn/SamzaAppMasterTaskManager.scala ---------------------------------------------------------------------- diff --git a/samza-yarn/src/main/scala/org/apache/samza/job/yarn/SamzaAppMasterTaskManager.scala b/samza-yarn/src/main/scala/org/apache/samza/job/yarn/SamzaAppMasterTaskManager.scala index 80d2972..9f4db17 100644 --- a/samza-yarn/src/main/scala/org/apache/samza/job/yarn/SamzaAppMasterTaskManager.scala +++ b/samza-yarn/src/main/scala/org/apache/samza/job/yarn/SamzaAppMasterTaskManager.scala @@ -65,7 +65,7 @@ import org.apache.hadoop.yarn.client.api.NMClient import org.apache.hadoop.yarn.client.api.impl.NMClientImpl object SamzaAppMasterTaskManager { - val DEFAULT_CONTAINER_MEM = 256 + val DEFAULT_CONTAINER_MEM = 1024 val DEFAULT_CPU_CORES = 1 val DEFAULT_CONTAINER_RETRY_COUNT = 8 val DEFAULT_CONTAINER_RETRY_WINDOW_MS = 300000 http://git-wip-us.apache.org/repos/asf/incubator-samza/blob/65fbc686/samza-yarn/src/main/scala/org/apache/samza/job/yarn/YarnJob.scala ---------------------------------------------------------------------- diff --git a/samza-yarn/src/main/scala/org/apache/samza/job/yarn/YarnJob.scala b/samza-yarn/src/main/scala/org/apache/samza/job/yarn/YarnJob.scala index b79c0aa..34af73b 100644 --- a/samza-yarn/src/main/scala/org/apache/samza/job/yarn/YarnJob.scala +++ b/samza-yarn/src/main/scala/org/apache/samza/job/yarn/YarnJob.scala @@ -37,17 +37,24 @@ import org.apache.samza.config.YarnConfig import org.apache.samza.config.ShellCommandConfig import org.apache.samza.SamzaException + +object YarnJob { + val DEFAULT_AM_CONTAINER_MEM = 1024 +} + /** * Starts the application manager */ class YarnJob(config: Config, hadoopConfig: Configuration) extends StreamJob { + import YarnJob._ + val client = new ClientHelper(hadoopConfig) var appId: Option[ApplicationId] = None - + def submit: YarnJob = { appId = client.submitApplication( new Path(config.getPackagePath.getOrElse(throw new SamzaException("No YARN package path defined in config."))), - config.getAMContainerMaxMemoryMb.getOrElse(512), + config.getAMContainerMaxMemoryMb.getOrElse(DEFAULT_AM_CONTAINER_MEM), 1, List( "export SAMZA_LOG_DIR=%s && ln -sfn %s logs && exec ./__package/bin/run-am.sh 1>logs/%s 2>logs/%s" @@ -55,7 +62,7 @@ class YarnJob(config: Config, hadoopConfig: Configuration) extends StreamJob { Some(Map( ShellCommandConfig.ENV_CONFIG -> Util.envVarEscape(JsonConfigSerializer.toJson(config)), ShellCommandConfig.ENV_CONTAINER_NAME -> Util.envVarEscape("application-master"), - ShellCommandConfig.ENV_SAMZA_OPTS -> Util.envVarEscape(config.getAmOpts.getOrElse("")))), + ShellCommandConfig.ENV_JAVA_OPTS -> Util.envVarEscape(config.getAmOpts.getOrElse("")))), Some("%s_%s" format (config.getName.get, config.getJobId.getOrElse(1)))) this
