[
https://issues.apache.org/jira/browse/FLINK-9777?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16546519#comment-16546519
]
ASF GitHub Bot commented on FLINK-9777:
---------------------------------------
Github user dawidwys commented on a diff in the pull request:
https://github.com/apache/flink/pull/6297#discussion_r203000806
--- Diff:
flink-yarn/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java ---
@@ -352,6 +352,76 @@ public void testConfigurationClusterSpecification()
throws Exception {
assertThat(clusterSpecification.getSlotsPerTaskManager(),
is(slotsPerTaskManager));
}
+ /**
+ * Tests the specifying heap memory for job manager and task manager.
+ */
+ @Test
+ public void testHeapMemoryProperty() throws Exception {
+ //without unit
+ String[] args = new String[] { "-yn", "2", "-yjm", "1024",
"-ytm", "2048" };
+
+ FlinkYarnSessionCli flinkYarnSessionCli = new
FlinkYarnSessionCli(
+ new Configuration(),
+ tmp.getRoot().getAbsolutePath(),
+ "y",
+ "yarn");
+
+ CommandLine commandLine =
flinkYarnSessionCli.parseCommandLineOptions(args, false);
+
+ ClusterSpecification clusterSpecification =
flinkYarnSessionCli.getClusterSpecification(commandLine);
+
+ assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
+ assertThat(clusterSpecification.getTaskManagerMemoryMB(),
is(2048));
+
+ //with unit "m"
+ args = new String[] { "-yn", "2", "-yjm", "1024m", "-ytm",
"2048m" };
+ commandLine = flinkYarnSessionCli.parseCommandLineOptions(args,
false);
+ clusterSpecification =
flinkYarnSessionCli.getClusterSpecification(commandLine);
+
+ assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
+ assertThat(clusterSpecification.getTaskManagerMemoryMB(),
is(2048));
+
+ //with unit non "m"
+ args = new String[] { "-yn", "2", "-yjm", "1g", "-ytm", "2g" };
+ commandLine = flinkYarnSessionCli.parseCommandLineOptions(args,
false);
+ clusterSpecification =
flinkYarnSessionCli.getClusterSpecification(commandLine);
+
+ assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
+ assertThat(clusterSpecification.getTaskManagerMemoryMB(),
is(2048));
+
+ //specify old config key
+ Configuration configuration = new Configuration();
+
configuration.setInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY_MB, 2048);
+
configuration.setInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY_MB, 4096);
+
+ flinkYarnSessionCli = new FlinkYarnSessionCli(
+ configuration,
+ tmp.getRoot().getAbsolutePath(),
+ "y",
+ "yarn");
+
+ commandLine = flinkYarnSessionCli.parseCommandLineOptions(new
String[0], false);
+
+ clusterSpecification =
flinkYarnSessionCli.getClusterSpecification(commandLine);
+
+ assertThat(clusterSpecification.getMasterMemoryMB(), is(2048));
+ assertThat(clusterSpecification.getTaskManagerMemoryMB(),
is(4096));
+
+ //set nothing use default value
--- End diff --
This should be a separate test
> YARN: JM and TM Memory must be specified with Units
> ----------------------------------------------------
>
> Key: FLINK-9777
> URL: https://issues.apache.org/jira/browse/FLINK-9777
> Project: Flink
> Issue Type: Bug
> Components: Documentation, YARN
> Affects Versions: 1.6.0
> Environment: commit 9f736d1927c62d220a82931c4f5ffa4955910f27
> Reporter: Gary Yao
> Assignee: vinoyang
> Priority: Blocker
> Labels: pull-request-available
> Fix For: 1.6.0
>
>
> FLINK-6469 breaks backwards compatibility because the JobManager and
> TaskManager memory must be specified with units (otherwise bytes are
> assumed). The command to start a YARN session as documented
> ([https://github.com/apache/flink/blob/9f736d1927c62d220a82931c4f5ffa4955910f27/docs/ops/deployment/yarn_setup.md)
>
> |https://github.com/apache/flink/blob/9f736d1927c62d220a82931c4f5ffa4955910f27/docs/ops/deployment/yarn_setup.md]
> would not work because 1024 bytes and 4096 bytes are not enough for the heap
> size. The command finishes with the following exception:
> {noformat}
> java.lang.reflect.UndeclaredThrowableException
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1854)
> at
> org.apache.flink.runtime.security.HadoopSecurityContext.runSecured(HadoopSecurityContext.java:41)
> at
> org.apache.flink.yarn.cli.FlinkYarnSessionCli.main(FlinkYarnSessionCli.java:802)
> Caused by: org.apache.flink.client.deployment.ClusterDeploymentException:
> Couldn't deploy Yarn session cluster
> at
> org.apache.flink.yarn.AbstractYarnClusterDescriptor.deploySessionCluster(AbstractYarnClusterDescriptor.java:420)
> at
> org.apache.flink.yarn.cli.FlinkYarnSessionCli.run(FlinkYarnSessionCli.java:599)
> at
> org.apache.flink.yarn.cli.FlinkYarnSessionCli.lambda$main$2(FlinkYarnSessionCli.java:802)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1836)
> ... 2 more
> Caused by: org.apache.flink.util.FlinkException: Cannot fulfill the minimum
> memory requirements with the provided cluster specification. Please increase
> the memory of the cluster.
> at
> org.apache.flink.yarn.AbstractYarnClusterDescriptor.validateClusterSpecification(AbstractYarnClusterDescriptor.java:453)
> at
> org.apache.flink.yarn.AbstractYarnClusterDescriptor.deployInternal(AbstractYarnClusterDescriptor.java:475)
> at
> org.apache.flink.yarn.AbstractYarnClusterDescriptor.deploySessionCluster(AbstractYarnClusterDescriptor.java:413)
> ... 7 more
> Caused by: java.lang.IllegalArgumentException
> at
> org.apache.flink.util.Preconditions.checkArgument(Preconditions.java:123)
> at
> org.apache.flink.runtime.clusterframework.ContaineredTaskManagerParameters.calculateCutoffMB(ContaineredTaskManagerParameters.java:115)
> at
> org.apache.flink.yarn.AbstractYarnClusterDescriptor.validateClusterSpecification(AbstractYarnClusterDescriptor.java:450)
> ... 9 more
> {noformat}
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
