Repository: samza Updated Branches: refs/heads/master 01a1795e5 -> 1d2f054ab
SAMZA-1794: setting application acl in launch context Currently we don't set application acl for container launch context. See https://hadoop.apache.org/docs/r2.6.4/api/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.html#setApplicationACLs(java.util.Map) This could potentially cause problem if samza job is running on a secured YARN cluster. Say user A submits the job, then by default only user A can view the log and the status of the job. Even worse case is that user A submits the job through some proxy account, then even user A herself/himself couldn't access to logs/status of the application. We need to make some changes for the YARN application submission to set application acls in launch context as configured. Author: Hai Lu <[email protected]> Reviewers: Jagadish<[email protected]> Closes #592 from lhaiesp/master Project: http://git-wip-us.apache.org/repos/asf/samza/repo Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/1d2f054a Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/1d2f054a Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/1d2f054a Branch: refs/heads/master Commit: 1d2f054ab0c250f33137c11b36c61549549bb0c2 Parents: 01a1795 Author: Hai Lu <[email protected]> Authored: Wed Aug 1 15:20:10 2018 -0700 Committer: Jagadish <[email protected]> Committed: Wed Aug 1 15:20:10 2018 -0700 ---------------------------------------------------------------------- .../versioned/jobs/configuration-table.html | 20 +++++++++ .../org/apache/samza/config/YarnConfig.java | 46 ++++++++++++++++++++ .../job/yarn/YarnClusterResourceManager.java | 7 +++ .../apache/samza/job/yarn/ClientHelper.scala | 7 ++- 4 files changed, 79 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/samza/blob/1d2f054a/docs/learn/documentation/versioned/jobs/configuration-table.html ---------------------------------------------------------------------- diff --git a/docs/learn/documentation/versioned/jobs/configuration-table.html b/docs/learn/documentation/versioned/jobs/configuration-table.html index 8557480..26b4661 100644 --- a/docs/learn/documentation/versioned/jobs/configuration-table.html +++ b/docs/learn/documentation/versioned/jobs/configuration-table.html @@ -2106,6 +2106,26 @@ </tr> <tr> + <td class="property" id="yarn-job-view-acl">yarn.job.view.acl</td> + <td class="default"></td> + <td class="description"> + This is for secured YARN cluster only. + The 'viewing' acl of the YARN application that controls who can view the application (e.g. application status, logs). + See <a href="https://hadoop.apache.org/docs/r2.6.0/api/org/apache/hadoop/yarn/api/records/ApplicationAccessType.html";>ApplicationAccessType</a> for more details. + </td> + </tr> + + <tr> + <td class="property" id="yarn-job-modify-acl">yarn.job.modify.acl</td> + <td class="default"></td> + <td class="description"> + This is for secured YARN cluster only. + The 'modify' acl of the YARN application that controls who can modify the application (e.g. killing the application). + See <a href="https://hadoop.apache.org/docs/r2.6.0/api/org/apache/hadoop/yarn/api/records/ApplicationAccessType.html";>ApplicationAccessType</a> for more details. + </td> + </tr> + + <tr> <td class="property" id="yarn-token-renewal-interval-seconds">yarn.token.renewal.interval.seconds</td> <td class="default"></td> <td class="description"> http://git-wip-us.apache.org/repos/asf/samza/blob/1d2f054a/samza-yarn/src/main/java/org/apache/samza/config/YarnConfig.java ---------------------------------------------------------------------- diff --git a/samza-yarn/src/main/java/org/apache/samza/config/YarnConfig.java b/samza-yarn/src/main/java/org/apache/samza/config/YarnConfig.java index 466b8cf..58adf26 100644 --- a/samza-yarn/src/main/java/org/apache/samza/config/YarnConfig.java +++ b/samza-yarn/src/main/java/org/apache/samza/config/YarnConfig.java @@ -19,6 +19,10 @@ package org.apache.samza.config; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.samza.SamzaException; public class YarnConfig extends MapConfig { @@ -115,6 +119,22 @@ public class YarnConfig extends MapConfig { */ public static final String YARN_JOB_STAGING_DIRECTORY = "yarn.job.staging.directory"; + /** + * For secured YARN cluster only. + * The 'viewing' acl of the YARN application. This controls who can view the application, + * for example, application status, logs. + * {@link org.apache.hadoop.yarn.api.records.ApplicationAccessType} for more details + */ + public static final String YARN_APPLICATION_VIEW_ACL = "yarn.job.view.acl"; + + /** + * For secured YARN cluster only. + * The 'modify' acl of the YARN application. This controls who can modify the application, + * for example, killing the job. + * {@link org.apache.hadoop.yarn.api.records.ApplicationAccessType} for more details + */ + public static final String YARN_APPLICATION_MODIFY_ACL = "yarn.job.modify.acl"; + public YarnConfig(Config config) { super(config); } @@ -190,4 +210,30 @@ public class YarnConfig extends MapConfig { public String getYarnJobStagingDirectory() { return get(YARN_JOB_STAGING_DIRECTORY, null); } + + public String getYarnApplicationViewAcl() { + return get(YARN_APPLICATION_VIEW_ACL, null); + } + + public String getYarnApplicationModifyAcl() { + return get(YARN_APPLICATION_MODIFY_ACL, null); + } + + /** + * Helper function to get all application acls + * @return a map of {@link ApplicationAccessType} to {@link String} for all the acls defined + */ + public Map<ApplicationAccessType, String> getYarnApplicationAcls() { + Map<ApplicationAccessType, String> acls = new HashMap<>(); + String viewAcl = getYarnApplicationViewAcl(); + String modifyAcl = getYarnApplicationModifyAcl(); + if (viewAcl != null) { + acls.put(ApplicationAccessType.VIEW_APP, viewAcl); + } + if (modifyAcl != null) { + acls.put(ApplicationAccessType.MODIFY_APP, modifyAcl); + } + return acls; + } + } http://git-wip-us.apache.org/repos/asf/samza/blob/1d2f054a/samza-yarn/src/main/java/org/apache/samza/job/yarn/YarnClusterResourceManager.java ---------------------------------------------------------------------- diff --git a/samza-yarn/src/main/java/org/apache/samza/job/yarn/YarnClusterResourceManager.java b/samza-yarn/src/main/java/org/apache/samza/job/yarn/YarnClusterResourceManager.java index 79a9083..6f175ea 100644 --- a/samza-yarn/src/main/java/org/apache/samza/job/yarn/YarnClusterResourceManager.java +++ b/samza-yarn/src/main/java/org/apache/samza/job/yarn/YarnClusterResourceManager.java @@ -651,6 +651,13 @@ public class YarnClusterResourceManager extends ClusterResourceManager implement context.setCommands(new ArrayList<String>() {{add(cmd);}}); context.setLocalResources(localResourceMap); + if (UserGroupInformation.isSecurityEnabled()) { + Map<ApplicationAccessType, String> acls = yarnConfig.getYarnApplicationAcls(); + if (!acls.isEmpty()) { + context.setApplicationACLs(acls); + } + } + log.debug("Setting localResourceMap to {}", localResourceMap); log.debug("Setting context to {}", context); http://git-wip-us.apache.org/repos/asf/samza/blob/1d2f054a/samza-yarn/src/main/scala/org/apache/samza/job/yarn/ClientHelper.scala ---------------------------------------------------------------------- diff --git a/samza-yarn/src/main/scala/org/apache/samza/job/yarn/ClientHelper.scala b/samza-yarn/src/main/scala/org/apache/samza/job/yarn/ClientHelper.scala index d193ddb..6f65ec4 100644 --- a/samza-yarn/src/main/scala/org/apache/samza/job/yarn/ClientHelper.scala +++ b/samza-yarn/src/main/scala/org/apache/samza/job/yarn/ClientHelper.scala @@ -191,12 +191,17 @@ class ClientHelper(conf: Configuration) extends Logging { } } - if (UserGroupInformation.isSecurityEnabled()) { + if (UserGroupInformation.isSecurityEnabled) { validateJobConfig(config) setupSecurityToken(fs, containerCtx) info("set security token for %s" format appId.get) + val acls = yarnConfig.getYarnApplicationAcls + if (!acls.isEmpty) { + containerCtx.setApplicationACLs(acls) + } + val amLocalResources = setupAMLocalResources(fs, Option(yarnConfig.getYarnKerberosPrincipal), Option(yarnConfig.getYarnKerberosKeytab)) localResources ++= amLocalResources
