This is an automated email from the ASF dual-hosted git repository.
suvasude pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new 7a74579 [GOBBLIN-834] Provide config for setting ACLs to control
visibility of Gobblin-on-Yarn application logs[]
7a74579 is described below
commit 7a74579c05eb5c0bf8965bfd96c3c1036c529ffa
Author: sv2000 <[email protected]>
AuthorDate: Fri Jul 26 10:41:13 2019 -0700
[GOBBLIN-834] Provide config for setting ACLs to control visibility of
Gobblin-on-Yarn application logs[]
Closes #2693 from sv2000/aclViewLogs
---
.../org/apache/gobblin/yarn/GobblinYarnAppLauncher.java | 11 +++++++++++
.../gobblin/yarn/GobblinYarnConfigurationKeys.java | 2 ++
.../main/java/org/apache/gobblin/yarn/YarnService.java | 17 ++++++++++++-----
3 files changed, 25 insertions(+), 5 deletions(-)
diff --git
a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java
b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java
index 8288604..cac61d8 100644
---
a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java
+++
b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.net.URI;
import java.nio.ByteBuffer;
import java.util.EnumSet;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -45,6 +46,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
@@ -163,6 +165,7 @@ public class GobblinYarnAppLauncher {
private final String applicationName;
private final String appQueueName;
+ private final String appViewAcl;
private final Config config;
@@ -261,6 +264,9 @@ public class GobblinYarnAppLauncher {
GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_OVERHEAD_MBS_KEY + "
cannot be more than "
+ GobblinYarnConfigurationKeys.CONTAINER_MEMORY_MBS_KEY + " * "
+ GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_XMX_RATIO_KEY);
+
+ this.appViewAcl = ConfigUtils.getString(this.config,
GobblinYarnConfigurationKeys.APP_VIEW_ACL,
+ GobblinYarnConfigurationKeys.DEFAULT_APP_VIEW_ACL);
}
/**
@@ -504,6 +510,11 @@ public class GobblinYarnAppLauncher {
amContainerLaunchContext.setLocalResources(appMasterLocalResources);
amContainerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration));
amContainerLaunchContext.setCommands(Lists.newArrayList(buildApplicationMasterCommand(resource.getMemory())));
+
+ Map<ApplicationAccessType, String> acls = new HashMap<>(1);
+ acls.put(ApplicationAccessType.VIEW_APP, this.appViewAcl);
+ amContainerLaunchContext.setApplicationACLs(acls);
+
if (UserGroupInformation.isSecurityEnabled()) {
setupSecurityTokens(amContainerLaunchContext);
}
diff --git
a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
index 7c80063..ce4ded6 100644
---
a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
+++
b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
@@ -35,6 +35,8 @@ public class GobblinYarnConfigurationKeys {
GOBBLIN_YARN_PREFIX + "email.notification.on.shutdown";
public static final String RELEASED_CONTAINERS_CACHE_EXPIRY_SECS =
GOBBLIN_YARN_PREFIX + "releasedContainersCacheExpirySecs";
public static final int DEFAULT_RELEASED_CONTAINERS_CACHE_EXPIRY_SECS = 300;
+ public static final String APP_VIEW_ACL = GOBBLIN_YARN_PREFIX + "appViewAcl";
+ public static final String DEFAULT_APP_VIEW_ACL = "*";
// Gobblin Yarn ApplicationMaster configuration properties.
public static final String APP_MASTER_MEMORY_MBS_KEY = GOBBLIN_YARN_PREFIX +
"app.master.memory.mbs";
diff --git
a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java
b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java
index ce53075..5a7e294 100644
--- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java
+++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java
@@ -24,6 +24,7 @@ import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -45,6 +46,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import
org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -64,7 +66,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.util.Records;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -86,25 +87,23 @@ import com.google.common.eventbus.EventBus;
import com.google.common.eventbus.Subscribe;
import com.google.common.io.Closer;
import com.google.common.util.concurrent.AbstractIdleService;
-
import com.typesafe.config.Config;
import lombok.AccessLevel;
import lombok.Getter;
-import org.apache.gobblin.configuration.ConfigurationKeys;
-
import org.apache.gobblin.cluster.GobblinClusterConfigurationKeys;
import org.apache.gobblin.cluster.GobblinClusterMetricTagNames;
import org.apache.gobblin.cluster.GobblinClusterUtils;
import org.apache.gobblin.cluster.HelixUtils;
+import org.apache.gobblin.cluster.event.ClusterManagerShutdownRequest;
+import org.apache.gobblin.configuration.ConfigurationKeys;
import org.apache.gobblin.metrics.GobblinMetrics;
import org.apache.gobblin.metrics.Tag;
import org.apache.gobblin.metrics.event.EventSubmitter;
import org.apache.gobblin.util.ConfigUtils;
import org.apache.gobblin.util.ExecutorsUtils;
import org.apache.gobblin.util.JvmUtils;
-import org.apache.gobblin.cluster.event.ClusterManagerShutdownRequest;
import org.apache.gobblin.yarn.event.ContainerReleaseRequest;
import org.apache.gobblin.yarn.event.ContainerShutdownRequest;
import org.apache.gobblin.yarn.event.NewContainerRequest;
@@ -124,6 +123,7 @@ public class YarnService extends AbstractIdleService {
private final String applicationName;
private final String applicationId;
+ private final String appViewAcl;
private final Config config;
@@ -254,6 +254,9 @@ public class YarnService extends AbstractIdleService {
GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_OVERHEAD_MBS_KEY + "
cannot be more than "
+ GobblinYarnConfigurationKeys.CONTAINER_MEMORY_MBS_KEY + " * "
+ GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_XMX_RATIO_KEY);
+
+ this.appViewAcl = ConfigUtils.getString(this.config,
GobblinYarnConfigurationKeys.APP_VIEW_ACL,
+ GobblinYarnConfigurationKeys.DEFAULT_APP_VIEW_ACL);
}
@SuppressWarnings("unused")
@@ -491,6 +494,10 @@ public class YarnService extends AbstractIdleService {
containerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration));
containerLaunchContext.setCommands(Lists.newArrayList(buildContainerCommand(container,
helixInstanceName)));
+ Map<ApplicationAccessType, String> acls = new HashMap<>(1);
+ acls.put(ApplicationAccessType.VIEW_APP, this.appViewAcl);
+ containerLaunchContext.setApplicationACLs(acls);
+
if (UserGroupInformation.isSecurityEnabled()) {
containerLaunchContext.setTokens(this.tokens.duplicate());
}
