AMBARI-22278. Improve Kerberos principal and keytab accounting (echekanskiy)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d03c24b9
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d03c24b9
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d03c24b9

Branch: refs/heads/trunk
Commit: d03c24b9f21cd91af78e0bfca2a1ce1e0de51bdf
Parents: 8a8d48f
Author: Eugene Chekanskiy <[email protected]>
Authored: Thu Oct 12 13:22:15 2017 +0300
Committer: Eugene Chekanskiy <[email protected]>
Committed: Wed Nov 1 17:52:04 2017 +0200

----------------------------------------------------------------------
 .../ambari/server/agent/HeartbeatProcessor.java |  72 ++++--
 .../controller/DeleteIdentityHandler.java       |   3 +-
 .../server/controller/KerberosHelper.java       |  21 +-
 .../server/controller/KerberosHelperImpl.java   | 138 +++++++++-
 .../HostKerberosIdentityResourceProvider.java   |  16 +-
 .../ServiceComponentUninstalledEvent.java       |  11 +-
 .../server/orm/dao/KerberosKeytabDAO.java       | 110 ++++++++
 .../server/orm/dao/KerberosPrincipalDAO.java    |   7 +
 .../orm/dao/KerberosPrincipalHostDAO.java       |  40 ++-
 .../orm/entities/KerberosKeytabEntity.java      |  86 +++++++
 .../entities/KerberosPrincipalHostEntity.java   |  57 +++-
 .../entities/KerberosPrincipalHostEntityPK.java |  19 +-
 .../AbstractPrepareKerberosServerAction.java    |  29 ++-
 .../kerberos/CleanupServerAction.java           |  14 +-
 .../server/serveraction/kerberos/Component.java |  13 +-
 .../ConfigureAmbariIdentitiesServerAction.java  |  31 ++-
 .../kerberos/CreateKeytabFilesServerAction.java |  64 +++--
 .../kerberos/CreatePrincipalsServerAction.java  |  48 ++--
 .../kerberos/KerberosIdentityDataFile.java      |   2 -
 .../KerberosIdentityDataFileWriter.java         |   9 +-
 .../kerberos/KerberosServerAction.java          |  27 +-
 .../PrepareDisableKerberosServerAction.java     |   2 +-
 .../PrepareEnableKerberosServerAction.java      |   2 +-
 .../PrepareKerberosIdentitiesServerAction.java  |   3 +-
 .../stageutils/ResolvedKerberosKeytab.java      | 257 +++++++++++++++++++
 .../upgrades/PreconfigureKerberosAction.java    |  48 +++-
 .../apache/ambari/server/state/ServiceImpl.java |   2 +-
 .../svccomphost/ServiceComponentHostImpl.java   |   2 +-
 .../main/resources/Ambari-DDL-Derby-CREATE.sql  |  15 +-
 .../main/resources/Ambari-DDL-MySQL-CREATE.sql  |  13 +-
 .../main/resources/Ambari-DDL-Oracle-CREATE.sql |  13 +-
 .../resources/Ambari-DDL-Postgres-CREATE.sql    |  11 +-
 .../resources/Ambari-DDL-SQLAnywhere-CREATE.sql |  13 +-
 .../resources/Ambari-DDL-SQLServer-CREATE.sql   |  13 +-
 .../src/main/resources/META-INF/persistence.xml |   1 +
 .../package/scripts/kerberos_common.py          |   7 +-
 .../package/scripts/kerberos_common.py          |   7 +-
 .../server/agent/TestHeartbeatHandler.java      |   2 +-
 .../server/controller/KerberosHelperTest.java   |   6 +
 ...ostKerberosIdentityResourceProviderTest.java |  12 +-
 .../utilities/KerberosIdentityCleanerTest.java  |  10 +-
 .../HostVersionOutOfSyncListenerTest.java       |   2 +-
 ...AbstractPrepareKerberosServerActionTest.java |  11 +-
 ...nfigureAmbariIdentitiesServerActionTest.java |  11 +-
 .../FinalizeKerberosServerActionTest.java       |   5 +
 .../kerberos/KerberosIdentityDataFileTest.java  |   8 +-
 .../kerberos/KerberosServerActionTest.java      |  10 +-
 .../PreconfigureKerberosActionTest.java         |  10 +
 48 files changed, 1097 insertions(+), 216 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/d03c24b9/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatProcessor.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatProcessor.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatProcessor.java
index 2690008..83d2c98 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatProcessor.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatProcessor.java
@@ -52,7 +52,9 @@ import 
org.apache.ambari.server.events.publishers.AlertEventPublisher;
 import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
 import org.apache.ambari.server.events.publishers.VersionEventPublisher;
 import org.apache.ambari.server.metadata.ActionMetadata;
+import org.apache.ambari.server.orm.dao.KerberosKeytabDAO;
 import org.apache.ambari.server.orm.dao.KerberosPrincipalHostDAO;
+import org.apache.ambari.server.orm.entities.KerberosPrincipalHostEntity;
 import org.apache.ambari.server.state.Alert;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
@@ -88,7 +90,6 @@ import com.google.inject.Injector;
 
 /**
  * HeartbeatProcessor class is used for bulk processing data retrieved from 
agents in background
- *
  */
 public class HeartbeatProcessor extends AbstractService{
   private static final Logger LOG = 
LoggerFactory.getLogger(HeartbeatProcessor.class);
@@ -135,6 +136,9 @@ public class HeartbeatProcessor extends AbstractService{
   KerberosPrincipalHostDAO kerberosPrincipalHostDAO;
 
   @Inject
+  KerberosKeytabDAO kerberosKeytabDao;
+
+  @Inject
   Gson gson;
 
   @Inject
@@ -153,7 +157,7 @@ public class HeartbeatProcessor extends AbstractService{
   @Override
   protected void doStart() {
     LOG.info("**** Starting heartbeats processing threads ****");
-    for (int i=0; i< poolSize; i++) {
+    for (int i = 0; i < poolSize; i++) {
       executor.scheduleAtFixedRate(new HeartbeatProcessingTask(), delay, 
period, TimeUnit.MILLISECONDS);
     }
   }
@@ -201,6 +205,7 @@ public class HeartbeatProcessor extends AbstractService{
 
   /**
    * Incapsulates logic for processing data from agent heartbeat
+   *
    * @param heartbeat Agent heartbeat object
    * @throws AmbariException
    */
@@ -217,14 +222,12 @@ public class HeartbeatProcessor extends AbstractService{
   }
 
 
-
   /**
    * Extracts all of the {@link Alert}s from the heartbeat and fires
    * {@link AlertEvent}s for each one. If there is a problem looking up the
    * cluster, then alerts will not be processed.
    *
-   * @param heartbeat
-   *          the heartbeat to process.
+   * @param heartbeat the heartbeat to process.
    */
   protected void processAlerts(HeartBeat heartbeat) {
     if (heartbeat == null) {
@@ -247,6 +250,7 @@ public class HeartbeatProcessor extends AbstractService{
 
   /**
    * Update host status basing on components statuses
+   *
    * @param heartbeat heartbeat to process
    * @throws AmbariException
    */
@@ -349,8 +353,9 @@ public class HeartbeatProcessor extends AbstractService{
 
   /**
    * Process reports of tasks executed on agents
+   *
    * @param heartbeat heartbeat to process
-   * @param now cached current time
+   * @param now       cached current time
    * @throws AmbariException
    */
   protected void processCommandReports(
@@ -424,8 +429,7 @@ public class HeartbeatProcessor extends AbstractService{
 
         String customCommand = report.getCustomCommand();
 
-        boolean adding = SET_KEYTAB.equalsIgnoreCase(customCommand);
-        if (adding || REMOVE_KEYTAB.equalsIgnoreCase(customCommand)) {
+        if (SET_KEYTAB.equalsIgnoreCase(customCommand) || 
REMOVE_KEYTAB.equalsIgnoreCase(customCommand)) {
           WriteKeytabsStructuredOut writeKeytabsStructuredOut;
           try {
             writeKeytabsStructuredOut = 
gson.fromJson(report.getStructuredOut(), WriteKeytabsStructuredOut.class);
@@ -435,25 +439,35 @@ public class HeartbeatProcessor extends AbstractService{
           }
 
           if (writeKeytabsStructuredOut != null) {
-            Map<String, String> keytabs = 
writeKeytabsStructuredOut.getKeytabs();
-            if (keytabs != null) {
-              for (Map.Entry<String, String> entry : keytabs.entrySet()) {
-                String principal = entry.getKey();
-                if (!kerberosPrincipalHostDAO.exists(principal, 
host.getHostId())) {
-                  if (adding) {
-                    kerberosPrincipalHostDAO.create(principal, 
host.getHostId());
-                  } else if ("_REMOVED_".equalsIgnoreCase(entry.getValue())) {
-                    kerberosPrincipalHostDAO.remove(principal, 
host.getHostId());
-                  }
+            if (SET_KEYTAB.equalsIgnoreCase(customCommand)) {
+              Map<String, String> keytabs = 
writeKeytabsStructuredOut.getKeytabs();
+              if (keytabs != null) {
+                for (Map.Entry<String, String> entry : keytabs.entrySet()) {
+                  String principal = entry.getKey();
+                  String keytabPath = entry.getValue();
+                  KerberosPrincipalHostEntity kphe = 
kerberosPrincipalHostDAO.find(principal, host.getHostId(), keytabPath);
+                  kphe.setDistributed(true);
+                  kerberosPrincipalHostDAO.merge(kphe);
+                }
+              }
+            } else if (REMOVE_KEYTAB.equalsIgnoreCase(customCommand)) {
+              Map<String, String> deletedKeytabs = 
writeKeytabsStructuredOut.getRemovedKeytabs();
+              if (deletedKeytabs != null) {
+                for (Map.Entry<String, String> entry : 
deletedKeytabs.entrySet()) {
+                  String keytabPath = entry.getValue();
+                  kerberosPrincipalHostDAO.removeByKeytabPath(keytabPath);
+                  kerberosKeytabDao.remove(keytabPath);
                 }
               }
             }
           }
         } else if (CHECK_KEYTABS.equalsIgnoreCase(customCommand)) {
           ListKeytabsStructuredOut structuredOut = 
gson.fromJson(report.getStructuredOut(), ListKeytabsStructuredOut.class);
-          for (MissingKeytab each : structuredOut.missingKeytabs){
-            LOG.info("Missing keytab: {} on host: {} principal: {}", 
each.keytabFilePath, hostname, each.principal);
-            kerberosPrincipalHostDAO.remove(each.principal, host.getHostId());
+          for (MissingKeytab each : structuredOut.missingKeytabs) {
+            LOG.info("Missing principal: {} for keytab: {} on host: {}", 
each.principal, each.keytabFilePath, hostname);
+            KerberosPrincipalHostEntity kphe = 
kerberosPrincipalHostDAO.find(each.principal, host.getHostId(), 
each.keytabFilePath);
+            kphe.setDistributed(false);
+            kerberosPrincipalHostDAO.merge(kphe);
           }
         }
       }
@@ -517,7 +531,7 @@ public class HeartbeatProcessor extends AbstractService{
             // Necessary for resetting clients stale configs after starting 
service
             if 
((RoleCommand.INSTALL.toString().equals(report.getRoleCommand()) ||
                 
(RoleCommand.CUSTOM_COMMAND.toString().equals(report.getRoleCommand()) &&
-                    "INSTALL".equals(report.getCustomCommand()))) && 
svcComp.isClientComponent()){
+                    "INSTALL".equals(report.getCustomCommand()))) && 
svcComp.isClientComponent()) {
               scHost.updateActualConfigs(report.getConfigurationTags());
               scHost.setRestartRequired(false);
             }
@@ -588,6 +602,7 @@ public class HeartbeatProcessor extends AbstractService{
 
   /**
    * Process reports of status commands
+   *
    * @param heartbeat heartbeat to process
    * @throws AmbariException
    */
@@ -701,7 +716,10 @@ public class HeartbeatProcessor extends AbstractService{
    */
   private static class WriteKeytabsStructuredOut {
     @SerializedName("keytabs")
-    private Map<String,String> keytabs;
+    private Map<String, String> keytabs;
+
+    @SerializedName("removedKeytabs")
+    private Map<String, String> removedKeytabs;
 
     public Map<String, String> getKeytabs() {
       return keytabs;
@@ -710,6 +728,14 @@ public class HeartbeatProcessor extends AbstractService{
     public void setKeytabs(Map<String, String> keytabs) {
       this.keytabs = keytabs;
     }
+
+    public Map<String, String> getRemovedKeytabs() {
+      return removedKeytabs;
+    }
+
+    public void setRemovedKeytabs(Map<String, String> removedKeytabs) {
+      this.removedKeytabs = removedKeytabs;
+    }
   }
 
   private static class ListKeytabsStructuredOut {

http://git-wip-us.apache.org/repos/asf/ambari/blob/d03c24b9/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java
index 29f8e2a..a7b9d80 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java
@@ -227,8 +227,7 @@ class DeleteIdentityHandler {
         calculateConfig(kerberosDescriptor, serviceNames()),
         new HashMap<>(),
         false,
-        new HashMap<>(),
-          false);
+        new HashMap<>());
       return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", 
actionLog.getStdOut(), actionLog.getStdErr());
     }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/d03c24b9/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
index b8e1be1..749943d 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
@@ -33,6 +33,7 @@ import 
org.apache.ambari.server.serveraction.kerberos.KerberosIdentityDataFileWr
 import 
org.apache.ambari.server.serveraction.kerberos.KerberosInvalidConfigurationException;
 import 
org.apache.ambari.server.serveraction.kerberos.KerberosMissingAdminCredentialsException;
 import 
org.apache.ambari.server.serveraction.kerberos.KerberosOperationException;
+import 
org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosKeytab;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.SecurityType;
 import org.apache.ambari.server.state.ServiceComponentHost;
@@ -588,15 +589,15 @@ public interface KerberosHelper {
    *                                       values
    * @param configurations                 a Map of configurations to use a 
replacements for variables
    *                                       in identity fields
-   * @param ignoreHeadless                 boolean value to specify if 
headless principals must not be processed
    * @return an integer indicating the number of identities added to the data 
file
    * @throws java.io.IOException if an error occurs while writing a record to 
the data file
    */
   int addIdentities(KerberosIdentityDataFileWriter 
kerberosIdentityDataFileWriter,
                     Collection<KerberosIdentityDescriptor> identities,
-                    Collection<String> identityFilter, String hostname, String 
serviceName,
+                    Collection<String> identityFilter, String hostname, Long 
hostId, String serviceName,
                     String componentName, Map<String, Map<String, String>> 
kerberosConfigurations,
-                    Map<String, Map<String, String>> configurations, boolean 
ignoreHeadless)
+                    Map<String, Map<String, String>> configurations,
+                    Map<String, ResolvedKerberosKeytab> resolvedKeytabs, 
String realm)
       throws IOException;
   /**
    * Calculates the map of configurations relative to the cluster and host.
@@ -735,6 +736,20 @@ public interface KerberosHelper {
   PrincipalKeyCredential getKDCAdministratorCredentials(String clusterName) 
throws AmbariException;
 
   /**
+   * Saves underlying entities in persistent storage.
+   *
+   * @param resolvedKerberosKeytab kerberos keytab to be persisted
+   */
+  void processResolvedKeytab(ResolvedKerberosKeytab resolvedKerberosKeytab);
+
+  /**
+   * Removes existent persisted keytabs if they are not in {@code 
expectedKeytabs} collection.
+   *
+   * @param expectedKeytabs collection to compare existent keytabs
+   */
+  void removeStaleKeytabs(Collection<ResolvedKerberosKeytab> expectedKeytabs);
+
+  /**
    * Translates a collection of configuration specifications 
(<code>config-type/property-name</code>)
    * to a map of configuration types to a set of property names.
    * <p>

http://git-wip-us.apache.org/repos/asf/ambari/blob/d03c24b9/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
index 4f14614..f913831 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
@@ -60,8 +60,11 @@ import 
org.apache.ambari.server.controller.internal.RequestStageContainer;
 import org.apache.ambari.server.controller.utilities.KerberosChecker;
 import org.apache.ambari.server.metadata.RoleCommandOrder;
 import org.apache.ambari.server.orm.dao.ArtifactDAO;
+import org.apache.ambari.server.orm.dao.KerberosKeytabDAO;
 import org.apache.ambari.server.orm.dao.KerberosPrincipalDAO;
+import org.apache.ambari.server.orm.dao.KerberosPrincipalHostDAO;
 import org.apache.ambari.server.orm.entities.ArtifactEntity;
+import org.apache.ambari.server.orm.entities.KerberosKeytabEntity;
 import org.apache.ambari.server.security.credential.Credential;
 import org.apache.ambari.server.security.credential.PrincipalKeyCredential;
 import org.apache.ambari.server.security.encryption.CredentialStoreService;
@@ -91,6 +94,7 @@ import 
org.apache.ambari.server.serveraction.kerberos.PrepareDisableKerberosServ
 import 
org.apache.ambari.server.serveraction.kerberos.PrepareEnableKerberosServerAction;
 import 
org.apache.ambari.server.serveraction.kerberos.PrepareKerberosIdentitiesServerAction;
 import 
org.apache.ambari.server.serveraction.kerberos.UpdateKerberosConfigsServerAction;
+import 
org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosKeytab;
 import org.apache.ambari.server.stageplanner.RoleGraph;
 import org.apache.ambari.server.stageplanner.RoleGraphFactory;
 import org.apache.ambari.server.state.Cluster;
@@ -125,12 +129,14 @@ import org.apache.ambari.server.utils.StageUtils;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.directory.server.kerberos.shared.keytab.Keytab;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
 import com.google.inject.Inject;
 import com.google.inject.Injector;
 import com.google.inject.Singleton;
@@ -202,6 +208,12 @@ public class KerberosHelperImpl implements KerberosHelper {
   @Inject
   private ArtifactDAO artifactDAO;
 
+  @Inject
+  private KerberosKeytabDAO kerberosKeytabDAO;
+
+  @Inject
+  KerberosPrincipalHostDAO kerberosPrincipalHostDAO;
+
   /**
    * The injector used to create new instances of helper classes like 
CreatePrincipalsServerAction
    * and CreateKeytabFilesServerAction.
@@ -1484,9 +1496,10 @@ public class KerberosHelperImpl implements 
KerberosHelper {
   @Override
   public int addIdentities(KerberosIdentityDataFileWriter 
kerberosIdentityDataFileWriter,
                            Collection<KerberosIdentityDescriptor> identities,
-                           Collection<String> identityFilter, String hostname, 
String serviceName,
+                           Collection<String> identityFilter, String hostname, 
Long hostId, String serviceName,
                            String componentName, Map<String, Map<String, 
String>> kerberosConfigurations,
-                           Map<String, Map<String, String>> configurations, 
boolean ignoreHeadless)
+                           Map<String, Map<String, String>> configurations,
+                           Map<String, ResolvedKerberosKeytab> 
resolvedKeytabs, String realm)
       throws IOException {
     int identitiesAdded = 0;
 
@@ -1514,7 +1527,6 @@ public class KerberosHelperImpl implements KerberosHelper 
{
             String keytabFileGroupName = null;
             String keytabFileGroupAccess = null;
             String keytabFileConfiguration = null;
-            boolean keytabIsCachable = false;
 
             if (keytabDescriptor != null) {
               keytabFilePath = 
variableReplacementHelper.replaceVariables(keytabDescriptor.getFile(), 
configurations);
@@ -1523,24 +1535,84 @@ public class KerberosHelperImpl implements 
KerberosHelper {
               keytabFileGroupName = 
variableReplacementHelper.replaceVariables(keytabDescriptor.getGroupName(), 
configurations);
               keytabFileGroupAccess = 
variableReplacementHelper.replaceVariables(keytabDescriptor.getGroupAccess(), 
configurations);
               keytabFileConfiguration = 
variableReplacementHelper.replaceVariables(keytabDescriptor.getConfiguration(), 
configurations);
-              keytabIsCachable = keytabDescriptor.isCachable();
+            }
+            // Evaluate the principal "pattern" found in the record to 
generate the "evaluated principal"
+            // by replacing the _HOST and _REALM variables.
+            String evaluatedPrincipal = principal.replace("_HOST", 
hostname).replace("_REALM", realm);
+
+            ResolvedKerberosKeytab resolvedKeytab = new ResolvedKerberosKeytab(
+                keytabFilePath,
+                keytabFileOwnerName,
+                keytabFileOwnerAccess,
+                keytabFileGroupName,
+                keytabFileGroupAccess,
+                Sets.newHashSet(Pair.of(hostId, evaluatedPrincipal)),
+                serviceName.equalsIgnoreCase("AMBARI"),
+                componentName.equalsIgnoreCase("AMBARI_SERVER_SELF")
+            );
+            if (resolvedKeytabs.containsKey(keytabFilePath)) {
+              ResolvedKerberosKeytab sameKeytab = 
resolvedKeytabs.get(keytabFilePath);
+              // validating owner and group
+              String warnTemplate = "Keytab '{}' on host '{}' have different 
{}, originally set to '{}' and '{}:{}' has '{}', using '{}'";
+              if 
(!resolvedKeytab.getOwnerName().equals(sameKeytab.getOwnerName())) {
+                LOG.warn(warnTemplate,
+                    keytabFilePath, hostname, "owners", 
sameKeytab.getOwnerName(),
+                    serviceName, componentName, resolvedKeytab.getOwnerName(),
+                    sameKeytab.getOwnerName());
+              }
+              if 
(!resolvedKeytab.getOwnerAccess().equals(sameKeytab.getOwnerAccess())) {
+                LOG.warn(warnTemplate,
+                    keytabFilePath, hostname, "owner access", 
sameKeytab.getOwnerAccess(),
+                    serviceName, componentName, 
resolvedKeytab.getOwnerAccess(),
+                    sameKeytab.getOwnerAccess());
+              }
+              // TODO probably fail on group difference. Some services can 
inject its principals to same keytab, but
+              // TODO with different owners, so make sure that keytabs are 
accessible through group acls
+              // TODO this includes same group name and group 'r' mode
+              if 
(!resolvedKeytab.getGroupName().equals(sameKeytab.getGroupName())) {
+                LOG.warn(warnTemplate,
+                    keytabFilePath, hostname, "groups", 
sameKeytab.getGroupName(),
+                    serviceName, componentName, resolvedKeytab.getGroupName(),
+                    sameKeytab.getGroupName());
+              }
+              if 
(!resolvedKeytab.getGroupAccess().equals(sameKeytab.getGroupAccess())) {
+                LOG.warn(warnTemplate,
+                    keytabFilePath, hostname, "group access", 
sameKeytab.getGroupAccess(),
+                    serviceName, componentName, 
resolvedKeytab.getGroupAccess(),
+                    sameKeytab.getGroupAccess());
+              }
+              // end validating
+              // merge principal to keytab
+              
sameKeytab.getMappedPrincipals().addAll(resolvedKeytab.getMappedPrincipals());
+              // ensure that keytab file on ambari-server host creating jass 
file
+              if (sameKeytab.isMustWriteAmbariJaasFile() || 
resolvedKeytab.isMustWriteAmbariJaasFile()) {
+                sameKeytab.setMustWriteAmbariJaasFile(true);
+              }
+              // ensure that this keytab is ambari-keytab, server will 
distribute it manually
+              if (sameKeytab.isAmbariServerKeytab() || 
resolvedKeytab.isAmbariServerKeytab()) {
+                sameKeytab.setAmbariServerKeytab(true);
+              }
+            } else {
+              resolvedKeytabs.put(keytabFilePath, resolvedKeytab);
+              LOG.info("Keytab {} owner:'{}:{}', group:'{}:{}' is defined", 
keytabFilePath,
+                  keytabFileOwnerName, keytabFileOwnerAccess, 
keytabFileGroupName, keytabFileGroupAccess);
             }
 
             // Append an entry to the action data file builder...
+            // TODO obsolete, move to ResolvedKerberosKeytab
             if(kerberosIdentityDataFileWriter != null) {
               kerberosIdentityDataFileWriter.writeRecord(
                   hostname,
                   serviceName,
                   componentName,
-                  principal,
+                  evaluatedPrincipal,
                   principalType,
                   keytabFilePath,
                   keytabFileOwnerName,
                   keytabFileOwnerAccess,
                   keytabFileGroupName,
                   keytabFileGroupAccess,
-                  (keytabIsCachable) ? "true" : "false",
-                  (ignoreHeadless && principalDescriptor.getType() == 
KerberosPrincipalType.USER) ? "true" : "false");
+                  "true");
             }
 
             // Add the principal-related configuration to the map of 
configurations
@@ -1793,6 +1865,46 @@ public class KerberosHelperImpl implements 
KerberosHelper {
     }
   }
 
+  /**
+   * Creates and saves  underlying  {@link 
org.apache.ambari.server.orm.entities.KerberosPrincipalEntity},
+   * {@link org.apache.ambari.server.orm.entities.KerberosKeytabEntity} and
+   * {@link org.apache.ambari.server.orm.entities.KerberosPrincipalHostEntity} 
entities in JPA storage.
+   *
+   * @param resolvedKerberosKeytab kerberos keytab to be persisted
+   */
+  @Override
+  public void processResolvedKeytab(ResolvedKerberosKeytab 
resolvedKerberosKeytab) {
+    if (kerberosKeytabDAO.find(resolvedKerberosKeytab.getFile()) == null) {
+      kerberosKeytabDAO.create(resolvedKerberosKeytab.getFile());
+    }
+    for (Pair<Long, String> principalPair : 
resolvedKerberosKeytab.getMappedPrincipals()) {
+      String principal = principalPair.getRight();
+      Long hostId = principalPair.getLeft();
+      if (!kerberosPrincipalDAO.exists(principal)) {
+        kerberosPrincipalDAO.create(principal, false);
+      }
+      if (hostId != null) {
+        if(!kerberosPrincipalHostDAO.exists(principal, hostId, 
resolvedKerberosKeytab.getFile())) {
+          kerberosPrincipalHostDAO.create(principal, hostId, 
resolvedKerberosKeytab.getFile());
+        }
+      }
+    }
+  }
+
+  @Override
+  public void removeStaleKeytabs(Collection<ResolvedKerberosKeytab> 
expectedKeytabs) {
+    List<KerberosKeytabEntity> allKeytabs = kerberosKeytabDAO.findAll();
+    Set<KerberosKeytabEntity> staleKeytabs;
+    staleKeytabs = allKeytabs != null ? new HashSet<>(allKeytabs) : 
Collections.emptySet();
+    for (ResolvedKerberosKeytab keytab : expectedKeytabs) {
+      staleKeytabs.remove(new KerberosKeytabEntity(keytab.getFile()));
+    }
+    for (KerberosKeytabEntity staleKeytab: staleKeytabs) {
+      kerberosPrincipalHostDAO.removeByKeytabPath(staleKeytab.getKeytabPath());
+      kerberosKeytabDAO.remove(staleKeytab);
+    }
+  }
+
   @Override
   public Map<String, Set<String>> 
translateConfigurationSpecifications(Collection<String> 
configurationSpecifications) {
     Map<String, Set<String>> translation = null;
@@ -2181,6 +2293,17 @@ public class KerberosHelperImpl implements 
KerberosHelper {
             if (sch.getState() == State.INSTALLED) {
               String hostname = sch.getHostName();
 
+              if(kerberosKeytabDAO.find(keytabFilePath) == null) {
+                kerberosKeytabDAO.create(keytabFilePath);
+              }
+              // create principals
+              if (!kerberosPrincipalDAO.exists(principal)) {
+                kerberosPrincipalDAO.create(principal, false);
+              }
+              if (!kerberosPrincipalHostDAO.exists(principal, 
sch.getHost().getHostId(), keytabFilePath)) {
+                kerberosPrincipalHostDAO.create(principal, 
sch.getHost().getHostId(), keytabFilePath);
+              }
+
               kerberosIdentityDataFileWriter.writeRecord(
                   hostname,
                   Service.Type.KERBEROS.name(),
@@ -2192,7 +2315,6 @@ public class KerberosHelperImpl implements KerberosHelper 
{
                   keytabFileOwnerAccess,
                   keytabFileGroupName,
                   keytabFileGroupAccess,
-                  "false",
                   "false");
 
               hostsWithValidKerberosClient.add(hostname);

http://git-wip-us.apache.org/repos/asf/ambari/blob/d03c24b9/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostKerberosIdentityResourceProvider.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostKerberosIdentityResourceProvider.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostKerberosIdentityResourceProvider.java
index bfaf7b4..0672500 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostKerberosIdentityResourceProvider.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostKerberosIdentityResourceProvider.java
@@ -76,7 +76,7 @@ public class HostKerberosIdentityResourceProvider extends 
ReadOnlyResourceProvid
   );
 
   protected static final Set<String> PK_PROPERTY_IDS = 
Collections.unmodifiableSet(
-    new HashSet<>(PK_PROPERTY_MAP.values())
+      new HashSet<>(PK_PROPERTY_MAP.values())
   );
 
   protected static final Set<String> PROPERTY_IDS = 
Collections.unmodifiableSet(
@@ -183,7 +183,7 @@ public class HostKerberosIdentityResourceProvider extends 
ReadOnlyResourceProvid
                     KerberosPrincipalType principalType = 
principalDescriptor.getType();
 
                     // Assume the principal is a service principal if not 
specified
-                    if(principalType == null) {
+                    if (principalType == null) {
                       principalType = KerberosPrincipalType.SERVICE;
                     }
 
@@ -194,10 +194,17 @@ public class HostKerberosIdentityResourceProvider extends 
ReadOnlyResourceProvid
                     setResourceProperty(resource, 
KERBEROS_IDENTITY_PRINCIPAL_TYPE_PROPERTY_ID, principalType, 
requestPropertyIds);
                     setResourceProperty(resource, 
KERBEROS_IDENTITY_PRINCIPAL_LOCAL_USERNAME_PROPERTY_ID, 
principalDescriptor.getLocalUsername(), requestPropertyIds);
 
+                    KerberosKeytabDescriptor keytabDescriptor = 
descriptor.getKeytabDescriptor();
+
                     String installedStatus;
+
                     if ((hostId != null) && 
kerberosPrincipalDAO.exists(principal)) {
-                      if (kerberosPrincipalHostDAO.exists(principal, hostId)) {
-                        installedStatus = "true";
+                      if (keytabDescriptor != null) {
+                        if (kerberosPrincipalHostDAO.exists(principal, hostId, 
keytabDescriptor.getFile())) {
+                          installedStatus = "true";
+                        } else {
+                          installedStatus = "false";
+                        }
                       } else {
                         installedStatus = "false";
                       }
@@ -207,7 +214,6 @@ public class HostKerberosIdentityResourceProvider extends 
ReadOnlyResourceProvid
 
                     setResourceProperty(resource, 
KERBEROS_IDENTITY_KEYTAB_FILE_INSTALLED_PROPERTY_ID, installedStatus, 
requestPropertyIds);
 
-                    KerberosKeytabDescriptor keytabDescriptor = 
descriptor.getKeytabDescriptor();
                     if (keytabDescriptor != null) {
                       String ownerAccess = keytabDescriptor.getOwnerAccess();
                       String groupAccess = keytabDescriptor.getGroupAccess();

http://git-wip-us.apache.org/repos/asf/ambari/blob/d03c24b9/ambari-server/src/main/java/org/apache/ambari/server/events/ServiceComponentUninstalledEvent.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/events/ServiceComponentUninstalledEvent.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/events/ServiceComponentUninstalledEvent.java
index 8acc401..b91135f 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/events/ServiceComponentUninstalledEvent.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/events/ServiceComponentUninstalledEvent.java
@@ -27,6 +27,7 @@ public class ServiceComponentUninstalledEvent extends 
ServiceEvent {
   private final String m_componentName;
   private final String m_hostName;
   private final boolean m_recoveryEnabled;
+  private final Long m_hostId;
 
   /**
    * Constructor.
@@ -40,7 +41,7 @@ public class ServiceComponentUninstalledEvent extends 
ServiceEvent {
    */
   public ServiceComponentUninstalledEvent(long clusterId, String stackName,
       String stackVersion, String serviceName, String componentName,
-      String hostName, boolean recoveryEnabled) {
+      String hostName, boolean recoveryEnabled, Long hostId) {
     super(AmbariEventType.SERVICE_COMPONENT_UNINSTALLED_SUCCESS, clusterId,
         stackName,
         stackVersion, serviceName);
@@ -48,6 +49,7 @@ public class ServiceComponentUninstalledEvent extends 
ServiceEvent {
     m_componentName = componentName;
     m_hostName = hostName;
     m_recoveryEnabled = recoveryEnabled;
+    m_hostId = hostId;
   }
 
   /**
@@ -71,6 +73,10 @@ public class ServiceComponentUninstalledEvent extends 
ServiceEvent {
     return m_recoveryEnabled;
   }
 
+  public Long getHostId() {
+    return m_hostId;
+  }
+
   /**
    * {@inheritDoc}
    */
@@ -84,11 +90,12 @@ public class ServiceComponentUninstalledEvent extends 
ServiceEvent {
     buffer.append(", componentName=").append(m_componentName);
     buffer.append(", hostName=").append(m_hostName);
     buffer.append(", recoveryEnabled=").append(m_recoveryEnabled);
+    buffer.append(", hostId=").append(m_hostId);
     buffer.append("}");
     return buffer.toString();
   }
 
   public Component getComponent() {
-    return new Component(getHostName(), getServiceName(), getComponentName());
+    return new Component(getHostName(), getServiceName(), getComponentName(), 
getHostId());
   }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/d03c24b9/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosKeytabDAO.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosKeytabDAO.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosKeytabDAO.java
new file mode 100644
index 0000000..a8723b7
--- /dev/null
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosKeytabDAO.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.orm.dao;
+
+import java.util.Collection;
+import java.util.List;
+
+import javax.persistence.EntityManager;
+import javax.persistence.TypedQuery;
+
+import org.apache.ambari.server.orm.RequiresSession;
+import org.apache.ambari.server.orm.entities.HostEntity;
+import org.apache.ambari.server.orm.entities.KerberosKeytabEntity;
+
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import com.google.inject.Singleton;
+import com.google.inject.persist.Transactional;
+
+@Singleton
+public class KerberosKeytabDAO {
+    @Inject
+    Provider<EntityManager> entityManagerProvider;
+
+    @Transactional
+    public void create(KerberosKeytabEntity kerberosKeytabEntity) {
+        entityManagerProvider.get().persist(kerberosKeytabEntity);
+    }
+
+    public void create(String keytabPath) {
+        create(new KerberosKeytabEntity(keytabPath));
+    }
+
+    @Transactional
+    public KerberosKeytabEntity merge(KerberosKeytabEntity 
kerberosKeytabEntity) {
+        return entityManagerProvider.get().merge(kerberosKeytabEntity);
+    }
+
+    @Transactional
+    public void remove(KerberosKeytabEntity kerberosKeytabEntity) {
+        entityManagerProvider.get().remove(merge(kerberosKeytabEntity));
+    }
+
+    public void remove(String keytabPath) {
+        KerberosKeytabEntity kke = find(keytabPath);
+        if (kke != null) {
+            remove(kke);
+        }
+    }
+
+    @Transactional
+    public void refresh(KerberosKeytabEntity kerberosKeytabEntity) {
+        entityManagerProvider.get().refresh(kerberosKeytabEntity);
+    }
+
+
+    @RequiresSession
+    public KerberosKeytabEntity find(String keytabPath) {
+        return entityManagerProvider.get().find(KerberosKeytabEntity.class, 
keytabPath);
+    }
+
+    @RequiresSession
+    public List<KerberosKeytabEntity> findAll() {
+        TypedQuery<KerberosKeytabEntity> query = entityManagerProvider.get().
+                createNamedQuery("KerberosKeytabEntity.findAll", 
KerberosKeytabEntity.class);
+
+        return query.getResultList();
+    }
+
+    @RequiresSession
+    public boolean exists(String keytabPath) {
+        return find(keytabPath) != null;
+    }
+
+    @RequiresSession
+    public Collection<KerberosKeytabEntity> findByHost(Long hostId) {
+        TypedQuery<KerberosKeytabEntity> query = entityManagerProvider.get().
+            createNamedQuery("KerberosKeytabEntity.findByHost", 
KerberosKeytabEntity.class);
+        query.setParameter("hostId", hostId);
+        return query.getResultList();
+    }
+
+    public Collection<KerberosKeytabEntity> findByHost(HostEntity hostEntity) {
+        return findByHost(hostEntity.getHostId());
+    }
+
+    public void remove(List<KerberosKeytabEntity> entities) {
+        if (entities != null) {
+            for (KerberosKeytabEntity entity : entities) {
+                entityManagerProvider.get().remove(entity);
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/d03c24b9/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosPrincipalDAO.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosPrincipalDAO.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosPrincipalDAO.java
index 93c55c1..81e4b3d 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosPrincipalDAO.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosPrincipalDAO.java
@@ -161,4 +161,11 @@ public class KerberosPrincipalDAO {
     return find(principalName) != null;
   }
 
+  public void remove(List<KerberosPrincipalEntity> entities) {
+    if (entities != null) {
+      for (KerberosPrincipalEntity entity : entities) {
+        remove(entity);
+      }
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/d03c24b9/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosPrincipalHostDAO.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosPrincipalHostDAO.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosPrincipalHostDAO.java
index 0c17f19..f27dc48 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosPrincipalHostDAO.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosPrincipalHostDAO.java
@@ -56,8 +56,8 @@ public class KerberosPrincipalHostDAO {
     entityManagerProvider.get().persist(kerberosPrincipalHostEntity);
   }
 
-  public void create(String principal, Long hostId) {
-    create(new KerberosPrincipalHostEntity(principal, hostId));
+  public void create(String principal, Long hostId, String keytabPath) {
+    create(new KerberosPrincipalHostEntity(principal, hostId, keytabPath));
   }
 
   /**
@@ -121,6 +121,19 @@ public class KerberosPrincipalHostDAO {
   }
 
   /**
+   * Find KerberosPrincipalHostEntities for the requested host
+   *
+   * @return a List of requested KerberosPrincipalHostEntities or null if none 
were found
+   */
+  @RequiresSession
+  public List<KerberosPrincipalHostEntity> findByKeytabPath(String keytabPath) 
{
+    final TypedQuery<KerberosPrincipalHostEntity> query = 
entityManagerProvider.get()
+        .createNamedQuery("KerberosPrincipalHostEntityFindByKeytabPath", 
KerberosPrincipalHostEntity.class);
+    query.setParameter("keytabPath", keytabPath);
+    return query.getResultList();
+  }
+
+  /**
    * Find the KerberosPrincipalHostEntity for the specified primary key
    *
    * @param primaryKey a KerberosPrincipalHostEntityPK containing the 
requested principal and host names
@@ -139,9 +152,9 @@ public class KerberosPrincipalHostDAO {
    * @return the KerberosPrincipalHostEntity or null if not found
    */
   @RequiresSession
-  public KerberosPrincipalHostEntity find(String principalName, Long hostId) {
+  public KerberosPrincipalHostEntity find(String principalName, Long hostId, 
String keytabPath) {
     return entityManagerProvider.get().find(KerberosPrincipalHostEntity.class,
-        new KerberosPrincipalHostEntityPK(principalName, hostId));
+        new KerberosPrincipalHostEntityPK(principalName, hostId, keytabPath));
   }
 
   /**
@@ -179,6 +192,15 @@ public class KerberosPrincipalHostDAO {
   }
 
   /**
+   * Remove KerberosPrincipalHostEntity instances for the specified host
+   *
+   * @param keytabPath a String indicating the keytab path of principal
+   */
+  @Transactional
+  public void removeByKeytabPath(String keytabPath) {
+    remove(findByKeytabPath(keytabPath));
+  }
+  /**
    * Remove KerberosPrincipalHostEntity instance for the specified principal 
and host
    *
    * @param principalName a String indicating the name of the principal
@@ -186,8 +208,8 @@ public class KerberosPrincipalHostDAO {
    * @see 
#remove(org.apache.ambari.server.orm.entities.KerberosPrincipalHostEntity)
    */
   @Transactional
-  public void remove(String principalName, Long hostId) {
-    remove(new KerberosPrincipalHostEntity(principalName, hostId));
+  public void remove(String principalName, Long hostId, String keytabPath) {
+    remove(new KerberosPrincipalHostEntity(principalName, hostId, keytabPath));
   }
 
   /**
@@ -210,8 +232,8 @@ public class KerberosPrincipalHostDAO {
    * @return true if the requested principal exists
    */
   @RequiresSession
-  public boolean exists(String principalName, Long hostId) {
-    return find(principalName, hostId) != null;
+  public boolean exists(String principalName, Long hostId, String keytabPath) {
+    return find(principalName, hostId, keytabPath) != null;
   }
 
   /**
@@ -219,7 +241,7 @@ public class KerberosPrincipalHostDAO {
    *
    * @param entities a collection of KerberosPrincipalHostEntity items to 
remove
    */
-  private void remove(List<KerberosPrincipalHostEntity> entities) {
+  public void remove(List<KerberosPrincipalHostEntity> entities) {
     if (entities != null) {
       for (KerberosPrincipalHostEntity entity : entities) {
         entityManagerProvider.get().remove(entity);

http://git-wip-us.apache.org/repos/asf/ambari/blob/d03c24b9/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosKeytabEntity.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosKeytabEntity.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosKeytabEntity.java
new file mode 100644
index 0000000..a25931b
--- /dev/null
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosKeytabEntity.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.orm.entities;
+
+import java.util.Collection;
+
+import javax.persistence.CascadeType;
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.FetchType;
+import javax.persistence.Id;
+import javax.persistence.NamedQueries;
+import javax.persistence.NamedQuery;
+import javax.persistence.OneToMany;
+import javax.persistence.Table;
+
+@Entity
+@Table(name = "kerberos_keytab")
+@NamedQueries({
+    @NamedQuery(name = "KerberosKeytabEntity.findAll", query = "SELECT kk FROM 
KerberosKeytabEntity kk"),
+    @NamedQuery(name = "KerberosKeytabEntity.findByHost",
+        query = "SELECT kk FROM KerberosKeytabEntity kk JOIN 
kk.kerberosPrincipalHostEntities he WHERE he.hostId=:hostId")
+})
+public class KerberosKeytabEntity {
+    @Id
+    @Column(name = "keytab_path", insertable = true, updatable = false, 
nullable = false)
+    private String keytabPath = null;
+
+    @OneToMany(mappedBy = "keytabEntity", cascade = CascadeType.REMOVE, fetch 
= FetchType.LAZY)
+    private Collection<KerberosPrincipalHostEntity> 
kerberosPrincipalHostEntities;
+
+    public KerberosKeytabEntity(){
+
+    }
+
+    public KerberosKeytabEntity(String keytabPath){
+        setKeytabPath(keytabPath);
+    }
+
+    public String getKeytabPath() {
+        return keytabPath;
+    }
+
+    public void setKeytabPath(String keytabPath) {
+        this.keytabPath = keytabPath;
+    }
+
+    public Collection<KerberosPrincipalHostEntity> 
getKerberosPrincipalHostEntities() {
+        return kerberosPrincipalHostEntities;
+    }
+
+    public void 
setKerberosPrincipalHostEntities(Collection<KerberosPrincipalHostEntity> 
kerberosPrincipalHostEntities) {
+        this.kerberosPrincipalHostEntities = kerberosPrincipalHostEntities;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        KerberosKeytabEntity that = (KerberosKeytabEntity) o;
+
+        return keytabPath.equals(that.keytabPath);
+    }
+
+    @Override
+    public int hashCode() {
+        return keytabPath.hashCode();
+    }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/d03c24b9/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosPrincipalHostEntity.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosPrincipalHostEntity.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosPrincipalHostEntity.java
index bb67131..d4e80c6 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosPrincipalHostEntity.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosPrincipalHostEntity.java
@@ -23,6 +23,7 @@ import javax.persistence.Entity;
 import javax.persistence.Id;
 import javax.persistence.IdClass;
 import javax.persistence.JoinColumn;
+import javax.persistence.JoinColumns;
 import javax.persistence.ManyToOne;
 import javax.persistence.NamedQueries;
 import javax.persistence.NamedQuery;
@@ -40,7 +41,9 @@ import javax.persistence.Table;
     @NamedQuery(name = "KerberosPrincipalHostEntityFindByPrincipal",
         query = "SELECT kph FROM KerberosPrincipalHostEntity kph WHERE 
kph.principalName=:principalName"),
     @NamedQuery(name = "KerberosPrincipalHostEntityFindByHost",
-        query = "SELECT kph FROM KerberosPrincipalHostEntity kph WHERE 
kph.hostId=:hostId")
+        query = "SELECT kph FROM KerberosPrincipalHostEntity kph WHERE 
kph.hostId=:hostId"),
+    @NamedQuery(name = "KerberosPrincipalHostEntityFindByKeytabPath",
+    query = "SELECT kph FROM KerberosPrincipalHostEntity kph WHERE 
kph.keytabPath=:keytabPath"),
 })
 public class KerberosPrincipalHostEntity {
 
@@ -52,6 +55,10 @@ public class KerberosPrincipalHostEntity {
   @Column(name = "host_id", insertable = true, updatable = false, nullable = 
false)
   private Long hostId;
 
+  @Id
+  @Column(name = "keytab_path", updatable = false, nullable = false)
+  private String keytabPath;
+
   @ManyToOne
   @JoinColumn(name = "principal_name", referencedColumnName = 
"principal_name", nullable = false, insertable = false, updatable = false)
   private KerberosPrincipalEntity principalEntity;
@@ -60,6 +67,14 @@ public class KerberosPrincipalHostEntity {
   @JoinColumn(name = "host_id", referencedColumnName = "host_id", nullable = 
false, insertable = false, updatable = false)
   private HostEntity hostEntity;
 
+  @ManyToOne
+  @JoinColumns({
+          @JoinColumn(name = "keytab_path", referencedColumnName = 
"keytab_path", nullable = false, insertable = false, updatable = false)
+  })
+  private KerberosKeytabEntity keytabEntity;
+
+  @Column(name = "is_distributed", insertable = true, updatable = true, 
nullable = false)
+  private Integer isDistributed = 0;
   /**
    * Constucts an empty KerberosPrincipalHostEntity
    */
@@ -72,9 +87,23 @@ public class KerberosPrincipalHostEntity {
    * @param principalName a String indicating this 
KerberosPrincipalHostEntity's principal name
    * @param hostId a Long indicating the KerberosPrincipalHostEntity's host id
    */
-  public KerberosPrincipalHostEntity(String principalName, Long hostId) {
+  public KerberosPrincipalHostEntity(String principalName, Long hostId, String 
keytabPath) {
+    setPrincipalName(principalName);
+    setHostId(hostId);
+    setKeytabPath(keytabPath);
+  }
+
+  /**
+   * Constructs a new KerberosPrincipalHostEntity
+   *
+   * @param principalName a String indicating this 
KerberosPrincipalHostEntity's principal name
+   * @param hostId a Long indicating the KerberosPrincipalHostEntity's host id
+   */
+  public KerberosPrincipalHostEntity(String principalName, Long hostId, String 
keytabPath, boolean isDistributed) {
     setPrincipalName(principalName);
     setHostId(hostId);
+    setKeytabPath(keytabPath);
+    setDistributed(isDistributed);
   }
 
   /**
@@ -157,4 +186,28 @@ public class KerberosPrincipalHostEntity {
   public void setPrincipalEntity(KerberosPrincipalEntity principalEntity) {
     this.principalEntity = principalEntity;
   }
+
+  public String getKeytabPath() {
+    return keytabPath;
+  }
+
+  public void setKeytabPath(String keytabPath) {
+    this.keytabPath = keytabPath;
+  }
+
+  public KerberosKeytabEntity getKeytabEntity() {
+    return keytabEntity;
+  }
+
+  public void setKeytabEntity(KerberosKeytabEntity keytabEntity) {
+    this.keytabEntity = keytabEntity;
+  }
+
+  public Boolean getDistributed() {
+    return isDistributed == 1;
+  }
+
+  public void setDistributed(Boolean isDistributed) {
+    this.isDistributed = (isDistributed) ? 1 : 0;
+  }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/d03c24b9/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosPrincipalHostEntityPK.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosPrincipalHostEntityPK.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosPrincipalHostEntityPK.java
index 600bb8b..7e57e4a 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosPrincipalHostEntityPK.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosPrincipalHostEntityPK.java
@@ -36,12 +36,17 @@ public class KerberosPrincipalHostEntityPK implements 
Serializable{
   @Column(name = "host_id", insertable = false, updatable = false, nullable = 
false)
   private Long hostId = null;
 
+  @Id
+  @Column(name = "keytab_path", insertable = false, updatable = false, 
nullable = false)
+  private String keytabPath = null;
+
   public KerberosPrincipalHostEntityPK() {
   }
 
-  public KerberosPrincipalHostEntityPK(String principalName, Long hostId) {
+  public KerberosPrincipalHostEntityPK(String principalName, Long hostId, 
String keytabPath) {
     setPrincipalName(principalName);
     setHostId(hostId);
+    setKeytabPath(keytabPath);
   }
 
   /**
@@ -92,11 +97,19 @@ public class KerberosPrincipalHostEntityPK implements 
Serializable{
     KerberosPrincipalHostEntityPK that = (KerberosPrincipalHostEntityPK) o;
 
     return this.principalName.equals(that.principalName) &&
-        this.hostId.equals(that.hostId);
+        this.hostId.equals(that.hostId) && 
this.keytabPath.equals(that.keytabPath);
   }
 
   @Override
   public int hashCode() {
-    return 31 * principalName.hashCode() + hostId.hashCode();
+    return 31 * principalName.hashCode() + hostId.hashCode() + 
keytabPath.hashCode();
+  }
+
+  public String getKeytabPath() {
+    return keytabPath;
+  }
+
+  public void setKeytabPath(String keytabPath) {
+    this.keytabPath = keytabPath;
   }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/d03c24b9/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java
index 7948a60..1dc8ca8 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java
@@ -33,6 +33,7 @@ import java.util.Set;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.agent.CommandReport;
 import org.apache.ambari.server.controller.KerberosHelper;
+import 
org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosKeytab;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.ServiceComponentHost;
 import org.apache.ambari.server.state.kerberos.KerberosComponentDescriptor;
@@ -77,13 +78,12 @@ public abstract class AbstractPrepareKerberosServerAction 
extends KerberosServer
                                     Map<String, Map<String, String>> 
currentConfigurations,
                                     Map<String, Map<String, String>> 
kerberosConfigurations,
                                     boolean includeAmbariIdentity,
-                                    Map<String, Set<String>> 
propertiesToBeIgnored,
-                                    boolean excludeHeadless) throws 
AmbariException {
+                                    Map<String, Set<String>> 
propertiesToBeIgnored) throws AmbariException {
     List<Component> components = new ArrayList<>();
     for (ServiceComponentHost each : schToProcess) {
       components.add(Component.fromServiceComponentHost(each));
     }
-    processServiceComponents(cluster, kerberosDescriptor, components, 
identityFilter, dataDirectory, currentConfigurations, kerberosConfigurations, 
includeAmbariIdentity, propertiesToBeIgnored, excludeHeadless);
+    processServiceComponents(cluster, kerberosDescriptor, components, 
identityFilter, dataDirectory, currentConfigurations, kerberosConfigurations, 
includeAmbariIdentity, propertiesToBeIgnored);
   }
 
   protected void processServiceComponents(Cluster cluster, KerberosDescriptor 
kerberosDescriptor,
@@ -92,8 +92,7 @@ public abstract class AbstractPrepareKerberosServerAction 
extends KerberosServer
                                           Map<String, Map<String, String>> 
currentConfigurations,
                                           Map<String, Map<String, String>> 
kerberosConfigurations,
                                           boolean includeAmbariIdentity,
-                                          Map<String, Set<String>> 
propertiesToBeIgnored,
-                                          boolean excludeHeadless) throws 
AmbariException {
+                                          Map<String, Set<String>> 
propertiesToBeIgnored) throws AmbariException {
 
     actionLog.writeStdOut("Processing Kerberos identities and configurations");
 
@@ -125,15 +124,17 @@ public abstract class AbstractPrepareKerberosServerAction 
extends KerberosServer
         throw new AmbariException(message, e);
       }
 
+      HashMap<String, ResolvedKerberosKeytab> resolvedKeytabs = new 
HashMap<>();
+      String realm = getDefaultRealm(getCommandParameters());
+
       try {
         Map<String, Set<String>> propertiesToIgnore = null;
-
         // Iterate over the components installed on the current host to get 
the service and
         // component-level Kerberos descriptors in order to determine which 
principals,
         // keytab files, and configurations need to be created or updated.
         for (Component sch : schToProcess) {
           String hostName = sch.getHostName();
-
+          Long hostId = sch.getHostId();
           String serviceName = sch.getServiceName();
           String componentName = sch.getServiceComponentName();
 
@@ -157,7 +158,8 @@ public abstract class AbstractPrepareKerberosServerAction 
extends KerberosServer
 
             // Add service-level principals (and keytabs)
             kerberosHelper.addIdentities(kerberosIdentityDataFileWriter, 
serviceIdentities,
-                identityFilter, hostName, serviceName, componentName, 
kerberosConfigurations, currentConfigurations, excludeHeadless);
+                identityFilter, hostName, hostId, serviceName, componentName, 
kerberosConfigurations, currentConfigurations,
+                resolvedKeytabs, realm);
             propertiesToIgnore = gatherPropertiesToIgnore(serviceIdentities, 
propertiesToIgnore);
 
             KerberosComponentDescriptor componentDescriptor = 
serviceDescriptor.getComponent(componentName);
@@ -172,7 +174,8 @@ public abstract class AbstractPrepareKerberosServerAction 
extends KerberosServer
 
               // Add component-level principals (and keytabs)
               kerberosHelper.addIdentities(kerberosIdentityDataFileWriter, 
componentIdentities,
-                  identityFilter, hostName, serviceName, componentName, 
kerberosConfigurations, currentConfigurations, excludeHeadless);
+                  identityFilter, hostName, hostId, serviceName, 
componentName, kerberosConfigurations, currentConfigurations,
+                  resolvedKeytabs, realm);
               propertiesToIgnore = 
gatherPropertiesToIgnore(componentIdentities, propertiesToIgnore);
             }
           }
@@ -193,7 +196,8 @@ public abstract class AbstractPrepareKerberosServerAction 
extends KerberosServer
 
               List<KerberosIdentityDescriptor> componentIdentities = 
Collections.singletonList(identity);
               kerberosHelper.addIdentities(kerberosIdentityDataFileWriter, 
componentIdentities,
-                  identityFilter, KerberosHelper.AMBARI_SERVER_HOST_NAME, 
"AMBARI", componentName, kerberosConfigurations, currentConfigurations, 
excludeHeadless);
+                  identityFilter, StageUtils.getHostName(), 
ambariServerHostID(), "AMBARI",componentName, kerberosConfigurations, 
currentConfigurations,
+                  resolvedKeytabs, realm);
               propertiesToIgnore = 
gatherPropertiesToIgnore(componentIdentities, propertiesToIgnore);
             }
           }
@@ -202,6 +206,11 @@ public abstract class AbstractPrepareKerberosServerAction 
extends KerberosServer
         if ((propertiesToBeIgnored != null) && (propertiesToIgnore != null)) {
           propertiesToBeIgnored.putAll(propertiesToIgnore);
         }
+
+        // create database records for keytabs that must be presented on 
cluster
+        for (ResolvedKerberosKeytab keytab : resolvedKeytabs.values()) {
+          kerberosHelper.processResolvedKeytab(keytab);
+        }
       } catch (IOException e) {
         String message = String.format("Failed to write index file - %s", 
identityDataFile.getAbsolutePath());
         LOG.error(message, e);

http://git-wip-us.apache.org/repos/asf/ambari/blob/d03c24b9/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CleanupServerAction.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CleanupServerAction.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CleanupServerAction.java
index dae8254..002076d 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CleanupServerAction.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CleanupServerAction.java
@@ -32,15 +32,24 @@ import org.apache.ambari.server.controller.spi.Resource;
 import org.apache.ambari.server.controller.spi.ResourceProvider;
 import org.apache.ambari.server.controller.utilities.ClusterControllerHelper;
 import org.apache.ambari.server.controller.utilities.PredicateBuilder;
+import org.apache.ambari.server.orm.dao.KerberosKeytabDAO;
+import org.apache.ambari.server.orm.dao.KerberosPrincipalDAO;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.SecurityType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.inject.Inject;
+
 /**
  * Used to perform Kerberos Cleanup Operations as part of the Unkerberization 
process
  */
 public class CleanupServerAction extends KerberosServerAction {
+  @Inject
+  KerberosKeytabDAO kerberosKeytabDAO;
+
+  @Inject
+  KerberosPrincipalDAO kerberosPrincipalDAO;
 
   private final static Logger LOG = 
LoggerFactory.getLogger(CleanupServerAction.class);
 
@@ -102,11 +111,12 @@ public class CleanupServerAction extends 
KerberosServerAction {
 
     ClusterController clusterController = 
ClusterControllerHelper.getClusterController();
 
-    ResourceProvider artifactProvider =
-      clusterController.ensureResourceProvider(Resource.Type.Artifact);
+    ResourceProvider artifactProvider = 
clusterController.ensureResourceProvider(Resource.Type.Artifact);
 
     try {
       artifactProvider.deleteResources(new RequestImpl(null, null, null, 
null), predicate);
+      kerberosPrincipalDAO.remove(kerberosPrincipalDAO.findAll());
+      kerberosKeytabDAO.remove(kerberosKeytabDAO.findAll());
       LOG.info("Kerberos descriptor removed successfully.");
       actionLog.writeStdOut("Kerberos descriptor removed successfully.");
     } catch (NoSuchResourceException e) {

http://git-wip-us.apache.org/repos/asf/ambari/blob/d03c24b9/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/Component.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/Component.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/Component.java
index 4f1ee52..ed7642c 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/Component.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/Component.java
@@ -25,18 +25,21 @@ public class Component {
   private final String hostName;
   private final String serviceName;
   private final String serviceComponentName;
+  private final Long hostId;
 
   public static Component fromServiceComponentHost(ServiceComponentHost 
serviceComponentHost) {
     return new Component(
       serviceComponentHost.getHostName(),
       serviceComponentHost.getServiceName(),
-      serviceComponentHost.getServiceComponentName());
+      serviceComponentHost.getServiceComponentName(),
+      serviceComponentHost.getHost().getHostId());
   }
 
-  public Component(String hostName, String serviceName, String 
serviceComponentName) {
+  public Component(String hostName, String serviceName, String 
serviceComponentName, Long hostId) {
     this.hostName = hostName;
     this.serviceName = serviceName;
     this.serviceComponentName = serviceComponentName;
+    this.hostId = hostId;
   }
 
   public String getHostName() {
@@ -51,6 +54,10 @@ public class Component {
     return serviceComponentName;
   }
 
+  public Long getHostId() {
+    return hostId;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) return true;
@@ -60,6 +67,7 @@ public class Component {
       .append(hostName, component.hostName)
       .append(serviceName, component.serviceName)
       .append(serviceComponentName, component.serviceComponentName)
+      .append(hostId, component.hostId)
       .isEquals();
   }
 
@@ -69,6 +77,7 @@ public class Component {
       .append(hostName)
       .append(serviceName)
       .append(serviceComponentName)
+      .append(hostId)
       .toHashCode();
   }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/d03c24b9/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerAction.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerAction.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerAction.java
index fca1b6f..3384152 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerAction.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerAction.java
@@ -26,11 +26,10 @@ import java.util.concurrent.ConcurrentMap;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.actionmanager.HostRoleStatus;
 import org.apache.ambari.server.agent.CommandReport;
-import org.apache.ambari.server.controller.KerberosHelper;
 import org.apache.ambari.server.controller.utilities.KerberosChecker;
-import org.apache.ambari.server.orm.dao.HostDAO;
+import org.apache.ambari.server.orm.dao.KerberosKeytabDAO;
 import org.apache.ambari.server.orm.dao.KerberosPrincipalHostDAO;
-import org.apache.ambari.server.orm.entities.HostEntity;
+import org.apache.ambari.server.orm.entities.KerberosPrincipalHostEntity;
 import org.apache.ambari.server.serveraction.ActionLog;
 import org.apache.ambari.server.utils.ShellCommandUtil;
 import org.apache.ambari.server.utils.StageUtils;
@@ -63,7 +62,7 @@ public class ConfigureAmbariIdentitiesServerAction extends 
KerberosServerAction
   private KerberosPrincipalHostDAO kerberosPrincipalHostDAO;
 
   @Inject
-  private HostDAO hostDAO;
+  private KerberosKeytabDAO kerberosKeytabDAO;
 
   /**
    * Called to execute this action.  Upon invocation, calls
@@ -121,7 +120,8 @@ public class ConfigureAmbariIdentitiesServerAction extends 
KerberosServerAction
       } else {
 
         String hostName = 
identityRecord.get(KerberosIdentityDataFileReader.HOSTNAME);
-        if (hostName != null && 
hostName.equalsIgnoreCase(KerberosHelper.AMBARI_SERVER_HOST_NAME)) {
+        String serviceName = 
identityRecord.get(KerberosIdentityDataFileReader.SERVICE);
+        if (hostName != null && serviceName.equals("AMBARI")) {
           String destKeytabFilePath = 
identityRecord.get(KerberosIdentityDataFileReader.KEYTAB_FILE_PATH);
           File hostDirectory = new File(dataDirectory, hostName);
           File srcKeytabFile = new File(hostDirectory, 
DigestUtils.sha1Hex(destKeytabFilePath));
@@ -182,11 +182,7 @@ public class ConfigureAmbariIdentitiesServerAction extends 
KerberosServerAction
           groupName, groupReadable, groupWritable);
 
       String ambariServerHostName = StageUtils.getHostName();
-      HostEntity ambariServerHostEntity = 
hostDAO.findByName(ambariServerHostName);
-      Long ambariServerHostID = (ambariServerHostEntity == null)
-          ? null
-          : ambariServerHostEntity.getHostId();
-
+      Long ambariServerHostID = ambariServerHostID();
       if (ambariServerHostID == null) {
         String message = String.format("Failed to add the 
kerberos_principal_host record for %s on " +
                 "the Ambari server host since the host id for Ambari server 
host, %s, was not found." +
@@ -196,8 +192,19 @@ public class ConfigureAmbariIdentitiesServerAction extends 
KerberosServerAction
         if (actionLog != null) {
           actionLog.writeStdErr(message);
         }
-      } else if (!kerberosPrincipalHostDAO.exists(principal, 
ambariServerHostID)) {
-        kerberosPrincipalHostDAO.create(principal, ambariServerHostID);
+      } else if (!kerberosPrincipalHostDAO.exists(principal, 
ambariServerHostID, destKeytabFilePath)) {
+        if (!kerberosKeytabDAO.exists(destKeytabFilePath)) {
+          kerberosKeytabDAO.create(destKeytabFilePath);
+        }
+        if(!kerberosPrincipalHostDAO.exists(principal, ambariServerHostID, 
destKeytabFilePath)) {
+          kerberosPrincipalHostDAO.create(
+              new KerberosPrincipalHostEntity(principal, ambariServerHostID, 
destKeytabFilePath, true)
+          );
+        } else {
+          KerberosPrincipalHostEntity kphe = 
kerberosPrincipalHostDAO.find(principal, ambariServerHostID, 
destKeytabFilePath);
+          kphe.setDistributed(true);
+          kerberosPrincipalHostDAO.merge(kphe);
+        }
       }
 
       if (actionLog != null) {

http://git-wip-us.apache.org/repos/asf/ambari/blob/d03c24b9/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java
index 355f515..aa65e61 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java
@@ -205,33 +205,29 @@ public class CreateKeytabFilesServerAction extends 
KerberosServerAction {
                 ensureAmbariOnlyAccess(hostDirectory);
               }
 
-            if (hostDirectory.exists()) {
-              File destinationKeytabFile = new File(hostDirectory, 
DigestUtils.sha1Hex(keytabFilePath));
-              HostEntity hostEntity = hostDAO.findByName(hostName);
-              // in case of ambari-server identity there's no host entity for 
ambari_server host
-              if (hostEntity == null && 
!hostName.equalsIgnoreCase(KerberosHelper.AMBARI_SERVER_HOST_NAME)) {
-                message = "Failed to find HostEntity for hostname = " + 
hostName;
-                actionLog.writeStdErr(message);
-                LOG.error(message);
-                commandReport = createCommandReport(1, HostRoleStatus.FAILED, 
"{}", actionLog.getStdOut(), actionLog.getStdErr());
-                return commandReport;
-              }
+              if (hostDirectory.exists()) {
+                File destinationKeytabFile = new File(hostDirectory, 
DigestUtils.sha1Hex(keytabFilePath));
+                HostEntity hostEntity = hostDAO.findByName(hostName);
+                // in case of ambari-server identity there's no host entity 
for ambari_server host
+                if (hostEntity == null && 
!hostName.equalsIgnoreCase(KerberosHelper.AMBARI_SERVER_HOST_NAME)) {
+                  message = "Failed to find HostEntity for hostname = " + 
hostName;
+                  actionLog.writeStdErr(message);
+                  LOG.error(message);
+                  commandReport = createCommandReport(1, 
HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
+                  return commandReport;
+                }
 
-              boolean regenerateKeytabs = 
getOperationType(getCommandParameters()) == OperationType.RECREATE_ALL;
-              boolean onlyKeytabWrite = 
"true".equalsIgnoreCase(identityRecord.get(KerberosIdentityDataFileReader.ONLY_KEYTAB_WRITE));
-              boolean grabKeytabFromCache = regenerateKeytabs && 
onlyKeytabWrite;
-              // if grabKeytabFromCache=true we will try to get keytab from 
cache and send to agent, it will be true for
-              // headless cached keytabs
-              if (password == null) {
-                if (!grabKeytabFromCache && 
(hostName.equalsIgnoreCase(KerberosHelper.AMBARI_SERVER_HOST_NAME) || 
kerberosPrincipalHostDAO
-                  .exists(evaluatedPrincipal, hostEntity.getHostId()))) {
-                  // There is nothing to do for this since it must already 
exist and we don't want to
-                  // regenerate the keytab
-                  message = String.format("Skipping keytab file for %s, 
missing password indicates nothing to do", evaluatedPrincipal);
-                  LOG.debug(message);
-                } else {
-                  KerberosPrincipalEntity principalEntity = 
kerberosPrincipalDAO.find(evaluatedPrincipal);
-                  String cachedKeytabPath = (principalEntity == null) ? null : 
principalEntity.getCachedKeytabPath();
+                boolean regenerateKeytabs = 
getOperationType(getCommandParameters()) == OperationType.RECREATE_ALL;
+                if (password == null) {
+                  if (!regenerateKeytabs && 
(hostName.equalsIgnoreCase(KerberosHelper.AMBARI_SERVER_HOST_NAME) || 
kerberosPrincipalHostDAO
+                      .exists(evaluatedPrincipal, hostEntity.getHostId(), 
keytabFilePath))) {
+                    // There is nothing to do for this since it must already 
exist and we don't want to
+                    // regenerate the keytab
+                    message = String.format("Skipping keytab file for %s, 
missing password indicates nothing to do", evaluatedPrincipal);
+                    LOG.debug(message);
+                  } else {
+                    KerberosPrincipalEntity principalEntity = 
kerberosPrincipalDAO.find(evaluatedPrincipal);
+                    String cachedKeytabPath = (principalEntity == null) ? null 
: principalEntity.getCachedKeytabPath();
 
                     if (cachedKeytabPath == null) {
                       message = String.format("Failed to create keytab for %s, 
missing cached file", evaluatedPrincipal);
@@ -250,9 +246,7 @@ public class CreateKeytabFilesServerAction extends 
KerberosServerAction {
                     }
                   }
                 } else {
-                  boolean canCache = 
("true".equalsIgnoreCase(identityRecord.get(KerberosIdentityDataFileReader.KEYTAB_FILE_IS_CACHABLE)));
-
-                  Keytab keytab = createKeytab(evaluatedPrincipal, password, 
keyNumber, operationHandler, visitedPrincipalKeys != null, canCache, actionLog);
+                  Keytab keytab = createKeytab(evaluatedPrincipal, password, 
keyNumber, operationHandler, visitedPrincipalKeys != null, true, actionLog);
 
                   if (keytab != null) {
                     try {
@@ -287,7 +281,7 @@ public class CreateKeytabFilesServerAction extends 
KerberosServerAction {
                 }
               } else {
                 message = String.format("Failed to create keytab file for %s, 
the container directory does not exist: %s",
-                  evaluatedPrincipal, hostDirectory.getAbsolutePath());
+                    evaluatedPrincipal, hostDirectory.getAbsolutePath());
                 actionLog.writeStdErr(message);
                 LOG.error(message);
                 commandReport = createCommandReport(1, HostRoleStatus.FAILED, 
"{}", actionLog.getStdOut(), actionLog.getStdErr());
@@ -299,10 +293,10 @@ public class CreateKeytabFilesServerAction extends 
KerberosServerAction {
         }
       }
     } finally {
-      if(commandReport != null && 
HostRoleStatus.FAILED.toString().equals(commandReport.getStatus())) {
+      if (commandReport != null && 
HostRoleStatus.FAILED.toString().equals(commandReport.getStatus())) {
         auditEventBuilder.withReasonOfFailure(message == null ? "Unknown 
error" : message);
       }
-      if(commandReport != null || auditEventBuilder.hasPrincipal()) {
+      if (commandReport != null || auditEventBuilder.hasPrincipal()) {
         auditLog(auditEventBuilder.build());
       }
     }
@@ -359,12 +353,12 @@ public class CreateKeytabFilesServerAction extends 
KerberosServerAction {
         // and store that location so it can be reused rather than recreate it.
         KerberosPrincipalEntity principalEntity = 
kerberosPrincipalDAO.find(principal);
         if (principalEntity != null) {
-          if (!principalEntity.isService() && canCache) {
+          if (canCache) {
             File cachedKeytabFile = cacheKeytab(principal, keytab);
             String previousCachedFilePath = 
principalEntity.getCachedKeytabPath();
             String cachedKeytabFilePath = ((cachedKeytabFile == null) || 
!cachedKeytabFile.exists())
-                    ? null
-                    : cachedKeytabFile.getAbsolutePath();
+                ? null
+                : cachedKeytabFile.getAbsolutePath();
 
             principalEntity.setCachedKeytabPath(cachedKeytabFilePath);
             kerberosPrincipalDAO.merge(principalEntity);

http://git-wip-us.apache.org/repos/asf/ambari/blob/d03c24b9/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java
index 1c0853b9..08e03bd 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java
@@ -18,6 +18,7 @@
 
 package org.apache.ambari.server.serveraction.kerberos;
 
+import java.io.File;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
@@ -30,6 +31,7 @@ import 
org.apache.ambari.server.audit.event.kerberos.CreatePrincipalKerberosAudi
 import org.apache.ambari.server.orm.dao.KerberosPrincipalDAO;
 import org.apache.ambari.server.orm.dao.KerberosPrincipalHostDAO;
 import org.apache.ambari.server.orm.entities.KerberosPrincipalEntity;
+import org.apache.ambari.server.orm.entities.KerberosPrincipalHostEntity;
 import org.apache.ambari.server.security.SecurePasswordHelper;
 import org.apache.ambari.server.serveraction.ActionLog;
 import org.apache.commons.lang.StringUtils;
@@ -128,27 +130,24 @@ public class CreatePrincipalsServerAction extends 
KerberosServerAction {
       seenPrincipals.add(evaluatedPrincipal);
 
       boolean processPrincipal;
+
+      // TODO add invalidate_principals option to make keytabs invalid all 
over the cluster.
+      KerberosPrincipalEntity kerberosPrincipalEntity = 
kerberosPrincipalDAO.find(evaluatedPrincipal);
+
       boolean regenerateKeytabs = getOperationType(getCommandParameters()) == 
OperationType.RECREATE_ALL;
 
       if (regenerateKeytabs) {
-        // do not process cached identities that can be passed as is(headless 
identities)
-        processPrincipal = 
"false".equals(identityRecord.get(KerberosIdentityDataFileReader.ONLY_KEYTAB_WRITE).toLowerCase());
+        // force recreation of principal due to keytab regeneration
+        processPrincipal = true;
+      } else if (kerberosPrincipalEntity == null) {
+        // This principal has not been processed before, process it.
+        processPrincipal = true;
+      } else if 
(!StringUtils.isEmpty(kerberosPrincipalEntity.getCachedKeytabPath())) {
+        // This principal has been processed and a keytab file has been cached 
for it... do not process it.
+        processPrincipal = false;
       } else {
-        KerberosPrincipalEntity kerberosPrincipalEntity = 
kerberosPrincipalDAO.find(evaluatedPrincipal);
-
-        if (kerberosPrincipalEntity == null) {
-          // This principal has not been processed before, process it.
-          processPrincipal = true;
-        } else if 
(!StringUtils.isEmpty(kerberosPrincipalEntity.getCachedKeytabPath())) {
-          // This principal has been processed and a keytab file has been 
cached for it... do not process it.
-          processPrincipal = false;
-        } else if (kerberosPrincipalHostDAO.exists(evaluatedPrincipal)) {
-          // This principal has been processed and a keytab file has been 
distributed... do not process it.
-          processPrincipal = false;
-        } else {
-          // This principal has been processed but a keytab file for it has 
not been distributed... process it.
-          processPrincipal = true;
-        }
+        // This principal has been processed but a keytab file for it has not 
been distributed... process it.
+        processPrincipal = true;
       }
 
       if (processPrincipal) {
@@ -159,7 +158,6 @@ public class CreatePrincipalsServerAction extends 
KerberosServerAction {
         if (password == null) {
           boolean servicePrincipal = 
"service".equalsIgnoreCase(identityRecord.get(KerberosIdentityDataFileReader.PRINCIPAL_TYPE));
           CreatePrincipalResult result = createPrincipal(evaluatedPrincipal, 
servicePrincipal, kerberosConfiguration, operationHandler, regenerateKeytabs, 
actionLog);
-
           if (result == null) {
             commandReport = createCommandReport(1, HostRoleStatus.FAILED, 
"{}", actionLog.getStdOut(), actionLog.getStdErr());
           } else {
@@ -167,6 +165,20 @@ public class CreatePrincipalsServerAction extends 
KerberosServerAction {
 
             principalPasswordMap.put(evaluatedPrincipal, result.getPassword());
             principalKeyNumberMap.put(evaluatedPrincipal, 
result.getKeyNumber());
+            // invalidate given principal for all keytabs to make them 
redistributed again
+            for (KerberosPrincipalHostEntity kphe: 
kerberosPrincipalHostDAO.findByPrincipal(evaluatedPrincipal)) {
+              kphe.setDistributed(false);
+              kerberosPrincipalHostDAO.merge(kphe);
+            }
+            // invalidate principal cache
+            KerberosPrincipalEntity principalEntity = 
kerberosPrincipalDAO.find(evaluatedPrincipal);
+            try {
+              new File(principalEntity.getCachedKeytabPath()).delete();
+            } catch (Exception e) {
+              LOG.debug("Failed to delete cache file '{}'", 
principalEntity.getCachedKeytabPath());
+            }
+            principalEntity.setCachedKeytabPath(null);
+            kerberosPrincipalDAO.merge(principalEntity);
           }
         }
       }

http://git-wip-us.apache.org/repos/asf/ambari/blob/d03c24b9/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosIdentityDataFile.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosIdentityDataFile.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosIdentityDataFile.java
index ddf3d1b..ae1217c 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosIdentityDataFile.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosIdentityDataFile.java
@@ -36,6 +36,4 @@ public interface KerberosIdentityDataFile extends 
KerberosDataFile {
   String KEYTAB_FILE_GROUP_NAME = "keytab_file_group_name";
   String KEYTAB_FILE_GROUP_ACCESS = "keytab_file_group_access";
   String KEYTAB_FILE_IS_CACHABLE = "keytab_file_is_cachable";
-  String ONLY_KEYTAB_WRITE = "only_keytab_write";
-
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/d03c24b9/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosIdentityDataFileWriter.java
----------------------------------------------------------------------
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosIdentityDataFileWriter.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosIdentityDataFileWriter.java
index ea742bd..f55c6f4 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosIdentityDataFileWriter.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosIdentityDataFileWriter.java
@@ -68,8 +68,7 @@ public class KerberosIdentityDataFileWriter extends 
AbstractKerberosDataFileWrit
                           String principal, String principalType,
                           String keytabFilePath, String keytabFileOwnerName,
                           String keytabFileOwnerAccess, String 
keytabFileGroupName,
-                          String keytabFileGroupAccess, String 
keytabFileCanCache,
-                          String onlyKeytabWrite)
+                          String keytabFileGroupAccess, String 
keytabFileCanCache)
       throws IOException {
     super.appendRecord(hostName,
         serviceName,
@@ -81,8 +80,7 @@ public class KerberosIdentityDataFileWriter extends 
AbstractKerberosDataFileWrit
         keytabFileOwnerAccess,
         keytabFileGroupName,
         keytabFileGroupAccess,
-        keytabFileCanCache,
-        onlyKeytabWrite);
+        keytabFileCanCache);
   }
 
   @Override
@@ -97,7 +95,6 @@ public class KerberosIdentityDataFileWriter extends 
AbstractKerberosDataFileWrit
         KEYTAB_FILE_OWNER_ACCESS,
         KEYTAB_FILE_GROUP_NAME,
         KEYTAB_FILE_GROUP_ACCESS,
-        KEYTAB_FILE_IS_CACHABLE,
-        ONLY_KEYTAB_WRITE);
+        KEYTAB_FILE_IS_CACHABLE);
   }
 }

Reply via email to