This is an automated email from the ASF dual-hosted git repository.
eyang pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new dfceffa YARN-9147. Rmove auxiliary services when manifest file is
removed. Contributed by Billie Rinaldi
dfceffa is described below
commit dfceffa70d54f5aa9cb2fe200653e83227c44f62
Author: Eric Yang <[email protected]>
AuthorDate: Thu Jan 3 12:54:16 2019 -0500
YARN-9147. Rmove auxiliary services when manifest file is removed.
Contributed by Billie Rinaldi
---
.../nodemanager/containermanager/AuxServices.java | 31 +++++++++++++++-------
.../containermanager/TestAuxServices.java | 13 +++++++++
2 files changed, 35 insertions(+), 9 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
index 59a76da..cde0683 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
@@ -40,6 +40,7 @@ import java.util.regex.Pattern;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.security.authorize.AccessControlList;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceConfiguration;
@@ -413,6 +414,7 @@ public class AuxServices extends AbstractService
serviceRecordMap.remove(sName);
serviceMetaData.remove(sName);
if (s != null) {
+ LOG.info("Removing aux service " + sName);
stopAuxService(s);
}
}
@@ -557,8 +559,24 @@ public class AuxServices extends AbstractService
* @param startServices if true starts services, otherwise only inits
services
* @throws IOException
*/
- private synchronized void loadManifest(Configuration conf, boolean
+ @VisibleForTesting
+ protected synchronized void loadManifest(Configuration conf, boolean
startServices) throws IOException {
+ if (manifest == null) {
+ return;
+ }
+ if (!manifestFS.exists(manifest)) {
+ if (serviceMap.isEmpty()) {
+ return;
+ }
+ LOG.info("Manifest file " + manifest + " doesn't exist, stopping " +
+ "auxiliary services");
+ Set<String> servicesToRemove = new HashSet<>(serviceMap.keySet());
+ for (String sName : servicesToRemove) {
+ maybeRemoveAuxService(sName);
+ }
+ return;
+ }
AuxServiceRecords services = maybeReadManifestFile();
if (services == null) {
// read did not occur or no changes detected
@@ -596,15 +614,10 @@ public class AuxServices extends AbstractService
}
// remove aux services that do not appear in the manifest
- List<String> servicesToRemove = new ArrayList<>();
- for (String sName : serviceMap.keySet()) {
- if (!loadedAuxServices.contains(sName)) {
- foundChanges = true;
- servicesToRemove.add(sName);
- }
- }
+ Set<String> servicesToRemove = new HashSet<>(serviceMap.keySet());
+ servicesToRemove.removeAll(loadedAuxServices);
for (String sName : servicesToRemove) {
- LOG.info("Removing aux service " + sName);
+ foundChanges = true;
maybeRemoveAuxService(sName);
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
index 1f944b4..cb8abc7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
@@ -885,4 +885,17 @@ public class TestAuxServices {
aux.init(conf);
assertEquals(2, aux.getServices().size());
}
+
+ @Test
+ public void testRemoveManifest() throws IOException {
+ Assume.assumeTrue(useManifest);
+ Configuration conf = getABConf();
+ final AuxServices aux = new AuxServices(MOCK_AUX_PATH_HANDLER,
+ MOCK_CONTEXT, MOCK_DEL_SERVICE);
+ aux.init(conf);
+ assertEquals(2, aux.getServices().size());
+ manifest.delete();
+ aux.loadManifest(conf, false);
+ assertEquals(0, aux.getServices().size());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
