[hadoop] branch trunk updated (176bd88 -> 6a55bae)
This is an automated email from the ASF dual-hosted git repository. ferhui pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git. from 176bd88 HADOOP-16080. hadoop-aws does not work with hadoop-client-api. (#2522) add 6a55bae HDFS-15875. Check whether file is being truncated before truncate (#2746) No new revisions were added by this update. Summary of changes: .../hdfs/server/datanode/BlockRecoveryWorker.java | 1 + .../server/datanode/DataNodeFaultInjector.java | 5 ++ .../hdfs/server/namenode/FSDirTruncateOp.java | 5 ++ .../hadoop/hdfs/server/namenode/FSNamesystem.java | 2 +- .../hdfs/server/namenode/TestFileTruncate.java | 69 +- 5 files changed, 80 insertions(+), 2 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated (b2a5656 -> 176bd88)
This is an automated email from the ASF dual-hosted git repository. stevel pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git. from b2a5656 YARN-10671.Fix Typo in TestSchedulingRequestContainerAllocation. Contributed by D M Murali Krishna Reddy. add 176bd88 HADOOP-16080. hadoop-aws does not work with hadoop-client-api. (#2522) No new revisions were added by this update. Summary of changes: .../org/apache/hadoop/fs/cosn/CosNFileSystem.java | 6 +++--- .../util/BlockingThreadPoolExecutorService.java | 5 + .../hadoop/util/SemaphoredDelegatingExecutor.java | 21 ++--- .../org/apache/hadoop/fs/TestFileSystemCaching.java | 6 -- .../hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java | 5 ++--- .../hadoop-aws/dev-support/findbugs-exclude.xml | 6 ++ .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 3 ++- .../apache/hadoop/fs/s3a/impl/DeleteOperation.java | 4 +++- .../org/apache/hadoop/fs/s3a/impl/StoreContext.java | 12 +++- .../hadoop/fs/s3a/impl/StoreContextBuilder.java | 7 +++ .../fs/s3a/s3guard/DynamoDBMetadataStore.java | 19 +++ .../s3a/ITestBlockingThreadPoolExecutorService.java | 4 ++-- .../fs/s3a/impl/ITestPartialRenamesDeletes.java | 12 +++- 13 files changed, 61 insertions(+), 49 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.3 updated: HADOOP-17528. SFTP File System: close the connection pool when closing a FileSystem (#2701)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 5ffcee8 HADOOP-17528. SFTP File System: close the connection pool
when closing a FileSystem (#2701)
5ffcee8 is described below
commit 5ffcee8979e1cfb043bec9b2c6d463911d5025dc
Author: Mike
AuthorDate: Tue Feb 23 20:03:27 2021 +0300
HADOOP-17528. SFTP File System: close the connection pool when closing a
FileSystem (#2701)
Contributed by Mike Pryakhin.
Change-Id: I59ef67c38c313f30c5e000b2fe41fcf715cf3a4b
---
.../org/apache/hadoop/fs/sftp/SFTPFileSystem.java | 31 +-
.../apache/hadoop/fs/sftp/TestSFTPFileSystem.java | 11
2 files changed, 41 insertions(+), 1 deletion(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
index 898f615..297ec04 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
@@ -24,6 +24,7 @@ import java.net.URI;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.Vector;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -50,6 +51,7 @@ public class SFTPFileSystem extends FileSystem {
private SFTPConnectionPool connectionPool;
private URI uri;
+ private final AtomicBoolean closed = new AtomicBoolean(false);
private static final int DEFAULT_SFTP_PORT = 22;
private static final int DEFAULT_MAX_CONNECTION = 5;
@@ -83,6 +85,7 @@ public class SFTPFileSystem extends FileSystem {
"Destination path %s already exist, cannot rename!";
public static final String E_FAILED_GETHOME = "Failed to get home directory";
public static final String E_FAILED_DISCONNECT = "Failed to disconnect";
+ public static final String E_FS_CLOSED = "FileSystem is closed!";
/**
* Set configuration from UI.
@@ -138,8 +141,9 @@ public class SFTPFileSystem extends FileSystem {
* @throws IOException
*/
private ChannelSftp connect() throws IOException {
-Configuration conf = getConf();
+checkNotClosed();
+Configuration conf = getConf();
String host = conf.get(FS_SFTP_HOST, null);
int port = conf.getInt(FS_SFTP_HOST_PORT, DEFAULT_SFTP_PORT);
String user = conf.get(FS_SFTP_USER_PREFIX + host, null);
@@ -703,6 +707,31 @@ public class SFTPFileSystem extends FileSystem {
}
}
+ @Override
+ public void close() throws IOException {
+if (closed.getAndSet(true)) {
+ return;
+}
+try {
+ super.close();
+} finally {
+ if (connectionPool != null) {
+connectionPool.shutdown();
+ }
+}
+ }
+
+ /**
+ * Verify that the input stream is open. Non blocking; this gives
+ * the last state of the volatile {@link #closed} field.
+ * @throws IOException if the connection is closed.
+ */
+ private void checkNotClosed() throws IOException {
+if (closed.get()) {
+ throw new IOException(uri + ": " + E_FS_CLOSED);
+}
+ }
+
@VisibleForTesting
SFTPConnectionPool getConnectionPool() {
return connectionPool;
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/sftp/TestSFTPFileSystem.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/sftp/TestSFTPFileSystem.java
index 6939262..58452f8 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/sftp/TestSFTPFileSystem.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/sftp/TestSFTPFileSystem.java
@@ -374,4 +374,15 @@ public class TestSFTPFileSystem {
assertThat(((SFTPFileSystem)
sftpFs).getConnectionPool().getLiveConnCount(),
is(1));
}
+
+ @Test
+ public void testCloseFileSystemClosesConnectionPool() throws Exception {
+SFTPFileSystem fs = (SFTPFileSystem) sftpFs;
+fs.getHomeDirectory();
+assertThat(fs.getConnectionPool().getLiveConnCount(), is(1));
+fs.close();
+assertThat(fs.getConnectionPool().getLiveConnCount(), is(0));
+///making sure that re-entrant close calls are safe
+fs.close();
+ }
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.3 updated: YARN-10671.Fix Typo in TestSchedulingRequestContainerAllocation. Contributed by D M Murali Krishna Reddy.
This is an automated email from the ASF dual-hosted git repository.
brahma pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new f12293f YARN-10671.Fix Typo in
TestSchedulingRequestContainerAllocation. Contributed by D M Murali Krishna
Reddy.
f12293f is described below
commit f12293fba28b3c512a7b639d6e809ea13f9fcd7a
Author: Brahma Reddy Battula
AuthorDate: Tue Mar 9 20:26:07 2021 +0530
YARN-10671.Fix Typo in TestSchedulingRequestContainerAllocation.
Contributed by D M Murali Krishna Reddy.
(cherry picked from commit b2a565629dba125be5b330e84c313ba26b50e80f)
---
.../scheduler/capacity/TestSchedulingRequestContainerAllocation.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocation.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocation.java
index f963e61..a4248c5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocation.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocation.java
@@ -862,7 +862,7 @@ public class TestSchedulingRequestContainerAllocation {
try {
rm.start();
- MockNM nm1 = rm.registerNode("192.168.0.1:1234:", 100*GB, 100);
+ MockNM nm1 = rm.registerNode("192.168.0.1:1234", 100*GB, 100);
MockNM nm2 = rm.registerNode("192.168.0.2:1234", 100*GB, 100);
MockNM nm3 = rm.registerNode("192.168.0.3:1234", 100*GB, 100);
MockNM nm4 = rm.registerNode("192.168.0.4:1234", 100*GB, 100);
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10671.Fix Typo in TestSchedulingRequestContainerAllocation. Contributed by D M Murali Krishna Reddy.
This is an automated email from the ASF dual-hosted git repository.
brahma pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new b2a5656 YARN-10671.Fix Typo in
TestSchedulingRequestContainerAllocation. Contributed by D M Murali Krishna
Reddy.
b2a5656 is described below
commit b2a565629dba125be5b330e84c313ba26b50e80f
Author: Brahma Reddy Battula
AuthorDate: Tue Mar 9 20:26:07 2021 +0530
YARN-10671.Fix Typo in TestSchedulingRequestContainerAllocation.
Contributed by D M Murali Krishna Reddy.
---
.../scheduler/capacity/TestSchedulingRequestContainerAllocation.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocation.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocation.java
index f963e61..a4248c5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocation.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestSchedulingRequestContainerAllocation.java
@@ -862,7 +862,7 @@ public class TestSchedulingRequestContainerAllocation {
try {
rm.start();
- MockNM nm1 = rm.registerNode("192.168.0.1:1234:", 100*GB, 100);
+ MockNM nm1 = rm.registerNode("192.168.0.1:1234", 100*GB, 100);
MockNM nm2 = rm.registerNode("192.168.0.2:1234", 100*GB, 100);
MockNM nm3 = rm.registerNode("192.168.0.3:1234", 100*GB, 100);
MockNM nm4 = rm.registerNode("192.168.0.4:1234", 100*GB, 100);
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] 02/02: YARN-9615. Add dispatcher metrics to RM. Contributed by Jonathan Hung and Qi Zhu.
This is an automated email from the ASF dual-hosted git repository.
pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit d5e035dbe1b93e6d17c0ceef68f67658a2ffbb11
Author: Peter Bacsko
AuthorDate: Tue Mar 9 14:33:14 2021 +0100
YARN-9615. Add dispatcher metrics to RM. Contributed by Jonathan Hung and
Qi Zhu.
---
.../apache/hadoop/yarn/event/AsyncDispatcher.java | 26 ++-
.../apache/hadoop/yarn/event/EventDispatcher.java | 20 ++-
.../yarn/metrics/DisableEventTypeMetrics.java | 42 +
.../hadoop/yarn/metrics/EventTypeMetrics.java | 32
.../yarn/metrics/GenericEventTypeMetrics.java | 160 ++
.../hadoop/yarn/event/TestAsyncDispatcher.java | 182 -
.../GenericEventTypeMetricsManager.java| 42 +
.../server/resourcemanager/ResourceManager.java| 18 +-
.../resourcemanager/TestResourceManager.java | 2 +
.../scheduler/TestSchedulerHealth.java | 2 +
.../scheduler/capacity/TestCapacityScheduler.java | 1 +
11 files changed, 521 insertions(+), 6 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index f9deab0..667515d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -26,6 +26,9 @@ import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
+import org.apache.hadoop.yarn.metrics.EventTypeMetrics;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.MonotonicClock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.Marker;
@@ -85,6 +88,11 @@ public class AsyncDispatcher extends AbstractService
implements Dispatcher {
protected final Map, EventHandler> eventDispatchers;
private boolean exitOnDispatchException = true;
+ private Map,
+ EventTypeMetrics> eventTypeMetricsMap;
+
+ private Clock clock = new MonotonicClock();
+
/**
* The thread name for dispatcher.
*/
@@ -98,6 +106,8 @@ public class AsyncDispatcher extends AbstractService
implements Dispatcher {
super("Dispatcher");
this.eventQueue = eventQueue;
this.eventDispatchers = new HashMap, EventHandler>();
+this.eventTypeMetricsMap = new HashMap,
+EventTypeMetrics>();
}
/**
@@ -135,7 +145,16 @@ public class AsyncDispatcher extends AbstractService
implements Dispatcher {
return;
}
if (event != null) {
-dispatch(event);
+if (eventTypeMetricsMap.
+get(event.getType().getDeclaringClass()) != null) {
+ long startTime = clock.getTime();
+ dispatch(event);
+ eventTypeMetricsMap.get(event.getType().getDeclaringClass())
+ .increment(event.getType(),
+ clock.getTime() - startTime);
+} else {
+ dispatch(event);
+}
if (printTrigger) {
//Log the latest dispatch event type
// may cause the too many events queued
@@ -369,4 +388,9 @@ public class AsyncDispatcher extends AbstractService
implements Dispatcher {
protected boolean isStopped() {
return stopped;
}
+
+ public void addMetrics(EventTypeMetrics metrics,
+ Class eventClass) {
+eventTypeMetricsMap.put(eventClass, metrics);
+ }
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
index 0969e99..f51bae6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
@@ -19,6 +19,9 @@
package org.apache.hadoop.yarn.event;
import
org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.metrics.EventTypeMetrics;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.MonotonicClock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.Marker;
@@ -47,12 +50,15 @@ public class EventDispatcher extends
private final Thread eventProcessor;
private volatile boolean stopped = false;
private boolean shouldExitOnError = true;
+ private EventTypeMetrics metrics;
private static final Logger LOG =
[hadoop] 01/02: Revert "YARN-9615. Add dispatcher metrics to RM. Contributed by Qi Zhu."
This is an automated email from the ASF dual-hosted git repository.
pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 3851994cd69516f82e4494774f9eb4d8c1e8f40d
Author: Peter Bacsko
AuthorDate: Tue Mar 9 14:32:02 2021 +0100
Revert "YARN-9615. Add dispatcher metrics to RM. Contributed by Qi Zhu."
This reverts commit 369f75b7a7f62ba454cc2a102ded6c2c947e80e5.
---
.../apache/hadoop/yarn/event/AsyncDispatcher.java | 26 +--
.../apache/hadoop/yarn/event/EventDispatcher.java | 20 +--
.../yarn/metrics/DisableEventTypeMetrics.java | 42 -
.../hadoop/yarn/metrics/EventTypeMetrics.java | 32
.../yarn/metrics/GenericEventTypeMetrics.java | 160 --
.../hadoop/yarn/event/TestAsyncDispatcher.java | 182 +
.../GenericEventTypeMetricsManager.java| 42 -
.../server/resourcemanager/ResourceManager.java| 18 +-
.../resourcemanager/TestResourceManager.java | 2 -
.../scheduler/TestSchedulerHealth.java | 2 -
.../scheduler/capacity/TestCapacityScheduler.java | 1 -
11 files changed, 6 insertions(+), 521 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index 667515d..f9deab0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -26,9 +26,6 @@ import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
-import org.apache.hadoop.yarn.metrics.EventTypeMetrics;
-import org.apache.hadoop.yarn.util.Clock;
-import org.apache.hadoop.yarn.util.MonotonicClock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.Marker;
@@ -88,11 +85,6 @@ public class AsyncDispatcher extends AbstractService
implements Dispatcher {
protected final Map, EventHandler> eventDispatchers;
private boolean exitOnDispatchException = true;
- private Map,
- EventTypeMetrics> eventTypeMetricsMap;
-
- private Clock clock = new MonotonicClock();
-
/**
* The thread name for dispatcher.
*/
@@ -106,8 +98,6 @@ public class AsyncDispatcher extends AbstractService
implements Dispatcher {
super("Dispatcher");
this.eventQueue = eventQueue;
this.eventDispatchers = new HashMap, EventHandler>();
-this.eventTypeMetricsMap = new HashMap,
-EventTypeMetrics>();
}
/**
@@ -145,16 +135,7 @@ public class AsyncDispatcher extends AbstractService
implements Dispatcher {
return;
}
if (event != null) {
-if (eventTypeMetricsMap.
-get(event.getType().getDeclaringClass()) != null) {
- long startTime = clock.getTime();
- dispatch(event);
- eventTypeMetricsMap.get(event.getType().getDeclaringClass())
- .increment(event.getType(),
- clock.getTime() - startTime);
-} else {
- dispatch(event);
-}
+dispatch(event);
if (printTrigger) {
//Log the latest dispatch event type
// may cause the too many events queued
@@ -388,9 +369,4 @@ public class AsyncDispatcher extends AbstractService
implements Dispatcher {
protected boolean isStopped() {
return stopped;
}
-
- public void addMetrics(EventTypeMetrics metrics,
- Class eventClass) {
-eventTypeMetricsMap.put(eventClass, metrics);
- }
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
index f51bae6..0969e99 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
@@ -19,9 +19,6 @@
package org.apache.hadoop.yarn.event;
import
org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.yarn.metrics.EventTypeMetrics;
-import org.apache.hadoop.yarn.util.Clock;
-import org.apache.hadoop.yarn.util.MonotonicClock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.Marker;
@@ -50,15 +47,12 @@ public class EventDispatcher extends
private final Thread eventProcessor;
private volatile boolean stopped = false;
private boolean shouldExitOnError = true;
- private EventTypeMetrics metrics;
private static final
[hadoop] branch trunk updated (369f75b -> d5e035d)
This is an automated email from the ASF dual-hosted git repository. pbacsko pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git. from 369f75b YARN-9615. Add dispatcher metrics to RM. Contributed by Qi Zhu. new 3851994 Revert "YARN-9615. Add dispatcher metrics to RM. Contributed by Qi Zhu." new d5e035d YARN-9615. Add dispatcher metrics to RM. Contributed by Jonathan Hung and Qi Zhu. The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-9615. Add dispatcher metrics to RM. Contributed by Qi Zhu.
This is an automated email from the ASF dual-hosted git repository.
pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 369f75b YARN-9615. Add dispatcher metrics to RM. Contributed by Qi
Zhu.
369f75b is described below
commit 369f75b7a7f62ba454cc2a102ded6c2c947e80e5
Author: Peter Bacsko
AuthorDate: Tue Mar 9 14:28:23 2021 +0100
YARN-9615. Add dispatcher metrics to RM. Contributed by Qi Zhu.
---
.../apache/hadoop/yarn/event/AsyncDispatcher.java | 26 ++-
.../apache/hadoop/yarn/event/EventDispatcher.java | 20 ++-
.../yarn/metrics/DisableEventTypeMetrics.java | 42 +
.../hadoop/yarn/metrics/EventTypeMetrics.java | 32
.../yarn/metrics/GenericEventTypeMetrics.java | 160 ++
.../hadoop/yarn/event/TestAsyncDispatcher.java | 182 -
.../GenericEventTypeMetricsManager.java| 42 +
.../server/resourcemanager/ResourceManager.java| 18 +-
.../resourcemanager/TestResourceManager.java | 2 +
.../scheduler/TestSchedulerHealth.java | 2 +
.../scheduler/capacity/TestCapacityScheduler.java | 1 +
11 files changed, 521 insertions(+), 6 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index f9deab0..667515d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -26,6 +26,9 @@ import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
+import org.apache.hadoop.yarn.metrics.EventTypeMetrics;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.MonotonicClock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.Marker;
@@ -85,6 +88,11 @@ public class AsyncDispatcher extends AbstractService
implements Dispatcher {
protected final Map, EventHandler> eventDispatchers;
private boolean exitOnDispatchException = true;
+ private Map,
+ EventTypeMetrics> eventTypeMetricsMap;
+
+ private Clock clock = new MonotonicClock();
+
/**
* The thread name for dispatcher.
*/
@@ -98,6 +106,8 @@ public class AsyncDispatcher extends AbstractService
implements Dispatcher {
super("Dispatcher");
this.eventQueue = eventQueue;
this.eventDispatchers = new HashMap, EventHandler>();
+this.eventTypeMetricsMap = new HashMap,
+EventTypeMetrics>();
}
/**
@@ -135,7 +145,16 @@ public class AsyncDispatcher extends AbstractService
implements Dispatcher {
return;
}
if (event != null) {
-dispatch(event);
+if (eventTypeMetricsMap.
+get(event.getType().getDeclaringClass()) != null) {
+ long startTime = clock.getTime();
+ dispatch(event);
+ eventTypeMetricsMap.get(event.getType().getDeclaringClass())
+ .increment(event.getType(),
+ clock.getTime() - startTime);
+} else {
+ dispatch(event);
+}
if (printTrigger) {
//Log the latest dispatch event type
// may cause the too many events queued
@@ -369,4 +388,9 @@ public class AsyncDispatcher extends AbstractService
implements Dispatcher {
protected boolean isStopped() {
return stopped;
}
+
+ public void addMetrics(EventTypeMetrics metrics,
+ Class eventClass) {
+eventTypeMetricsMap.put(eventClass, metrics);
+ }
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
index 0969e99..f51bae6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
@@ -19,6 +19,9 @@
package org.apache.hadoop.yarn.event;
import
org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.metrics.EventTypeMetrics;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.MonotonicClock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.Marker;
@@ -47,12 +50,15 @@ public class EventDispatcher extends
private final Thread eventProcessor;
private volatile boolean stopped = false;
[hadoop] branch trunk updated: YARN-10679. Better logging of uncaught exceptions throughout SLS. Contributed by Szilard Nemeth.
This is an automated email from the ASF dual-hosted git repository.
pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new c3aa413 YARN-10679. Better logging of uncaught exceptions throughout
SLS. Contributed by Szilard Nemeth.
c3aa413 is described below
commit c3aa413ee3f8b24fb216d7136e71d3b0b770
Author: Peter Bacsko
AuthorDate: Tue Mar 9 14:02:12 2021 +0100
YARN-10679. Better logging of uncaught exceptions throughout SLS.
Contributed by Szilard Nemeth.
---
.../apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java| 7 +++
.../org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java| 6 +++---
.../java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java | 6 +-
.../src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java | 8 ++--
4 files changed, 17 insertions(+), 10 deletions(-)
diff --git
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
index a318204..e0cb151 100644
---
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
+++
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
@@ -63,7 +63,6 @@ import com.codahale.metrics.Timer;
@Unstable
public class SLSCapacityScheduler extends CapacityScheduler implements
SchedulerWrapper,Configurable {
-
private Configuration conf;
private Map appQueueMap =
@@ -99,7 +98,7 @@ public class SLSCapacityScheduler extends CapacityScheduler
implements
CapacityScheduler.class);
schedulerMetrics.init(this, conf);
} catch (Exception e) {
-e.printStackTrace();
+LOG.error("Caught exception while initializing schedulerMetrics", e);
}
}
}
@@ -129,7 +128,7 @@ public class SLSCapacityScheduler extends CapacityScheduler
implements
updateQueueWithAllocateRequest(allocation, attemptId,
resourceRequests, containerIds);
} catch (IOException e) {
- e.printStackTrace();
+ LOG.error("Caught exception while executing finally block", e);
}
}
} else {
@@ -376,7 +375,7 @@ public class SLSCapacityScheduler extends CapacityScheduler
implements
schedulerMetrics.tearDown();
}
} catch (Exception e) {
- e.printStackTrace();
+ LOG.error("Caught exception while stopping service", e);
}
super.serviceStop();
}
diff --git
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
index 7298c05..84549bc 100644
---
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
+++
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
@@ -92,7 +92,7 @@ public class SLSFairScheduler extends FairScheduler
FairScheduler.class);
schedulerMetrics.init(this, conf);
} catch (Exception e) {
-e.printStackTrace();
+LOG.error("Caught exception while initializing schedulerMetrics", e);
}
}
}
@@ -122,7 +122,7 @@ public class SLSFairScheduler extends FairScheduler
updateQueueWithAllocateRequest(allocation, attemptId,
resourceRequests, containerIds);
} catch (IOException e) {
- e.printStackTrace();
+ LOG.error("Caught exception while executing finally block", e);
}
}
} else {
@@ -332,7 +332,7 @@ public class SLSFairScheduler extends FairScheduler
schedulerMetrics.tearDown();
}
} catch (Exception e) {
- e.printStackTrace();
+ LOG.error("Caught exception while stopping service", e);
}
super.serviceStop();
}
diff --git
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java
index 19cfe88..1678455 100644
---
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java
+++
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java
@@ -26,10 +26,14 @@ import java.util.concurrent.TimeUnit;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@Private
@Unstable
public class TaskRunner {
+ private static final Logger LOG = LoggerFactory.getLogger(TaskRunner.class);
+
@Private
@Unstable
public abstract static class Task implements Runnable,
[hadoop] branch trunk updated: YARN-10681. Fix assertion failure message in BaseSLSRunnerTest. Contributed by Szilard Nemeth.
This is an automated email from the ASF dual-hosted git repository.
pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 099f58f YARN-10681. Fix assertion failure message in
BaseSLSRunnerTest. Contributed by Szilard Nemeth.
099f58f is described below
commit 099f58f8f41d6439643e625794c66ad932bae17b
Author: Peter Bacsko
AuthorDate: Tue Mar 9 13:22:48 2021 +0100
YARN-10681. Fix assertion failure message in BaseSLSRunnerTest. Contributed
by Szilard Nemeth.
---
.../src/test/java/org/apache/hadoop/yarn/sls/BaseSLSRunnerTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/BaseSLSRunnerTest.java
b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/BaseSLSRunnerTest.java
index bfbd592..1bb4710 100644
---
a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/BaseSLSRunnerTest.java
+++
b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/BaseSLSRunnerTest.java
@@ -126,7 +126,7 @@ public abstract class BaseSLSRunnerTest {
if (!exceptionList.isEmpty()) {
sls.stop();
-Assert.fail("TestSLSRunner catched exception from child thread "
+Assert.fail("TestSLSRunner caught exception from child thread "
+ "(TaskRunner.TaskDefinition): " + exceptionList);
break;
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10677. Logger of SLSFairScheduler is provided with the wrong class. Contributed by Szilard Nemeth.
This is an automated email from the ASF dual-hosted git repository.
pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7f522c9 YARN-10677. Logger of SLSFairScheduler is provided with the
wrong class. Contributed by Szilard Nemeth.
7f522c9 is described below
commit 7f522c92fabc142c1b2e694e85a731a815405504
Author: Peter Bacsko
AuthorDate: Tue Mar 9 12:53:32 2021 +0100
YARN-10677. Logger of SLSFairScheduler is provided with the wrong class.
Contributed by Szilard Nemeth.
---
.../java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java| 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
index c832436..7298c05 100644
---
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
+++
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
@@ -66,7 +66,8 @@ public class SLSFairScheduler extends FairScheduler
new ConcurrentHashMap<>();
// logger
- private static final Logger LOG =
LoggerFactory.getLogger(SLSCapacityScheduler.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SLSFairScheduler.class);
public SchedulerMetrics getSchedulerMetrics() {
return schedulerMetrics;
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10678. Try blocks without catch blocks in SLS scheduler classes can swallow other exceptions. Contributed by Szilard Nemeth.
This is an automated email from the ASF dual-hosted git repository.
pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new ea90cd3 YARN-10678. Try blocks without catch blocks in SLS scheduler
classes can swallow other exceptions. Contributed by Szilard Nemeth.
ea90cd3 is described below
commit ea90cd3556f9f4c53f657266dfbfcc7bd4b031ce
Author: Peter Bacsko
AuthorDate: Tue Mar 9 12:03:53 2021 +0100
YARN-10678. Try blocks without catch blocks in SLS scheduler classes can
swallow other exceptions. Contributed by Szilard Nemeth.
---
.../org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java | 3 +++
.../java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java| 3 +++
2 files changed, 6 insertions(+)
diff --git
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
index f34a3d3..a318204 100644
---
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
+++
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
@@ -119,6 +119,9 @@ public class SLSCapacityScheduler extends CapacityScheduler
implements
containerIds, strings,
strings2, updateRequests);
return allocation;
+ } catch (Exception e) {
+LOG.error("Caught exception from allocate", e);
+throw e;
} finally {
context.stop();
schedulerMetrics.increaseSchedulerAllocationCounter();
diff --git
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
index 09ef2cf..c832436 100644
---
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
+++
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
@@ -111,6 +111,9 @@ public class SLSFairScheduler extends FairScheduler
schedulingRequests, containerIds,
blacklistAdditions, blacklistRemovals, updateRequests);
return allocation;
+ } catch (Exception e) {
+LOG.error("Caught exception from allocate", e);
+throw e;
} finally {
context.stop();
schedulerMetrics.increaseSchedulerAllocationCounter();
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10675. Consolidate YARN-10672 and YARN-10447. Contributed by Szilard Nemeth.
This is an automated email from the ASF dual-hosted git repository.
pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 20416bc YARN-10675. Consolidate YARN-10672 and YARN-10447.
Contributed by Szilard Nemeth.
20416bc is described below
commit 20416bc62dce85c2c1aecbc36b8373993a3c55c3
Author: Peter Bacsko
AuthorDate: Tue Mar 9 11:34:37 2021 +0100
YARN-10675. Consolidate YARN-10672 and YARN-10447. Contributed by Szilard
Nemeth.
---
.../scheduler/capacity/CapacityScheduler.java| 20 +++-
.../scheduler/capacity/TestLeafQueue.java| 7 +--
2 files changed, 8 insertions(+), 19 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 467dacb..d259c93 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -249,8 +249,6 @@ public class CapacityScheduler extends
private CSMaxRunningAppsEnforcer maxRunningEnforcer;
- private boolean activitiesManagerEnabled = true;
-
public CapacityScheduler() {
super(CapacityScheduler.class.getName());
this.maxRunningEnforcer = new CSMaxRunningAppsEnforcer(this);
@@ -351,9 +349,7 @@ public class CapacityScheduler extends
this.workflowPriorityMappingsMgr = new WorkflowPriorityMappingsManager();
this.activitiesManager = new ActivitiesManager(rmContext);
- if (activitiesManagerEnabled) {
-activitiesManager.init(conf);
- }
+ activitiesManager.init(conf);
initializeQueues(this.conf);
this.isLazyPreemptionEnabled = conf.getLazyPreemptionEnabled();
@@ -411,9 +407,7 @@ public class CapacityScheduler extends
private void startSchedulerThreads() {
writeLock.lock();
try {
- if (activitiesManagerEnabled) {
-activitiesManager.start();
- }
+ activitiesManager.start();
if (scheduleAsynchronously) {
Preconditions.checkNotNull(asyncSchedulerThreads,
"asyncSchedulerThreads is null");
@@ -447,9 +441,7 @@ public class CapacityScheduler extends
public void serviceStop() throws Exception {
writeLock.lock();
try {
- if (activitiesManagerEnabled) {
-this.activitiesManager.stop();
- }
+ this.activitiesManager.stop();
if (scheduleAsynchronously && asyncSchedulerThreads != null) {
for (Thread t : asyncSchedulerThreads) {
t.interrupt();
@@ -3480,7 +3472,6 @@ public class CapacityScheduler extends
this.maxRunningEnforcer = enforcer;
}
-
/**
* Returning true as capacity scheduler supports placement constraints.
*/
@@ -3490,11 +3481,6 @@ public class CapacityScheduler extends
}
@VisibleForTesting
- public void setActivitiesManagerEnabled(boolean enabled) {
-this.activitiesManagerEnabled = enabled;
- }
-
- @VisibleForTesting
public void setQueueManager(CapacitySchedulerQueueManager qm) {
this.queueManager = qm;
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index 6262d85..4b02310 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -162,9 +162,13 @@ public class TestLeafQueue {
private void setUpInternal(ResourceCalculator rC, boolean withNodeLabels)
throws Exception {
CapacityScheduler spyCs = new CapacityScheduler();
-spyCs.setActivitiesManagerEnabled(false);
queues = new CSQueueStore();
cs = spy(spyCs);
+
+//All stub calls on the spy object of the 'cs' field should happen
+//before cs.start() is invoked. See YARN-10672 for more details.
+when(cs.getNumClusterNodes()).thenReturn(3);
+
[hadoop] branch trunk updated: YARN-10676. Improve code quality in TestTimelineAuthenticationFilterForV1. Contributed by Szilard Nemeth.
This is an automated email from the ASF dual-hosted git repository.
pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new e472ee2 YARN-10676. Improve code quality in
TestTimelineAuthenticationFilterForV1. Contributed by Szilard Nemeth.
e472ee2 is described below
commit e472ee2aa580f9af3d95ff0267eef69ef5e3a7d5
Author: Peter Bacsko
AuthorDate: Tue Mar 9 11:28:24 2021 +0100
YARN-10676. Improve code quality in TestTimelineAuthenticationFilterForV1.
Contributed by Szilard Nemeth.
---
.../TestTimelineAuthenticationFilterForV1.java | 98 ++
1 file changed, 62 insertions(+), 36 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java
index 0e1310e..cf2db2f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.yarn.server.timeline.security;
-import static org.junit.Assert.assertTrue;
-
import java.io.File;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
@@ -49,12 +47,16 @@ import
org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistor
import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import static
org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_HTTP_AUTH_PREFIX;
+import static org.junit.Assert.fail;
+
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Test cases for authentication via TimelineAuthenticationFilter while
@@ -62,17 +64,20 @@ import org.junit.runners.Parameterized;
*/
@RunWith(Parameterized.class)
public class TestTimelineAuthenticationFilterForV1 {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestTimelineAuthenticationFilterForV1.class);
private static final String FOO_USER = "foo";
private static final String BAR_USER = "bar";
private static final String HTTP_USER = "HTTP";
+ private static final String PRINCIPAL = HTTP_USER + "/localhost";
private static final File TEST_ROOT_DIR = new File(
System.getProperty("test.build.dir", "target/test-dir"),
TestTimelineAuthenticationFilterForV1.class.getName() + "-root");
- private static File httpSpnegoKeytabFile = new File(
+ private static final File httpSpnegoKeytabFile = new File(
KerberosTestUtils.getKeytabFile());
- private static String httpSpnegoPrincipal =
+ private static final String httpSpnegoPrincipal =
KerberosTestUtils.getServerPrincipal();
private static final String BASEDIR =
System.getProperty("test.build.dir", "target/test-dir") + "/"
@@ -100,16 +105,16 @@ public class TestTimelineAuthenticationFilterForV1 {
testMiniKDC = new MiniKdc(MiniKdc.createConf(), TEST_ROOT_DIR);
testMiniKDC.start();
testMiniKDC.createPrincipal(
- httpSpnegoKeytabFile, HTTP_USER + "/localhost");
+ httpSpnegoKeytabFile, PRINCIPAL);
} catch (Exception e) {
- assertTrue("Couldn't setup MiniKDC", false);
+ LOG.error("Failed to setup MiniKDC", e);
+ fail("Couldn't setup MiniKDC");
}
try {
testTimelineServer = new ApplicationHistoryServer();
conf = new Configuration(false);
- conf.setStrings(TIMELINE_HTTP_AUTH_PREFIX + "type",
- "kerberos");
+ conf.setStrings(TIMELINE_HTTP_AUTH_PREFIX + "type", "kerberos");
conf.set(TIMELINE_HTTP_AUTH_PREFIX +
KerberosAuthenticationHandler.PRINCIPAL, httpSpnegoPrincipal);
conf.set(TIMELINE_HTTP_AUTH_PREFIX +
@@ -150,8 +155,8 @@ public class TestTimelineAuthenticationFilterForV1 {
testTimelineServer.init(conf);
testTimelineServer.start();
} catch (Exception e) {
- e.printStackTrace();
- assertTrue("Couldn't setup TimelineServer", false);
+ LOG.error("Failed to setup TimelineServer", e);
+ fail("Couldn't setup TimelineServer");
}
}
@@ -181,7 +186,7 @@ public class TestTimelineAuthenticationFilterForV1 {
