[hadoop] branch trunk updated: YARN-9460. QueueACLsManager and ReservationsACLManager should not use instanceof checks. Contributed by Bilwa S T.
This is an automated email from the ASF dual-hosted git repository.
surendralilhore pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new b2facc8 YARN-9460. QueueACLsManager and ReservationsACLManager should
not use instanceof checks. Contributed by Bilwa S T.
b2facc8 is described below
commit b2facc84a1b48b9dcbe0816e120778d2100b320e
Author: Surendra Singh Lilhore
AuthorDate: Sat Jun 20 19:55:23 2020 +0530
YARN-9460. QueueACLsManager and ReservationsACLManager should not use
instanceof checks. Contributed by Bilwa S T.
---
.../server/resourcemanager/ResourceManager.java| 2 +-
.../reservation/AbstractReservationSystem.java | 10 +-
.../security/CapacityQueueACLsManager.java | 111
.../security/CapacityReservationsACLsManager.java | 46
.../security/FairQueueACLsManager.java | 72 +
.../security/FairReservationsACLsManager.java | 42
.../security/GenericQueueACLsManager.java | 55 ++
.../resourcemanager/security/QueueACLsManager.java | 116 +
.../security/ReservationsACLsManager.java | 44 ++--
.../resourcemanager/security/package-info.java | 28 +
.../server/resourcemanager/TestClientRMTokens.java | 5 +-
11 files changed, 402 insertions(+), 129 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 48cbd8f..836a5ec 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -438,7 +438,7 @@ public class ResourceManager extends CompositeService
protected QueueACLsManager createQueueACLsManager(ResourceScheduler
scheduler,
Configuration conf) {
-return new QueueACLsManager(scheduler, conf);
+return QueueACLsManager.getQueueACLsManager(scheduler, conf);
}
@VisibleForTesting
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
index 5b8772c..d9e4be9 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
@@ -50,6 +50,8 @@ import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import
org.apache.hadoop.yarn.server.resourcemanager.security.CapacityReservationsACLsManager;
+import
org.apache.hadoop.yarn.server.resourcemanager.security.FairReservationsACLsManager;
import
org.apache.hadoop.yarn.server.resourcemanager.security.ReservationsACLsManager;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.UTCClock;
@@ -173,7 +175,13 @@ public abstract class AbstractReservationSystem extends
AbstractService
YarnConfiguration.DEFAULT_YARN_RESERVATION_ACL_ENABLE)
&& conf.getBoolean(YarnConfiguration.YARN_ACL_ENABLE,
YarnConfiguration.DEFAULT_YARN_ACL_ENABLE)) {
- reservationsACLsManager = new ReservationsACLsManager(scheduler, conf);
+ if (scheduler instanceof CapacityScheduler) {
+reservationsACLsManager = new
CapacityReservationsACLsManager(scheduler,
+conf);
+ } else if (scheduler instanceof FairScheduler) {
+reservationsACLsManager = new FairReservationsACLsManager(scheduler,
+conf);
+ }
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/CapacityQueueACLsManager.java
[hadoop] branch branch-3.3 updated: MAPREDUCE-7281. Fix NoClassDefFoundError on 'mapred minicluster'. (#2077)
This is an automated email from the ASF dual-hosted git repository.
iwasakims pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 56d72ad MAPREDUCE-7281. Fix NoClassDefFoundError on 'mapred
minicluster'. (#2077)
56d72ad is described below
commit 56d72adbdd2a1c8647ebd974a2ecb2904f92799c
Author: Masatake Iwasaki
AuthorDate: Sat Jun 20 07:37:55 2020 +0900
MAPREDUCE-7281. Fix NoClassDefFoundError on 'mapred minicluster'. (#2077)
(cherry picked from commit 8fd0fdf8890b4c0cf3ea977be8fae8fa17e6599b)
---
hadoop-mapreduce-project/bin/mapred | 2 ++
1 file changed, 2 insertions(+)
diff --git a/hadoop-mapreduce-project/bin/mapred
b/hadoop-mapreduce-project/bin/mapred
index 9773ec8..3e52556 100755
--- a/hadoop-mapreduce-project/bin/mapred
+++ b/hadoop-mapreduce-project/bin/mapred
@@ -105,6 +105,8 @@ function mapredcmd_case
minicluster)
hadoop_add_classpath
"${HADOOP_YARN_HOME}/${YARN_DIR}/timelineservice"'/*'
hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_DIR}/test"'/*'
+ junitjar=$(echo "${HADOOP_TOOLS_LIB_JARS_DIR}"/junit-[0-9]*.jar)
+ hadoop_add_classpath "${junitjar}"
HADOOP_CLASSNAME=org.apache.hadoop.mapreduce.MiniHadoopClusterManager
;;
*)
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: HDFS-15418. ViewFileSystemOverloadScheme should represent mount links as non symlinks. Contributed by Uma Maheswara Rao G.
This is an automated email from the ASF dual-hosted git repository.
umamahesh pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new b27810a HDFS-15418. ViewFileSystemOverloadScheme should represent
mount links as non symlinks. Contributed by Uma Maheswara Rao G.
b27810a is described below
commit b27810aa6015253866ccc0ccc7247ad7024c0730
Author: Uma Maheswara Rao G
AuthorDate: Sat Jun 20 00:32:02 2020 -0700
HDFS-15418. ViewFileSystemOverloadScheme should represent mount links as
non symlinks. Contributed by Uma Maheswara Rao G.
---
.../org/apache/hadoop/fs/viewfs/Constants.java | 8 ++
.../apache/hadoop/fs/viewfs/ViewFileSystem.java| 71 +++
.../fs/viewfs/ViewFileSystemOverloadScheme.java| 20 +++-
.../java/org/apache/hadoop/fs/viewfs/ViewFs.java | 80 -
.../viewfs/TestViewFsOverloadSchemeListStatus.java | 132 +
.../hadoop/fs/viewfs/TestViewfsFileStatus.java | 4 +-
.../src/site/markdown/ViewFsOverloadScheme.md | 42 +++
...SystemOverloadSchemeHdfsFileSystemContract.java | 5 +
...ViewFileSystemOverloadSchemeWithHdfsScheme.java | 9 ++
9 files changed, 295 insertions(+), 76 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
index 0a5d4b4..f454f63 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
@@ -90,4 +90,12 @@ public interface Constants {
String CONFIG_VIEWFS_ENABLE_INNER_CACHE = "fs.viewfs.enable.inner.cache";
boolean CONFIG_VIEWFS_ENABLE_INNER_CACHE_DEFAULT = true;
+
+ /**
+ * Enable ViewFileSystem to show mountlinks as symlinks.
+ */
+ String CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS =
+ "fs.viewfs.mount.links.as.symlinks";
+
+ boolean CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS_DEFAULT = true;
}
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index 895edc0..1ee06e0 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.fs.viewfs;
import static
org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs;
import static
org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE;
import static
org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE_DEFAULT;
+import static
org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS;
+import static
org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS_DEFAULT;
import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555;
import java.io.FileNotFoundException;
@@ -527,10 +529,18 @@ public class ViewFileSystem extends FileSystem {
* the target path FileStatus object. The target path will be available via
* getSymlink on that children's FileStatus object. Since it represents as
* symlink, isDirectory on that children's FileStatus will return false.
+ * This behavior can be changed by setting an advanced configuration
+ * fs.viewfs.mount.links.as.symlinks to false. In this case, mount points
will
+ * be represented as non-symlinks and all the file/directory attributes like
+ * permissions, isDirectory etc will be assigned from it's resolved target
+ * directory/file.
*
* If you want to get the FileStatus of target path for that children, you
may
* want to use GetFileStatus API with that children's symlink path. Please
see
* {@link ViewFileSystem#getFileStatus(Path f)}
+ *
+ * Note: In ViewFileSystem, by default the mount links are represented as
+ * symlinks.
*/
@Override
public FileStatus[] listStatus(final Path f) throws AccessControlException,
@@ -1114,6 +1124,7 @@ public class ViewFileSystem extends FileSystem {
final long creationTime; // of the the mount table
final UserGroupInformation ugi; // the user/group of user who created
mtable
final URI myUri;
+private final boolean showMountLinksAsSymlinks;
public InternalDirOfViewFs(final InodeTree.INodeDir dir,
final long cTime, final UserGroupInformation ugi, URI uri,
@@ -1127,6 +1138,9 @@ public class ViewFileSystem extends FileSystem {
theInternalDir = dir;
creationTime = cTime;
this.ugi = ugi;
+ showMountLinksAsSymlinks = config
+
