This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 0bf6cb5c39 HDDS-8573. Verify default setting for DN root dir to
restrict non-admin access (#4682)
0bf6cb5c39 is described below
commit 0bf6cb5c397eb2bda9dd455a1edab4ef4177dcc0
Author: Arafat2198 <[email protected]>
AuthorDate: Wed May 24 22:20:50 2023 +0530
HDDS-8573. Verify default setting for DN root dir to restrict non-admin
access (#4682)
---
.../apache/hadoop/hdds/recon/ReconConfigKeys.java | 5 +
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 4 +
.../org/apache/hadoop/hdds/server/ServerUtils.java | 96 +++++++++++-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 8 +
.../common/src/main/resources/ozone-default.xml | 36 +++++
.../apache/hadoop/hdds/server/TestServerUtils.java | 162 ++++++++++++++++++++-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 4 +
7 files changed, 307 insertions(+), 8 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/recon/ReconConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/recon/ReconConfigKeys.java
index c1f36fb6de..21eb4a1a4d 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/recon/ReconConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/recon/ReconConfigKeys.java
@@ -32,6 +32,11 @@ public final class ReconConfigKeys {
public static final String RECON_SCM_CONFIG_PREFIX = "ozone.recon.scmconfig";
+ public static final String OZONE_RECON_DB_DIR = "ozone.recon.db.dir";
+
+ public static final String OZONE_RECON_DB_DIRS_PERMISSIONS =
+ "ozone.recon.db.dirs.permissions";
+
public static final String OZONE_RECON_DATANODE_ADDRESS_KEY =
"ozone.recon.datanode.address";
public static final String OZONE_RECON_ADDRESS_KEY =
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 2c80c74f29..1ed7e0a806 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -36,6 +36,10 @@ public final class ScmConfigKeys {
// performance.
public static final String OZONE_SCM_DB_DIRS = "ozone.scm.db.dirs";
+ // SCM DB directory permission
+ public static final String OZONE_SCM_DB_DIRS_PERMISSIONS =
+ "ozone.scm.db.dirs.permissions";
+
public static final String DFS_CONTAINER_RATIS_ENABLED_KEY
= "dfs.container.ratis.enabled";
public static final boolean DFS_CONTAINER_RATIS_ENABLED_DEFAULT
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java
index 08f04f1cde..6c0272e256 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java
@@ -19,14 +19,20 @@ package org.apache.hadoop.hdds.server;
import java.io.File;
import java.net.InetSocketAddress;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.PosixFilePermissions;
import java.util.Collection;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.recon.ReconConfigKeys;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.http.client.methods.HttpRequestBase;
@@ -41,6 +47,7 @@ public final class ServerUtils {
private static final Logger LOG = LoggerFactory.getLogger(
ServerUtils.class);
+
private ServerUtils() {
}
@@ -152,8 +159,10 @@ public final class ServerUtils {
}
/**
- * Utility method to get value of a given key that corresponds to a DB
- * directory.
+ * Utility method to retrieve the value of a key representing a DB directory
+ * and create a File object for the directory. The method also sets the
+ * directory permissions based on the configuration.
+ *
* @param conf configuration bag
* @param key Key to test
* @param componentName Which component's key is this
@@ -163,7 +172,6 @@ public final class ServerUtils {
String key,
String componentName) {
final Collection<String> metadirs = conf.getTrimmedStringCollection(key);
-
if (metadirs.size() > 1) {
throw new IllegalArgumentException(
"Bad config setting " + key +
@@ -178,12 +186,90 @@ public final class ServerUtils {
dbDirPath + " specified in configuration setting " +
key);
}
+ try {
+ Path path = dbDirPath.toPath();
+ // Fetch the permissions for the respective component from the config
+ String permissionValue = getPermissions(key, conf);
+ String symbolicPermission = getSymbolicPermission(permissionValue);
+
+ // Set the permissions for the directory
+ Files.setPosixFilePermissions(path,
+ PosixFilePermissions.fromString(symbolicPermission));
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to set directory permissions for " +
+ dbDirPath + ": " + e.getMessage(), e);
+ }
return dbDirPath;
}
return null;
}
+ /**
+ * Fetches the symbolic representation of the permission value.
+ *
+ * @param permissionValue the permission value (octal or symbolic)
+ * @return the symbolic representation of the permission value
+ */
+ private static String getSymbolicPermission(String permissionValue) {
+ if (isSymbolic(permissionValue)) {
+ // For symbolic representation, use it directly
+ return permissionValue;
+ } else {
+ // For octal representation, convert it to FsPermission object and then
+ // to symbolic representation
+ short octalPermission = Short.parseShort(permissionValue, 8);
+ FsPermission fsPermission = new FsPermission(octalPermission);
+ return fsPermission.toString();
+ }
+ }
+
+ /**
+ * Checks if the permission value is in symbolic representation.
+ *
+ * @param permissionValue the permission value to check
+ * @return true if the permission value is in symbolic representation,
+ * false otherwise
+ */
+ private static boolean isSymbolic(String permissionValue) {
+ return permissionValue.matches(".*[rwx].*");
+ }
+
+
+ /**
+ * Retrieves the permissions' configuration value for a given config key.
+ *
+ * @param key The configuration key.
+ * @param conf The ConfigurationSource object containing the config
+ * @return The permissions' configuration value for the specified key.
+ * @throws IllegalArgumentException If the configuration value is not defined
+ */
+ public static String getPermissions(String key, ConfigurationSource conf) {
+ String configName = "";
+
+ // Assign the appropriate config name based on the KEY
+ if (key.equals(ReconConfigKeys.OZONE_RECON_DB_DIR)) {
+ configName = ReconConfigKeys.OZONE_RECON_DB_DIRS_PERMISSIONS;
+ } else if (key.equals(ScmConfigKeys.OZONE_SCM_DB_DIRS)) {
+ configName = ScmConfigKeys.OZONE_SCM_DB_DIRS_PERMISSIONS;
+ } else if (key.equals(OzoneConfigKeys.OZONE_OM_DB_DIRS)) {
+ configName = OzoneConfigKeys.OZONE_OM_DB_DIRS_PERMISSIONS;
+ } else {
+ // If the permissions are not defined for the config, we make it fall
+ // back to the default permissions for metadata files and directories
+ configName = OzoneConfigKeys.OZONE_METADATA_DIRS_PERMISSIONS;
+ }
+
+ String configValue = conf.get(configName);
+ if (configValue != null) {
+ return configValue;
+ }
+
+ throw new IllegalArgumentException(
+ "Invalid configuration value for key: " + key);
+ }
+
+
/**
* Checks and creates Ozone Metadir Path if it does not exist.
*
@@ -202,7 +288,7 @@ public final class ServerUtils {
}
public static void setOzoneMetaDirPath(OzoneConfiguration conf,
- String path) {
+ String path) {
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, path);
}
@@ -212,7 +298,7 @@ public final class ServerUtils {
* If the directory is missing the method tries to create it.
*
* @param conf The ozone configuration object
- * @param key The configuration key which specify the directory.
+ * @param key The configuration key which specify the directory.
* @return The path of the directory.
*/
public static File getDBPath(ConfigurationSource conf, String key) {
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 9493dcb1a7..587f45db5d 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -42,6 +42,14 @@ public final class OzoneConfigKeys {
public static final String OZONE_METADATA_DIRS = "ozone.metadata.dirs";
+ public static final String OZONE_METADATA_DIRS_PERMISSIONS =
+ "ozone.metadata.dirs.permissions";
+ public static final String OZONE_OM_DB_DIRS_PERMISSIONS =
+ "ozone.om.db.dirs.permissions";
+
+
+ public static final String OZONE_OM_DB_DIRS = "ozone.om.db.dirs";
+
/**
*
* When set to true, allocate a random free port for ozone container,
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 5457dc04eb..c79112d3bb 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -680,6 +680,15 @@
production environments.
</description>
</property>
+ <property>
+ <name>ozone.om.db.dirs.permissions</name>
+ <value>750</value>
+ <description>
+ Permissions for the metadata directories for Ozone Manager. The
+ permissions have to be octal or symbolic. If the default permissions are
not set
+ then the default value of 750 will be used.
+ </description>
+ </property>
<property>
<name>ozone.metadata.dirs</name>
<value/>
@@ -694,6 +703,15 @@
dfs.container.ratis.datanode.storage.dir be configured separately.
</description>
</property>
+ <property>
+ <name>ozone.metadata.dirs.permissions</name>
+ <value>750</value>
+ <description>
+ Permissions for the metadata directories for fallback location for SCM,
OM, Recon and DataNodes
+ to store their metadata. The permissions have to be octal or symbolic.
This is the fallback used in case the
+ default permissions for OM,SCM,Recon,Datanode are not set.
+ </description>
+ </property>
<property>
<name>ozone.metastore.rocksdb.statistics</name>
@@ -731,6 +749,15 @@
production environments.
</description>
</property>
+ <property>
+ <name>ozone.scm.db.dirs.permissions</name>
+ <value>750</value>
+ <description>
+ Permissions for the metadata directories for Storage Container Manager.
The
+ permissions can either be octal or symbolic. If the default permissions
are not set
+ then the default value of 750 will be used.
+ </description>
+ </property>
<property>
<name>ozone.scm.block.client.address</name>
<value/>
@@ -2786,6 +2813,15 @@
production environments.
</description>
</property>
+ <property>
+ <name>ozone.recon.db.dirs.permissions</name>
+ <value>750</value>
+ <description>
+ Permissions for the metadata directories for Recon. The
+ permissions can either be octal or symbolic. If the default permissions
are not set
+ then the default value of 750 will be used.
+ </description>
+ </property>
<property>
<name>ozone.scm.network.topology.schema.file</name>
<value>network-topology-default.xml</value>
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/TestServerUtils.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/TestServerUtils.java
index 61b1fe4b57..2799ec0878 100644
---
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/TestServerUtils.java
+++
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/TestServerUtils.java
@@ -18,25 +18,181 @@
package org.apache.hadoop.hdds.server;
import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.Set;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.recon.ReconConfigKeys;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.test.PathUtils;
import org.apache.commons.io.FileUtils;
-import org.junit.jupiter.api.Test;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
-import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
+
/**
* Unit tests for {@link ServerUtils}.
*/
public class TestServerUtils {
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+
+ /**
+ * Test case for {@link ServerUtils#getPermissions}.
+ * Verifies the retrieval of permissions for different configs.
+ */
+ @Test
+ public void testGetPermissions() {
+ // Create an OzoneConfiguration object and set the permissions
+ // for different keys
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(ReconConfigKeys.OZONE_RECON_DB_DIRS_PERMISSIONS, "750");
+ conf.set(ScmConfigKeys.OZONE_SCM_DB_DIRS_PERMISSIONS, "775");
+ conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS_PERMISSIONS, "770");
+ conf.set(OzoneConfigKeys.OZONE_OM_DB_DIRS_PERMISSIONS, "700");
+
+ // Test getPermissions for different config names and assert the
+ // returned permissions
+ assertEquals("750",
+ ServerUtils.getPermissions(ReconConfigKeys.OZONE_RECON_DB_DIR, conf));
+ assertEquals("775",
+ ServerUtils.getPermissions(ScmConfigKeys.OZONE_SCM_DB_DIRS, conf));
+ assertEquals("770",
+ ServerUtils.getPermissions(OzoneConfigKeys.OZONE_METADATA_DIRS, conf));
+ assertEquals("700",
+ ServerUtils.getPermissions(OzoneConfigKeys.OZONE_OM_DB_DIRS, conf));
+
+ }
+
+ @Test
+ public void testGetDirectoryFromConfigWithOctalPermissions()
+ throws IOException {
+ // Create a temporary directory
+ String filePath = folder.getRoot().getAbsolutePath();
+
+ // Create an OzoneConfiguration object
+ OzoneConfiguration conf = new OzoneConfiguration();
+ String key = OzoneConfigKeys.OZONE_METADATA_DIRS;
+ String componentName = "Ozone";
+ conf.set(key, filePath);
+
+ // Set octal permissions
+ String octalPermissionValue = "750";
+ String octalExpectedPermissions = "rwxr-x---";
+ conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS_PERMISSIONS,
+ octalPermissionValue);
+
+ // Get the directory using ServerUtils method
+ File directory =
+ ServerUtils.getDirectoryFromConfig(conf, key, componentName);
+
+ // Assert that the directory is not null and exists
+ assertNotNull(directory);
+ assertTrue(directory.exists());
+ assertTrue(directory.isDirectory());
+
+ // Get the path of the directory
+ Path directoryPath = directory.toPath();
+
+ // Convert the expected octal permissions string to a
+ // set of PosixFilePermission
+ Set<PosixFilePermission> octalExpectedPermissionSet =
+ PosixFilePermissions.fromString(octalExpectedPermissions);
+
+ // Get the actual permissions of the directory
+ Set<PosixFilePermission> actualPermissionSet =
+ Files.getPosixFilePermissions(directoryPath);
+
+ // Assert that the expected and actual permissions sets are equal
+ assertEquals(octalExpectedPermissionSet, actualPermissionSet);
+ }
+
+ @Test
+ public void testGetDirectoryFromConfigWithSymbolicPermissions()
+ throws IOException {
+ // Create a temporary directory
+ String filePath = folder.getRoot().getAbsolutePath();
+
+ OzoneConfiguration conf = new OzoneConfiguration();
+ String key = OzoneConfigKeys.OZONE_METADATA_DIRS;
+ String componentName = "Ozone";
+ conf.set(key, filePath);
+
+ // Set symbolic permissions
+ String symbolicPermissionValue = "rwx------";
+ String symbolicExpectedPermissions = "rwx------";
+ conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS_PERMISSIONS,
+ symbolicPermissionValue);
+
+ File directory =
+ ServerUtils.getDirectoryFromConfig(conf, key, componentName);
+
+ assertNotNull(directory);
+ assertTrue(directory.exists());
+ assertTrue(directory.isDirectory());
+
+ Path directoryPath = directory.toPath();
+
+ Set<PosixFilePermission> symbolicExpectedPermissionSet =
+ PosixFilePermissions.fromString(symbolicExpectedPermissions);
+
+ Set<PosixFilePermission> actualPermissionSet =
+ Files.getPosixFilePermissions(directoryPath);
+
+ assertEquals(symbolicExpectedPermissionSet, actualPermissionSet);
+ }
+
+ @Test
+ public void testGetDirectoryFromConfigWithMultipleDirectories()
+ throws IOException {
+ // Create temporary directories
+ File dir1 = folder.newFolder("dir1");
+ File dir2 = folder.newFolder("dir2");
+
+ // Create an OzoneConfiguration object
+ OzoneConfiguration conf = new OzoneConfiguration();
+ String key = OzoneConfigKeys.OZONE_METADATA_DIRS;
+ String componentName = "Ozone";
+ conf.set(key, dir1.getAbsolutePath() + "," + dir2.getAbsolutePath());
+
+ // Assert that an IllegalArgumentException is thrown because Recon does not
+ // support multiple metadata dirs currently
+ assertThrows(IllegalArgumentException.class, () -> {
+ ServerUtils.getDirectoryFromConfig(conf, key, componentName);
+ });
+ }
+
+ @Test
+ public void testGetDirectoryFromConfigWithNoDirectory() {
+ // Create an empty OzoneConfiguration object
+ OzoneConfiguration conf = new OzoneConfiguration();
+ String key = OzoneConfigKeys.OZONE_METADATA_DIRS;
+ String componentName = "Ozone";
+
+ // Get the directory using ServerUtils method
+ File directory =
+ ServerUtils.getDirectoryFromConfig(conf, key, componentName);
+
+ // Assert that the directory is null
+ assertNull(directory);
+ }
+
/**
* Test {@link ServerUtils#getScmDbDir}.
*/
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index 5c6a39d198..4442b6d061 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -53,6 +53,10 @@ public final class OMConfigKeys {
// multiple entries for performance (sharding)..
public static final String OZONE_OM_DB_DIRS = "ozone.om.db.dirs";
+ // SCM DB directory permission
+ public static final String OZONE_OM_DB_DIRS_PERMISSIONS =
+ "ozone.om.db.dirs.permissions";
+
public static final String OZONE_OM_HANDLER_COUNT_KEY =
"ozone.om.handler.count.key";
public static final int OZONE_OM_HANDLER_COUNT_DEFAULT = 100;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
