Repository: hadoop Updated Branches: refs/heads/HDFS-7240 75ce3fcc7 -> a8fea9668
HDFS-12568. Ozone: Cleanup the ozone-default.xml. Contributed by Anu Engineer. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a8fea966 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a8fea966 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a8fea966 Branch: refs/heads/HDFS-7240 Commit: a8fea966812158d2ce52db725a802f664f5edad1 Parents: 75ce3fc Author: Anu Engineer <[email protected]> Authored: Tue Oct 3 11:41:27 2017 -0700 Committer: Anu Engineer <[email protected]> Committed: Tue Oct 3 11:41:27 2017 -0700 ---------------------------------------------------------------------- .../apache/hadoop/conf/OzonePropertyTag.java | 1 + .../apache/hadoop/cblock/CBlockConfigKeys.java | 6 +- .../apache/hadoop/ozone/OzoneConfigKeys.java | 8 +- .../apache/hadoop/ozone/ksm/KSMConfigKeys.java | 2 +- .../container/common/utils/ContainerCache.java | 4 +- .../src/main/resources/ozone-default.xml | 1304 +++++++++--------- .../ozone/TestOzoneConfigurationFields.java | 32 - 7 files changed, 666 insertions(+), 691 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8fea966/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/OzonePropertyTag.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/OzonePropertyTag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/OzonePropertyTag.java index 162e4e1..c1e6cbd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/OzonePropertyTag.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/OzonePropertyTag.java @@ -28,6 +28,7 @@ import org.apache.hadoop.classification.InterfaceStability; @InterfaceStability.Evolving public enum OzonePropertyTag implements PropertyTag { OZONE, + CBLOCK, MANAGEMENT, SECURITY, PERFORMANCE, http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8fea966/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java index 87f40b3..c1a72f4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java @@ -49,7 +49,7 @@ public final class CBlockConfigKeys { 4096; public static final String DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY = - "dfs.storage.service.handler.count"; + "dfs.cblock.service.handler.count"; public static final int DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_DEFAULT = 10; public static final String DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY = @@ -82,7 +82,7 @@ public final class CBlockConfigKeys { * Cache size in 1000s of entries. 256 indicates 256 * 1024. */ public static final String DFS_CBLOCK_CACHE_QUEUE_SIZE_KB = - "dfs.cblock.cache.cache.size.in.kb"; + "dfs.cblock.cache.queue.size.in.kb"; public static final int DFS_CBLOCK_CACHE_QUEUE_SIZE_KB_DEFAULT = 256; /** @@ -148,7 +148,7 @@ public final class CBlockConfigKeys { public static final int DFS_CBLOCK_SCM_PORT_DEFAULT = 9860; public static final String DFS_CBLOCK_CONTAINER_SIZE_GB_KEY = - "dfs.cblock.container.size"; + "dfs.cblock.container.size.gb"; public static final int DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT = 5; http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8fea966/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 67161e3..171b634 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -89,8 +89,9 @@ public final class OzoneConfigKeys { public static final String OZONE_METADATA_STORE_IMPL_DEFAULT = OZONE_METADATA_STORE_IMPL_ROCKSDB; - public static final String OZONE_KEY_CACHE = "ozone.key.cache.size"; - public static final int OZONE_KEY_CACHE_DEFAULT = 1024; + public static final String OZONE_CONTAINER_CACHE_SIZE = + "ozone.container.cache.size"; + public static final int OZONE_CONTAINER_CACHE_DEFAULT = 1024; public static final String OZONE_SCM_BLOCK_SIZE_KEY = "ozone.scm.block.size"; @@ -158,9 +159,6 @@ public final class OzoneConfigKeys { = ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY; public static final String DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT = ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT; - /** A unique ID to identify a Ratis server. */ - public static final String DFS_CONTAINER_RATIS_SERVER_ID = - "dfs.container.ratis.server.id"; public static final String DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR = "dfs.container.ratis.datanode.storage.dir"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8fea966/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java index 6710b0a..75cf613 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java @@ -57,7 +57,7 @@ public final class KSMConfigKeys { // LevelDB cache file uses an off-heap cache in LevelDB of 128 MB. public static final String OZONE_KSM_DB_CACHE_SIZE_MB = - "ozone.ksm.leveldb.cache.size.mb"; + "ozone.ksm.db.cache.size.mb"; public static final int OZONE_KSM_DB_CACHE_SIZE_DEFAULT = 128; public static final String OZONE_KSM_USER_MAX_VOLUME = http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8fea966/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java index e8bee35..6ae45b6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java @@ -59,8 +59,8 @@ public final class ContainerCache extends LRUMap { */ public synchronized static ContainerCache getInstance(Configuration conf) { if (cache == null) { - int cacheSize = conf.getInt(OzoneConfigKeys.OZONE_KEY_CACHE, - OzoneConfigKeys.OZONE_KEY_CACHE_DEFAULT); + int cacheSize = conf.getInt(OzoneConfigKeys.OZONE_CONTAINER_CACHE_SIZE, + OzoneConfigKeys.OZONE_CONTAINER_CACHE_DEFAULT); cache = new ContainerCache(cacheSize, LOAD_FACTOR, true); } return cache; http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8fea966/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml index 5e5469b..5bc15ac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml @@ -1,6 +1,5 @@ -<?xml version="1.0"?> +<?xml version="1.0" encoding="UTF-8"?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> - <!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with @@ -17,1041 +16,1050 @@ See the License for the specific language governing permissions and limitations under the License. --> - <!-- Do not modify this file directly. Instead, copy entries that you --> <!-- wish to modify from this file into ozone-site.xml and change them --> <!-- there. If ozone-site.xml does not already exist, create it. --> +<!--Tags supported are OZONE, CBLOCK, MANAGEMENT, SECURITY, PERFORMANCE, --> +<!--DEBUG, CLIENT, SERVER, KSM, SCM, CRITICAL, RATIS, CONTAINER, REQUIRED, --> +<!--REST, STORAGE, PIPELINE, STANDALONE --> + <configuration> + <!--CBlock Settings--> <property> - <name>ozone.enabled</name> - <value>false</value> - <tag>OZONE,REQUIRED</tag> + <name>dfs.cblock.block.buffer.flush.interval.seconds</name> + <value>60</value> + <tag>CBLOCK, PERFORMANCE</tag> <description> - Status of the Ozone Object Storage service is enabled. - Set to true to enable Ozone. - Set to false to disable Ozone. + Controls the frequency at this the local cache flushes the + blocks to the remote containers. </description> </property> - <property> - <name>ozone.handler.type</name> - <value>distributed</value> - <tag>OZONE,REST</tag> + <name>dfs.cblock.cache.block.buffer.size</name> + <value>512</value> + <tag>CBLOCK, PERFORMANCE</tag> <description> - The second key dfs.storage.handler.type tells ozone which storage - handler to use. The possible values are: - distributed - The Ozone distributed storage handler. - local - Local Storage handler strictly for testing. + Size of the local cache for blocks. So cache size will be block + size multiplied by this number. </description> </property> - <property> - <name>ozone.localstorage.root</name> - <value>/tmp/ozone</value> - <tag>OZONE,REST</tag> + <name>dfs.cblock.cache.core.min.pool.size</name> + <value>16</value> + <tag>CBLOCK, PERFORMANCE</tag> <description> + A minimum number of threads in the pool that cBlock cache will + use for the background I/O to remote containers. </description> </property> - <property> - <name>ozone.trace.enabled</name> - <value>false</value> - <tag>OZONE,DEBUG</tag> + <name>dfs.cblock.cache.max.pool.size</name> + <value>256</value> + <tag>CBLOCK, PERFORMANCE</tag> <description> + Maximum number of threads in the pool that cBlock cache will + use for background I/O to remote containers. </description> </property> - <property> - <name>ozone.metadata.dirs</name> - <value></value> - <tag>OZONE,KSM,SCM,CONTAINER,REQUIRED,STORAGE</tag> + <name>dfs.cblock.cache.keep.alive.seconds</name> + <value>60</value> + <tag>CBLOCK, PERFORMANCE</tag> <description> - Ozone metadata dir path. + If the cblock cache has no I/O, then the threads in the cache + pool are kept idle for this many seconds before shutting down. </description> </property> - <property> - <name>ozone.key.cache.size</name> - <value>1024</value> - <tag>PERFORMANCE,CONTAINER,STORAGE</tag> + <name>dfs.cblock.cache.leveldb.cache.size.mb</name> + <value>256</value> + <tag>CBLOCK, PERFORMANCE</tag> <description> + The amount of physical memory allocated to the local cache. The + SCSI driver will allocate this much RAM cache instances. </description> </property> - <property> - <name>ozone.administrators</name> - <value></value> - <tag>OZONE,SECURITY</tag> + <name>dfs.cblock.cache.max.retry</name> + <value>65536</value> + <tag>CBLOCK, PERFORMANCE</tag> <description> - Ozone administrator users delimited by comma. - If not set, only the user who launches an ozone service will be the - admin user. This property must be set if ozone services are started by - different users. Otherwise the RPC layer will reject calls from - other servers which are started by users not in the list. + If the local cache is enabled then, CBlock writes to the local + cache when I/O happens. Then the background I/O threads write this + block to the remote containers. This value controls how many times the + background thread should attempt to do I/O to the remote containers + before giving up. </description> </property> - <property> - <name>ozone.client.protocol</name> - <value>org.apache.hadoop.ozone.client.rpc.RpcClient</value> - <tag>OZONE</tag> + <name>dfs.cblock.cache.queue.size.in.kb</name> + <value>256</value> + <tag>CBLOCK, PERFORMANCE</tag> <description> - Protocol class to be used by client to connect to ozone cluster. - The build-in implementation includes: - org.apache.hadoop.ozone.client.rpc.RpcClient for RPC - org.apache.hadoop.ozone.client.rest.RestClient for REST + Size of the in memory cache queue, that is flushed to local + disk. </description> </property> - <property> - <name>ozone.client.socket.timeout.ms</name> - <value>5000</value> - <tag>OZONE</tag> + <name>dfs.cblock.cache.thread.priority</name> + <value>5</value> + <tag>CBLOCK, PERFORMANCE</tag> <description> - Socket timeout for Ozone client in milliseconds. + Priority of cache flusher thread, affecting the relative performance of + write and read. Supported values are 1, 5, 10. + Use 10 for high priority and 1 for low priority. </description> </property> - <property> - <name>ozone.client.connection.timeout.ms</name> - <value>5000</value> - <tag>OZONE,PERFORMANCE,CLIENT</tag> + <name>dfs.cblock.container.size.gb</name> + <value>5</value> + <tag>CBLOCK, MANAGEMENT</tag> <description> - Connection timeout for Ozone client in milliseconds. + The size of ozone container in the number of GBs. Note that + this is not setting container size for ozone. This setting is + instructing CBlock to manage containers at a standard size. </description> </property> - <property> - <name>ozone.scm.client.address</name> - <value></value> - <tag>OZONE</tag> + <name>dfs.cblock.disk.cache.path</name> + <value>${hadoop.tmp.dir}/cblockCacheDB</value> + <tag>CBLOCK, REQUIRED</tag> <description> - The address of the Ozone SCM client service. This is a required - setting. - - It is a string in the host:port format. The port number is optional - and defaults to 9860. + The default path for the cblock local cache. If the cblock + local cache is enabled, then it must be set to a valid path. This cache + *should* be mapped to the fastest disk on a given machine, For example, + an SSD drive would be a good idea. Currently, all mounted disk on a + data node is mapped to a single path, so having a large number of IOPS + is essential. </description> </property> - <property> - <name>ozone.scm.client.port</name> - <value>9860</value> - <tag>OZONE</tag> + <name>dfs.cblock.jscsi-address</name> + <value/> + <tag>CBLOCK, MANAGEMENT</tag> <description> - The port number of the Ozone SCM client service. + The address that cblock will be bind to, should be a host:port + format, This setting is required for cblock server to start. + This address to be used by jscsi to mount volume. </description> </property> - <property> - <name>ozone.scm.datanode.address</name> - <value></value> - <tag>OZONE</tag> + <name>dfs.cblock.jscsi.cblock.server.address</name> + <value>127.0.0.1</value> + <tag>CBLOCK, MANAGEMENT</tag> <description> - The address of the Ozone SCM service used for internal communication - between the DataNodes and the SCM. - - It is a string in the host:port format. The port number is optional - and defaults to 9861. - - This setting is optional. If unspecified then the hostname portion - is picked from the ozone.scm.client.address setting and the - default service port of 9861 is chosen. + The address local jscsi server will use to talk to cblock manager. </description> </property> - <property> - <name>ozone.scm.datanode.bind.host</name> - <value></value> - <tag>OZONE</tag> + <name>dfs.cblock.jscsi.port</name> + <value>9811</value> + <tag>CBLOCK, MANAGEMENT</tag> <description> - The hostname or IP address used by the SCM service endpoint to bind. + The port on CBlockManager node for jSCSI to talk to. </description> </property> - <property> - <name>ozone.scm.datanode.port</name> - <value>9861</value> - <tag>OZONE</tag> + <name>dfs.cblock.jscsi.rpc-bind-host</name> + <value>0.0.0.0</value> + <tag>CBLOCK, MANAGEMENT</tag> <description> - The port number of the Ozone SCM service. + The actual address the cblock jscsi rpc server will bind to. If + this optional address is set, it overrides only the hostname portion of + dfs.cblock.jscsi-address. </description> </property> - <property> - <name>ozone.scm.client.bind.host</name> + <name>dfs.cblock.jscsi.server.address</name> <value>0.0.0.0</value> - <tag>OZONE</tag> + <tag>CBLOCK, MANAGEMENT</tag> <description> - The hostname or IP address used by the SCM client endpoint to bind. - This setting is used by the SCM only and never used by clients. - - The setting can be useful in multi-homed setups to restrict the - availability of the SCM client service to a specific interface. - - The default is appropriate for most clusters. + The address that jscsi server will be running, it is nice have one + local jscsi server for each client(Linux JSCSI client) that tries to + mount cblock. </description> </property> - <property> - <name>ozone.scm.block.client.address</name> - <value></value> - <tag>OZONE</tag> + <name>dfs.cblock.manager.pool.size</name> + <value>16</value> + <tag>CBLOCK, PERFORMANCE</tag> <description> - The address of the Ozone SCM block client service. If not defined - value of ozone.scm.client.address is used. + Number of threads that cblock manager will use for container + operations. </description> </property> - <property> - <name>ozone.scm.block.client.bind.host</name> - <value>0.0.0.0</value> - <tag>OZONE</tag> + <name>dfs.cblock.rpc.timeout.seconds</name> + <value>300</value> + <tag>CBLOCK, MANAGEMENT</tag> <description> - The hostname or IP address used by the SCM block client - endpoint to bind. + RPC timeout in seconds used for cblock CLI operations. When you + create very large disks, like 5TB, etc. The number of containers + allocated in the system is huge. It is will 5TB/5GB, which is 1000 + containers. The client CLI might timeout even though the cblock manager + creates the specified disk. This value allows the user to wait for a + longer period. </description> </property> - <property> - <name>ozone.scm.block.client.port</name> - <value>9863</value> - <tag>OZONE</tag> + <name>dfs.cblock.scm.ipaddress</name> + <value>127.0.0.1</value> + <tag>CBLOCK, MANAGEMENT</tag> <description> - The port number of the Ozone SCM block client service. + IP address used by cblock to connect to SCM. </description> </property> - <property> - <name>ozone.scm.handler.count.key</name> - <value>10</value> - <tag>OZONE</tag> + <name>dfs.cblock.scm.port</name> + <value>9860</value> + <tag>CBLOCK, MANAGEMENT</tag> <description> - The number of RPC handler threads for each SCM service endpoint. - - The default is appropriate for small clusters (tens of nodes). + Port used by cblock to connect to SCM. </description> </property> - <property> - <name>ozone.scm.chunk.size</name> - <value>1048576</value> - <tag>OZONE</tag> + <name>dfs.cblock.service.handler.count</name> + <value>10</value> + <tag>CBLOCK, MANAGEMENT</tag> <description> - The chunk size for read/write chunk operations in bytes. - - The chunk size defaults to 1MB. If the value configured is more - than the maximum size (1MB), it will be reset to the maximum - size. + Default number of handlers for CBlock service rpc. </description> </property> - <property> - <name>ozone.scm.heartbeat.interval.seconds</name> - <value>30</value> - <tag>OZONE</tag> + <name>dfs.cblock.service.leveldb.path</name> + <value>${hadoop.tmp.dir}/cblock_server.dat</value> + <tag>CBLOCK, REQUIRED</tag> <description> - The heartbeat interval from a datanode to SCM in seconds. + Default path for the cblock meta data store. </description> </property> - <property> - <name>ozone.scm.stale.node.interval.ms</name> - <value>90000</value> - <tag>OZONE</tag> + <name>dfs.cblock.service.rpc-bind-host</name> + <value>0.0.0.0</value> + <tag>CBLOCK, MANAGEMENT</tag> <description> - The interval in milliseconds for stale node flagging. + The actual address the cblock service RPC server will bind to. + If the optional address is set, it overrides only the hostname portion of + dfs.cblock.servicerpc-address. </description> </property> - <property> - <name>ozone.scm.dead.node.interval.ms</name> - <value>600000</value> - <tag>OZONE</tag> + <name>dfs.cblock.servicerpc-address</name> + <value/> + <tag>CBLOCK, MANAGEMENT, REQUIRED</tag> <description> - The interval in milliseconds for dead node flagging. + The address that cblock will be bind to, should be a host:port + format, this setting is required for cblock server to start. + This address is used for cblock management operations like create, delete, + info and list volumes </description> </property> - <property> - <name>ozone.scm.max.hb.count.to.process</name> - <value>5000</value> - <tag>OZONE</tag> + <name>dfs.cblock.short.circuit.io</name> + <value>false</value> + <tag>CBLOCK, PERFORMANCE</tag> <description> - The maximum number of heartbeat to process per loop of the process - thread. + Enables use of the local cache in cblock. Enabling this allows + I/O against the local cache and background threads do actual I/O against + the + containers. </description> </property> - <property> - <name>ozone.scm.heartbeat.thread.interval.ms</name> - <value>3000</value> - <tag>OZONE</tag> - <description> - The interval in milliseconds that the heartbeat processor thread runs. + <name>dfs.cblock.trace.io</name> + <value>false</value> + <tag>CBLOCK, DEBUG</tag> + <description>Default flag for enabling trace io, Trace I/O logs all I/O with + hashes of + data. This is useful for detecting things like data corruption. </description> </property> + <!--Container Settings used by Datanode--> <property> - <name>ozone.scm.heartbeat.rpc-timeout</name> - <value>1000</value> - <tag>OZONE</tag> - <description> - Timeout value for the RPC from Datanode to SCM in milliseconds. + <name>ozone.container.cache.size</name> + <value>1024</value> + <tag>PERFORMANCE, CONTAINER, STORAGE</tag> + <description>The open container is cached on the data node side. We maintain + an LRU + cache for caching the recently used containers. This setting controls the + size of that cache. </description> </property> - <property> - <name>ozone.scm.block.deletion.max.retry</name> - <value>4096</value> - <tag>OZONE</tag> - <description> - SCM wraps up a number of blocks in a deletion transaction and send that - to datanode for physically deletion periodically. This property - determines how many times at most for SCM to retry sending a deletion - transaction to datanode. The default value 4096 is relatively big so - that SCM could try enough times before giving up, as the actual deletion - is async so time required is unpredictable. - </description> + <name>dfs.container.ipc</name> + <value>9859</value> + <tag>OZONE, CONTAINER, MANAGEMENT</tag> + <description>The ipc port number of container.</description> </property> - <property> - <name>ozone.scm.heartbeat.log.warn.interval.count</name> - <value>10</value> - <tag>OZONE</tag> - <description> - Defines how frequently we will log the missing of heartbeat to SCM. - For example in the default case, we will write a warning message for each 10 - sequential heartbeats that we miss to SCM. + <name>dfs.container.ipc.random.port</name> + <value>false</value> + <tag>OZONE, DEBUG, CONTAINER</tag> + <description>Allocates a random free port for ozone container. This is used + only while + running unit tests. </description> </property> - <property> - <name>ozone.scm.names</name> - <value></value> - <tag>OZONE</tag> - <description> - The value of this property is a set of DNS | DNS:PORT | IP Address | IP:PORT. - Written as a comma separated string. e.g. scm1, scm2:8020, 7.7.7.7:7777. - This property allows datanodes to discover where SCM is, so that - datanodes can send heartbeat to SCM. + <name>dfs.container.ratis.datanode.storage.dir</name> + <value/> + <tag>OZONE, CONTAINER, STORAGE, MANAGEMENT, RATIS</tag> + <description>This directory is used for storing Ratis metadata like logs. If + this is + not set then default metadata dirs is used. A warning will be logged if + this not set. Ideally, this should be mapped to a fast disk like an SSD. </description> </property> - <property> - <name>ozone.scm.datanode.id</name> - <value></value> - <tag>OZONE</tag> - <description> - The path that datanodes will created to store the datanode ID. - If this value is not set, datanodes will fail to come up. + <name>dfs.container.ratis.enabled</name> + <value>false</value> + <tag>OZONE, MANAGEMENT, PIPELINE, RATIS</tag> + <description>Ozone supports different kinds of replication pipelines. Ratis + is one of + the replication pipeline supported by ozone. </description> </property> - <property> - <name>ozone.scm.db.cache.size.mb</name> - <value>128</value> - <tag>OZONE</tag> - <description> - The cache size for SCM level db store in MB. + <name>dfs.container.ratis.ipc</name> + <value>9858</value> + <tag>OZONE, CONTAINER, PIPELINE, RATIS, MANAGEMENT</tag> + <description>The ipc port number of container.</description> + </property> + <property> + <name>dfs.container.ratis.ipc.random.port</name> + <value>false</value> + <tag>OZONE,DEBUG</tag> + <description>Allocates a random free port for ozone ratis port for the + container. This + is used only while running unit tests. </description> </property> - <property> - <name>ozone.scm.container.size.gb</name> - <value>5</value> - <tag>OZONE</tag> - <description> - Storage container size in GB for block allocations. + <name>dfs.container.ratis.rpc.type</name> + <value>GRPC</value> + <tag>OZONE, RATIS, MANAGEMENT</tag> + <description>Ratis supports different kinds of transports like netty, GRPC, + Hadoop RPC + etc. This picks one of those for this cluster. </description> </property> - + <!--Ozone Settings--> <property> - <name>ozone.scm.container.placement.impl</name> - <value>org.apache.hadoop.ozone.scm.container.placement.algorithms.SCMContainerPlacementRandom</value> - <tag>OZONE</tag> - <description> - Placement policy class for containers. - Defaults to SCMContainerPlacementRandom.class + <name>ozone.administrators</name> + <value/> + <tag>OZONE, SECURITY</tag> + <description>Ozone administrator users delimited by the comma. + If not set, only the user who launches an ozone service will be the admin + user. This property must be set if ozone services are started by different + users. Otherwise, the RPC layer will reject calls from other servers which + are started by users not in the list. </description> </property> - <property> - <name>ozone.scm.container.provision_batch_size</name> - <value>1</value> - <tag>OZONE</tag> - <description> - Pre-provision specified number of containers to creare for - block creation. + <name>ozone.block.deleting.container.limit.per.interval</name> + <value>10</value> + <tag>OZONE, PERFORMANCE, SCM</tag> + <description>A maximum number of containers to be scanned by block deleting + service per + time interval. The block deleting service spawns a thread to handle block + deletions in a container. This property is used to throttle the number of + threads spawned for block deletions. </description> </property> - <property> - <name>ozone.scm.max.container.report.threads</name> - <value>100</value> - <tag>OZONE</tag> - <description> - Maximum number of threads to process container reports in scm. - Each container report from a datanode is processed by scm in - a worker thread, fetched from a thread pool. This property is - used to control the maximum size of the thread pool. + <name>ozone.block.deleting.limit.per.task</name> + <value>1000</value> + <tag>OZONE, PERFORMANCE, SCM</tag> + <description>A maximum number of blocks to be deleted by block deleting + service per + time interval. This property is used to throttle the actual number of + block deletions on a data node per container. </description> </property> - <property> - <name>ozone.scm.container.reports.wait.timeout.seconds</name> - <value>300</value> - <tag>OZONE</tag> - <description> - Maximum time to wait in seconds for processing all container - reports from a node pool. It determines the timeout for a - node pool reports. + <name>ozone.block.deleting.service.interval.ms</name> + <value>60000</value> + <tag>OZONE, PERFORMANCE, SCM</tag> + <description>Time interval in milliseconds of the block deleting service. + The block deleting service runs on each datanode periodically and + deletes blocks queued for deletion. </description> </property> - <property> - <name>ozone.scm.container.report.processing.interval.seconds</name> - <value>60</value> - <tag>OZONE</tag> - <description> - Time interval in seconds for scm to process container reports - for a node pool. Scm handles node pool reports in a cyclic clock - manner, it fetches pools periodically with this time interval. + <name>ozone.block.deleting.service.timeout</name> + <value>300000ms</value> + <tag>OZONE, PERFORMANCE, SCM</tag> + <description>A timeout value of block deletion service. If this is set + greater than 0, + the service will stop waiting for the block deleting completion after this + time. If timeout happens to a large proportion of block deletion, this + needs to be increased with ozone.block.deleting.limit.per.task. This + setting supports multiple time unit suffixes as described in + dfs.heartbeat.interval. If no suffix is specified, then milliseconds is + assumed. </description> </property> - <property> - <name>ozone.block.deleting.service.interval.ms</name> - <value>60000</value> - <tag>OZONE,PERFORMANCE,SCM</tag> - <description> - Time interval in milliseconds of the block deleting service. - The block deleting service runs on each datanode to scan staled - blocks and delete them asynchronously. + <name>ozone.client.connection.timeout.ms</name> + <value>5000</value> + <tag>OZONE, PERFORMANCE, CLIENT</tag> + <description>Connection timeout for Ozone client in milliseconds. </description> </property> - <property> - <name>ozone.block.deleting.service.timeout</name> - <value>300000ms</value> - <tag>OZONE,PERFORMANCE,SCM</tag> - <description> - Timeout value of block deletion service. If this is set greater than 0, - the service will stop waiting for the block deleting completion after this - time. If timeout happens to a large proportion of block deletion, this needs - to be increased with ozone.block.deleting.limit.per.task. This setting supports - multiple time unit suffixes as described in dfs.heartbeat.interval. If no suffix - is specified then milliseconds is assumed. + <name>ozone.client.protocol</name> + <value>org.apache.hadoop.ozone.client.rpc.RpcClient</value> + <tag>OZONE, CLIENT, MANAGEMENT</tag> + <description>Protocol class to be used by the client to connect to ozone + cluster. + The build-in implementation includes: + org.apache.hadoop.ozone.client.rpc.RpcClient for RPC + org.apache.hadoop.ozone.client.rest.RestClient for REST + The default is the RpClient. Please do not change this unless you have a + very good understanding of what you are doing. </description> </property> - <property> - <name>ozone.block.deleting.limit.per.task</name> - <value>1000</value> - <tag>OZONE,PERFORMANCE,SCM</tag> - <description> - Maximum number of blocks to be deleted by block deleting service - per time interval. This property is used to throttle the actual number - of block deletions on a datanode per container. - </description> + <name>ozone.client.socket.timeout.ms</name> + <value>5000</value> + <tag>OZONE, CLIENT</tag> + <description>Socket timeout for Ozone client in milliseconds.</description> </property> - <property> - <name>ozone.block.deleting.container.limit.per.interval</name> - <value>10</value> - <tag>OZONE,PERFORMANCE,SCM</tag> + <name>ozone.enabled</name> + <value>false</value> + <tag>OZONE, REQUIRED</tag> <description> - Maximum number of containers to be scanned by block deleting service - per time interval. The block deleting service spawns a thread to handle - block deletions in a container. This property is used to throttle - the number of threads spawned for block deletions. + Status of the Ozone Object Storage service is enabled. + Set to true to enable Ozone. + Set to false to disable Ozone. + Unless this value is set to true, Ozone services will not be started in + the cluster. + + Please note: By default ozone is disabled on a hadoop cluster. </description> </property> - <property> - <name>ozone.scm.container.deletion-choosing.policy</name> - <value>org.apache.hadoop.ozone.container.common.impl.TopNOrderedContainerDeletionChoosingPolicy</value> - <tag>OZONE</tag> + <name>ozone.handler.type</name> + <value>distributed</value> + <tag>OZONE, REST</tag> <description> - The policy used for choosing desire containers for block deletion. - Datanode selects a number of containers to process block deletion - in a certain interval defined by ozone.block.deleting.service.interval.ms, - the number of containers to process in each interval is defined - by ozone.block.deleting.container.limit.per.interval. This property - is used to configure the policy applied while selecting containers. - There are two policies supporting now: RandomContainerDeletionChoosingPolicy and - TopNOrderedContainerDeletionChoosingPolicy. - org.apache.hadoop.ozone.container.common.impl.RandomContainerDeletionChoosingPolicy - implements a simply random policy that to return a random list of containers. - org.apache.hadoop.ozone.container.common.impl.TopNOrderedContainerDeletionChoosingPolicy - implements a policy that choosing top count number of containers in a pending-deletion-blocks's num - based descending order. + Tells ozone which storage handler to use. The possible values are: + distributed - The Ozone distributed storage handler, which speaks to + KSM/SCM on the backend and provides REST services to clients. + local - Local Storage handler strictly for testing - To be removed. </description> </property> - <property> <name>ozone.key.deleting.limit.per.task</name> <value>1000</value> - <tag>PERFORMANCE,KSM</tag> + <tag>KSM, PERFORMANCE</tag> <description> - Maximum number of keys to be scanned by key deleting service per - time interval in KSM. Those keys are sent to delete metadata and + A maximum number of keys to be scanned by key deleting service + per time interval in KSM. Those keys are sent to delete metadata and generate transactions in SCM for next async deletion between SCM and DataNode. </description> </property> - <property> - <name>dfs.container.ipc</name> - <value>9859</value> - <tag>OZONE</tag> + <name>ozone.ksm.address</name> + <value/> + <tag>KSM, REQUIRED</tag> <description> - The ipc port number of container. + The address of the Ozone KSM service. This allows clients to discover + the KSMs address. </description> </property> - <property> - <name>dfs.container.ipc.random.port</name> - <value>false</value> - <tag>OZONE,CONTAINER,PIPELINE,STANDALONE</tag> + <name>ozone.ksm.group.rights</name> + <value>READ_WRITE</value> + <tag>KSM, SECURITY</tag> <description> - Whether allocates a random free port for ozone container. + Default group permissions in Ozone KSM. </description> </property> - <property> - <name>scm.container.client.idle.threshold</name> - <value>10000</value> - <tag>OZONE</tag> + <name>ozone.ksm.handler.count.key</name> + <value>200</value> + <tag>KSM, PERFORMANCE</tag> <description> + The number of RPC handler threads for KSM service endpoints. </description> </property> - <property> - <name>scm.container.client.max.size</name> - <value>256</value> - <tag>OZONE</tag> + <name>ozone.ksm.http-address</name> + <value>0.0.0.0:9874</value> + <tag>KSM, MANAGEMENT</tag> <description> - </description> - </property> + The address and the base port where the KSM web UI will listen on. - <property> - <name>dfs.container.ratis.enabled</name> - <value>false</value> - <tag>OZONE</tag> - <description> + If the port is 0, then the server will start on a free port. However, it + is best to specify a well-known port, so it is easy to connect and see + the KSM management UI. </description> </property> - <property> - <name>dfs.container.ratis.rpc.type</name> - <value>GRPC</value> - <tag>OZONE</tag> + <name>ozone.ksm.http-bind-host</name> + <value>0.0.0.0</value> + <tag>KSM, MANAGEMENT</tag> <description> + The actual address the KSM web server will bind to. If this optional + the address is set, it overrides only the hostname portion of + ozone.ksm.http-address. </description> </property> - <property> - <name>dfs.container.ratis.server.id</name> - <value></value> - <tag>OZONE</tag> + <name>ozone.ksm.http.enabled</name> + <value>true</value> + <tag>KSM, MANAGEMENT</tag> <description> - The unique ID to identify a Ratis server. + Property to enable or disable KSM web user interface. </description> </property> - <property> - <name>dfs.container.ratis.datanode.storage.dir</name> - <value></value> - <tag>OZONE,CONTAINER,STORAGE</tag> + <name>ozone.ksm.https-address</name> + <value>0.0.0.0:9875</value> + <tag>KSM, MANAGEMENT, SECURITY</tag> <description> + The address and the base port where the KSM web UI will listen + on using HTTPS. + If the port is 0 then the server will start on a free port. </description> </property> - - <!--KSM properties--> - <property> - <name>ozone.ksm.handler.count.key</name> - <value>200</value> - <tag>PERFORMANCE,KSM</tag> + <name>ozone.ksm.https-bind-host</name> + <value>0.0.0.0</value> + <tag>KSM, MANAGEMENT, SECURITY</tag> <description> - The number of RPC handler threads for each KSM service endpoint. + The actual address the KSM web server will bind to using HTTPS. + If this optional address is set, it overrides only the hostname portion of + ozone.ksm.http-address. </description> </property> - <property> - <name>ozone.ksm.address</name> - <value></value> - <tag>KSM,REQUIRED</tag> + <name>ozone.ksm.keytab.file</name> + <value/> + <tag>KSM, SECURITY</tag> <description> - The address of the Ozone KSM service. + The keytab file for Kerberos authentication in KSM. </description> </property> - <property> - <name>ozone.ksm.leveldb.cache.size.mb</name> + <name>ozone.ksm.db.cache.size.mb</name> <value>128</value> - <tag>PERFORMANCE,KSM</tag> + <tag>KSM, PERFORMANCE</tag> <description> - The size of KSM LevelDB cache in MB that used for caching files. + The size of KSM DB cache in MB that used for caching files. + This value is set to an abnormally low value in the default configuration. + That is to make unit testing easy. Generally, this value should be set to + something like 16GB or more, if you intend to use Ozone at scale. + + A large value for this key allows a proportionally larger amount of KSM + metadata to be cached in memory. This makes KSM operations faster. </description> </property> - <property> <name>ozone.ksm.user.max.volume</name> <value>1024</value> - <tag>KSM,MANAGEMENT</tag> + <tag>KSM, MANAGEMENT</tag> <description> - The maximum number of volumes that each user can create. + The maximum number of volumes a user can have on a cluster.Increasing or + decreasing this number has no real impact on ozone cluster. This is + defined only for operational purposes. Only an administrator can create a + volume, once a volume is created there are no restrictions on the number + of buckets or keys inside each bucket a user can create. </description> </property> - <property> <name>ozone.ksm.user.rights</name> <value>READ_WRITE</value> - <tag>SECURITY,KSM</tag> - <description> - Default user permissions in Ozone KSM. - </description> - </property> - - <property> - <name>ozone.ksm.group.rights</name> - <value>READ_WRITE</value> - <tag>SECURITY,KSM</tag> + <tag>KSM, SECURITY</tag> <description> - Default group permissions in Ozone KSM. + Default user permissions used in KSM. </description> </property> - - <property> - <name>ozone.scm.http.enabled</name> - <value>true</value> - <tag>OZONE</tag> + <name>ozone.localstorage.root</name> + <value>${hadoop.tmp.dir}/ozone</value> + <tag>OZONE, DEBUG</tag> <description> - Property to enable or disable SCM web ui. + This is used only for testing purposes. This value is used by the local + storage handler to simulate a REST backend. This is useful only when + debugging the REST front end independent of KSM and SCM. To be removed. </description> </property> - <property> - <name>ozone.scm.http-address</name> - <value>0.0.0.0:9876</value> - <tag>OZONE</tag> + <name>ozone.metadata.dirs</name> + <value/> + <tag>OZONE, KSM, SCM, CONTAINER, REQUIRED, STORAGE</tag> <description> - The address and the base port where the SCM web ui will listen on. + Ozone metadata is shared among KSM, which acts as the namespace + manager for ozone, SCM which acts as the block manager and data nodes + which maintain the name of the key(Key Name and BlockIDs). This + replicated and distributed metadata store is maintained under the + directory pointed by this key. Since metadata can be I/O intensive, at + least on KSM and SCM we recommend having SSDs. If you have the luxury + of mapping this path to SSDs on all machines in the cluster, that will + be excellent. - If the port is 0 then the server will start on a free port. + If Ratis metadata directories are not specified, Ratis server will emit a + warning and use this path for storing its metadata too. </description> </property> - <property> - <name>ozone.scm.http-bind-host</name> - <value>0.0.0.0</value> - <tag>OZONE</tag> + <name>ozone.metastore.impl</name> + <value>RocksDB</value> + <tag>OZONE, KSM, SCM, CONTAINER, STORAGE</tag> <description> - The actual address the SCM web server will bind to. If this optional - address is set, it overrides only the hostname portion of - ozone.scm.http-address. + Ozone metadata store implementation. Ozone metadata are well + distributed to multiple services such as ksm, scm. They are stored in + some local key-value databases. This property determines which database + library to use. Supported value is either LevelDB or RocksDB. </description> </property> <property> - <name>ozone.scm.https-address</name> - <value>0.0.0.0:9877</value> - <tag>OZONE</tag> - <description> - The address and the base port where the SCM web ui will listen on - using HTTPS. - - If the port is 0 then the server will start on a free port. + <name>ozone.scm.block.client.address</name> + <value/> + <tag>OZONE, SCM</tag> + <description>The address of the Ozone SCM block client service. If not + defined value of ozone.scm.client.address is used. </description> </property> - <property> - <name>ozone.scm.https-bind-host</name> + <name>ozone.scm.block.client.bind.host</name> <value>0.0.0.0</value> - <tag>OZONE</tag> + <tag>OZONE, SCM</tag> <description> - The actual address the SCM web server will bind to using HTTPS. If this - optional address is set, it overrides only the hostname portion of - ozone.scm.http-address. + The hostname or IP address used by the SCM block client + endpoint to bind. </description> </property> - <property> - <name>ozone.ksm.http.enabled</name> - <value>true</value> - <tag>KSM,MANAGEMENT</tag> + <name>ozone.scm.block.client.port</name> + <value>9863</value> + <tag>OZONE, SCM</tag> <description> - Property to enable or disable KSM web ui. + The port number of the Ozone SCM block client service. </description> </property> - <property> - <name>ozone.ksm.http-address</name> - <value>0.0.0.0:9874</value> - <tag>KSM,MANAGEMENT</tag> + <name>ozone.scm.block.deletion.max.retry</name> + <value>4096</value> + <tag>OZONE, SCM</tag> <description> - The address and the base port where the KSM web ui will listen on. - - If the port is 0 then the server will start on a free port. + SCM wraps up many blocks in a deletion transaction and sends that to data + node for physical deletion periodically. This property determines how many + times SCM is going to retry sending a deletion operation to the data node. </description> </property> - <property> - <name>ozone.ksm.http-bind-host</name> - <value>0.0.0.0</value> - <tag>KSM,MANAGEMENT</tag> + <name>ozone.scm.block.size</name> + <value>268435456</value> + <tag>OZONE, SCM</tag> <description> - The actual address the KSM web server will bind to. If this optional - address is set, it overrides only the hostname portion of - ozone.ksm.http-address. + The default size of a scm block in bytes. This is maps to the default + Ozone block size. </description> </property> - <property> - <name>ozone.ksm.https-address</name> - <value>0.0.0.0:9875</value> - <tag>KSM,MANAGEMENT,SECURITY</tag> + <name>ozone.scm.chunk.size</name> + <value>1048576</value> + <tag>OZONE, SCM, CONTAINER, PERFORMANCE</tag> <description> - The address and the base port where the KSM web ui will listen on - using HTTPS. + The chunk size for reading/writing chunk operations in bytes. - If the port is 0 then the server will start on a free port. + The chunk size defaults to 1MB. If the value configured is more than the + maximum size (1MB), it will be reset to the maximum size. This maps to the + network packet sizes and file write operations in the client to datanode + protocol. </description> </property> - <property> - <name>ozone.ksm.https-bind-host</name> - <value>0.0.0.0</value> - <tag>KSM,MANAGEMENT,SECURITY</tag> + <name>ozone.scm.client.address</name> + <value/> + <tag>OZONE, SCM, REQUIRED</tag> <description> - The actual address the KSM web server will bind to using HTTPS. If this - optional address is set, it overrides only the hostname portion of - ozone.ksm.http-address. + The address of the Ozone SCM client service. This is a required setting. + + It is a string in the host:port format. The port number is optional + and defaults to 9860. </description> </property> - <property> - <name>ozone.web.authentication.kerberos.principal</name> - <value></value> - <tag>OZONE,SECURITY</tag> - <description> - The server principal used by the SCM and KSM for web UI SPNEGO - authentication when Kerberos security is enabled. This is - typically set to HTTP/[email protected] The SPNEGO server principal - begins with the prefix HTTP/ by convention. + <name>ozone.scm.client.bind.host</name> + <value>0.0.0.0</value> + <tag>OZONE, SCM, MANAGEMENT</tag> + <description>The hostname or IP address used by the SCM client endpoint to + bind. + This setting is used by the SCM only and never used by clients. - If the value is '*', the web server will attempt to login with - every principal specified in the keytab file. + The setting can be useful in multi-homed setups to restrict the + availability of the SCM client service to a specific interface. + + The default is appropriate for most clusters. </description> </property> - <property> - <name>ozone.scm.keytab.file</name> - <value></value> - <tag>OZONE</tag> - <description> - The keytab file for Kerberos authentication in SCM. - </description> + <name>ozone.scm.client.port</name> + <value>9860</value> + <tag>OZONE, SCM, MANAGEMENT</tag> + <description>The port number of the Ozone SCM client service.</description> </property> - <property> - <name>ozone.ksm.keytab.file</name> - <value></value> - <tag>SECURITY,KSM</tag> + <name>ozone.scm.container.deletion-choosing.policy</name> + <value> + org.apache.hadoop.ozone.container.common.impl.TopNOrderedContainerDeletionChoosingPolicy + </value> + <tag>OZONE, MANAGEMENT</tag> <description> - The keytab file for Kerberos authentication in KSM. + The policy used for choosing desire containers for block deletion. + Datanode selects some containers to process block deletion + in a certain interval defined by ozone.block.deleting.service.interval.ms. + The number of containers to process in each interval is defined + by ozone.block.deleting.container.limit.per.interval. This property is + used to configure the policy applied while selecting containers. + There are two policies supporting now: + RandomContainerDeletionChoosingPolicy and + TopNOrderedContainerDeletionChoosingPolicy. + org.apache.hadoop.ozone.container.common.impl.RandomContainerDeletionChoosingPolicy + implements a simply random policy that to return a random list of + containers. + org.apache.hadoop.ozone.container.common.impl.TopNOrderedContainerDeletionChoosingPolicy + implements a policy that choosing top count number of containers in a + pending-deletion-blocks's num + based descending order. </description> </property> - <property> - <name>ozone.metastore.impl</name> - <value>RocksDB</value> - <tag>OZONE,KSM,SCM,CONTAINER,STORAGE</tag> - <description> - Ozone metadata store implementation. Ozone metadata are well distributed - to multiple services such as ksm, scm. They are stored in some local - key-value databases. This property determines which database library to - use. Supported value is either LevelDB or RocksDB. + <name>ozone.scm.container.placement.impl</name> + <value> + org.apache.hadoop.ozone.scm.container.placement.algorithms.SCMContainerPlacementRandom + </value> + <tag>OZONE, MANAGEMENT</tag> + <description>Placement policy class for containers. + Defaults to SCMContainerPlacementRandom.class </description> </property> - <property> - <name>dfs.cblock.servicerpc-address</name> - <value></value> - <tag>OZONE</tag> - <description> - The address that cblock will be bind to, should be a host:port format, - this setting is required for cblock server to start. - This address is used for cblock management operations like create, - delete, info and list volumes + <name>ozone.scm.container.provision_batch_size</name> + <value>1</value> + <tag>OZONE, PERFORMANCE</tag> + <description>Pre-provision specified number of containers for block + allocation. </description> </property> - <property> - <name>dfs.cblock.service.rpc-bind-host</name> - <value>0.0.0.0</value> - <tag>OZONE</tag> - <description> - The actual address the cblock service rpc server will bind to. If this - optional address is set, it overrides only the hostname portion of - dfs.cblock.servicerpc-address. + <name>ozone.scm.container.report.processing.interval.seconds</name> + <value>60</value> + <tag>OZONE, PERFORMANCE</tag> + <description>Time interval in seconds for scm to process container reports + for a node pool. Scm handles node pool reports in a cyclic clock + manner, it fetches pools periodically with this time interval. </description> </property> - <property> - <name>dfs.cblock.jscsi-address</name> - <value></value> - <tag>OZONE</tag> - <description> - The address that cblock will be bind to, should be a host:port format, - this setting is required for cblock server to start. - This address to be used by jscsi to mount volume. + <name>ozone.scm.container.reports.wait.timeout.seconds</name> + <value>300</value> + <tag>OZONE, PERFORMANCE, MANAGEMENT</tag> + <description>Maximum time to wait in seconds for processing all container + reports from + a node pool. It determines the timeout for a + node pool report. </description> </property> - <property> - <name>dfs.cblock.jscsi.rpc-bind-host</name> - <value>0.0.0.0</value> - <tag>OZONE</tag> + <name>ozone.scm.container.size.gb</name> + <value>5</value> + <tag>OZONE, PERFORMANCE, MANAGEMENT</tag> <description> - The actual address the cblock jscsi rpc server will bind to. If this - optional address is set, it overrides only the hostname portion of - dfs.cblock.jscsi-address. + Default container size used by Ozone. This value is specified + in GB. + There are two considerations while picking this number. The speed at which + a container can be replicated, determined by the network speed and the + metadata that each container generates. So selecting a large number + creates less SCM metadata, but recovery time will be more. 5GB is a number + that maps to quick replication times in gigabit networks, but still + balances the amount of metadata. </description> </property> - <property> - <name>dfs.cblock.jscsi.port</name> - <value>9811</value> - <tag>OZONE</tag> + <name>ozone.scm.datanode.address</name> + <value/> + <tag>OZONE, MANAGEMENT</tag> <description> - The port on CBlockManager node for jSCSI to talk to. + The address of the Ozone SCM service used for internal + communication between the DataNodes and the SCM. + + It is a string in the host:port format. The port number is optional + and defaults to 9861. + + This setting is optional. If unspecified then the hostname portion + is picked from the ozone.scm.client.address setting and the + default service port of 9861 is chosen. </description> </property> - <property> - <name>dfs.storage.service.handler.count</name> - <value>10</value> - <tag>OZONE</tag> + <name>ozone.scm.datanode.bind.host</name> + <value/> + <tag>OZONE, MANAGEMENT</tag> <description> - Default number of handlers for CBlock service rpc. + The hostname or IP address used by the SCM service endpoint to + bind. </description> </property> - <property> - <name>dfs.cblock.service.leveldb.path</name> - <value>/tmp/cblock_levelDB.dat</value> - <tag>OZONE</tag> - <description> - Default path for the cblock meta data disk store. + <name>ozone.scm.datanode.id</name> + <value/> + <tag>OZONE, MANAGEMENT</tag> + <description>The path that datanodes will use to store the datanode ID. + If this value is not set, then datanode ID is created under the + metadata directory. </description> </property> - <property> - <name>dfs.cblock.disk.cache.path</name> - <value>/tmp/cblockCacheDB</value> - <tag>OZONE</tag> + <name>ozone.scm.datanode.port</name> + <value>9861</value> + <tag>OZONE, MANAGEMENT</tag> <description> - Default path for the cblock local cache. + The port number of the Ozone SCM service. </description> </property> - <property> - <name>dfs.cblock.trace.io</name> - <value>false</value> - <tag>OZONE</tag> - <description> - Default flag for enabling trace io. + <name>ozone.scm.db.cache.size.mb</name> + <value>128</value> + <tag>OZONE, PERFORMANCE</tag> + <description>SCM keeps track of the Containers in the cluster. This DB holds + the container metadata. This value is set to a small value to make the + unit + testing runs smooth. In production, we recommend a value of 16GB or + higher. This allows SCM to avoid disk I/O's while looking up the container + location. </description> </property> - <property> - <name>dfs.cblock.short.circuit.io</name> - <value>false</value> - <tag>OZONE</tag> + <name>ozone.scm.dead.node.interval.ms</name> + <value>600000</value> + <tag>OZONE, MANAGEMENT</tag> <description> - Default flag to enable cblock local cache. + The interval between heartbeats before a node is tagged as dead. This + value is in milliseconds. </description> </property> - <property> - <name>dfs.cblock.cache.cache.size.in.kb</name> - <value>256</value> - <tag>OZONE</tag> + <name>ozone.scm.handler.count.key</name> + <value>10</value> + <tag>OZONE, MANAGEMENT, PERFORMANCE</tag> <description> - Default cblock cache queue size, in number of kb. + The number of RPC handler threads for each SCM service + endpoint. + + The default is appropriate for small clusters (tens of nodes). + + Set a value that is appropriate for the cluster size. Generally, HDFS + recommends RPC handler count is set to 20 * log2(Cluster Size) with an + upper limit of 200. However, SCM will not have the same amount of + traffic as Namenode, so a value much smaller than that will work well too. </description> </property> - <property> - <name>dfs.cblock.cache.core.min.pool.size</name> - <value>16</value> - <tag>OZONE</tag> + <name>ozone.scm.heartbeat.interval.seconds</name> + <value>30</value> + <tag>OZONE, MANAGEMENT</tag> <description> - Minimum number of thread pool thread that cBlock cache will use for background I/O. + The heartbeat interval from a data node to SCM in seconds. Yes, + it is not three but 30, since most data nodes will heart beating via Ratis + heartbeats. If a client is not able to talk to a data node, it will notify + KSM/SCM eventually. So a 30 second HB seems to work. This assumes that + replication strategy used is Ratis if not, this value should be set to + something smaller like 3 seconds. </description> </property> - <property> - <name>dfs.cblock.cache.max.pool.size</name> - <value>256</value> - <tag>OZONE</tag> + <name>ozone.scm.heartbeat.log.warn.interval.count</name> + <value>10</value> + <tag>OZONE, MANAGEMENT</tag> <description> - Maximum number of thread pool thread that cBlcok cache will use for background I/O. + Defines how frequently we will log the missing of a heartbeat to SCM. + For example in the default case, we will write a warning message for each + ten consecutive heartbeats that we miss to SCM. This helps in reducing + clutter in a data node log, but trade off is that logs will have less of + this statement. </description> </property> - <property> - <name>dfs.cblock.cache.keep.alive.seconds</name> - <value>60</value> - <tag>OZONE</tag> + <name>ozone.scm.heartbeat.rpc-timeout</name> + <value>1000</value> + <tag>OZONE, MANAGEMENT</tag> <description> - Number of seconds to keep the Thread alive when it is idle. + Timeout value for the RPC from Datanode to SCM in milliseconds. </description> </property> - <property> - <name>dfs.cblock.cache.thread.priority</name> - <value>5</value> - <tag>OZONE</tag> + <name>ozone.scm.heartbeat.thread.interval.ms</name> + <value>3000</value> + <tag>OZONE, MANAGEMENT</tag> <description> - Priority of cache flusher thread, affecting the relative - performance of write and read.upported values are 1, 5, 10. - Use 10 for high priority and 1 for low priority. + When a heartbeat from the data node arrives on SCM, It is queued for + processing with the time stamp of when the heartbeat arrived. There is a + heartbeat processing thread inside SCM that runs at a specified interval. + This value controls how frequently this thread is run. + + There are some assumptions build into SCM such as this value should allow + the heartbeat processing thread to run at least three times more + frequently than heartbeats and at least five times more than stale node + detection time. If you specify a wrong value, SCM will gracefully refuse + to run. For more info look at the node manager tests in SCM. + + In short, you don't need to change this. </description> </property> - <property> - <name>dfs.cblock.cache.block.buffer.size</name> - <value>512</value> - <tag>OZONE</tag> + <name>ozone.scm.http-address</name> + <value>0.0.0.0:9876</value> + <tag>OZONE, MANAGEMENT</tag> <description> - Block Buffer size in terms of blockID entries, in number of blockIDs. + The address and the base port where the SCM web ui will listen on. + + If the port is 0 then the server will start on a free port. </description> </property> - <property> - <name>dfs.cblock.block.buffer.flush.interval.seconds</name> - <value>60</value> - <tag>OZONE</tag> + <name>ozone.scm.http-bind-host</name> + <value>0.0.0.0</value> + <tag>OZONE, MANAGEMENT</tag> <description> - The interval to flush cblock buffer in seconds. + The actual address the SCM web server will bind to. If this + optional address is set, it overrides only the hostname portion of + ozone.scm.http-address. </description> </property> - <property> - <name>dfs.cblock.jscsi.server.address</name> - <value>0.0.0.0</value> - <tag>OZONE</tag> + <name>ozone.scm.http.enabled</name> + <value>true</value> + <tag>OZONE, MANAGEMENT</tag> <description> - The address that jscsi server will be running, should have one - local jscsi server for each client that tries to mount cblock. + Property to enable or disable SCM web ui. </description> </property> - <property> - <name>dfs.cblock.jscsi.cblock.server.address</name> - <value>127.0.0.1</value> - <tag>OZONE</tag> + <name>ozone.scm.https-address</name> + <value>0.0.0.0:9877</value> + <tag>OZONE, MANAGEMENT</tag> <description> - The address local jscsi server will use to talk to cblock manager. + The address and the base port where the SCM web UI will listen + on using HTTPS. + + If the port is 0 then the server will start on a free port. </description> </property> - <property> - <name>dfs.cblock.container.size</name> - <value>5</value> - <tag>OZONE</tag> + <name>ozone.scm.https-bind-host</name> + <value>0.0.0.0</value> + <tag>OZONE, MANAGEMENT</tag> <description> - The size of ozone container in number of GBs. Note that this is - not setting container size for ozone, but rather, this is setting - that assumption that cblock manager will make about ozone. + The actual address the SCM web server will bind to using HTTPS. + If this optional address is set, it overrides only the hostname portion of + ozone.scm.http-address. </description> </property> - <property> - <name>dfs.cblock.cache.leveldb.cache.size.mb</name> - <value>256</value> - <tag>OZONE</tag> + <name>ozone.scm.keytab.file</name> + <value/> + <tag>OZONE, SECURITY</tag> <description> - The size of LevelDB cache file which uses an off-heap cache in LevelDB. + The keytab file for Kerberos authentication in SCM. </description> </property> - <property> - <name>dfs.cblock.cache.max.retry</name> - <value>65536</value> - <tag>OZONE</tag> + <name>ozone.scm.max.container.report.threads</name> + <value>100</value> + <tag>OZONE, PERFORMANCE</tag> <description> - The maximum number of retries when writing a block to container. + Maximum number of threads to process container reports in scm. + Each container report from a data node is processed by scm in a worker + thread, fetched from a thread pool. This property is used to control the + maximum size of the thread pool. </description> </property> - <property> - <name>dfs.cblock.manager.pool.size</name> - <value>16</value> - <tag>OZONE</tag> + <name>ozone.scm.max.hb.count.to.process</name> + <value>5000</value> + <tag>OZONE, MANAGEMENT, PERFORMANCE</tag> <description> - Number of threads that cblock manager will use for container operations. + The maximum number of heartbeat to process per loop of the + heartbeat process thread. Please see + ozone.scm.heartbeat.thread.interval.ms + for more info. </description> </property> - <property> - <name>dfs.cblock.rpc.timeout.seconds</name> - <value>300</value> + <name>ozone.scm.names</name> + <value/> <tag>OZONE</tag> <description> - RPC timeout in seconds used for cblock CLI operations. + The value of this property is a set of DNS | DNS:PORT | IP + Address | IP:PORT. Written as a comma separated string. e.g. scm1, + scm2:8020, 7.7.7.7:7777. + This property allows datanodes to discover where SCM is, so that + datanodes can send heartbeat to SCM. </description> </property> - <property> - <name>dfs.cblock.scm.ipaddress</name> - <value>127.0.0.1</value> - <tag>OZONE</tag> + <name>ozone.scm.stale.node.interval.ms</name> + <value>90000</value> + <tag>OZONE, MANAGEMENT</tag> <description> - IP address used by cblock to connect to SCM. + The interval in milliseconds for stale node flagging. Please + see ozone.scm.heartbeat.thread.interval.ms before changing this value. </description> </property> - <property> - <name>dfs.cblock.scm.port</name> - <value>9860</value> - <tag>OZONE</tag> + <name>ozone.trace.enabled</name> + <value>false</value> + <tag>OZONE, DEBUG</tag> <description> - Port used by cblock to connect to SCM. + Setting this flag to true dumps the HTTP request/ response in + the logs. Very useful when debugging REST protocol. </description> </property> - <property> - <name>ozone.scm.block.size</name> - <value>268435456</value> - <tag>OZONE</tag> + <name>ozone.web.authentication.kerberos.principal</name> + <value/> + <tag>OZONE, SECURITY</tag> <description> - The default size of a scm block in bytes. + The server principal used by the SCM and KSM for web UI SPNEGO + authentication when Kerberos security is enabled. This is typically set to + HTTP/[email protected] The SPNEGO server principal begins with the prefix + HTTP/ by convention. + + If the value is '*', the web server will attempt to login with + every principal specified in the keytab file. </description> </property> + <!--Client Settings--> <property> - <name>dfs.container.ratis.ipc</name> - <value>9858</value> - <tag>OZONE,CONTAINER,PIPELINE,RATIS</tag> + <name>scm.container.client.idle.threshold</name> + <value>10000</value> + <tag>OZONE, PERFORMANCE</tag> <description> - The ipc port number of container. + In the standalone pipelines, the SCM clients use netty to + communicate with the container. It also uses connection pooling to + reduce client side overheads. This allows a connection to stay idle for + a while before the connection is closed. </description> </property> - <property> - <name>dfs.container.ratis.ipc.random.port</name> - <value>false</value> - <tag>OZONE,CONTAINER,PIPELINE,RATIS</tag> + <name>scm.container.client.max.size</name> + <value>256</value> + <tag>OZONE, PERFORMANCE</tag> <description> - Whether allocates a random free port for ozone ratis port for container. + Controls the maximum number of connections that we cached via + clientconnection pooling. If the number of connection + exceed this count then the oldest idle connection is evicted. </description> </property> - -</configuration> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8fea966/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java index b4f4fee..43ba40d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java @@ -18,13 +18,9 @@ package org.apache.hadoop.ozone; import org.apache.hadoop.cblock.CBlockConfigKeys; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.OzonePropertyTag; import org.apache.hadoop.conf.TestConfigurationFieldsBase; import org.apache.hadoop.ozone.ksm.KSMConfigKeys; import org.apache.hadoop.scm.ScmConfigKeys; -import org.junit.Test; -import static org.junit.Assert.assertEquals; /** * Tests if configuration constants documented in ozone-defaults.xml. @@ -40,32 +36,4 @@ public class TestOzoneConfigurationFields extends TestConfigurationFieldsBase { errorIfMissingConfigProps = true; errorIfMissingXmlProps = true; } - - @Test - public void testOzoneTags() { - Configuration config = new OzoneConfiguration(); - config.reloadConfiguration(); - - // To load default resources - config.get("ozone.enabled"); - assertEquals(87, - config.getAllPropertiesByTag(OzonePropertyTag.OZONE).size()); - assertEquals(15, config.getAllPropertiesByTag(OzonePropertyTag.KSM) - .size()); - assertEquals(6, config.getAllPropertiesByTag(OzonePropertyTag.SCM) - .size()); - assertEquals(6, config.getAllPropertiesByTag(OzonePropertyTag.MANAGEMENT) - .size()); - assertEquals(7, config.getAllPropertiesByTag(OzonePropertyTag.SECURITY) - .size()); - assertEquals(9, config.getAllPropertiesByTag(OzonePropertyTag.PERFORMANCE) - .size()); - assertEquals(1, config.getAllPropertiesByTag(OzonePropertyTag.DEBUG) - .size()); - assertEquals(3, config.getAllPropertiesByTag(OzonePropertyTag.REQUIRED) - .size()); - assertEquals(2, config.getAllPropertiesByTag(OzonePropertyTag.RATIS) - .size()); - } - } \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
