http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/main/resources/ozone-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/main/resources/ozone-default.xml b/hadoop-hdsl/common/src/main/resources/ozone-default.xml deleted file mode 100644 index 9feadcf..0000000 --- a/hadoop-hdsl/common/src/main/resources/ozone-default.xml +++ /dev/null @@ -1,1031 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<!-- Do not modify this file directly. Instead, copy entries that you --> -<!-- wish to modify from this file into ozone-site.xml and change them --> -<!-- there. If ozone-site.xml does not already exist, create it. --> - -<!--Tags supported are OZONE, CBLOCK, MANAGEMENT, SECURITY, PERFORMANCE, --> -<!--DEBUG, CLIENT, SERVER, KSM, SCM, CRITICAL, RATIS, CONTAINER, REQUIRED, --> -<!--REST, STORAGE, PIPELINE, STANDALONE --> - -<configuration> - - <!--Container Settings used by Datanode--> - <property> - <name>ozone.container.cache.size</name> - <value>1024</value> - <tag>PERFORMANCE, CONTAINER, STORAGE</tag> - <description>The open container is cached on the data node side. We maintain - an LRU - cache for caching the recently used containers. This setting controls the - size of that cache. - </description> - </property> - <property> - <name>dfs.container.ipc</name> - <value>9859</value> - <tag>OZONE, CONTAINER, MANAGEMENT</tag> - <description>The ipc port number of container.</description> - </property> - <property> - <name>dfs.container.ipc.random.port</name> - <value>false</value> - <tag>OZONE, DEBUG, CONTAINER</tag> - <description>Allocates a random free port for ozone container. This is used - only while - running unit tests. - </description> - </property> - <property> - <name>dfs.container.ratis.datanode.storage.dir</name> - <value/> - <tag>OZONE, CONTAINER, STORAGE, MANAGEMENT, RATIS</tag> - <description>This directory is used for storing Ratis metadata like logs. If - this is - not set then default metadata dirs is used. A warning will be logged if - this not set. Ideally, this should be mapped to a fast disk like an SSD. - </description> - </property> - <property> - <name>dfs.container.ratis.enabled</name> - <value>false</value> - <tag>OZONE, MANAGEMENT, PIPELINE, RATIS</tag> - <description>Ozone supports different kinds of replication pipelines. Ratis - is one of - the replication pipeline supported by ozone. - </description> - </property> - <property> - <name>dfs.container.ratis.ipc</name> - <value>9858</value> - <tag>OZONE, CONTAINER, PIPELINE, RATIS, MANAGEMENT</tag> - <description>The ipc port number of container.</description> - </property> - <property> - <name>dfs.container.ratis.ipc.random.port</name> - <value>false</value> - <tag>OZONE,DEBUG</tag> - <description>Allocates a random free port for ozone ratis port for the - container. This - is used only while running unit tests. - </description> - </property> - <property> - <name>dfs.container.ratis.rpc.type</name> - <value>GRPC</value> - <tag>OZONE, RATIS, MANAGEMENT</tag> - <description>Ratis supports different kinds of transports like netty, GRPC, - Hadoop RPC - etc. This picks one of those for this cluster. - </description> - </property> - <property> - <name>dfs.container.ratis.num.write.chunk.threads</name> - <value>60</value> - <tag>OZONE, RATIS, PERFORMANCE</tag> - <description>Maximum number of threads in the thread pool that Ratis - will use for writing chunks (60 by default). - </description> - </property> - <property> - <name>dfs.container.ratis.segment.size</name> - <value>1073741824</value> - <tag>OZONE, RATIS, PERFORMANCE</tag> - <description>The size of the raft segment used by Apache Ratis on datanodes. - (1 GB by default) - </description> - </property> - <property> - <name>dfs.container.ratis.segment.preallocated.size</name> - <value>134217728</value> - <tag>OZONE, RATIS, PERFORMANCE</tag> - <description>The size of the buffer which is preallocated for raft segment - used by Apache Ratis on datanodes.(128 MB by default) - </description> - </property> - <property> - <name>ozone.container.report.interval</name> - <value>60000ms</value> - <tag>OZONE, CONTAINER, MANAGEMENT</tag> - <description>Time interval of the datanode to send container report. Each - datanode periodically send container report upon receive - sendContainerReport from SCM. Unit could be defined with - postfix (ns,ms,s,m,h,d)</description> - </property> - <!--Ozone Settings--> - <property> - <name>ozone.administrators</name> - <value/> - <tag>OZONE, SECURITY</tag> - <description>Ozone administrator users delimited by the comma. - If not set, only the user who launches an ozone service will be the admin - user. This property must be set if ozone services are started by different - users. Otherwise, the RPC layer will reject calls from other servers which - are started by users not in the list. - </description> - </property> - <property> - <name>ozone.block.deleting.container.limit.per.interval</name> - <value>10</value> - <tag>OZONE, PERFORMANCE, SCM</tag> - <description>A maximum number of containers to be scanned by block deleting - service per - time interval. The block deleting service spawns a thread to handle block - deletions in a container. This property is used to throttle the number of - threads spawned for block deletions. - </description> - </property> - <property> - <name>ozone.block.deleting.limit.per.task</name> - <value>1000</value> - <tag>OZONE, PERFORMANCE, SCM</tag> - <description>A maximum number of blocks to be deleted by block deleting - service per - time interval. This property is used to throttle the actual number of - block deletions on a data node per container. - </description> - </property> - <property> - <name>ozone.block.deleting.service.interval</name> - <value>1m</value> - <tag>OZONE, PERFORMANCE, SCM</tag> - <description>Time interval of the block deleting service. - The block deleting service runs on each datanode periodically and - deletes blocks queued for deletion. Unit could be defined with - postfix (ns,ms,s,m,h,d) - </description> - </property> - <property> - <name>ozone.block.deleting.service.timeout</name> - <value>300000ms</value> - <tag>OZONE, PERFORMANCE, SCM</tag> - <description>A timeout value of block deletion service. If this is set - greater than 0, - the service will stop waiting for the block deleting completion after this - time. If timeout happens to a large proportion of block deletion, this - needs to be increased with ozone.block.deleting.limit.per.task. This - setting supports multiple time unit suffixes as described in - dfs.heartbeat.interval. If no suffix is specified, then milliseconds is - assumed. - </description> - </property> - <property> - <name>ozone.client.connection.timeout</name> - <value>5000ms</value> - <tag>OZONE, PERFORMANCE, CLIENT</tag> - <description>Connection timeout for Ozone client in milliseconds. - </description> - </property> - <property> - <name>ozone.client.protocol</name> - <value>org.apache.hadoop.ozone.client.rpc.RpcClient</value> - <tag>OZONE, CLIENT, MANAGEMENT</tag> - <description>Protocol class to be used by the client to connect to ozone - cluster. - The build-in implementation includes: - org.apache.hadoop.ozone.client.rpc.RpcClient for RPC - org.apache.hadoop.ozone.client.rest.RestClient for REST - The default is the RpClient. Please do not change this unless you have a - very good understanding of what you are doing. - </description> - </property> - <property> - <name>ozone.client.socket.timeout</name> - <value>5000ms</value> - <tag>OZONE, CLIENT</tag> - <description>Socket timeout for Ozone client. Unit could be defined with - postfix (ns,ms,s,m,h,d)</description> - </property> - <property> - <name>ozone.enabled</name> - <value>false</value> - <tag>OZONE, REQUIRED</tag> - <description> - Status of the Ozone Object Storage service is enabled. - Set to true to enable Ozone. - Set to false to disable Ozone. - Unless this value is set to true, Ozone services will not be started in - the cluster. - - Please note: By default ozone is disabled on a hadoop cluster. - </description> - </property> - <property> - <name>ozone.handler.type</name> - <value>distributed</value> - <tag>OZONE, REST</tag> - <description> - Tells ozone which storage handler to use. The possible values are: - distributed - The Ozone distributed storage handler, which speaks to - KSM/SCM on the backend and provides REST services to clients. - local - Local Storage handler strictly for testing - To be removed. - </description> - </property> - <property> - <name>ozone.key.deleting.limit.per.task</name> - <value>1000</value> - <tag>KSM, PERFORMANCE</tag> - <description> - A maximum number of keys to be scanned by key deleting service - per time interval in KSM. Those keys are sent to delete metadata and - generate transactions in SCM for next async deletion between SCM - and DataNode. - </description> - </property> - <property> - <name>ozone.ksm.address</name> - <value/> - <tag>KSM, REQUIRED</tag> - <description> - The address of the Ozone KSM service. This allows clients to discover - the KSMs address. - </description> - </property> - <property> - <name>ozone.ksm.group.rights</name> - <value>READ_WRITE</value> - <tag>KSM, SECURITY</tag> - <description> - Default group permissions in Ozone KSM. - </description> - </property> - <property> - <name>ozone.ksm.handler.count.key</name> - <value>20</value> - <tag>KSM, PERFORMANCE</tag> - <description> - The number of RPC handler threads for KSM service endpoints. - </description> - </property> - <property> - <name>ozone.ksm.http-address</name> - <value>0.0.0.0:9874</value> - <tag>KSM, MANAGEMENT</tag> - <description> - The address and the base port where the KSM web UI will listen on. - - If the port is 0, then the server will start on a free port. However, it - is best to specify a well-known port, so it is easy to connect and see - the KSM management UI. - </description> - </property> - <property> - <name>ozone.ksm.http-bind-host</name> - <value>0.0.0.0</value> - <tag>KSM, MANAGEMENT</tag> - <description> - The actual address the KSM web server will bind to. If this optional - the address is set, it overrides only the hostname portion of - ozone.ksm.http-address. - </description> - </property> - <property> - <name>ozone.ksm.http.enabled</name> - <value>true</value> - <tag>KSM, MANAGEMENT</tag> - <description> - Property to enable or disable KSM web user interface. - </description> - </property> - <property> - <name>ozone.ksm.https-address</name> - <value>0.0.0.0:9875</value> - <tag>KSM, MANAGEMENT, SECURITY</tag> - <description> - The address and the base port where the KSM web UI will listen - on using HTTPS. - If the port is 0 then the server will start on a free port. - </description> - </property> - <property> - <name>ozone.ksm.https-bind-host</name> - <value>0.0.0.0</value> - <tag>KSM, MANAGEMENT, SECURITY</tag> - <description> - The actual address the KSM web server will bind to using HTTPS. - If this optional address is set, it overrides only the hostname portion of - ozone.ksm.http-address. - </description> - </property> - <property> - <name>ozone.ksm.keytab.file</name> - <value/> - <tag>KSM, SECURITY</tag> - <description> - The keytab file for Kerberos authentication in KSM. - </description> - </property> - <property> - <name>ozone.ksm.db.cache.size.mb</name> - <value>128</value> - <tag>KSM, PERFORMANCE</tag> - <description> - The size of KSM DB cache in MB that used for caching files. - This value is set to an abnormally low value in the default configuration. - That is to make unit testing easy. Generally, this value should be set to - something like 16GB or more, if you intend to use Ozone at scale. - - A large value for this key allows a proportionally larger amount of KSM - metadata to be cached in memory. This makes KSM operations faster. - </description> - </property> - <property> - <name>ozone.ksm.user.max.volume</name> - <value>1024</value> - <tag>KSM, MANAGEMENT</tag> - <description> - The maximum number of volumes a user can have on a cluster.Increasing or - decreasing this number has no real impact on ozone cluster. This is - defined only for operational purposes. Only an administrator can create a - volume, once a volume is created there are no restrictions on the number - of buckets or keys inside each bucket a user can create. - </description> - </property> - <property> - <name>ozone.ksm.user.rights</name> - <value>READ_WRITE</value> - <tag>KSM, SECURITY</tag> - <description> - Default user permissions used in KSM. - </description> - </property> - <property> - <name>ozone.localstorage.root</name> - <value>${hadoop.tmp.dir}/ozone</value> - <tag>OZONE, DEBUG</tag> - <description> - This is used only for testing purposes. This value is used by the local - storage handler to simulate a REST backend. This is useful only when - debugging the REST front end independent of KSM and SCM. To be removed. - </description> - </property> - <property> - <name>ozone.metadata.dirs</name> - <value/> - <tag>OZONE, KSM, SCM, CONTAINER, REQUIRED, STORAGE</tag> - <description> - Ozone metadata is shared among KSM, which acts as the namespace - manager for ozone, SCM which acts as the block manager and data nodes - which maintain the name of the key(Key Name and BlockIDs). This - replicated and distributed metadata store is maintained under the - directory pointed by this key. Since metadata can be I/O intensive, at - least on KSM and SCM we recommend having SSDs. If you have the luxury - of mapping this path to SSDs on all machines in the cluster, that will - be excellent. - - If Ratis metadata directories are not specified, Ratis server will emit a - warning and use this path for storing its metadata too. - </description> - </property> - <property> - <name>ozone.metastore.impl</name> - <value>RocksDB</value> - <tag>OZONE, KSM, SCM, CONTAINER, STORAGE</tag> - <description> - Ozone metadata store implementation. Ozone metadata are well - distributed to multiple services such as ksm, scm. They are stored in - some local key-value databases. This property determines which database - library to use. Supported value is either LevelDB or RocksDB. - </description> - </property> - - <property> - <name>ozone.metastore.rocksdb.statistics</name> - <value>ALL</value> - <tag>OZONE, KSM, SCM, STORAGE, PERFORMANCE</tag> - <description> - The statistics level of the rocksdb store. If you use any value from - org.rocksdb.StatsLevel (eg. ALL or EXCEPT_DETAILED_TIMERS), the rocksdb - statistics will be exposed over JMX bean with the choosed setting. Set - it to OFF to not initialize rocksdb statistics at all. Please note that - collection of statistics could have 5-10% performance penalty. - Check the rocksdb documentation for more details. - </description> - </property> - - <property> - <name>ozone.scm.block.client.address</name> - <value/> - <tag>OZONE, SCM</tag> - <description>The address of the Ozone SCM block client service. If not - defined value of ozone.scm.client.address is used. - </description> - </property> - <property> - <name>ozone.scm.block.client.bind.host</name> - <value>0.0.0.0</value> - <tag>OZONE, SCM</tag> - <description> - The hostname or IP address used by the SCM block client - endpoint to bind. - </description> - </property> - <property> - <name>ozone.scm.block.client.port</name> - <value>9863</value> - <tag>OZONE, SCM</tag> - <description> - The port number of the Ozone SCM block client service. - </description> - </property> - <property> - <name>ozone.scm.block.deletion.max.retry</name> - <value>4096</value> - <tag>OZONE, SCM</tag> - <description> - SCM wraps up many blocks in a deletion transaction and sends that to data - node for physical deletion periodically. This property determines how many - times SCM is going to retry sending a deletion operation to the data node. - </description> - </property> - <property> - <name>ozone.scm.block.size.in.mb</name> - <value>256</value> - <tag>OZONE, SCM</tag> - <description> - The default size of a scm block in bytes. This is maps to the default - Ozone block size. - </description> - </property> - <property> - <name>ozone.scm.chunk.size</name> - <value>16777216</value> - <tag>OZONE, SCM, CONTAINER, PERFORMANCE</tag> - <description> - The chunk size for reading/writing chunk operations in bytes. - - The chunk size defaults to 8MB. If the value configured is more than the - maximum size (16MB), it will be reset to the maximum size. This maps to - the network packet sizes and file write operations in the client to - datanode protocol. - </description> - </property> - <property> - <name>ozone.scm.client.address</name> - <value/> - <tag>OZONE, SCM, REQUIRED</tag> - <description> - The address of the Ozone SCM client service. This is a required setting. - - It is a string in the host:port format. The port number is optional - and defaults to 9860. - </description> - </property> - <property> - <name>ozone.scm.client.bind.host</name> - <value>0.0.0.0</value> - <tag>OZONE, SCM, MANAGEMENT</tag> - <description>The hostname or IP address used by the SCM client endpoint to - bind. - This setting is used by the SCM only and never used by clients. - - The setting can be useful in multi-homed setups to restrict the - availability of the SCM client service to a specific interface. - - The default is appropriate for most clusters. - </description> - </property> - <property> - <name>ozone.scm.client.port</name> - <value>9860</value> - <tag>OZONE, SCM, MANAGEMENT</tag> - <description>The port number of the Ozone SCM client service.</description> - </property> - <property> - <name>ozone.scm.container.deletion-choosing.policy</name> - <value> - org.apache.hadoop.ozone.container.common.impl.TopNOrderedContainerDeletionChoosingPolicy - </value> - <tag>OZONE, MANAGEMENT</tag> - <description> - The policy used for choosing desire containers for block deletion. - Datanode selects some containers to process block deletion - in a certain interval defined by ozone.block.deleting.service.interval. - The number of containers to process in each interval is defined - by ozone.block.deleting.container.limit.per.interval. This property is - used to configure the policy applied while selecting containers. - There are two policies supporting now: - RandomContainerDeletionChoosingPolicy and - TopNOrderedContainerDeletionChoosingPolicy. - org.apache.hadoop.ozone.container.common.impl.RandomContainerDeletionChoosingPolicy - implements a simply random policy that to return a random list of - containers. - org.apache.hadoop.ozone.container.common.impl.TopNOrderedContainerDeletionChoosingPolicy - implements a policy that choosing top count number of containers in a - pending-deletion-blocks's num - based descending order. - </description> - </property> - <property> - <name>ozone.scm.container.placement.impl</name> - <value> - org.apache.hadoop.ozone.scm.container.placement.algorithms.SCMContainerPlacementRandom - </value> - <tag>OZONE, MANAGEMENT</tag> - <description>Placement policy class for containers. - Defaults to SCMContainerPlacementRandom.class - </description> - </property> - <property> - <name>ozone.scm.container.provision_batch_size</name> - <value>20</value> - <tag>OZONE, PERFORMANCE</tag> - <description>Pre-provision specified number of containers for block - allocation. - </description> - </property> - <property> - <name>ozone.scm.container.report.processing.interval</name> - <value>60s</value> - <tag>OZONE, PERFORMANCE</tag> - <description>Time interval for scm to process container reports - for a node pool. Scm handles node pool reports in a cyclic clock - manner, it fetches pools periodically with this time interval. - </description> - </property> - <property> - <name>ozone.scm.container.reports.wait.timeout</name> - <value>300s</value> - <tag>OZONE, PERFORMANCE, MANAGEMENT</tag> - <description>Maximum time to wait in seconds for processing all container - reports from - a node pool. It determines the timeout for a - node pool report. - </description> - </property> - <property> - <name>ozone.scm.container.size.gb</name> - <value>5</value> - <tag>OZONE, PERFORMANCE, MANAGEMENT</tag> - <description> - Default container size used by Ozone. This value is specified - in GB. - There are two considerations while picking this number. The speed at which - a container can be replicated, determined by the network speed and the - metadata that each container generates. So selecting a large number - creates less SCM metadata, but recovery time will be more. 5GB is a number - that maps to quick replication times in gigabit networks, but still - balances the amount of metadata. - </description> - </property> - <property> - <name>ozone.scm.datanode.address</name> - <value/> - <tag>OZONE, MANAGEMENT</tag> - <description> - The address of the Ozone SCM service used for internal - communication between the DataNodes and the SCM. - - It is a string in the host:port format. The port number is optional - and defaults to 9861. - - This setting is optional. If unspecified then the hostname portion - is picked from the ozone.scm.client.address setting and the - default service port of 9861 is chosen. - </description> - </property> - <property> - <name>ozone.scm.datanode.bind.host</name> - <value/> - <tag>OZONE, MANAGEMENT</tag> - <description> - The hostname or IP address used by the SCM service endpoint to - bind. - </description> - </property> - <property> - <name>ozone.scm.datanode.id</name> - <value/> - <tag>OZONE, MANAGEMENT</tag> - <description>The path that datanodes will use to store the datanode ID. - If this value is not set, then datanode ID is created under the - metadata directory. - </description> - </property> - <property> - <name>ozone.scm.datanode.port</name> - <value>9861</value> - <tag>OZONE, MANAGEMENT</tag> - <description> - The port number of the Ozone SCM service. - </description> - </property> - <property> - <name>ozone.scm.db.cache.size.mb</name> - <value>128</value> - <tag>OZONE, PERFORMANCE</tag> - <description>SCM keeps track of the Containers in the cluster. This DB holds - the container metadata. This value is set to a small value to make the - unit - testing runs smooth. In production, we recommend a value of 16GB or - higher. This allows SCM to avoid disk I/O's while looking up the container - location. - </description> - </property> - <property> - <name>ozone.scm.dead.node.interval</name> - <value>10m</value> - <tag>OZONE, MANAGEMENT</tag> - <description> - The interval between heartbeats before a node is tagged as dead. - </description> - </property> - <property> - <name>ozone.scm.handler.count.key</name> - <value>10</value> - <tag>OZONE, MANAGEMENT, PERFORMANCE</tag> - <description> - The number of RPC handler threads for each SCM service - endpoint. - - The default is appropriate for small clusters (tens of nodes). - - Set a value that is appropriate for the cluster size. Generally, HDFS - recommends RPC handler count is set to 20 * log2(Cluster Size) with an - upper limit of 200. However, SCM will not have the same amount of - traffic as Namenode, so a value much smaller than that will work well too. - </description> - </property> - <property> - <name>ozone.scm.heartbeat.interval</name> - <value>30s</value> - <tag>OZONE, MANAGEMENT</tag> - <description> - The heartbeat interval from a data node to SCM. Yes, - it is not three but 30, since most data nodes will heart beating via Ratis - heartbeats. If a client is not able to talk to a data node, it will notify - KSM/SCM eventually. So a 30 second HB seems to work. This assumes that - replication strategy used is Ratis if not, this value should be set to - something smaller like 3 seconds. - </description> - </property> - <property> - <name>ozone.scm.heartbeat.log.warn.interval.count</name> - <value>10</value> - <tag>OZONE, MANAGEMENT</tag> - <description> - Defines how frequently we will log the missing of a heartbeat to SCM. - For example in the default case, we will write a warning message for each - ten consecutive heartbeats that we miss to SCM. This helps in reducing - clutter in a data node log, but trade off is that logs will have less of - this statement. - </description> - </property> - <property> - <name>ozone.scm.heartbeat.rpc-timeout</name> - <value>1000</value> - <tag>OZONE, MANAGEMENT</tag> - <description> - Timeout value for the RPC from Datanode to SCM in milliseconds. - </description> - </property> - <property> - <name>ozone.scm.heartbeat.thread.interval</name> - <value>3s</value> - <tag>OZONE, MANAGEMENT</tag> - <description> - When a heartbeat from the data node arrives on SCM, It is queued for - processing with the time stamp of when the heartbeat arrived. There is a - heartbeat processing thread inside SCM that runs at a specified interval. - This value controls how frequently this thread is run. - - There are some assumptions build into SCM such as this value should allow - the heartbeat processing thread to run at least three times more - frequently than heartbeats and at least five times more than stale node - detection time. If you specify a wrong value, SCM will gracefully refuse - to run. For more info look at the node manager tests in SCM. - - In short, you don't need to change this. - </description> - </property> - <property> - <name>ozone.scm.http-address</name> - <value>0.0.0.0:9876</value> - <tag>OZONE, MANAGEMENT</tag> - <description> - The address and the base port where the SCM web ui will listen on. - - If the port is 0 then the server will start on a free port. - </description> - </property> - <property> - <name>ozone.scm.http-bind-host</name> - <value>0.0.0.0</value> - <tag>OZONE, MANAGEMENT</tag> - <description> - The actual address the SCM web server will bind to. If this - optional address is set, it overrides only the hostname portion of - ozone.scm.http-address. - </description> - </property> - <property> - <name>ozone.scm.http.enabled</name> - <value>true</value> - <tag>OZONE, MANAGEMENT</tag> - <description> - Property to enable or disable SCM web ui. - </description> - </property> - <property> - <name>ozone.scm.https-address</name> - <value>0.0.0.0:9877</value> - <tag>OZONE, MANAGEMENT</tag> - <description> - The address and the base port where the SCM web UI will listen - on using HTTPS. - - If the port is 0 then the server will start on a free port. - </description> - </property> - <property> - <name>ozone.scm.https-bind-host</name> - <value>0.0.0.0</value> - <tag>OZONE, MANAGEMENT</tag> - <description> - The actual address the SCM web server will bind to using HTTPS. - If this optional address is set, it overrides only the hostname portion of - ozone.scm.http-address. - </description> - </property> - <property> - <name>ozone.scm.keytab.file</name> - <value/> - <tag>OZONE, SECURITY</tag> - <description> - The keytab file for Kerberos authentication in SCM. - </description> - </property> - <property> - <name>ozone.scm.max.container.report.threads</name> - <value>100</value> - <tag>OZONE, PERFORMANCE</tag> - <description> - Maximum number of threads to process container reports in scm. - Each container report from a data node is processed by scm in a worker - thread, fetched from a thread pool. This property is used to control the - maximum size of the thread pool. - </description> - </property> - <property> - <name>ozone.scm.max.hb.count.to.process</name> - <value>5000</value> - <tag>OZONE, MANAGEMENT, PERFORMANCE</tag> - <description> - The maximum number of heartbeat to process per loop of the - heartbeat process thread. Please see - ozone.scm.heartbeat.thread.interval - for more info. - </description> - </property> - <property> - <name>ozone.scm.max.nodepool.processing.threads</name> - <value>1</value> - <tag>OZONE, MANAGEMENT, PERFORMANCE</tag> - <description> - Number of node pools to process in parallel. - </description> - </property> - <property> - <name>ozone.scm.names</name> - <value/> - <tag>OZONE</tag> - <description> - The value of this property is a set of DNS | DNS:PORT | IP - Address | IP:PORT. Written as a comma separated string. e.g. scm1, - scm2:8020, 7.7.7.7:7777. - This property allows datanodes to discover where SCM is, so that - datanodes can send heartbeat to SCM. - </description> - </property> - <property> - <name>ozone.scm.stale.node.interval</name> - <value>90s</value> - <tag>OZONE, MANAGEMENT</tag> - <description> - The interval for stale node flagging. Please - see ozone.scm.heartbeat.thread.interval before changing this value. - </description> - </property> - <property> - <name>ozone.scm.max.nodepool.processing.threads</name> - <value>1</value> - <tag>OZONE, SCM</tag> - <description> - Controls the number of node pools that can be processed in parallel by - Container Supervisor. - </description> - </property> - <property> - <name>ozone.trace.enabled</name> - <value>false</value> - <tag>OZONE, DEBUG</tag> - <description> - Setting this flag to true dumps the HTTP request/ response in - the logs. Very useful when debugging REST protocol. - </description> - </property> - <property> - <name>ozone.web.authentication.kerberos.principal</name> - <value/> - <tag>OZONE, SECURITY</tag> - <description> - The server principal used by the SCM and KSM for web UI SPNEGO - authentication when Kerberos security is enabled. This is typically set to - HTTP/_h...@realm.tld The SPNEGO server principal begins with the prefix - HTTP/ by convention. - - If the value is '*', the web server will attempt to login with - every principal specified in the keytab file. - </description> - </property> - - <!--Client Settings--> - <property> - <name>scm.container.client.idle.threshold</name> - <value>10s</value> - <tag>OZONE, PERFORMANCE</tag> - <description> - In the standalone pipelines, the SCM clients use netty to - communicate with the container. It also uses connection pooling to - reduce client side overheads. This allows a connection to stay idle for - a while before the connection is closed. - </description> - </property> - <property> - <name>scm.container.client.max.size</name> - <value>256</value> - <tag>OZONE, PERFORMANCE</tag> - <description> - Controls the maximum number of connections that we cached via - clientconnection pooling. If the number of connection - exceed this count then the oldest idle connection is evicted. - </description> - </property> - - <property> - <name>scm.container.client.max.outstanding.requests</name> - <value>100</value> - <tag>OZONE, PERFORMANCE</tag> - <description> - Controls the maximum number of outstanding async requests that can be - handled by the Standalone as well as Ratis client. - </description> - </property> - - <property> - <name>ozone.scm.container.creation.lease.timeout</name> - <value>60s</value> - <tag>OZONE, SCM</tag> - <description> - Container creation timeout in milliseconds to be used by SCM. When - BEGIN_CREATE event happens the container is moved from ALLOCATED to - CREATING state, SCM will now wait for the configured amount of time - to get COMPLETE_CREATE event if it doesn't receive it will move the - container to DELETING. - </description> - </property> - - <property> - <name>ozone.key.preallocation.maxsize</name> - <value>134217728</value> - <tag>OZONE, KSM, PERFORMANCE</tag> - <description> - When a new key write request is sent to KSM, if a size is requested, at most - 128MB of size is allocated at request time. If client needs more space for the - write, separate block allocation requests will be made. - </description> - </property> - - <property> - <name>ozone.client.list.cache</name> - <value>1000</value> - <tag>OZONE, PERFORMANCE</tag> - <description> - Configuration property to configure the cache size of client list calls. - </description> - </property> - - <property> - <name>ozone.replication</name> - <value>3</value> - <tag>OZONE, CLIENT</tag> - <description> - Default replication value. The actual number of replications can be - specified when writing the key. The default is used if replication - is not specified. Supported values: 1 and 3. - </description> - </property> - - <property> - <name>ozone.replication.type</name> - <value>RATIS</value> - <tag>OZONE, CLIENT</tag> - <description> - Default replication type to be used while writing key into ozone. The - value can be specified when writing the key, default is used when - nothing is specified. Supported values: RATIS, STAND_ALONE and CHAINED. - </description> - </property> - <property> - <name>ozone.scm.container.close.threshold</name> - <value>0.9f</value> - <tag>OZONE, SCM</tag> - <description> - This determines the threshold to be used for closing a container. - When the container used percentage reaches this threshold, - the container will be closed. Value should be a positive, non-zero - percentage in float notation (X.Yf), with 1.0f meaning 100%. - </description> - </property> - <property> - <name>ozone.rest.client.http.connection.max</name> - <value>100</value> - <tag>OZONE, CLIENT</tag> - <description> - This defines the overall connection limit for the connection pool used in - RestClient. - </description> - </property> - <property> - <name>ozone.rest.client.http.connection.per-route.max</name> - <value>20</value> - <tag>OZONE, CLIENT</tag> - <description> - This defines the connection limit per one HTTP route/host. Total max - connection is limited by ozone.rest.client.http.connection.max property. - </description> - </property> - - <property> - <name>ozone.open.key.cleanup.service.interval.seconds</name> - <value>86400</value> - <tag>OZONE, KSM, PERFORMANCE</tag> - <description> - A background job periodically checks open key entries and delete the expired ones. This entry controls the - interval of this cleanup check. - </description> - </property> - - <property> - <name>ozone.open.key.expire.threshold</name> - <value>86400</value> - <tag>OZONE, KSM, PERFORMANCE</tag> - <description> - Controls how long an open key operation is considered active. Specifically, if a key - has been open longer than the value of this config entry, that open key is considered as - expired (e.g. due to client crash). Default to 24 hours. - </description> - </property> - - - <property> - <name>hdsl.rest.rest-csrf.enabled</name> - <value>false</value> - <description> - If true, then enables Object Store REST server protection against - cross-site request forgery (CSRF). - </description> - </property> - - <property> - <name>hdsl.rest.http-address</name> - <value>0.0.0.0:9880</value> - <description>The http address of Object Store REST server inside the - datanode.</description> - </property> - - - <property> - <name>hdsl.rest.netty.high.watermark</name> - <value>65535</value> - <description> - High watermark configuration to Netty for Object Store REST server. - </description> - </property> - - <property> - <name>hdsl.rest.netty.low.watermark</name> - <value>32768</value> - <description> - Low watermark configuration to Netty for Object Store REST server. - </description> - </property> - - -</configuration> \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java b/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java deleted file mode 100644 index cee66ca..0000000 --- a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java +++ /dev/null @@ -1,414 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone; - -import com.google.common.collect.Lists; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdsl.conf.OzoneConfiguration; -import org.apache.hadoop.hdfs.DFSUtil; -import org.apache.hadoop.hdfs.DFSUtilClient; -import org.apache.hadoop.utils.BatchOperation; -import org.apache.hadoop.utils.MetadataStore; -import org.apache.hadoop.test.GenericTestUtils; -import org.apache.hadoop.utils.MetadataKeyFilters.KeyPrefixFilter; -import org.apache.hadoop.utils.MetadataKeyFilters.MetadataKeyFilter; -import org.apache.hadoop.utils.MetadataStoreBuilder; -import org.junit.Rule; -import org.junit.Before; -import org.junit.After; -import org.junit.Test; -import org.junit.Assert; -import org.junit.rules.ExpectedException; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.io.File; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.ArrayList; -import java.util.UUID; -import java.util.Collection; -import java.util.Arrays; -import java.util.Iterator; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.junit.runners.Parameterized.*; - -/** - * Test class for ozone metadata store. - */ -@RunWith(Parameterized.class) -public class TestMetadataStore { - - private final String storeImpl; - - public TestMetadataStore(String metadataImpl) { - this.storeImpl = metadataImpl; - } - - @Parameters - public static Collection<Object[]> data() { - return Arrays.asList(new Object[][] { - {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB}, - {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB} - }); - } - - private MetadataStore store; - private File testDir; - private final static int MAX_GETRANGE_LENGTH = 100; - - @Rule - public ExpectedException expectedException = ExpectedException.none(); - - @Before - public void init() throws IOException { - testDir = GenericTestUtils.getTestDir(getClass().getSimpleName() - + "-" + storeImpl.toLowerCase()); - - Configuration conf = new OzoneConfiguration(); - conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl); - - store = MetadataStoreBuilder.newBuilder() - .setConf(conf) - .setCreateIfMissing(true) - .setDbFile(testDir) - .build(); - - // Add 20 entries. - // {a0 : a-value0} to {a9 : a-value9} - // {b0 : b-value0} to {b9 : b-value9} - for (int i=0; i<10; i++) { - store.put(getBytes("a" + i), getBytes("a-value" + i)); - store.put(getBytes("b" + i), getBytes("b-value" + i)); - } - } - - @After - public void cleanup() throws IOException { - store.close(); - store.destroy(); - FileUtils.deleteDirectory(testDir); - } - - private byte[] getBytes(String str) { - return str == null ? null : - DFSUtilClient.string2Bytes(str); - } - - private String getString(byte[] bytes) { - return bytes == null ? null : - DFSUtilClient.bytes2String(bytes); - } - - @Test - public void testGetDelete() throws IOException { - for (int i=0; i<10; i++) { - byte[] va = store.get(getBytes("a" + i)); - Assert.assertEquals("a-value" + i, getString(va)); - - byte[] vb = store.get(getBytes("b" + i)); - Assert.assertEquals("b-value" + i, getString(vb)); - } - - String keyToDel = "del-" + UUID.randomUUID().toString(); - store.put(getBytes(keyToDel), getBytes(keyToDel)); - Assert.assertEquals(keyToDel, getString(store.get(getBytes(keyToDel)))); - store.delete(getBytes(keyToDel)); - Assert.assertEquals(null, store.get(getBytes(keyToDel))); - } - - @Test - public void testPeekFrom() throws IOException { - // Test peek from an element that has prev as well as next - testPeek("a3", "a2", "a4"); - - // Test peek from an element that only has prev - testPeek("b9", "b8", null); - - // Test peek from an element that only has next - testPeek("a0", null, "a1"); - } - - private String getExpectedValue(String key) { - if (key == null) { - return null; - } - char[] arr = key.toCharArray(); - return new StringBuffer().append(arr[0]).append("-value") - .append(arr[arr.length - 1]).toString(); - } - - private void testPeek(String peekKey, String prevKey, String nextKey) - throws IOException { - // Look for current - String k = null; - String v = null; - ImmutablePair<byte[], byte[]> current = - store.peekAround(0, getBytes(peekKey)); - if (current != null) { - k = getString(current.getKey()); - v = getString(current.getValue()); - } - Assert.assertEquals(peekKey, k); - Assert.assertEquals(v, getExpectedValue(peekKey)); - - // Look for prev - k = null; - v = null; - ImmutablePair<byte[], byte[]> prev = - store.peekAround(-1, getBytes(peekKey)); - if (prev != null) { - k = getString(prev.getKey()); - v = getString(prev.getValue()); - } - Assert.assertEquals(prevKey, k); - Assert.assertEquals(v, getExpectedValue(prevKey)); - - // Look for next - k = null; - v = null; - ImmutablePair<byte[], byte[]> next = - store.peekAround(1, getBytes(peekKey)); - if (next != null) { - k = getString(next.getKey()); - v = getString(next.getValue()); - } - Assert.assertEquals(nextKey, k); - Assert.assertEquals(v, getExpectedValue(nextKey)); - } - - @Test - public void testIterateKeys() throws IOException { - // iterate keys from b0 - ArrayList<String> result = Lists.newArrayList(); - store.iterate(getBytes("b0"), (k, v) -> { - // b-value{i} - String value = getString(v); - char num = value.charAt(value.length() - 1); - // each value adds 1 - int i = Character.getNumericValue(num) + 1; - value = value.substring(0, value.length() - 1) + i; - result.add(value); - return true; - }); - - Assert.assertFalse(result.isEmpty()); - for (int i=0; i<result.size(); i++) { - Assert.assertEquals("b-value" + (i+1), result.get(i)); - } - - // iterate from a non exist key - result.clear(); - store.iterate(getBytes("xyz"), (k, v) -> { - result.add(getString(v)); - return true; - }); - Assert.assertTrue(result.isEmpty()); - - // iterate from the beginning - result.clear(); - store.iterate(null, (k, v) -> { - result.add(getString(v)); - return true; - }); - Assert.assertEquals(20, result.size()); - } - - @Test - public void testGetRangeKVs() throws IOException { - List<Map.Entry<byte[], byte[]>> result = null; - - // Set empty startKey will return values from beginning. - result = store.getRangeKVs(null, 5); - Assert.assertEquals(5, result.size()); - Assert.assertEquals("a-value2", getString(result.get(2).getValue())); - - // Empty list if startKey doesn't exist. - result = store.getRangeKVs(getBytes("a12"), 5); - Assert.assertEquals(0, result.size()); - - // Returns max available entries after a valid startKey. - result = store.getRangeKVs(getBytes("b0"), MAX_GETRANGE_LENGTH); - Assert.assertEquals(10, result.size()); - Assert.assertEquals("b0", getString(result.get(0).getKey())); - Assert.assertEquals("b-value0", getString(result.get(0).getValue())); - result = store.getRangeKVs(getBytes("b0"), 5); - Assert.assertEquals(5, result.size()); - - // Both startKey and count are honored. - result = store.getRangeKVs(getBytes("a9"), 2); - Assert.assertEquals(2, result.size()); - Assert.assertEquals("a9", getString(result.get(0).getKey())); - Assert.assertEquals("a-value9", getString(result.get(0).getValue())); - Assert.assertEquals("b0", getString(result.get(1).getKey())); - Assert.assertEquals("b-value0", getString(result.get(1).getValue())); - - // Filter keys by prefix. - // It should returns all "b*" entries. - MetadataKeyFilter filter1 = new KeyPrefixFilter("b"); - result = store.getRangeKVs(null, 100, filter1); - Assert.assertEquals(10, result.size()); - Assert.assertTrue(result.stream().allMatch(entry -> - new String(entry.getKey()).startsWith("b") - )); - Assert.assertEquals(20, filter1.getKeysScannedNum()); - Assert.assertEquals(10, filter1.getKeysHintedNum()); - result = store.getRangeKVs(null, 3, filter1); - Assert.assertEquals(3, result.size()); - result = store.getRangeKVs(getBytes("b3"), 1, filter1); - Assert.assertEquals("b-value3", getString(result.get(0).getValue())); - - // Define a customized filter that filters keys by suffix. - // Returns all "*2" entries. - MetadataKeyFilter filter2 = (preKey, currentKey, nextKey) - -> getString(currentKey).endsWith("2"); - result = store.getRangeKVs(null, MAX_GETRANGE_LENGTH, filter2); - Assert.assertEquals(2, result.size()); - Assert.assertEquals("a2", getString(result.get(0).getKey())); - Assert.assertEquals("b2", getString(result.get(1).getKey())); - result = store.getRangeKVs(null, 1, filter2); - Assert.assertEquals(1, result.size()); - Assert.assertEquals("a2", getString(result.get(0).getKey())); - - // Apply multiple filters. - result = store.getRangeKVs(null, MAX_GETRANGE_LENGTH, filter1, filter2); - Assert.assertEquals(1, result.size()); - Assert.assertEquals("b2", getString(result.get(0).getKey())); - Assert.assertEquals("b-value2", getString(result.get(0).getValue())); - - // If filter is null, no effect. - result = store.getRangeKVs(null, 1, null); - Assert.assertEquals(1, result.size()); - Assert.assertEquals("a0", getString(result.get(0).getKey())); - } - - @Test - public void testGetSequentialRangeKVs() throws IOException { - MetadataKeyFilter suffixFilter = (preKey, currentKey, nextKey) - -> DFSUtil.bytes2String(currentKey).endsWith("2"); - // Suppose to return a2 and b2 - List<Map.Entry<byte[], byte[]>> result = - store.getRangeKVs(null, MAX_GETRANGE_LENGTH, suffixFilter); - Assert.assertEquals(2, result.size()); - Assert.assertEquals("a2", DFSUtil.bytes2String(result.get(0).getKey())); - Assert.assertEquals("b2", DFSUtil.bytes2String(result.get(1).getKey())); - - // Suppose to return just a2, because when it iterates to a3, - // the filter no long matches and it should stop from there. - result = store.getSequentialRangeKVs(null, - MAX_GETRANGE_LENGTH, suffixFilter); - Assert.assertEquals(1, result.size()); - Assert.assertEquals("a2", DFSUtil.bytes2String(result.get(0).getKey())); - } - - @Test - public void testGetRangeLength() throws IOException { - List<Map.Entry<byte[], byte[]>> result = null; - - result = store.getRangeKVs(null, 0); - Assert.assertEquals(0, result.size()); - - result = store.getRangeKVs(null, 1); - Assert.assertEquals(1, result.size()); - - // Count less than zero is invalid. - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("Invalid count given"); - store.getRangeKVs(null, -1); - } - - @Test - public void testInvalidStartKey() throws IOException { - // If startKey is invalid, the returned list should be empty. - List<Map.Entry<byte[], byte[]>> kvs = - store.getRangeKVs(getBytes("unknownKey"), MAX_GETRANGE_LENGTH); - Assert.assertEquals(kvs.size(), 0); - } - - @Test - public void testDestroyDB() throws IOException { - // create a new DB to test db destroy - Configuration conf = new OzoneConfiguration(); - conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl); - - File dbDir = GenericTestUtils.getTestDir(getClass().getSimpleName() - + "-" + storeImpl.toLowerCase() + "-toDestroy"); - MetadataStore dbStore = MetadataStoreBuilder.newBuilder() - .setConf(conf) - .setCreateIfMissing(true) - .setDbFile(dbDir) - .build(); - - dbStore.put(getBytes("key1"), getBytes("value1")); - dbStore.put(getBytes("key2"), getBytes("value2")); - - Assert.assertFalse(dbStore.isEmpty()); - Assert.assertTrue(dbDir.exists()); - Assert.assertTrue(dbDir.listFiles().length > 0); - - dbStore.destroy(); - - Assert.assertFalse(dbDir.exists()); - } - - @Test - public void testBatchWrite() throws IOException { - Configuration conf = new OzoneConfiguration(); - conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl); - - File dbDir = GenericTestUtils.getTestDir(getClass().getSimpleName() - + "-" + storeImpl.toLowerCase() + "-batchWrite"); - MetadataStore dbStore = MetadataStoreBuilder.newBuilder() - .setConf(conf) - .setCreateIfMissing(true) - .setDbFile(dbDir) - .build(); - - List<String> expectedResult = Lists.newArrayList(); - for (int i = 0; i<10; i++) { - dbStore.put(getBytes("batch-" + i), getBytes("batch-value-" + i)); - expectedResult.add("batch-" + i); - } - - BatchOperation batch = new BatchOperation(); - batch.delete(getBytes("batch-2")); - batch.delete(getBytes("batch-3")); - batch.delete(getBytes("batch-4")); - batch.put(getBytes("batch-new-2"), getBytes("batch-new-value-2")); - - expectedResult.remove("batch-2"); - expectedResult.remove("batch-3"); - expectedResult.remove("batch-4"); - expectedResult.add("batch-new-2"); - - dbStore.writeBatch(batch); - - Iterator<String> it = expectedResult.iterator(); - AtomicInteger count = new AtomicInteger(0); - dbStore.iterate(null, (key, value) -> { - count.incrementAndGet(); - return it.hasNext() && it.next().equals(getString(key)); - }); - - Assert.assertEquals(8, count.get()); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java b/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java deleted file mode 100644 index 03c45c5..0000000 --- a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java +++ /dev/null @@ -1,141 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone; - -import org.junit.Test; - -import java.util.HashMap; -import java.util.Set; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -/** - * This class is to test acl stoarge and retreival in ozone store. - */ -public class TestOzoneAcls { - - @Test - public void testAclParse() { - HashMap<String, Boolean> testMatrix; - testMatrix = new HashMap<>(); - - testMatrix.put("user:bilbo:r", Boolean.TRUE); - testMatrix.put("user:bilbo:w", Boolean.TRUE); - testMatrix.put("user:bilbo:rw", Boolean.TRUE); - testMatrix.put("user:bilbo:wr", Boolean.TRUE); - testMatrix.put(" user:bilbo:wr ", Boolean.TRUE); - - - // ACLs makes no judgement on the quality of - // user names. it is for the userAuth interface - // to determine if a user name is really a name - testMatrix.put(" user:*:rw", Boolean.TRUE); - testMatrix.put(" user:~!:rw", Boolean.TRUE); - - - testMatrix.put("", Boolean.FALSE); - testMatrix.put(null, Boolean.FALSE); - testMatrix.put(" user:bilbo:", Boolean.FALSE); - testMatrix.put(" user:bilbo:rx", Boolean.FALSE); - testMatrix.put(" user:bilbo:mk", Boolean.FALSE); - testMatrix.put(" user::rw", Boolean.FALSE); - testMatrix.put("user11:bilbo:rw", Boolean.FALSE); - testMatrix.put(" user:::rw", Boolean.FALSE); - - testMatrix.put(" group:hobbit:r", Boolean.TRUE); - testMatrix.put(" group:hobbit:w", Boolean.TRUE); - testMatrix.put(" group:hobbit:rw", Boolean.TRUE); - testMatrix.put(" group:hobbit:wr", Boolean.TRUE); - testMatrix.put(" group:*:rw", Boolean.TRUE); - testMatrix.put(" group:~!:rw", Boolean.TRUE); - - testMatrix.put(" group:hobbit:", Boolean.FALSE); - testMatrix.put(" group:hobbit:rx", Boolean.FALSE); - testMatrix.put(" group:hobbit:mk", Boolean.FALSE); - testMatrix.put(" group::", Boolean.FALSE); - testMatrix.put(" group::rw", Boolean.FALSE); - testMatrix.put(" group22:hobbit:", Boolean.FALSE); - testMatrix.put(" group:::rw", Boolean.FALSE); - - testMatrix.put("JUNK group:hobbit:r", Boolean.FALSE); - testMatrix.put("JUNK group:hobbit:w", Boolean.FALSE); - testMatrix.put("JUNK group:hobbit:rw", Boolean.FALSE); - testMatrix.put("JUNK group:hobbit:wr", Boolean.FALSE); - testMatrix.put("JUNK group:*:rw", Boolean.FALSE); - testMatrix.put("JUNK group:~!:rw", Boolean.FALSE); - - testMatrix.put(" world::r", Boolean.TRUE); - testMatrix.put(" world::w", Boolean.TRUE); - testMatrix.put(" world::rw", Boolean.TRUE); - testMatrix.put(" world::wr", Boolean.TRUE); - - testMatrix.put(" world:bilbo:w", Boolean.FALSE); - testMatrix.put(" world:bilbo:rw", Boolean.FALSE); - - Set<String> keys = testMatrix.keySet(); - for (String key : keys) { - if (testMatrix.get(key)) { - OzoneAcl.parseAcl(key); - } else { - try { - OzoneAcl.parseAcl(key); - // should never get here since parseAcl will throw - fail("An exception was expected but did not happen."); - } catch (IllegalArgumentException e) { - // nothing to do - } - } - } - } - - @Test - public void testAclValues() { - OzoneAcl acl = OzoneAcl.parseAcl("user:bilbo:rw"); - assertEquals(acl.getName(), "bilbo"); - assertEquals(OzoneAcl.OzoneACLRights.READ_WRITE, acl.getRights()); - assertEquals(OzoneAcl.OzoneACLType.USER, acl.getType()); - - acl = OzoneAcl.parseAcl("user:bilbo:wr"); - assertEquals("bilbo", acl.getName()); - assertEquals(OzoneAcl.OzoneACLRights.READ_WRITE, acl.getRights()); - assertEquals(OzoneAcl.OzoneACLType.USER, acl.getType()); - - acl = OzoneAcl.parseAcl("user:bilbo:r"); - assertEquals("bilbo", acl.getName()); - assertEquals(OzoneAcl.OzoneACLRights.READ, acl.getRights()); - assertEquals(OzoneAcl.OzoneACLType.USER, acl.getType()); - - acl = OzoneAcl.parseAcl("user:bilbo:w"); - assertEquals("bilbo", acl.getName()); - assertEquals(OzoneAcl.OzoneACLRights.WRITE, acl.getRights()); - assertEquals(OzoneAcl.OzoneACLType.USER, acl.getType()); - - acl = OzoneAcl.parseAcl("group:hobbit:wr"); - assertEquals(acl.getName(), "hobbit"); - assertEquals(OzoneAcl.OzoneACLRights.READ_WRITE, acl.getRights()); - assertEquals(OzoneAcl.OzoneACLType.GROUP, acl.getType()); - - acl = OzoneAcl.parseAcl("world::wr"); - assertEquals(acl.getName(), ""); - assertEquals(OzoneAcl.OzoneACLRights.READ_WRITE, acl.getRights()); - assertEquals(OzoneAcl.OzoneACLType.WORLD, acl.getType()); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java b/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java deleted file mode 100644 index d4c626e..0000000 --- a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.ozone.common; - -import org.apache.commons.collections.SetUtils; -import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException; -import org.apache.hadoop.ozone.common.statemachine.StateMachine; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import java.util.HashSet; -import java.util.Set; - -import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.INIT; -import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.CREATING; -import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.OPERATIONAL; -import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.CLOSED; -import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.CLEANUP; -import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.FINAL; - -/** - * This class is to test ozone common state machine. - */ -public class TestStateMachine { - - /** - * STATES used by the test state machine. - */ - public enum STATES {INIT, CREATING, OPERATIONAL, CLOSED, CLEANUP, FINAL}; - - /** - * EVENTS used by the test state machine. - */ - public enum EVENTS {ALLOCATE, CREATE, UPDATE, CLOSE, DELETE, TIMEOUT}; - - @Rule - public ExpectedException exception = ExpectedException.none(); - - @Test - public void testStateMachineStates() throws InvalidStateTransitionException { - Set<STATES> finals = new HashSet<>(); - finals.add(FINAL); - - StateMachine<STATES, EVENTS> stateMachine = - new StateMachine<>(INIT, finals); - - stateMachine.addTransition(INIT, CREATING, EVENTS.ALLOCATE); - stateMachine.addTransition(CREATING, OPERATIONAL, EVENTS.CREATE); - stateMachine.addTransition(OPERATIONAL, OPERATIONAL, EVENTS.UPDATE); - stateMachine.addTransition(OPERATIONAL, CLEANUP, EVENTS.DELETE); - stateMachine.addTransition(OPERATIONAL, CLOSED, EVENTS.CLOSE); - stateMachine.addTransition(CREATING, CLEANUP, EVENTS.TIMEOUT); - - // Initial and Final states - Assert.assertEquals("Initial State", INIT, stateMachine.getInitialState()); - Assert.assertTrue("Final States", SetUtils.isEqualSet(finals, - stateMachine.getFinalStates())); - - // Valid state transitions - Assert.assertEquals("STATE should be OPERATIONAL after being created", - OPERATIONAL, stateMachine.getNextState(CREATING, EVENTS.CREATE)); - Assert.assertEquals("STATE should be OPERATIONAL after being updated", - OPERATIONAL, stateMachine.getNextState(OPERATIONAL, EVENTS.UPDATE)); - Assert.assertEquals("STATE should be CLEANUP after being deleted", - CLEANUP, stateMachine.getNextState(OPERATIONAL, EVENTS.DELETE)); - Assert.assertEquals("STATE should be CLEANUP after being timeout", - CLEANUP, stateMachine.getNextState(CREATING, EVENTS.TIMEOUT)); - Assert.assertEquals("STATE should be CLOSED after being closed", - CLOSED, stateMachine.getNextState(OPERATIONAL, EVENTS.CLOSE)); - - // Negative cases: invalid transition - expectException(); - stateMachine.getNextState(OPERATIONAL, EVENTS.CREATE); - - expectException(); - stateMachine.getNextState(CREATING, EVENTS.CLOSE); - } - - /** - * We expect an InvalidStateTransitionException. - */ - private void expectException() { - exception.expect(InvalidStateTransitionException.class); - exception.expectMessage("Invalid event"); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/lease/TestLeaseManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/lease/TestLeaseManager.java b/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/lease/TestLeaseManager.java deleted file mode 100644 index 517c1a7..0000000 --- a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/lease/TestLeaseManager.java +++ /dev/null @@ -1,374 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -/** - * A generic lease management API which can be used if a service - * needs any kind of lease management. - */ - -package org.apache.hadoop.ozone.lease; - -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import java.util.HashMap; -import java.util.Map; - -/** - * Test class to check functionality and consistency of LeaseManager. - */ -public class TestLeaseManager { - - @Rule - public ExpectedException exception = ExpectedException.none(); - - /** - * Dummy resource on which leases can be acquired. - */ - private final class DummyResource { - - private final String name; - - private DummyResource(String name) { - this.name = name; - } - - @Override - public int hashCode() { - return name.hashCode(); - } - - @Override - public boolean equals(Object obj) { - if(obj instanceof DummyResource) { - return name.equals(((DummyResource) obj).name); - } - return false; - } - } - - @Test - public void testLeaseAcquireAndRelease() throws LeaseException { - //It is assumed that the test case execution won't take more than 5 seconds, - //if it takes more time increase the defaultTimeout value of LeaseManager. - LeaseManager<DummyResource> manager = new LeaseManager<>(5000); - manager.start(); - DummyResource resourceOne = new DummyResource("one"); - DummyResource resourceTwo = new DummyResource("two"); - DummyResource resourceThree = new DummyResource("three"); - Lease<DummyResource> leaseOne = manager.acquire(resourceOne); - Lease<DummyResource> leaseTwo = manager.acquire(resourceTwo); - Lease<DummyResource> leaseThree = manager.acquire(resourceThree); - Assert.assertEquals(leaseOne, manager.get(resourceOne)); - Assert.assertEquals(leaseTwo, manager.get(resourceTwo)); - Assert.assertEquals(leaseThree, manager.get(resourceThree)); - Assert.assertFalse(leaseOne.hasExpired()); - Assert.assertFalse(leaseTwo.hasExpired()); - Assert.assertFalse(leaseThree.hasExpired()); - //The below releases should not throw LeaseNotFoundException. - manager.release(resourceOne); - manager.release(resourceTwo); - manager.release(resourceThree); - Assert.assertTrue(leaseOne.hasExpired()); - Assert.assertTrue(leaseTwo.hasExpired()); - Assert.assertTrue(leaseThree.hasExpired()); - manager.shutdown(); - } - - @Test - public void testLeaseAlreadyExist() throws LeaseException { - LeaseManager<DummyResource> manager = new LeaseManager<>(5000); - manager.start(); - DummyResource resourceOne = new DummyResource("one"); - DummyResource resourceTwo = new DummyResource("two"); - Lease<DummyResource> leaseOne = manager.acquire(resourceOne); - Lease<DummyResource> leaseTwo = manager.acquire(resourceTwo); - Assert.assertEquals(leaseOne, manager.get(resourceOne)); - Assert.assertEquals(leaseTwo, manager.get(resourceTwo)); - - exception.expect(LeaseAlreadyExistException.class); - exception.expectMessage("Resource: " + resourceOne); - manager.acquire(resourceOne); - - manager.release(resourceOne); - manager.release(resourceTwo); - manager.shutdown(); - } - - @Test - public void testLeaseNotFound() throws LeaseException, InterruptedException { - LeaseManager<DummyResource> manager = new LeaseManager<>(5000); - manager.start(); - DummyResource resourceOne = new DummyResource("one"); - DummyResource resourceTwo = new DummyResource("two"); - DummyResource resourceThree = new DummyResource("three"); - - //Case 1: lease was never acquired. - exception.expect(LeaseNotFoundException.class); - exception.expectMessage("Resource: " + resourceOne); - manager.get(resourceOne); - - //Case 2: lease is acquired and released. - Lease<DummyResource> leaseTwo = manager.acquire(resourceTwo); - Assert.assertEquals(leaseTwo, manager.get(resourceTwo)); - Assert.assertFalse(leaseTwo.hasExpired()); - manager.release(resourceTwo); - Assert.assertTrue(leaseTwo.hasExpired()); - exception.expect(LeaseNotFoundException.class); - exception.expectMessage("Resource: " + resourceTwo); - manager.get(resourceTwo); - - //Case 3: lease acquired and timed out. - Lease<DummyResource> leaseThree = manager.acquire(resourceThree); - Assert.assertEquals(leaseThree, manager.get(resourceThree)); - Assert.assertFalse(leaseThree.hasExpired()); - long sleepTime = leaseThree.getRemainingTime() + 1000; - try { - Thread.sleep(sleepTime); - } catch (InterruptedException ex) { - //even in case of interrupt we have to wait till lease times out. - Thread.sleep(sleepTime); - } - Assert.assertTrue(leaseThree.hasExpired()); - exception.expect(LeaseNotFoundException.class); - exception.expectMessage("Resource: " + resourceThree); - manager.get(resourceThree); - manager.shutdown(); - } - - @Test - public void testCustomLeaseTimeout() throws LeaseException { - LeaseManager<DummyResource> manager = new LeaseManager<>(5000); - manager.start(); - DummyResource resourceOne = new DummyResource("one"); - DummyResource resourceTwo = new DummyResource("two"); - DummyResource resourceThree = new DummyResource("three"); - Lease<DummyResource> leaseOne = manager.acquire(resourceOne); - Lease<DummyResource> leaseTwo = manager.acquire(resourceTwo, 10000); - Lease<DummyResource> leaseThree = manager.acquire(resourceThree, 50000); - Assert.assertEquals(leaseOne, manager.get(resourceOne)); - Assert.assertEquals(leaseTwo, manager.get(resourceTwo)); - Assert.assertEquals(leaseThree, manager.get(resourceThree)); - Assert.assertFalse(leaseOne.hasExpired()); - Assert.assertFalse(leaseTwo.hasExpired()); - Assert.assertFalse(leaseThree.hasExpired()); - Assert.assertEquals(5000, leaseOne.getLeaseLifeTime()); - Assert.assertEquals(10000, leaseTwo.getLeaseLifeTime()); - Assert.assertEquals(50000, leaseThree.getLeaseLifeTime()); - // Releasing of leases is done in shutdown, so don't have to worry about - // lease release - manager.shutdown(); - } - - @Test - public void testLeaseCallback() throws LeaseException, InterruptedException { - Map<DummyResource, String> leaseStatus = new HashMap<>(); - LeaseManager<DummyResource> manager = new LeaseManager<>(5000); - manager.start(); - DummyResource resourceOne = new DummyResource("one"); - Lease<DummyResource> leaseOne = manager.acquire(resourceOne); - leaseStatus.put(resourceOne, "lease in use"); - leaseOne.registerCallBack(() -> { - leaseStatus.put(resourceOne, "lease expired"); - return null; - }); - // wait for lease to expire - long sleepTime = leaseOne.getRemainingTime() + 1000; - try { - Thread.sleep(sleepTime); - } catch (InterruptedException ex) { - //even in case of interrupt we have to wait till lease times out. - Thread.sleep(sleepTime); - } - Assert.assertTrue(leaseOne.hasExpired()); - exception.expect(LeaseNotFoundException.class); - exception.expectMessage("Resource: " + resourceOne); - manager.get(resourceOne); - // check if callback has been executed - Assert.assertEquals("lease expired", leaseStatus.get(resourceOne)); - } - - @Test - public void testCallbackExecutionInCaseOfLeaseRelease() - throws LeaseException, InterruptedException { - // Callbacks should not be executed in case of lease release - Map<DummyResource, String> leaseStatus = new HashMap<>(); - LeaseManager<DummyResource> manager = new LeaseManager<>(5000); - manager.start(); - DummyResource resourceOne = new DummyResource("one"); - Lease<DummyResource> leaseOne = manager.acquire(resourceOne); - leaseStatus.put(resourceOne, "lease in use"); - leaseOne.registerCallBack(() -> { - leaseStatus.put(resourceOne, "lease expired"); - return null; - }); - leaseStatus.put(resourceOne, "lease released"); - manager.release(resourceOne); - Assert.assertTrue(leaseOne.hasExpired()); - exception.expect(LeaseNotFoundException.class); - exception.expectMessage("Resource: " + resourceOne); - manager.get(resourceOne); - Assert.assertEquals("lease released", leaseStatus.get(resourceOne)); - } - - @Test - public void testLeaseCallbackWithMultipleLeases() - throws LeaseException, InterruptedException { - Map<DummyResource, String> leaseStatus = new HashMap<>(); - LeaseManager<DummyResource> manager = new LeaseManager<>(5000); - manager.start(); - DummyResource resourceOne = new DummyResource("one"); - DummyResource resourceTwo = new DummyResource("two"); - DummyResource resourceThree = new DummyResource("three"); - DummyResource resourceFour = new DummyResource("four"); - DummyResource resourceFive = new DummyResource("five"); - Lease<DummyResource> leaseOne = manager.acquire(resourceOne); - Lease<DummyResource> leaseTwo = manager.acquire(resourceTwo); - Lease<DummyResource> leaseThree = manager.acquire(resourceThree); - Lease<DummyResource> leaseFour = manager.acquire(resourceFour); - Lease<DummyResource> leaseFive = manager.acquire(resourceFive); - leaseStatus.put(resourceOne, "lease in use"); - leaseStatus.put(resourceTwo, "lease in use"); - leaseStatus.put(resourceThree, "lease in use"); - leaseStatus.put(resourceFour, "lease in use"); - leaseStatus.put(resourceFive, "lease in use"); - leaseOne.registerCallBack(() -> { - leaseStatus.put(resourceOne, "lease expired"); - return null; - }); - leaseTwo.registerCallBack(() -> { - leaseStatus.put(resourceTwo, "lease expired"); - return null; - }); - leaseThree.registerCallBack(() -> { - leaseStatus.put(resourceThree, "lease expired"); - return null; - }); - leaseFour.registerCallBack(() -> { - leaseStatus.put(resourceFour, "lease expired"); - return null; - }); - leaseFive.registerCallBack(() -> { - leaseStatus.put(resourceFive, "lease expired"); - return null; - }); - - // release lease one, two and three - leaseStatus.put(resourceOne, "lease released"); - manager.release(resourceOne); - leaseStatus.put(resourceTwo, "lease released"); - manager.release(resourceTwo); - leaseStatus.put(resourceThree, "lease released"); - manager.release(resourceThree); - - // wait for other leases to expire - long sleepTime = leaseFive.getRemainingTime() + 1000; - - try { - Thread.sleep(sleepTime); - } catch (InterruptedException ex) { - //even in case of interrupt we have to wait till lease times out. - Thread.sleep(sleepTime); - } - Assert.assertTrue(leaseOne.hasExpired()); - Assert.assertTrue(leaseTwo.hasExpired()); - Assert.assertTrue(leaseThree.hasExpired()); - Assert.assertTrue(leaseFour.hasExpired()); - Assert.assertTrue(leaseFive.hasExpired()); - - Assert.assertEquals("lease released", leaseStatus.get(resourceOne)); - Assert.assertEquals("lease released", leaseStatus.get(resourceTwo)); - Assert.assertEquals("lease released", leaseStatus.get(resourceThree)); - Assert.assertEquals("lease expired", leaseStatus.get(resourceFour)); - Assert.assertEquals("lease expired", leaseStatus.get(resourceFive)); - manager.shutdown(); - } - - @Test - public void testReuseReleasedLease() throws LeaseException { - LeaseManager<DummyResource> manager = new LeaseManager<>(5000); - manager.start(); - DummyResource resourceOne = new DummyResource("one"); - Lease<DummyResource> leaseOne = manager.acquire(resourceOne); - Assert.assertEquals(leaseOne, manager.get(resourceOne)); - Assert.assertFalse(leaseOne.hasExpired()); - - manager.release(resourceOne); - Assert.assertTrue(leaseOne.hasExpired()); - - Lease<DummyResource> sameResourceLease = manager.acquire(resourceOne); - Assert.assertEquals(sameResourceLease, manager.get(resourceOne)); - Assert.assertFalse(sameResourceLease.hasExpired()); - - manager.release(resourceOne); - Assert.assertTrue(sameResourceLease.hasExpired()); - manager.shutdown(); - } - - @Test - public void testReuseTimedOutLease() - throws LeaseException, InterruptedException { - LeaseManager<DummyResource> manager = new LeaseManager<>(5000); - manager.start(); - DummyResource resourceOne = new DummyResource("one"); - Lease<DummyResource> leaseOne = manager.acquire(resourceOne); - Assert.assertEquals(leaseOne, manager.get(resourceOne)); - Assert.assertFalse(leaseOne.hasExpired()); - - // wait for lease to expire - long sleepTime = leaseOne.getRemainingTime() + 1000; - try { - Thread.sleep(sleepTime); - } catch (InterruptedException ex) { - //even in case of interrupt we have to wait till lease times out. - Thread.sleep(sleepTime); - } - Assert.assertTrue(leaseOne.hasExpired()); - - Lease<DummyResource> sameResourceLease = manager.acquire(resourceOne); - Assert.assertEquals(sameResourceLease, manager.get(resourceOne)); - Assert.assertFalse(sameResourceLease.hasExpired()); - - manager.release(resourceOne); - Assert.assertTrue(sameResourceLease.hasExpired()); - manager.shutdown(); - } - - @Test - public void testRenewLease() throws LeaseException, InterruptedException { - LeaseManager<DummyResource> manager = new LeaseManager<>(5000); - manager.start(); - DummyResource resourceOne = new DummyResource("one"); - Lease<DummyResource> leaseOne = manager.acquire(resourceOne); - Assert.assertEquals(leaseOne, manager.get(resourceOne)); - Assert.assertFalse(leaseOne.hasExpired()); - - // add 5 more seconds to the lease - leaseOne.renew(5000); - - Thread.sleep(5000); - - // lease should still be active - Assert.assertEquals(leaseOne, manager.get(resourceOne)); - Assert.assertFalse(leaseOne.hasExpired()); - manager.release(resourceOne); - manager.shutdown(); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/lease/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/lease/package-info.java b/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/lease/package-info.java deleted file mode 100644 index 1071309..0000000 --- a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/lease/package-info.java +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.ozone.lease; -/* - This package contains lease management unit test classes. - */ \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/package-info.java b/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/package-info.java deleted file mode 100644 index 0030d2e..0000000 --- a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/ozone/package-info.java +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone; -/** - * Ozone related test helper classes and tests of common utils. - */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/test/java/org/apache/hadoop/scm/TestArchive.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/scm/TestArchive.java b/hadoop-hdsl/common/src/test/java/org/apache/hadoop/scm/TestArchive.java deleted file mode 100644 index 86adddb..0000000 --- a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/scm/TestArchive.java +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.scm; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.RandomUtils; -import org.apache.hadoop.fs.FileUtil; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Iterator; -import java.util.Random; -import java.util.zip.Adler32; -import java.util.zip.Checksum; - -/** - * Test archive creation and unpacking. - */ -public class TestArchive { - private static final int DIR_COUNT = 10; - private static final int SUB_DIR_COUNT = 3; - private static final int FILE_COUNT = 10; - private long checksumWrite = 0L; - private long checksumRead = 0L; - private long tmp = 0L; - private Checksum crc = new Adler32(); - - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - @Rule - public TemporaryFolder outputFolder = new TemporaryFolder(); - - - @Before - public void setUp() throws Exception { - Random r = new Random(); - final int megaByte = 1024 * 1024; - - for (int x = 0; x < DIR_COUNT; x++) { - File subdir = folder.newFolder(String.format("dir%d", x)); - for (int y = 0; y < SUB_DIR_COUNT; y++) { - File targetDir = new File(subdir.getPath().concat(File.separator) - .concat(String.format("subdir%d%d", x, y))); - if(!targetDir.mkdirs()) { - throw new IOException("Failed to create subdirectory. " + - targetDir.toString()); - } - for (int z = 0; z < FILE_COUNT; z++) { - Path temp = Paths.get(targetDir.getPath().concat(File.separator) - .concat(String.format("File%d.txt", z))); - byte[] buf = RandomUtils.nextBytes(r.nextInt(megaByte)); - Files.write(temp, buf); - crc.reset(); - crc.update(buf, 0, buf.length); - tmp = crc.getValue(); - checksumWrite +=tmp; - } - } - } - } - - @Test - public void testArchive() throws Exception { - File archiveFile = new File(outputFolder.getRoot() + File.separator - + "test.container.zip"); - long zipCheckSum = FileUtil.zip(folder.getRoot(), archiveFile); - Assert.assertTrue(zipCheckSum > 0); - File decomp = new File(outputFolder.getRoot() + File.separator + - "decompress"); - if (!decomp.exists() && !decomp.mkdirs()) { - throw new IOException("Unable to create the destination directory. " + - decomp.getPath()); - } - - FileUtil.unZip(archiveFile, decomp); - String[] patterns = {"txt"}; - Iterator<File> iter = FileUtils.iterateFiles(decomp, patterns, true); - int count = 0; - while (iter.hasNext()) { - count++; - byte[] buf = Files.readAllBytes(iter.next().toPath()); - crc.reset(); - crc.update(buf, 0, buf.length); - tmp = crc.getValue(); - checksumRead += tmp; - } - Assert.assertEquals(DIR_COUNT * SUB_DIR_COUNT * FILE_COUNT, count); - Assert.assertEquals(checksumWrite, checksumRead); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/test/java/org/apache/hadoop/scm/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/scm/package-info.java b/hadoop-hdsl/common/src/test/java/org/apache/hadoop/scm/package-info.java deleted file mode 100644 index 9c480d6..0000000 --- a/hadoop-hdsl/common/src/test/java/org/apache/hadoop/scm/package-info.java +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.scm; -/** - Test cases for SCM client classes. - */ \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
