Repository: hadoop Updated Branches: refs/heads/HDFS-7240 49c92ea73 -> 350220bfb
HDFS-11631. Block Storage : allow cblock server to be started from hdfs commandline. Contributed by Chen Liang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/350220bf Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/350220bf Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/350220bf Branch: refs/heads/HDFS-7240 Commit: 350220bfb3c555ddf8ab4f0f5a444e37a6d3d9db Parents: 49c92ea Author: Anu Engineer <[email protected]> Authored: Mon Apr 17 15:54:39 2017 -0700 Committer: Anu Engineer <[email protected]> Committed: Mon Apr 17 15:54:39 2017 -0700 ---------------------------------------------------------------------- .../hadoop-hdfs/src/main/bin/hdfs | 11 +++++ .../apache/hadoop/cblock/CBlockConfigKeys.java | 15 ++++++- .../org/apache/hadoop/cblock/CBlockManager.java | 47 ++++++++++++++++++++ 3 files changed, 71 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/350220bf/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs index 1ae5de4..afb27c3 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs @@ -33,6 +33,8 @@ function hadoop_usage hadoop_add_subcommand "balancer" "run a cluster balancing utility" hadoop_add_subcommand "cacheadmin" "configure the HDFS cache" + hadoop_add_subcommand "cblock" "cblock CLI" + hadoop_add_subcommand "cblockserver" "run cblock server" hadoop_add_subcommand "classpath" "prints the class path needed to get the hadoop jar and the required libraries" hadoop_add_subcommand "crypto" "configure HDFS encryption zones" hadoop_add_subcommand "datanode" "run a DFS datanode" @@ -86,6 +88,15 @@ function hdfscmd_case cacheadmin) HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.CacheAdmin ;; + cblock) + HADOOP_CLASSNAME=org.apache.hadoop.cblock.cli.CBlockCli + ;; + cblockserver) + HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + HADOOP_CLASSNAME=org.apache.hadoop.cblock.CBlockManager + hadoop_debug "Appending HADOOP_CBLOCK_OPTS onto HADOOP_OPTS" + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CBLOCK_OPTS}" + ;; classpath) hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@" ;; http://git-wip-us.apache.org/repos/asf/hadoop/blob/350220bf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java index 8aa4ae8..b1fba41 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java @@ -142,9 +142,20 @@ public final class CBlockConfigKeys { public static final String DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_DEFAULT = "127.0.0.1"; + // to what address cblock server should talk to scm? + public static final String DFS_CBLOCK_SCM_IPADDRESS_KEY = + "dfs.cblock.scm.ipaddress"; + public static final String DFS_CBLOCK_SCM_IPADDRESS_DEFAULT = + "127.0.0.1"; + public static final String DFS_CBLOCK_SCM_PORT_KEY = + "dfs.cblock.scm.port"; + public static final int DFS_CBLOCK_SCM_PORT_DEFAULT = 9860; + public static final String DFS_CBLOCK_CONTAINER_SIZE_GB_KEY = - "dfs.cblock.container.size.gb"; - public static final int DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT = 5; + "dfs.cblock.container.size"; + public static final int DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT = + 5; + private CBlockConfigKeys() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/350220bf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockManager.java index c7abed6..9f8d5b1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockManager.java @@ -30,6 +30,11 @@ import org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocolPB; import org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocolServerSideTranslatorPB; import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB; import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolServerSideTranslatorPB; +import org.apache.hadoop.ipc.Client; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.scm.XceiverClientManager; +import org.apache.hadoop.scm.client.ContainerOperationClient; import org.apache.hadoop.scm.client.ScmClient; import org.apache.hadoop.cblock.storage.StorageManager; import org.apache.hadoop.cblock.util.KeyUtil; @@ -37,6 +42,9 @@ import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.ozone.OzoneConfiguration; +import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; +import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.utils.LevelDBStore; import org.iq80.leveldb.DBIterator; import org.slf4j.Logger; @@ -50,9 +58,15 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT; +import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_CONTAINER_SIZE_GB_KEY; import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSIRPC_ADDRESS_DEFAULT; import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY; import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSIRPC_BIND_HOST_KEY; +import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SCM_IPADDRESS_DEFAULT; +import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SCM_IPADDRESS_KEY; +import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SCM_PORT_DEFAULT; +import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SCM_PORT_KEY; import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICERPC_ADDRESS_DEFAULT; import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICERPC_ADDRESS_KEY; import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICERPC_BIND_HOST_KEY; @@ -60,6 +74,7 @@ import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICERPC_HA import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY; import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICE_LEVELDB_PATH_DEFAULT; import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT; /** * The main entry point of CBlock operations, ALL the CBlock operations @@ -317,4 +332,36 @@ public class CBlockManager implements CBlockServiceProtocol, } return response; } + + public static void main(String[] args) throws Exception { + long version = RPC.getProtocolVersion( + StorageContainerLocationProtocolPB.class); + OzoneConfiguration ozoneConf = new OzoneConfiguration(); + String scmAddress = ozoneConf.get(DFS_CBLOCK_SCM_IPADDRESS_KEY, + DFS_CBLOCK_SCM_IPADDRESS_DEFAULT); + int scmPort = ozoneConf.getInt(DFS_CBLOCK_SCM_PORT_KEY, + DFS_CBLOCK_SCM_PORT_DEFAULT); + int containerSizeGB = ozoneConf.getInt(DFS_CBLOCK_CONTAINER_SIZE_GB_KEY, + DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT); + ContainerOperationClient.setContainerSizeB(containerSizeGB* OzoneConsts.GB); + InetSocketAddress address = new InetSocketAddress(scmAddress, scmPort); + + ozoneConf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, + OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT); + LOG.info( + "Creating StorageContainerLocationProtocol RPC client with address {}", + address); + RPC.setProtocolEngine(ozoneConf, StorageContainerLocationProtocolPB.class, + ProtobufRpcEngine.class); + StorageContainerLocationProtocolClientSideTranslatorPB client = + new StorageContainerLocationProtocolClientSideTranslatorPB( + RPC.getProxy(StorageContainerLocationProtocolPB.class, version, + address, UserGroupInformation.getCurrentUser(), ozoneConf, + NetUtils.getDefaultSocketFactory(ozoneConf), + Client.getRpcTimeout(ozoneConf))); + ScmClient storageClient = new ContainerOperationClient( + client, new XceiverClientManager(ozoneConf)); + CBlockManager cbm = new CBlockManager(ozoneConf, storageClient); + cbm.start(); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
