Repository: hadoop Updated Branches: refs/heads/HDFS-7240 d63ec0ca8 -> ad16978e6
HDFS-11414. Ozone : move StorageContainerLocation protocol to hdfs-client. Contributed by Chen Liang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ad16978e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ad16978e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ad16978e Branch: refs/heads/HDFS-7240 Commit: ad16978e65df67cf735b6255f4f062dac0ee1523 Parents: d63ec0c Author: Arpit Agarwal <[email protected]> Authored: Tue Feb 28 10:28:05 2017 -0800 Committer: Arpit Agarwal <[email protected]> Committed: Tue Feb 28 10:28:05 2017 -0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 1 + .../hadoop/scm/protocol/LocatedContainer.java | 127 +++++++++++++++++ .../StorageContainerLocationProtocol.java | 52 +++++++ .../hadoop/scm/protocol/package-info.java | 19 +++ ...rLocationProtocolClientSideTranslatorPB.java | 141 +++++++++++++++++++ .../StorageContainerLocationProtocolPB.java | 34 +++++ .../hadoop/scm/protocolPB/package-info.java | 24 ++++ .../StorageContainerLocationProtocol.proto | 99 +++++++++++++ hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 - .../server/datanode/ObjectStoreHandler.java | 4 +- .../hadoop/ozone/protocol/LocatedContainer.java | 127 ----------------- .../StorageContainerLocationProtocol.java | 54 ------- ...rLocationProtocolClientSideTranslatorPB.java | 141 ------------------- .../StorageContainerLocationProtocolPB.java | 34 ----- ...rLocationProtocolServerSideTranslatorPB.java | 5 +- .../ozone/scm/StorageContainerManager.java | 6 +- .../web/storage/DistributedStorageHandler.java | 4 +- .../StorageContainerLocationProtocol.proto | 99 ------------- .../apache/hadoop/ozone/MiniOzoneCluster.java | 4 +- .../ozone/TestStorageContainerManager.java | 4 +- .../hadoop/ozone/scm/TestAllocateContainer.java | 3 +- 21 files changed, 512 insertions(+), 471 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad16978e/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml index 9c8dd1b..31e6408 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml @@ -165,6 +165,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <include>inotify.proto</include> <include>erasurecoding.proto</include> <include>ReconfigurationProtocol.proto</include> + <include>StorageContainerLocationProtocol.proto</include> <include>DatanodeContainerProtocol.proto</include> </includes> </source> http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad16978e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/LocatedContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/LocatedContainer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/LocatedContainer.java new file mode 100644 index 0000000..6e89d0c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/LocatedContainer.java @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.scm.protocol; + +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; + +/** + * Holds the nodes that currently host the container for an object key hash. + */ [email protected] +public final class LocatedContainer { + private final String key; + private final String matchedKeyPrefix; + private final String containerName; + private final Set<DatanodeInfo> locations; + private final DatanodeInfo leader; + + /** + * Creates a LocatedContainer. + * + * @param key object key + * @param matchedKeyPrefix prefix of key that was used to find the location + * @param containerName container name + * @param locations nodes that currently host the container + * @param leader node that currently acts as pipeline leader + */ + public LocatedContainer(String key, String matchedKeyPrefix, + String containerName, Set<DatanodeInfo> locations, DatanodeInfo leader) { + this.key = key; + this.matchedKeyPrefix = matchedKeyPrefix; + this.containerName = containerName; + this.locations = locations; + this.leader = leader; + } + + /** + * Returns the container name. + * + * @return container name + */ + public String getContainerName() { + return this.containerName; + } + + /** + * Returns the object key. + * + * @return object key + */ + public String getKey() { + return this.key; + } + + /** + * Returns the node that currently acts as pipeline leader. + * + * @return node that currently acts as pipeline leader + */ + public DatanodeInfo getLeader() { + return this.leader; + } + + /** + * Returns the nodes that currently host the container. + * + * @return Set<DatanodeInfo> nodes that currently host the container + */ + public Set<DatanodeInfo> getLocations() { + return this.locations; + } + + /** + * Returns the prefix of the key that was used to find the location. + * + * @return prefix of the key that was used to find the location + */ + public String getMatchedKeyPrefix() { + return this.matchedKeyPrefix; + } + + @Override + public boolean equals(Object otherObj) { + if (otherObj == null) { + return false; + } + if (!(otherObj instanceof LocatedContainer)) { + return false; + } + LocatedContainer other = (LocatedContainer)otherObj; + return this.key == null ? other.key == null : this.key.equals(other.key); + } + + @Override + public int hashCode() { + return key.hashCode(); + } + + @Override + public String toString() { + return getClass().getSimpleName() + + "{key=" + key + + "; matchedKeyPrefix=" + matchedKeyPrefix + + "; containerName=" + containerName + + "; locations=" + locations + + "; leader=" + leader + + "}"; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad16978e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java new file mode 100644 index 0000000..ba15ac0 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.scm.protocol; + +import java.io.IOException; +import java.util.Set; + +import org.apache.hadoop.scm.container.common.helpers.Pipeline; + +/** + * ContainerLocationProtocol is used by an HDFS node to find the set of nodes + * that currently host a container. + */ +public interface StorageContainerLocationProtocol { + + /** + * Find the set of nodes that currently host the container of an object, as + * identified by the object key hash. This method supports batch lookup by + * passing multiple key hashes. + * + * @param keys batch of object keys to find + * @return located containers for each object key + * @throws IOException if there is any failure + */ + Set<LocatedContainer> getStorageContainerLocations(Set<String> keys) + throws IOException; + + /** + * Asks SCM where a container should be allocated. SCM responds with the + * set of datanodes that should be used creating this container. + * @param containerName - Name of the container. + * @return Pipeline. + * @throws IOException + */ + Pipeline allocateContainer(String containerName) throws IOException; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad16978e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/package-info.java new file mode 100644 index 0000000..274f859 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/package-info.java @@ -0,0 +1,19 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.scm.protocol; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad16978e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java new file mode 100644 index 0000000..b5686c2 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java @@ -0,0 +1,141 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.scm.protocolPB; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Sets; +import com.google.protobuf.RpcController; +import com.google.protobuf.ServiceException; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfoProto; +import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; +import org.apache.hadoop.ipc.ProtobufHelper; +import org.apache.hadoop.ipc.ProtocolTranslator; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.scm.protocol.LocatedContainer; +import org.apache.hadoop.scm.protocol.StorageContainerLocationProtocol; +import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ContainerRequestProto; +import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ContainerResponseProto; +import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.GetStorageContainerLocationsRequestProto; +import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.GetStorageContainerLocationsResponseProto; +import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.LocatedContainerProto; +import org.apache.hadoop.scm.container.common.helpers.Pipeline; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Set; + +/** + * This class is the client-side translator to translate the requests made on + * the {@link StorageContainerLocationProtocol} interface to the RPC server + * implementing {@link StorageContainerLocationProtocolPB}. + */ [email protected] +public final class StorageContainerLocationProtocolClientSideTranslatorPB + implements StorageContainerLocationProtocol, ProtocolTranslator, Closeable { + + /** + * RpcController is not used and hence is set to null. + */ + private static final RpcController NULL_RPC_CONTROLLER = null; + + private final StorageContainerLocationProtocolPB rpcProxy; + + /** + * Creates a new StorageContainerLocationProtocolClientSideTranslatorPB. + * + * @param rpcProxy {@link StorageContainerLocationProtocolPB} RPC proxy + */ + public StorageContainerLocationProtocolClientSideTranslatorPB( + StorageContainerLocationProtocolPB rpcProxy) { + this.rpcProxy = rpcProxy; + } + + @Override + public Set<LocatedContainer> getStorageContainerLocations(Set<String> keys) + throws IOException { + GetStorageContainerLocationsRequestProto.Builder req = + GetStorageContainerLocationsRequestProto.newBuilder(); + for (String key : keys) { + req.addKeys(key); + } + final GetStorageContainerLocationsResponseProto resp; + try { + resp = rpcProxy.getStorageContainerLocations(NULL_RPC_CONTROLLER, + req.build()); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + Set<LocatedContainer> locatedContainers = + Sets.newLinkedHashSetWithExpectedSize(resp.getLocatedContainersCount()); + for (LocatedContainerProto locatedContainer : + resp.getLocatedContainersList()) { + Set<DatanodeInfo> locations = Sets.newLinkedHashSetWithExpectedSize( + locatedContainer.getLocationsCount()); + for (DatanodeInfoProto location : locatedContainer.getLocationsList()) { + locations.add(PBHelperClient.convert(location)); + } + locatedContainers.add(new LocatedContainer(locatedContainer.getKey(), + locatedContainer.getMatchedKeyPrefix(), + locatedContainer.getContainerName(), locations, + PBHelperClient.convert(locatedContainer.getLeader()))); + } + return locatedContainers; + } + + /** + * Asks SCM where a container should be allocated. SCM responds with the set + * of datanodes that should be used creating this container. + * + * @param containerName - Name of the container. + * @return Pipeline. + * @throws IOException + */ + @Override + public Pipeline allocateContainer(String containerName) throws IOException { + + Preconditions.checkNotNull(containerName, "Container Name cannot be Null"); + Preconditions.checkState(!containerName.isEmpty(), "Container name cannot" + + " be empty"); + + ContainerRequestProto request = ContainerRequestProto.newBuilder() + .setContainerName(containerName).build(); + + final ContainerResponseProto response; + try { + response = rpcProxy.allocateContainer(NULL_RPC_CONTROLLER, request); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + if (response.getErrorCode() != ContainerResponseProto.Error.success) { + throw new IOException(response.hasErrorMessage() ? + response.getErrorMessage() : "Allocate container failed."); + } + return Pipeline.getFromProtoBuf(response.getPipeline()); + } + + @Override + public Object getUnderlyingProxyObject() { + return rpcProxy; + } + + @Override + public void close() { + RPC.stopProxy(rpcProxy); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad16978e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolPB.java new file mode 100644 index 0000000..9ee1307 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolPB.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.scm.protocolPB; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.ipc.ProtocolInfo; +import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.StorageContainerLocationProtocolService; + +/** + * Protocol used from an HDFS node to StorageContainerManager. This extends the + * Protocol Buffers service interface to add Hadoop-specific annotations. + */ +@ProtocolInfo(protocolName = + "org.apache.hadoop.ozone.protocol.StorageContainerLocationProtocol", + protocolVersion = 1) [email protected] +public interface StorageContainerLocationProtocolPB + extends StorageContainerLocationProtocolService.BlockingInterface { +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad16978e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/package-info.java new file mode 100644 index 0000000..f9a2c09 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/package-info.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.scm.protocolPB; + +/** + * This package contains classes for the client of the storage container + * protocol. + */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad16978e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto new file mode 100644 index 0000000..5b2fa49 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * These .proto interfaces are private and unstable. + * Please see http://wiki.apache.org/hadoop/Compatibility + * for what changes are allowed for a *unstable* .proto interface. + */ + +option java_package = "org.apache.hadoop.ozone.protocol.proto"; +option java_outer_classname = "StorageContainerLocationProtocolProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +package hadoop.hdfs; + +import "hdfs.proto"; +import "DatanodeContainerProtocol.proto"; + +/** + * keys - batch of object keys to find + */ +message GetStorageContainerLocationsRequestProto { + repeated string keys = 1; +} + +/** + * locatedContainers - for each requested hash, nodes that currently host the + * container for that object key hash + */ +message GetStorageContainerLocationsResponseProto { + repeated LocatedContainerProto locatedContainers = 1; +} + +/** + * Holds the nodes that currently host the container for an object key. + */ +message LocatedContainerProto { + required string key = 1; + required string matchedKeyPrefix = 2; + required string containerName = 3; + repeated DatanodeInfoProto locations = 4; + required DatanodeInfoProto leader = 5; +} + +/** +* Request send to SCM asking where the container should be created. +*/ +message ContainerRequestProto { + required string containerName = 1; +} + +/** + * Reply from SCM indicating that the container. + */ +message ContainerResponseProto { + enum Error { + success = 1; + errorContainerAlreadyExists = 2; + errorContainerMissing = 3; + } + required Error errorCode = 1; + required hadoop.hdfs.ozone.Pipeline pipeline = 2; + optional string errorMessage = 3; +} + +/** + * Protocol used from an HDFS node to StorageContainerManager. See the request + * and response messages for details of the RPC calls. + */ +service StorageContainerLocationProtocolService { + /** + * Find the set of nodes that currently host the container of an object, as + * identified by the object key hash. This method supports batch lookup by + * passing multiple key hashes. + */ + rpc getStorageContainerLocations(GetStorageContainerLocationsRequestProto) + returns(GetStorageContainerLocationsResponseProto); + + /** + Creates a container entry in SCM. + */ + rpc allocateContainer(ContainerRequestProto) returns (ContainerResponseProto); + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad16978e/hadoop-hdfs-project/hadoop-hdfs/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index d375cc7..f9d2ee5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -350,7 +350,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <include>QJournalProtocol.proto</include> <include>editlog.proto</include> <include>fsimage.proto</include> - <include>StorageContainerLocationProtocol.proto</include> <include>StorageContainerDatanodeProtocol.proto</include> <include>CBlockServiceProtocol.proto</include> <include>CBlockClientServerProtocol.proto</include> http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad16978e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ObjectStoreHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ObjectStoreHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ObjectStoreHandler.java index 5eed3ac..65d0256 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ObjectStoreHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ObjectStoreHandler.java @@ -40,8 +40,8 @@ import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.ozone.OzoneConfiguration; -import org.apache.hadoop.ozone.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; -import org.apache.hadoop.ozone.protocolPB.StorageContainerLocationProtocolPB; +import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; +import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB; import org.apache.hadoop.ozone.web.handlers.ServiceFilter; import org.apache.hadoop.ozone.web.interfaces.StorageHandler; import org.apache.hadoop.ozone.web.ObjectStoreApplication; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad16978e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/LocatedContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/LocatedContainer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/LocatedContainer.java deleted file mode 100644 index 1915caa..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/LocatedContainer.java +++ /dev/null @@ -1,127 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.protocol; - -import java.util.Set; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hdfs.protocol.DatanodeInfo; - -/** - * Holds the nodes that currently host the container for an object key hash. - */ [email protected] -public final class LocatedContainer { - private final String key; - private final String matchedKeyPrefix; - private final String containerName; - private final Set<DatanodeInfo> locations; - private final DatanodeInfo leader; - - /** - * Creates a LocatedContainer. - * - * @param key object key - * @param matchedKeyPrefix prefix of key that was used to find the location - * @param containerName container name - * @param locations nodes that currently host the container - * @param leader node that currently acts as pipeline leader - */ - public LocatedContainer(String key, String matchedKeyPrefix, - String containerName, Set<DatanodeInfo> locations, DatanodeInfo leader) { - this.key = key; - this.matchedKeyPrefix = matchedKeyPrefix; - this.containerName = containerName; - this.locations = locations; - this.leader = leader; - } - - /** - * Returns the container name. - * - * @return container name - */ - public String getContainerName() { - return this.containerName; - } - - /** - * Returns the object key. - * - * @return object key - */ - public String getKey() { - return this.key; - } - - /** - * Returns the node that currently acts as pipeline leader. - * - * @return node that currently acts as pipeline leader - */ - public DatanodeInfo getLeader() { - return this.leader; - } - - /** - * Returns the nodes that currently host the container. - * - * @return Set<DatanodeInfo> nodes that currently host the container - */ - public Set<DatanodeInfo> getLocations() { - return this.locations; - } - - /** - * Returns the prefix of the key that was used to find the location. - * - * @return prefix of the key that was used to find the location - */ - public String getMatchedKeyPrefix() { - return this.matchedKeyPrefix; - } - - @Override - public boolean equals(Object otherObj) { - if (otherObj == null) { - return false; - } - if (!(otherObj instanceof LocatedContainer)) { - return false; - } - LocatedContainer other = (LocatedContainer)otherObj; - return this.key == null ? other.key == null : this.key.equals(other.key); - } - - @Override - public int hashCode() { - return key.hashCode(); - } - - @Override - public String toString() { - return getClass().getSimpleName() - + "{key=" + key - + "; matchedKeyPrefix=" + matchedKeyPrefix - + "; containerName=" + containerName - + "; locations=" + locations - + "; leader=" + leader - + "}"; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad16978e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerLocationProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerLocationProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerLocationProtocol.java deleted file mode 100644 index 6f51925..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerLocationProtocol.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.protocol; - -import java.io.IOException; -import java.util.Set; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.scm.container.common.helpers.Pipeline; - -/** - * ContainerLocationProtocol is used by an HDFS node to find the set of nodes - * that currently host a container. - */ [email protected] -public interface StorageContainerLocationProtocol { - - /** - * Find the set of nodes that currently host the container of an object, as - * identified by the object key hash. This method supports batch lookup by - * passing multiple key hashes. - * - * @param keys batch of object keys to find - * @return located containers for each object key - * @throws IOException if there is any failure - */ - Set<LocatedContainer> getStorageContainerLocations(Set<String> keys) - throws IOException; - - /** - * Asks SCM where a container should be allocated. SCM responds with the - * set of datanodes that should be used creating this container. - * @param containerName - Name of the container. - * @return Pipeline. - * @throws IOException - */ - Pipeline allocateContainer(String containerName) throws IOException; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad16978e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java deleted file mode 100644 index c8dec01..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java +++ /dev/null @@ -1,141 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.hadoop.ozone.protocolPB; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Sets; -import com.google.protobuf.RpcController; -import com.google.protobuf.ServiceException; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hdfs.protocol.DatanodeInfo; -import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfoProto; -import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; -import org.apache.hadoop.ipc.ProtobufHelper; -import org.apache.hadoop.ipc.ProtocolTranslator; -import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.ozone.protocol.LocatedContainer; -import org.apache.hadoop.ozone.protocol.StorageContainerLocationProtocol; -import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ContainerRequestProto; -import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ContainerResponseProto; -import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.GetStorageContainerLocationsRequestProto; -import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.GetStorageContainerLocationsResponseProto; -import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.LocatedContainerProto; -import org.apache.hadoop.scm.container.common.helpers.Pipeline; - -import java.io.Closeable; -import java.io.IOException; -import java.util.Set; - -/** - * This class is the client-side translator to translate the requests made on - * the {@link StorageContainerLocationProtocol} interface to the RPC server - * implementing {@link StorageContainerLocationProtocolPB}. - */ [email protected] -public final class StorageContainerLocationProtocolClientSideTranslatorPB - implements StorageContainerLocationProtocol, ProtocolTranslator, Closeable { - - /** - * RpcController is not used and hence is set to null. - */ - private static final RpcController NULL_RPC_CONTROLLER = null; - - private final StorageContainerLocationProtocolPB rpcProxy; - - /** - * Creates a new StorageContainerLocationProtocolClientSideTranslatorPB. - * - * @param rpcProxy {@link StorageContainerLocationProtocolPB} RPC proxy - */ - public StorageContainerLocationProtocolClientSideTranslatorPB( - StorageContainerLocationProtocolPB rpcProxy) { - this.rpcProxy = rpcProxy; - } - - @Override - public Set<LocatedContainer> getStorageContainerLocations(Set<String> keys) - throws IOException { - GetStorageContainerLocationsRequestProto.Builder req = - GetStorageContainerLocationsRequestProto.newBuilder(); - for (String key : keys) { - req.addKeys(key); - } - final GetStorageContainerLocationsResponseProto resp; - try { - resp = rpcProxy.getStorageContainerLocations(NULL_RPC_CONTROLLER, - req.build()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } - Set<LocatedContainer> locatedContainers = - Sets.newLinkedHashSetWithExpectedSize(resp.getLocatedContainersCount()); - for (LocatedContainerProto locatedContainer : - resp.getLocatedContainersList()) { - Set<DatanodeInfo> locations = Sets.newLinkedHashSetWithExpectedSize( - locatedContainer.getLocationsCount()); - for (DatanodeInfoProto location : locatedContainer.getLocationsList()) { - locations.add(PBHelperClient.convert(location)); - } - locatedContainers.add(new LocatedContainer(locatedContainer.getKey(), - locatedContainer.getMatchedKeyPrefix(), - locatedContainer.getContainerName(), locations, - PBHelperClient.convert(locatedContainer.getLeader()))); - } - return locatedContainers; - } - - /** - * Asks SCM where a container should be allocated. SCM responds with the set - * of datanodes that should be used creating this container. - * - * @param containerName - Name of the container. - * @return Pipeline. - * @throws IOException - */ - @Override - public Pipeline allocateContainer(String containerName) throws IOException { - - Preconditions.checkNotNull(containerName, "Container Name cannot be Null"); - Preconditions.checkState(!containerName.isEmpty(), "Container name cannot" + - " be empty"); - - ContainerRequestProto request = ContainerRequestProto.newBuilder() - .setContainerName(containerName).build(); - - final ContainerResponseProto response; - try { - response = rpcProxy.allocateContainer(NULL_RPC_CONTROLLER, request); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } - if (response.getErrorCode() != ContainerResponseProto.Error.success) { - throw new IOException(response.hasErrorMessage() ? - response.getErrorMessage() : "Allocate container failed."); - } - return Pipeline.getFromProtoBuf(response.getPipeline()); - } - - @Override - public Object getUnderlyingProxyObject() { - return rpcProxy; - } - - @Override - public void close() { - RPC.stopProxy(rpcProxy); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad16978e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolPB.java deleted file mode 100644 index 0587655..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolPB.java +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.protocolPB; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.ipc.ProtocolInfo; -import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.StorageContainerLocationProtocolService; - -/** - * Protocol used from an HDFS node to StorageContainerManager. This extends the - * Protocol Buffers service interface to add Hadoop-specific annotations. - */ -@ProtocolInfo(protocolName = - "org.apache.hadoop.ozone.protocol.StorageContainerLocationProtocol", - protocolVersion = 1) [email protected] -public interface StorageContainerLocationProtocolPB - extends StorageContainerLocationProtocolService.BlockingInterface { -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad16978e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java index 6590112..b634e56 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java @@ -27,8 +27,8 @@ import com.google.protobuf.ServiceException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; -import org.apache.hadoop.ozone.protocol.LocatedContainer; -import org.apache.hadoop.ozone.protocol.StorageContainerLocationProtocol; +import org.apache.hadoop.scm.protocol.LocatedContainer; +import org.apache.hadoop.scm.protocol.StorageContainerLocationProtocol; import org.apache.hadoop.ozone.protocol.proto .StorageContainerLocationProtocolProtos; import org.apache.hadoop.ozone.protocol.proto @@ -43,6 +43,7 @@ import org.apache.hadoop.ozone.protocol.proto .StorageContainerLocationProtocolProtos.ContainerResponseProto; import org.apache.hadoop.scm.container.common.helpers.Pipeline; +import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB; /** * This class is the server-side translator that forwards requests received on http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad16978e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java index 0a6f35f..b6fd4c0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java @@ -28,9 +28,9 @@ import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ozone.OzoneClientUtils; import org.apache.hadoop.ozone.OzoneConfiguration; -import org.apache.hadoop.ozone.protocol.LocatedContainer; +import org.apache.hadoop.scm.protocol.LocatedContainer; import org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol; -import org.apache.hadoop.ozone.protocol.StorageContainerLocationProtocol; +import org.apache.hadoop.scm.protocol.StorageContainerLocationProtocol; import org.apache.hadoop.ozone.protocol.commands.NullCommand; import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand; import org.apache.hadoop.ozone.protocol.commands.SCMCommand; @@ -57,7 +57,7 @@ import org.apache.hadoop.ozone.protocol.proto import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolPB; import org.apache.hadoop.ozone.protocolPB .StorageContainerDatanodeProtocolServerSideTranslatorPB; -import org.apache.hadoop.ozone.protocolPB.StorageContainerLocationProtocolPB; +import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB; import org.apache.hadoop.ozone.protocolPB .StorageContainerLocationProtocolServerSideTranslatorPB; import org.apache.hadoop.ozone.scm.container.ContainerMapping; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad16978e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java index e8e5830..8480a88 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java @@ -42,8 +42,8 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.scm.container.common.helpers.Pipeline; import org.apache.hadoop.scm.XceiverClient; import org.apache.hadoop.scm.XceiverClientManager; -import org.apache.hadoop.ozone.protocol.LocatedContainer; -import org.apache.hadoop.ozone.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; +import org.apache.hadoop.scm.protocol.LocatedContainer; +import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.ozone.web.exceptions.OzoneException; import org.apache.hadoop.ozone.web.handlers.BucketArgs; import org.apache.hadoop.ozone.web.handlers.KeyArgs; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad16978e/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerLocationProtocol.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerLocationProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerLocationProtocol.proto deleted file mode 100644 index 5b2fa49..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerLocationProtocol.proto +++ /dev/null @@ -1,99 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * These .proto interfaces are private and unstable. - * Please see http://wiki.apache.org/hadoop/Compatibility - * for what changes are allowed for a *unstable* .proto interface. - */ - -option java_package = "org.apache.hadoop.ozone.protocol.proto"; -option java_outer_classname = "StorageContainerLocationProtocolProtos"; -option java_generic_services = true; -option java_generate_equals_and_hash = true; -package hadoop.hdfs; - -import "hdfs.proto"; -import "DatanodeContainerProtocol.proto"; - -/** - * keys - batch of object keys to find - */ -message GetStorageContainerLocationsRequestProto { - repeated string keys = 1; -} - -/** - * locatedContainers - for each requested hash, nodes that currently host the - * container for that object key hash - */ -message GetStorageContainerLocationsResponseProto { - repeated LocatedContainerProto locatedContainers = 1; -} - -/** - * Holds the nodes that currently host the container for an object key. - */ -message LocatedContainerProto { - required string key = 1; - required string matchedKeyPrefix = 2; - required string containerName = 3; - repeated DatanodeInfoProto locations = 4; - required DatanodeInfoProto leader = 5; -} - -/** -* Request send to SCM asking where the container should be created. -*/ -message ContainerRequestProto { - required string containerName = 1; -} - -/** - * Reply from SCM indicating that the container. - */ -message ContainerResponseProto { - enum Error { - success = 1; - errorContainerAlreadyExists = 2; - errorContainerMissing = 3; - } - required Error errorCode = 1; - required hadoop.hdfs.ozone.Pipeline pipeline = 2; - optional string errorMessage = 3; -} - -/** - * Protocol used from an HDFS node to StorageContainerManager. See the request - * and response messages for details of the RPC calls. - */ -service StorageContainerLocationProtocolService { - /** - * Find the set of nodes that currently host the container of an object, as - * identified by the object key hash. This method supports batch lookup by - * passing multiple key hashes. - */ - rpc getStorageContainerLocations(GetStorageContainerLocationsRequestProto) - returns(GetStorageContainerLocationsResponseProto); - - /** - Creates a container entry in SCM. - */ - rpc allocateContainer(ContainerRequestProto) returns (ContainerResponseProto); - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad16978e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java index 85968e4..7e1cda3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java @@ -26,8 +26,8 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.ipc.Client; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.ozone.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; -import org.apache.hadoop.ozone.protocolPB.StorageContainerLocationProtocolPB; +import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; +import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB; import org.apache.hadoop.ozone.scm.StorageContainerManager; import org.apache.hadoop.ozone.scm.node.SCMNodeManager; import org.apache.hadoop.ozone.web.client.OzoneClient; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad16978e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java index 5f1348a..e8ca6b1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java @@ -32,8 +32,8 @@ import org.junit.Rule; import org.junit.rules.ExpectedException; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.ozone.protocol.LocatedContainer; -import org.apache.hadoop.ozone.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; +import org.apache.hadoop.scm.protocol.LocatedContainer; +import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; import org.junit.rules.Timeout; /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad16978e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestAllocateContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestAllocateContainer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestAllocateContainer.java index 727055a..c310647 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestAllocateContainer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestAllocateContainer.java @@ -21,8 +21,7 @@ import org.apache.commons.lang.RandomStringUtils; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfiguration; -import org.apache.hadoop.ozone.protocolPB - .StorageContainerLocationProtocolClientSideTranslatorPB; +import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.scm.container.common.helpers.Pipeline; import org.junit.AfterClass; import org.junit.Assert; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
