YARN-8953. [CSI] CSI driver adaptor module support in NodeManager. Contributed by Weiwei Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5fb14e06 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5fb14e06 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5fb14e06 Branch: refs/heads/HDFS-13891 Commit: 5fb14e0635f850c63e009f954b8539a401d0f90a Parents: 27ffec7 Author: Sunil G <[email protected]> Authored: Tue Nov 20 08:02:11 2018 +0530 Committer: Sunil G <[email protected]> Committed: Tue Nov 20 08:02:11 2018 +0530 ---------------------------------------------------------------------- .../hadoop-yarn/hadoop-yarn-api/pom.xml | 2 + .../apache/hadoop/yarn/api/CsiAdaptorPB.java | 31 +++ .../hadoop/yarn/api/CsiAdaptorProtocol.java | 39 +++ .../protocolrecords/GetPluginInfoRequest.java | 30 +++ .../protocolrecords/GetPluginInfoResponse.java | 43 ++++ .../ValidateVolumeCapabilitiesRequest.java | 117 +++++++++ .../ValidateVolumeCapabilitiesResponse.java | 46 ++++ .../hadoop/yarn/conf/YarnConfiguration.java | 12 + .../src/main/proto/YarnCsiAdaptor.proto | 34 +++ .../src/main/proto/yarn_csi_adaptor.proto | 69 +++++ .../client/CsiAdaptorProtocolPBClientImpl.java | 91 +++++++ .../CsiAdaptorProtocolPBServiceImpl.java | 75 ++++++ .../impl/pb/GetPluginInfoRequestPBImpl.java | 60 +++++ .../impl/pb/GetPluginInfoResponsePBImpl.java | 84 ++++++ ...ValidateVolumeCapabilitiesRequestPBImpl.java | 121 +++++++++ ...alidateVolumeCapabilitiesResponsePBImpl.java | 87 +++++++ .../src/main/resources/yarn-default.xml | 13 + .../hadoop-yarn/hadoop-yarn-csi/pom.xml | 12 + .../csi/adaptor/CsiAdaptorProtocolService.java | 122 +++++++++ .../hadoop/yarn/csi/adaptor/package-info.java | 21 ++ .../hadoop/yarn/csi/client/CsiClient.java | 4 + .../hadoop/yarn/csi/client/CsiClientImpl.java | 11 + .../GetPluginInfoResponseProtoTranslator.java | 44 ++++ .../yarn/csi/translator/ProtoTranslator.java | 49 ++++ .../csi/translator/ProtoTranslatorFactory.java | 66 +++++ ...olumeCapabilitiesRequestProtoTranslator.java | 93 +++++++ ...lumeCapabilitiesResponseProtoTranslator.java | 48 ++++ .../yarn/csi/translator/package-info.java | 21 ++ .../hadoop/yarn/csi/utils/ConfigUtils.java | 61 +++++ .../yarn/csi/adaptor/TestCsiAdaptorService.java | 256 +++++++++++++++++++ .../TestGetPluginInfoRequestResponse.java | 66 +++++ .../TestValidateVolumeCapabilityRequest.java | 113 ++++++++ .../TestValidateVolumeCapabilityResponse.java | 61 +++++ .../hadoop/yarn/csi/adaptor/package-info.java | 21 ++ .../volume/csi/CsiAdaptorClientProtocol.java | 34 --- .../volume/csi/CsiAdaptorClient.java | 36 --- .../volume/csi/VolumeManager.java | 23 +- .../volume/csi/VolumeManagerImpl.java | 91 ++++++- .../volume/csi/lifecycle/Volume.java | 8 + .../volume/csi/lifecycle/VolumeImpl.java | 58 +++-- .../csi/processor/VolumeAMSProcessor.java | 17 +- .../volume/csi/TestVolumeLifecycle.java | 100 +++++--- .../volume/csi/TestVolumeProcessor.java | 32 ++- 43 files changed, 2273 insertions(+), 149 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml index 4f11ad6..832e98c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml @@ -124,6 +124,8 @@ <include>server/application_history_server.proto</include> <include>client_SCM_protocol.proto</include> <include>server/SCM_Admin_protocol.proto</include> + <include>yarn_csi_adaptor.proto</include> + <include>YarnCsiAdaptor.proto</include> </includes> </source> </configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/CsiAdaptorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/CsiAdaptorPB.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/CsiAdaptorPB.java new file mode 100644 index 0000000..2a7a114 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/CsiAdaptorPB.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api; + +import org.apache.hadoop.ipc.ProtocolInfo; +import org.apache.hadoop.yarn.proto.CsiAdaptorProtocol; + +/** + * Interface for the CSI adaptor protocol. + */ +@ProtocolInfo( + protocolName = "CsiAdaptorPB", + protocolVersion = 1) +public interface CsiAdaptorPB extends + CsiAdaptorProtocol.CsiAdaptorProtocolService.BlockingInterface { +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/CsiAdaptorProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/CsiAdaptorProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/CsiAdaptorProtocol.java new file mode 100644 index 0000000..0822163 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/CsiAdaptorProtocol.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api; + +import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoResponse; +import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesResponse; +import org.apache.hadoop.yarn.exceptions.YarnException; + +import java.io.IOException; + +/** + * CSI adaptor delegates all the calls from YARN to a CSI driver. + */ +public interface CsiAdaptorProtocol { + + GetPluginInfoResponse getPluginInfo(GetPluginInfoRequest request) + throws YarnException, IOException; + + ValidateVolumeCapabilitiesResponse validateVolumeCapacity( + ValidateVolumeCapabilitiesRequest request) throws YarnException, + IOException; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetPluginInfoRequest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetPluginInfoRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetPluginInfoRequest.java new file mode 100644 index 0000000..a32c93c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetPluginInfoRequest.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.yarn.util.Records; + +/** + * Get plugin info request. + */ +public abstract class GetPluginInfoRequest { + + public static GetPluginInfoRequest newInstance() { + return Records.newRecord(GetPluginInfoRequest.class); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetPluginInfoResponse.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetPluginInfoResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetPluginInfoResponse.java new file mode 100644 index 0000000..9577200 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetPluginInfoResponse.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.yarn.util.Records; + +/** + * Get plugin info response. + */ +public abstract class GetPluginInfoResponse { + + public static GetPluginInfoResponse newInstance( + String driverName, String version) { + GetPluginInfoResponse response = + Records.newRecord(GetPluginInfoResponse.class); + response.setDriverName(driverName); + response.setVersion(version); + return response; + } + + public abstract void setDriverName(String driverName); + + public abstract String getDriverName(); + + public abstract void setVersion(String version); + + public abstract String getVersion(); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ValidateVolumeCapabilitiesRequest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ValidateVolumeCapabilitiesRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ValidateVolumeCapabilitiesRequest.java new file mode 100644 index 0000000..0ab1b1f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ValidateVolumeCapabilitiesRequest.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.yarn.util.Records; + +import java.util.List; +import java.util.Map; + +/** + * YARN internal message used to validate volume capabilities + * with a CSI driver controller plugin. + */ +public abstract class ValidateVolumeCapabilitiesRequest { + + /** + * Volume access mode. + */ + public enum AccessMode { + UNKNOWN, + SINGLE_NODE_WRITER, + SINGLE_NODE_READER_ONLY, + MULTI_NODE_READER_ONLY, + MULTI_NODE_SINGLE_WRITER, + MULTI_NODE_MULTI_WRITER, + } + + /** + * Volume type. + */ + public enum VolumeType { + BLOCK, + FILE_SYSTEM + } + + /** + * Volume capability. + */ + public static class VolumeCapability { + + private AccessMode mode; + private VolumeType type; + private List<String> flags; + + public VolumeCapability(AccessMode accessMode, VolumeType volumeType, + List<String> mountFlags) { + this.mode = accessMode; + this.type = volumeType; + this.flags = mountFlags; + } + + public AccessMode getAccessMode() { + return mode; + } + + public VolumeType getVolumeType() { + return type; + } + + public List<String> getMountFlags() { + return flags; + } + } + + public static ValidateVolumeCapabilitiesRequest newInstance( + String volumeId, List<VolumeCapability> volumeCapabilities, + Map<String, String> volumeAttributes) { + ValidateVolumeCapabilitiesRequest + request = + Records.newRecord( + ValidateVolumeCapabilitiesRequest.class); + request.setVolumeId(volumeId); + request.setVolumeAttributes(volumeAttributes); + for (VolumeCapability capability : volumeCapabilities) { + request.addVolumeCapability(capability); + } + return request; + } + + public static ValidateVolumeCapabilitiesRequest newInstance( + String volumeId, Map<String, String> volumeAttributes) { + ValidateVolumeCapabilitiesRequest + request = + Records.newRecord( + ValidateVolumeCapabilitiesRequest.class); + request.setVolumeId(volumeId); + request.setVolumeAttributes(volumeAttributes); + return request; + } + + public abstract void setVolumeId(String volumeId); + + public abstract String getVolumeId(); + + public abstract void setVolumeAttributes(Map<String, String> attributes); + + public abstract Map<String, String> getVolumeAttributes(); + + public abstract void addVolumeCapability(VolumeCapability volumeCapability); + + public abstract List<VolumeCapability> getVolumeCapabilities(); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ValidateVolumeCapabilitiesResponse.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ValidateVolumeCapabilitiesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ValidateVolumeCapabilitiesResponse.java new file mode 100644 index 0000000..a72ba0a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ValidateVolumeCapabilitiesResponse.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.yarn.util.Records; + +/** + * YARN internal message used to represent the response of + * volume capabilities validation with a CSI driver controller plugin. + */ +public abstract class ValidateVolumeCapabilitiesResponse { + + public static ValidateVolumeCapabilitiesResponse newInstance( + boolean supported, String responseMessage) { + ValidateVolumeCapabilitiesResponse + record = + Records.newRecord( + ValidateVolumeCapabilitiesResponse.class); + record.setResponseMessage(responseMessage); + record.setSupported(supported); + return record; + } + + public abstract void setSupported(boolean supported); + + public abstract boolean isSupported(); + + public abstract void setResponseMessage(String responseMessage); + + public abstract String getResponseMessage(); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index e88d594..2f25284 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3428,6 +3428,18 @@ public class YarnConfiguration extends Configuration { false; //////////////////////////////// + // CSI Volume configs + //////////////////////////////// + /** + * One or more socket addresses for csi-adaptor. + * Multiple addresses are delimited by ",". + */ + public static final String NM_CSI_ADAPTOR_PREFIX = + NM_PREFIX + "csi-driver-adaptor."; + public static final String NM_CSI_ADAPTOR_ADDRESSES = + NM_CSI_ADAPTOR_PREFIX + "addresses"; + + //////////////////////////////// // Other Configs //////////////////////////////// http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/YarnCsiAdaptor.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/YarnCsiAdaptor.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/YarnCsiAdaptor.proto new file mode 100644 index 0000000..9dcb8a7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/YarnCsiAdaptor.proto @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +option java_package = "org.apache.hadoop.yarn.proto"; +option java_outer_classname = "CsiAdaptorProtocol"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +package hadoop.yarn; + +import "yarn_csi_adaptor.proto"; + +service CsiAdaptorProtocolService { + + rpc getPluginInfo (GetPluginInfoRequest) + returns (GetPluginInfoResponse); + + rpc validateVolumeCapacity (ValidateVolumeCapabilitiesRequest) + returns (ValidateVolumeCapabilitiesResponse); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_csi_adaptor.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_csi_adaptor.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_csi_adaptor.proto new file mode 100644 index 0000000..c9adbea --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_csi_adaptor.proto @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +option java_package = "org.apache.hadoop.yarn.proto"; +option java_outer_classname = "CsiAdaptorProtos"; +option java_generate_equals_and_hash = true; +package hadoop.yarn; + +import "yarn_protos.proto"; + +message ValidateVolumeCapabilitiesRequest { + required string volume_id = 1; + repeated VolumeCapability volume_capabilities = 2; + repeated StringStringMapProto volume_attributes = 3; +} + +message ValidateVolumeCapabilitiesResponse { + // True if the Plugin supports the specified capabilities for the + // given volume. This field is REQUIRED. + required bool supported = 1; + + // Message to the CO if `supported` above is false. This field is + // OPTIONAL. + // An empty string is equal to an unspecified field value. + optional string message = 2; +} + +message VolumeCapability { + enum VolumeType { + BLOCK = 0; + FILE_SYSTEM = 1; + } + + enum AccessMode { + UNKNOWN = 0; + SINGLE_NODE_WRITER = 1; + SINGLE_NODE_READER_ONLY = 2; + MULTI_NODE_READER_ONLY = 3; + MULTI_NODE_SINGLE_WRITER = 4; + MULTI_NODE_MULTI_WRITER = 5; + } + + required VolumeType volume_type = 1; + required AccessMode access_mode = 2; + repeated string mount_flags = 3; +} + +message GetPluginInfoRequest { + // Intentionally empty. +} + +message GetPluginInfoResponse { + required string name = 1; + required string vendor_version = 2; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/CsiAdaptorProtocolPBClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/CsiAdaptorProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/CsiAdaptorProtocolPBClientImpl.java new file mode 100644 index 0000000..2e10f72 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/CsiAdaptorProtocolPBClientImpl.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.impl.pb.client; + +import com.google.protobuf.ServiceException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.yarn.api.CsiAdaptorPB; +import org.apache.hadoop.yarn.api.CsiAdaptorProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoResponse; +import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetPluginInfoRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetPluginInfoResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ValidateVolumeCapabilitiesRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ValidateVolumeCapabilitiesResponsePBImpl; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.ipc.RPCUtil; +import org.apache.hadoop.yarn.proto.CsiAdaptorProtos; + +import java.io.Closeable; +import java.io.IOException; +import java.net.InetSocketAddress; + +/** + * CSI adaptor client implementation. + */ +public class CsiAdaptorProtocolPBClientImpl + implements CsiAdaptorProtocol, Closeable { + + private final CsiAdaptorPB proxy; + + public CsiAdaptorProtocolPBClientImpl(long clientVersion, + InetSocketAddress addr, Configuration conf) throws IOException { + RPC.setProtocolEngine(conf, CsiAdaptorPB.class, ProtobufRpcEngine.class); + this.proxy = RPC.getProxy(CsiAdaptorPB.class, clientVersion, addr, conf); + } + + @Override + public GetPluginInfoResponse getPluginInfo( + GetPluginInfoRequest request) throws YarnException, IOException { + CsiAdaptorProtos.GetPluginInfoRequest requestProto = + ((GetPluginInfoRequestPBImpl) request).getProto(); + try { + return new GetPluginInfoResponsePBImpl( + proxy.getPluginInfo(null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } + + @Override + public ValidateVolumeCapabilitiesResponse validateVolumeCapacity( + ValidateVolumeCapabilitiesRequest request) + throws YarnException, IOException { + CsiAdaptorProtos.ValidateVolumeCapabilitiesRequest requestProto = + ((ValidateVolumeCapabilitiesRequestPBImpl) request).getProto(); + try { + return new ValidateVolumeCapabilitiesResponsePBImpl( + proxy.validateVolumeCapacity(null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } + + @Override + public void close() throws IOException { + if(this.proxy != null) { + RPC.stopProxy(this.proxy); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/CsiAdaptorProtocolPBServiceImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/CsiAdaptorProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/CsiAdaptorProtocolPBServiceImpl.java new file mode 100644 index 0000000..9a19435 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/CsiAdaptorProtocolPBServiceImpl.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.impl.pb.service; + +import com.google.protobuf.RpcController; +import com.google.protobuf.ServiceException; +import org.apache.hadoop.yarn.api.CsiAdaptorPB; +import org.apache.hadoop.yarn.api.CsiAdaptorProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoResponse; +import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetPluginInfoRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetPluginInfoResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ValidateVolumeCapabilitiesRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ValidateVolumeCapabilitiesResponsePBImpl; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.proto.CsiAdaptorProtos; + +import java.io.IOException; + +/** + * CSI adaptor server side implementation, this is hosted on a node manager. + */ +public class CsiAdaptorProtocolPBServiceImpl implements CsiAdaptorPB { + + private final CsiAdaptorProtocol real; + public CsiAdaptorProtocolPBServiceImpl(CsiAdaptorProtocol impl) { + this.real = impl; + } + + @Override + public CsiAdaptorProtos.GetPluginInfoResponse getPluginInfo( + RpcController controller, CsiAdaptorProtos.GetPluginInfoRequest request) + throws ServiceException { + try { + GetPluginInfoRequest req = + new GetPluginInfoRequestPBImpl(request); + GetPluginInfoResponse response = real.getPluginInfo(req); + return ((GetPluginInfoResponsePBImpl) response).getProto(); + } catch (YarnException | IOException e) { + throw new ServiceException(e); + } + } + + @Override + public CsiAdaptorProtos.ValidateVolumeCapabilitiesResponse + validateVolumeCapacity(RpcController controller, + CsiAdaptorProtos.ValidateVolumeCapabilitiesRequest request) + throws ServiceException { + try { + ValidateVolumeCapabilitiesRequestPBImpl req = + new ValidateVolumeCapabilitiesRequestPBImpl(request); + ValidateVolumeCapabilitiesResponse response = + real.validateVolumeCapacity(req); + return ((ValidateVolumeCapabilitiesResponsePBImpl) response).getProto(); + } catch (YarnException | IOException e) { + throw new ServiceException(e); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetPluginInfoRequestPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetPluginInfoRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetPluginInfoRequestPBImpl.java new file mode 100644 index 0000000..0d1c2e5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetPluginInfoRequestPBImpl.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoRequest; +import org.apache.hadoop.yarn.proto.CsiAdaptorProtos; + +/** + * Get plugin info request protobuf impl. + */ +public class GetPluginInfoRequestPBImpl extends GetPluginInfoRequest { + + private CsiAdaptorProtos.GetPluginInfoRequest.Builder builder; + + public GetPluginInfoRequestPBImpl( + CsiAdaptorProtos.GetPluginInfoRequest requestProto) { + this.builder = requestProto.toBuilder(); + } + + public GetPluginInfoRequestPBImpl() { + this.builder = CsiAdaptorProtos.GetPluginInfoRequest.newBuilder(); + } + + public CsiAdaptorProtos.GetPluginInfoRequest getProto() { + Preconditions.checkNotNull(builder); + return builder.build(); + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetPluginInfoResponsePBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetPluginInfoResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetPluginInfoResponsePBImpl.java new file mode 100644 index 0000000..141fd6d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetPluginInfoResponsePBImpl.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoResponse; +import org.apache.hadoop.yarn.proto.CsiAdaptorProtos; + +/** + * Get plugin info response protobuf impl. + */ +public class GetPluginInfoResponsePBImpl extends GetPluginInfoResponse { + + private CsiAdaptorProtos.GetPluginInfoResponse.Builder builder; + + public GetPluginInfoResponsePBImpl( + CsiAdaptorProtos.GetPluginInfoResponse responseProto) { + this.builder = responseProto.toBuilder(); + } + + public GetPluginInfoResponsePBImpl() { + this.builder = CsiAdaptorProtos.GetPluginInfoResponse.newBuilder(); + } + + @Override + public void setDriverName(String driverName) { + Preconditions.checkNotNull(builder); + builder.setName(driverName); + } + + @Override + public String getDriverName() { + Preconditions.checkNotNull(builder); + return builder.getName(); + } + + @Override + public void setVersion(String version) { + Preconditions.checkNotNull(builder); + builder.setVendorVersion(version); + } + + @Override + public String getVersion() { + Preconditions.checkNotNull(builder); + return builder.getVendorVersion(); + } + + public CsiAdaptorProtos.GetPluginInfoResponse getProto() { + Preconditions.checkNotNull(builder); + return builder.build(); + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesRequestPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesRequestPBImpl.java new file mode 100644 index 0000000..14bd89d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesRequestPBImpl.java @@ -0,0 +1,121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest; +import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; +import org.apache.hadoop.yarn.proto.CsiAdaptorProtos; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * PB wrapper for CsiAdaptorProtos.ValidateVolumeCapabilitiesRequest. + */ +public class ValidateVolumeCapabilitiesRequestPBImpl extends + ValidateVolumeCapabilitiesRequest { + + private CsiAdaptorProtos.ValidateVolumeCapabilitiesRequest.Builder builder; + + public ValidateVolumeCapabilitiesRequestPBImpl( + CsiAdaptorProtos.ValidateVolumeCapabilitiesRequest proto) { + this.builder = proto.toBuilder(); + } + + public ValidateVolumeCapabilitiesRequestPBImpl() { + this.builder = CsiAdaptorProtos.ValidateVolumeCapabilitiesRequest + .newBuilder(); + } + + @Override + public String getVolumeId() { + Preconditions.checkNotNull(builder); + return builder.getVolumeId(); + } + + @Override + public void setVolumeAttributes(Map<String, String> attributes) { + Preconditions.checkNotNull(builder); + builder.addAllVolumeAttributes(ProtoUtils.convertToProtoFormat(attributes)); + } + + @Override + public void setVolumeId(String volumeId) { + Preconditions.checkNotNull(builder); + builder.setVolumeId(volumeId); + } + + @Override + public void addVolumeCapability(VolumeCapability volumeCapability) { + Preconditions.checkNotNull(builder); + CsiAdaptorProtos.VolumeCapability vc = + CsiAdaptorProtos.VolumeCapability.newBuilder() + .setAccessMode(CsiAdaptorProtos.VolumeCapability.AccessMode + .valueOf(volumeCapability.getAccessMode().ordinal())) + .setVolumeType(CsiAdaptorProtos.VolumeCapability.VolumeType + .valueOf(volumeCapability.getVolumeType().ordinal())) + .addAllMountFlags(volumeCapability.getMountFlags()) + .build(); + builder.addVolumeCapabilities(vc); + } + + @Override + public List<VolumeCapability> getVolumeCapabilities() { + Preconditions.checkNotNull(builder); + List<VolumeCapability> caps = new ArrayList<>( + builder.getVolumeCapabilitiesCount()); + builder.getVolumeCapabilitiesList().forEach(capability -> { + VolumeCapability vc = new VolumeCapability( + AccessMode.valueOf(capability.getAccessMode().name()), + VolumeType.valueOf(capability.getVolumeType().name()), + capability.getMountFlagsList()); + caps.add(vc); + }); + return caps; + } + + @Override + public Map<String, String> getVolumeAttributes() { + Preconditions.checkNotNull(builder); + return ProtoUtils.convertStringStringMapProtoListToMap( + builder.getVolumeAttributesList()); + } + + public CsiAdaptorProtos.ValidateVolumeCapabilitiesRequest getProto() { + Preconditions.checkNotNull(builder); + return builder.build(); + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesResponsePBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesResponsePBImpl.java new file mode 100644 index 0000000..aa33ab7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesResponsePBImpl.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesResponse; +import org.apache.hadoop.yarn.proto.CsiAdaptorProtos; + +/** + * PB wrapper for CsiAdaptorProtos.ValidateVolumeCapabilitiesResponse. + */ +public class ValidateVolumeCapabilitiesResponsePBImpl + extends ValidateVolumeCapabilitiesResponse { + + private CsiAdaptorProtos.ValidateVolumeCapabilitiesResponse.Builder builder; + + public ValidateVolumeCapabilitiesResponsePBImpl() { + this.builder = CsiAdaptorProtos.ValidateVolumeCapabilitiesResponse + .newBuilder(); + } + + public ValidateVolumeCapabilitiesResponsePBImpl( + CsiAdaptorProtos.ValidateVolumeCapabilitiesResponse response) { + this.builder = response.toBuilder(); + } + + @Override + public void setSupported(boolean supported) { + Preconditions.checkNotNull(builder); + this.builder.setSupported(supported); + } + + @Override + public boolean isSupported() { + Preconditions.checkNotNull(builder); + return builder.getSupported(); + } + + @Override + public void setResponseMessage(String message) { + Preconditions.checkNotNull(builder); + this.builder.setMessage(message); + } + + @Override + public String getResponseMessage() { + Preconditions.checkNotNull(builder); + return this.builder.getMessage(); + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + public CsiAdaptorProtos.ValidateVolumeCapabilitiesResponse getProto() { + Preconditions.checkNotNull(builder); + return builder.build(); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index f5493bc..e7a0e14 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -4037,4 +4037,17 @@ <name>yarn.node-attribute.fs-store.impl.class</name> <value>org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore</value> </property> + + <!-- CSI configuration --> + <property> + <description> + CSI driver adaptor addresses on a node manager. + This configuration will be loaded by the resource manager to initiate + a client for each adaptor in order to communicate with CSI drivers. + Note, these addresses should be mapped to the adaptor addresses which + runs the controller plugin. + </description> + <name>yarn.nodemanager.csi-driver-adaptor.addresses</name> + <value></value> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml index 41f5098..27d8452 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml @@ -84,6 +84,18 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-api</artifactId> + </dependency> + <dependency> <groupId>javax.annotation</groupId> <artifactId>javax.annotation-api</artifactId> <scope>compile</scope> http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/adaptor/CsiAdaptorProtocolService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/adaptor/CsiAdaptorProtocolService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/adaptor/CsiAdaptorProtocolService.java new file mode 100644 index 0000000..f94275f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/adaptor/CsiAdaptorProtocolService.java @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.csi.adaptor; + +import com.google.common.annotations.VisibleForTesting; +import csi.v0.Csi; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.api.CsiAdaptorProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoResponse; +import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesResponse; +import org.apache.hadoop.yarn.csi.client.CsiClient; +import org.apache.hadoop.yarn.csi.client.CsiClientImpl; +import org.apache.hadoop.yarn.csi.translator.ProtoTranslatorFactory; +import org.apache.hadoop.yarn.csi.utils.ConfigUtils; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.InetSocketAddress; + +/** + * This is a Hadoop RPC server, we uses the Hadoop RPC framework here + * because we need to stick to the security model current Hadoop supports. + */ +public class CsiAdaptorProtocolService extends AbstractService + implements CsiAdaptorProtocol { + + private static final Logger LOG = + LoggerFactory.getLogger(CsiAdaptorProtocolService.class); + + private Server server; + private InetSocketAddress adaptorServiceAddress; + private CsiClient csiClient; + private String csiDriverName; + + public CsiAdaptorProtocolService(String driverName, + String domainSocketPath) { + super(CsiAdaptorProtocolService.class.getName()); + this.csiClient = new CsiClientImpl(domainSocketPath); + this.csiDriverName = driverName; + } + + @VisibleForTesting + public void setCsiClient(CsiClient client) { + this.csiClient = client; + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + adaptorServiceAddress = ConfigUtils + .getCsiAdaptorAddressForDriver(csiDriverName, conf); + super.serviceInit(conf); + } + + @Override + protected void serviceStart() throws Exception { + Configuration conf = getConfig(); + YarnRPC rpc = YarnRPC.create(conf); + this.server = rpc.getServer( + CsiAdaptorProtocol.class, + this, adaptorServiceAddress, conf, null, 1); + this.server.start(); + LOG.info("{} started, listening on address: {}", + CsiAdaptorProtocolService.class.getName(), + adaptorServiceAddress.toString()); + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + if (this.server != null) { + this.server.stop(); + } + super.serviceStop(); + } + + @Override + public GetPluginInfoResponse getPluginInfo( + GetPluginInfoRequest request) throws YarnException, IOException { + Csi.GetPluginInfoResponse response = csiClient.getPluginInfo(); + return ProtoTranslatorFactory.getTranslator( + GetPluginInfoResponse.class, Csi.GetPluginInfoResponse.class) + .convertFrom(response); + } + + @Override + public ValidateVolumeCapabilitiesResponse validateVolumeCapacity( + ValidateVolumeCapabilitiesRequest request) throws YarnException, + IOException { + Csi.ValidateVolumeCapabilitiesRequest req = ProtoTranslatorFactory + .getTranslator(ValidateVolumeCapabilitiesRequest.class, + Csi.ValidateVolumeCapabilitiesRequest.class) + .convertTo(request); + Csi.ValidateVolumeCapabilitiesResponse response = + csiClient.validateVolumeCapabilities(req); + return ProtoTranslatorFactory.getTranslator( + ValidateVolumeCapabilitiesResponse.class, + Csi.ValidateVolumeCapabilitiesResponse.class) + .convertFrom(response); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/adaptor/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/adaptor/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/adaptor/package-info.java new file mode 100644 index 0000000..919aab5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/adaptor/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * This package contains CSI adaptor classes. + */ +package org.apache.hadoop.yarn.csi.adaptor; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/client/CsiClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/client/CsiClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/client/CsiClient.java index 5bb9ce9..d31c0c9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/client/CsiClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/client/CsiClient.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.csi.client; +import csi.v0.Csi; import csi.v0.Csi.GetPluginInfoResponse; import java.io.IOException; @@ -36,4 +37,7 @@ public interface CsiClient { * @throws IOException when unable to get plugin info from the driver. */ GetPluginInfoResponse getPluginInfo() throws IOException; + + Csi.ValidateVolumeCapabilitiesResponse validateVolumeCapabilities( + Csi.ValidateVolumeCapabilitiesRequest request) throws IOException; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/client/CsiClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/client/CsiClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/client/CsiClientImpl.java index 58dd292..5b3d2e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/client/CsiClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/client/CsiClientImpl.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.csi.client; +import csi.v0.Csi; import csi.v0.Csi.GetPluginInfoRequest; import csi.v0.Csi.GetPluginInfoResponse; import org.apache.hadoop.yarn.csi.utils.GrpcHelper; @@ -48,4 +49,14 @@ public class CsiClientImpl implements CsiClient { return client.createIdentityBlockingStub().getPluginInfo(request); } } + + @Override + public Csi.ValidateVolumeCapabilitiesResponse validateVolumeCapabilities( + Csi.ValidateVolumeCapabilitiesRequest request) throws IOException { + try (CsiGrpcClient client = CsiGrpcClient.newBuilder() + .setDomainSocketAddress(address).build()) { + return client.createControllerBlockingStub() + .validateVolumeCapabilities(request); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/GetPluginInfoResponseProtoTranslator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/GetPluginInfoResponseProtoTranslator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/GetPluginInfoResponseProtoTranslator.java new file mode 100644 index 0000000..c4f042e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/GetPluginInfoResponseProtoTranslator.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.csi.translator; + +import csi.v0.Csi; +import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoResponse; +import org.apache.hadoop.yarn.exceptions.YarnException; + +/** + * Protobuf message translator for GetPluginInfoResponse and + * Csi.GetPluginInfoResponse. + */ +public class GetPluginInfoResponseProtoTranslator implements + ProtoTranslator<GetPluginInfoResponse, Csi.GetPluginInfoResponse> { + + @Override public Csi.GetPluginInfoResponse convertTo( + GetPluginInfoResponse messageA) throws YarnException { + return Csi.GetPluginInfoResponse.newBuilder() + .setName(messageA.getDriverName()) + .setVendorVersion(messageA.getVersion()) + .build(); + } + + @Override public GetPluginInfoResponse convertFrom( + Csi.GetPluginInfoResponse messageB) throws YarnException { + return GetPluginInfoResponse.newInstance(messageB.getName(), + messageB.getVendorVersion()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ProtoTranslator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ProtoTranslator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ProtoTranslator.java new file mode 100644 index 0000000..93cb441 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ProtoTranslator.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.csi.translator; + +import org.apache.hadoop.yarn.exceptions.YarnException; + +/** + * ProtoTranslator converts a YARN side message to CSI proto message + * and vice versa. Each CSI proto message should have a corresponding + * YARN side message implementation, and a transformer to convert them + * one to the other. This layer helps we to hide CSI spec messages + * from YARN components. + * + * @param <A> YARN side internal messages + * @param <B> CSI proto messages + */ +public interface ProtoTranslator<A, B> { + + /** + * Convert message from type A to type B. + * @param messageA + * @return messageB + * @throws YarnException + */ + B convertTo(A messageA) throws YarnException; + + /** + * Convert message from type B to type A. + * @param messageB + * @return messageA + * @throws YarnException + */ + A convertFrom(B messageB) throws YarnException; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ProtoTranslatorFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ProtoTranslatorFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ProtoTranslatorFactory.java new file mode 100644 index 0000000..5eb76ff --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ProtoTranslatorFactory.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.csi.translator; + +import csi.v0.Csi; +import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesResponse; + +/** + * Factory class to get desired proto transformer instance. + */ +public final class ProtoTranslatorFactory { + + private ProtoTranslatorFactory() { + // hide constructor for the factory class + } + + /** + * Get a {@link ProtoTranslator} based on the given input message + * types. If the type is not supported, a IllegalArgumentException + * will be thrown. When adding more transformers to this factory class, + * note each transformer works exactly for one message to another + * (and vice versa). For each type of the message, make sure there is + * a corresponding unit test added, such as + * TestValidateVolumeCapabilitiesRequest. + * + * @param yarnProto yarn proto message + * @param csiProto CSI proto message + * @param <A> yarn proto message + * @param <B> CSI proto message + * @throws IllegalArgumentException + * when given types are not supported + * @return + * a proto message transformer that transforms + * YARN internal proto message to CSI + */ + public static <A, B> ProtoTranslator<A, B> getTranslator( + Class<A> yarnProto, Class<B> csiProto) { + if (yarnProto == ValidateVolumeCapabilitiesRequest.class + && csiProto == Csi.ValidateVolumeCapabilitiesRequest.class) { + return new ValidateVolumeCapabilitiesRequestProtoTranslator(); + } else if (yarnProto == ValidateVolumeCapabilitiesResponse.class + && csiProto == Csi.ValidateVolumeCapabilitiesResponse.class) { + return new ValidationVolumeCapabilitiesResponseProtoTranslator(); + } + throw new IllegalArgumentException("A problem is found while processing" + + " proto message translating. Unexpected message types," + + " no transformer is found can handle the transformation from type " + + yarnProto.getName() + " <-> " + csiProto.getName()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ValidateVolumeCapabilitiesRequestProtoTranslator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ValidateVolumeCapabilitiesRequestProtoTranslator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ValidateVolumeCapabilitiesRequestProtoTranslator.java new file mode 100644 index 0000000..a74c47a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ValidateVolumeCapabilitiesRequestProtoTranslator.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.csi.translator; + +import csi.v0.Csi; +import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest.VolumeCapability; +import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest.VolumeType; +import org.apache.hadoop.yarn.exceptions.YarnException; + +import java.util.ArrayList; +import java.util.List; + +/** + * Proto message translator for ValidateVolumeCapabilitiesRequest. + * @param <A> ValidateVolumeCapabilitiesRequest + * @param <B> Csi.ValidateVolumeCapabilitiesRequest + */ +public class ValidateVolumeCapabilitiesRequestProtoTranslator<A, B> + implements ProtoTranslator<ValidateVolumeCapabilitiesRequest, + Csi.ValidateVolumeCapabilitiesRequest> { + + @Override + public Csi.ValidateVolumeCapabilitiesRequest convertTo( + ValidateVolumeCapabilitiesRequest request) throws YarnException { + Csi.ValidateVolumeCapabilitiesRequest.Builder buidler = + Csi.ValidateVolumeCapabilitiesRequest.newBuilder(); + buidler.setVolumeId(request.getVolumeId()); + if (request.getVolumeCapabilities() != null + && request.getVolumeCapabilities().size() > 0) { + buidler.putAllVolumeAttributes(request.getVolumeAttributes()); + } + for (VolumeCapability cap : + request.getVolumeCapabilities()) { + Csi.VolumeCapability.AccessMode accessMode = + Csi.VolumeCapability.AccessMode.newBuilder() + .setModeValue(cap.getAccessMode().ordinal()) + .build(); + Csi.VolumeCapability.MountVolume mountVolume = + Csi.VolumeCapability.MountVolume.newBuilder() + .addAllMountFlags(cap.getMountFlags()) + .build(); + Csi.VolumeCapability capability = + Csi.VolumeCapability.newBuilder() + .setAccessMode(accessMode) + .setMount(mountVolume) + .build(); + buidler.addVolumeCapabilities(capability); + } + return buidler.build(); + } + + @Override + public ValidateVolumeCapabilitiesRequest convertFrom( + Csi.ValidateVolumeCapabilitiesRequest request) throws YarnException { + ValidateVolumeCapabilitiesRequest result = ValidateVolumeCapabilitiesRequest + .newInstance(request.getVolumeId(), request.getVolumeAttributesMap()); + for (Csi.VolumeCapability csiCap : + request.getVolumeCapabilitiesList()) { + ValidateVolumeCapabilitiesRequest.AccessMode mode = + ValidateVolumeCapabilitiesRequest.AccessMode + .valueOf(csiCap.getAccessMode().getMode().name()); + if (!csiCap.hasMount()) { + throw new YarnException("Invalid request," + + " mount is not found in the request."); + } + List<String> mountFlags = new ArrayList<>(); + for (int i=0; i<csiCap.getMount().getMountFlagsCount(); i++) { + mountFlags.add(csiCap.getMount().getMountFlags(i)); + } + VolumeCapability capability = new VolumeCapability(mode, + VolumeType.FILE_SYSTEM, mountFlags); + result.addVolumeCapability(capability); + } + return result; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ValidationVolumeCapabilitiesResponseProtoTranslator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ValidationVolumeCapabilitiesResponseProtoTranslator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ValidationVolumeCapabilitiesResponseProtoTranslator.java new file mode 100644 index 0000000..fd42712 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/ValidationVolumeCapabilitiesResponseProtoTranslator.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.csi.translator; + +import csi.v0.Csi; +import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesResponse; +import org.apache.hadoop.yarn.exceptions.YarnException; + +/** + * Proto message translator for ValidateVolumeCapabilitiesResponse. + * @param <A> ValidateVolumeCapabilitiesResponse + * @param <B> Csi.ValidateVolumeCapabilitiesResponse + */ +public class ValidationVolumeCapabilitiesResponseProtoTranslator<A, B> + implements ProtoTranslator<ValidateVolumeCapabilitiesResponse, + Csi.ValidateVolumeCapabilitiesResponse> { + + @Override + public Csi.ValidateVolumeCapabilitiesResponse convertTo( + ValidateVolumeCapabilitiesResponse response) throws YarnException { + return Csi.ValidateVolumeCapabilitiesResponse.newBuilder() + .setSupported(response.isSupported()) + .setMessage(response.getResponseMessage()) + .build(); + } + + @Override + public ValidateVolumeCapabilitiesResponse convertFrom( + Csi.ValidateVolumeCapabilitiesResponse response) throws YarnException { + return ValidateVolumeCapabilitiesResponse.newInstance( + response.getSupported(), response.getMessage()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/package-info.java new file mode 100644 index 0000000..c0964cd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/translator/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * This package contains classes for protocol translation between YARN and CSI. + */ +package org.apache.hadoop.yarn.csi.translator; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb14e06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/utils/ConfigUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/utils/ConfigUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/utils/ConfigUtils.java new file mode 100644 index 0000000..77e6955 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/main/java/org/apache/hadoop/yarn/csi/utils/ConfigUtils.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.csi.utils; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; + +import java.net.InetSocketAddress; + +/** + * Utility class to load configurations. + */ +public final class ConfigUtils { + + private ConfigUtils() { + // Hide constructor for utility class. + } + /** + * Resolve the CSI adaptor address for a CSI driver from configuration. + * Expected configuration property name is + * yarn.nodemanager.csi-driver-adaptor.${driverName}.address. + * @param driverName + * @param conf + * @return adaptor service address + * @throws YarnException + */ + public static InetSocketAddress getCsiAdaptorAddressForDriver( + String driverName, Configuration conf) throws YarnException { + String configName = YarnConfiguration.NM_CSI_ADAPTOR_PREFIX + + driverName + ".address"; + String errorMessage = "Failed to load CSI adaptor address for driver " + + driverName + ", configuration property " + configName + + " is not defined or invalid."; + try { + InetSocketAddress address = conf + .getSocketAddr(configName, null, -1); + if (address == null) { + throw new YarnException(errorMessage); + } + return address; + } catch (IllegalArgumentException e) { + throw new YarnException(errorMessage); + } + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
