http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Catalog/Capabilities/RAM.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Catalog/Capabilities/RAM.cs b/lang/cs/Org.Apache.REEF.Common/Catalog/Capabilities/RAM.cs new file mode 100644 index 0000000..94358e3 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Catalog/Capabilities/RAM.cs @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System.Globalization; + +namespace Org.Apache.REEF.Common.Catalog.Capabilities +{ + public class RAM : ICapability + { + private int _megaBytes; + + public RAM(int megaBytes) + { + _megaBytes = megaBytes; + } + + public int MegaBytes + { + get + { + return _megaBytes; + } + } + + public override string ToString() + { + return string.Format(CultureInfo.InvariantCulture, "RAM(in mega bytes) = [{0}]", MegaBytes); + } + + public override int GetHashCode() + { + return MegaBytes.GetHashCode(); + } + } +}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Catalog/INodeDescriptor.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Catalog/INodeDescriptor.cs b/lang/cs/Org.Apache.REEF.Common/Catalog/INodeDescriptor.cs new file mode 100644 index 0000000..e6222a9 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Catalog/INodeDescriptor.cs @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System.Net; +using Org.Apache.REEF.Common.Catalog.Capabilities; + +namespace Org.Apache.REEF.Common.Catalog +{ + public interface INodeDescriptor + { + IPEndPoint InetSocketAddress { get; set; } + + string HostName { get; set; } + + CPU Cpu { get; set; } + + RAM Ram { get; set; } + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Catalog/IRackDescriptor.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Catalog/IRackDescriptor.cs b/lang/cs/Org.Apache.REEF.Common/Catalog/IRackDescriptor.cs new file mode 100644 index 0000000..5e6bb32 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Catalog/IRackDescriptor.cs @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +namespace Org.Apache.REEF.Common.Catalog +{ + public interface IRackDescriptor : IResourceCatalog + { + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Catalog/IResourceCatalog.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Catalog/IResourceCatalog.cs b/lang/cs/Org.Apache.REEF.Common/Catalog/IResourceCatalog.cs new file mode 100644 index 0000000..dd2adf3 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Catalog/IResourceCatalog.cs @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System.Collections.Generic; +using Org.Apache.REEF.Common.Catalog.Capabilities; + +namespace Org.Apache.REEF.Common.Catalog +{ + public interface IResourceCatalog + { + string Name { get; set; } + + ICollection<ICapability> Capabilities { get; set; } + + ICollection<INodeDescriptor> Nodes { get; set; } + + ICollection<IRackDescriptor> Racks { get; set; } + + INodeDescriptor GetNode(string nodeId); + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Catalog/NodeDescriptorImpl.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Catalog/NodeDescriptorImpl.cs b/lang/cs/Org.Apache.REEF.Common/Catalog/NodeDescriptorImpl.cs new file mode 100644 index 0000000..0e2cb90 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Catalog/NodeDescriptorImpl.cs @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System.Collections.Generic; +using System.Net; +using Org.Apache.REEF.Common.Catalog.Capabilities; + +namespace Org.Apache.REEF.Common.Catalog +{ + public class NodeDescriptorImpl : INodeDescriptor + { + private RackDescriptorImpl _rack; + + private string _id; + + private IPEndPoint _address; + + private RAM _ram; + + private IList<ICapability> _capabilities; + + public NodeDescriptorImpl() + { + } + + public NodeDescriptorImpl(string id, IPEndPoint addresss, RackDescriptorImpl rack, RAM ram) + { + _id = id; + _address = addresss; + _rack = rack; + _ram = ram; + _capabilities = new List<ICapability>(); + _rack.AddNodeDescriptor(this); + } + + public RackDescriptorImpl Rack + { + get + { + return _rack; + } + } + + public string Id + { + get + { + return _id; + } + } + + public string HostName { get; set; } + + public CPU Cpu + { + get + { + return new CPU(1); + } + + set + { + } + } + + public RAM Ram + { + get + { + return _ram; + } + + set + { + _ram = value; + } + } + + public IList<ICapability> Capabilities + { + get + { + return _capabilities; + } + } + + public IPEndPoint InetSocketAddress + { + get + { + return _address; + } + + set + { + _address = value; + } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Catalog/RackDescriptorImpl.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Catalog/RackDescriptorImpl.cs b/lang/cs/Org.Apache.REEF.Common/Catalog/RackDescriptorImpl.cs new file mode 100644 index 0000000..02a3715 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Catalog/RackDescriptorImpl.cs @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using System.Collections.Generic; +using System.Text; +using Org.Apache.REEF.Common.Catalog.Capabilities; + +namespace Org.Apache.REEF.Common.Catalog +{ + public class RackDescriptorImpl : IRackDescriptor + { + public RackDescriptorImpl(string name) + { + Name = name; + Capabilities = new List<ICapability>(); + Nodes = new List<INodeDescriptor>(); + } + + public string Name { get; set; } + + public ICollection<ICapability> Capabilities { get; set; } + + public ICollection<INodeDescriptor> Nodes { get; set; } + + public ICollection<IRackDescriptor> Racks { get; set; } + + public INodeDescriptor GetNode(string nodeId) + { + throw new NotImplementedException(); + } + + public void AddNodeDescriptor(NodeDescriptorImpl node) + { + Nodes.Add(node); + } + + public override string ToString() + { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.Append("Rack " + Name); + foreach (INodeDescriptor nodeDescriptor in Nodes) + { + stringBuilder.Append(Environment.NewLine + nodeDescriptor); + } + return stringBuilder.ToString(); + } + + public override int GetHashCode() + { + return Name.GetHashCode(); + } + + public override bool Equals(object obj) + { + return base.Equals(obj); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Catalog/ResourceCatalogImpl.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Catalog/ResourceCatalogImpl.cs b/lang/cs/Org.Apache.REEF.Common/Catalog/ResourceCatalogImpl.cs new file mode 100644 index 0000000..22947ca --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Catalog/ResourceCatalogImpl.cs @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using Org.Apache.REEF.Utilities.Logging; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Net; +using System.Text; +using Org.Apache.REEF.Common.Catalog.Capabilities; +using Org.Apache.REEF.Common.Protobuf.ReefProtocol; + +namespace Org.Apache.REEF.Common.Catalog +{ + public class ResourceCatalogImpl : IResourceCatalog + { + private static readonly Logger LOGGER = Logger.GetLogger(typeof(ResourceCatalogImpl)); + + private static string defaultRackName = "default-rack"; + + private Dictionary<string, RackDescriptorImpl> _racks = new Dictionary<string, RackDescriptorImpl>(); + + private Dictionary<string, NodeDescriptorImpl> _nodes = new Dictionary<string, NodeDescriptorImpl>(); + + public string Name { get; set; } + + public ICollection<ICapability> Capabilities { get; set; } + + public ICollection<INodeDescriptor> Nodes { get; set; } + + public ICollection<IRackDescriptor> Racks { get; set; } + + public INodeDescriptor GetNode(string nodeId) + { + return _nodes[nodeId]; + } + + public void Handle(NodeDescriptorProto node) + { + string rackName = node.rack_name == null ? node.rack_name : defaultRackName; + string message = string.Format( + CultureInfo.InvariantCulture, + "Catalog new node: id[{0}], rack[{1}], host[{2}], port[{3}], memory[{4}]", + node.identifier, + rackName, + node.host_name, + node.port, + node.memory_size); + LOGGER.Log(Level.Info, message); + if (!string.IsNullOrWhiteSpace(rackName) && !_racks.ContainsKey(rackName)) + { + RackDescriptorImpl newRack = new RackDescriptorImpl(rackName); + _racks.Add(rackName, newRack); + } + RackDescriptorImpl rack = _racks[rackName]; + IPAddress ipAddress = null; + IPAddress.TryParse(node.host_name, out ipAddress); + if (ipAddress == null) + { + Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(new ArgumentException("cannot parse host ipaddress: " + node.host_name), LOGGER); + } + IPEndPoint ipEndPoint = new IPEndPoint(ipAddress, node.port); + RAM ram = new RAM(node.memory_size); + NodeDescriptorImpl nodeDescriptor = new NodeDescriptorImpl(node.identifier, ipEndPoint, rack, ram); + _nodes.Add(nodeDescriptor.Id, nodeDescriptor); + } + + public override string ToString() + { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.Append("=== Resource Catalog ==="); + foreach (IRackDescriptor rackDescriptor in Racks) + { + stringBuilder.Append(Environment.NewLine + rackDescriptor); + } + return stringBuilder.ToString(); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/ClientJobStatusHandler.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/ClientJobStatusHandler.cs b/lang/cs/Org.Apache.REEF.Common/ClientJobStatusHandler.cs index 84e916f..158161c 100644 --- a/lang/cs/Org.Apache.REEF.Common/ClientJobStatusHandler.cs +++ b/lang/cs/Org.Apache.REEF.Common/ClientJobStatusHandler.cs @@ -17,15 +17,13 @@ * under the License. */ -using Org.Apache.REEF.Common.Client; -using Org.Apache.REEF.Common.ProtoBuf.ClienRuntimeProto; -using Org.Apache.REEF.Common.ProtoBuf.ReefProtocol; -using Org.Apache.REEF.Common.ProtoBuf.ReefServiceProto; using Org.Apache.REEF.Utilities; using Org.Apache.REEF.Utilities.Logging; using Org.Apache.REEF.Wake.Remote; using Org.Apache.REEF.Wake.Time; using System; +using Org.Apache.REEF.Common.Protobuf.ReefProtocol; +using Org.Apache.REEF.Wake.Time.Event; namespace Org.Apache.REEF.Common { http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Context/ContextMessage.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Context/ContextMessage.cs b/lang/cs/Org.Apache.REEF.Common/Context/ContextMessage.cs new file mode 100644 index 0000000..7f9b226 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Context/ContextMessage.cs @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using Org.Apache.REEF.Utilities.Logging; + +namespace Org.Apache.REEF.Common.Context +{ + public class ContextMessage + { + private static readonly Logger LOGGER = Logger.GetLogger(typeof(ContextMessage)); + private readonly string _messageSourcId; + private readonly byte[] _bytes; + + private ContextMessage(string messageSourceId, byte[] bytes) + { + _messageSourcId = messageSourceId; + _bytes = bytes; + } + + public string MessageSourceId + { + get { return _messageSourcId; } + } + + public byte[] Bytes + { + get { return _bytes; } + } + + /// <summary> + /// construt a new new ContextMessage with the given content. + /// </summary> + /// <param name="messageSourceId">The message's sourceID. This will be accessible in the Driver for routing.</param> + /// <param name="bytes">The actual content of the message, serialized into a byte[]</param> + /// <returns>new ContextMessage with the given content.</returns> + public static ContextMessage From(string messageSourceId, byte[] bytes) + { + if (string.IsNullOrEmpty(messageSourceId)) + { + Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(new ArgumentNullException("messageSourceId"), LOGGER); + } + if (bytes == null) + { + Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(new ArgumentNullException("bytes"), LOGGER); + } + return new ContextMessage(messageSourceId, bytes); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Context/IContextMessage.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Context/IContextMessage.cs b/lang/cs/Org.Apache.REEF.Common/Context/IContextMessage.cs new file mode 100644 index 0000000..7d7a298 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Context/IContextMessage.cs @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using Org.Apache.REEF.Utilities; + +namespace Org.Apache.REEF.Common.Context +{ + public interface IContextMessage : IMessage, IIdentifiable + { + string MessageSourceId { get; } + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Context/IContextMessageHandler.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Context/IContextMessageHandler.cs b/lang/cs/Org.Apache.REEF.Common/Context/IContextMessageHandler.cs new file mode 100644 index 0000000..044d0af --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Context/IContextMessageHandler.cs @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; + +namespace Org.Apache.REEF.Common.Context +{ + public interface IContextMessageHandler : IObserver<byte[]> + { + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Context/IContextMessageSource.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Context/IContextMessageSource.cs b/lang/cs/Org.Apache.REEF.Common/Context/IContextMessageSource.cs new file mode 100644 index 0000000..d1eb08c --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Context/IContextMessageSource.cs @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using Org.Apache.REEF.Utilities; + +namespace Org.Apache.REEF.Common.Context +{ + public interface IContextMessageSource + { + Optional<ContextMessage> Message { get; set; } + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Evaluator/DefaultLocalHttpDriverConnection.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Evaluator/DefaultLocalHttpDriverConnection.cs b/lang/cs/Org.Apache.REEF.Common/Evaluator/DefaultLocalHttpDriverConnection.cs new file mode 100644 index 0000000..26049e6 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Evaluator/DefaultLocalHttpDriverConnection.cs @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using Org.Apache.REEF.Tang.Annotations; +using System; + +namespace Org.Apache.REEF.Common.Evaluator +{ + public class DefaultLocalHttpDriverConnection : IDriverConnection + { + private readonly Uri _queryUri; + + [Inject] + public DefaultLocalHttpDriverConnection() + { + _queryUri = new Uri( + string.Concat( + Constants.LocalHttpEndpointBaseUri, + Constants.HttpReefUriSpecification, + Constants.HttpDriverUriTarget)); + } + + public DriverInformation GetDriverInformation(string applicationId) + { + // application id not needed for local runtime + return DriverInformation.GetDriverInformationFromHttp(_queryUri); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Evaluator/DefaultYarnClusterHttpDriverConnection.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Evaluator/DefaultYarnClusterHttpDriverConnection.cs b/lang/cs/Org.Apache.REEF.Common/Evaluator/DefaultYarnClusterHttpDriverConnection.cs new file mode 100644 index 0000000..e0076e7 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Evaluator/DefaultYarnClusterHttpDriverConnection.cs @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using Org.Apache.REEF.Tang.Annotations; +using System; + +namespace Org.Apache.REEF.Common.Evaluator +{ + public class DefaultYarnClusterHttpDriverConnection : IDriverConnection + { + [Inject] + public DefaultYarnClusterHttpDriverConnection() + { + } + + public DriverInformation GetDriverInformation(string applicationId) + { + // e.g., http://headnodehost:9014/proxy/application_1407519727821_0012/reef/v1/driver + Uri queryUri = new Uri( + string.Concat( + Constants.HDInsightClusterHttpEndpointBaseUri, + applicationId, + Constants.HttpReefUriSpecification, + Constants.HttpDriverUriTarget)); + return DriverInformation.GetDriverInformationFromHttp(queryUri); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Evaluator/DefaultYarnOneBoxHttpDriverConnection.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Evaluator/DefaultYarnOneBoxHttpDriverConnection.cs b/lang/cs/Org.Apache.REEF.Common/Evaluator/DefaultYarnOneBoxHttpDriverConnection.cs new file mode 100644 index 0000000..9a4974c --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Evaluator/DefaultYarnOneBoxHttpDriverConnection.cs @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using Org.Apache.REEF.Tang.Annotations; +using System; +using System.Globalization; + +namespace Org.Apache.REEF.Common.Evaluator +{ + public class DefaultYarnOneBoxHttpDriverConnection : IDriverConnection + { + [Inject] + public DefaultYarnOneBoxHttpDriverConnection() + { + } + + public DriverInformation GetDriverInformation(string applicationId) + { + // e.g., http://yingdac1:8088/proxy/application_1407519727821_0012/reef/v1/driver + string oneBoxHost = string.Format(CultureInfo.InvariantCulture, "http://{0}:8088/proxy/";, Environment.MachineName); + Uri queryUri = new Uri( + string.Concat( + oneBoxHost, + applicationId, + Constants.HttpReefUriSpecification, + Constants.HttpDriverUriTarget)); + return DriverInformation.GetDriverInformationFromHttp(queryUri); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Evaluator/DriverInformation.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Evaluator/DriverInformation.cs b/lang/cs/Org.Apache.REEF.Common/Evaluator/DriverInformation.cs new file mode 100644 index 0000000..055784e --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Evaluator/DriverInformation.cs @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using Org.Apache.REEF.Common.Avro; +using Org.Apache.REEF.Utilities.Logging; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net; +using System.Text; + +namespace Org.Apache.REEF.Common.Evaluator +{ + public class DriverInformation + { + private static readonly Logger LOGGER = Logger.GetLogger(typeof(DriverInformation)); + + private string _rid; + + private string _startTime; + + private string _nameServerId; + + private IList<AvroReefServiceInfo> _services; + + public DriverInformation(string rid, string startTime, IList<AvroReefServiceInfo> services) + { + _rid = rid; + _startTime = startTime; + _services = services; + + if (_services == null) + { + LOGGER.Log(Level.Warning, "no services information from driver."); + } + else + { + AvroReefServiceInfo nameServerInfo = + _services.FirstOrDefault( + s => s.serviceName.Equals(Constants.NameServerServiceName, StringComparison.OrdinalIgnoreCase)); + if (nameServerInfo != null) + { + _nameServerId = nameServerInfo.serviceInfo; + } + } + } + + public string DriverRemoteIdentifier + { + get + { + return _rid; + } + } + + public string DriverStartTime + { + get + { + return _startTime; + } + } + + public string NameServerId + { + get + { + return _nameServerId; + } + } + + public static DriverInformation GetDriverInformationFromHttp(Uri queryUri) + { + HttpWebRequest request = (HttpWebRequest)WebRequest.Create(queryUri); + request.AllowAutoRedirect = false; + request.KeepAlive = false; + request.ContentType = "text/html"; + + string driverInfomation; + AvroDriverInfo info = null; + try + { + using (HttpWebResponse webResponse = (HttpWebResponse)request.GetResponse()) + { + Stream stream = webResponse.GetResponseStream(); + if (stream == null) + { + return null; + } + using (StreamReader streamReader = new StreamReader(stream, Encoding.UTF8)) + { + driverInfomation = streamReader.ReadToEnd(); + LOGGER.Log(Level.Verbose, "Http response line: " + driverInfomation); + info = AvroJsonSerializer<AvroDriverInfo>.FromString(driverInfomation); + } + } + } + catch (WebException) + { + LOGGER.Log(Level.Warning, string.Format(CultureInfo.InvariantCulture, "In RECOVERY mode, cannot connect to [{0}] for driver information, will try again later.", queryUri)); + return null; + } + catch (Exception e) + { + Org.Apache.REEF.Utilities.Diagnostics.Exceptions.CaughtAndThrow(e, Level.Error, string.Format(CultureInfo.InvariantCulture, "Cannot read content from {0}.", queryUri), LOGGER); + } + + if (info != null) + { + LOGGER.Log( + Level.Verbose, + string.Format(CultureInfo.InvariantCulture, "Driver information extracted with remote identier [{0}], start time [{1}], and servics [{2}]", info.remoteId, info.startTime, info.services)); + return new DriverInformation(info.remoteId, info.startTime, info.services); + } + return null; + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Evaluator/EvaluatorOperationState.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Evaluator/EvaluatorOperationState.cs b/lang/cs/Org.Apache.REEF.Common/Evaluator/EvaluatorOperationState.cs new file mode 100644 index 0000000..22ffa67 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Evaluator/EvaluatorOperationState.cs @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +namespace Org.Apache.REEF.Common.Evaluator +{ + public enum EvaluatorOperationState + { + /// <summary> + /// default state + /// </summary> + UNINITIATED = 0, + + /// <summary> + /// normal operational state + /// </summary> + OPERATIONAL = 1, + + /// <summary> + /// in the process of recovering + /// </summary> + RECOVERY = 2 + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Evaluator/EvaluatorRuntimeState.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Evaluator/EvaluatorRuntimeState.cs b/lang/cs/Org.Apache.REEF.Common/Evaluator/EvaluatorRuntimeState.cs new file mode 100644 index 0000000..cc879bb --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Evaluator/EvaluatorRuntimeState.cs @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +namespace Org.Apache.REEF.Common.Evaluator +{ + public enum EvaluatorRuntimeState + { + /// <summary> + /// default state + /// </summary> + UNINITIATED = 0, + + /// <summary> + /// normal operational state + /// </summary> + RUNNING = 1, + + /// <summary> + /// in the process of recovering + /// </summary> + RECOVERY = 2 + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Evaluator/EvaluatorType.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Evaluator/EvaluatorType.cs b/lang/cs/Org.Apache.REEF.Common/Evaluator/EvaluatorType.cs new file mode 100644 index 0000000..4269dd2 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Evaluator/EvaluatorType.cs @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +namespace Org.Apache.REEF.Common.Evaluator +{ + public enum EvaluatorType + { + /// <summary> + /// default type + /// </summary> + UNDECIDED = 0, + + /// <summary> + /// Indicates an Evaluator that runs on the JVM + /// </summary> + JVM = 1, + + /// <summary> + /// Indicates an Evaluator that runs on the CLR + /// </summary> + CLR = 2 + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Evaluator/IDriverConnection.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Evaluator/IDriverConnection.cs b/lang/cs/Org.Apache.REEF.Common/Evaluator/IDriverConnection.cs new file mode 100644 index 0000000..10c6d6e --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Evaluator/IDriverConnection.cs @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +namespace Org.Apache.REEF.Common.Evaluator +{ + public interface IDriverConnection + { + DriverInformation GetDriverInformation(string applicationId); + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Events/IContextStart.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Events/IContextStart.cs b/lang/cs/Org.Apache.REEF.Common/Events/IContextStart.cs new file mode 100644 index 0000000..924f2c4 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Events/IContextStart.cs @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +namespace Org.Apache.REEF.Common.Events +{ + public interface IContextStart + { + string Id { get; set; } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Events/IContextStop.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Events/IContextStop.cs b/lang/cs/Org.Apache.REEF.Common/Events/IContextStop.cs new file mode 100644 index 0000000..bdc5a73 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Events/IContextStop.cs @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +namespace Org.Apache.REEF.Common.Events +{ + public interface IContextStop + { + string Id { get; set; } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Exceptions/EvaluatorException.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Exceptions/EvaluatorException.cs b/lang/cs/Org.Apache.REEF.Common/Exceptions/EvaluatorException.cs new file mode 100644 index 0000000..d1bb558 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Exceptions/EvaluatorException.cs @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using Org.Apache.REEF.Utilities; +using System; +using Org.Apache.REEF.Common.Tasks; + +namespace Org.Apache.REEF.Common.Exceptions +{ + public class EvaluatorException : System.Exception, IIdentifiable + { + private readonly string _evaluatorId; + + public EvaluatorException(string evaluatorId) + { + _evaluatorId = evaluatorId; + RunningTask = null; + } + + public EvaluatorException(string evaluatorId, string message, System.Exception cause) + : base(message, cause) + { + _evaluatorId = evaluatorId; + RunningTask = null; + } + + public EvaluatorException(string evaluatorId, string message) + : this(evaluatorId, message, (IRunningTask)null) + { + } + + public EvaluatorException(string evaluatorId, string message, IRunningTask runningTask) + : base(message) + { + _evaluatorId = evaluatorId; + RunningTask = runningTask; + } + + public EvaluatorException(string evaluatorId, System.Exception cause) + : this(evaluatorId, cause, null) + { + } + + public EvaluatorException(string evaluatorId, Exception cause, IRunningTask runningTask) + : base(string.Empty, cause) + { + _evaluatorId = evaluatorId; + RunningTask = runningTask; + } + + public IRunningTask RunningTask { get; set; } + + public string Id + { + get { return _evaluatorId; } + set { } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Exceptions/JobException.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Exceptions/JobException.cs b/lang/cs/Org.Apache.REEF.Common/Exceptions/JobException.cs new file mode 100644 index 0000000..e379b92 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Exceptions/JobException.cs @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using Org.Apache.REEF.Utilities; + +namespace Org.Apache.REEF.Common.Exceptions +{ + public class JobException : Exception, IIdentifiable + { + private string _jobId; + + public JobException(string jobId) + { + _jobId = jobId; + } + + public JobException(string jobId, string message, Exception cause) + : base(message, cause) + { + _jobId = jobId; + } + + public JobException(string jobId, string message) + : base(message) + { + _jobId = jobId; + } + + public JobException(string jobId, Exception cause) + : base(string.Empty, cause) + { + _jobId = jobId; + } + + public string Id + { + get { return _jobId; } + set { } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/FailedRuntime.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/FailedRuntime.cs b/lang/cs/Org.Apache.REEF.Common/FailedRuntime.cs index 078feb8..2ac6781 100644 --- a/lang/cs/Org.Apache.REEF.Common/FailedRuntime.cs +++ b/lang/cs/Org.Apache.REEF.Common/FailedRuntime.cs @@ -18,9 +18,9 @@ */ using Org.Apache.REEF.Common.Api; -using Org.Apache.REEF.Common.ProtoBuf.ReefServiceProto; using Org.Apache.REEF.Utilities; using System; +using Org.Apache.REEF.Common.Protobuf.ReefProtocol; namespace Org.Apache.REEF.Common { http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/IJobMessageObserver.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/IJobMessageObserver.cs b/lang/cs/Org.Apache.REEF.Common/IJobMessageObserver.cs index 0de5039..b1ba4ed 100644 --- a/lang/cs/Org.Apache.REEF.Common/IJobMessageObserver.cs +++ b/lang/cs/Org.Apache.REEF.Common/IJobMessageObserver.cs @@ -19,7 +19,7 @@ using System; -namespace Org.Apache.REEF.Common.Client +namespace Org.Apache.REEF.Common { /// <summary> /// The driver uses this interface to communicate with the job client. http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Io/INameClient.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Io/INameClient.cs b/lang/cs/Org.Apache.REEF.Common/Io/INameClient.cs new file mode 100644 index 0000000..8fc1f2d --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Io/INameClient.cs @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using System.Collections.Generic; +using System.Net; + +namespace Org.Apache.REEF.Common.Io +{ + /// <summary> + /// Client for the Reef name service. + /// Used to register, unregister, and lookup IP Addresses of known hosts. + /// </summary> + public interface INameClient : IDisposable + { + /// <summary> + /// Registers the identifier with the NameService. + /// Overwrites the previous mapping if the identifier has already + /// been registered. + /// </summary> + /// <param name="id">The key used to map the remote endpoint</param> + /// <param name="endpoint">The endpoint to map</param> + void Register(string id, IPEndPoint endpoint); + + /// <summary> + /// Unregisters the remote identifier with the NameService + /// </summary> + /// <param name="id">The identifier to unregister</param> + void Unregister(string id); + + /// <summary> + /// Looks up the IPEndpoint for the registered identifier. + /// </summary> + /// <param name="id">The identifier to look up</param> + /// <returns>The mapped IPEndpoint for the identifier, or null if + /// the identifier has not been registered with the NameService</returns> + IPEndPoint Lookup(string id); + + /// <summary> + /// Looks up the IPEndpoint for each of the registered identifiers in the list. + /// </summary> + /// <param name="ids">The list of identifiers to look up</param> + /// <returns>The list of NameAssignments representing a pair of identifer + /// and mapped IPEndpoint for that identifier. If any of the requested identifiers + /// are not registered with the NameService, their corresponding NameAssignment + /// IPEndpoint value will be null.</returns> + List<NameAssignment> Lookup(List<string> ids); + + /// <summary> + /// Restart the name client in case of failure. + /// </summary> + /// <param name="serverEndpoint">The new server endpoint to connect to</param> + void Restart(IPEndPoint serverEndpoint); + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Io/NameAssignment.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Io/NameAssignment.cs b/lang/cs/Org.Apache.REEF.Common/Io/NameAssignment.cs new file mode 100644 index 0000000..bfd97a2 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Io/NameAssignment.cs @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System.Net; +using System.Net.Sockets; + +namespace Org.Apache.REEF.Common.Io +{ + /// <summary> + /// Tuple containing the string identifier and IPEndpoint. + /// Used by NameServer and NameClient + /// </summary> + public class NameAssignment + { + public NameAssignment(string id, IPEndPoint endpoint) + { + Identifier = id; + Endpoint = endpoint; + } + + public NameAssignment(string id, string address, int port) + { + Identifier = id; + IPAddress ipAddress; + if (!IPAddress.TryParse(address, out ipAddress)) + { + IPHostEntry hostEntry = Dns.GetHostEntry(address); + foreach (var ip in hostEntry.AddressList) + { + if (ip.AddressFamily == AddressFamily.InterNetwork) + { + ipAddress = ip; + break; + } + } + } + Endpoint = new IPEndPoint(ipAddress, port); + } + + public string Identifier { get; set; } + + public IPEndPoint Endpoint { get; set; } + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Io/NamingConfiguration.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Io/NamingConfiguration.cs b/lang/cs/Org.Apache.REEF.Common/Io/NamingConfiguration.cs new file mode 100644 index 0000000..0f922dd --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Io/NamingConfiguration.cs @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System.Diagnostics.CodeAnalysis; +using Org.Apache.REEF.Tang.Formats; +using Org.Apache.REEF.Tang.Util; + +namespace Org.Apache.REEF.Common.Io +{ + public class NamingConfiguration : ConfigurationModuleBuilder + { + [SuppressMessage("Microsoft.Security", "CA2104:Do not declare read only mutable reference types", Justification = "not applicable")] + public static readonly RequiredParameter<string> NameServerAddress = new RequiredParameter<string>(); + + [SuppressMessage("Microsoft.Security", "CA2104:Do not declare read only mutable reference types", Justification = "not applicable")] + public static readonly RequiredParameter<int> NameServerPort = new RequiredParameter<int>(); + + public static ConfigurationModule ConfigurationModule + { + get + { + return new NamingConfiguration() + .BindNamedParameter(GenericType<NamingConfigurationOptions.NameServerAddress>.Class, NameServerAddress) + .BindNamedParameter(GenericType<NamingConfigurationOptions.NameServerPort>.Class, NameServerPort) + .Build(); + } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Io/NamingConfigurationOptions.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Io/NamingConfigurationOptions.cs b/lang/cs/Org.Apache.REEF.Common/Io/NamingConfigurationOptions.cs new file mode 100644 index 0000000..8e7e91d --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Io/NamingConfigurationOptions.cs @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using Org.Apache.REEF.Tang.Annotations; + +namespace Org.Apache.REEF.Common.Io +{ + public class NamingConfigurationOptions + { + [NamedParameter("IP address of NameServer")] + public class NameServerAddress : Name<string> + { + } + + [NamedParameter("Port of NameServer")] + public class NameServerPort : Name<int> + { + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Org.Apache.REEF.Common.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Org.Apache.REEF.Common.csproj b/lang/cs/Org.Apache.REEF.Common/Org.Apache.REEF.Common.csproj index 572af49..85e48b7 100644 --- a/lang/cs/Org.Apache.REEF.Common/Org.Apache.REEF.Common.csproj +++ b/lang/cs/Org.Apache.REEF.Common/Org.Apache.REEF.Common.csproj @@ -28,7 +28,7 @@ under the License. <TargetFrameworkVersion>v4.5</TargetFrameworkVersion> <FileAlignment>512</FileAlignment> <RestorePackages>true</RestorePackages> - <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\..\..</SolutionDir> + <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir> </PropertyGroup> <Import Project="$(SolutionDir)\build.props" /> <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' "> @@ -95,117 +95,114 @@ under the License. <Reference Include="System.Xml" /> </ItemGroup> <ItemGroup> - <Compile Include="api\AbstractFailure.cs" /> - <Compile Include="api\IAbstractFailure.cs" /> - <Compile Include="api\IFailure.cs" /> - <Compile Include="api\IResourceLaunchHandler.cs" /> - <Compile Include="api\IResourceReleaseHandler.cs" /> - <Compile Include="api\IResourceRequestHandler.cs" /> - <Compile Include="avro\AvroDriverInfo.cs" /> - <Compile Include="avro\AvroHttpRequest.cs" /> - <Compile Include="avro\AvroHttpSerializer.cs" /> - <Compile Include="avro\AvroJsonSerializer.cs" /> - <Compile Include="avro\AvroReefServiceInfo.cs" /> - <Compile Include="catalog\capabilities\CPU.cs" /> - <Compile Include="catalog\capabilities\ICapability.cs" /> - <Compile Include="catalog\capabilities\RAM.cs" /> - <Compile Include="catalog\INodeDescriptor.cs" /> - <Compile Include="catalog\IRackDescriptor.cs" /> - <Compile Include="catalog\IResourceCatalog.cs" /> - <Compile Include="catalog\NodeDescriptorImpl.cs" /> - <Compile Include="catalog\RackDescriptorImpl.cs" /> - <Compile Include="catalog\ResourceCatalogImpl.cs" /> + <Compile Include="Api\AbstractFailure.cs" /> + <Compile Include="Api\IAbstractFailure.cs" /> + <Compile Include="Api\IFailure.cs" /> + <Compile Include="Api\IResourceLaunchHandler.cs" /> + <Compile Include="Api\IResourceReleaseHandler.cs" /> + <Compile Include="Api\IResourceRequestHandler.cs" /> + <Compile Include="Avro\AvroDriverInfo.cs" /> + <Compile Include="Avro\AvroHttpRequest.cs" /> + <Compile Include="Avro\AvroHttpSerializer.cs" /> + <Compile Include="Avro\AvroJsonSerializer.cs" /> + <Compile Include="Avro\AvroReefServiceInfo.cs" /> + <Compile Include="Catalog\Capabilities\CPU.cs" /> + <Compile Include="Catalog\Capabilities\ICapability.cs" /> + <Compile Include="Catalog\Capabilities\RAM.cs" /> + <Compile Include="Catalog\INodeDescriptor.cs" /> + <Compile Include="Catalog\IRackDescriptor.cs" /> + <Compile Include="Catalog\IResourceCatalog.cs" /> + <Compile Include="Catalog\NodeDescriptorImpl.cs" /> + <Compile Include="Catalog\RackDescriptorImpl.cs" /> + <Compile Include="Catalog\ResourceCatalogImpl.cs" /> <Compile Include="ClientJobStatusHandler.cs" /> <Compile Include="Constants.cs" /> - <Compile Include="context\ContextMessage.cs" /> - <Compile Include="context\IContextMessage.cs" /> - <Compile Include="context\IContextMessageHandler.cs" /> - <Compile Include="context\IContextMessageSource.cs" /> + <Compile Include="Context\ContextMessage.cs" /> + <Compile Include="Context\IContextMessage.cs" /> + <Compile Include="Context\IContextMessageHandler.cs" /> + <Compile Include="Context\IContextMessageSource.cs" /> <Compile Include="EvaluatorHeartBeatSanityChecker.cs" /> - <Compile Include="evaluator\DefaultLocalHttpDriverConnection.cs" /> - <Compile Include="evaluator\DefaultYarnClusterHttpDriverConnection.cs" /> - <Compile Include="evaluator\DefaultYarnOneBoxHttpDriverConnection.cs" /> - <Compile Include="evaluator\DriverInformation.cs" /> - <Compile Include="evaluator\EvaluatorOperationState.cs" /> - <Compile Include="evaluator\EvaluatorRuntimeState.cs" /> - <Compile Include="evaluator\EvaluatorType.cs" /> - <Compile Include="evaluator\IDriverConnection.cs" /> - <Compile Include="events\IContextStart.cs" /> - <Compile Include="events\IContextStop.cs" /> - <Compile Include="exceptions\EvaluatorException.cs" /> - <Compile Include="exceptions\JobException.cs" /> + <Compile Include="Evaluator\DefaultLocalHttpDriverConnection.cs" /> + <Compile Include="Evaluator\DefaultYarnClusterHttpDriverConnection.cs" /> + <Compile Include="Evaluator\DefaultYarnOneBoxHttpDriverConnection.cs" /> + <Compile Include="Evaluator\DriverInformation.cs" /> + <Compile Include="Evaluator\EvaluatorOperationState.cs" /> + <Compile Include="Evaluator\EvaluatorRuntimeState.cs" /> + <Compile Include="Evaluator\EvaluatorType.cs" /> + <Compile Include="Evaluator\IDriverConnection.cs" /> + <Compile Include="Events\IContextStart.cs" /> + <Compile Include="Events\IContextStop.cs" /> + <Compile Include="Exceptions\EvaluatorException.cs" /> + <Compile Include="Exceptions\JobException.cs" /> <Compile Include="FailedRuntime.cs" /> <Compile Include="IContextAndTaskSubmittable.cs" /> <Compile Include="IContextSubmittable.cs" /> <Compile Include="IJobMessageObserver.cs" /> - <Compile Include="io\INameClient.cs" /> - <Compile Include="io\NameAssignment.cs" /> - <Compile Include="io\NamingConfiguration.cs" /> - <Compile Include="io\NamingConfigurationOptions.cs" /> + <Compile Include="Io\INameClient.cs" /> + <Compile Include="Io\NameAssignment.cs" /> + <Compile Include="Io\NamingConfiguration.cs" /> + <Compile Include="Io\NamingConfigurationOptions.cs" /> <Compile Include="ITaskSubmittable.cs" /> <Compile Include="Properties\AssemblyInfo.cs" /> - <Compile Include="protobuf\cs\ClientRuntime.pb.cs" /> - <Compile Include="protobuf\cs\codec\EvaluatorHeartbeatProtoCodec.cs" /> - <Compile Include="protobuf\cs\codec\REEFMessageCodec.cs" /> - <Compile Include="protobuf\cs\DriverRuntime.pb.cs" /> - <Compile Include="protobuf\cs\EvaluatorRunTime.pb.cs" /> - <Compile Include="protobuf\cs\ReefProtocol.pb.cs" /> - <Compile Include="protobuf\cs\ReefService.pb.cs" /> - <Compile Include="protobuf\cs\Serializer.cs" /> - <Compile Include="runtime\evaluator\Constants.cs" /> - <Compile Include="runtime\evaluator\context\ContextClientCodeException.cs" /> - <Compile Include="runtime\evaluator\context\ContextConfiguration.cs" /> - <Compile Include="runtime\evaluator\context\ContextLifeCycle.cs" /> - <Compile Include="runtime\evaluator\context\ContextManager.cs" /> - <Compile Include="runtime\evaluator\context\ContextRuntime.cs" /> - <Compile Include="runtime\evaluator\context\ContextStartImpl.cs" /> - <Compile Include="runtime\evaluator\context\ContextStopImpl.cs" /> - <Compile Include="runtime\evaluator\context\RootContextLauncher.cs" /> - <Compile Include="runtime\evaluator\EvaluatorRuntime.cs" /> - <Compile Include="runtime\evaluator\EvaluatorSettings.cs" /> - <Compile Include="runtime\evaluator\HeartBeatManager.cs" /> - <Compile Include="runtime\evaluator\ReefMessageProtoObserver.cs" /> - <Compile Include="runtime\evaluator\task\CloseEventImpl.cs" /> - <Compile Include="runtime\evaluator\task\DriverMessageImpl.cs" /> - <Compile Include="runtime\evaluator\task\SuspendEventImpl.cs" /> - <Compile Include="runtime\evaluator\task\TaskClientCodeException.cs" /> - <Compile Include="runtime\evaluator\task\TaskLifeCycle.cs" /> - <Compile Include="runtime\evaluator\task\TaskRuntime.cs" /> - <Compile Include="runtime\evaluator\task\TaskStartImpl.cs" /> - <Compile Include="runtime\evaluator\task\TaskState.cs" /> - <Compile Include="runtime\evaluator\task\TaskStatus.cs" /> - <Compile Include="runtime\evaluator\task\TaskStopImpl.cs" /> - <Compile Include="runtime\evaluator\utils\EvaluatorConfigurations.cs" /> - <Compile Include="runtime\evaluator\utils\RemoteManager.cs" /> + <Compile Include="Protobuf\ReefProtocol\ClientRuntime.pb.cs" /> + <Compile Include="Protobuf\ReefProtocol\DriverRuntime.pb.cs" /> + <Compile Include="Protobuf\ReefProtocol\EvaluatorHeartbeatProtoCodec.cs" /> + <Compile Include="Protobuf\ReefProtocol\EvaluatorRunTime.pb.cs" /> + <Compile Include="Protobuf\ReefProtocol\REEFMessageCodec.cs" /> + <Compile Include="Protobuf\ReefProtocol\ReefProtocol.pb.cs" /> + <Compile Include="Protobuf\ReefProtocol\ReefService.pb.cs" /> + <Compile Include="Protobuf\ReefProtocol\Serializer.cs" /> + <Compile Include="Runtime\Evaluator\Constants.cs" /> + <Compile Include="Runtime\Evaluator\Context\ContextClientCodeException.cs" /> + <Compile Include="Runtime\Evaluator\Context\ContextConfiguration.cs" /> + <Compile Include="Runtime\Evaluator\Context\ContextLifeCycle.cs" /> + <Compile Include="Runtime\Evaluator\Context\ContextManager.cs" /> + <Compile Include="Runtime\Evaluator\Context\ContextRuntime.cs" /> + <Compile Include="Runtime\Evaluator\Context\ContextStartImpl.cs" /> + <Compile Include="Runtime\Evaluator\Context\ContextStopImpl.cs" /> + <Compile Include="Runtime\Evaluator\Context\RootContextLauncher.cs" /> + <Compile Include="Runtime\Evaluator\EvaluatorRuntime.cs" /> + <Compile Include="Runtime\Evaluator\EvaluatorSettings.cs" /> + <Compile Include="Runtime\Evaluator\HeartBeatManager.cs" /> + <Compile Include="Runtime\Evaluator\ReefMessageProtoObserver.cs" /> + <Compile Include="Runtime\Evaluator\Task\CloseEventImpl.cs" /> + <Compile Include="Runtime\Evaluator\Task\DriverMessageImpl.cs" /> + <Compile Include="Runtime\Evaluator\Task\SuspendEventImpl.cs" /> + <Compile Include="Runtime\Evaluator\Task\TaskClientCodeException.cs" /> + <Compile Include="Runtime\Evaluator\Task\TaskLifeCycle.cs" /> + <Compile Include="Runtime\Evaluator\Task\TaskRuntime.cs" /> + <Compile Include="Runtime\Evaluator\Task\TaskStartImpl.cs" /> + <Compile Include="Runtime\Evaluator\Task\TaskState.cs" /> + <Compile Include="Runtime\Evaluator\Task\TaskStatus.cs" /> + <Compile Include="Runtime\Evaluator\Task\TaskStopImpl.cs" /> + <Compile Include="Runtime\Evaluator\Utils\EvaluatorConfigurations.cs" /> + <Compile Include="Runtime\Evaluator\Utils\RemoteManager.cs" /> <Compile Include="runtime\MachineStatus.cs" /> - <Compile Include="services\IService.cs" /> - <Compile Include="services\ServiceConfiguration.cs" /> - <Compile Include="services\ServicesConfigurationOptions.cs" /> - <Compile Include="tasks\defaults\DefaultDriverMessageHandler.cs" /> - <Compile Include="tasks\defaults\DefaultTaskMessageSource.cs" /> - <Compile Include="tasks\events\ICloseEvent.cs" /> - <Compile Include="tasks\events\IDriverMessage.cs" /> - <Compile Include="tasks\events\ISuspendEvent.cs" /> - <Compile Include="tasks\events\ITaskStart.cs" /> - <Compile Include="tasks\events\ITaskStop.cs" /> - <Compile Include="tasks\IDriverMessageHandler.cs" /> - <Compile Include="tasks\IRunningTask.cs" /> - <Compile Include="tasks\ITask.cs" /> - <Compile Include="tasks\ITaskMessageSource.cs" /> - <Compile Include="tasks\TaskConfiguration.cs" /> - <Compile Include="tasks\TaskConfigurationOptions.cs" /> - <Compile Include="tasks\TaskMessage.cs" /> + <Compile Include="Services\IService.cs" /> + <Compile Include="Services\ServiceConfiguration.cs" /> + <Compile Include="Services\ServicesConfigurationOptions.cs" /> + <Compile Include="Tasks\Defaults\DefaultDriverMessageHandler.cs" /> + <Compile Include="Tasks\Defaults\DefaultTaskMessageSource.cs" /> + <Compile Include="Tasks\Events\ICloseEvent.cs" /> + <Compile Include="Tasks\Events\IDriverMessage.cs" /> + <Compile Include="Tasks\Events\ISuspendEvent.cs" /> + <Compile Include="Tasks\Events\ITaskStart.cs" /> + <Compile Include="Tasks\Events\ITaskStop.cs" /> + <Compile Include="Tasks\IDriverMessageHandler.cs" /> + <Compile Include="Tasks\IRunningTask.cs" /> + <Compile Include="Tasks\ITask.cs" /> + <Compile Include="Tasks\ITaskMessageSource.cs" /> + <Compile Include="Tasks\TaskConfiguration.cs" /> + <Compile Include="Tasks\TaskConfigurationOptions.cs" /> + <Compile Include="Tasks\TaskMessage.cs" /> </ItemGroup> <ItemGroup> <None Include="packages.config" /> - <None Include="protobuf\proto\client_runtime.proto" /> - <None Include="protobuf\proto\driver_runtime.proto" /> - <None Include="protobuf\proto\evaluator_runtime.proto" /> - <None Include="protobuf\proto\reef_protocol.proto" /> - <None Include="protobuf\proto\reef_service_protos.proto" /> - </ItemGroup> - <ItemGroup> - <Folder Include="protobuf\tools\" /> + <None Include="Protobuf\Proto\client_runtime.proto" /> + <None Include="Protobuf\Proto\driver_runtime.proto" /> + <None Include="Protobuf\Proto\evaluator_runtime.proto" /> + <None Include="Protobuf\Proto\reef_protocol.proto" /> + <None Include="Protobuf\Proto\reef_service_protos.proto" /> </ItemGroup> <ItemGroup> <ProjectReference Include="$(SolutionDir)\Org.Apache.REEF.Tang\Org.Apache.REEF.Tang.csproj"> @@ -216,7 +213,7 @@ under the License. <Project>{79e7f89a-1dfb-45e1-8d43-d71a954aeb98}</Project> <Name>Org.Apache.REEF.Utilities</Name> </ProjectReference> - <ProjectReference Include="..\Org.Apache.REEF.Wake\Org.Apache.REEF.Wake.csproj"> + <ProjectReference Include="$(SolutionDir)\Org.Apache.REEF.Wake\Org.Apache.REEF.Wake.csproj"> <Project>{cdfb3464-4041-42b1-9271-83af24cd5008}</Project> <Name>Org.Apache.REEF.Wake</Name> </ProjectReference> http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Protobuf/Proto/client_runtime.proto ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Protobuf/Proto/client_runtime.proto b/lang/cs/Org.Apache.REEF.Common/Protobuf/Proto/client_runtime.proto new file mode 100644 index 0000000..3d1f927 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Protobuf/Proto/client_runtime.proto @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +option java_package = "org.apache.reef.proto"; +option java_outer_classname = "ClientRuntimeProtocol"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; + +import "reef_service_protos.proto"; + +// Messages from REEF Client -> Driver Runtime + +message JobSubmissionProto { + required string identifier = 1; // the job identifier + required string remote_id = 2; // the remote identifier + required string configuration = 5; // the runtime configuration + required string user_name = 6; // the user name + + optional SIZE driver_size = 7; + optional int32 driver_memory = 8; + optional int32 priority = 9; + optional string queue = 10; + + repeated FileResourceProto global_file = 11; // files that should be placed on the driver and all subsequent evaluators + repeated FileResourceProto local_File = 12; // files that should be placed on the driver only + +} + +enum Signal { + SIG_TERMINATE = 1; + SIG_SUSPEND = 2; + SIG_RESUME = 3; +} + +message JobControlProto { + required string identifier = 1; + optional Signal signal = 2; + optional bytes message = 3; +} + + http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Protobuf/Proto/driver_runtime.proto ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Protobuf/Proto/driver_runtime.proto b/lang/cs/Org.Apache.REEF.Common/Protobuf/Proto/driver_runtime.proto new file mode 100644 index 0000000..2b21ac7 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Protobuf/Proto/driver_runtime.proto @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +option java_package = "org.apache.reef.proto"; +option java_outer_classname = "DriverRuntimeProtocol"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; + + +import "reef_service_protos.proto"; + +// Messages from Driver Runtime -> Driver Process + +message DriverProcessRegistrationProto { + required string remote_identifier = 1; +} + + +message NodeDescriptorProto { + required string identifier = 1; + required string host_name = 2; // e.g., IP address + required int32 port = 3; // e.g., IP port + required int32 memory_size = 4; + optional string rack_name = 5; // e.g., /default-rack +} + +message ResourceAllocationProto { + required string identifier = 1; // e.g., the container id, or the thread id + required int32 resource_memory = 2; // megabytes + required string node_id = 3; +} + +message ResourceStatusProto { + required string identifier = 1; + required State state = 2; + optional string diagnostics = 3; + optional int32 exit_code = 4; + optional bool is_from_previous_driver = 5; +} + +message RuntimeStatusProto { + required string name = 1; // e.g., local, yarn21 + required State state = 2; + optional RuntimeErrorProto error = 3; // runtime (e.g., YARN) error + + optional int32 outstanding_container_requests = 5; + repeated string container_allocation = 6; +} + +////////////////////////////////////////////////////// +// Messages from Driver Process -> Driver Runtime + +message ResourceRequestProto { + // optional SIZE resource_size = 1; // Removed in REEF 0.3 in favor of memory_size. + optional int32 memory_size = 2; // Memory size of the evaluator in MB + optional int32 priority = 3; + + required int32 resource_count = 5; + repeated string node_name = 6; // a list of specific nodes + repeated string rack_name = 7; // a list of specific racks + + optional bool relax_locality = 10; +} + +message ResourceReleaseProto { + required string identifier = 1; +} + +message ResourceLaunchProto { + required string identifier = 1; + required string remote_id = 2; + required string evaluator_conf = 3; + required ProcessType type = 4; + repeated FileResourceProto file = 10; +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Protobuf/Proto/evaluator_runtime.proto ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Protobuf/Proto/evaluator_runtime.proto b/lang/cs/Org.Apache.REEF.Common/Protobuf/Proto/evaluator_runtime.proto new file mode 100644 index 0000000..1415e5c --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Protobuf/Proto/evaluator_runtime.proto @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +option java_package = "org.apache.reef.proto"; +option java_outer_classname = "EvaluatorRuntimeProtocol"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; + +import "reef_service_protos.proto"; + +// Stop the evaluator +message StopEvaluatorProto { +} + +// Kill the evaluator +message KillEvaluatorProto { +} + +// Start a task +message StartTaskProto { + required string context_id = 1; + required string configuration = 2; +} + +message AddContextProto { + required string parent_context_id = 1; + required string context_configuration = 2; + optional string service_configuration = 3; +} + +message RemoveContextProto { + required string context_id = 1; +} + +// Stop the task +message StopTaskProto { +} + +// Suspend the task +message SuspendTaskProto { +} + +///////////////////////////////////////// +// Message aggregators + +message ContextMessageProto { + required string context_id = 1; + required bytes message = 2; +} + +message ContextControlProto { + optional bytes task_message = 1; + optional ContextMessageProto context_message = 2; + + optional AddContextProto add_context = 5; + optional RemoveContextProto remove_context = 6; + optional StartTaskProto start_task = 7; + optional StopTaskProto stop_task = 8; + optional SuspendTaskProto suspend_task = 9; +} + +message EvaluatorHeartbeatProto { + required int64 timestamp = 1; + required EvaluatorStatusProto evaluator_status = 2; + repeated ContextStatusProto context_status = 3; + optional TaskStatusProto task_status = 4; + optional bool recovery = 5; +} + +message EvaluatorControlProto { + required int64 timestamp = 1; + required string identifier = 2; + + optional ContextControlProto context_control = 3; + optional KillEvaluatorProto kill_evaluator = 4; +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7edb8570/lang/cs/Org.Apache.REEF.Common/Protobuf/Proto/reef_protocol.proto ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Common/Protobuf/Proto/reef_protocol.proto b/lang/cs/Org.Apache.REEF.Common/Protobuf/Proto/reef_protocol.proto new file mode 100644 index 0000000..6b99415 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.Common/Protobuf/Proto/reef_protocol.proto @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import "client_runtime.proto"; + +import "evaluator_runtime.proto"; + +import "reef_service_protos.proto"; + + +option java_package = "com.Org.Apache.REEF.proto"; + +option java_generic_services = true; + +option java_generate_equals_and_hash = true; + +option java_outer_classname = "REEFProtocol"; + +message REEFMessage { + // Messages defined in client_runtime.proto + optional JobSubmissionProto jobSubmission = 1; + optional JobControlProto jobControl = 2; + // Messages defined in reef_service_protos.proto + optional RuntimeErrorProto runtimeError = 3; + optional JobStatusProto jobStatus = 4; + // Messages from evaluator_runtime.proto + optional EvaluatorControlProto evaluatorControl = 5; + optional EvaluatorHeartbeatProto evaluatorHeartBeat = 6; +} \ No newline at end of file
