Repository: incubator-reef Updated Branches: refs/heads/master 41ff14187 -> 28e5ec030
[REEF-450] Introduce Org.Apache.REEF.IO.PartitionedData This change adds `Org.Apache.REEF.IO.PartitionedData` which is a minimal API to represent partitioned data sets. This change also introduces `Org.Apache.REEF.IO.PartitionedData.Random`, which is an implementation of `Org.Apache.REEF.IO.PartitionedData` for a given number of partitions with a fixed number of random bytes per partition. Lastly, this introduces `Org.Apache.REEF.IO.Tests` as the test project for `Org.Apache.REEF.IO`. Unit Tests are included for `Org.Apache.REEF.IO.PartitionedData.Random`. JIRA: [REEF-450](https://issues.apache.org/jira/browse/REEF-450) Pull Request: This closes #285 Author: Markus Weimer <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-reef/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-reef/commit/28e5ec03 Tree: http://git-wip-us.apache.org/repos/asf/incubator-reef/tree/28e5ec03 Diff: http://git-wip-us.apache.org/repos/asf/incubator-reef/diff/28e5ec03 Branch: refs/heads/master Commit: 28e5ec030c3fab0118224a6ad147ffa05de1ead8 Parents: 41ff141 Author: Markus Weimer <[email protected]> Authored: Tue Jul 7 16:16:07 2015 -0700 Committer: Julia Wang <[email protected]> Committed: Fri Jul 10 12:32:12 2015 -0700 ---------------------------------------------------------------------- .../Org.Apache.REEF.IO.Tests.csproj | 72 +++++++++++++ .../Properties/AssemblyInfo.cs | 34 ++++++ .../TestRandomDataSet.cs | 108 +++++++++++++++++++ .../Org.Apache.REEF.IO.csproj | 17 +++ .../PartitionedData/IPartition.cs | 40 +++++++ .../PartitionedData/IPartitionDescriptor.cs | 39 +++++++ .../PartitionedData/IPartitionedDataSet.cs | 46 ++++++++ .../Parameters/NumberOfDoublesPerPartition.cs | 28 +++++ .../Random/Parameters/NumberOfPartitions.cs | 28 +++++ .../Random/Parameters/PartitionId.cs | 28 +++++ .../Random/RandomDataConfiguration.cs | 47 ++++++++ .../PartitionedData/Random/RandomDataSet.cs | 79 ++++++++++++++ .../PartitionedData/Random/RandomPartition.cs | 68 ++++++++++++ .../Random/RandomPartitionDescriptor.cs | 51 +++++++++ lang/cs/Org.Apache.REEF.sln | Bin 26956 -> 27944 bytes 15 files changed, 685 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/28e5ec03/lang/cs/Org.Apache.REEF.IO.Tests/Org.Apache.REEF.IO.Tests.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/Org.Apache.REEF.IO.Tests.csproj b/lang/cs/Org.Apache.REEF.IO.Tests/Org.Apache.REEF.IO.Tests.csproj new file mode 100644 index 0000000..33b53c2 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.IO.Tests/Org.Apache.REEF.IO.Tests.csproj @@ -0,0 +1,72 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> +<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003";> + <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" /> + <PropertyGroup> + <ProjectGuid>{CCFAFD31-D057-42CC-AD2C-2001E921D949}</ProjectGuid> + <OutputType>Library</OutputType> + <AppDesignerFolder>Properties</AppDesignerFolder> + <RootNamespace>Org.Apache.REEF.IO.Tests</RootNamespace> + <AssemblyName>Org.Apache.REEF.IO.Tests</AssemblyName> + <TargetFrameworkVersion>v4.5</TargetFrameworkVersion> + <FileAlignment>512</FileAlignment> + <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..</SolutionDir> + <RestorePackages>true</RestorePackages> + </PropertyGroup> + <Import Project="$(SolutionDir)\build.props" /> + <ItemGroup> + <Reference Include="Microsoft.VisualStudio.QualityTools.UnitTestFramework, Version=10.1.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL" /> + <Reference Include="System" /> + <Reference Include="System.Core" /> + <Reference Include="System.Xml.Linq" /> + <Reference Include="System.Data.DataSetExtensions" /> + <Reference Include="Microsoft.CSharp" /> + <Reference Include="System.Data" /> + <Reference Include="System.Xml" /> + </ItemGroup> + <ItemGroup> + <Compile Include="Properties\AssemblyInfo.cs" /> + <Compile Include="TestRandomDataSet.cs" /> + </ItemGroup> + <ItemGroup> + <ProjectReference Include="$(SolutionDir)\Org.Apache.REEF.Common\Org.Apache.REEF.Common.csproj"> + <Project>{545a0582-4105-44ce-b99c-b1379514a630}</Project> + <Name>Org.Apache.REEF.Common</Name> + </ProjectReference> + <ProjectReference Include="$(SolutionDir)\Org.Apache.REEF.IO\Org.Apache.REEF.IO.csproj"> + <Project>{dec0f0a8-dbef-4ebf-b69c-e2369c15abf1}</Project> + <Name>Org.Apache.REEF.IO</Name> + </ProjectReference> + <ProjectReference Include="$(SolutionDir)\Org.Apache.REEF.Tang\Org.Apache.REEF.Tang.csproj"> + <Project>{97dbb573-3994-417a-9f69-ffa25f00d2a6}</Project> + <Name>Org.Apache.REEF.Tang</Name> + </ProjectReference> + <ProjectReference Include="$(SolutionDir)\Org.Apache.REEF.Utilities\Org.Apache.REEF.Utilities.csproj"> + <Project>{79e7f89a-1dfb-45e1-8d43-d71a954aeb98}</Project> + <Name>Org.Apache.REEF.Utilities</Name> + </ProjectReference> + <ProjectReference Include="$(SolutionDir)\Org.Apache.REEF.Wake\Org.Apache.REEF.Wake.csproj"> + <Project>{cdfb3464-4041-42b1-9271-83af24cd5008}</Project> + <Name>Org.Apache.REEF.Wake</Name> + </ProjectReference> + </ItemGroup> + <ItemGroup> + <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" /> + </ItemGroup> + <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> +</Project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/28e5ec03/lang/cs/Org.Apache.REEF.IO.Tests/Properties/AssemblyInfo.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/Properties/AssemblyInfo.cs b/lang/cs/Org.Apache.REEF.IO.Tests/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..6ddaf7e --- /dev/null +++ b/lang/cs/Org.Apache.REEF.IO.Tests/Properties/AssemblyInfo.cs @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("Org.Apache.REEF.IO.Tests")] +[assembly: AssemblyDescription("Tests for the IO Library for REEF")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("The Apache Software Foundation.")] +[assembly: AssemblyProduct("Org.Apache.REEF.IO.Tests")] +[assembly: AssemblyCopyright("The Apache Software Foundation")] +[assembly: AssemblyTrademark("The Apache Software Foundation")] +[assembly: AssemblyCulture("")] +[assembly: ComVisible(false)] +[assembly: AssemblyVersion("0.12.0.0")] +[assembly: AssemblyFileVersion("0.12.0.0")] +[assembly: Guid("a22b790a-0432-4395-9949-bf8e3958f061")] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/28e5ec03/lang/cs/Org.Apache.REEF.IO.Tests/TestRandomDataSet.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestRandomDataSet.cs b/lang/cs/Org.Apache.REEF.IO.Tests/TestRandomDataSet.cs new file mode 100644 index 0000000..bf5e85a --- /dev/null +++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestRandomDataSet.cs @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Org.Apache.REEF.IO.PartitionedData; +using Org.Apache.REEF.IO.PartitionedData.Random; +using Org.Apache.REEF.Tang.Implementations.Tang; + +namespace Org.Apache.REEF.IO.Tests +{ + /// <summary> + /// Tests for Org.Apache.REEF.IO.PartitionedData.Random. + /// </summary> + [TestClass] + public sealed class TestRandomDataSet + { + /// <summary> + /// The number of partitions the tested dataset will have. + /// </summary> + private const int NumberOfPartitions = 3; + + /// <summary> + /// The number of doubles the stream of each partition will return. + /// </summary> + private const int NumberOfDoublesPerPartition = 7; + + /// <summary> + /// Number of bytes per double in the stream. + /// </summary> + private const int BytesPerDouble = 8; + + /// <summary> + /// The number of bytes we expect each partition's stream to return. + /// </summary> + private const int ExpectedNumberOfBytesPerPartition = NumberOfDoublesPerPartition*BytesPerDouble; + + /// <summary> + /// Test for the driver side APIs of RandomDataSet. + /// </summary> + [TestMethod] + public void TestDriverSide() + { + var dataSet = MakeRandomDataSet(); + Assert.IsNotNull(dataSet); + Assert.IsNotNull(dataSet.Id); + Assert.IsNotNull(dataSet.GetEnumerator()); + Assert.AreEqual(NumberOfPartitions, dataSet.Count); + Assert.IsNull(dataSet.GetPartitionDescriptorForId("THIS_IS_AN_ID_THAT_CANNOT_BE_IN_THE_DATASET")); + } + + /// <summary> + /// Tests the Evaluator side of the IPartionedDataSet. + /// </summary> + /// <remarks> + /// This instantiates each IPartition using the IConfiguration provided by the IPartitionDescriptor. + /// </remarks> + [TestMethod] + public void TestEvaluatorSide() + { + var dataSet = MakeRandomDataSet(); + foreach (var partitionDescriptor in dataSet) + { + var partition = + TangFactory.GetTang() + .NewInjector(partitionDescriptor.GetPartitionConfiguration()) + .GetInstance<IPartition>(); + Assert.IsNotNull(partition); + Assert.IsNotNull(partition.Id); + + using (var partitionStream = partition.Open()) + { + Assert.IsNotNull(partitionStream); + Assert.IsTrue(partitionStream.CanRead); + Assert.IsFalse(partitionStream.CanWrite); + Assert.AreEqual(ExpectedNumberOfBytesPerPartition, partitionStream.Length); + } + } + } + + /// <summary> + /// Make a DataSet instance using the RandomDataConfiguration. + /// </summary> + /// <returns></returns> + private IPartitionedDataSet MakeRandomDataSet() + { + return TangFactory.GetTang().NewInjector(RandomDataConfiguration.ConfigurationModule + .Set(RandomDataConfiguration.NumberOfDoublesPerPartition, NumberOfDoublesPerPartition.ToString()) + .Set(RandomDataConfiguration.NumberOfPartitions, NumberOfPartitions.ToString()) + .Build()).GetInstance<IPartitionedDataSet>(); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/28e5ec03/lang/cs/Org.Apache.REEF.IO/Org.Apache.REEF.IO.csproj ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO/Org.Apache.REEF.IO.csproj b/lang/cs/Org.Apache.REEF.IO/Org.Apache.REEF.IO.csproj index 2a995dc..e02bac3 100644 --- a/lang/cs/Org.Apache.REEF.IO/Org.Apache.REEF.IO.csproj +++ b/lang/cs/Org.Apache.REEF.IO/Org.Apache.REEF.IO.csproj @@ -41,9 +41,26 @@ under the License. <Reference Include="System.Xml" /> </ItemGroup> <ItemGroup> + <Compile Include="PartitionedData\IPartition.cs" /> + <Compile Include="PartitionedData\IPartitionDescriptor.cs" /> + <Compile Include="PartitionedData\IPartitionedDataSet.cs" /> + <Compile Include="PartitionedData\Random\Parameters\NumberOfDoublesPerPartition.cs" /> + <Compile Include="PartitionedData\Random\Parameters\NumberOfPartitions.cs" /> + <Compile Include="PartitionedData\Random\Parameters\PartitionId.cs" /> + <Compile Include="PartitionedData\Random\RandomDataConfiguration.cs" /> + <Compile Include="PartitionedData\Random\RandomDataSet.cs" /> + <Compile Include="PartitionedData\Random\RandomPartition.cs" /> + <Compile Include="PartitionedData\Random\RandomPartitionDescriptor.cs" /> <Compile Include="Properties\AssemblyInfo.cs" /> </ItemGroup> <ItemGroup> <None Include="Org.Apache.REEF.IO.nuspec" /> </ItemGroup> + <ItemGroup> + <ProjectReference Include="$(SolutionDir)\Org.Apache.REEF.Tang\Org.Apache.REEF.Tang.csproj"> + <Project>{97DBB573-3994-417A-9F69-FFA25F00D2A6}</Project> + <Name>Org.Apache.REEF.Tang</Name> + </ProjectReference> + </ItemGroup> + <ItemGroup /> </Project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/28e5ec03/lang/cs/Org.Apache.REEF.IO/PartitionedData/IPartition.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO/PartitionedData/IPartition.cs b/lang/cs/Org.Apache.REEF.IO/PartitionedData/IPartition.cs new file mode 100644 index 0000000..73f422f --- /dev/null +++ b/lang/cs/Org.Apache.REEF.IO/PartitionedData/IPartition.cs @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System.IO; + +namespace Org.Apache.REEF.IO.PartitionedData +{ + /// <summary> + /// Evaluator-Side representation of a data set partition. + /// </summary> + public interface IPartition + { + /// <summary> + /// The id of the partition. + /// </summary> + string Id { get; } + + /// <summary> + /// Opens the stream to the underlying partition. + /// </summary> + /// <returns>A read-only stream.</returns> + Stream Open(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/28e5ec03/lang/cs/Org.Apache.REEF.IO/PartitionedData/IPartitionDescriptor.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO/PartitionedData/IPartitionDescriptor.cs b/lang/cs/Org.Apache.REEF.IO/PartitionedData/IPartitionDescriptor.cs new file mode 100644 index 0000000..521883a --- /dev/null +++ b/lang/cs/Org.Apache.REEF.IO/PartitionedData/IPartitionDescriptor.cs @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using Org.Apache.REEF.Tang.Interface; + +namespace Org.Apache.REEF.IO.PartitionedData +{ + /// <summary> + /// Driver-Side representation of a partition. + /// </summary> + public interface IPartitionDescriptor + { + /// <summary> + /// The id of the partition. + /// </summary> + string Id { get; } + + /// <summary> + /// The Configuration used to instantiate an IPartition connected to this partition. + /// </summary> + IConfiguration GetPartitionConfiguration(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/28e5ec03/lang/cs/Org.Apache.REEF.IO/PartitionedData/IPartitionedDataSet.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO/PartitionedData/IPartitionedDataSet.cs b/lang/cs/Org.Apache.REEF.IO/PartitionedData/IPartitionedDataSet.cs new file mode 100644 index 0000000..52748be --- /dev/null +++ b/lang/cs/Org.Apache.REEF.IO/PartitionedData/IPartitionedDataSet.cs @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System.Collections.Generic; + +namespace Org.Apache.REEF.IO.PartitionedData +{ + /// <summary> + /// Driver-Side representation of a partitioned data set. + /// </summary> + public interface IPartitionedDataSet : IEnumerable<IPartitionDescriptor> + { + /// <summary> + /// The number of partitions. + /// </summary> + int Count { get; } + + /// <summary> + /// The Id of this dataset + /// </summary> + string Id { get; } + + /// <summary> + /// Get the PartitionDescriptor for the given Id. + /// </summary> + /// <param name="partitionId"></param> + /// <returns>The IPartitionDescriptor found or null if none exists.</returns> + IPartitionDescriptor GetPartitionDescriptorForId(string partitionId); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/28e5ec03/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/Parameters/NumberOfDoublesPerPartition.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/Parameters/NumberOfDoublesPerPartition.cs b/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/Parameters/NumberOfDoublesPerPartition.cs new file mode 100644 index 0000000..7482627 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/Parameters/NumberOfDoublesPerPartition.cs @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using Org.Apache.REEF.Tang.Annotations; + +namespace Org.Apache.REEF.IO.PartitionedData.Random.Parameters +{ + [NamedParameter("Number of random partitions to create.", defaultValue: "2")] + internal sealed class NumberOfDoublesPerPartition : Name<int> + { + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/28e5ec03/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/Parameters/NumberOfPartitions.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/Parameters/NumberOfPartitions.cs b/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/Parameters/NumberOfPartitions.cs new file mode 100644 index 0000000..36e8cc6 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/Parameters/NumberOfPartitions.cs @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using Org.Apache.REEF.Tang.Annotations; + +namespace Org.Apache.REEF.IO.PartitionedData.Random.Parameters +{ + [NamedParameter("Number of random partitions to create.", defaultValue: "2")] + internal sealed class NumberOfPartitions : Name<int> + { + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/28e5ec03/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/Parameters/PartitionId.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/Parameters/PartitionId.cs b/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/Parameters/PartitionId.cs new file mode 100644 index 0000000..97ee0f1 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/Parameters/PartitionId.cs @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using Org.Apache.REEF.Tang.Annotations; + +namespace Org.Apache.REEF.IO.PartitionedData.Random.Parameters +{ + [NamedParameter("The id of the partition.")] + internal sealed class PartitionId : Name<string> + { + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/28e5ec03/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/RandomDataConfiguration.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/RandomDataConfiguration.cs b/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/RandomDataConfiguration.cs new file mode 100644 index 0000000..c8422ca --- /dev/null +++ b/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/RandomDataConfiguration.cs @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using Org.Apache.REEF.IO.PartitionedData.Random.Parameters; +using Org.Apache.REEF.Tang.Formats; +using Org.Apache.REEF.Tang.Util; + +namespace Org.Apache.REEF.IO.PartitionedData.Random +{ + /// <summary> + /// Configuration module for the random dataset generator. + /// </summary> + public sealed class RandomDataConfiguration : ConfigurationModuleBuilder + { + /// <summary> + /// The number of doubles to serialize into the stream per partition. + /// </summary> + public static readonly OptionalParameter<int> NumberOfDoublesPerPartition = new OptionalParameter<int>(); + + /// <summary> + /// Number of random partitions to create. + /// </summary> + public static readonly OptionalParameter<int> NumberOfPartitions = new OptionalParameter<int>(); + + public static ConfigurationModule ConfigurationModule = new RandomDataConfiguration() + .BindImplementation(GenericType<IPartitionedDataSet>.Class, GenericType<RandomDataSet>.Class) + .BindNamedParameter(GenericType<NumberOfPartitions>.Class, NumberOfPartitions) + .BindNamedParameter(GenericType<NumberOfDoublesPerPartition>.Class, NumberOfDoublesPerPartition) + .Build(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/28e5ec03/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/RandomDataSet.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/RandomDataSet.cs b/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/RandomDataSet.cs new file mode 100644 index 0000000..7eb74b6 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/RandomDataSet.cs @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System.Collections; +using System.Collections.Generic; +using Org.Apache.REEF.IO.PartitionedData.Random.Parameters; +using Org.Apache.REEF.Tang.Annotations; + +namespace Org.Apache.REEF.IO.PartitionedData.Random +{ + /// <summary> + /// An implementation of IPartitionedDataSet that produces a given number of partitions containing a given number of random + /// doubles. + /// </summary> + internal sealed class RandomDataSet : IPartitionedDataSet + { + private readonly Dictionary<string, IPartitionDescriptor> _partitions; + + [Inject] + private RandomDataSet([Parameter(typeof(NumberOfPartitions))] int numberOfPartitions, + [Parameter(typeof(NumberOfDoublesPerPartition))] int numberOfDoublesPerPartition) + { + _partitions = new Dictionary<string, IPartitionDescriptor>(numberOfPartitions); + for (var i = 0; i < numberOfPartitions; ++i) + { + var id = "RandomPartition-" + i; + _partitions[id] = new RandomPartitionDescriptor(id, numberOfDoublesPerPartition); + } + } + + public IEnumerator<IPartitionDescriptor> GetEnumerator() + { + return _partitions.Values.GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return _partitions.Values.GetEnumerator(); + } + + public int Count + { + get { return _partitions.Count; } + } + + public string Id + { + get { return "RandomDataSet"; } + } + + public IPartitionDescriptor GetPartitionDescriptorForId(string partitionId) + { + try + { + return _partitions[partitionId]; + } + catch (KeyNotFoundException) + { + return null; + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/28e5ec03/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/RandomPartition.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/RandomPartition.cs b/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/RandomPartition.cs new file mode 100644 index 0000000..03c6035 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/RandomPartition.cs @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System; +using System.Diagnostics; +using System.IO; +using Org.Apache.REEF.IO.PartitionedData.Random.Parameters; +using Org.Apache.REEF.Tang.Annotations; + +namespace Org.Apache.REEF.IO.PartitionedData.Random +{ + /// <summary> + /// An implementation of IPartition that returns a configurable number of random doubles. + /// </summary> + internal sealed class RandomPartition : IPartition + { + private readonly string _id; + private readonly byte[] _randomData; + + [Inject] + private RandomPartition([Parameter(typeof(PartitionId))] string id, + [Parameter(typeof(NumberOfDoublesPerPartition))] int numberOfDoubles) + { + _id = id; + _randomData = new byte[numberOfDoubles*8]; + var random = new System.Random(); + + for (var i = 0; i < numberOfDoubles; ++i) + { + var randomDouble = random.NextDouble(); + var randomDoubleAsBytes = BitConverter.GetBytes(randomDouble); + Debug.Assert(randomDoubleAsBytes.Length == 8); + for (var j = 0; j < 8; ++j) + { + var index = i*8 + j; + Debug.Assert(index < _randomData.Length); + _randomData[index] = randomDoubleAsBytes[j]; + } + } + } + + public string Id + { + get { return _id; } + } + + public Stream Open() + { + return new MemoryStream(_randomData, false); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/28e5ec03/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/RandomPartitionDescriptor.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/RandomPartitionDescriptor.cs b/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/RandomPartitionDescriptor.cs new file mode 100644 index 0000000..9b37f21 --- /dev/null +++ b/lang/cs/Org.Apache.REEF.IO/PartitionedData/Random/RandomPartitionDescriptor.cs @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using Org.Apache.REEF.IO.PartitionedData.Random.Parameters; +using Org.Apache.REEF.Tang.Implementations.Tang; +using Org.Apache.REEF.Tang.Interface; + +namespace Org.Apache.REEF.IO.PartitionedData.Random +{ + internal sealed class RandomPartitionDescriptor : IPartitionDescriptor + { + private readonly string _id; + private readonly int _numberOfDoubles; + + internal RandomPartitionDescriptor(string id, int numberOfDoubles) + { + _id = id; + _numberOfDoubles = numberOfDoubles; + } + + public string Id + { + get { return _id; } + } + + public IConfiguration GetPartitionConfiguration() + { + return TangFactory.GetTang().NewConfigurationBuilder() + .BindImplementation(typeof(IPartition), typeof(RandomPartition)) + .BindNamedParameter(typeof(PartitionId), _id) + .BindNamedParameter(typeof(NumberOfDoublesPerPartition), _numberOfDoubles.ToString()) + .Build(); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/28e5ec03/lang/cs/Org.Apache.REEF.sln ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.sln b/lang/cs/Org.Apache.REEF.sln index 726683e..2626a8e 100644 Binary files a/lang/cs/Org.Apache.REEF.sln and b/lang/cs/Org.Apache.REEF.sln differ
