Repository: incubator-reef Updated Branches: refs/heads/master fea1fbd3d -> cfa149a6e
[REEF-646] Remove APIs deprecated since 0.12 from reef-io This removes deprecated fields/methods in DataLoader, DataLoadingRequestBuilder and InputFormatLoadingService. JIRA: [REEF-646](https://issues.apache.org/jira/browse/REEF-646) Pull Request: Closes #430 Project: http://git-wip-us.apache.org/repos/asf/incubator-reef/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-reef/commit/cfa149a6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-reef/tree/cfa149a6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-reef/diff/cfa149a6 Branch: refs/heads/master Commit: cfa149a6e35d0ab61ed8cd35b2d83a1c70fc8f5b Parents: fea1fbd Author: Mariia Mykhailova <michayl...@gmail.com> Authored: Thu Aug 27 14:27:06 2015 -0700 Committer: Markus Weimer <wei...@apache.org> Committed: Thu Aug 27 16:26:33 2015 -0700 ---------------------------------------------------------------------- .../examples/data/loading/DataLoadingREEF.java | 3 +- .../reef/examples/group/bgd/BGDClient.java | 3 +- .../reef/io/data/loading/api/DataLoader.java | 23 ----- .../loading/api/DataLoadingRequestBuilder.java | 95 +------------------- .../loading/impl/InputFormatLoadingService.java | 26 +----- 5 files changed, 5 insertions(+), 145 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/cfa149a6/lang/java/reef-examples/src/main/java/org/apache/reef/examples/data/loading/DataLoadingREEF.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-examples/src/main/java/org/apache/reef/examples/data/loading/DataLoadingREEF.java b/lang/java/reef-examples/src/main/java/org/apache/reef/examples/data/loading/DataLoadingREEF.java index ec253a3..4c15957 100644 --- a/lang/java/reef-examples/src/main/java/org/apache/reef/examples/data/loading/DataLoadingREEF.java +++ b/lang/java/reef-examples/src/main/java/org/apache/reef/examples/data/loading/DataLoadingREEF.java @@ -95,11 +95,10 @@ public final class DataLoadingREEF { .build(); final Configuration dataLoadConfiguration = new DataLoadingRequestBuilder() - .setMemoryMB(1024) .setInputFormatClass(TextInputFormat.class) .setInputPath(inputDir) .setNumberOfDesiredSplits(NUM_SPLITS) - .setComputeRequest(computeRequest) + .addComputeRequest(computeRequest) .setDriverConfigurationModule(DriverConfiguration.CONF .set(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getClassLocation(LineCounter.class)) .set(DriverConfiguration.ON_CONTEXT_ACTIVE, LineCounter.ContextActiveHandler.class) http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/cfa149a6/lang/java/reef-examples/src/main/java/org/apache/reef/examples/group/bgd/BGDClient.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-examples/src/main/java/org/apache/reef/examples/group/bgd/BGDClient.java b/lang/java/reef-examples/src/main/java/org/apache/reef/examples/group/bgd/BGDClient.java index 0877df3..6f5af66 100644 --- a/lang/java/reef-examples/src/main/java/org/apache/reef/examples/group/bgd/BGDClient.java +++ b/lang/java/reef-examples/src/main/java/org/apache/reef/examples/group/bgd/BGDClient.java @@ -101,11 +101,10 @@ public class BGDClient { .setMemory(memory) .build(); final Configuration dataLoadConfiguration = new DataLoadingRequestBuilder() - .setMemoryMB(memory) .setInputFormatClass(TextInputFormat.class) .setInputPath(input) .setNumberOfDesiredSplits(numSplits) - .setComputeRequest(computeRequest) + .addComputeRequest(computeRequest) .renewFailedEvaluators(false) .setDriverConfigurationModule(DriverConfiguration.CONF .setMultiple(DriverConfiguration.GLOBAL_LIBRARIES, EnvironmentUtils.getAllClasspathJars()) http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/cfa149a6/lang/java/reef-io/src/main/java/org/apache/reef/io/data/loading/api/DataLoader.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-io/src/main/java/org/apache/reef/io/data/loading/api/DataLoader.java b/lang/java/reef-io/src/main/java/org/apache/reef/io/data/loading/api/DataLoader.java index 5e38d8c..8fd71df 100644 --- a/lang/java/reef-io/src/main/java/org/apache/reef/io/data/loading/api/DataLoader.java +++ b/lang/java/reef-io/src/main/java/org/apache/reef/io/data/loading/api/DataLoader.java @@ -39,8 +39,6 @@ import org.apache.reef.wake.time.event.StartTime; import javax.inject.Inject; -import java.util.Arrays; -import java.util.HashSet; import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; @@ -85,27 +83,6 @@ public class DataLoader { private final EvaluatorRequestor requestor; /** - * @deprecated since 0.12. Should use the other constructor instead, which - * allows to specify different compute requests (i.e. masters) and - * data requests (i.e. slaves), in particular racks - */ - @Deprecated - @Inject - public DataLoader( - final Clock clock, - final EvaluatorRequestor requestor, - final DataLoadingService dataLoadingService, - @Parameter(DataLoadingRequestBuilder.DataLoadingEvaluatorMemoryMB.class) final int dataEvalMemoryMB, - @Parameter(DataLoadingRequestBuilder.DataLoadingEvaluatorNumberOfCores.class) final int dataEvalCore, - @Parameter(DataLoadingRequestBuilder.DataLoadingComputeRequest.class) final String serializedComputeRequest) { - this(clock, requestor, dataLoadingService, new HashSet<String>( - Arrays.asList(serializedComputeRequest)), new HashSet<String>( - Arrays.asList(AvroEvaluatorRequestSerializer.toString(EvaluatorRequest - .newBuilder().setMemory(dataEvalMemoryMB) - .setNumberOfCores(dataEvalCore).build())))); - } - - /** * Allows to specify compute and data evaluator requests in particular * locations. * http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/cfa149a6/lang/java/reef-io/src/main/java/org/apache/reef/io/data/loading/api/DataLoadingRequestBuilder.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-io/src/main/java/org/apache/reef/io/data/loading/api/DataLoadingRequestBuilder.java b/lang/java/reef-io/src/main/java/org/apache/reef/io/data/loading/api/DataLoadingRequestBuilder.java index 46085fe..d835aac 100644 --- a/lang/java/reef-io/src/main/java/org/apache/reef/io/data/loading/api/DataLoadingRequestBuilder.java +++ b/lang/java/reef-io/src/main/java/org/apache/reef/io/data/loading/api/DataLoadingRequestBuilder.java @@ -39,7 +39,6 @@ import org.apache.reef.tang.exceptions.BindException; import org.apache.reef.tang.formats.ConfigurationModule; import java.util.ArrayList; -import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.Set; @@ -52,19 +51,6 @@ public final class DataLoadingRequestBuilder // constant used in several places. private static final int UNINITIALIZED = -1; - - /** - * @deprecated since 0.12. Should use instead - * {@link DataLoadingRequestBuilder#dataRequests} - */ - @Deprecated - private int memoryMB = UNINITIALIZED; - /** - * @deprecated since 0.12. Should use instead - * {@link DataLoadingRequestBuilder#dataRequests} - */ - @Deprecated - private int numberOfCores = UNINITIALIZED; private int numberOfDesiredSplits = UNINITIALIZED; private List<EvaluatorRequest> computeRequests = new ArrayList<>(); private final List<EvaluatorRequest> dataRequests = new ArrayList<>(); @@ -94,28 +80,6 @@ public final class DataLoadingRequestBuilder } /** - * Set the memory to be used for Evaluator allocated. - * - * @param memoryMB the amount of memory in MB - * @return this - */ - public DataLoadingRequestBuilder setMemoryMB(final int memoryMB) { - this.memoryMB = memoryMB; - return this; - } - - /** - * Set the core number to be used for Evaluator allocated. - * - * @param numberOfCores the number of cores - * @return this - */ - public DataLoadingRequestBuilder setNumberOfCores(final int numberOfCores) { - this.numberOfCores = numberOfCores; - return this; - } - - /** * Adds the requests to the compute requests list. * * @param computeRequests @@ -169,22 +133,6 @@ public final class DataLoadingRequestBuilder return this; } - /** - * Sets the compute request. - * - * @deprecated since 0.12. Should use instead - * {@link DataLoadingRequestBuilder#addComputeRequest(EvaluatorRequest)} - * or {@link DataLoadingRequestBuilder#addComputeRequests(List)} - * @param computeRequest - * the compute request - * @return this - */ - @Deprecated - public DataLoadingRequestBuilder setComputeRequest(final EvaluatorRequest computeRequest) { - this.computeRequests = new ArrayList<>(Arrays.asList(computeRequest)); - return this; - } - @SuppressWarnings("checkstyle:hiddenfield") public DataLoadingRequestBuilder loadIntoMemory(final boolean inMemory) { this.inMemory = inMemory; @@ -295,25 +243,8 @@ public final class DataLoadingRequestBuilder final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder(driverConfiguration); - // if empty, then the user code still uses the deprecated fields. - // we create a dataLoadRequest object based on them (or their default values) - if (this.dataRequests.isEmpty()) { - final int dataMemoryMB = this.memoryMB > 0 ? this.memoryMB : Integer - .valueOf(DataLoadingEvaluatorMemoryMB.DEFAULT_DATA_MEMORY); - final int dataCores = this.numberOfCores > 0 ? this.numberOfCores : Integer - .valueOf(DataLoadingEvaluatorNumberOfCores.DEFAULT_DATA_CORES); - final EvaluatorRequest defaultDataRequest = EvaluatorRequest.newBuilder().setMemory(dataMemoryMB) - .setNumberOfCores(dataCores).build(); - this.dataRequests.add(defaultDataRequest); - } else { - // if there are dataRequests, make sure the user did not configure the - // memory or the number of cores (deprecated API), as they will be discarded - Validate.isTrue(this.numberOfCores == UNINITIALIZED && this.memoryMB == UNINITIALIZED, - "Should not set number of cores or memory if you added specific data requests"); - } - - // at this point data requests cannot be empty, either we use the one we created based on the - // deprecated fields, or the ones created by the user + Validate.isTrue(!this.dataRequests.isEmpty(), + "Number of cores and memory are deprecated; you have to add specific data requests"); for (final EvaluatorRequest request : this.dataRequests) { jcb.bindSetEntry(DataLoadingDataRequests.class, AvroEvaluatorRequestSerializer.toString(request)); } @@ -353,28 +284,6 @@ public final class DataLoadingRequestBuilder static final String DEFAULT_DESIRED_SPLITS = "0"; } - @NamedParameter(short_name = "dataLoadingEvaluatorMemoryMB", - default_value = DataLoadingEvaluatorMemoryMB.DEFAULT_DATA_MEMORY) - public static final class DataLoadingEvaluatorMemoryMB implements Name<Integer> { - static final String DEFAULT_DATA_MEMORY = "4096"; - } - - @NamedParameter(short_name = "dataLoadingEvaluatorCore", - default_value = DataLoadingEvaluatorNumberOfCores.DEFAULT_DATA_CORES) - public static final class DataLoadingEvaluatorNumberOfCores implements Name<Integer> { - static final String DEFAULT_DATA_CORES = "1"; - } - - /** - * @deprecated since 0.12. Should use instead DataLoadingComputeRequests. No - * need for the default value anymore, it is handled in the - * DataLoader side in order to disambiguate constructors - */ - @Deprecated - @NamedParameter - public static final class DataLoadingComputeRequest implements Name<String> { - } - /** * Allows to specify a set of compute requests to send to the DataLoader. */ http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/cfa149a6/lang/java/reef-io/src/main/java/org/apache/reef/io/data/loading/impl/InputFormatLoadingService.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-io/src/main/java/org/apache/reef/io/data/loading/impl/InputFormatLoadingService.java b/lang/java/reef-io/src/main/java/org/apache/reef/io/data/loading/impl/InputFormatLoadingService.java index 40b76ce..6136d96 100644 --- a/lang/java/reef-io/src/main/java/org/apache/reef/io/data/loading/impl/InputFormatLoadingService.java +++ b/lang/java/reef-io/src/main/java/org/apache/reef/io/data/loading/impl/InputFormatLoadingService.java @@ -18,7 +18,6 @@ */ package org.apache.reef.io.data.loading.impl; -import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.reef.annotations.audience.DriverSide; @@ -37,15 +36,13 @@ import org.apache.reef.tang.exceptions.BindException; import javax.inject.Inject; -import java.util.Arrays; -import java.util.HashSet; import java.util.Random; import java.util.logging.Level; import java.util.logging.Logger; /** * An implementation of {@link DataLoadingService} - * that uses the Hadoop {@link InputFormat} to find + * that uses the Hadoop {@link org.apache.hadoop.mapred.InputFormat} to find * partitions of data & request resources. * <p/> * The InputFormat is taken from the job configurations @@ -69,27 +66,6 @@ public class InputFormatLoadingService<K, V> implements DataLoadingService { private final String inputFormatClass; - - /** - * @deprecated since 0.12. Should use the other constructor instead, which - * allows to specify the strategy on how to assign partitions to - * evaluators. This one by default uses {@link SingleDataCenterEvaluatorToPartitionStrategy} - * - */ - @Deprecated - @Inject - public InputFormatLoadingService( - final InputFormat<K, V> inputFormat, - final JobConf jobConf, - @Parameter(DataLoadingRequestBuilder.NumberOfDesiredSplits.class) final int numberOfDesiredSplits, - @Parameter(DataLoadingRequestBuilder.LoadDataIntoMemory.class) final boolean inMemory, - @Parameter(JobConfExternalConstructor.InputFormatClass.class) final String inputFormatClass, - @Parameter(JobConfExternalConstructor.InputPath.class) final String inputPath) { - this(new SingleDataCenterEvaluatorToPartitionStrategy(inputFormatClass, new HashSet<String>( - Arrays.asList(DistributedDataSetPartitionSerializer.serialize(new DistributedDataSetPartition(inputPath, - DistributedDataSetPartition.LOAD_INTO_ANY_LOCATION, numberOfDesiredSplits))))), inMemory, inputFormatClass); - } - @Inject public InputFormatLoadingService( final EvaluatorToPartitionStrategy<InputSplit> evaluatorToPartitionStrategy,
