http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorService.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorService.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorService.java deleted file mode 100644 index 933332e..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorService.java +++ /dev/null @@ -1,238 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.service; - -import java.io.IOException; -import java.io.InputStream; -import java.lang.reflect.Type; -import java.util.List; -import java.util.Map; - -import javax.ws.rs.DELETE; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.resourceestimator.common.api.RecurrenceId; -import org.apache.hadoop.resourceestimator.common.api.ResourceSkyline; -import org.apache.hadoop.resourceestimator.common.config.ResourceEstimatorConfiguration; -import org.apache.hadoop.resourceestimator.common.config.ResourceEstimatorUtil; -import org.apache.hadoop.resourceestimator.common.exception.ResourceEstimatorException; -import org.apache.hadoop.resourceestimator.common.serialization.RLESparseResourceAllocationSerDe; -import org.apache.hadoop.resourceestimator.common.serialization.ResourceSerDe; -import org.apache.hadoop.resourceestimator.skylinestore.api.SkylineStore; -import org.apache.hadoop.resourceestimator.skylinestore.exceptions.SkylineStoreException; -import org.apache.hadoop.resourceestimator.solver.api.Solver; -import org.apache.hadoop.resourceestimator.solver.exceptions.SolverException; -import org.apache.hadoop.resourceestimator.translator.api.LogParser; -import org.apache.hadoop.resourceestimator.translator.impl.LogParserUtil; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.reflect.TypeToken; -import com.google.inject.Singleton; - -/** - * Resource Estimator Service which provides a set of REST APIs for users to - * use the estimation service. - */ -@Singleton @Path("/resourceestimator") public class ResourceEstimatorService { - private static final Logger LOGGER = - LoggerFactory.getLogger(ResourceEstimatorService.class); - private static SkylineStore skylineStore; - private static Solver solver; - private static LogParser logParser; - private static LogParserUtil logParserUtil = new LogParserUtil(); - private static Configuration config; - private static Gson gson; - private static Type rleType; - private static Type skylineStoreType; - - public ResourceEstimatorService() throws ResourceEstimatorException { - if (skylineStore == null) { - try { - config = new Configuration(); - config.addResource(ResourceEstimatorConfiguration.CONFIG_FILE); - skylineStore = ResourceEstimatorUtil.createProviderInstance(config, - ResourceEstimatorConfiguration.SKYLINESTORE_PROVIDER, - ResourceEstimatorConfiguration.DEFAULT_SKYLINESTORE_PROVIDER, - SkylineStore.class); - logParser = ResourceEstimatorUtil.createProviderInstance(config, - ResourceEstimatorConfiguration.TRANSLATOR_PROVIDER, - ResourceEstimatorConfiguration.DEFAULT_TRANSLATOR_PROVIDER, - LogParser.class); - logParser.init(config, skylineStore); - logParserUtil.setLogParser(logParser); - solver = ResourceEstimatorUtil.createProviderInstance(config, - ResourceEstimatorConfiguration.SOLVER_PROVIDER, - ResourceEstimatorConfiguration.DEFAULT_SOLVER_PROVIDER, - Solver.class); - solver.init(config, skylineStore); - } catch (Exception ex) { - LOGGER - .error("Server initialization failed due to: {}", ex.getMessage()); - throw new ResourceEstimatorException(ex.getMessage(), ex); - } - gson = new GsonBuilder() - .registerTypeAdapter(Resource.class, new ResourceSerDe()) - .registerTypeAdapter(RLESparseResourceAllocation.class, - new RLESparseResourceAllocationSerDe()) - .enableComplexMapKeySerialization().create(); - rleType = new TypeToken<RLESparseResourceAllocation>() { - }.getType(); - skylineStoreType = - new TypeToken<Map<RecurrenceId, List<ResourceSkyline>>>() { - }.getType(); - } - } - - /** - * Parse the log file. See also {@link LogParser#parseStream(InputStream)}. - * - * @param logFile file/directory of the log to be parsed. - * @throws IOException if fails to parse the log. - * @throws SkylineStoreException if fails to addHistory to - * {@link SkylineStore}. - * @throws ResourceEstimatorException if the {@link LogParser} - * is not initialized. - */ - @POST @Path("/translator/{logFile : .+}") public void parseFile( - @PathParam("logFile") String logFile) - throws IOException, SkylineStoreException, ResourceEstimatorException { - logParserUtil.parseLog(logFile); - LOGGER.debug("Parse logFile: {}.", logFile); - } - - /** - * Get predicted {code Resource} allocation for the pipeline. If the - * prediction for the pipeline already exists in the {@link SkylineStore}, it - * will directly get the prediction from {@link SkylineStore}, otherwise it - * will call the {@link Solver} to make prediction, and store the predicted - * {code Resource} allocation to the {@link SkylineStore}. Note that invoking - * {@link Solver} could be a time-consuming operation. - * - * @param pipelineId the id of the pipeline. - * @return Json format of {@link RLESparseResourceAllocation}. - * @throws SolverException if {@link Solver} fails; - * @throws SkylineStoreException if fails to get history - * {@link ResourceSkyline} or predicted {code Resource} allocation - * from {@link SkylineStore}. - */ - @GET @Path("/estimator/{pipelineId}") @Produces(MediaType.APPLICATION_JSON) - public String getPrediction( - @PathParam(value = "pipelineId") String pipelineId) - throws SolverException, SkylineStoreException { - // first, try to grab the predicted resource allocation from the skyline - // store - RLESparseResourceAllocation result = skylineStore.getEstimation(pipelineId); - // if received resource allocation is null, then run the solver - if (result == null) { - RecurrenceId recurrenceId = new RecurrenceId(pipelineId, "*"); - Map<RecurrenceId, List<ResourceSkyline>> jobHistory = - skylineStore.getHistory(recurrenceId); - result = solver.solve(jobHistory); - } - final String prediction = gson.toJson(result, rleType); - LOGGER.debug("Predict resource requests for pipelineId: {}." + pipelineId); - - return prediction; - } - - /** - * Get history {@link ResourceSkyline} from {@link SkylineStore}. This - * function supports the following special wildcard operations regarding - * {@link RecurrenceId}: If the {@code pipelineId} is "*", it will return all - * entries in the store; else, if the {@code runId} is "*", it will return all - * {@link ResourceSkyline}s belonging to the {@code pipelineId}; else, it will - * return all {@link ResourceSkyline}s belonging to the {{@code pipelineId}, - * {@code runId}}. If the {@link RecurrenceId} does not exist, it will not do - * anything. - * - * @param pipelineId pipelineId of the history run. - * @param runId runId of the history run. - * @return Json format of history {@link ResourceSkyline}s. - * @throws SkylineStoreException if fails to getHistory - * {@link ResourceSkyline} from {@link SkylineStore}. - */ - @GET @Path("/skylinestore/history/{pipelineId}/{runId}") - @Produces(MediaType.APPLICATION_JSON) - public String getHistoryResourceSkyline( - @PathParam("pipelineId") String pipelineId, - @PathParam("runId") String runId) throws SkylineStoreException { - RecurrenceId recurrenceId = new RecurrenceId(pipelineId, runId); - Map<RecurrenceId, List<ResourceSkyline>> jobHistory = - skylineStore.getHistory(recurrenceId); - final String skyline = gson.toJson(jobHistory, skylineStoreType); - LOGGER - .debug("Query the skyline store for recurrenceId: {}." + recurrenceId); - - recurrenceId = new RecurrenceId("*", "*"); - jobHistory = skylineStore.getHistory(recurrenceId); - - return skyline; - } - - /** - * Get estimated {code Resource} allocation for the pipeline. - * - * @param pipelineId id of the pipeline. - * @return Json format of {@link RLESparseResourceAllocation}. - * @throws SkylineStoreException if fails to get estimated {code Resource} - * allocation from {@link SkylineStore}. - */ - @GET @Path("/skylinestore/estimation/{pipelineId}") - @Produces(MediaType.APPLICATION_JSON) - public String getEstimatedResourceAllocation( - @PathParam("pipelineId") String pipelineId) throws SkylineStoreException { - RLESparseResourceAllocation result = skylineStore.getEstimation(pipelineId); - final String skyline = gson.toJson(result, rleType); - LOGGER.debug("Query the skyline store for pipelineId: {}." + pipelineId); - - return skyline; - } - - /** - * Delete history {@link ResourceSkyline}s from {@link SkylineStore}. - * <p> Note that for safety considerations, we only allow users to delete - * history {@link ResourceSkyline}s of one job run. - * - * @param pipelineId pipelineId of the history run. - * @param runId runId runId of the history run. - * @throws SkylineStoreException if fails to deleteHistory - * {@link ResourceSkyline}s. - */ - @DELETE @Path("/skylinestore/history/{pipelineId}/{runId}") - public void deleteHistoryResourceSkyline( - @PathParam("pipelineId") String pipelineId, - @PathParam("runId") String runId) throws SkylineStoreException { - RecurrenceId recurrenceId = new RecurrenceId(pipelineId, runId); - skylineStore.deleteHistory(recurrenceId); - LOGGER.info("Delete ResourceSkyline for recurrenceId: {}.", recurrenceId); - } -}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ShutdownHook.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ShutdownHook.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ShutdownHook.java deleted file mode 100644 index 23e1413..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ShutdownHook.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.service; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Simple shutdown hook for {@link ResourceEstimatorServer}. - */ -public class ShutdownHook extends Thread { - private static final Logger LOGGER = - LoggerFactory.getLogger(ShutdownHook.class); - private final ResourceEstimatorServer server; - - ShutdownHook(ResourceEstimatorServer server) { - this.server = server; - } - - public void run() { - try { - server.shutdown(); - } catch (Exception e) { - LOGGER.error("HttpServer fails to shut down!"); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/package-info.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/package-info.java deleted file mode 100644 index 3571736..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Resource estimator service. - */ - -package org.apache.hadoop.resourceestimator.service; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/api/HistorySkylineStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/api/HistorySkylineStore.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/api/HistorySkylineStore.java deleted file mode 100644 index 8fe4619..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/api/HistorySkylineStore.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.skylinestore.api; - -import java.util.List; -import java.util.Map; - -import org.apache.hadoop.resourceestimator.common.api.RecurrenceId; -import org.apache.hadoop.resourceestimator.common.api.ResourceSkyline; -import org.apache.hadoop.resourceestimator.skylinestore.exceptions.SkylineStoreException; - -/** - * HistorySkylineStore stores pipeline job's {@link ResourceSkyline}s in all - * runs. {@code Estimator} will query the {@link ResourceSkyline}s for pipeline - * jobs. {@code Parser} will parse various types of job logs, construct - * {@link ResourceSkyline}s out of the logs and store them in the SkylineStore. - */ -public interface HistorySkylineStore { - /** - * Add job's resource skyline to the <em>store</em> indexed by the job's - * {@link RecurrenceId}. {@link RecurrenceId} is used to identify recurring - * pipeline jobs, and we assume that {@code - * ResourceEstimatorServer} users will provide the correct - * {@link RecurrenceId}. <p> If {@link ResourceSkyline}s to be added contain - * <em>null</em> elements, the function will skip them. - * - * @param recurrenceId the unique id of user's recurring pipeline jobs. - * @param resourceSkylines the list of {@link ResourceSkyline}s in one run. - * @throws SkylineStoreException if: (1) input parameters are invalid; (2) - * {@link ResourceSkyline}s to be added contain some duplicate - * {@link RecurrenceId}s which already exist in the - * {@link HistorySkylineStore}. - */ - void addHistory(RecurrenceId recurrenceId, - List<ResourceSkyline> resourceSkylines) throws SkylineStoreException; - - /** - * Delete all {@link ResourceSkyline}s belonging to given - * {@link RecurrenceId}. - * <p> Note that for safety considerations, we only allow users to - * deleteHistory {@link ResourceSkyline}s of one job run. - * - * @param recurrenceId the unique id of user's recurring pipeline jobs. - * @throws SkylineStoreException if: (1) input parameters are invalid; (2) - * recurrenceId does not exist in the {@link HistorySkylineStore}. - */ - void deleteHistory(RecurrenceId recurrenceId) throws SkylineStoreException; - - /** - * Update {@link RecurrenceId} with given {@link ResourceSkyline}s. This - * function will deleteHistory all the {@link ResourceSkyline}s belonging to - * the {@link RecurrenceId}, and re-insert the given {@link ResourceSkyline}s - * to the SkylineStore. - * <p> If {@link ResourceSkyline}s contain <em>null</em> elements, - * the function will skip them. - * - * @param recurrenceId the unique id of the pipeline job. - * @param resourceSkylines the list of {@link ResourceSkyline}s in one run. - * @throws SkylineStoreException if: (1) input parameters are invalid; (2) - * recurrenceId does not exist in the SkylineStore. - */ - void updateHistory(RecurrenceId recurrenceId, - List<ResourceSkyline> resourceSkylines) throws SkylineStoreException; - - /** - * Return all {@link ResourceSkyline}s belonging to {@link RecurrenceId}. - * <p> This function supports the following special wildcard operations - * regarding {@link RecurrenceId}: If the {@code pipelineId} is "*", it will - * return all entries in the store; else, if the {@code runId} is "*", it - * will return all {@link ResourceSkyline}s belonging to the - * {@code pipelineId}; else, it will return all {@link ResourceSkyline}s - * belonging to the {{@code pipelineId}, {@code runId}}. If the - * {@link RecurrenceId} does not exist, it will return <em>null</em>. - * - * @param recurrenceId the unique id of the pipeline job. - * @return all {@link ResourceSkyline}s belonging to the recurrenceId. - * @throws SkylineStoreException if recurrenceId is <em>null</em>. - */ - Map<RecurrenceId, List<ResourceSkyline>> getHistory(RecurrenceId recurrenceId) - throws SkylineStoreException; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/api/PredictionSkylineStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/api/PredictionSkylineStore.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/api/PredictionSkylineStore.java deleted file mode 100644 index c3fedce..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/api/PredictionSkylineStore.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.skylinestore.api; - -import org.apache.hadoop.resourceestimator.skylinestore.exceptions.SkylineStoreException; -import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation; - -/** - * PredictionSkylineStore stores the predicted - * {@code RLESparseResourceAllocation} of a job as computed by the - * {@code Estimator} based on the {@code ResourceSkyline}s of past executions in - * the {@code HistorySkylineStore}. - */ -public interface PredictionSkylineStore { - - /** - * Add job's predicted {@code Resource} allocation to the <em>store</em> - * indexed by the {@code - * pipelineId}. - * <p> Note that right now we only keep the latest copy of predicted - * {@code Resource} allocation for the recurring pipeline. - * - * @param pipelineId the id of the recurring pipeline. - * @param resourceOverTime the predicted {@code Resource} allocation for the - * pipeline. - * @throws SkylineStoreException if input parameters are invalid. - */ - void addEstimation(String pipelineId, - RLESparseResourceAllocation resourceOverTime) - throws SkylineStoreException; - - /** - * Return the predicted {@code Resource} allocation for the pipeline. - * <p> If the pipelineId does not exist, it will return <em>null</em>. - * - * @param pipelineId the unique id of the pipeline. - * @return the predicted {@code Resource} allocation for the pipeline. - * @throws SkylineStoreException if pipelineId is <em>null</em>. - */ - RLESparseResourceAllocation getEstimation(String pipelineId) - throws SkylineStoreException; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/api/SkylineStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/api/SkylineStore.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/api/SkylineStore.java deleted file mode 100644 index f352ed4..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/api/SkylineStore.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.skylinestore.api; - -/** - * SkylineStore is composable interface for storing the history - * {@code ResourceSkyline}s of past job runs and the predicted - * {@code RLESparseResourceAllocation} for future execution. - */ -public interface SkylineStore - extends HistorySkylineStore, PredictionSkylineStore { -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/api/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/api/package-info.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/api/package-info.java deleted file mode 100644 index e833486..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/api/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * APIs for the {@code SkylineStore}. - */ - -package org.apache.hadoop.resourceestimator.skylinestore.api; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/DuplicateRecurrenceIdException.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/DuplicateRecurrenceIdException.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/DuplicateRecurrenceIdException.java deleted file mode 100644 index 7c92480..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/DuplicateRecurrenceIdException.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.skylinestore.exceptions; - -/** - * Exception thrown the {@code RecurrenceId} already exists in the - * {@code SkylineStore}. - */ -public class DuplicateRecurrenceIdException extends SkylineStoreException { - private static final long serialVersionUID = -684069387367879218L; - - public DuplicateRecurrenceIdException(final String message) { - super(message); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/EmptyResourceSkylineException.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/EmptyResourceSkylineException.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/EmptyResourceSkylineException.java deleted file mode 100644 index 55a8fa7..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/EmptyResourceSkylineException.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.skylinestore.exceptions; - -/** - * Exception thrown if the @link{ResourceSkyline}s to be added to the - * {@code SkylineStore} is empty. - */ -public class EmptyResourceSkylineException extends SkylineStoreException { - private static final long serialVersionUID = -684069387367879218L; - - public EmptyResourceSkylineException(final String message) { - super(message); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/NullPipelineIdException.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/NullPipelineIdException.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/NullPipelineIdException.java deleted file mode 100644 index d48be7d..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/NullPipelineIdException.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.skylinestore.exceptions; - -/** - * Exception thrown when pipelineId to be added is <em>null</em>. - */ -public class NullPipelineIdException extends SkylineStoreException { - private static final long serialVersionUID = -684069387367879218L; - - public NullPipelineIdException(final String message) { - super(message); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/NullRLESparseResourceAllocationException.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/NullRLESparseResourceAllocationException.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/NullRLESparseResourceAllocationException.java deleted file mode 100644 index 9aee0b6..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/NullRLESparseResourceAllocationException.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.skylinestore.exceptions; - -/** - * Exception thrown if the {@code ResourceSkyline} to be added is <em>null</em>. - */ -public class NullRLESparseResourceAllocationException - extends SkylineStoreException { - private static final long serialVersionUID = -684069387367879218L; - - public NullRLESparseResourceAllocationException(final String message) { - super(message); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/NullRecurrenceIdException.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/NullRecurrenceIdException.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/NullRecurrenceIdException.java deleted file mode 100644 index 518c065..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/NullRecurrenceIdException.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.skylinestore.exceptions; - -/** - * Exception thrown the {@code RecurrenceId} to be added is <em>null</em>. - */ -public class NullRecurrenceIdException extends SkylineStoreException { - private static final long serialVersionUID = -684069387367879218L; - - public NullRecurrenceIdException(final String message) { - super(message); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/NullResourceSkylineException.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/NullResourceSkylineException.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/NullResourceSkylineException.java deleted file mode 100644 index b70c764..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/NullResourceSkylineException.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.skylinestore.exceptions; - -/** - * Exception thrown if the {@code ResourceSkyline} to be added is <em>null</em>. - */ -public class NullResourceSkylineException extends SkylineStoreException { - private static final long serialVersionUID = -684069387367879218L; - - public NullResourceSkylineException(final String message) { - super(message); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/RecurrenceIdNotFoundException.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/RecurrenceIdNotFoundException.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/RecurrenceIdNotFoundException.java deleted file mode 100644 index b5e734d..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/RecurrenceIdNotFoundException.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.skylinestore.exceptions; - -/** - * Exception thrown if {@code RecurrenceId} is not found in the - * {@code SkylineStore}. - */ -public class RecurrenceIdNotFoundException extends SkylineStoreException { - private static final long serialVersionUID = -684069387367879218L; - - public RecurrenceIdNotFoundException(final String message) { - super(message); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/SkylineStoreException.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/SkylineStoreException.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/SkylineStoreException.java deleted file mode 100644 index 751b5dd..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/SkylineStoreException.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.skylinestore.exceptions; - -/** - * Exception thrown the @link{SkylineStore} or the {@code Estimator} tries to - * addHistory or query pipeline job's resource skylines. - */ -public abstract class SkylineStoreException extends Exception { - private static final long serialVersionUID = -684069387367879218L; - - public SkylineStoreException(final String message) { - super(message); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/package-info.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/package-info.java deleted file mode 100644 index 716e090..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/exceptions/package-info.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -/** - * SkylineStore exception module. - */ -package org.apache.hadoop.resourceestimator.skylinestore.exceptions; http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/impl/InMemoryStore.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/impl/InMemoryStore.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/impl/InMemoryStore.java deleted file mode 100644 index e00f3a0..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/impl/InMemoryStore.java +++ /dev/null @@ -1,256 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.skylinestore.impl; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import org.apache.hadoop.resourceestimator.common.api.RecurrenceId; -import org.apache.hadoop.resourceestimator.common.api.ResourceSkyline; -import org.apache.hadoop.resourceestimator.skylinestore.api.SkylineStore; -import org.apache.hadoop.resourceestimator.skylinestore.exceptions.DuplicateRecurrenceIdException; -import org.apache.hadoop.resourceestimator.skylinestore.exceptions.EmptyResourceSkylineException; -import org.apache.hadoop.resourceestimator.skylinestore.exceptions.RecurrenceIdNotFoundException; -import org.apache.hadoop.resourceestimator.skylinestore.exceptions.SkylineStoreException; -import org.apache.hadoop.resourceestimator.skylinestore.validator.SkylineStoreValidator; -import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * An in-memory implementation of {@link SkylineStore}. - */ -public class InMemoryStore implements SkylineStore { - private static final Logger LOGGER = - LoggerFactory.getLogger(InMemoryStore.class); - private final ReentrantReadWriteLock readWriteLock = - new ReentrantReadWriteLock(); - private final Lock readLock = readWriteLock.readLock(); - private final Lock writeLock = readWriteLock.writeLock(); - private final SkylineStoreValidator inputValidator = - new SkylineStoreValidator(); - /** - * A pipeline job's history {@link ResourceSkyline}s. TODO: we may flatten it - * out for quick access. - */ - private final Map<RecurrenceId, List<ResourceSkyline>> skylineStore = - new HashMap<>(); // pipelineId, resource skyline - // Recurring pipeline's predicted {@link ResourceSkyline}s. - private final Map<String, RLESparseResourceAllocation> estimationStore = - new HashMap<>(); // pipelineId, ResourceSkyline - - private List<ResourceSkyline> eliminateNull( - final List<ResourceSkyline> resourceSkylines) { - final List<ResourceSkyline> result = new ArrayList<>(); - for (final ResourceSkyline resourceSkyline : resourceSkylines) { - if (resourceSkyline != null) { - result.add(resourceSkyline); - } - } - return result; - } - - @Override public final void addHistory(final RecurrenceId recurrenceId, - final List<ResourceSkyline> resourceSkylines) - throws SkylineStoreException { - inputValidator.validate(recurrenceId, resourceSkylines); - writeLock.lock(); - try { - // remove the null elements in the resourceSkylines - final List<ResourceSkyline> filteredInput = - eliminateNull(resourceSkylines); - if (filteredInput.size() > 0) { - if (skylineStore.containsKey(recurrenceId)) { - // if filteredInput has duplicate jobIds with existing skylines in the - // store, - // throw out an exception - final List<ResourceSkyline> jobHistory = - skylineStore.get(recurrenceId); - final List<String> oldJobIds = new ArrayList<>(); - for (final ResourceSkyline resourceSkyline : jobHistory) { - oldJobIds.add(resourceSkyline.getJobId()); - } - if (!oldJobIds.isEmpty()) { - for (ResourceSkyline elem : filteredInput) { - if (oldJobIds.contains(elem.getJobId())) { - StringBuilder errMsg = new StringBuilder(); - errMsg.append( - "Trying to addHistory duplicate resource skylines for " - + recurrenceId - + ". Use updateHistory function instead."); - LOGGER.error(errMsg.toString()); - throw new DuplicateRecurrenceIdException(errMsg.toString()); - } - } - } - skylineStore.get(recurrenceId).addAll(filteredInput); - LOGGER.info("Successfully addHistory new resource skylines for {}.", - recurrenceId); - } else { - skylineStore.put(recurrenceId, filteredInput); - LOGGER.info("Successfully addHistory new resource skylines for {}.", - recurrenceId); - } - } - } finally { - writeLock.unlock(); - } - } - - @Override public void addEstimation(String pipelineId, - RLESparseResourceAllocation resourceSkyline) - throws SkylineStoreException { - inputValidator.validate(pipelineId, resourceSkyline); - writeLock.lock(); - try { - estimationStore.put(pipelineId, resourceSkyline); - LOGGER.info("Successfully add estimated resource allocation for {}.", - pipelineId); - } finally { - writeLock.unlock(); - } - } - - @Override public final void deleteHistory(final RecurrenceId recurrenceId) - throws SkylineStoreException { - inputValidator.validate(recurrenceId); - writeLock.lock(); - try { - if (skylineStore.containsKey(recurrenceId)) { - skylineStore.remove(recurrenceId); - LOGGER.warn("Delete resource skylines for {}.", recurrenceId); - } else { - StringBuilder errMsg = new StringBuilder(); - errMsg.append( - "Trying to deleteHistory non-existing recurring pipeline " - + recurrenceId + "\'s resource skylines"); - LOGGER.error(errMsg.toString()); - throw new RecurrenceIdNotFoundException(errMsg.toString()); - } - } finally { - writeLock.unlock(); - } - } - - @Override public final void updateHistory(final RecurrenceId recurrenceId, - final List<ResourceSkyline> resourceSkylines) - throws SkylineStoreException { - inputValidator.validate(recurrenceId, resourceSkylines); - writeLock.lock(); - try { - if (skylineStore.containsKey(recurrenceId)) { - // remove the null elements in the resourceSkylines - List<ResourceSkyline> filteredInput = eliminateNull(resourceSkylines); - if (filteredInput.size() > 0) { - skylineStore.put(recurrenceId, filteredInput); - LOGGER.info("Successfully updateHistory resource skylines for {}.", - recurrenceId); - } else { - StringBuilder errMsg = new StringBuilder(); - errMsg.append("Trying to updateHistory " + recurrenceId - + " with empty resource skyline"); - LOGGER.error(errMsg.toString()); - throw new EmptyResourceSkylineException(errMsg.toString()); - } - } else { - StringBuilder errMsg = new StringBuilder(); - errMsg.append( - "Trying to updateHistory non-existing resource skylines for " - + recurrenceId); - LOGGER.error(errMsg.toString()); - throw new RecurrenceIdNotFoundException(errMsg.toString()); - } - } finally { - writeLock.unlock(); - } - } - - @Override public final Map<RecurrenceId, List<ResourceSkyline>> getHistory( - final RecurrenceId recurrenceId) throws SkylineStoreException { - inputValidator.validate(recurrenceId); - readLock.lock(); - try { - String pipelineId = recurrenceId.getPipelineId(); - // User tries to getHistory all resource skylines in the skylineStore - if (pipelineId.equals("*")) { - LOGGER - .info("Successfully query resource skylines for {}.", recurrenceId); - return Collections.unmodifiableMap(skylineStore); - } - String runId = recurrenceId.getRunId(); - Map<RecurrenceId, List<ResourceSkyline>> result = - new HashMap<RecurrenceId, List<ResourceSkyline>>(); - // User tries to getHistory pipelineId's all resource skylines in the - // skylineStore - if (runId.equals("*")) { - // TODO: this for loop is expensive, so we may change the type of - // skylineStore to - // speed up this loop. - for (Map.Entry<RecurrenceId, List<ResourceSkyline>> entry : skylineStore - .entrySet()) { - RecurrenceId index = entry.getKey(); - if (index.getPipelineId().equals(pipelineId)) { - result.put(index, entry.getValue()); - } - } - if (result.size() > 0) { - LOGGER.info("Successfully query resource skylines for {}.", - recurrenceId); - return Collections.unmodifiableMap(result); - } else { - LOGGER.warn( - "Trying to getHistory non-existing resource skylines for {}.", - recurrenceId); - return null; - } - } - // User tries to getHistory {pipelineId, runId}'s resource skylines - if (skylineStore.containsKey(recurrenceId)) { - result.put(recurrenceId, skylineStore.get(recurrenceId)); - } else { - LOGGER - .warn("Trying to getHistory non-existing resource skylines for {}.", - recurrenceId); - return null; - } - LOGGER.info("Successfully query resource skylines for {}.", recurrenceId); - return Collections.unmodifiableMap(result); - } finally { - readLock.unlock(); - } - } - - @Override public final RLESparseResourceAllocation getEstimation( - String pipelineId) throws SkylineStoreException { - inputValidator.validate(pipelineId); - readLock.lock(); - try { - return estimationStore.get(pipelineId); - } finally { - readLock.unlock(); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/impl/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/impl/package-info.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/impl/package-info.java deleted file mode 100644 index ffccd5d..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/impl/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Implementation for {@code SkylineStore}. - */ - -package org.apache.hadoop.resourceestimator.skylinestore.impl; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/validator/SkylineStoreValidator.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/validator/SkylineStoreValidator.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/validator/SkylineStoreValidator.java deleted file mode 100644 index f5f50f5..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/validator/SkylineStoreValidator.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.skylinestore.validator; - -import java.util.List; - -import org.apache.hadoop.resourceestimator.common.api.RecurrenceId; -import org.apache.hadoop.resourceestimator.common.api.ResourceSkyline; -import org.apache.hadoop.resourceestimator.skylinestore.api.SkylineStore; -import org.apache.hadoop.resourceestimator.skylinestore.exceptions.NullPipelineIdException; -import org.apache.hadoop.resourceestimator.skylinestore.exceptions.NullRLESparseResourceAllocationException; -import org.apache.hadoop.resourceestimator.skylinestore.exceptions.NullRecurrenceIdException; -import org.apache.hadoop.resourceestimator.skylinestore.exceptions.NullResourceSkylineException; -import org.apache.hadoop.resourceestimator.skylinestore.exceptions.SkylineStoreException; -import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * SkylineStoreValidator validates input parameters for {@link SkylineStore}. - */ -public class SkylineStoreValidator { - private static final Logger LOGGER = - LoggerFactory.getLogger(SkylineStoreValidator.class); - - /** - * Check if recurrenceId is <em>null</em>. - * - * @param recurrenceId the id of the recurring pipeline job. - * @throws SkylineStoreException if input parameters are invalid. - */ - public final void validate(final RecurrenceId recurrenceId) - throws SkylineStoreException { - if (recurrenceId == null) { - StringBuilder sb = new StringBuilder(); - sb.append("Recurrence id is null, please try again by specifying" - + " a valid Recurrence id."); - LOGGER.error(sb.toString()); - throw new NullRecurrenceIdException(sb.toString()); - } - } - - /** - * Check if pipelineId is <em>null</em>. - * - * @param pipelineId the id of the recurring pipeline job. - * @throws SkylineStoreException if input parameters are invalid. - */ - public final void validate(final String pipelineId) - throws SkylineStoreException { - if (pipelineId == null) { - StringBuilder sb = new StringBuilder(); - sb.append("pipelineId is null, please try again by specifying" - + " a valid pipelineId."); - LOGGER.error(sb.toString()); - throw new NullPipelineIdException(sb.toString()); - } - } - - /** - * Check if recurrenceId is <em>null</em> or resourceSkylines is - * <em>null</em>. - * - * @param recurrenceId the id of the recurring pipeline job. - * @param resourceSkylines the list of {@link ResourceSkyline}s to be added. - * @throws SkylineStoreException if input parameters are invalid. - */ - public final void validate(final RecurrenceId recurrenceId, - final List<ResourceSkyline> resourceSkylines) - throws SkylineStoreException { - validate(recurrenceId); - if (resourceSkylines == null) { - StringBuilder sb = new StringBuilder(); - sb.append("ResourceSkylines for " + recurrenceId - + " is null, please try again by " - + "specifying valid ResourceSkylines."); - LOGGER.error(sb.toString()); - throw new NullResourceSkylineException(sb.toString()); - } - } - - /** - * Check if pipelineId is <em>null</em> or resourceOverTime is <em>null</em>. - * - * @param pipelineId the id of the recurring pipeline. - * @param resourceOverTime predicted {@code Resource} allocation to be added. - * @throws SkylineStoreException if input parameters are invalid. - */ - public final void validate(final String pipelineId, - final RLESparseResourceAllocation resourceOverTime) - throws SkylineStoreException { - validate(pipelineId); - if (resourceOverTime == null) { - StringBuilder sb = new StringBuilder(); - sb.append("Resource allocation for " + pipelineId + " is null."); - LOGGER.error(sb.toString()); - throw new NullRLESparseResourceAllocationException(sb.toString()); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/validator/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/validator/package-info.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/validator/package-info.java deleted file mode 100644 index 23d67c5..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/skylinestore/validator/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Validator for {@code SkylineStore}. - */ - -package org.apache.hadoop.resourceestimator.skylinestore.validator; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/api/Solver.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/api/Solver.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/api/Solver.java deleted file mode 100644 index 7958a6f..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/api/Solver.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.solver.api; - -import java.util.List; -import java.util.Map; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.resourceestimator.common.api.RecurrenceId; -import org.apache.hadoop.resourceestimator.common.api.ResourceSkyline; -import org.apache.hadoop.resourceestimator.skylinestore.api.PredictionSkylineStore; -import org.apache.hadoop.resourceestimator.skylinestore.exceptions.SkylineStoreException; -import org.apache.hadoop.resourceestimator.solver.exceptions.SolverException; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation; - -/** - * Solver takes recurring pipeline's {@link ResourceSkyline} history as input, - * predicts its {@link Resource} requirement at each time t for the next run, - * and translate them into {@link ResourceSkyline} which will be used to make - * recurring resource reservations. - */ -public interface Solver { - /** - * Initializing the Solver, including loading solver parameters from - * configuration file. - * - * @param config {@link Configuration} for the Solver. - * @param skylineStore the {@link PredictionSkylineStore} which stores - * predicted {@code Resource} allocations. - */ - void init(Configuration config, PredictionSkylineStore skylineStore); - - /** - * The Solver reads recurring pipeline's {@link ResourceSkyline} history, and - * precits its {@link ResourceSkyline} requirements for the next run. - * - * @param jobHistory the {@link ResourceSkyline}s of the recurring pipeline in - * previous runs. The {@link RecurrenceId} identifies one run of the - * recurring pipeline, and the list of {@link ResourceSkyline}s - * records the {@link ResourceSkyline} of each job within the pipeline. - * @return the amount of {@link Resource} requested by the pipeline for the - * next run (discretized by timeInterval). - * @throws SolverException if: (1) input is invalid; (2) the number of - * instances in the jobHistory is smaller than the minimum - * requirement; (3) solver runtime has unexpected behaviors; - * @throws SkylineStoreException if it fails to add predicted {@code Resource} - * allocation to the {@link PredictionSkylineStore}. - */ - RLESparseResourceAllocation solve( - Map<RecurrenceId, List<ResourceSkyline>> jobHistory) - throws SolverException, SkylineStoreException; - - /** - * Release the resource used by the Solver. - */ - void close(); -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/api/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/api/package-info.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/api/package-info.java deleted file mode 100644 index fc8363d..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/api/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * API for {@code Solver}. - */ - -package org.apache.hadoop.resourceestimator.solver.api; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/exceptions/InvalidInputException.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/exceptions/InvalidInputException.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/exceptions/InvalidInputException.java deleted file mode 100644 index ff51f5f..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/exceptions/InvalidInputException.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.solver.exceptions; - -/** - * Exception thrown the {@code SkylineStore} or the {@code Estimator} tries to - * addHistory or query pipeline job's resource skylines. - */ -public class InvalidInputException extends SolverException { - - private static final long serialVersionUID = -684069387367879218L; - - public InvalidInputException(final String entity, final String reason) { - super(entity + " is " + reason + ", please try again with valid " + entity); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/exceptions/InvalidSolverException.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/exceptions/InvalidSolverException.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/exceptions/InvalidSolverException.java deleted file mode 100644 index 9b614b6..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/exceptions/InvalidSolverException.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.solver.exceptions; - -/** - * Exception thrown the @link{SkylineStore} or the {@code Estimator} tries to - * addHistory or query pipeline job's resource skylines. - */ -public class InvalidSolverException extends SolverException { - - private static final long serialVersionUID = -684069387367879218L; - - public InvalidSolverException(final String message) { - super(message); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/exceptions/SolverException.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/exceptions/SolverException.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/exceptions/SolverException.java deleted file mode 100644 index 57507ea..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/exceptions/SolverException.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.solver.exceptions; - -/** - * Exception thrown the @link{SkylineStore} or the {@code Estimator} tries to - * addHistory or query pipeline job's resource skylines. - */ -public abstract class SolverException extends Exception { - - private static final long serialVersionUID = -684069387367879218L; - - public SolverException(final String message) { - super(message); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/exceptions/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/exceptions/package-info.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/exceptions/package-info.java deleted file mode 100644 index bd45324..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/exceptions/package-info.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -/** - * Exception module. - */ -package org.apache.hadoop.resourceestimator.solver.exceptions; http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/impl/BaseSolver.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/impl/BaseSolver.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/impl/BaseSolver.java deleted file mode 100644 index 55abb1c..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/impl/BaseSolver.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.solver.impl; - -import java.util.ArrayList; -import java.util.List; -import java.util.Random; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.resourceestimator.common.config.ResourceEstimatorConfiguration; -import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest; -import org.apache.hadoop.yarn.api.records.ReservationDefinition; -import org.apache.hadoop.yarn.api.records.ReservationId; -import org.apache.hadoop.yarn.api.records.ReservationRequest; -import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter; -import org.apache.hadoop.yarn.api.records.ReservationRequests; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation; - -/** - * Common functions shared by {@code Solver} (translate predicted resource - * allocation into Hadoop's {@link ReservationSubmissionRequest}. - */ -public abstract class BaseSolver { - /** - * Used to generate {@link ReservationId}. - */ - private static final Random RAND = new Random(); - - /** - * Translate the estimated {@link Resource} requirements of the pipeline to - * Hadoop's {@link ReservationSubmissionRequest}. - * - * @param containerSpec the {@link Resource} to be allocated to each - * container; - * @param containerRequests the predicted {@link Resource} to be allocated to - * the job in each discrete time intervals; - * @param config configuration file for BaseSolver. - * @return {@link ReservationSubmissionRequest} to be submitted to Hadoop to - * make recurring resource reservation for the pipeline. - */ - public final ReservationSubmissionRequest toRecurringRDL( - final Resource containerSpec, - final RLESparseResourceAllocation containerRequests, - final Configuration config) { - final int timeInterval = - config.getInt(ResourceEstimatorConfiguration.TIME_INTERVAL_KEY, 5); - long pipelineSubmissionTime = containerRequests.getEarliestStartTime(); - long pipelineFinishTime = containerRequests.getLatestNonNullTime(); - final long containerMemAlloc = containerSpec.getMemorySize(); - final long jobLen = - (pipelineFinishTime - pipelineSubmissionTime) / timeInterval; - List<ReservationRequest> reservationRequestList = new ArrayList<>(); - for (int i = 0; i < jobLen; i++) { - // container spec, # of containers, concurrency, duration - ReservationRequest reservationRequest = ReservationRequest - .newInstance(containerSpec, (int) ( - containerRequests.getCapacityAtTime(i * timeInterval) - .getMemorySize() / containerMemAlloc), 1, timeInterval); - reservationRequestList.add(reservationRequest); - } - ReservationRequests reservationRequests = ReservationRequests - .newInstance(reservationRequestList, - ReservationRequestInterpreter.R_ALL); - ReservationDefinition reservationDefinition = ReservationDefinition - .newInstance(pipelineSubmissionTime, pipelineFinishTime, - reservationRequests, "LpSolver#toRecurringRDL"); - ReservationId reservationId = - ReservationId.newInstance(RAND.nextLong(), RAND.nextLong()); - ReservationSubmissionRequest reservationSubmissionRequest = - ReservationSubmissionRequest - .newInstance(reservationDefinition, "resourceestimator", - reservationId); - return reservationSubmissionRequest; - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
