http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/impl/LpSolver.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/impl/LpSolver.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/impl/LpSolver.java deleted file mode 100644 index c944d20..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/impl/LpSolver.java +++ /dev/null @@ -1,340 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.solver.impl; - -import java.math.BigDecimal; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.resourceestimator.common.api.RecurrenceId; -import org.apache.hadoop.resourceestimator.common.api.ResourceSkyline; -import org.apache.hadoop.resourceestimator.common.config.ResourceEstimatorConfiguration; -import org.apache.hadoop.resourceestimator.skylinestore.api.PredictionSkylineStore; -import org.apache.hadoop.resourceestimator.skylinestore.exceptions.SkylineStoreException; -import org.apache.hadoop.resourceestimator.solver.api.Solver; -import org.apache.hadoop.resourceestimator.solver.exceptions.SolverException; -import org.apache.hadoop.resourceestimator.solver.preprocess.SolverPreprocessor; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation; -import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationInterval; -import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; -import org.ojalgo.optimisation.Expression; -import org.ojalgo.optimisation.ExpressionsBasedModel; -import org.ojalgo.optimisation.Optimisation.Result; -import org.ojalgo.optimisation.Variable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A LP(Linear Programming) solution to predict recurring pipeline's - * {@link Resource} requirements, and generate Hadoop {@code RDL} requests which - * will be used to make recurring resource reservation. - */ -public class LpSolver extends BaseSolver implements Solver { - private static final Logger LOGGER = LoggerFactory.getLogger(LpSolver.class); - private final SolverPreprocessor preprocessor = new SolverPreprocessor(); - /** - * Controls the balance between over-allocation and under-allocation. - */ - private double alpha; - /** - * Controls the generalization of the solver. - */ - private double beta; - /** - * The minimum number of job runs required to run the solver. - */ - private int minJobRuns; - /** - * The time interval which is used to discretize job execution. - */ - private int timeInterval; - /** - * The PredictionSkylineStore to store the predicted ResourceSkyline for new - * run. - */ - private PredictionSkylineStore predictionSkylineStore; - - @Override public final void init(final Configuration config, - PredictionSkylineStore skylineStore) { - this.alpha = - config.getDouble(ResourceEstimatorConfiguration.SOLVER_ALPHA_KEY, 0.1); - this.beta = - config.getDouble(ResourceEstimatorConfiguration.SOLVER_BETA_KEY, 0.1); - this.minJobRuns = - config.getInt(ResourceEstimatorConfiguration.SOLVER_MIN_JOB_RUN_KEY, 1); - this.timeInterval = - config.getInt(ResourceEstimatorConfiguration.TIME_INTERVAL_KEY, 5); - this.predictionSkylineStore = skylineStore; - } - - /** - * Generate over-allocation constraints. - * - * @param lpModel the LP model. - * @param cJobITimeK actual container allocation for job i in time - * interval k. - * @param oa container over-allocation. - * @param x predicted container allocation. - * @param indexJobITimeK index for job i at time interval k. - * @param timeK index for time interval k. - */ - private void generateOverAllocationConstraints( - final ExpressionsBasedModel lpModel, final double cJobITimeK, - final Variable[] oa, final Variable[] x, final int indexJobITimeK, - final int timeK) { - // oa_job_i_timeK >= x_timeK - cJobITimeK - Expression overAllocExpression = - lpModel.addExpression("over_alloc_" + indexJobITimeK); - overAllocExpression.set(oa[indexJobITimeK], 1); - overAllocExpression.set(x[timeK], -1); - overAllocExpression.lower(-cJobITimeK); // >= - } - - /** - * Generate under-allocation constraints. - * - * @param lpModel the LP model. - * @param cJobITimeK actual container allocation for job i in time - * interval k. - * @param uaPredict absolute container under-allocation. - * @param ua recursive container under-allocation. - * @param x predicted container allocation. - * @param indexJobITimeK index for job i at time interval k. - * @param timeK index for time interval k. - */ - private void generateUnderAllocationConstraints( - final ExpressionsBasedModel lpModel, final double cJobITimeK, - final Variable[] uaPredict, final Variable[] ua, final Variable[] x, - final int indexJobITimeK, final int timeK) { - // uaPredict_job_i_timeK + x_timeK >= cJobITimeK - Expression underAllocPredictExpression = - lpModel.addExpression("under_alloc_predict_" + indexJobITimeK); - underAllocPredictExpression.set(uaPredict[indexJobITimeK], 1); - underAllocPredictExpression.set(x[timeK], 1); - underAllocPredictExpression.lower(cJobITimeK); // >= - if (timeK >= 1) { - /** Recursively calculate container under-allocation. */ - // ua_job_i_timeK >= ua_job_i_time_(k-1) + cJobITimeK - x_timeK - Expression underAllocExpression = - lpModel.addExpression("under_alloc_" + indexJobITimeK); - underAllocExpression.set(ua[indexJobITimeK], 1); - underAllocExpression.set(ua[indexJobITimeK - 1], -1); - underAllocExpression.set(x[timeK], 1); - underAllocExpression.lower(cJobITimeK); // >= - } else { - /** Initial value for container under-allocation. */ - // ua_job_i_time_0 >= cJobI_time_0 - x_time_0 - Expression underAllocExpression = - lpModel.addExpression("under_alloc_" + indexJobITimeK); - underAllocExpression.set(ua[indexJobITimeK], 1); - underAllocExpression.set(x[timeK], 1); - underAllocExpression.lower(cJobITimeK); // >= - } - } - - /** - * Generate solver objective. - * - * @param objective LP solver objective. - * @param numJobs number of history runs of the recurring pipeline. - * @param jobLen (maximum) job lenght of the recurring pipeline. - * @param oa container over-allocation. - * @param ua recursive container under-allocation. - * @param eps regularization parameter. - */ - private void generateObjective(final Expression objective, final int numJobs, - final int jobLen, final Variable[] oa, final Variable[] ua, - final Variable eps) { - int indexJobITimeK; - // sum Over_Allocation - for (int indexJobI = 0; indexJobI < numJobs; indexJobI++) { - for (int timeK = 0; timeK < jobLen; timeK++) { - indexJobITimeK = indexJobI * jobLen + timeK; - objective.set(oa[indexJobITimeK], alpha / numJobs); - } - } - // sum Under_Allocation - int indexJobITimeN; - for (int indexJobI = 0; indexJobI < numJobs; indexJobI++) { - indexJobITimeN = indexJobI * jobLen + jobLen - 1; - objective.set(ua[indexJobITimeN], (1 - alpha) / numJobs); - } - objective.set(eps, beta); - objective.weight(BigDecimal.valueOf(1)); - } - - /** - * Get the job length of recurring pipeline. - * - * @param resourceSkylines the history ResourceSkylines allocated to the - * recurring pipeline. - * @param numJobs number of history runs of the recurring pipeline. - * @return length of (discretized time intervals of) the recurring pipeline. - */ - private int getJobLen(final List<ResourceSkyline> resourceSkylines, - final int numJobs) { - int curLen = 0; - int jobLen = 0; - for (int indexJobI = 0; indexJobI < numJobs; indexJobI++) { - curLen = (int) (resourceSkylines.get(indexJobI).getSkylineList() - .getLatestNonNullTime() - resourceSkylines.get(indexJobI) - .getSkylineList().getEarliestStartTime() + timeInterval - 1) - / timeInterval; // for round up - if (jobLen < curLen) { - jobLen = curLen; - } - } - return jobLen; - } - - @Override public final RLESparseResourceAllocation solve( - final Map<RecurrenceId, List<ResourceSkyline>> jobHistory) - throws SolverException, SkylineStoreException { - // TODO: addHistory timeout support for this function, and ideally we should - // return the confidence - // level associated with the predicted resource. - preprocessor.validate(jobHistory, timeInterval); - final List<ResourceSkyline> resourceSkylines = - preprocessor.aggregateSkylines(jobHistory, minJobRuns); - final int numJobs = resourceSkylines.size(); - final int jobLen = getJobLen(resourceSkylines, numJobs); - - /** Create variables. */ - final ExpressionsBasedModel lpModel = new ExpressionsBasedModel(); - - Variable[] oa = new Variable[jobLen * numJobs]; - Variable[] ua = new Variable[jobLen * numJobs]; - Variable[] uaPredict = new Variable[jobLen * numJobs]; - Variable[] x = new Variable[jobLen]; - for (int i = 0; i < jobLen * numJobs; i++) { - oa[i] = new Variable("oa" + i).lower(BigDecimal.valueOf(0)); - ua[i] = new Variable("ua" + i).lower(BigDecimal.valueOf(0)); - uaPredict[i] = new Variable("uaPredict" + i).lower(BigDecimal.valueOf(0)); - } - for (int i = 0; i < jobLen; i++) { - x[i] = new Variable("x").lower(BigDecimal.valueOf(0)); - } - lpModel.addVariables(x); - lpModel.addVariables(oa); - lpModel.addVariables(ua); - lpModel.addVariables(uaPredict); - Variable eps = new Variable("epsilon").lower(BigDecimal.valueOf(0)); - lpModel.addVariable(eps); - - /** Set constraints. */ - int indexJobITimeK = 0; - double cJobI = 0; - double cJobITimeK = 0; - ResourceSkyline resourceSkyline; - int[] containerNums; - // 1. sum(job_i){sum(timeK){1/cJobI * uaPredict_job_i_timeK}} <= numJobs - // * eps - Expression regularizationConstraint = - lpModel.addExpression("regularization"); - regularizationConstraint.set(eps, -numJobs); - regularizationConstraint.upper(BigDecimal.valueOf(0)); // <= 0 - for (int indexJobI = 0; - indexJobI < resourceSkylines.size(); indexJobI++) { - resourceSkyline = resourceSkylines.get(indexJobI); - // the # of containers consumed by job i in discretized time intervals - containerNums = preprocessor - .getDiscreteSkyline(resourceSkyline.getSkylineList(), timeInterval, - resourceSkyline.getContainerSpec().getMemorySize(), jobLen); - // the aggregated # of containers consumed by job i during its lifespan - cJobI = 0; - for (int i = 0; i < containerNums.length; i++) { - cJobI = cJobI + containerNums[i]; - } - for (int timeK = 0; timeK < jobLen; timeK++) { - indexJobITimeK = indexJobI * jobLen + timeK; - // the # of containers consumed by job i in the k-th time interval - cJobITimeK = containerNums[timeK]; - regularizationConstraint - .set(uaPredict[indexJobITimeK], 1 / cJobI); - generateOverAllocationConstraints(lpModel, cJobITimeK, oa, x, - indexJobITimeK, timeK); - generateUnderAllocationConstraints(lpModel, cJobITimeK, uaPredict, - ua, x, indexJobITimeK, timeK); - } - } - - /** Set objective. */ - Expression objective = lpModel.addExpression("objective"); - generateObjective(objective, numJobs, jobLen, oa, ua, eps); - - /** Solve the model. */ - final Result lpResult = lpModel.minimise(); - final TreeMap<Long, Resource> treeMap = new TreeMap<>(); - RLESparseResourceAllocation result = - new RLESparseResourceAllocation(treeMap, - new DefaultResourceCalculator()); - ReservationInterval riAdd; - Resource containerSpec = resourceSkylines.get(0).getContainerSpec(); - String pipelineId = - ((RecurrenceId) jobHistory.keySet().toArray()[0]).getPipelineId(); - Resource resource; - for (int indexTimeK = 0; indexTimeK < jobLen; indexTimeK++) { - riAdd = new ReservationInterval(indexTimeK * timeInterval, - (indexTimeK + 1) * timeInterval); - resource = Resource.newInstance( - containerSpec.getMemorySize() * (int) lpResult - .doubleValue(indexTimeK), - containerSpec.getVirtualCores() * (int) lpResult - .doubleValue(indexTimeK)); - result.addInterval(riAdd, resource); - LOGGER.debug("time interval: {}, container: {}.", indexTimeK, - lpResult.doubleValue(indexTimeK)); - } - - predictionSkylineStore.addEstimation(pipelineId, result); - - /** - * TODO: 1. We can calculate the estimated error (over-allocation, - * under-allocation) of our prediction which could be used to generate - * confidence level for our prediction; 2. Also, we can modify our model to - * take job input data size (and maybe stage info) into consideration; 3. We - * can also try to generate such conclusion: our prediction under-allocates - * X amount of resources from time 0 to time 100 compared with 95% of - * history runs; 4. We can build framework-specific versions of estimator - * (such as scope/spark/hive, etc.) and provides more specific suggestions. - * For example, we may say: for spark job i, its task size is X GB while the - * container memory allocation is Y GB; as a result, its shuffling stage is - * 20% slower than ideal case due to the disk spilling operations, etc. 5. - * If we have more information of jobs (other than ResourceSkyline), we may - * have such conclusion: job i is 20% slower than 90% of history runs, and - * it is because part of its tasks are running together with job j's tasks. - * In this case, we not only predict the amount of resource needed for job - * i, but also how to place the resource requirements to clusters; 6. We may - * monitor job progress, and dynamically increase/decrease container - * allocations to satisfy job deadline while minimizing the cost; 7. We may - * allow users to specify a budget (say $100 per job run), and optimize the - * resource allocation under the budget constraints. 8. ... - */ - return result; - } - - @Override public final void close() { - // TODO: currently place holder - } -}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/impl/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/impl/package-info.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/impl/package-info.java deleted file mode 100644 index 3344f60..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/impl/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Implementation for {@code Solver}. - */ - -package org.apache.hadoop.resourceestimator.solver.impl; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/preprocess/SolverPreprocessor.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/preprocess/SolverPreprocessor.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/preprocess/SolverPreprocessor.java deleted file mode 100644 index af61334..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/preprocess/SolverPreprocessor.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.solver.preprocess; - -import static java.lang.Math.toIntExact; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; - -import org.apache.hadoop.resourceestimator.common.api.RecurrenceId; -import org.apache.hadoop.resourceestimator.common.api.ResourceSkyline; -import org.apache.hadoop.resourceestimator.solver.api.Solver; -import org.apache.hadoop.resourceestimator.solver.exceptions.InvalidInputException; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation; -import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationInterval; -import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Common preprocessing functions for {@link Solver}. - */ -public class SolverPreprocessor { - private static final Logger LOGGER = - LoggerFactory.getLogger(SolverPreprocessor.class); - - /** - * Check if Solver's input parameters are valid. - * - * @param jobHistory the history {@link ResourceSkyline}s of the recurring - * pipeline job. - * @param timeInterval the time interval which is used to discretize the - * history {@link ResourceSkyline}s. - * @throws InvalidInputException if: (1) jobHistory is <em>null</em>; - * (2) jobHistory is empty; (3) timeout is non-positive; - * (4) timeInterval is non-positive; - */ - public final void validate( - final Map<RecurrenceId, List<ResourceSkyline>> jobHistory, - final int timeInterval) throws InvalidInputException { - if ((jobHistory == null) || (jobHistory.size() == 0)) { - LOGGER.error( - "Job resource skyline history is invalid, please try again with" - + " valid resource skyline history."); - throw new InvalidInputException("Job ResourceSkyline history", "invalid"); - } - - if (timeInterval <= 0) { - LOGGER.error( - "Solver timeInterval {} is invalid, please specify a positive value.", - timeInterval); - throw new InvalidInputException("Solver timeInterval", "non-positive"); - } - } - - /** - * Return the multi-dimension resource vector consumed by the job at specified - * time. - * - * @param skyList the list of {@link Resource}s used by the job. - * @param index the discretized time index. - * @param containerMemAlloc the multi-dimension resource vector allocated to - * one container. - * @return the multi-dimension resource vector consumed by the job. - */ - public final long getResourceVector(final RLESparseResourceAllocation skyList, - final int index, final long containerMemAlloc) { - return skyList.getCapacityAtTime(index).getMemorySize() / containerMemAlloc; - } - - /** - * Discretize job's lifespan into intervals, and return the number of - * containers used by the job within each interval. - * <p> Note that here we assume all containers allocated to the job have the - * same {@link Resource}. This is due to the limit of - * {@link RLESparseResourceAllocation}. - * - * @param skyList the list of {@link Resource}s used by the job. - * @param timeInterval the time interval used to discretize the job's - * lifespan. - * @param containerMemAlloc the amount of memory allocated to each container. - * @param jobLen the duration of the job. - * @return the number of containers allocated to the job within discretized - * time intervals. - */ - public final int[] getDiscreteSkyline( - final RLESparseResourceAllocation skyList, final int timeInterval, - final long containerMemAlloc, final int jobLen) { - long jobLifeSpan = - skyList.getLatestNonNullTime() - skyList.getEarliestStartTime(); - int[] result = new int[jobLen]; - Arrays.fill(result, 0); - - int index = 0; - long numContainerAt = 0; - for (int i = 0; i < jobLifeSpan; i++) { - index = (int) Math.floor((double) i / timeInterval); - numContainerAt = getResourceVector(skyList, i, containerMemAlloc); - if (result[index] < numContainerAt) { - result[index] = (int) numContainerAt; - } - } - return result; - } - - /** - * Merge different jobs' resource skylines into one within the same pipeline. - * - * @param resourceSkylines different jobs' resource skylines within the same - * pipeline. - * @return an aggregated resource skyline for the pipeline. - */ - public final ResourceSkyline mergeSkyline( - final List<ResourceSkyline> resourceSkylines) { - // TODO: - // rewrite this function with shift and merge once YARN-5328 is committed - /** First, getHistory the pipeline submission time. */ - long pipelineSubmission = Long.MAX_VALUE; - for (int i = 0; i < resourceSkylines.size(); i++) { - long jobSubmission = resourceSkylines.get(i).getJobSubmissionTime(); - if (pipelineSubmission > jobSubmission) { - pipelineSubmission = jobSubmission; - } - } - final TreeMap<Long, Resource> resourceOverTime = new TreeMap<>(); - final RLESparseResourceAllocation skylineListAgg = - new RLESparseResourceAllocation(resourceOverTime, - new DefaultResourceCalculator()); - /** - * Second, adjust different jobs' ResourceSkyline starting time based on - * pipeline submission time, and merge them into one ResourceSkyline. - */ - for (int i = 0; i < resourceSkylines.size(); i++) { - long jobSubmission = resourceSkylines.get(i).getJobSubmissionTime(); - long diff = (jobSubmission - pipelineSubmission) / 1000; - RLESparseResourceAllocation tmp = - resourceSkylines.get(i).getSkylineList(); - Object[] timePoints = tmp.getCumulative().keySet().toArray(); - for (int j = 0; j < timePoints.length - 2; j++) { - ReservationInterval riAdd = - new ReservationInterval(toIntExact((long) timePoints[j]) + diff, - toIntExact((long) timePoints[j + 1] + diff)); - skylineListAgg.addInterval(riAdd, - tmp.getCapacityAtTime(toIntExact((long) timePoints[j]))); - } - } - ResourceSkyline skylineAgg = - new ResourceSkyline(resourceSkylines.get(0).getJobId(), - resourceSkylines.get(0).getJobInputDataSize(), - resourceSkylines.get(0).getJobSubmissionTime(), - resourceSkylines.get(0).getJobFinishTime(), - resourceSkylines.get(0).getContainerSpec(), skylineListAgg); - - return skylineAgg; - } - - /** - * Aggregate all job's {@link ResourceSkyline}s in the one run of recurring - * pipeline, and return the aggregated {@link ResourceSkyline}s in different - * runs. - * - * @param jobHistory the history {@link ResourceSkyline} of the recurring - * pipeline job. - * @param minJobRuns the minimum number of job runs required to run the - * solver. - * @return the aggregated {@link ResourceSkyline}s in different runs. - * @throws InvalidInputException if: (1) job submission time parsing fails; - * (2) jobHistory has less job runs than the minimum requirement; - */ - public final List<ResourceSkyline> aggregateSkylines( - final Map<RecurrenceId, List<ResourceSkyline>> jobHistory, - final int minJobRuns) throws InvalidInputException { - List<ResourceSkyline> resourceSkylines = new ArrayList<ResourceSkyline>(); - for (Map.Entry<RecurrenceId, List<ResourceSkyline>> entry : jobHistory - .entrySet()) { - // TODO: identify different jobs within the same pipeline - // right now, we do prediction at the granularity of pipeline, i.e., we - // will merge the - // resource skylines of jobs within the same pipeline into one aggregated - // resource skyline - ResourceSkyline skylineAgg = null; - skylineAgg = mergeSkyline(entry.getValue()); - resourceSkylines.add(skylineAgg); - } - int numJobs = resourceSkylines.size(); - if (numJobs < minJobRuns) { - LOGGER.error( - "Solver requires job resource skyline history for at least {} runs," - + " but it only receives history info for {} runs.", - minJobRuns, numJobs); - throw new InvalidInputException("Job ResourceSkyline history", - "containing less job runs" + " than " + minJobRuns); - } - - return resourceSkylines; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/preprocess/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/preprocess/package-info.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/preprocess/package-info.java deleted file mode 100644 index b58cef9..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/solver/preprocess/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Preprocessor for {@code Solver}. - */ - -package org.apache.hadoop.resourceestimator.solver.preprocess; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/api/JobMetaData.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/api/JobMetaData.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/api/JobMetaData.java deleted file mode 100644 index 2bfab3e..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/api/JobMetaData.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.translator.api; - -import java.util.HashMap; -import java.util.Map; -import java.util.TreeMap; - -import org.apache.hadoop.resourceestimator.common.api.RecurrenceId; -import org.apache.hadoop.resourceestimator.common.api.ResourceSkyline; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation; -import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationInterval; -import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Job metadata collected when parsing the log file. - */ -public class JobMetaData { - // containerId, releaseTime - private static final Logger LOGGER = - LoggerFactory.getLogger(JobMetaData.class); - private final ResourceSkyline resourceSkyline = new ResourceSkyline(); - private final Map<String, Long> rawStart = new HashMap<String, Long>(); - // containerId, startTime - private final Map<String, Long> rawEnd = new HashMap<String, Long>(); - private RecurrenceId recurrenceId; - - /** - * Constructor. - * - * @param jobSubmissionTimeConfig job submission time. - */ - public JobMetaData(final long jobSubmissionTimeConfig) { - resourceSkyline.setJobSubmissionTime(jobSubmissionTimeConfig); - } - - /** - * Set job finish time. - * - * @param jobFinishTimeConfig job finish time. - * @return the reference to current {@link JobMetaData}. - */ - public final JobMetaData setJobFinishTime(final long jobFinishTimeConfig) { - resourceSkyline.setJobFinishTime(jobFinishTimeConfig); - return this; - } - - /** - * Add container launch time. - * - * @param containerId id of the container. - * @param time container launch time. - * @return the reference to current {@link JobMetaData}. - */ - public final JobMetaData setContainerStart(final String containerId, - final long time) { - if (rawStart.put(containerId, time) != null) { - LOGGER.warn("find duplicate container launch time for {}, so we replace" - + " it with {}.", containerId, time); - } - return this; - } - - /** - * Add container release time. - * - * @param containerId id of the container. - * @param time container release time. - * @return the reference to current {@link JobMetaData}. - */ - public final JobMetaData setContainerEnd(final String containerId, - final long time) { - if (rawEnd.put(containerId, time) != null) { - LOGGER.warn("find duplicate container release time for {}, so we replace" - + " it with {}.", containerId, time); - } - return this; - } - - /** - * Get {@link RecurrenceId}. - * - * @return {@link RecurrenceId}. - */ - public final RecurrenceId getRecurrenceId() { - return recurrenceId; - } - - /** - * Set {@link RecurrenceId}. - * - * @param recurrenceIdConfig the {@link RecurrenceId}. - * @return the reference to current {@link JobMetaData}. - */ - public final JobMetaData setRecurrenceId( - final RecurrenceId recurrenceIdConfig) { - this.recurrenceId = recurrenceIdConfig; - return this; - } - - /** - * Get {@link ResourceSkyline}. - * - * @return {@link ResourceSkyline}. - */ - public final ResourceSkyline getResourceSkyline() { - return resourceSkyline; - } - - /** - * Normalized container launch/release time, and generate the - * {@link ResourceSkyline}. - */ - public final void createSkyline() { - final long jobSubmissionTime = resourceSkyline.getJobSubmissionTime(); - Resource containerSpec = resourceSkyline.getContainerSpec(); - final TreeMap<Long, Resource> resourceOverTime = new TreeMap<>(); - final RLESparseResourceAllocation skylineList = - new RLESparseResourceAllocation(resourceOverTime, - new DefaultResourceCalculator()); - resourceSkyline.setSkylineList(skylineList); - if (containerSpec == null) { - // if RmParser fails to extract container resource spec from logs, we will - // statically set - // it to be <1core, 1GB> - containerSpec = Resource.newInstance(1024, 1); - } - resourceSkyline.setContainerSpec(containerSpec); - for (final Map.Entry<String, Long> entry : rawStart.entrySet()) { - final long timeStart = entry.getValue(); - final Long timeEnd = rawEnd.get(entry.getKey()); - if (timeEnd == null) { - LOGGER.warn("container release time not found for {}.", entry.getKey()); - } else { - final ReservationInterval riAdd = - new ReservationInterval((timeStart - jobSubmissionTime) / 1000, - (timeEnd - jobSubmissionTime) / 1000); - resourceSkyline.getSkylineList().addInterval(riAdd, containerSpec); - } - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/api/LogParser.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/api/LogParser.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/api/LogParser.java deleted file mode 100644 index b7efe5a..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/api/LogParser.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.translator.api; - -import java.io.IOException; -import java.io.InputStream; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.resourceestimator.common.exception.ResourceEstimatorException; -import org.apache.hadoop.resourceestimator.skylinestore.api.HistorySkylineStore; -import org.apache.hadoop.resourceestimator.skylinestore.exceptions.SkylineStoreException; - -/** - * LogParser iterates over a stream of logs, uses {@link SingleLineParser} to - * parse each line, and adds extracted {@code ResourceSkyline}s to the - * {@code SkylineStore}. - */ -public interface LogParser extends AutoCloseable { - - /** - * Initializing the LogParser, including loading solver parameters from - * configuration file. - * - * @param config {@link Configuration} for the LogParser. - * @param skylineStore the {@link HistorySkylineStore} which stores recurring - * pipeline's {@code - * ResourceSkyline}s. - * @throws ResourceEstimatorException if initialization of a - * {@code SingleLineParser} fails. - */ - void init(Configuration config, HistorySkylineStore skylineStore) - throws ResourceEstimatorException; - - /** - * Parses each line in the log stream, and adds extracted - * {@code ResourceSkyline}s to the {@code - * SkylineStore}. - * - * @param logs the stream of input logs. - * @throws SkylineStoreException if it fails to addHistory extracted - * {@code ResourceSkyline}s to the {@code SkylineStore}. - * @throws IOException if it fails to read from the {@link InputStream}. - */ - void parseStream(InputStream logs) throws SkylineStoreException, IOException; - - @Override void close(); -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/api/SingleLineParser.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/api/SingleLineParser.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/api/SingleLineParser.java deleted file mode 100644 index b9607fe..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/api/SingleLineParser.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.translator.api; - -import java.text.ParseException; -import java.util.List; -import java.util.Map; - -import org.apache.hadoop.resourceestimator.common.api.RecurrenceId; -import org.apache.hadoop.resourceestimator.common.api.ResourceSkyline; -import org.apache.hadoop.resourceestimator.translator.exceptions.DataFieldNotFoundException; - -/** - * SingleLineParser parses one line in the log file, extracts the - * {@link ResourceSkyline}s and stores them. - */ -public interface SingleLineParser { - /** - * Parse one line in the log file, extract the {@link ResourceSkyline}s and - * store them. - * - * @param logLine one line in the log file. - * @param jobMetas the job metadata collected during parsing. - * @param skylineRecords the valid {@link ResourceSkyline}s extracted from the - * log. - * @throws DataFieldNotFoundException if certain data fields are not found in - * the log. - * @throws ParseException if it fails to convert date string to - * unix timestamp successfully. - */ - void parseLine(String logLine, Map<String, JobMetaData> jobMetas, - Map<RecurrenceId, List<ResourceSkyline>> skylineRecords) - throws DataFieldNotFoundException, ParseException; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/api/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/api/package-info.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/api/package-info.java deleted file mode 100644 index a1b79c0..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/api/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * API for {@code Translator}. - */ - -package org.apache.hadoop.resourceestimator.translator.api; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/exceptions/DataFieldNotFoundException.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/exceptions/DataFieldNotFoundException.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/exceptions/DataFieldNotFoundException.java deleted file mode 100644 index 6eae614b..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/exceptions/DataFieldNotFoundException.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.translator.exceptions; - -/** - * Exception thrown when job attributes are not found. - */ -public class DataFieldNotFoundException extends Exception { - private static final long serialVersionUID = -684069387367879218L; - - public DataFieldNotFoundException(final String log) { - super("Fail to extract data fields properly from " + log); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/exceptions/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/exceptions/package-info.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/exceptions/package-info.java deleted file mode 100644 index 104388d..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/exceptions/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Exception for {@code Translator}. - */ - -package org.apache.hadoop.resourceestimator.translator.exceptions; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/BaseLogParser.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/BaseLogParser.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/BaseLogParser.java deleted file mode 100644 index 50d911f..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/BaseLogParser.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.translator.impl; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.text.ParseException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.resourceestimator.common.api.RecurrenceId; -import org.apache.hadoop.resourceestimator.common.api.ResourceSkyline; -import org.apache.hadoop.resourceestimator.common.config.ResourceEstimatorConfiguration; -import org.apache.hadoop.resourceestimator.common.config.ResourceEstimatorUtil; -import org.apache.hadoop.resourceestimator.common.exception.ResourceEstimatorException; -import org.apache.hadoop.resourceestimator.skylinestore.api.HistorySkylineStore; -import org.apache.hadoop.resourceestimator.skylinestore.exceptions.SkylineStoreException; -import org.apache.hadoop.resourceestimator.translator.api.JobMetaData; -import org.apache.hadoop.resourceestimator.translator.api.LogParser; -import org.apache.hadoop.resourceestimator.translator.api.SingleLineParser; -import org.apache.hadoop.resourceestimator.translator.exceptions.DataFieldNotFoundException; -import org.apache.hadoop.resourceestimator.translator.validator.ParserValidator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Base class to implement {@link LogParser}. It wraps a - * {@link SingleLineParser} from the {@link Configuration} to parse a log - * dir/file. - */ -public class BaseLogParser implements LogParser { - private static final Logger LOGGER = - LoggerFactory.getLogger(BaseLogParser.class); - private static final ParserValidator INPUT_VALIDATOR = new ParserValidator(); - private SingleLineParser singleLineParser; - private HistorySkylineStore historySkylineStore; - - @Override public void init(Configuration config, - HistorySkylineStore skylineStore) throws ResourceEstimatorException { - singleLineParser = ResourceEstimatorUtil.createProviderInstance(config, - ResourceEstimatorConfiguration.TRANSLATOR_LINE_PARSER, - ResourceEstimatorConfiguration.DEFAULT_TRANSLATOR_LINE_PARSER, - SingleLineParser.class); - this.historySkylineStore = skylineStore; - } - - /** - * Add job's {@link ResourceSkyline}s to the {@link HistorySkylineStore}. - * - * @param skylineRecords the {@link Map} which records the completed recurring - * pipeline's {@link ResourceSkyline}s. - * @throws SkylineStoreException if it failes to addHistory job's - * {@link ResourceSkyline}s to the {@link HistorySkylineStore}. - */ - private void addToSkylineStore( - final Map<RecurrenceId, List<ResourceSkyline>> skylineRecords) - throws SkylineStoreException { - for (final Map.Entry<RecurrenceId, List<ResourceSkyline>> entry : - skylineRecords.entrySet()) { - historySkylineStore.addHistory(entry.getKey(), entry.getValue()); - } - } - - public void parseLine(final String logLine, - final Map<String, JobMetaData> jobMetas, - final Map<RecurrenceId, List<ResourceSkyline>> skylineRecords) - throws DataFieldNotFoundException, ParseException { - singleLineParser.parseLine(logLine, jobMetas, skylineRecords); - } - - @Override public final void parseStream(final InputStream logs) - throws SkylineStoreException, IOException { - if (!INPUT_VALIDATOR.validate(logs)) { - LOGGER.error("Input validation fails, please specify with" - + " valid input parameters."); - return; - } - final Map<RecurrenceId, List<ResourceSkyline>> skylineRecords = - new HashMap<>(); - final Map<String, JobMetaData> jobMetas = - new HashMap<String, JobMetaData>(); - final BufferedReader bf = new BufferedReader(new InputStreamReader(logs)); - String line = null; - while ((line = bf.readLine()) != null) { - try { - parseLine(line, jobMetas, skylineRecords); - } catch (DataFieldNotFoundException e) { - LOGGER.debug("Data field not found", e); - } catch (ParseException e) { - LOGGER.debug("Date conversion error", e); - } - } - - addToSkylineStore(skylineRecords); - } - - /** - * Release the resource used by the ParserUtil. - */ - @Override public final void close() { - historySkylineStore = null; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/LogParserUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/LogParserUtil.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/LogParserUtil.java deleted file mode 100644 index 35da799..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/LogParserUtil.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.translator.impl; - -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.text.DateFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; - -import org.apache.hadoop.resourceestimator.common.exception.ResourceEstimatorException; -import org.apache.hadoop.resourceestimator.skylinestore.api.SkylineStore; -import org.apache.hadoop.resourceestimator.skylinestore.exceptions.SkylineStoreException; -import org.apache.hadoop.resourceestimator.translator.api.LogParser; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Common utility functions for {@link LogParser}. - */ -public class LogParserUtil { - private static final Logger LOGGER = - LoggerFactory.getLogger(LogParserUtil.class); - private LogParser logParser; - private DateFormat format = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss"); - - /** - * Set the {@link LogParser} to use. - * - * @param logParser the {@link LogParser} to use. - */ - public void setLogParser(final LogParser logParser) { - this.logParser = logParser; - } - - /** - * Set date format for the {@link LogParser}. - * - * @param datePattern the date pattern in the log. - */ - public void setDateFormat(final String datePattern) { - this.format = new SimpleDateFormat(datePattern); - } - - /** - * Converts String date to unix timestamp. Note that we assume the time in the - * logs has the same time zone with the machine which runs the - * {@link RmSingleLineParser}. - * - * @param date The String date. - * @return Unix time stamp. - * @throws ParseException if data conversion from String to unix timestamp - * fails. - */ - public long stringToUnixTimestamp(final String date) throws ParseException { - return format.parse(date).getTime(); - } - - /** - * Parse the log file/directory. - * - * @param logFile the file/directory of the log. - * @throws SkylineStoreException if fails to addHistory to - * {@link SkylineStore}. - * @throws IOException if fails to parse the log. - * @throws ResourceEstimatorException if the {@link LogParser} - * is not initialized. - */ - public final void parseLog(final String logFile) - throws SkylineStoreException, IOException, ResourceEstimatorException { - if (logParser == null) { - throw new ResourceEstimatorException("The log parser is not initialized," - + " please try again after initializing."); - } - InputStream inputStream = new FileInputStream(logFile); - logParser.parseStream(inputStream); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/NativeSingleLineParser.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/NativeSingleLineParser.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/NativeSingleLineParser.java deleted file mode 100644 index 83557d3..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/NativeSingleLineParser.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.translator.impl; - -import java.text.ParseException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.resourceestimator.common.api.RecurrenceId; -import org.apache.hadoop.resourceestimator.common.api.ResourceSkyline; -import org.apache.hadoop.resourceestimator.common.config.ResourceEstimatorConfiguration; -import org.apache.hadoop.resourceestimator.translator.api.JobMetaData; -import org.apache.hadoop.resourceestimator.translator.api.SingleLineParser; -import org.apache.hadoop.resourceestimator.translator.exceptions.DataFieldNotFoundException; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation; -import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationInterval; -import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This sample parser will parse the sample log and extract the resource - * skyline. - * <p> The expected log format is: NormalizedJobName NumInstances SubmitTime - * StartTime EndTime JobInstanceName memUsage coreUsage - */ -public class NativeSingleLineParser implements SingleLineParser { - private static final Logger LOGGER = - LoggerFactory.getLogger(NativeSingleLineParser.class); - - /** - * Aggregates different jobs' {@link ResourceSkyline}s within the same - * pipeline together. - * - * @param resourceSkyline newly extracted {@link ResourceSkyline}. - * @param recurrenceId the {@link RecurrenceId} which the resourceSkyline - * belongs to. - * @param skylineRecords a {@link Map} which stores the - * {@link ResourceSkyline}s for all pipelines during this parsing. - */ - private void aggregateSkyline(final ResourceSkyline resourceSkyline, - final RecurrenceId recurrenceId, - final Map<RecurrenceId, List<ResourceSkyline>> skylineRecords) { - List<ResourceSkyline> resourceSkylines = skylineRecords.get(recurrenceId); - if (resourceSkylines == null) { - resourceSkylines = new ArrayList<ResourceSkyline>(); - skylineRecords.put(recurrenceId, resourceSkylines); - } - resourceSkylines.add(resourceSkyline); - } - - @Override public void parseLine(String logLine, - Map<String, JobMetaData> jobMetas, - Map<RecurrenceId, List<ResourceSkyline>> skylineRecords) - throws DataFieldNotFoundException, ParseException { - Configuration config = new Configuration(); - config.addResource(new org.apache.hadoop.fs.Path( - ResourceEstimatorConfiguration.CONFIG_FILE)); - int timeInterval = - config.getInt(ResourceEstimatorConfiguration.TIME_INTERVAL_KEY, 5); - // note that for native log, we assume each container is allocated <1 core, - // 1GB RAM> - long containerMemAlloc = 1024; - int containerCpuAlloc = 1; - String[] splitString = logLine.split("\\s+"); - String pipelineId = splitString[0]; - String jobId = splitString[5]; - String[] skylineUnits = splitString[7].split("\\|"); - - JobMetaData appMeta = new JobMetaData(0); - RecurrenceId recurrenceId = new RecurrenceId(pipelineId, jobId); - appMeta.setRecurrenceId(recurrenceId); - Resource containerAlloc; - int numContainers; - ResourceSkyline resourceSkyline = appMeta.getResourceSkyline(); - final TreeMap<Long, Resource> resourceOverTime = new TreeMap<>(); - final RLESparseResourceAllocation skylineList = - new RLESparseResourceAllocation(resourceOverTime, - new DefaultResourceCalculator()); - resourceSkyline.setSkylineList(skylineList); - for (String elem : skylineUnits) { - numContainers = Integer.parseInt(elem.split("\\:")[0]); - containerAlloc = Resource.newInstance(containerMemAlloc * numContainers, - containerCpuAlloc * numContainers); - final ReservationInterval riAdd = - new ReservationInterval(Long.parseLong(elem.split("\\:")[1]), - Long.parseLong(elem.split("\\:")[1]) + timeInterval); - resourceSkyline.getSkylineList().addInterval(riAdd, containerAlloc); - } - resourceSkyline.setContainerSpec( - Resource.newInstance(containerMemAlloc, containerCpuAlloc)); - appMeta.setJobFinishTime( - appMeta.getResourceSkyline().getSkylineList().getLatestNonNullTime()); - resourceSkyline.setJobInputDataSize(0); - resourceSkyline.setJobId(jobId); - aggregateSkyline(resourceSkyline, recurrenceId, skylineRecords); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/RmSingleLineParser.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/RmSingleLineParser.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/RmSingleLineParser.java deleted file mode 100644 index 1904934..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/RmSingleLineParser.java +++ /dev/null @@ -1,203 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.translator.impl; - -import java.text.ParseException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.hadoop.resourceestimator.common.api.RecurrenceId; -import org.apache.hadoop.resourceestimator.common.api.ResourceSkyline; -import org.apache.hadoop.resourceestimator.translator.api.JobMetaData; -import org.apache.hadoop.resourceestimator.translator.api.SingleLineParser; -import org.apache.hadoop.resourceestimator.translator.exceptions.DataFieldNotFoundException; -import org.apache.hadoop.yarn.api.records.Resource; - -/** - * {@link SingleLineParser} for Hadoop Resource Manager logs. - */ -public class RmSingleLineParser implements SingleLineParser { - private static final LogParserUtil PARSERUTIL = new LogParserUtil(); - private static final Pattern FILTER_PATTERN = Pattern.compile( - "(Submit Application Request|AM Allocated Container|" - + "AM Released Container|finalState=FAILED|" - + "ApplicationSummary|, Resource:)"); - private static final Pattern SUBMISSION_PATTERN = - Pattern.compile("APPID=(\\w+)"); - private static final Pattern FAIL_PATTERN = - Pattern.compile("appattempt_(\\d+_\\d+)_\\d+"); - private static final Pattern FINISH_PATTERN = - Pattern.compile("appId=(\\w+).*?name=(\\w+)\\-(\\w+)"); - private static final Pattern CONTAINER_EVENT_PATTERN = - Pattern.compile("APPID=(\\w+).*?CONTAINERID=(\\w+)"); - private static final Pattern CONTAINER_SPEC_PATTERN = Pattern.compile( - "(container_[^_]+|appattempt)_(\\d+_\\d+).*?memory:(\\d+)," - + "\\svCores:(\\d+)"); - - /** - * Aggregates different jobs' {@link ResourceSkyline}s within the same - * pipeline together. - * - * @param resourceSkyline newly extracted {@link ResourceSkyline}. - * @param recurrenceId the {@link RecurrenceId} which the resourceSkyline - * belongs to. - * @param skylineRecords a {@link Map} which stores the - * {@link ResourceSkyline}s for all pipelines during this parsing. - */ - private void aggregateSkyline(final ResourceSkyline resourceSkyline, - final RecurrenceId recurrenceId, - final Map<RecurrenceId, List<ResourceSkyline>> skylineRecords) { - List<ResourceSkyline> resourceSkylines = skylineRecords.get(recurrenceId); - if (resourceSkylines == null) { - resourceSkylines = new ArrayList<ResourceSkyline>(); - skylineRecords.put(recurrenceId, resourceSkylines); - } - resourceSkylines.add(resourceSkyline); - } - - @Override public final void parseLine(final String logLine, - final Map<String, JobMetaData> jobMetas, - final Map<RecurrenceId, List<ResourceSkyline>> skylineRecords) - throws DataFieldNotFoundException, ParseException { - final String[] splits = logLine.split(",", 5); // Limit the max number of 5 - // splits - if (splits.length < 5) { - return; - } - final Matcher jobEventMatcher = FILTER_PATTERN.matcher(splits[4]); // search - // only - // the - // tail - if (!jobEventMatcher.find()) { // this line of log does not contain targeted - // events - return; - } - - // now we have the match, let's do some parsing - final long date = PARSERUTIL.stringToUnixTimestamp(splits[1]); - final String tail = splits[4].split("\\s+", 4)[3]; // use the tail of the - // tail only - switch (jobEventMatcher.group(1)) { - case "Submit Application Request": { - /** Submit job. */ - final Matcher appIdMatch = SUBMISSION_PATTERN.matcher(tail); - if (appIdMatch.find()) { - final String appId = appIdMatch.group(1); - jobMetas.put(appId, new JobMetaData(date)); - } else { - throw new DataFieldNotFoundException(tail); - } - break; - } - case "AM Allocated Container": { - /** Allocate container. */ - final Matcher containerEventMatcher = - CONTAINER_EVENT_PATTERN.matcher(tail); - if (containerEventMatcher.find()) { - final String appId = containerEventMatcher.group(1); - final String containerId = containerEventMatcher.group(2); - final JobMetaData appMeta = jobMetas.get(appId); - if (appMeta != null) { - appMeta.setContainerStart(containerId, date); - } - } else { - throw new DataFieldNotFoundException(tail); - } - break; - } - case ", Resource:": { - final Matcher containerSpecMatcher = CONTAINER_SPEC_PATTERN.matcher(tail); - if (containerSpecMatcher.find()) { - final String appId = "application_" + containerSpecMatcher.group(2); - final JobMetaData appMeta = jobMetas.get(appId); - if (appMeta != null) { - final long memAlloc = Long.parseLong(containerSpecMatcher.group(3)); - final int cpuAlloc = Integer.parseInt(containerSpecMatcher.group(4)); - final Resource containerAlloc = - Resource.newInstance(memAlloc, cpuAlloc); - appMeta.getResourceSkyline().setContainerSpec(containerAlloc); - } - } else { - throw new DataFieldNotFoundException(tail); - } - break; - } - case "AM Released Container": { - final Matcher containerEventMatcher = - CONTAINER_EVENT_PATTERN.matcher(tail); - if (containerEventMatcher.find()) { - final String appId = containerEventMatcher.group(1); - final JobMetaData appMeta = jobMetas.get(appId); - if (appMeta != null) { - final String containerId = containerEventMatcher.group(2); - appMeta.setContainerEnd(containerId, date); - } - } else { - throw new DataFieldNotFoundException(tail); - } - break; - } - case "finalState=FAILED": { - /** In case of appAttempt failed: discard previous records. */ - final Matcher failMatcher = FAIL_PATTERN.matcher(tail); - if (failMatcher.find()) { - final String appId = "application_" + failMatcher.group(1); - if (jobMetas.containsKey(appId)) { - jobMetas.put(appId, new JobMetaData(date)); - } - } else { - throw new DataFieldNotFoundException(tail); - } - break; - } - case "ApplicationSummary": { - /** Finish a job. */ - final Matcher finishMatcher = FINISH_PATTERN.matcher(tail); - if (finishMatcher.find()) { - final String appId = finishMatcher.group(1); - final String pipelineId = finishMatcher.group(2); - final String runId = finishMatcher.group(3); - final RecurrenceId recurrenceId = new RecurrenceId(pipelineId, runId); - final JobMetaData appMeta = jobMetas.remove(appId); - if (appMeta != null) { - appMeta.setRecurrenceId(recurrenceId).setJobFinishTime(date) - .getResourceSkyline().setJobInputDataSize(0); // TODO: need to - // read job input - // data size from - // logs - appMeta.createSkyline(); - final ResourceSkyline resourceSkyline = appMeta.getResourceSkyline(); - resourceSkyline.setJobId(appId); - aggregateSkyline(resourceSkyline, recurrenceId, skylineRecords); - } - } else { - throw new DataFieldNotFoundException(tail); - } - break; - } - default: - break; - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/package-info.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/package-info.java deleted file mode 100644 index 13b5138..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Implementation for {@code Translator}. - */ - -package org.apache.hadoop.resourceestimator.translator.impl; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/validator/ParserValidator.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/validator/ParserValidator.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/validator/ParserValidator.java deleted file mode 100644 index ebf50ba..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/validator/ParserValidator.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.translator.validator; - -import java.io.InputStream; - -import org.apache.hadoop.resourceestimator.translator.api.LogParser; - -/** - * Validates the input parameters for the {@link LogParser}. - */ -public class ParserValidator { - /** - * Validates the input parameters for the {@link LogParser}. - * - * @param logs input log streams to the {@link LogParser}. - * @return whether the input parameters are valid or not. - */ - public final boolean validate(final InputStream logs) { - // TODO - return true; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/validator/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/validator/package-info.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/validator/package-info.java deleted file mode 100644 index c48b262..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/validator/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Validator for {@code Translator}. - */ - -package org.apache.hadoop.resourceestimator.translator.validator; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/resources/webapps/ResourceEstimatorServer/.gitignore ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/resources/webapps/ResourceEstimatorServer/.gitignore b/hadoop-tools/hadoop-resourceestimator/src/main/resources/webapps/ResourceEstimatorServer/.gitignore deleted file mode 100644 index ae1e83e..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/resources/webapps/ResourceEstimatorServer/.gitignore +++ /dev/null @@ -1,14 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
