Revert "HADOOP-14840. Tool to estimate resource requirements of an application pipeline based on prior executions. (Rui Li via Subru)."
This reverts commit da7f989d86d28ee6f4a7ac410fc8a4a163e11012. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/13f1a7e0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/13f1a7e0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/13f1a7e0 Branch: refs/heads/branch-2 Commit: 13f1a7e0edf05e18963b1b00999dc9f320c8cd53 Parents: da7f989 Author: Akira Ajisaka <[email protected]> Authored: Thu Oct 26 15:33:53 2017 +0900 Committer: Akira Ajisaka <[email protected]> Committed: Thu Oct 26 15:33:53 2017 +0900 ---------------------------------------------------------------------- LICENSE.txt | 1 - .../assemblies/hadoop-resourceestimator.xml | 42 -- .../main/resources/assemblies/hadoop-tools.xml | 11 - hadoop-project/pom.xml | 5 - hadoop-project/src/site/site.xml | 1 - hadoop-tools/hadoop-resourceestimator/README.md | 19 - hadoop-tools/hadoop-resourceestimator/pom.xml | 175 ------- .../src/config/checkstyle.xml | 50 -- .../src/main/bin/estimator.cmd | 52 --- .../src/main/bin/estimator.sh | 71 --- .../src/main/bin/start-estimator.cmd | 37 -- .../src/main/bin/start-estimator.sh | 42 -- .../src/main/bin/stop-estimator.cmd | 37 -- .../src/main/bin/stop-estimator.sh | 42 -- .../src/main/conf/resourceestimator-config.xml | 85 ---- .../src/main/data/resourceEstimatorService.txt | 2 - .../common/api/RecurrenceId.java | 95 ---- .../common/api/ResourceSkyline.java | 211 --------- .../common/api/package-info.java | 23 - .../config/ResourceEstimatorConfiguration.java | 125 ----- .../common/config/ResourceEstimatorUtil.java | 81 ---- .../common/config/package-info.java | 23 - .../exception/ResourceEstimatorException.java | 35 -- .../common/exception/package-info.java | 23 - .../RLESparseResourceAllocationSerDe.java | 77 --- .../common/serialization/ResourceSerDe.java | 61 --- .../common/serialization/package-info.java | 24 - .../service/ResourceEstimatorServer.java | 146 ------ .../service/ResourceEstimatorService.java | 238 ---------- .../resourceestimator/service/ShutdownHook.java | 45 -- .../resourceestimator/service/package-info.java | 23 - .../skylinestore/api/HistorySkylineStore.java | 99 ---- .../api/PredictionSkylineStore.java | 60 --- .../skylinestore/api/SkylineStore.java | 30 -- .../skylinestore/api/package-info.java | 23 - .../DuplicateRecurrenceIdException.java | 33 -- .../EmptyResourceSkylineException.java | 33 -- .../exceptions/NullPipelineIdException.java | 32 -- ...ullRLESparseResourceAllocationException.java | 33 -- .../exceptions/NullRecurrenceIdException.java | 32 -- .../NullResourceSkylineException.java | 32 -- .../RecurrenceIdNotFoundException.java | 33 -- .../exceptions/SkylineStoreException.java | 33 -- .../skylinestore/exceptions/package-info.java | 24 - .../skylinestore/impl/InMemoryStore.java | 256 ---------- .../skylinestore/impl/package-info.java | 23 - .../validator/SkylineStoreValidator.java | 118 ----- .../skylinestore/validator/package-info.java | 23 - .../resourceestimator/solver/api/Solver.java | 76 --- .../solver/api/package-info.java | 23 - .../exceptions/InvalidInputException.java | 34 -- .../exceptions/InvalidSolverException.java | 34 -- .../solver/exceptions/SolverException.java | 34 -- .../solver/exceptions/package-info.java | 24 - .../solver/impl/BaseSolver.java | 94 ---- .../resourceestimator/solver/impl/LpSolver.java | 340 -------------- .../solver/impl/package-info.java | 23 - .../solver/preprocess/SolverPreprocessor.java | 219 --------- .../solver/preprocess/package-info.java | 23 - .../translator/api/JobMetaData.java | 163 ------- .../translator/api/LogParser.java | 65 --- .../translator/api/SingleLineParser.java | 52 --- .../translator/api/package-info.java | 23 - .../exceptions/DataFieldNotFoundException.java | 32 -- .../translator/exceptions/package-info.java | 23 - .../translator/impl/BaseLogParser.java | 125 ----- .../translator/impl/LogParserUtil.java | 97 ---- .../translator/impl/NativeSingleLineParser.java | 120 ----- .../translator/impl/RmSingleLineParser.java | 203 -------- .../translator/impl/package-info.java | 23 - .../translator/validator/ParserValidator.java | 41 -- .../translator/validator/package-info.java | 23 - .../webapps/ResourceEstimatorServer/.gitignore | 14 - .../src/site/markdown/ResourceEstimator.md | 181 -------- .../src/site/resources/css/site.css | 29 -- .../common/api/TestResourceSkyline.java | 128 ----- .../serialization/TestHistorySkylineSerDe.java | 134 ------ .../common/serialization/TestResourceSerDe.java | 64 --- .../serialization/TestResourceSkylineSerDe.java | 112 ----- .../common/serialization/package-info.java | 24 - .../service/GuiceServletConfig.java | 42 -- .../service/TestResourceEstimatorService.java | 282 ----------- .../skylinestore/impl/TestInMemoryStore.java | 32 -- .../skylinestore/impl/TestSkylineStore.java | 464 ------------------- .../solver/impl/TestLpSolver.java | 112 ----- .../solver/impl/TestSolver.java | 73 --- .../translator/api/TestJobMetaData.java | 163 ------- .../translator/impl/TestNativeParser.java | 115 ----- .../translator/impl/TestRmParser.java | 239 ---------- .../src/test/resources/log4j.properties | 25 - .../src/test/resources/lp/answer.txt | 7 - .../src/test/resources/lp/tinySample.txt | 2 - .../src/test/resources/lp/tpch_q12.txt | 13 - .../test/resources/resourceEstimatorService.txt | 2 - .../test/resources/resourceestimator-config.xml | 85 ---- .../src/test/resources/trace/invalidLog1.txt | 4 - .../src/test/resources/trace/invalidLog2.txt | 3 - .../src/test/resources/trace/invalidLog3.txt | 5 - .../src/test/resources/trace/invalidLog4.txt | 5 - .../src/test/resources/trace/invalidLog5.txt | 7 - .../src/test/resources/trace/invalidLog6.txt | 5 - .../src/test/resources/trace/invalidLog7.txt | 5 - .../src/test/resources/trace/invalidLog8.txt | 6 - .../src/test/resources/trace/invalidLog9.txt | 6 - .../src/test/resources/trace/nativeLog.txt | 1 - .../src/test/resources/trace/rmLog.txt | 30 -- hadoop-tools/hadoop-tools-dist/pom.xml | 6 - hadoop-tools/pom.xml | 1 - 108 files changed, 7159 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/LICENSE.txt ---------------------------------------------------------------------- diff --git a/LICENSE.txt b/LICENSE.txt index 09a25d8..ebabcf9 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -618,7 +618,6 @@ hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery Apache HBase - Server which contains JQuery minified javascript library version 1.8.3 Microsoft JDBC Driver for SQLServer - version 6.2.1.jre7 -oj! Algorithms - version 43.0 -------------------------------------------------------------------------------- Copyright jQuery Foundation and other contributors, https://jquery.org/ http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-assemblies/src/main/resources/assemblies/hadoop-resourceestimator.xml ---------------------------------------------------------------------- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-resourceestimator.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-resourceestimator.xml deleted file mode 100644 index d1d941d..0000000 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-resourceestimator.xml +++ /dev/null @@ -1,42 +0,0 @@ -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd";> - <id>hadoop-resourceestimator</id> - <formats> - <format>dir</format> - </formats> - <includeBaseDirectory>false</includeBaseDirectory> - - <fileSets> - <fileSet> - <directory>${basedir}/src/main/bin</directory> - <outputDirectory>resourceestimator/bin</outputDirectory> - <fileMode>0755</fileMode> - </fileSet> - <fileSet> - <directory>${basedir}/src/main/conf</directory> - <outputDirectory>resourceestimator/conf</outputDirectory> - </fileSet> - <fileSet> - <directory>${basedir}/src/main/data</directory> - <outputDirectory>resourceestimator/data</outputDirectory> - </fileSet> - </fileSets> - </assembly> http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml ---------------------------------------------------------------------- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml index e6a9a76..d7c3c95 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml @@ -118,17 +118,6 @@ <directory>../hadoop-sls/target/hadoop-sls-${project.version}/sls</directory> <outputDirectory>/share/hadoop/${hadoop.component}/sls</outputDirectory> </fileSet> - <fileSet> - <directory>../hadoop-resourceestimator/target</directory> - <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory> - <includes> - <include>*-sources.jar</include> - </includes> - </fileSet> - <fileSet> - <directory>../hadoop-resourceestimator/target/hadoop-resourceestimator-${project.version}/resourceestimator</directory> - <outputDirectory>/share/hadoop/${hadoop.component}/resourceestimator</outputDirectory> - </fileSet> </fileSets> <dependencySets> <dependencySet> http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-project/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index ea02765..2c8f4a9 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -618,11 +618,6 @@ <version>3.2</version> </dependency> <dependency> - <groupId>org.ojalgo</groupId> - <artifactId>ojalgo</artifactId> - <version>43.0</version> - </dependency> - <dependency> <groupId>com.sun.jersey</groupId> <artifactId>jersey-core</artifactId> <version>${jersey.version}</version> http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-project/src/site/site.xml ---------------------------------------------------------------------- diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml index 232aa78..62e9deb 100644 --- a/hadoop-project/src/site/site.xml +++ b/hadoop-project/src/site/site.xml @@ -172,7 +172,6 @@ <item name="DistCp" href="hadoop-distcp/DistCp.html"/> <item name="GridMix" href="hadoop-gridmix/GridMix.html"/> <item name="Rumen" href="hadoop-rumen/Rumen.html"/> - <item name="Resource Estimator Service" href="hadoop-resourceestimator/ResourceEstimator.html"/> <item name="Scheduler Load Simulator" href="hadoop-sls/SchedulerLoadSimulator.html"/> <item name="Hadoop Benchmarking" href="hadoop-project-dist/hadoop-common/Benchmarking.html"/> </menu> http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/README.md ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/README.md b/hadoop-tools/hadoop-resourceestimator/README.md deleted file mode 100644 index 1cb8cb5..0000000 --- a/hadoop-tools/hadoop-resourceestimator/README.md +++ /dev/null @@ -1,19 +0,0 @@ -Resource Estimator Service -========================== - -Resource Estimator Service can parse the history logs of production jobs, extract their resource consumption skylines in the past runs and predict their resource requirements for the new run. - -## Current Status - - * Support [Hadoop YARN ResourceManager](http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/YARN.html) logs. - * In-memory store for parsed history resource skyline and estimation. - * A [Linear Programming](https://github.com/optimatika/ojAlgo) based estimator. - * Provides REST interface to parse logs, query history store and estimations. - -## Upcoming features - - * UI to query history and edit and save estimations. - * Persisent store implementation for store (either DB based or distributed key-value like HBase). - * Integrate directly with the [Hadoop YARN Reservation System](http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/ReservationSystem.html) to make a recurring reservation based on the estimated resources. - -Refer to the [design document](https://issues.apache.org/jira/secure/attachment/12886714/ResourceEstimator-design-v1.pdf) for more details. http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/pom.xml b/hadoop-tools/hadoop-resourceestimator/pom.xml deleted file mode 100644 index 6790a41..0000000 --- a/hadoop-tools/hadoop-resourceestimator/pom.xml +++ /dev/null @@ -1,175 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - ~ - ~ Licensed to the Apache Software Foundation (ASF) under one - ~ or more contributor license agreements. See the NOTICE file - ~ distributed with this work for additional information - ~ regarding copyright ownership. The ASF licenses this file - ~ to you under the Apache License, Version 2.0 (the - ~ "License"); you may not use this file except in compliance - ~ with the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - ~ - --> -<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xmlns="http://maven.apache.org/POM/4.0.0"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-project</artifactId> - <version>2.9.0-SNAPSHOT</version> - <relativePath>../../hadoop-project</relativePath> - </parent> - <artifactId>hadoop-resourceestimator</artifactId> - <version>2.9.0-SNAPSHOT</version> - <name>Apache Resource Estimator Service</name> - <packaging>jar</packaging> - <dependencies> - <dependency> - <groupId>com.google.code.gson</groupId> - <artifactId>gson</artifactId> - </dependency> - <dependency> - <groupId>org.ojalgo</groupId> - <artifactId>ojalgo</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-common</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-api</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-server-resourcemanager</artifactId> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </dependency> - <dependency> - <groupId>javax.inject</groupId> - <artifactId>javax.inject</artifactId> - <version>1</version> - </dependency> - <dependency> - <groupId>com.sun.jersey.jersey-test-framework</groupId> - <artifactId>jersey-test-framework-grizzly2</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>javax.servlet</groupId> - <artifactId>servlet-api</artifactId> - </dependency> - <dependency> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-server</artifactId> - </dependency> - <dependency> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-json</artifactId> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>org.apache.rat</groupId> - <artifactId>apache-rat-plugin</artifactId> - <configuration> - <excludes> - <exclude>README.md</exclude> - <exclude>**/*.txt</exclude> - </excludes> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-checkstyle-plugin</artifactId> - <configuration> - <configLocation>src/config/checkstyle.xml</configLocation> - </configuration> - </plugin> - </plugins> - </build> - - <profiles> - <profile> - <id>dist</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-assembly-plugin</artifactId> - <configuration> - <appendAssemblyId>false</appendAssemblyId> - <attach>false</attach> - <finalName> - ${project.artifactId}-${project.version} - </finalName> - <descriptorRefs> - <descriptorRef>hadoop-resourceestimator - </descriptorRef> - </descriptorRefs> - </configuration> - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-assemblies</artifactId> - <version>${project.version}</version> - </dependency> - </dependencies> - <executions> - <execution> - <id>dist</id> - <phase>prepare-package</phase> - <goals> - <goal>single</goal> - </goals> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-dependency-plugin</artifactId> - <executions> - <execution> - <id>deplist</id> - <phase>compile</phase> - <goals> - <goal>list</goal> - </goals> - <configuration> - <outputFile> - ${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt - </outputFile> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - </profiles> -</project> http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/config/checkstyle.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/config/checkstyle.xml b/hadoop-tools/hadoop-resourceestimator/src/config/checkstyle.xml deleted file mode 100644 index 7f22659..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/config/checkstyle.xml +++ /dev/null @@ -1,50 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE module PUBLIC - "-//Puppy Crawl//DTD Check Configuration 1.2//EN" - "http://www.puppycrawl.com/dtds/configuration_1_2.dtd";> - -<!-- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<!-- - Checkstyle configuration that checks the sun coding conventions from: - - - the Java Language Specification at - http://java.sun.com/docs/books/jls/second_edition/html/index.html - - - the Sun Code Conventions at http://java.sun.com/docs/codeconv/ - - - the Javadoc guidelines at - http://java.sun.com/j2se/javadoc/writingdoccomments/index.html - - - the JDK Api documentation http://java.sun.com/j2se/docs/api/index.html - - - some best practices - - Checkstyle is very configurable. Be sure to read the documentation at - http://checkstyle.sf.net (or in your downloaded distribution). - - Most Checks are configurable, be sure to consult the documentation. - To completely disable a check, just comment it out or delete it from the file. - Finally, it is worth reading the documentation. ---> - -<module name="Checker"> - <module name="TreeWalker"> - <module name="AvoidNestedBlocks"> - <property name="id" value="checkstyle:avoidnestedblocks"/> - <property name="allowInSwitchCase" value="true"/> - </module> - </module> -</module> http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/bin/estimator.cmd ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/bin/estimator.cmd b/hadoop-tools/hadoop-resourceestimator/src/main/bin/estimator.cmd deleted file mode 100644 index 317025c..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/bin/estimator.cmd +++ /dev/null @@ -1,52 +0,0 @@ -@echo off -@rem Licensed to the Apache Software Foundation (ASF) under one or more -@rem contributor license agreements. See the NOTICE file distributed with -@rem this work for additional information regarding copyright ownership. -@rem The ASF licenses this file to You under the Apache License, Version 2.0 -@rem (the "License"); you may not use this file except in compliance with -@rem the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. -@rem - -setlocal enabledelayedexpansion - -if not defined HADOOP_BIN_PATH ( - set HADOOP_BIN_PATH=%~dp0 -) - -if "%HADOOP_BIN_PATH:~-1%" == "\" ( - set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1% -) - -set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec -if not defined HADOOP_LIBEXEC_DIR ( - set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR% -) - -:main - @rem CLASSPATH initially contains HADOOP_CONF_DIR - if not defined HADOOP_CONF_DIR ( - echo No HADOOP_CONF_DIR set. - echo Please specify it. - goto :eof - ) - - set CLASSPATH=%HADOOP_CONF_DIR%;%CLASSPATH% -goto :eof - -:classpath - set CLASS=org.apache.hadoop.util.Classpath - goto :eof - -:resourceestimator - set CLASS=org.apache.hadoop.resourceestimator.service.ResourceEstimatorServer - goto :eof - -endlocal http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/bin/estimator.sh ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/bin/estimator.sh b/hadoop-tools/hadoop-resourceestimator/src/main/bin/estimator.sh deleted file mode 100644 index 3e1ec27..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/bin/estimator.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. See accompanying LICENSE file. -# - -## @audience public -## @stability stable -function hadoop_usage() -{ - echo "Usage: estimator.sh" - #hadoop-daemon.sh. need both start and stop, status (query the status). run as background process. -} - -## @audience public -## @stability stable -function calculate_classpath -{ - hadoop_add_client_opts - hadoop_add_to_classpath_tools hadoop-resourceestimator -} - -## @audience public -## @stability stable -function resourceestimatorcmd_case -{ - # shellcheck disable=SC2034 - HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" - # shellcheck disable=SC2034 - HADOOP_CLASSNAME='org.apache.hadoop.resourceestimator.service.ResourceEstimatorServer' -} - -# let's locate libexec... -if [[ -n "${HADOOP_HOME}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" -else - this="${BASH_SOURCE-$0}" - bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) - HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../../../../../libexec" -fi - -HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}" -# shellcheck disable=SC2034 -HADOOP_NEW_CONFIG=true -if [[ -f "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then - # shellcheck source=./hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh - . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" -else - echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh." 2>&1 - exit 1 -fi - -# get arguments -HADOOP_SUBCMD=$1 -shift - -HADOOP_SUBCMD_ARGS=("$@") - -resourceestimatorcmd_case "${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}" - -calculate_classpath -hadoop_generic_java_subcmd_handler http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/bin/start-estimator.cmd ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/bin/start-estimator.cmd b/hadoop-tools/hadoop-resourceestimator/src/main/bin/start-estimator.cmd deleted file mode 100644 index 463e91c..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/bin/start-estimator.cmd +++ /dev/null @@ -1,37 +0,0 @@ -@echo off -@rem Licensed to the Apache Software Foundation (ASF) under one or more -@rem contributor license agreements. See the NOTICE file distributed with -@rem this work for additional information regarding copyright ownership. -@rem The ASF licenses this file to You under the Apache License, Version 2.0 -@rem (the "License"); you may not use this file except in compliance with -@rem the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. -@rem -setlocal enabledelayedexpansion - -echo starting resource estimator daemons - -if not defined HADOOP_BIN_PATH ( - set HADOOP_BIN_PATH=%~dp0 -) - -if "%HADOOP_BIN_PATH:~-1%" == "\" ( - set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1% -) - -set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec -if not defined HADOOP_LIBEXEC_DIR ( - set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR% -) - -@rem start resource estimator -start "Resource Estimator Service" estimator resourceestimator - -endlocal http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/bin/start-estimator.sh ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/bin/start-estimator.sh b/hadoop-tools/hadoop-resourceestimator/src/main/bin/start-estimator.sh deleted file mode 100644 index bf4328a..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/bin/start-estimator.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. See accompanying LICENSE file. -# - -# let's locate libexec... -if [[ -n "${HADOOP_HOME}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" -else - this="${BASH_SOURCE-$0}" - bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) - HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../../../../../libexec" -fi - -HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}" -# shellcheck disable=SC2034 -HADOOP_NEW_CONFIG=true -if [[ -f "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then - # shellcheck source=./hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh - . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" -else - echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh." 2>&1 - exit 1 -fi - -# start resource estimator -echo "Starting resource estimator" -hadoop_uservar_su estimator resourceestimator "bin/estimator.sh" \ - --config "${HADOOP_CONF_DIR}" \ - --daemon start \ - resourceestimator -(( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? )) http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/bin/stop-estimator.cmd ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/bin/stop-estimator.cmd b/hadoop-tools/hadoop-resourceestimator/src/main/bin/stop-estimator.cmd deleted file mode 100644 index e08c7e0..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/bin/stop-estimator.cmd +++ /dev/null @@ -1,37 +0,0 @@ -@echo off -@rem Licensed to the Apache Software Foundation (ASF) under one or more -@rem contributor license agreements. See the NOTICE file distributed with -@rem this work for additional information regarding copyright ownership. -@rem The ASF licenses this file to You under the Apache License, Version 2.0 -@rem (the "License"); you may not use this file except in compliance with -@rem the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. -@rem -setlocal enabledelayedexpansion - -echo stopping resource estimator daemons - -if not defined HADOOP_BIN_PATH ( - set HADOOP_BIN_PATH=%~dp0 -) - -if "%HADOOP_BIN_PATH:~-1%" == "\" ( - set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1% -) - -set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec -if not defined HADOOP_LIBEXEC_DIR ( - set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR% -) - -@rem start resource estimator -stop "Resource Estimator Service" estimator resourceestimator - -endlocal http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/bin/stop-estimator.sh ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/bin/stop-estimator.sh b/hadoop-tools/hadoop-resourceestimator/src/main/bin/stop-estimator.sh deleted file mode 100644 index e78c798..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/bin/stop-estimator.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. See accompanying LICENSE file. -# - -# let's locate libexec... -if [[ -n "${HADOOP_HOME}" ]]; then - HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" -else - this="${BASH_SOURCE-$0}" - bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) - HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../../../../../libexec" -fi - -HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}" -# shellcheck disable=SC2034 -HADOOP_NEW_CONFIG=true -if [[ -f "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then - # shellcheck source=./hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh - . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" -else - echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh." 2>&1 - exit 1 -fi - -# stop resource estimator -echo "Stopping resource estimator" -hadoop_uservar_su estimator resourceestimator "bin/estimator.sh" \ - --config "${HADOOP_CONF_DIR}" \ - --daemon stop \ - resourceestimator -(( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? )) http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/conf/resourceestimator-config.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/conf/resourceestimator-config.xml b/hadoop-tools/hadoop-resourceestimator/src/main/conf/resourceestimator-config.xml deleted file mode 100644 index f78085a..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/conf/resourceestimator-config.xml +++ /dev/null @@ -1,85 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<configuration> - <property> - <name>resourceestimator.solver.lp.alpha</name> - <value>0.1</value> - <description> - The resource estimator has an integrated Linear Programming solver to make the prediction, and this parameter tunes the tradeoff between resource over-allocation and under-allocation in the Linear Programming model. This parameter varies from 0 to 1, and a larger alpha value means the model minimizes over-allocation better. Default value is 0.1. - </description> - </property> - <property> - <name>resourceestimator.solver.lp.beta</name> - <value>0.1</value> - <description> - This parameter controls the generalization of the Linear Programming model. This parameter varies from 0 to 1. Deafult value is 0.1. - </description> - </property> - <property> - <name>resourceestimator.solver.lp.minJobRuns</name> - <value>2</value> - <description> - The minimum number of job runs required in order to make the prediction. Default value is 2. - </description> - </property> - <property> - <name>resourceestimator.timeInterval</name> - <value>5</value> - <description> - The time length which is used to discretize job execution into intervals. Note that the estimator makes resource allocation prediction for each interval. A smaller time interval has more fine-grained granularity for prediction, but it also takes longer time and more space for prediction. Default value is 5 (seconds). - </description> - </property> - <property> - <name>resourceestimator.translator.line-parser</name> - <value>org.apache.hadoop.resourceestimator.translator.impl.NativeSingleLineParser</value> - <description> - The class name of the translator single-line parser, which parses a single line in the log. Default value is org.apache.hadoop.resourceestimator.translator.impl.NativeSingleLineParser, which can parse one line in the sample log. Note that if users want to parse Hadoop Resource Manager logs, they need to set the value to be org.apache.hadoop.resourceestimator.translator.impl.RmSingleLineParser. If they want to implement single-line parser to parse their customized log file, they need to change this value accordingly. - </description> - </property> - <!-- Optional configuration parameters --> - <!-- - <property> - <name>resourceestimator.solver.provider</name> - <value>org.apache.hadoop.resourceestimator.solver.impl.LpSolver</value> - <description> - The class name of the solver provider. Default value is org.apache.hadoop.resourceestimator.solver.impl.LpSolver, which incorporates a Linear Programming model to make the prediction. If users want to implement their own models, they need to change this value accordingly. - </description> - </property> - <property> - <name>resourceestimator.skylinestore.provider</name> - <value>org.apache.hadoop.resourceestimator.skylinestore.impl.InMemoryStore</value> - <description> - The class name of the skylinestore provider. Default value is org.apache.hadoop.resourceestimator.skylinestore.impl.InMemoryStore, which is an in-memory implementation of skylinestore. If users want to use their own skylinestore implementation, they need to change this value accordingly. - </description> - </property> - <property> - <name>resourceestimator.translator.provider</name> - <value>org.apache.hadoop.resourceestimator.translator.impl.BaseLogParser</value> - <description> - The class name of the translator provider. Default value is org.apache.hadoop.resourceestimator.translator.impl.BaseLogParser, which extracts resourceskylines from log streams. If users want to use their own translator implementation, they need to change this value accordingly. - </description> - </property> - <property> - <name>resourceestimator.service-port</name> - <value>9998</value> - <description> - The port which ResourceEstimatorService listens to. The default value is 9998. - </description> - </property> - --> -</configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/data/resourceEstimatorService.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/data/resourceEstimatorService.txt b/hadoop-tools/hadoop-resourceestimator/src/main/data/resourceEstimatorService.txt deleted file mode 100644 index 15fb84e..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/data/resourceEstimatorService.txt +++ /dev/null @@ -1,2 +0,0 @@ -tpch_q12 50 1462148517000 1462148527345 1462148952345 tpch_q12_0 8192:0| 1:0|1:5|1074:10|2538:15|2468:20| -tpch_q12 50 1462148949000 1462148960455 1462149205455 tpch_q12_1 8192:0| 1:0|1:5|794:10|2517:15|2484:20| \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/RecurrenceId.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/RecurrenceId.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/RecurrenceId.java deleted file mode 100644 index 01ef977..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/RecurrenceId.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.common.api; - -/** - * RecurrenceId is the id for the recurring pipeline jobs. - * <p> We assume that the pipeline job can be uniquely identified with - * {pipelineId, runId}. - */ -public class RecurrenceId { - /** - * pipelineId is the unique id for the pipeline jobs. - */ - private String pipelineId; - /** - * runId is the unique instance id for the pipeline job in one run, and it - * will change across runs. - */ - private String runId; - // TODO: we may addHistory more ids of the pipeline jobs to identify them. - - /** - * Constructor. - * - * @param pipelineIdConfig the unique id for the pipeline jobs. - * @param runIdConfig the unique instance id for the pipeline job in one run. - */ - public RecurrenceId(final String pipelineIdConfig, final String runIdConfig) { - this.pipelineId = pipelineIdConfig; - this.runId = runIdConfig; - } - - /** - * Return the pipelineId for the pipeline jobs. - * - * @return the pipelineId. - */ - public final String getPipelineId() { - return pipelineId; - } - - public void setPipelineId(String pipelineId) { - this.pipelineId = pipelineId; - } - - /** - * Return the runId for the pipeline job in one run. - * - * @return the runId. - */ - public final String getRunId() { - return runId; - } - - public void setRunId(String runId) { - this.runId = runId; - } - - @Override public final String toString() { - return String.format("{pipelineId: %s, runId: %s}", pipelineId, runId); - } - - @Override public final int hashCode() { - return getPipelineId().hashCode() ^ getRunId().hashCode(); - } - - @Override public final boolean equals(final Object obj) { - if (this == obj) { - return true; - } - if ((obj == null) || (getClass() != obj.getClass())) { - return false; - } - final RecurrenceId other = (RecurrenceId) obj; - return pipelineId.equals(other.pipelineId) && runId.equals(other.runId); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/ResourceSkyline.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/ResourceSkyline.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/ResourceSkyline.java deleted file mode 100644 index a4bc69f..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/ResourceSkyline.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.common.api; - -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation; - -/** - * ResourceSkyline records the job identification information as well as job's - * requested {@code - * container}s information during its lifespan. - */ -public class ResourceSkyline { - /** - * The auto-generated {@code ApplicationId} in job's one run. - * <p> - * <p> - * For a pipeline job, we assume the {@code jobId} changes each time we run - * the pipeline job. - */ - private String jobId; - /** - * The input data size of the job. - */ - private double jobInputDataSize; - /** - * Job submission time. Different logs could have different time format, so we - * store the original string directly extracted from logs. - */ - private long jobSubmissionTime; - /** - * Job finish time. Different logs could have different time format, so we - * store the original string directly extracted from logs. - */ - private long jobFinishTime; - /** - * The resource spec of containers allocated to the job. - */ - private Resource containerSpec; - /** - * The list of {@link Resource} allocated to the job in its lifespan. - */ - private RLESparseResourceAllocation skylineList; - // TODO - // We plan to record pipeline job's actual resource consumptions in one run - // here. - // TODO - // We might need to addHistory more features to the ResourceSkyline, such as - // users, VC, etc. - - /** - * Constructor. - * - * @param jobIdConfig the id of the job. - * @param jobInputDataSizeConfig the input data size of the job. - * @param jobSubmissionTimeConfig the submission time of the job. - * @param jobFinishTimeConfig the finish time of the job. - * @param containerSpecConfig the resource spec of containers allocated - * to the job. - * @param skylineListConfig the list of {@link Resource} allocated in - * one run. - */ - public ResourceSkyline(final String jobIdConfig, - final double jobInputDataSizeConfig, final long jobSubmissionTimeConfig, - final long jobFinishTimeConfig, final Resource containerSpecConfig, - final RLESparseResourceAllocation skylineListConfig) { - this.jobId = jobIdConfig; - this.jobInputDataSize = jobInputDataSizeConfig; - this.jobSubmissionTime = jobSubmissionTimeConfig; - this.jobFinishTime = jobFinishTimeConfig; - this.containerSpec = containerSpecConfig; - this.skylineList = skylineListConfig; - } - - /** - * Empty constructor. - */ - public ResourceSkyline() { - } - - /** - * Get the id of the job. - * - * @return the id of this job. - */ - public final String getJobId() { - return jobId; - } - - /** - * Set jobId. - * - * @param jobIdConfig jobId. - */ - public final void setJobId(final String jobIdConfig) { - this.jobId = jobIdConfig; - } - - /** - * Get the job's input data size. - * - * @return job's input data size. - */ - public final double getJobInputDataSize() { - return jobInputDataSize; - } - - /** - * Set jobInputDataSize. - * - * @param jobInputDataSizeConfig jobInputDataSize. - */ - public final void setJobInputDataSize(final double jobInputDataSizeConfig) { - this.jobInputDataSize = jobInputDataSizeConfig; - } - - /** - * Get the job's submission time. - * - * @return job's submission time. - */ - public final long getJobSubmissionTime() { - return jobSubmissionTime; - } - - /** - * Set jobSubmissionTime. - * - * @param jobSubmissionTimeConfig jobSubmissionTime. - */ - public final void setJobSubmissionTime(final long jobSubmissionTimeConfig) { - this.jobSubmissionTime = jobSubmissionTimeConfig; - } - - /** - * Get the job's finish time. - * - * @return job's finish time. - */ - public final long getJobFinishTime() { - return jobFinishTime; - } - - /** - * Set jobFinishTime. - * - * @param jobFinishTimeConfig jobFinishTime. - */ - public final void setJobFinishTime(final long jobFinishTimeConfig) { - this.jobFinishTime = jobFinishTimeConfig; - } - - /** - * Get the resource spec of the job's allocated {@code container}s. - * <p> Key assumption: during job's lifespan, its allocated {@code container}s - * have the same {@link Resource} spec. - * - * @return the {@link Resource} spec of the job's allocated - * {@code container}s. - */ - public final Resource getContainerSpec() { - return containerSpec; - } - - /** - * Set containerSpec. - * - * @param containerSpecConfig containerSpec. - */ - public final void setContainerSpec(final Resource containerSpecConfig) { - this.containerSpec = containerSpecConfig; - } - - /** - * Get the list of {@link Resource}s allocated to the job. - * - * @return the {@link RLESparseResourceAllocation} which contains the list of - * {@link Resource}s allocated to the job. - */ - public final RLESparseResourceAllocation getSkylineList() { - return skylineList; - } - - /** - * Set skylineList. - * - * @param skylineListConfig skylineList. - */ - public final void setSkylineList( - final RLESparseResourceAllocation skylineListConfig) { - this.skylineList = skylineListConfig; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/package-info.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/package-info.java deleted file mode 100644 index a7c7527..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * The common APIs for the resource estimator. - */ - -package org.apache.hadoop.resourceestimator.common.api; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/ResourceEstimatorConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/ResourceEstimatorConfiguration.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/ResourceEstimatorConfiguration.java deleted file mode 100644 index bb041b6..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/ResourceEstimatorConfiguration.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.common.config; - -import org.apache.hadoop.resourceestimator.skylinestore.impl.InMemoryStore; -import org.apache.hadoop.resourceestimator.solver.impl.LpSolver; -import org.apache.hadoop.resourceestimator.translator.impl.BaseLogParser; -import org.apache.hadoop.resourceestimator.translator.impl.NativeSingleLineParser; - -/** - * Defines configuration keys for ResourceEstimatorServer. - */ -public final class ResourceEstimatorConfiguration { - - /** - * The location of the configuration file for ResourceEstimatorService. - */ - public static final String CONFIG_FILE = "resourceestimator-config.xml"; - - /** - * The URI for ResourceEstimatorService. - */ - public static final String SERVICE_URI = "http://0.0.0.0/";; - - /** - * The port which ResourceEstimatorService listens to. - */ - public static final String SERVICE_PORT = "resourceestimator.service-port"; - - /** - * Default port number of ResourceEstimatorService. - */ - public static final int DEFAULT_SERVICE_PORT = 9998; - - /** - * The class name of the skylinestore provider. - */ - public static final String SKYLINESTORE_PROVIDER = - "resourceestimator.skylinestore.provider"; - - /** - * Default value for skylinestore provider, which is an in-memory implementation of skylinestore. - */ - public static final String DEFAULT_SKYLINESTORE_PROVIDER = - InMemoryStore.class.getName(); - - /** - * The class name of the translator provider. - */ - public static final String TRANSLATOR_PROVIDER = - "resourceestimator.translator.provider"; - - /** - * Default value for translator provider, which extracts resourceskylines from log streams. - */ - public static final String DEFAULT_TRANSLATOR_PROVIDER = - BaseLogParser.class.getName(); - - /** - * The class name of the translator single-line parser, which parses a single line in the log. - */ - public static final String TRANSLATOR_LINE_PARSER = - "resourceestimator.translator.line-parser"; - - /** - * Default value for translator single-line parser, which can parse one line in the sample log. - */ - public static final String DEFAULT_TRANSLATOR_LINE_PARSER = - NativeSingleLineParser.class.getName(); - - /** - * The class name of the solver provider. - */ - public static final String SOLVER_PROVIDER = - "resourceestimator.solver.provider"; - - /** - * Default value for solver provider, which incorporates a Linear Programming model to make the prediction. - */ - public static final String DEFAULT_SOLVER_PROVIDER = LpSolver.class.getName(); - - /** - * The time length which is used to discretize job execution into intervals. - */ - public static final String TIME_INTERVAL_KEY = - "resourceestimator.timeInterval"; - - /** - * The parameter which tunes the tradeoff between resource over-allocation and under-allocation in the Linear Programming model. - */ - public static final String SOLVER_ALPHA_KEY = - "resourceestimator.solver.lp.alpha"; - - /** - * This parameter which controls the generalization of the Linear Programming model. - */ - public static final String SOLVER_BETA_KEY = - "resourceestimator.solver.lp.beta"; - - /** - * The minimum number of job runs required in order to make the prediction. - */ - public static final String SOLVER_MIN_JOB_RUN_KEY = - "resourceestimator.solver.lp.minJobRuns"; - - private ResourceEstimatorConfiguration() {} -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/ResourceEstimatorUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/ResourceEstimatorUtil.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/ResourceEstimatorUtil.java deleted file mode 100644 index c00dd00..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/ResourceEstimatorUtil.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.common.config; - -import java.lang.reflect.Constructor; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.resourceestimator.common.exception.ResourceEstimatorException; - -/** - * General resourceestimator utils. - */ -public final class ResourceEstimatorUtil { - - private static final Class<?>[] EMPTY_ARRAY = new Class[0]; - - private ResourceEstimatorUtil() {} - - /** - * Helper method to create instances of Object using the class name specified - * in the configuration object. - * - * @param conf the yarn configuration - * @param configuredClassName the configuration provider key - * @param defaultValue the default implementation class - * @param type the required interface/base class - * @param <T> The type of the instance to create - * @return the instances created - * @throws ResourceEstimatorException if the provider initialization fails. - */ - @SuppressWarnings("unchecked") public static <T> T createProviderInstance( - Configuration conf, String configuredClassName, String defaultValue, - Class<T> type) throws ResourceEstimatorException { - String className = conf.get(configuredClassName); - if (className == null) { - className = defaultValue; - } - try { - Class<?> concreteClass = Class.forName(className); - if (type.isAssignableFrom(concreteClass)) { - Constructor<T> meth = - (Constructor<T>) concreteClass.getDeclaredConstructor(EMPTY_ARRAY); - meth.setAccessible(true); - return meth.newInstance(); - } else { - StringBuilder errMsg = new StringBuilder(); - errMsg.append("Class: ").append(className).append(" not instance of ") - .append(type.getCanonicalName()); - throw new ResourceEstimatorException(errMsg.toString()); - } - } catch (ClassNotFoundException e) { - StringBuilder errMsg = new StringBuilder(); - errMsg.append("Could not instantiate : ").append(className) - .append(" due to exception: ").append(e.getCause()); - throw new ResourceEstimatorException(errMsg.toString()); - } catch (ReflectiveOperationException e) { - StringBuilder errMsg = new StringBuilder(); - errMsg.append("Could not instantiate : ").append(className) - .append(" due to exception: ").append(e.getCause()); - throw new ResourceEstimatorException(errMsg.toString()); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/package-info.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/package-info.java deleted file mode 100644 index ecf140a..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * The common configuration for the resource estimator. - */ - -package org.apache.hadoop.resourceestimator.common.config; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/exception/ResourceEstimatorException.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/exception/ResourceEstimatorException.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/exception/ResourceEstimatorException.java deleted file mode 100644 index ba5666d..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/exception/ResourceEstimatorException.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.common.exception; - -/** - * Exception thrown by ResourceEstimatorServer utility classes. - */ -public class ResourceEstimatorException extends Exception { - - public ResourceEstimatorException(String message) { - super(message); - } - - public ResourceEstimatorException(String message, Exception ex) { - super(message, ex); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/exception/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/exception/package-info.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/exception/package-info.java deleted file mode 100644 index 6ebecb6..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/exception/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * The common exception for the resource estimator. - */ - -package org.apache.hadoop.resourceestimator.common.exception; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/RLESparseResourceAllocationSerDe.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/RLESparseResourceAllocationSerDe.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/RLESparseResourceAllocationSerDe.java deleted file mode 100644 index 379dc3a..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/RLESparseResourceAllocationSerDe.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.common.serialization; - -import java.lang.reflect.Type; -import java.util.NavigableMap; - -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation; -import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; -import org.apache.hadoop.yarn.util.resource.ResourceCalculator; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonDeserializationContext; -import com.google.gson.JsonDeserializer; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParseException; -import com.google.gson.JsonSerializationContext; -import com.google.gson.JsonSerializer; -import com.google.gson.reflect.TypeToken; - -/** - * Serialize/deserialize RLESparseResourceAllocation object to/from JSON. - */ -public class RLESparseResourceAllocationSerDe - implements JsonSerializer<RLESparseResourceAllocation>, - JsonDeserializer<RLESparseResourceAllocation> { - private static final String KEY = "resourceAllocation"; - private final Gson gson = - new GsonBuilder().registerTypeAdapter(Resource.class, new ResourceSerDe()) - .create(); - private final Type type = new TypeToken<NavigableMap<Long, Resource>>() { - }.getType(); - private final ResourceCalculator resourceCalculator = - new DefaultResourceCalculator(); - - @Override public final JsonElement serialize( - final RLESparseResourceAllocation resourceAllocation, - final Type typeOfSrc, final JsonSerializationContext context) { - NavigableMap<Long, Resource> myMap = resourceAllocation.getCumulative(); - JsonObject jo = new JsonObject(); - JsonElement element = gson.toJsonTree(myMap, type); - jo.add(KEY, element); - - return jo; - } - - @Override public final RLESparseResourceAllocation deserialize( - final JsonElement json, final Type typeOfT, - final JsonDeserializationContext context) throws JsonParseException { - NavigableMap<Long, Resource> resAllocation = - gson.fromJson(json.getAsJsonObject().get(KEY), type); - RLESparseResourceAllocation rleSparseResourceAllocation = - new RLESparseResourceAllocation(resAllocation, resourceCalculator); - return rleSparseResourceAllocation; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/ResourceSerDe.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/ResourceSerDe.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/ResourceSerDe.java deleted file mode 100644 index 22b079c..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/ResourceSerDe.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.common.serialization; - -import java.lang.reflect.Type; - -import org.apache.hadoop.yarn.api.records.Resource; - -import com.google.gson.JsonDeserializationContext; -import com.google.gson.JsonDeserializer; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParseException; -import com.google.gson.JsonSerializationContext; -import com.google.gson.JsonSerializer; - -/** - * Serialize/deserialize Resource object to/from JSON. - */ -public class ResourceSerDe - implements JsonSerializer<Resource>, JsonDeserializer<Resource> { - private static final String KEY1 = "memory"; - private static final String KEY2 = "vcores"; - - @Override public final JsonElement serialize(final Resource resource, - final Type type, final JsonSerializationContext context) { - JsonObject jo = new JsonObject(); - jo.addProperty(KEY1, resource.getMemorySize()); - jo.addProperty(KEY2, resource.getVirtualCores()); - return jo; - } - - @Override public final Resource deserialize(final JsonElement json, - final Type type, final JsonDeserializationContext context) - throws JsonParseException { - JsonObject jo = json.getAsJsonObject(); - long mem = jo.getAsJsonPrimitive(KEY1).getAsLong(); - int vcore = jo.getAsJsonPrimitive(KEY2).getAsInt(); - Resource resource = Resource.newInstance(mem, vcore); - - return resource; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/package-info.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/package-info.java deleted file mode 100644 index 06957b3..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/package-info.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -/** - * SkylineStore serialization module. - */ -package org.apache.hadoop.resourceestimator.common.serialization; http://git-wip-us.apache.org/repos/asf/hadoop/blob/13f1a7e0/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorServer.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorServer.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorServer.java deleted file mode 100644 index 27a5864..0000000 --- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorServer.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.resourceestimator.service; - -import java.io.IOException; -import java.net.URI; - -import javax.ws.rs.core.UriBuilder; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.http.HttpServer2; -import org.apache.hadoop.resourceestimator.common.config.ResourceEstimatorConfiguration; -import org.apache.hadoop.service.CompositeService; -import org.apache.hadoop.util.ShutdownHookManager; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; -import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A simple embedded Hadoop HTTP server. - */ -public final class ResourceEstimatorServer extends CompositeService { - private static final Logger LOGGER = - LoggerFactory.getLogger(ResourceEstimatorServer.class.getName()); - private HttpServer2 webServer; - private static URI baseURI; - - public ResourceEstimatorServer() { - super(ResourceEstimatorServer.class.getName()); - } - - private static URI getBaseURI(Configuration config) { - baseURI = UriBuilder.fromUri(ResourceEstimatorConfiguration.SERVICE_URI) - .port(getPort(config)).build(); - return baseURI; - } - - private static int getPort(Configuration config) { - return config.getInt(ResourceEstimatorConfiguration.SERVICE_PORT, - ResourceEstimatorConfiguration.DEFAULT_SERVICE_PORT); - } - - @Override protected void serviceInit(Configuration conf) throws Exception { - super.serviceInit(conf); - } - - @Override protected void serviceStart() throws Exception { - super.serviceStart(); - startResourceEstimatorApp(); - } - - private void join() { - // keep the main thread that started the server up until it receives a stop - // signal - if (webServer != null) { - try { - webServer.join(); - } catch (InterruptedException ignore) { - } - } - } - - @Override protected void serviceStop() throws Exception { - if (webServer != null) { - webServer.stop(); - } - super.serviceStop(); - } - - private void startResourceEstimatorApp() throws IOException { - Configuration config = new YarnConfiguration(); - config.addResource(ResourceEstimatorConfiguration.CONFIG_FILE); - HttpServer2.Builder builder = - new HttpServer2.Builder().setName("ResourceEstimatorServer") - .setConf(config) - //.setFindPort(true) - .addEndpoint(getBaseURI(config)); - webServer = builder.build(); - webServer.addJerseyResourcePackage( - ResourceEstimatorService.class.getPackage().getName() + ";" - + GenericExceptionHandler.class.getPackage().getName() + ";" - + YarnJacksonJaxbJsonProvider.class.getPackage().getName(), "/*"); - webServer.start(); - } - - /** - * Start embedded Hadoop HTTP server. - * - * @return an instance of the started HTTP server. - * @throws IOException in case there is an error while starting server. - */ - static ResourceEstimatorServer startResourceEstimatorServer() - throws IOException, InterruptedException { - Configuration config = new YarnConfiguration(); - config.addResource(ResourceEstimatorConfiguration.CONFIG_FILE); - ResourceEstimatorServer resourceEstimatorServer = null; - try { - resourceEstimatorServer = new ResourceEstimatorServer(); - ShutdownHookManager.get().addShutdownHook( - new CompositeServiceShutdownHook(resourceEstimatorServer), 30); - resourceEstimatorServer.init(config); - resourceEstimatorServer.start(); - } catch (Throwable t) { - LOGGER.error("Error starting ResourceEstimatorServer", t); - } - - return resourceEstimatorServer; - } - - public static void main(String[] args) - throws InterruptedException, IOException { - ResourceEstimatorServer server = startResourceEstimatorServer(); - server.join(); - } - - /** - * Stop embedded Hadoop HTTP server. - * - * @throws Exception in case the HTTP server fails to shut down. - */ - public void shutdown() throws Exception { - LOGGER.info("Stopping resourceestimator service at: {}.", - baseURI.toString()); - webServer.stop(); - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
