HADOOP-14840. Tool to estimate resource requirements of an application pipeline based on prior executions. (Rui Li via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/625039ef Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/625039ef Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/625039ef Branch: refs/heads/trunk Commit: 625039ef20e6011ab360131d70582a6e4bf2ec1d Parents: 3fae675 Author: Subru Krishnan <[email protected]> Authored: Wed Oct 25 15:51:27 2017 -0700 Committer: Subru Krishnan <[email protected]> Committed: Wed Oct 25 15:51:27 2017 -0700 ---------------------------------------------------------------------- LICENSE.txt | 1 + .../assemblies/hadoop-resourceestimator.xml | 42 ++ .../main/resources/assemblies/hadoop-tools.xml | 11 + hadoop-project/pom.xml | 5 + hadoop-project/src/site/site.xml | 1 + hadoop-tools/hadoop-resourceestimator/README.md | 19 + hadoop-tools/hadoop-resourceestimator/pom.xml | 174 +++++++ .../src/config/checkstyle.xml | 50 ++ .../src/main/bin/estimator.cmd | 52 +++ .../src/main/bin/estimator.sh | 71 +++ .../src/main/bin/start-estimator.cmd | 37 ++ .../src/main/bin/start-estimator.sh | 42 ++ .../src/main/bin/stop-estimator.cmd | 37 ++ .../src/main/bin/stop-estimator.sh | 42 ++ .../src/main/conf/resourceestimator-config.xml | 85 ++++ .../src/main/data/resourceEstimatorService.txt | 2 + .../common/api/RecurrenceId.java | 95 ++++ .../common/api/ResourceSkyline.java | 211 +++++++++ .../common/api/package-info.java | 23 + .../config/ResourceEstimatorConfiguration.java | 125 +++++ .../common/config/ResourceEstimatorUtil.java | 81 ++++ .../common/config/package-info.java | 23 + .../exception/ResourceEstimatorException.java | 35 ++ .../common/exception/package-info.java | 23 + .../RLESparseResourceAllocationSerDe.java | 77 +++ .../common/serialization/ResourceSerDe.java | 61 +++ .../common/serialization/package-info.java | 24 + .../service/ResourceEstimatorServer.java | 146 ++++++ .../service/ResourceEstimatorService.java | 238 ++++++++++ .../resourceestimator/service/ShutdownHook.java | 45 ++ .../resourceestimator/service/package-info.java | 23 + .../skylinestore/api/HistorySkylineStore.java | 99 ++++ .../api/PredictionSkylineStore.java | 60 +++ .../skylinestore/api/SkylineStore.java | 30 ++ .../skylinestore/api/package-info.java | 23 + .../DuplicateRecurrenceIdException.java | 33 ++ .../EmptyResourceSkylineException.java | 33 ++ .../exceptions/NullPipelineIdException.java | 32 ++ ...ullRLESparseResourceAllocationException.java | 33 ++ .../exceptions/NullRecurrenceIdException.java | 32 ++ .../NullResourceSkylineException.java | 32 ++ .../RecurrenceIdNotFoundException.java | 33 ++ .../exceptions/SkylineStoreException.java | 33 ++ .../skylinestore/exceptions/package-info.java | 24 + .../skylinestore/impl/InMemoryStore.java | 256 ++++++++++ .../skylinestore/impl/package-info.java | 23 + .../validator/SkylineStoreValidator.java | 118 +++++ .../skylinestore/validator/package-info.java | 23 + .../resourceestimator/solver/api/Solver.java | 76 +++ .../solver/api/package-info.java | 23 + .../exceptions/InvalidInputException.java | 34 ++ .../exceptions/InvalidSolverException.java | 34 ++ .../solver/exceptions/SolverException.java | 34 ++ .../solver/exceptions/package-info.java | 24 + .../solver/impl/BaseSolver.java | 94 ++++ .../resourceestimator/solver/impl/LpSolver.java | 340 ++++++++++++++ .../solver/impl/package-info.java | 23 + .../solver/preprocess/SolverPreprocessor.java | 219 +++++++++ .../solver/preprocess/package-info.java | 23 + .../translator/api/JobMetaData.java | 163 +++++++ .../translator/api/LogParser.java | 65 +++ .../translator/api/SingleLineParser.java | 52 +++ .../translator/api/package-info.java | 23 + .../exceptions/DataFieldNotFoundException.java | 32 ++ .../translator/exceptions/package-info.java | 23 + .../translator/impl/BaseLogParser.java | 125 +++++ .../translator/impl/LogParserUtil.java | 97 ++++ .../translator/impl/NativeSingleLineParser.java | 120 +++++ .../translator/impl/RmSingleLineParser.java | 203 ++++++++ .../translator/impl/package-info.java | 23 + .../translator/validator/ParserValidator.java | 41 ++ .../translator/validator/package-info.java | 23 + .../webapps/ResourceEstimatorServer/.gitignore | 14 + .../src/site/markdown/ResourceEstimator.md | 181 ++++++++ .../src/site/resources/css/site.css | 29 ++ .../common/api/TestResourceSkyline.java | 128 +++++ .../serialization/TestHistorySkylineSerDe.java | 134 ++++++ .../common/serialization/TestResourceSerDe.java | 64 +++ .../serialization/TestResourceSkylineSerDe.java | 112 +++++ .../common/serialization/package-info.java | 24 + .../service/GuiceServletConfig.java | 42 ++ .../service/TestResourceEstimatorService.java | 282 +++++++++++ .../skylinestore/impl/TestInMemoryStore.java | 32 ++ .../skylinestore/impl/TestSkylineStore.java | 464 +++++++++++++++++++ .../solver/impl/TestLpSolver.java | 112 +++++ .../solver/impl/TestSolver.java | 73 +++ .../translator/api/TestJobMetaData.java | 163 +++++++ .../translator/impl/TestNativeParser.java | 115 +++++ .../translator/impl/TestRmParser.java | 239 ++++++++++ .../src/test/resources/log4j.properties | 25 + .../src/test/resources/lp/answer.txt | 7 + .../src/test/resources/lp/tinySample.txt | 2 + .../src/test/resources/lp/tpch_q12.txt | 13 + .../test/resources/resourceEstimatorService.txt | 2 + .../test/resources/resourceestimator-config.xml | 85 ++++ .../src/test/resources/trace/invalidLog1.txt | 4 + .../src/test/resources/trace/invalidLog2.txt | 3 + .../src/test/resources/trace/invalidLog3.txt | 5 + .../src/test/resources/trace/invalidLog4.txt | 5 + .../src/test/resources/trace/invalidLog5.txt | 7 + .../src/test/resources/trace/invalidLog6.txt | 5 + .../src/test/resources/trace/invalidLog7.txt | 5 + .../src/test/resources/trace/invalidLog8.txt | 6 + .../src/test/resources/trace/invalidLog9.txt | 6 + .../src/test/resources/trace/nativeLog.txt | 1 + .../src/test/resources/trace/rmLog.txt | 30 ++ hadoop-tools/hadoop-tools-dist/pom.xml | 6 + hadoop-tools/pom.xml | 1 + 108 files changed, 7158 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/LICENSE.txt ---------------------------------------------------------------------- diff --git a/LICENSE.txt b/LICENSE.txt index 3f50521..67472b6 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -699,6 +699,7 @@ hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery Apache HBase - Server which contains JQuery minified javascript library version 1.8.3 Microsoft JDBC Driver for SQLServer - version 6.2.1.jre7 +oj! Algorithms - version 43.0 -------------------------------------------------------------------------------- Copyright 2005, 2012, 2013 jQuery Foundation and other contributors, https://jquery.org/ http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-assemblies/src/main/resources/assemblies/hadoop-resourceestimator.xml ---------------------------------------------------------------------- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-resourceestimator.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-resourceestimator.xml new file mode 100644 index 0000000..d1d941d --- /dev/null +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-resourceestimator.xml @@ -0,0 +1,42 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd";> + <id>hadoop-resourceestimator</id> + <formats> + <format>dir</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + + <fileSets> + <fileSet> + <directory>${basedir}/src/main/bin</directory> + <outputDirectory>resourceestimator/bin</outputDirectory> + <fileMode>0755</fileMode> + </fileSet> + <fileSet> + <directory>${basedir}/src/main/conf</directory> + <outputDirectory>resourceestimator/conf</outputDirectory> + </fileSet> + <fileSet> + <directory>${basedir}/src/main/data</directory> + <outputDirectory>resourceestimator/data</outputDirectory> + </fileSet> + </fileSets> + </assembly> http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml ---------------------------------------------------------------------- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml index 0a4367d..7d34154 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml @@ -175,6 +175,17 @@ <outputDirectory>/share/hadoop/${hadoop.component}/sls</outputDirectory> </fileSet> <fileSet> + <directory>../hadoop-resourceestimator/target</directory> + <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory> + <includes> + <include>*-sources.jar</include> + </includes> + </fileSet> + <fileSet> + <directory>../hadoop-resourceestimator/target/hadoop-resourceestimator-${project.version}/resourceestimator</directory> + <outputDirectory>/share/hadoop/${hadoop.component}/resourceestimator</outputDirectory> + </fileSet> + <fileSet> <directory>../hadoop-aws/src/main/bin</directory> <outputDirectory>/bin</outputDirectory> <fileMode>0755</fileMode> http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-project/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 3baa8f3..d7156d7 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -667,6 +667,11 @@ <version>5.0.4</version> </dependency> <dependency> + <groupId>org.ojalgo</groupId> + <artifactId>ojalgo</artifactId> + <version>43.0</version> + </dependency> + <dependency> <groupId>com.sun.jersey</groupId> <artifactId>jersey-core</artifactId> <version>${jersey.version}</version> http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-project/src/site/site.xml ---------------------------------------------------------------------- diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml index ae92605..6fc3cf9 100644 --- a/hadoop-project/src/site/site.xml +++ b/hadoop-project/src/site/site.xml @@ -178,6 +178,7 @@ <item name="DistCp" href="hadoop-distcp/DistCp.html"/> <item name="GridMix" href="hadoop-gridmix/GridMix.html"/> <item name="Rumen" href="hadoop-rumen/Rumen.html"/> + <item name="Resource Estimator Service" href="hadoop-resourceestimator/ResourceEstimator.html"/> <item name="Scheduler Load Simulator" href="hadoop-sls/SchedulerLoadSimulator.html"/> <item name="Hadoop Benchmarking" href="hadoop-project-dist/hadoop-common/Benchmarking.html"/> </menu> http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-tools/hadoop-resourceestimator/README.md ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/README.md b/hadoop-tools/hadoop-resourceestimator/README.md new file mode 100644 index 0000000..1cb8cb5 --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/README.md @@ -0,0 +1,19 @@ +Resource Estimator Service +========================== + +Resource Estimator Service can parse the history logs of production jobs, extract their resource consumption skylines in the past runs and predict their resource requirements for the new run. + +## Current Status + + * Support [Hadoop YARN ResourceManager](http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/YARN.html) logs. + * In-memory store for parsed history resource skyline and estimation. + * A [Linear Programming](https://github.com/optimatika/ojAlgo) based estimator. + * Provides REST interface to parse logs, query history store and estimations. + +## Upcoming features + + * UI to query history and edit and save estimations. + * Persisent store implementation for store (either DB based or distributed key-value like HBase). + * Integrate directly with the [Hadoop YARN Reservation System](http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/ReservationSystem.html) to make a recurring reservation based on the estimated resources. + +Refer to the [design document](https://issues.apache.org/jira/secure/attachment/12886714/ResourceEstimator-design-v1.pdf) for more details. http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-tools/hadoop-resourceestimator/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/pom.xml b/hadoop-tools/hadoop-resourceestimator/pom.xml new file mode 100644 index 0000000..d44c9bc --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/pom.xml @@ -0,0 +1,174 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + ~ + --> +<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xmlns="http://maven.apache.org/POM/4.0.0"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-project</artifactId> + <version>3.1.0-SNAPSHOT</version> + <relativePath>../../hadoop-project</relativePath> + </parent> + <artifactId>hadoop-resourceestimator</artifactId> + <name>Apache Resource Estimator Service</name> + <packaging>jar</packaging> + <dependencies> + <dependency> + <groupId>com.google.code.gson</groupId> + <artifactId>gson</artifactId> + </dependency> + <dependency> + <groupId>org.ojalgo</groupId> + <artifactId>ojalgo</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-resourcemanager</artifactId> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + <dependency> + <groupId>javax.inject</groupId> + <artifactId>javax.inject</artifactId> + <version>1</version> + </dependency> + <dependency> + <groupId>com.sun.jersey.jersey-test-framework</groupId> + <artifactId>jersey-test-framework-grizzly2</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>javax.servlet</groupId> + <artifactId>javax.servlet-api</artifactId> + </dependency> + <dependency> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-server</artifactId> + </dependency> + <dependency> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-json</artifactId> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <configuration> + <excludes> + <exclude>README.md</exclude> + <exclude>**/*.txt</exclude> + </excludes> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-checkstyle-plugin</artifactId> + <configuration> + <configLocation>src/config/checkstyle.xml</configLocation> + </configuration> + </plugin> + </plugins> + </build> + + <profiles> + <profile> + <id>dist</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <appendAssemblyId>false</appendAssemblyId> + <attach>false</attach> + <finalName> + ${project.artifactId}-${project.version} + </finalName> + <descriptorRefs> + <descriptorRef>hadoop-resourceestimator + </descriptorRef> + </descriptorRefs> + </configuration> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-assemblies</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + <executions> + <execution> + <id>dist</id> + <phase>prepare-package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>deplist</id> + <phase>compile</phase> + <goals> + <goal>list</goal> + </goals> + <configuration> + <outputFile> + ${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt + </outputFile> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> +</project> http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-tools/hadoop-resourceestimator/src/config/checkstyle.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/config/checkstyle.xml b/hadoop-tools/hadoop-resourceestimator/src/config/checkstyle.xml new file mode 100644 index 0000000..7f22659 --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/config/checkstyle.xml @@ -0,0 +1,50 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE module PUBLIC + "-//Puppy Crawl//DTD Check Configuration 1.2//EN" + "http://www.puppycrawl.com/dtds/configuration_1_2.dtd";> + +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- + Checkstyle configuration that checks the sun coding conventions from: + + - the Java Language Specification at + http://java.sun.com/docs/books/jls/second_edition/html/index.html + + - the Sun Code Conventions at http://java.sun.com/docs/codeconv/ + + - the Javadoc guidelines at + http://java.sun.com/j2se/javadoc/writingdoccomments/index.html + + - the JDK Api documentation http://java.sun.com/j2se/docs/api/index.html + + - some best practices + + Checkstyle is very configurable. Be sure to read the documentation at + http://checkstyle.sf.net (or in your downloaded distribution). + + Most Checks are configurable, be sure to consult the documentation. + To completely disable a check, just comment it out or delete it from the file. + Finally, it is worth reading the documentation. +--> + +<module name="Checker"> + <module name="TreeWalker"> + <module name="AvoidNestedBlocks"> + <property name="id" value="checkstyle:avoidnestedblocks"/> + <property name="allowInSwitchCase" value="true"/> + </module> + </module> +</module> http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-tools/hadoop-resourceestimator/src/main/bin/estimator.cmd ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/bin/estimator.cmd b/hadoop-tools/hadoop-resourceestimator/src/main/bin/estimator.cmd new file mode 100644 index 0000000..317025c --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/main/bin/estimator.cmd @@ -0,0 +1,52 @@ +@echo off +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +setlocal enabledelayedexpansion + +if not defined HADOOP_BIN_PATH ( + set HADOOP_BIN_PATH=%~dp0 +) + +if "%HADOOP_BIN_PATH:~-1%" == "\" ( + set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1% +) + +set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec +if not defined HADOOP_LIBEXEC_DIR ( + set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR% +) + +:main + @rem CLASSPATH initially contains HADOOP_CONF_DIR + if not defined HADOOP_CONF_DIR ( + echo No HADOOP_CONF_DIR set. + echo Please specify it. + goto :eof + ) + + set CLASSPATH=%HADOOP_CONF_DIR%;%CLASSPATH% +goto :eof + +:classpath + set CLASS=org.apache.hadoop.util.Classpath + goto :eof + +:resourceestimator + set CLASS=org.apache.hadoop.resourceestimator.service.ResourceEstimatorServer + goto :eof + +endlocal http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-tools/hadoop-resourceestimator/src/main/bin/estimator.sh ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/bin/estimator.sh b/hadoop-tools/hadoop-resourceestimator/src/main/bin/estimator.sh new file mode 100644 index 0000000..3e1ec27 --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/main/bin/estimator.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. +# + +## @audience public +## @stability stable +function hadoop_usage() +{ + echo "Usage: estimator.sh" + #hadoop-daemon.sh. need both start and stop, status (query the status). run as background process. +} + +## @audience public +## @stability stable +function calculate_classpath +{ + hadoop_add_client_opts + hadoop_add_to_classpath_tools hadoop-resourceestimator +} + +## @audience public +## @stability stable +function resourceestimatorcmd_case +{ + # shellcheck disable=SC2034 + HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + # shellcheck disable=SC2034 + HADOOP_CLASSNAME='org.apache.hadoop.resourceestimator.service.ResourceEstimatorServer' +} + +# let's locate libexec... +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" +else + this="${BASH_SOURCE-$0}" + bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../../../../../libexec" +fi + +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then + # shellcheck source=./hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh + . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh." 2>&1 + exit 1 +fi + +# get arguments +HADOOP_SUBCMD=$1 +shift + +HADOOP_SUBCMD_ARGS=("$@") + +resourceestimatorcmd_case "${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}" + +calculate_classpath +hadoop_generic_java_subcmd_handler http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-tools/hadoop-resourceestimator/src/main/bin/start-estimator.cmd ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/bin/start-estimator.cmd b/hadoop-tools/hadoop-resourceestimator/src/main/bin/start-estimator.cmd new file mode 100644 index 0000000..463e91c --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/main/bin/start-estimator.cmd @@ -0,0 +1,37 @@ +@echo off +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +setlocal enabledelayedexpansion + +echo starting resource estimator daemons + +if not defined HADOOP_BIN_PATH ( + set HADOOP_BIN_PATH=%~dp0 +) + +if "%HADOOP_BIN_PATH:~-1%" == "\" ( + set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1% +) + +set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec +if not defined HADOOP_LIBEXEC_DIR ( + set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR% +) + +@rem start resource estimator +start "Resource Estimator Service" estimator resourceestimator + +endlocal http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-tools/hadoop-resourceestimator/src/main/bin/start-estimator.sh ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/bin/start-estimator.sh b/hadoop-tools/hadoop-resourceestimator/src/main/bin/start-estimator.sh new file mode 100644 index 0000000..bf4328a --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/main/bin/start-estimator.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. +# + +# let's locate libexec... +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" +else + this="${BASH_SOURCE-$0}" + bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../../../../../libexec" +fi + +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then + # shellcheck source=./hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh + . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh." 2>&1 + exit 1 +fi + +# start resource estimator +echo "Starting resource estimator" +hadoop_uservar_su estimator resourceestimator "bin/estimator.sh" \ + --config "${HADOOP_CONF_DIR}" \ + --daemon start \ + resourceestimator +(( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? )) http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-tools/hadoop-resourceestimator/src/main/bin/stop-estimator.cmd ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/bin/stop-estimator.cmd b/hadoop-tools/hadoop-resourceestimator/src/main/bin/stop-estimator.cmd new file mode 100644 index 0000000..e08c7e0 --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/main/bin/stop-estimator.cmd @@ -0,0 +1,37 @@ +@echo off +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +setlocal enabledelayedexpansion + +echo stopping resource estimator daemons + +if not defined HADOOP_BIN_PATH ( + set HADOOP_BIN_PATH=%~dp0 +) + +if "%HADOOP_BIN_PATH:~-1%" == "\" ( + set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1% +) + +set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec +if not defined HADOOP_LIBEXEC_DIR ( + set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR% +) + +@rem start resource estimator +stop "Resource Estimator Service" estimator resourceestimator + +endlocal http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-tools/hadoop-resourceestimator/src/main/bin/stop-estimator.sh ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/bin/stop-estimator.sh b/hadoop-tools/hadoop-resourceestimator/src/main/bin/stop-estimator.sh new file mode 100644 index 0000000..e78c798 --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/main/bin/stop-estimator.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. +# + +# let's locate libexec... +if [[ -n "${HADOOP_HOME}" ]]; then + HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" +else + this="${BASH_SOURCE-$0}" + bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../../../../../libexec" +fi + +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then + # shellcheck source=./hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh + . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh." 2>&1 + exit 1 +fi + +# stop resource estimator +echo "Stopping resource estimator" +hadoop_uservar_su estimator resourceestimator "bin/estimator.sh" \ + --config "${HADOOP_CONF_DIR}" \ + --daemon stop \ + resourceestimator +(( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? )) http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-tools/hadoop-resourceestimator/src/main/conf/resourceestimator-config.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/conf/resourceestimator-config.xml b/hadoop-tools/hadoop-resourceestimator/src/main/conf/resourceestimator-config.xml new file mode 100644 index 0000000..f78085a --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/main/conf/resourceestimator-config.xml @@ -0,0 +1,85 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration> + <property> + <name>resourceestimator.solver.lp.alpha</name> + <value>0.1</value> + <description> + The resource estimator has an integrated Linear Programming solver to make the prediction, and this parameter tunes the tradeoff between resource over-allocation and under-allocation in the Linear Programming model. This parameter varies from 0 to 1, and a larger alpha value means the model minimizes over-allocation better. Default value is 0.1. + </description> + </property> + <property> + <name>resourceestimator.solver.lp.beta</name> + <value>0.1</value> + <description> + This parameter controls the generalization of the Linear Programming model. This parameter varies from 0 to 1. Deafult value is 0.1. + </description> + </property> + <property> + <name>resourceestimator.solver.lp.minJobRuns</name> + <value>2</value> + <description> + The minimum number of job runs required in order to make the prediction. Default value is 2. + </description> + </property> + <property> + <name>resourceestimator.timeInterval</name> + <value>5</value> + <description> + The time length which is used to discretize job execution into intervals. Note that the estimator makes resource allocation prediction for each interval. A smaller time interval has more fine-grained granularity for prediction, but it also takes longer time and more space for prediction. Default value is 5 (seconds). + </description> + </property> + <property> + <name>resourceestimator.translator.line-parser</name> + <value>org.apache.hadoop.resourceestimator.translator.impl.NativeSingleLineParser</value> + <description> + The class name of the translator single-line parser, which parses a single line in the log. Default value is org.apache.hadoop.resourceestimator.translator.impl.NativeSingleLineParser, which can parse one line in the sample log. Note that if users want to parse Hadoop Resource Manager logs, they need to set the value to be org.apache.hadoop.resourceestimator.translator.impl.RmSingleLineParser. If they want to implement single-line parser to parse their customized log file, they need to change this value accordingly. + </description> + </property> + <!-- Optional configuration parameters --> + <!-- + <property> + <name>resourceestimator.solver.provider</name> + <value>org.apache.hadoop.resourceestimator.solver.impl.LpSolver</value> + <description> + The class name of the solver provider. Default value is org.apache.hadoop.resourceestimator.solver.impl.LpSolver, which incorporates a Linear Programming model to make the prediction. If users want to implement their own models, they need to change this value accordingly. + </description> + </property> + <property> + <name>resourceestimator.skylinestore.provider</name> + <value>org.apache.hadoop.resourceestimator.skylinestore.impl.InMemoryStore</value> + <description> + The class name of the skylinestore provider. Default value is org.apache.hadoop.resourceestimator.skylinestore.impl.InMemoryStore, which is an in-memory implementation of skylinestore. If users want to use their own skylinestore implementation, they need to change this value accordingly. + </description> + </property> + <property> + <name>resourceestimator.translator.provider</name> + <value>org.apache.hadoop.resourceestimator.translator.impl.BaseLogParser</value> + <description> + The class name of the translator provider. Default value is org.apache.hadoop.resourceestimator.translator.impl.BaseLogParser, which extracts resourceskylines from log streams. If users want to use their own translator implementation, they need to change this value accordingly. + </description> + </property> + <property> + <name>resourceestimator.service-port</name> + <value>9998</value> + <description> + The port which ResourceEstimatorService listens to. The default value is 9998. + </description> + </property> + --> +</configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-tools/hadoop-resourceestimator/src/main/data/resourceEstimatorService.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/data/resourceEstimatorService.txt b/hadoop-tools/hadoop-resourceestimator/src/main/data/resourceEstimatorService.txt new file mode 100644 index 0000000..15fb84e --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/main/data/resourceEstimatorService.txt @@ -0,0 +1,2 @@ +tpch_q12 50 1462148517000 1462148527345 1462148952345 tpch_q12_0 8192:0| 1:0|1:5|1074:10|2538:15|2468:20| +tpch_q12 50 1462148949000 1462148960455 1462149205455 tpch_q12_1 8192:0| 1:0|1:5|794:10|2517:15|2484:20| \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/RecurrenceId.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/RecurrenceId.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/RecurrenceId.java new file mode 100644 index 0000000..01ef977 --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/RecurrenceId.java @@ -0,0 +1,95 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.resourceestimator.common.api; + +/** + * RecurrenceId is the id for the recurring pipeline jobs. + * <p> We assume that the pipeline job can be uniquely identified with + * {pipelineId, runId}. + */ +public class RecurrenceId { + /** + * pipelineId is the unique id for the pipeline jobs. + */ + private String pipelineId; + /** + * runId is the unique instance id for the pipeline job in one run, and it + * will change across runs. + */ + private String runId; + // TODO: we may addHistory more ids of the pipeline jobs to identify them. + + /** + * Constructor. + * + * @param pipelineIdConfig the unique id for the pipeline jobs. + * @param runIdConfig the unique instance id for the pipeline job in one run. + */ + public RecurrenceId(final String pipelineIdConfig, final String runIdConfig) { + this.pipelineId = pipelineIdConfig; + this.runId = runIdConfig; + } + + /** + * Return the pipelineId for the pipeline jobs. + * + * @return the pipelineId. + */ + public final String getPipelineId() { + return pipelineId; + } + + public void setPipelineId(String pipelineId) { + this.pipelineId = pipelineId; + } + + /** + * Return the runId for the pipeline job in one run. + * + * @return the runId. + */ + public final String getRunId() { + return runId; + } + + public void setRunId(String runId) { + this.runId = runId; + } + + @Override public final String toString() { + return String.format("{pipelineId: %s, runId: %s}", pipelineId, runId); + } + + @Override public final int hashCode() { + return getPipelineId().hashCode() ^ getRunId().hashCode(); + } + + @Override public final boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + final RecurrenceId other = (RecurrenceId) obj; + return pipelineId.equals(other.pipelineId) && runId.equals(other.runId); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/ResourceSkyline.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/ResourceSkyline.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/ResourceSkyline.java new file mode 100644 index 0000000..a4bc69f --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/ResourceSkyline.java @@ -0,0 +1,211 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.resourceestimator.common.api; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation; + +/** + * ResourceSkyline records the job identification information as well as job's + * requested {@code + * container}s information during its lifespan. + */ +public class ResourceSkyline { + /** + * The auto-generated {@code ApplicationId} in job's one run. + * <p> + * <p> + * For a pipeline job, we assume the {@code jobId} changes each time we run + * the pipeline job. + */ + private String jobId; + /** + * The input data size of the job. + */ + private double jobInputDataSize; + /** + * Job submission time. Different logs could have different time format, so we + * store the original string directly extracted from logs. + */ + private long jobSubmissionTime; + /** + * Job finish time. Different logs could have different time format, so we + * store the original string directly extracted from logs. + */ + private long jobFinishTime; + /** + * The resource spec of containers allocated to the job. + */ + private Resource containerSpec; + /** + * The list of {@link Resource} allocated to the job in its lifespan. + */ + private RLESparseResourceAllocation skylineList; + // TODO + // We plan to record pipeline job's actual resource consumptions in one run + // here. + // TODO + // We might need to addHistory more features to the ResourceSkyline, such as + // users, VC, etc. + + /** + * Constructor. + * + * @param jobIdConfig the id of the job. + * @param jobInputDataSizeConfig the input data size of the job. + * @param jobSubmissionTimeConfig the submission time of the job. + * @param jobFinishTimeConfig the finish time of the job. + * @param containerSpecConfig the resource spec of containers allocated + * to the job. + * @param skylineListConfig the list of {@link Resource} allocated in + * one run. + */ + public ResourceSkyline(final String jobIdConfig, + final double jobInputDataSizeConfig, final long jobSubmissionTimeConfig, + final long jobFinishTimeConfig, final Resource containerSpecConfig, + final RLESparseResourceAllocation skylineListConfig) { + this.jobId = jobIdConfig; + this.jobInputDataSize = jobInputDataSizeConfig; + this.jobSubmissionTime = jobSubmissionTimeConfig; + this.jobFinishTime = jobFinishTimeConfig; + this.containerSpec = containerSpecConfig; + this.skylineList = skylineListConfig; + } + + /** + * Empty constructor. + */ + public ResourceSkyline() { + } + + /** + * Get the id of the job. + * + * @return the id of this job. + */ + public final String getJobId() { + return jobId; + } + + /** + * Set jobId. + * + * @param jobIdConfig jobId. + */ + public final void setJobId(final String jobIdConfig) { + this.jobId = jobIdConfig; + } + + /** + * Get the job's input data size. + * + * @return job's input data size. + */ + public final double getJobInputDataSize() { + return jobInputDataSize; + } + + /** + * Set jobInputDataSize. + * + * @param jobInputDataSizeConfig jobInputDataSize. + */ + public final void setJobInputDataSize(final double jobInputDataSizeConfig) { + this.jobInputDataSize = jobInputDataSizeConfig; + } + + /** + * Get the job's submission time. + * + * @return job's submission time. + */ + public final long getJobSubmissionTime() { + return jobSubmissionTime; + } + + /** + * Set jobSubmissionTime. + * + * @param jobSubmissionTimeConfig jobSubmissionTime. + */ + public final void setJobSubmissionTime(final long jobSubmissionTimeConfig) { + this.jobSubmissionTime = jobSubmissionTimeConfig; + } + + /** + * Get the job's finish time. + * + * @return job's finish time. + */ + public final long getJobFinishTime() { + return jobFinishTime; + } + + /** + * Set jobFinishTime. + * + * @param jobFinishTimeConfig jobFinishTime. + */ + public final void setJobFinishTime(final long jobFinishTimeConfig) { + this.jobFinishTime = jobFinishTimeConfig; + } + + /** + * Get the resource spec of the job's allocated {@code container}s. + * <p> Key assumption: during job's lifespan, its allocated {@code container}s + * have the same {@link Resource} spec. + * + * @return the {@link Resource} spec of the job's allocated + * {@code container}s. + */ + public final Resource getContainerSpec() { + return containerSpec; + } + + /** + * Set containerSpec. + * + * @param containerSpecConfig containerSpec. + */ + public final void setContainerSpec(final Resource containerSpecConfig) { + this.containerSpec = containerSpecConfig; + } + + /** + * Get the list of {@link Resource}s allocated to the job. + * + * @return the {@link RLESparseResourceAllocation} which contains the list of + * {@link Resource}s allocated to the job. + */ + public final RLESparseResourceAllocation getSkylineList() { + return skylineList; + } + + /** + * Set skylineList. + * + * @param skylineListConfig skylineList. + */ + public final void setSkylineList( + final RLESparseResourceAllocation skylineListConfig) { + this.skylineList = skylineListConfig; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/package-info.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/package-info.java new file mode 100644 index 0000000..a7c7527 --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/api/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * The common APIs for the resource estimator. + */ + +package org.apache.hadoop.resourceestimator.common.api; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/ResourceEstimatorConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/ResourceEstimatorConfiguration.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/ResourceEstimatorConfiguration.java new file mode 100644 index 0000000..bb041b6 --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/ResourceEstimatorConfiguration.java @@ -0,0 +1,125 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.resourceestimator.common.config; + +import org.apache.hadoop.resourceestimator.skylinestore.impl.InMemoryStore; +import org.apache.hadoop.resourceestimator.solver.impl.LpSolver; +import org.apache.hadoop.resourceestimator.translator.impl.BaseLogParser; +import org.apache.hadoop.resourceestimator.translator.impl.NativeSingleLineParser; + +/** + * Defines configuration keys for ResourceEstimatorServer. + */ +public final class ResourceEstimatorConfiguration { + + /** + * The location of the configuration file for ResourceEstimatorService. + */ + public static final String CONFIG_FILE = "resourceestimator-config.xml"; + + /** + * The URI for ResourceEstimatorService. + */ + public static final String SERVICE_URI = "http://0.0.0.0/";; + + /** + * The port which ResourceEstimatorService listens to. + */ + public static final String SERVICE_PORT = "resourceestimator.service-port"; + + /** + * Default port number of ResourceEstimatorService. + */ + public static final int DEFAULT_SERVICE_PORT = 9998; + + /** + * The class name of the skylinestore provider. + */ + public static final String SKYLINESTORE_PROVIDER = + "resourceestimator.skylinestore.provider"; + + /** + * Default value for skylinestore provider, which is an in-memory implementation of skylinestore. + */ + public static final String DEFAULT_SKYLINESTORE_PROVIDER = + InMemoryStore.class.getName(); + + /** + * The class name of the translator provider. + */ + public static final String TRANSLATOR_PROVIDER = + "resourceestimator.translator.provider"; + + /** + * Default value for translator provider, which extracts resourceskylines from log streams. + */ + public static final String DEFAULT_TRANSLATOR_PROVIDER = + BaseLogParser.class.getName(); + + /** + * The class name of the translator single-line parser, which parses a single line in the log. + */ + public static final String TRANSLATOR_LINE_PARSER = + "resourceestimator.translator.line-parser"; + + /** + * Default value for translator single-line parser, which can parse one line in the sample log. + */ + public static final String DEFAULT_TRANSLATOR_LINE_PARSER = + NativeSingleLineParser.class.getName(); + + /** + * The class name of the solver provider. + */ + public static final String SOLVER_PROVIDER = + "resourceestimator.solver.provider"; + + /** + * Default value for solver provider, which incorporates a Linear Programming model to make the prediction. + */ + public static final String DEFAULT_SOLVER_PROVIDER = LpSolver.class.getName(); + + /** + * The time length which is used to discretize job execution into intervals. + */ + public static final String TIME_INTERVAL_KEY = + "resourceestimator.timeInterval"; + + /** + * The parameter which tunes the tradeoff between resource over-allocation and under-allocation in the Linear Programming model. + */ + public static final String SOLVER_ALPHA_KEY = + "resourceestimator.solver.lp.alpha"; + + /** + * This parameter which controls the generalization of the Linear Programming model. + */ + public static final String SOLVER_BETA_KEY = + "resourceestimator.solver.lp.beta"; + + /** + * The minimum number of job runs required in order to make the prediction. + */ + public static final String SOLVER_MIN_JOB_RUN_KEY = + "resourceestimator.solver.lp.minJobRuns"; + + private ResourceEstimatorConfiguration() {} +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/ResourceEstimatorUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/ResourceEstimatorUtil.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/ResourceEstimatorUtil.java new file mode 100644 index 0000000..c00dd00 --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/ResourceEstimatorUtil.java @@ -0,0 +1,81 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.resourceestimator.common.config; + +import java.lang.reflect.Constructor; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.resourceestimator.common.exception.ResourceEstimatorException; + +/** + * General resourceestimator utils. + */ +public final class ResourceEstimatorUtil { + + private static final Class<?>[] EMPTY_ARRAY = new Class[0]; + + private ResourceEstimatorUtil() {} + + /** + * Helper method to create instances of Object using the class name specified + * in the configuration object. + * + * @param conf the yarn configuration + * @param configuredClassName the configuration provider key + * @param defaultValue the default implementation class + * @param type the required interface/base class + * @param <T> The type of the instance to create + * @return the instances created + * @throws ResourceEstimatorException if the provider initialization fails. + */ + @SuppressWarnings("unchecked") public static <T> T createProviderInstance( + Configuration conf, String configuredClassName, String defaultValue, + Class<T> type) throws ResourceEstimatorException { + String className = conf.get(configuredClassName); + if (className == null) { + className = defaultValue; + } + try { + Class<?> concreteClass = Class.forName(className); + if (type.isAssignableFrom(concreteClass)) { + Constructor<T> meth = + (Constructor<T>) concreteClass.getDeclaredConstructor(EMPTY_ARRAY); + meth.setAccessible(true); + return meth.newInstance(); + } else { + StringBuilder errMsg = new StringBuilder(); + errMsg.append("Class: ").append(className).append(" not instance of ") + .append(type.getCanonicalName()); + throw new ResourceEstimatorException(errMsg.toString()); + } + } catch (ClassNotFoundException e) { + StringBuilder errMsg = new StringBuilder(); + errMsg.append("Could not instantiate : ").append(className) + .append(" due to exception: ").append(e.getCause()); + throw new ResourceEstimatorException(errMsg.toString()); + } catch (ReflectiveOperationException e) { + StringBuilder errMsg = new StringBuilder(); + errMsg.append("Could not instantiate : ").append(className) + .append(" due to exception: ").append(e.getCause()); + throw new ResourceEstimatorException(errMsg.toString()); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/package-info.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/package-info.java new file mode 100644 index 0000000..ecf140a --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/config/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * The common configuration for the resource estimator. + */ + +package org.apache.hadoop.resourceestimator.common.config; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/exception/ResourceEstimatorException.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/exception/ResourceEstimatorException.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/exception/ResourceEstimatorException.java new file mode 100644 index 0000000..ba5666d --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/exception/ResourceEstimatorException.java @@ -0,0 +1,35 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.resourceestimator.common.exception; + +/** + * Exception thrown by ResourceEstimatorServer utility classes. + */ +public class ResourceEstimatorException extends Exception { + + public ResourceEstimatorException(String message) { + super(message); + } + + public ResourceEstimatorException(String message, Exception ex) { + super(message, ex); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/exception/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/exception/package-info.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/exception/package-info.java new file mode 100644 index 0000000..6ebecb6 --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/exception/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * The common exception for the resource estimator. + */ + +package org.apache.hadoop.resourceestimator.common.exception; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/RLESparseResourceAllocationSerDe.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/RLESparseResourceAllocationSerDe.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/RLESparseResourceAllocationSerDe.java new file mode 100644 index 0000000..379dc3a --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/RLESparseResourceAllocationSerDe.java @@ -0,0 +1,77 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.resourceestimator.common.serialization; + +import java.lang.reflect.Type; +import java.util.NavigableMap; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation; +import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; +import org.apache.hadoop.yarn.util.resource.ResourceCalculator; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; +import com.google.gson.reflect.TypeToken; + +/** + * Serialize/deserialize RLESparseResourceAllocation object to/from JSON. + */ +public class RLESparseResourceAllocationSerDe + implements JsonSerializer<RLESparseResourceAllocation>, + JsonDeserializer<RLESparseResourceAllocation> { + private static final String KEY = "resourceAllocation"; + private final Gson gson = + new GsonBuilder().registerTypeAdapter(Resource.class, new ResourceSerDe()) + .create(); + private final Type type = new TypeToken<NavigableMap<Long, Resource>>() { + }.getType(); + private final ResourceCalculator resourceCalculator = + new DefaultResourceCalculator(); + + @Override public final JsonElement serialize( + final RLESparseResourceAllocation resourceAllocation, + final Type typeOfSrc, final JsonSerializationContext context) { + NavigableMap<Long, Resource> myMap = resourceAllocation.getCumulative(); + JsonObject jo = new JsonObject(); + JsonElement element = gson.toJsonTree(myMap, type); + jo.add(KEY, element); + + return jo; + } + + @Override public final RLESparseResourceAllocation deserialize( + final JsonElement json, final Type typeOfT, + final JsonDeserializationContext context) throws JsonParseException { + NavigableMap<Long, Resource> resAllocation = + gson.fromJson(json.getAsJsonObject().get(KEY), type); + RLESparseResourceAllocation rleSparseResourceAllocation = + new RLESparseResourceAllocation(resAllocation, resourceCalculator); + return rleSparseResourceAllocation; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/ResourceSerDe.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/ResourceSerDe.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/ResourceSerDe.java new file mode 100644 index 0000000..22b079c --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/ResourceSerDe.java @@ -0,0 +1,61 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.resourceestimator.common.serialization; + +import java.lang.reflect.Type; + +import org.apache.hadoop.yarn.api.records.Resource; + +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; + +/** + * Serialize/deserialize Resource object to/from JSON. + */ +public class ResourceSerDe + implements JsonSerializer<Resource>, JsonDeserializer<Resource> { + private static final String KEY1 = "memory"; + private static final String KEY2 = "vcores"; + + @Override public final JsonElement serialize(final Resource resource, + final Type type, final JsonSerializationContext context) { + JsonObject jo = new JsonObject(); + jo.addProperty(KEY1, resource.getMemorySize()); + jo.addProperty(KEY2, resource.getVirtualCores()); + return jo; + } + + @Override public final Resource deserialize(final JsonElement json, + final Type type, final JsonDeserializationContext context) + throws JsonParseException { + JsonObject jo = json.getAsJsonObject(); + long mem = jo.getAsJsonPrimitive(KEY1).getAsLong(); + int vcore = jo.getAsJsonPrimitive(KEY2).getAsInt(); + Resource resource = Resource.newInstance(mem, vcore); + + return resource; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/package-info.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/package-info.java new file mode 100644 index 0000000..06957b3 --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/common/serialization/package-info.java @@ -0,0 +1,24 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/** + * SkylineStore serialization module. + */ +package org.apache.hadoop.resourceestimator.common.serialization; http://git-wip-us.apache.org/repos/asf/hadoop/blob/625039ef/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorServer.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorServer.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorServer.java new file mode 100644 index 0000000..27a5864 --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorServer.java @@ -0,0 +1,146 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.resourceestimator.service; + +import java.io.IOException; +import java.net.URI; + +import javax.ws.rs.core.UriBuilder; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.HttpServer2; +import org.apache.hadoop.resourceestimator.common.config.ResourceEstimatorConfiguration; +import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.util.ShutdownHookManager; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; +import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A simple embedded Hadoop HTTP server. + */ +public final class ResourceEstimatorServer extends CompositeService { + private static final Logger LOGGER = + LoggerFactory.getLogger(ResourceEstimatorServer.class.getName()); + private HttpServer2 webServer; + private static URI baseURI; + + public ResourceEstimatorServer() { + super(ResourceEstimatorServer.class.getName()); + } + + private static URI getBaseURI(Configuration config) { + baseURI = UriBuilder.fromUri(ResourceEstimatorConfiguration.SERVICE_URI) + .port(getPort(config)).build(); + return baseURI; + } + + private static int getPort(Configuration config) { + return config.getInt(ResourceEstimatorConfiguration.SERVICE_PORT, + ResourceEstimatorConfiguration.DEFAULT_SERVICE_PORT); + } + + @Override protected void serviceInit(Configuration conf) throws Exception { + super.serviceInit(conf); + } + + @Override protected void serviceStart() throws Exception { + super.serviceStart(); + startResourceEstimatorApp(); + } + + private void join() { + // keep the main thread that started the server up until it receives a stop + // signal + if (webServer != null) { + try { + webServer.join(); + } catch (InterruptedException ignore) { + } + } + } + + @Override protected void serviceStop() throws Exception { + if (webServer != null) { + webServer.stop(); + } + super.serviceStop(); + } + + private void startResourceEstimatorApp() throws IOException { + Configuration config = new YarnConfiguration(); + config.addResource(ResourceEstimatorConfiguration.CONFIG_FILE); + HttpServer2.Builder builder = + new HttpServer2.Builder().setName("ResourceEstimatorServer") + .setConf(config) + //.setFindPort(true) + .addEndpoint(getBaseURI(config)); + webServer = builder.build(); + webServer.addJerseyResourcePackage( + ResourceEstimatorService.class.getPackage().getName() + ";" + + GenericExceptionHandler.class.getPackage().getName() + ";" + + YarnJacksonJaxbJsonProvider.class.getPackage().getName(), "/*"); + webServer.start(); + } + + /** + * Start embedded Hadoop HTTP server. + * + * @return an instance of the started HTTP server. + * @throws IOException in case there is an error while starting server. + */ + static ResourceEstimatorServer startResourceEstimatorServer() + throws IOException, InterruptedException { + Configuration config = new YarnConfiguration(); + config.addResource(ResourceEstimatorConfiguration.CONFIG_FILE); + ResourceEstimatorServer resourceEstimatorServer = null; + try { + resourceEstimatorServer = new ResourceEstimatorServer(); + ShutdownHookManager.get().addShutdownHook( + new CompositeServiceShutdownHook(resourceEstimatorServer), 30); + resourceEstimatorServer.init(config); + resourceEstimatorServer.start(); + } catch (Throwable t) { + LOGGER.error("Error starting ResourceEstimatorServer", t); + } + + return resourceEstimatorServer; + } + + public static void main(String[] args) + throws InterruptedException, IOException { + ResourceEstimatorServer server = startResourceEstimatorServer(); + server.join(); + } + + /** + * Stop embedded Hadoop HTTP server. + * + * @throws Exception in case the HTTP server fails to shut down. + */ + public void shutdown() throws Exception { + LOGGER.info("Stopping resourceestimator service at: {}.", + baseURI.toString()); + webServer.stop(); + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
