Repository: hadoop Updated Branches: refs/heads/branch-2 7c26ae506 -> 9897538a9
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9897538a/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/translator/api/TestJobMetaData.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/translator/api/TestJobMetaData.java b/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/translator/api/TestJobMetaData.java new file mode 100644 index 0000000..69ba480 --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/translator/api/TestJobMetaData.java @@ -0,0 +1,163 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.resourceestimator.translator.api; + +import java.text.ParseException; + +import org.apache.hadoop.resourceestimator.common.api.RecurrenceId; +import org.apache.hadoop.resourceestimator.translator.impl.LogParserUtil; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +/** + * Test JobMetaData. + */ +public class TestJobMetaData { + /** + * TODO: parametrize this test. + */ + private LogParserUtil logParserUtil = new LogParserUtil(); + + private JobMetaData jobMetaData; + private RecurrenceId recurrenceId; + + @Before public final void setup() throws ParseException { + recurrenceId = new RecurrenceId("Fraud Detection", "17/07/16 16:27:25"); + jobMetaData = new JobMetaData( + logParserUtil.stringToUnixTimestamp("17/07/16 16:27:25")); + jobMetaData.setRecurrenceId(recurrenceId); + jobMetaData.setContainerStart("C1", + logParserUtil.stringToUnixTimestamp("17/07/16 16:27:30")); + jobMetaData.setContainerEnd("C1", + logParserUtil.stringToUnixTimestamp("17/07/16 16:37:30")); + jobMetaData.setContainerStart("C2", + logParserUtil.stringToUnixTimestamp("17/07/16 16:27:40")); + jobMetaData.setContainerEnd("C2", + logParserUtil.stringToUnixTimestamp("17/07/16 16:37:40")); + jobMetaData.setJobFinishTime( + logParserUtil.stringToUnixTimestamp("17/07/16 16:37:45")); + final Resource containerAlloc = Resource.newInstance(1, 1); + jobMetaData.getResourceSkyline().setContainerSpec(containerAlloc); + jobMetaData.getResourceSkyline().setJobInputDataSize(1024.5); + jobMetaData.createSkyline(); + } + + @Test public final void testGetContainerSpec() { + final Resource containerAlloc = + jobMetaData.getResourceSkyline().getContainerSpec(); + final Resource containerAlloc2 = Resource.newInstance(1, 1); + Assert.assertEquals(containerAlloc.getMemorySize(), + containerAlloc2.getMemorySize()); + Assert.assertEquals(containerAlloc.getVirtualCores(), + containerAlloc2.getVirtualCores()); + } + + @Test public final void testGetJobSize() { + Assert.assertEquals(jobMetaData.getResourceSkyline().getJobInputDataSize(), + 1024.5, 0); + } + + @Test public final void testGetRecurrenceeId() { + final RecurrenceId recurrenceIdTest = + new RecurrenceId("Fraud Detection", "17/07/16 16:27:25"); + Assert.assertEquals(recurrenceIdTest, jobMetaData.getRecurrenceId()); + } + + @Test public final void testStringToUnixTimestamp() throws ParseException { + final long submissionTime = + logParserUtil.stringToUnixTimestamp("17/07/16 16:27:25"); + Assert.assertEquals(jobMetaData.getResourceSkyline().getJobSubmissionTime(), + submissionTime); + } + + @Test public final void testResourceSkyline() { + final RLESparseResourceAllocation skylineList = + jobMetaData.getResourceSkyline().getSkylineList(); + final int containerCPU = + jobMetaData.getResourceSkyline().getContainerSpec().getVirtualCores(); + int k; + for (k = 0; k < 5; k++) { + Assert.assertEquals(0, + skylineList.getCapacityAtTime(k).getVirtualCores() / containerCPU); + } + for (k = 5; k < 15; k++) { + Assert.assertEquals(1, + skylineList.getCapacityAtTime(k).getVirtualCores() / containerCPU); + } + for (k = 15; k < 605; k++) { + Assert.assertEquals(2, + skylineList.getCapacityAtTime(k).getVirtualCores() / containerCPU); + } + for (k = 605; k < 615; k++) { + Assert.assertEquals(1, + skylineList.getCapacityAtTime(k).getVirtualCores() / containerCPU); + } + Assert.assertEquals(0, + skylineList.getCapacityAtTime(615).getVirtualCores() / containerCPU); + } + + @Test public final void testContainerReleaseTimeMissing() + throws ParseException { + // create an invalid JobMetaData + recurrenceId = new RecurrenceId("Fraud Detection", "17/07/16 16:27:25"); + jobMetaData = new JobMetaData( + logParserUtil.stringToUnixTimestamp("17/07/16 16:27:25")); + jobMetaData.setRecurrenceId(recurrenceId); + jobMetaData.setContainerStart("C1", + logParserUtil.stringToUnixTimestamp("17/07/16 16:27:30")); + jobMetaData.setContainerEnd("C1", + logParserUtil.stringToUnixTimestamp("17/07/16 16:37:30")); + jobMetaData.setContainerStart("C2", + logParserUtil.stringToUnixTimestamp("17/07/16 16:27:40")); + jobMetaData.setJobFinishTime( + logParserUtil.stringToUnixTimestamp("17/07/16 16:37:45")); + final Resource containerAlloc = Resource.newInstance(1, 1); + jobMetaData.getResourceSkyline().setContainerSpec(containerAlloc); + jobMetaData.getResourceSkyline().setJobInputDataSize(1024.5); + jobMetaData.createSkyline(); + // test the generated ResourceSkyline + final RLESparseResourceAllocation skylineList = + jobMetaData.getResourceSkyline().getSkylineList(); + final int containerCPU = + jobMetaData.getResourceSkyline().getContainerSpec().getVirtualCores(); + int k; + for (k = 0; k < 5; k++) { + Assert.assertEquals(0, + skylineList.getCapacityAtTime(k).getVirtualCores() / containerCPU); + } + for (k = 5; k < 605; k++) { + Assert.assertEquals(1, + skylineList.getCapacityAtTime(k).getVirtualCores() / containerCPU); + } + Assert.assertEquals(0, + skylineList.getCapacityAtTime(605).getVirtualCores() / containerCPU); + } + + @After public final void cleanUp() { + jobMetaData = null; + recurrenceId = null; + logParserUtil = null; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/9897538a/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/translator/impl/TestNativeParser.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/translator/impl/TestNativeParser.java b/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/translator/impl/TestNativeParser.java new file mode 100644 index 0000000..f399bf9 --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/translator/impl/TestNativeParser.java @@ -0,0 +1,115 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.resourceestimator.translator.impl; + +import java.io.IOException; +import java.text.ParseException; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.resourceestimator.common.api.RecurrenceId; +import org.apache.hadoop.resourceestimator.common.api.ResourceSkyline; +import org.apache.hadoop.resourceestimator.common.config.ResourceEstimatorConfiguration; +import org.apache.hadoop.resourceestimator.common.exception.ResourceEstimatorException; +import org.apache.hadoop.resourceestimator.skylinestore.api.SkylineStore; +import org.apache.hadoop.resourceestimator.skylinestore.exceptions.SkylineStoreException; +import org.apache.hadoop.resourceestimator.skylinestore.impl.InMemoryStore; +import org.apache.hadoop.resourceestimator.translator.api.LogParser; +import org.apache.hadoop.resourceestimator.translator.exceptions.DataFieldNotFoundException; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +/** + * This sample parser will parse the sample log and extract the resource + * skyline. + */ +public class TestNativeParser { + private LogParserUtil logParserUtil = new LogParserUtil(); + private SkylineStore skylineStore; + + @Before public final void setup() throws ResourceEstimatorException { + skylineStore = new InMemoryStore(); + final LogParser nativeParser = new BaseLogParser(); + Configuration config = new Configuration(); + config.addResource(ResourceEstimatorConfiguration.CONFIG_FILE); + nativeParser.init(config, skylineStore); + logParserUtil.setLogParser(nativeParser); + } + + private void parseFile(final String logFile) + throws SkylineStoreException, IOException, ParseException, + ResourceEstimatorException, DataFieldNotFoundException { + logParserUtil.parseLog(logFile); + } + + @Test public final void testParse() + throws SkylineStoreException, IOException, ParseException, + ResourceEstimatorException, DataFieldNotFoundException { + final String logFile = "src/test/resources/trace/nativeLog.txt"; + parseFile(logFile); + final RecurrenceId recurrenceId = + new RecurrenceId("tpch_q12", "tpch_q12_0"); + final Map<RecurrenceId, List<ResourceSkyline>> jobSkylineLists = + skylineStore.getHistory(recurrenceId); + Assert.assertEquals(1, jobSkylineLists.size()); + final List<ResourceSkyline> jobHistory = jobSkylineLists.get(recurrenceId); + Assert.assertEquals(1, jobHistory.size()); + final ResourceSkyline resourceSkyline = jobHistory.get(0); + Assert.assertEquals(0, resourceSkyline.getJobInputDataSize(), 0); + Assert.assertEquals("tpch_q12_0", resourceSkyline.getJobId()); + Assert.assertEquals(0, resourceSkyline.getJobSubmissionTime()); + Assert.assertEquals(25, resourceSkyline.getJobFinishTime()); + Assert + .assertEquals(1024, resourceSkyline.getContainerSpec().getMemorySize()); + Assert + .assertEquals(1, resourceSkyline.getContainerSpec().getVirtualCores()); + final RLESparseResourceAllocation skylineLists = + resourceSkyline.getSkylineList(); + int k; + for (k = 0; k < 10; k++) { + Assert.assertEquals(1, + skylineLists.getCapacityAtTime(k).getMemorySize() / 1024); + } + for (k = 10; k < 15; k++) { + Assert.assertEquals(1074, + skylineLists.getCapacityAtTime(k).getMemorySize() / 1024); + } + for (k = 15; k < 20; k++) { + Assert.assertEquals(2538, + skylineLists.getCapacityAtTime(k).getMemorySize() / 1024); + } + for (k = 20; k < 25; k++) { + Assert.assertEquals(2468, + skylineLists.getCapacityAtTime(k).getMemorySize() / 1024); + } + Assert.assertEquals(0, + skylineLists.getCapacityAtTime(25).getMemorySize() / 1024); + } + + @After public final void cleanUp() { + skylineStore = null; + logParserUtil = null; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/9897538a/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/translator/impl/TestRmParser.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/translator/impl/TestRmParser.java b/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/translator/impl/TestRmParser.java new file mode 100644 index 0000000..4a39ed7 --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/translator/impl/TestRmParser.java @@ -0,0 +1,239 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.resourceestimator.translator.impl; + +import java.io.IOException; +import java.text.ParseException; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.resourceestimator.common.api.RecurrenceId; +import org.apache.hadoop.resourceestimator.common.api.ResourceSkyline; +import org.apache.hadoop.resourceestimator.common.config.ResourceEstimatorConfiguration; +import org.apache.hadoop.resourceestimator.common.exception.ResourceEstimatorException; +import org.apache.hadoop.resourceestimator.skylinestore.api.SkylineStore; +import org.apache.hadoop.resourceestimator.skylinestore.exceptions.SkylineStoreException; +import org.apache.hadoop.resourceestimator.skylinestore.impl.InMemoryStore; +import org.apache.hadoop.resourceestimator.translator.api.LogParser; +import org.apache.hadoop.resourceestimator.translator.exceptions.DataFieldNotFoundException; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +/** + * This sample parser will parse the sample log and extract the resource + * skyline. + */ +public class TestRmParser { + private LogParserUtil logParserUtil = new LogParserUtil(); + private SkylineStore skylineStore; + + @Before public final void setup() throws ResourceEstimatorException { + skylineStore = new InMemoryStore(); + final LogParser rmParser = new BaseLogParser(); + Configuration config = new Configuration(); + config.addResource(new org.apache.hadoop.fs.Path( + ResourceEstimatorConfiguration.CONFIG_FILE)); + config.set(ResourceEstimatorConfiguration.TRANSLATOR_LINE_PARSER, + RmSingleLineParser.class.getName()); + rmParser.init(config, skylineStore); + logParserUtil.setLogParser(rmParser); + } + + private void parseFile(final String logFile) + throws SkylineStoreException, IOException, ParseException, + ResourceEstimatorException, DataFieldNotFoundException { + logParserUtil.parseLog(logFile); + } + + @Test public final void testParse() + throws SkylineStoreException, IOException, ParseException, + ResourceEstimatorException, DataFieldNotFoundException { + final String logFile = "src/test/resources/trace/rmLog.txt"; + parseFile(logFile); + final RecurrenceId recurrenceId = new RecurrenceId("FraudDetection", "1"); + final Map<RecurrenceId, List<ResourceSkyline>> jobSkylineLists = + skylineStore.getHistory(recurrenceId); + Assert.assertEquals(1, jobSkylineLists.size()); + final List<ResourceSkyline> jobHistory = jobSkylineLists.get(recurrenceId); + Assert.assertEquals(1, jobHistory.size()); + final ResourceSkyline resourceSkyline = jobHistory.get(0); + Assert.assertEquals(0, resourceSkyline.getJobInputDataSize(), 0); + Assert.assertEquals("application_1497832133857_0330", + resourceSkyline.getJobId()); + Assert.assertEquals( + logParserUtil.stringToUnixTimestamp("06/21/2017 16:10:13"), + resourceSkyline.getJobSubmissionTime()); + Assert.assertEquals( + logParserUtil.stringToUnixTimestamp("06/21/2017 16:18:35"), + resourceSkyline.getJobFinishTime()); + final Resource resource = Resource.newInstance(1800, 1); + Assert.assertEquals(resource.getMemorySize(), + resourceSkyline.getContainerSpec().getMemorySize()); + Assert.assertEquals(resource.getVirtualCores(), + resourceSkyline.getContainerSpec().getVirtualCores()); + final RLESparseResourceAllocation skylineLists = + resourceSkyline.getSkylineList(); + + int k; + for (k = 0; k < 142; k++) { + Assert.assertEquals(1, + skylineLists.getCapacityAtTime(k).getMemorySize() / resource + .getMemorySize()); + } + for (k = 142; k < 345; k++) { + Assert.assertEquals(2, + skylineLists.getCapacityAtTime(k).getMemorySize() / resource + .getMemorySize()); + } + for (k = 345; k < 502; k++) { + Assert.assertEquals(1, + skylineLists.getCapacityAtTime(k).getMemorySize() / resource + .getMemorySize()); + } + } + + @Test(expected = ParseException.class) + public final void testInvalidDateFormat() + throws ParseException { + logParserUtil.stringToUnixTimestamp("2017.07.16 16:37:45"); + } + + @Test public final void testDuplicateJobSubmissionTime() + throws SkylineStoreException, IOException, ParseException, + ResourceEstimatorException, DataFieldNotFoundException { + final String logFile = "src/test/resources/trace/invalidLog1.txt"; + parseFile(logFile); + final RecurrenceId recurrenceId = new RecurrenceId("Test", "1"); + final Map<RecurrenceId, List<ResourceSkyline>> jobSkylineLists = + skylineStore.getHistory(recurrenceId); + Assert.assertEquals( + logParserUtil.stringToUnixTimestamp("06/21/2017 16:10:23"), + jobSkylineLists.get(recurrenceId).get(0).getJobSubmissionTime()); + } + + @Test public final void testJobIdNotFoundInJobSubmission() + throws SkylineStoreException, IOException, ParseException, + ResourceEstimatorException, DataFieldNotFoundException { + final String logFile = "src/test/resources/trace/invalidLog2.txt"; + parseFile(logFile); + final RecurrenceId recurrenceId = new RecurrenceId("Test", "2"); + Assert.assertNull(skylineStore.getHistory(recurrenceId)); + } + + @Test public final void testJobIdNotFoundInContainerAlloc() + throws SkylineStoreException, IOException, ParseException, + ResourceEstimatorException, DataFieldNotFoundException { + final String logFile = "src/test/resources/trace/invalidLog3.txt"; + parseFile(logFile); + final RecurrenceId recurrenceId = new RecurrenceId("Test", "3"); + final Map<RecurrenceId, List<ResourceSkyline>> jobSkylineLists = + skylineStore.getHistory(recurrenceId); + Assert.assertEquals(0, + jobSkylineLists.get(recurrenceId).get(0).getSkylineList() + .getCumulative().size()); + } + + @Test public final void testContainerIdNotFoundInContainerAlloc() + throws SkylineStoreException, IOException, ParseException, + ResourceEstimatorException, DataFieldNotFoundException { + final String logFile = "src/test/resources/trace/invalidLog4.txt"; + parseFile(logFile); + final RecurrenceId recurrenceId = new RecurrenceId("Test", "4"); + final Map<RecurrenceId, List<ResourceSkyline>> jobSkylineLists = + skylineStore.getHistory(recurrenceId); + Assert.assertEquals(0, + jobSkylineLists.get(recurrenceId).get(0).getSkylineList() + .getCumulative().size()); + } + + @Test public final void testJobIdNotFoundInJobFailure() + throws SkylineStoreException, IOException, ParseException, + ResourceEstimatorException, DataFieldNotFoundException { + final String logFile = "src/test/resources/trace/invalidLog5.txt"; + parseFile(logFile); + final RecurrenceId recurrenceId = new RecurrenceId("Test", "5"); + final Map<RecurrenceId, List<ResourceSkyline>> jobSkylineLists = + skylineStore.getHistory(recurrenceId); + Assert.assertEquals( + logParserUtil.stringToUnixTimestamp("06/21/2017 16:10:13"), + jobSkylineLists.get(recurrenceId).get(0).getJobSubmissionTime()); + } + + @Test public final void testJobIdNotFoundInJobFinish() + throws SkylineStoreException, IOException, ParseException, + ResourceEstimatorException, DataFieldNotFoundException { + final String logFile = "src/test/resources/trace/invalidLog6.txt"; + parseFile(logFile); + final RecurrenceId recurrenceId = new RecurrenceId("Test", "6"); + Assert.assertNull(skylineStore.getHistory(recurrenceId)); + } + + @Test public final void testRecurrenceIdNotFoundInJobFinish() + throws SkylineStoreException, IOException, ParseException, + ResourceEstimatorException, DataFieldNotFoundException { + final String logFile = "src/test/resources/trace/invalidLog7.txt"; + parseFile(logFile); + final RecurrenceId recurrenceId = new RecurrenceId("Test", "7"); + Assert.assertNull(skylineStore.getHistory(recurrenceId)); + } + + @Test public final void testJobIdNotFoundInResourceSpec() + throws SkylineStoreException, IOException, ParseException, + ResourceEstimatorException, DataFieldNotFoundException { + final String logFile = "src/test/resources/trace/invalidLog8.txt"; + parseFile(logFile); + final RecurrenceId recurrenceId = new RecurrenceId("Test", "8"); + final Map<RecurrenceId, List<ResourceSkyline>> jobSkylineLists = + skylineStore.getHistory(recurrenceId); + Assert.assertEquals(1024, + jobSkylineLists.get(recurrenceId).get(0).getContainerSpec() + .getMemorySize()); + Assert.assertEquals(1, + jobSkylineLists.get(recurrenceId).get(0).getContainerSpec() + .getVirtualCores()); + } + + @Test public final void testResourceSpecNotFoundInResourceSpec() + throws SkylineStoreException, IOException, ParseException, + ResourceEstimatorException, DataFieldNotFoundException { + final String logFile = "src/test/resources/trace/invalidLog9.txt"; + parseFile(logFile); + final RecurrenceId recurrenceId = new RecurrenceId("Test", "9"); + final Map<RecurrenceId, List<ResourceSkyline>> jobSkylineLists = + skylineStore.getHistory(recurrenceId); + Assert.assertEquals(1024, + jobSkylineLists.get(recurrenceId).get(0).getContainerSpec() + .getMemorySize()); + Assert.assertEquals(1, + jobSkylineLists.get(recurrenceId).get(0).getContainerSpec() + .getVirtualCores()); + } + + @After public final void cleanUp() { + skylineStore = null; + logParserUtil = null; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/9897538a/hadoop-tools/hadoop-resourceestimator/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/log4j.properties b/hadoop-tools/hadoop-resourceestimator/src/test/resources/log4j.properties new file mode 100644 index 0000000..60b8fbe --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/test/resources/log4j.properties @@ -0,0 +1,25 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +log4j.rootLogger=INFO, stdout +# Redirect log messages to console +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n http://git-wip-us.apache.org/repos/asf/hadoop/blob/9897538a/hadoop-tools/hadoop-resourceestimator/src/test/resources/lp/answer.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/lp/answer.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/lp/answer.txt new file mode 100644 index 0000000..94565de --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/test/resources/lp/answer.txt @@ -0,0 +1,7 @@ +1 +1 +1738 +2535 +2484 +2478 +2468 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/9897538a/hadoop-tools/hadoop-resourceestimator/src/test/resources/lp/tinySample.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/lp/tinySample.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/lp/tinySample.txt new file mode 100644 index 0000000..15fb84e --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/test/resources/lp/tinySample.txt @@ -0,0 +1,2 @@ +tpch_q12 50 1462148517000 1462148527345 1462148952345 tpch_q12_0 8192:0| 1:0|1:5|1074:10|2538:15|2468:20| +tpch_q12 50 1462148949000 1462148960455 1462149205455 tpch_q12_1 8192:0| 1:0|1:5|794:10|2517:15|2484:20| \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/9897538a/hadoop-tools/hadoop-resourceestimator/src/test/resources/lp/tpch_q12.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/lp/tpch_q12.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/lp/tpch_q12.txt new file mode 100644 index 0000000..ee19625 --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/test/resources/lp/tpch_q12.txt @@ -0,0 +1,13 @@ +tpch_q12 50 1462148517000 1462148527345 1462148952345 tpch_q12_0 8192:0|8192:5|8798208:10|20791296:15|20348928:20|20275200:25|20217856:30|20160512:35|20045824:40|18554880:45|15736832:50|15302656:55|14991360:60|14737408:65|14508032:70|14360576:75|14196736:80|14049280:85|13901824:90|13795328:95|13721600:100|13664256:105|13565952:110|12877824:115|12148736:120|11476992:125|10829824:130|10444800:135|10059776:140|9633792:145|9355264:150|9125888:155|9027584:160|8937472:165|8839168:170|8757248:175|8642560:180|8527872:185|8421376:190|8396800:195|8355840:200|8323072:205|8298496:210|8282112:215|8282112:220|8273920:225|8273920:230|8273920:235|8273920:240|8273920:245|8273920:250|8273920:255|7700480:260|5160960:265|4087808:270|3907584:275|3702784:280|3407872:285|2965504:290|2686976:295|2473984:300|2244608:305|2056192:310|1933312:315|1810432:320|1613824:325|1417216:330|1277952:335|1130496:340|843776:345|679936:350|647168:355|630784:360|630784:365|630784:370|614400:375|581632:380|548864:385|540672: 390|540672:395|540672:400|507904:405|491520:410|278528:415|16384:420|0:425| 1:0|1:5|1074:10|2538:15|2484:20|2475:25|2468:30| +tpch_q12 50 1462148949000 1462148960455 1462149205455 tpch_q12_1 8192:0|8192:5|6504448:10|20619264:15|20348928:20|20299776:25|20201472:30|20127744:35|20021248:40|18366464:45|15392768:50|15605760:55|15400960:60|16064512:65|13664256:70|15753216:75|15843328:80|15843328:85|13615104:90|11943936:95|11501568:100|10895360:105|10223616:110|9822208:115|9478144:120|9142272:125|8855552:130|8683520:135|8544256:140|8413184:145|8331264:150|8290304:155|8282112:160|8273920:165|8273920:170|8273920:175|8273920:180|8273920:185|8273920:190|8265728:195|4890624:200|1089536:205|671744:210|565248:215|516096:220|483328:225|360448:230|16384:235|16384:240|8192:245|0:250| 1:0|1:5|794:10|2517:15|2484:20|2478:25|2466:30| +tpch_q12 50 1462150611000 1462150620642 1462150895642 tpch_q12_7 8192:0|8192:5|11362304:10|20365312:15|20348928:20|20242432:25|20176896:30|20029440:35|19783680:40|16703488:45|15286272:50|14589952:55|14172160:60|13770752:65|13598720:70|12992512:75|12468224:80|12025856:85|11640832:90|11460608:95|11272192:100|10510336:105|9789440:110|9207808:115|8781824:120|8585216:125|8478720:130|8396800:135|8314880:140|8290304:145|8273920:150|8273920:155|8273920:160|8273920:165|8273920:170|8273920:175|8273920:180|7077888:185|4407296:190|3735552:195|3260416:200|2736128:205|2228224:210|1777664:215|1433600:220|1155072:225|983040:230|843776:235|720896:240|647168:245|630784:250|622592:255|565248:260|303104:265|16384:270|8192:275|0:280| 1:0|1:5|1387:10|2486:15|2484:20|2471:25|2463:30| +tpch_q12 50 1462150893000 1462150904200 1462151134200 tpch_q12_8 8192:0|8192:5|7946240:10|20389888:15|20348928:20|20267008:25|20160512:30|20054016:35|19824640:40|17866752:45|14041088:50|14114816:55|12615680:60|14336000:65|12771328:70|12189696:75|11796480:80|11599872:85|11517952:90|11460608:95|11403264:100|11116544:105|10010624:110|9306112:115|8847360:120|8593408:125|8404992:130|8331264:135|8298496:140|8290304:145|8282112:150|8282112:155|8282112:160|8282112:165|8282112:170|8282112:175|8273920:180|8273920:185|8273920:190|8273920:195|8273920:200|8273920:205|8118272:210|2883584:215|376832:220|16384:225|8192:230|0:235| 1:0|1:5|970:10|2489:15|2484:20|2474:25|2461:30| +tpch_q12 50 1462151132000 1462151141241 1462151421241 tpch_q12_9 8192:0|16384:5|16031744:10|20348928:15|20348928:20|20201472:25|20152320:30|20004864:35|19562496:40|16121856:45|15319040:50|14589952:55|14139392:60|13770752:65|13131776:70|12378112:75|11878400:80|11649024:85|11534336:90|11444224:95|11157504:100|10362880:105|9650176:110|9134080:115|8749056:120|8519680:125|8372224:130|8306688:135|8290304:140|8282112:145|8282112:150|8273920:155|8273920:160|8273920:165|8273920:170|8273920:175|8273920:180|8273920:185|7569408:190|4505600:195|3522560:200|3072000:205|2801664:210|2523136:215|2080768:220|1679360:225|1343488:230|1007616:235|892928:240|819200:245|688128:250|573440:255|540672:260|524288:265|491520:270|24576:275|8192:280|0:285| 1:0|2:5|1957:10|2484:15|2484:20|2466:25|2460:30| +tpch_q12 50 1462151420000 1462151429196 1462151739196 tpch_q12_10 8192:0|8192:5|10543104:10|20357120:15|20348928:20|20234240:25|20168704:30|20045824:35|19750912:40|17293312:45|15327232:50|14770176:55|14336000:60|13926400:65|13410304:70|12517376:75|12034048:80|11714560:85|11526144:90|11427840:95|11247616:100|10616832:105|9928704:110|9412608:115|8962048:120|8585216:125|8380416:130|8323072:135|8290304:140|8282112:145|8282112:150|8282112:155|8273920:160|8273920:165|8273920:170|8273920:175|8273920:180|8273920:185|8273920:190|6889472:195|4120576:200|3465216:205|3153920:210|2924544:215|2514944:220|2056192:225|1794048:230|1646592:235|1417216:240|1122304:245|1048576:250|1007616:255|892928:260|778240:265|647168:270|557056:275|532480:280|524288:285|507904:290|475136:295|229376:300|16384:305|8192:310|0:315| 1:0|1:5|1287:10|2485:15|2484:20|2470:25|2462:30| +tpch_q12 50 1462151735000 1462151744905 1462152069905 tpch_q12_11 8192:0|8192:5|9494528:10|20389888:15|20348928:20|20267008:25|20168704:30|20086784:35|19652608:40|17334272:45|15351808:50|15040512:55|14639104:60|14295040:65|13967360:70|13672448:75|12853248:80|12181504:85|11730944:90|11493376:95|11255808:100|10592256:105|10002432:110|9633792:115|9224192:120|8896512:125|8634368:130|8462336:135|8355840:140|8298496:145|8282112:150|8273920:155|8273920:160|8273920:165|8273920:170|8273920:175|8273920:180|8142848:185|5718016:190|4440064:195|3989504:200|3604480:205|3334144:210|2981888:215|2572288:220|2269184:225|1966080:230|1744896:235|1507328:240|1310720:245|1089536:250|942080:255|851968:260|794624:265|745472:270|655360:275|598016:280|565248:285|548864:290|532480:295|532480:300|532480:305|491520:310|270336:315|16384:320|8192:325|0:330| 1:0|1:5|1159:10|2489:15|2484:20|2474:25|2462:30| +tpch_q12 50 1462152067000 1462152075862 1462152365862 tpch_q12_12 8192:0|8192:5|11583488:10|20357120:15|20348928:20|20234240:25|20168704:30|20037632:35|19734528:40|16744448:45|15351808:50|14704640:55|14229504:60|13901824:65|13631488:70|13107200:75|12500992:80|12140544:85|11804672:90|11558912:95|11386880:100|10723328:105|9928704:110|9355264:115|8937472:120|8691712:125|8519680:130|8421376:135|8372224:140|8339456:145|8306688:150|8273920:155|8273920:160|8273920:165|8273920:170|8273920:175|8273920:180|8273920:185|7577600:190|4898816:195|3719168:200|3104768:205|2613248:210|2252800:215|2064384:220|1835008:225|1515520:230|1171456:235|974848:240|892928:245|802816:250|729088:255|712704:260|663552:265|557056:270|483328:275|368640:280|16384:285|8192:290|0:295| 1:0|1:5|1414:10|2485:15|2484:20|2470:25|2462:30| +tpch_q12 50 1462161466000 1462161474815 1462161689815 tpch_q12_45 8192:0|8192:5|13836288:10|20365312:15|20348928:20|20209664:25|20160512:30|20029440:35|19906560:40|18333696:45|15794176:50|16015360:55|13451264:60|15269888:65|12730368:70|11862016:75|11657216:80|11509760:85|11460608:90|11403264:95|11296768:100|10698752:105|9814016:110|9175040:115|8757248:120|8486912:125|8339456:130|8290304:135|8282112:140|8273920:145|8273920:150|8273920:155|8273920:160|8273920:165|8273920:170|8273920:175|5447680:180|1245184:185|737280:190|589824:195|483328:200|147456:205|16384:210|8192:215|0:220| 1:0|1:5|1689:10|2486:15|2484:20|2467:25|2461:30| +tpch_q12 50 1462161685000 1462161693550 1462161903550 tpch_q12_46 8192:0|8192:5|13500416:10|20348928:15|20348928:20|20201472:25|20152320:30|20037632:35|19734528:40|17047552:45|14548992:50|13778944:55|14696448:60|16064512:65|14540800:70|12468224:75|11935744:80|11640832:85|11517952:90|11436032:95|11190272:100|10379264:105|9740288:110|9199616:115|8863744:120|8585216:125|8429568:130|8331264:135|8290304:140|8282112:145|8282112:150|8282112:155|8273920:160|8273920:165|8273920:170|8273920:175|8273920:180|8273920:185|8257536:190|3710976:195|393216:200|16384:205|8192:210|0:215| 1:0|1:5|1648:10|2484:15|2484:20|2466:25|2460:30| +tpch_q12 50 1462161901000 1462161909461 1462162219461 tpch_q12_47 8192:0|8192:5|14802944:10|20348928:15|20348928:20|20217856:25|20152320:30|20013056:35|19734528:40|16285696:45|15310848:50|14573568:55|14090240:60|13639680:65|13025280:70|12271616:75|11911168:80|11673600:85|11501568:90|11411456:95|11206656:100|10493952:105|9797632:110|9175040:115|8749056:120|8503296:125|8396800:130|8347648:135|8331264:140|8290304:145|8273920:150|8273920:155|8273920:160|8273920:165|8273920:170|8273920:175|8273920:180|6823936:185|4120576:190|3342336:195|2711552:200|2170880:205|1712128:210|1466368:215|1294336:220|1187840:225|1089536:230|917504:235|770048:240|663552:245|589824:250|548864:255|540672:260|540672:265|540672:270|540672:275|540672:280|540672:285|540672:290|516096:295|409600:300|16384:305|8192:310|0:315| 1:0|1:5|1807:10|2484:15|2484:20|2468:25|2460:30| +tpch_q12 50 1462162218000 1462162226670 1462162536670 tpch_q12_48 8192:0|8192:5|13295616:10|20348928:15|20348928:20|20234240:25|20160512:30|20070400:35|19898368:40|15876096:45|15138816:50|14344192:55|14024704:60|13787136:65|13623296:70|13312000:75|12607488:80|12042240:85|11780096:90|11640832:95|11493376:100|10649600:105|9732096:110|9232384:115|8953856:120|8749056:125|8577024:130|8429568:135|8347648:140|8298496:145|8290304:150|8290304:155|8290304:160|8282112:165|8282112:170|8282112:175|8282112:180|8273920:185|8273920:190|8273920:195|8273920:200|8273920:205|8273920:210|8273920:215|7798784:220|7307264:225|4808704:230|3891200:235|3530752:240|3162112:245|2596864:250|2195456:255|1835008:260|1589248:265|1425408:270|1204224:275|974848:280|745472:285|581632:290|483328:295|237568:300|16384:305|8192:310|0:315| 1:0|1:5|1623:10|2484:15|2484:20|2470:25|2461:30| +tpch_q12 50 1462162533000 1462162541572 1462162766572 tpch_q12_49 8192:0|8192:5|13320192:10|20348928:15|20348928:20|20217856:25|20152320:30|20004864:35|19750912:40|18366464:45|18063360:50|17530880:55|12361728:60|14123008:65|12713984:70|12181504:75|11943936:80|11730944:85|11624448:90|11493376:95|11329536:100|10723328:105|9887744:110|9322496:115|8921088:120|8634368:125|8429568:130|8339456:135|8314880:140|8306688:145|8298496:150|8282112:155|8273920:160|8273920:165|8273920:170|8273920:175|8273920:180|8273920:185|8232960:190|3375104:195|696320:200|581632:205|507904:210|335872:215|16384:220|8192:225|0:230| 1:0|1:5|1626:10|2484:15|2484:20|2468:25|2460:30| http://git-wip-us.apache.org/repos/asf/hadoop/blob/9897538a/hadoop-tools/hadoop-resourceestimator/src/test/resources/resourceEstimatorService.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/resourceEstimatorService.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/resourceEstimatorService.txt new file mode 100644 index 0000000..15fb84e --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/test/resources/resourceEstimatorService.txt @@ -0,0 +1,2 @@ +tpch_q12 50 1462148517000 1462148527345 1462148952345 tpch_q12_0 8192:0| 1:0|1:5|1074:10|2538:15|2468:20| +tpch_q12 50 1462148949000 1462148960455 1462149205455 tpch_q12_1 8192:0| 1:0|1:5|794:10|2517:15|2484:20| \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/9897538a/hadoop-tools/hadoop-resourceestimator/src/test/resources/resourceestimator-config.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/resourceestimator-config.xml b/hadoop-tools/hadoop-resourceestimator/src/test/resources/resourceestimator-config.xml new file mode 100644 index 0000000..f78085a --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/test/resources/resourceestimator-config.xml @@ -0,0 +1,85 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration> + <property> + <name>resourceestimator.solver.lp.alpha</name> + <value>0.1</value> + <description> + The resource estimator has an integrated Linear Programming solver to make the prediction, and this parameter tunes the tradeoff between resource over-allocation and under-allocation in the Linear Programming model. This parameter varies from 0 to 1, and a larger alpha value means the model minimizes over-allocation better. Default value is 0.1. + </description> + </property> + <property> + <name>resourceestimator.solver.lp.beta</name> + <value>0.1</value> + <description> + This parameter controls the generalization of the Linear Programming model. This parameter varies from 0 to 1. Deafult value is 0.1. + </description> + </property> + <property> + <name>resourceestimator.solver.lp.minJobRuns</name> + <value>2</value> + <description> + The minimum number of job runs required in order to make the prediction. Default value is 2. + </description> + </property> + <property> + <name>resourceestimator.timeInterval</name> + <value>5</value> + <description> + The time length which is used to discretize job execution into intervals. Note that the estimator makes resource allocation prediction for each interval. A smaller time interval has more fine-grained granularity for prediction, but it also takes longer time and more space for prediction. Default value is 5 (seconds). + </description> + </property> + <property> + <name>resourceestimator.translator.line-parser</name> + <value>org.apache.hadoop.resourceestimator.translator.impl.NativeSingleLineParser</value> + <description> + The class name of the translator single-line parser, which parses a single line in the log. Default value is org.apache.hadoop.resourceestimator.translator.impl.NativeSingleLineParser, which can parse one line in the sample log. Note that if users want to parse Hadoop Resource Manager logs, they need to set the value to be org.apache.hadoop.resourceestimator.translator.impl.RmSingleLineParser. If they want to implement single-line parser to parse their customized log file, they need to change this value accordingly. + </description> + </property> + <!-- Optional configuration parameters --> + <!-- + <property> + <name>resourceestimator.solver.provider</name> + <value>org.apache.hadoop.resourceestimator.solver.impl.LpSolver</value> + <description> + The class name of the solver provider. Default value is org.apache.hadoop.resourceestimator.solver.impl.LpSolver, which incorporates a Linear Programming model to make the prediction. If users want to implement their own models, they need to change this value accordingly. + </description> + </property> + <property> + <name>resourceestimator.skylinestore.provider</name> + <value>org.apache.hadoop.resourceestimator.skylinestore.impl.InMemoryStore</value> + <description> + The class name of the skylinestore provider. Default value is org.apache.hadoop.resourceestimator.skylinestore.impl.InMemoryStore, which is an in-memory implementation of skylinestore. If users want to use their own skylinestore implementation, they need to change this value accordingly. + </description> + </property> + <property> + <name>resourceestimator.translator.provider</name> + <value>org.apache.hadoop.resourceestimator.translator.impl.BaseLogParser</value> + <description> + The class name of the translator provider. Default value is org.apache.hadoop.resourceestimator.translator.impl.BaseLogParser, which extracts resourceskylines from log streams. If users want to use their own translator implementation, they need to change this value accordingly. + </description> + </property> + <property> + <name>resourceestimator.service-port</name> + <value>9998</value> + <description> + The port which ResourceEstimatorService listens to. The default value is 9998. + </description> + </property> + --> +</configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/9897538a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog1.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog1.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog1.txt new file mode 100644 index 0000000..ba05a89 --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog1.txt @@ -0,0 +1,4 @@ +// Test case 1: duplicate job submission time +252823:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop IP=25.163.46.150 OPERATION=Submit Application Request TARGET=ClientRMService RESULT=SUCCESS APPID=application_1 CALLERCONTEXT=HIVE_QUERY_ID:ExoAdmin_20170621160339_7fed8f3c-3558-4abe-a0db-029eaa44d70e" +252823:e,06/21/2017 16:10:23,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop IP=25.163.46.150 OPERATION=Submit Application Request TARGET=ClientRMService RESULT=SUCCESS APPID=application_1 CALLERCONTEXT=HIVE_QUERY_ID:ExoAdmin_20170621160339_7fed8f3c-3558-4abe-a0db-029eaa44d70e" +253114:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAppManager$ApplicationSummary: appId=application_1,name=Test-1,user=hadoop, queue=PROD,state=FINISHED,trackingUrl=http://BY1PR00OC0019.namprd00.prod.outlook.com:8088/proxy/1,appMasterHost=by2pr08mb1799.namprd08.prod.outlook.com,startTime=1498061413073,finishTime=1498061905698,finalStatus=SUCCEEDED,memorySeconds=1330655,vcoreSeconds=704,preemptedAMContainers=0,preemptedNonAMContainers=0,preemptedResources=<memory:0\, vCores:0>,applicationType=MAPREDUCE" http://git-wip-us.apache.org/repos/asf/hadoop/blob/9897538a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog2.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog2.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog2.txt new file mode 100644 index 0000000..fe24943 --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog2.txt @@ -0,0 +1,3 @@ +// Test case 2: no jobId found in job submission log +252823:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop IP=25.163.46.150 OPERATION=Submit Application Request TARGET=ClientRMService RESULT=SUCCESS APPIDs=2 CALLERCONTEXT=HIVE_QUERY_ID:ExoAdmin_20170621160339_7fed8f3c-3558-4abe-a0db-029eaa44d70e" +253114:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAppManager$ApplicationSummary: appId=application_2,name=Test-2,user=hadoop, queue=PROD,state=FINISHED,trackingUrl=http://BY1PR00OC0019.namprd00.prod.outlook.com:8088/proxy/1,appMasterHost=by2pr08mb1799.namprd08.prod.outlook.com,startTime=1498061413073,finishTime=1498061905698,finalStatus=SUCCEEDED,memorySeconds=1330655,vcoreSeconds=704,preemptedAMContainers=0,preemptedNonAMContainers=0,preemptedResources=<memory:0\, vCores:0>,applicationType=MAPREDUCE" http://git-wip-us.apache.org/repos/asf/hadoop/blob/9897538a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog3.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog3.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog3.txt new file mode 100644 index 0000000..63d9c2c --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog3.txt @@ -0,0 +1,5 @@ +// Test case 3: no jobId found in container allocation log +252823:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop IP=25.163.46.150 OPERATION=Submit Application Request TARGET=ClientRMService RESULT=SUCCESS APPID=application_3 CALLERCONTEXT=HIVE_QUERY_ID:ExoAdmin_20170621160339_7fed8f3c-3558-4abe-a0db-029eaa44d70e" +252839:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Allocated Container TARGET=SchedulerApp RESULT=SUCCESS APPIDs=3 CONTAINERID=container_e26_1497832133857_0330_01_000001" +253100:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Released Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_3 CONTAINERID=container_e26_1497832133857_0330_01_000001" +253114:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAppManager$ApplicationSummary: appId=application_3,name=Test-3,user=hadoop, queue=PROD,state=FINISHED,trackingUrl=http://BY1PR00OC0019.namprd00.prod.outlook.com:8088/proxy/1,appMasterHost=by2pr08mb1799.namprd08.prod.outlook.com,startTime=1498061413073,finishTime=1498061905698,finalStatus=SUCCEEDED,memorySeconds=1330655,vcoreSeconds=704,preemptedAMContainers=0,preemptedNonAMContainers=0,preemptedResources=<memory:0\, vCores:0>,applicationType=MAPREDUCE" http://git-wip-us.apache.org/repos/asf/hadoop/blob/9897538a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog4.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog4.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog4.txt new file mode 100644 index 0000000..d787319 --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog4.txt @@ -0,0 +1,5 @@ +// Test case 4: no containerId found in container allocation log +252823:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop IP=25.163.46.150 OPERATION=Submit Application Request TARGET=ClientRMService RESULT=SUCCESS APPID=application_4 CALLERCONTEXT=HIVE_QUERY_ID:ExoAdmin_20170621160339_7fed8f3c-3558-4abe-a0db-029eaa44d70e" +252839:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Allocated Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_4 CONTAINERIDs=container_e26_1497832133857_0330_01_000001" +253100:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Released Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_4 CONTAINERID=container_e26_1497832133857_0330_01_000001" +253114:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAppManager$ApplicationSummary: appId=application_4,name=Test-4,user=hadoop, queue=PROD,state=FINISHED,trackingUrl=http://BY1PR00OC0019.namprd00.prod.outlook.com:8088/proxy/1,appMasterHost=by2pr08mb1799.namprd08.prod.outlook.com,startTime=1498061413073,finishTime=1498061905698,finalStatus=SUCCEEDED,memorySeconds=1330655,vcoreSeconds=704,preemptedAMContainers=0,preemptedNonAMContainers=0,preemptedResources=<memory:0\, vCores:0>,applicationType=MAPREDUCE" http://git-wip-us.apache.org/repos/asf/hadoop/blob/9897538a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog5.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog5.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog5.txt new file mode 100644 index 0000000..2bafa67 --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog5.txt @@ -0,0 +1,7 @@ +// Test case 5: jobId not found in app failure log +252823:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop IP=25.163.46.150 OPERATION=Submit Application Request TARGET=ClientRMService RESULT=SUCCESS APPID=application_5 CALLERCONTEXT=HIVE_QUERY_ID:ExoAdmin_20170621160339_7fed8f3c-3558-4abe-a0db-029eaa44d70e" +252839:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Allocated Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_5 CONTAINERID=container_e26_1497832133857_0330_01_000001" +253100:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Released Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_5 CONTAINERID=container_e26_1497832133857_0330_01_000001" +e,06/20/2017 22:30:21,yarn resourcemanager,DefaultTag,Pid="5156" Tid="9896" TS="0x01D2EA14CDBCF8F1" String1="17/06/20 22:30:21 INFO capacity.CapacityScheduler: Application Attempt appattempt_5 is done. finalState=FAILED" +252844:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO attempt.RMAppAttemptImpl: Storing attempt: AppId: application_5 AttemptId: appattempt_5_000001 MasterContainer: Container: [ContainerId: container_e26_1497832133857_0330_01_000001, NodeId: by2pr08mb1799.namprd08.prod.outlook.com:10131, NodeHttpAddress: by2pr08mb1799.namprd08.prod.outlook.com:8042, Resource: <memory:1800, vCores:1>, Priority: 0, Token: Token { kind: ContainerToken, service: by2pr08mb1799.namprd08.prod.outlook.com:10131 }, ]" +253114:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAppManager$ApplicationSummary: appId=application_5,name=Test-5,user=hadoop, queue=PROD,state=FINISHED,trackingUrl=http://BY1PR00OC0019.namprd00.prod.outlook.com:8088/proxy/1,appMasterHost=by2pr08mb1799.namprd08.prod.outlook.com,startTime=1498061413073,finishTime=1498061905698,finalStatus=SUCCEEDED,memorySeconds=1330655,vcoreSeconds=704,preemptedAMContainers=0,preemptedNonAMContainers=0,preemptedResources=<memory:0\, vCores:0>,applicationType=MAPREDUCE" http://git-wip-us.apache.org/repos/asf/hadoop/blob/9897538a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog6.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog6.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog6.txt new file mode 100644 index 0000000..bcb5112 --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog6.txt @@ -0,0 +1,5 @@ +// Test case 6: no jobId found in job finish log +252823:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop IP=25.163.46.150 OPERATION=Submit Application Request TARGET=ClientRMService RESULT=SUCCESS APPID=application_6 CALLERCONTEXT=HIVE_QUERY_ID:ExoAdmin_20170621160339_7fed8f3c-3558-4abe-a0db-029eaa44d70e" +252839:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Allocated Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_6 CONTAINERID=container_e26_1497832133857_0330_01_000001" +253100:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Released Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_6 CONTAINERID=container_e26_1497832133857_0330_01_000001" +253114:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAppManager$ApplicationSummary: appIds=application_6,name=Test-6,user=hadoop, queue=PROD,state=FINISHED,trackingUrl=http://BY1PR00OC0019.namprd00.prod.outlook.com:8088/proxy/1,appMasterHost=by2pr08mb1799.namprd08.prod.outlook.com,startTime=1498061413073,finishTime=1498061905698,finalStatus=SUCCEEDED,memorySeconds=1330655,vcoreSeconds=704,preemptedAMContainers=0,preemptedNonAMContainers=0,preemptedResources=<memory:0\, vCores:0>,applicationType=MAPREDUCE" http://git-wip-us.apache.org/repos/asf/hadoop/blob/9897538a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog7.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog7.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog7.txt new file mode 100644 index 0000000..46676be --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog7.txt @@ -0,0 +1,5 @@ +// Test case 7: no recurrenceId found in job finish log +252823:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop IP=25.163.46.150 OPERATION=Submit Application Request TARGET=ClientRMService RESULT=SUCCESS APPID=application_7 CALLERCONTEXT=HIVE_QUERY_ID:ExoAdmin_20170621160339_7fed8f3c-3558-4abe-a0db-029eaa44d70e" +252839:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Allocated Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_7 CONTAINERID=container_e26_1497832133857_0330_01_000001" +253100:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Released Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_7 CONTAINERID=container_e26_1497832133857_0330_01_000001" +253114:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAppManager$ApplicationSummary: appId=application_7,name=Test:7,user=hadoop, queue=PROD,state=FINISHED,trackingUrl=http://BY1PR00OC0019.namprd00.prod.outlook.com:8088/proxy/1,appMasterHost=by2pr08mb1799.namprd08.prod.outlook.com,startTime=1498061413073,finishTime=1498061905698,finalStatus=SUCCEEDED,memorySeconds=1330655,vcoreSeconds=704,preemptedAMContainers=0,preemptedNonAMContainers=0,preemptedResources=<memory:0\, vCores:0>,applicationType=MAPREDUCE" http://git-wip-us.apache.org/repos/asf/hadoop/blob/9897538a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog8.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog8.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog8.txt new file mode 100644 index 0000000..b8a8e20 --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog8.txt @@ -0,0 +1,6 @@ +// Test case 8: no jobId found in resource spec log +252823:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop IP=25.163.46.150 OPERATION=Submit Application Request TARGET=ClientRMService RESULT=SUCCESS APPID=application_8 CALLERCONTEXT=HIVE_QUERY_ID:ExoAdmin_20170621160339_7fed8f3c-3558-4abe-a0db-029eaa44d70e" +252839:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Allocated Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_8 CONTAINERID=container_e26_1497832133857_0330_01_000001" +253100:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Released Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_8 CONTAINERID=container_e26_1497832133857_0330_01_000001" +252844:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO attempt.RMAppAttemptImpl: Storing attempt: AppId: application_8 AttemptIds: appattempt_8_000001 MasterContainer: Container: [ContainerId: container_e26_1497832133857_0330_01_000001, NodeId: by2pr08mb1799.namprd08.prod.outlook.com:10131, NodeHttpAddress: by2pr08mb1799.namprd08.prod.outlook.com:8042, Resource: <memory:1800, vCores:1>, Priority: 0, Token: Token { kind: ContainerToken, service: by2pr08mb1799.namprd08.prod.outlook.com:10131 }, ]" +253114:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAppManager$ApplicationSummary: appId=application_8,name=Test-8,user=hadoop, queue=PROD,state=FINISHED,trackingUrl=http://BY1PR00OC0019.namprd00.prod.outlook.com:8088/proxy/1,appMasterHost=by2pr08mb1799.namprd08.prod.outlook.com,startTime=1498061413073,finishTime=1498061905698,finalStatus=SUCCEEDED,memorySeconds=1330655,vcoreSeconds=704,preemptedAMContainers=0,preemptedNonAMContainers=0,preemptedResources=<memory:0\, vCores:0>,applicationType=MAPREDUCE" http://git-wip-us.apache.org/repos/asf/hadoop/blob/9897538a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog9.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog9.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog9.txt new file mode 100644 index 0000000..598f935 --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/invalidLog9.txt @@ -0,0 +1,6 @@ +// Test case 9: no resource spec found in resource spec log +252823:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop IP=25.163.46.150 OPERATION=Submit Application Request TARGET=ClientRMService RESULT=SUCCESS APPID=application_9 CALLERCONTEXT=HIVE_QUERY_ID:ExoAdmin_20170621160339_7fed8f3c-3558-4abe-a0db-029eaa44d70e" +252839:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Allocated Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_9 CONTAINERID=container_e26_1497832133857_0330_01_000001" +253100:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Released Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_9 CONTAINERID=container_e26_1497832133857_0330_01_000001" +252844:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO attempt.RMAppAttemptImpl: Storing attempt: AppId: application_9 AttemptId: appattempt_9_000001 MasterContainer: Container: [ContainerId: container_e26_1497832133857_0330_01_000001, NodeId: by2pr08mb1799.namprd08.prod.outlook.com:10131, NodeHttpAddress: by2pr08mb1799.namprd08.prod.outlook.com:8042, Resource: <memory:1800, CPU:1>, Priority: 0, Token: Token { kind: ContainerToken, service: by2pr08mb1799.namprd08.prod.outlook.com:10131 }, ]" +253114:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAppManager$ApplicationSummary: appId=application_9,name=Test-9,user=hadoop, queue=PROD,state=FINISHED,trackingUrl=http://BY1PR00OC0019.namprd00.prod.outlook.com:8088/proxy/1,appMasterHost=by2pr08mb1799.namprd08.prod.outlook.com,startTime=1498061413073,finishTime=1498061905698,finalStatus=SUCCEEDED,memorySeconds=1330655,vcoreSeconds=704,preemptedAMContainers=0,preemptedNonAMContainers=0,preemptedResources=<memory:0\, vCores:0>,applicationType=MAPREDUCE" http://git-wip-us.apache.org/repos/asf/hadoop/blob/9897538a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/nativeLog.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/nativeLog.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/nativeLog.txt new file mode 100644 index 0000000..fd03ea3 --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/nativeLog.txt @@ -0,0 +1 @@ +tpch_q12 50 1462148517000 1462148527345 1462148952345 tpch_q12_0 8192:0| 1:0|1:5|1074:10|2538:15|2468:20| \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/9897538a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/rmLog.txt ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/rmLog.txt b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/rmLog.txt new file mode 100644 index 0000000..7532a73 --- /dev/null +++ b/hadoop-tools/hadoop-resourceestimator/src/test/resources/trace/rmLog.txt @@ -0,0 +1,30 @@ +252823:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop IP=25.163.46.150 OPERATION=Submit Application Request TARGET=ClientRMService RESULT=SUCCESS APPID=application_1497832133857_0330 CALLERCONTEXT=HIVE_QUERY_ID:ExoAdmin_20170621160339_7fed8f3c-3558-4abe-a0db-029eaa44d70e" +252825:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO rmapp.RMAppImpl: Storing application with id application_1497832133857_0330" +252826:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO rmapp.RMAppImpl: application_1497832133857_0330 State change from NEW to NEW_SAVING" +252827:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD060D14" String1="17/06/21 16:10:13 INFO recovery.RMStateStore: Storing info for app: application_1497832133857_0330" +252828:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO rmapp.RMAppImpl: application_1497832133857_0330 State change from NEW_SAVING to SUBMITTED" +252829:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO capacity.ParentQueue: Application added - appId: application_1497832133857_0330 user: hadoop leaf-queue of parent: root #applications: 3" +252830:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO capacity.CapacityScheduler: Accepted application application_1497832133857_0330 from user: hadoop, in queue: PROD" +252831:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO rmapp.RMAppImpl: application_1497832133857_0330 State change from SUBMITTED to ACCEPTED" +252834:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO capacity.LeafQueue: Application application_1497832133857_0330 from user: hadoop activated in queue: PROD" +252835:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO capacity.LeafQueue: Application added - appId: application_1497832133857_0330 user: org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue$User@7e2d171f, leaf-queue: PROD #user-pending-applications: 0 #user-active-applications: 1 #queue-pending-applications: 0 #queue-active-applications: 1" +252839:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Allocated Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_1497832133857_0330 CONTAINERID=container_e26_1497832133857_0330_01_000001" +252844:e,06/21/2017 16:10:13,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA8DD11F8F0" String1="17/06/21 16:10:13 INFO attempt.RMAppAttemptImpl: Storing attempt: AppId: application_1497832133857_0330 AttemptId: appattempt_1497832133857_0330_000001 MasterContainer: Container: [ContainerId: container_e26_1497832133857_0330_01_000001, NodeId: by2pr08mb1799.namprd08.prod.outlook.com:10131, NodeHttpAddress: by2pr08mb1799.namprd08.prod.outlook.com:8042, Resource: <memory:1800, vCores:1>, Priority: 0, Token: Token { kind: ContainerToken, service: by2pr08mb1799.namprd08.prod.outlook.com:10131 }, ]" +252897:e,06/21/2017 16:12:34,yarn resourcemanager,DefaultTag,Pid="5156" Tid="41712" TS="0x01D2EAA931905154" String1="17/06/21 16:12:34 INFO resourcemanager.RMAuditLogger: USER=hadoop IP=25.163.46.25 OPERATION=Register App Master TARGET=ApplicationMasterService RESULT=SUCCESS APPID=application_1497832133857_0330 APPATTEMPTID=appattempt_1497832133857_0330_000001" +252900:e,06/21/2017 16:12:34,yarn resourcemanager,DefaultTag,Pid="5156" Tid="41712" TS="0x01D2EAA931905154" String1="17/06/21 16:12:34 INFO rmapp.RMAppImpl: application_1497832133857_0330 State change from ACCEPTED to RUNNING" +252906:e,06/21/2017 16:12:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="41712" TS="0x01D2EAA93234D4E8" String1="17/06/21 16:12:35 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Allocated Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_1497832133857_0330 CONTAINERID=container_e26_1497832133857_0330_01_000002" +252910:e,06/21/2017 16:12:38,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39272" TS="0x01D2EAA933A6643E" String1="17/06/21 16:12:38 INFO scheduler.AppSchedulingInfo: checking for deactivate of application :application_1497832133857_0330" +253016:e,06/21/2017 16:15:58,yarn resourcemanager,DefaultTag,Pid="5156" Tid="39188" TS="0x01D2EAA9AB29C5EE" String1="17/06/21 16:15:58 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Released Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_1497832133857_0330 CONTAINERID=container_e26_1497832133857_0330_01_000002" +253086:e,06/21/2017 16:18:25,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA02A6A418" String1="17/06/21 16:18:25 INFO rmapp.RMAppImpl: Updating application application_1497832133857_0330 with final state: FINISHING" +253087:e,06/21/2017 16:18:25,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA02A6A418" String1="17/06/21 16:18:25 INFO rmapp.RMAppImpl: application_1497832133857_0330 State change from RUNNING to FINAL_SAVING" +253089:e,06/21/2017 16:18:25,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA02B28FF0" String1="17/06/21 16:18:25 INFO recovery.RMStateStore: Updating info for app: application_1497832133857_0330" +253090:e,06/21/2017 16:18:25,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA02B28FF0" String1="17/06/21 16:18:25 INFO rmapp.RMAppImpl: application_1497832133857_0330 State change from FINAL_SAVING to FINISHING" +253091:e,06/21/2017 16:18:26,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA033F3BC8" String1="17/06/21 16:18:26 INFO resourcemanager.ApplicationMasterService: application_1497832133857_0330 unregistered successfully. " +253100:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=AM Released Container TARGET=SchedulerApp RESULT=SUCCESS APPID=application_1497832133857_0330 CONTAINERID=container_e26_1497832133857_0330_01_000001" +253105:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO rmapp.RMAppImpl: application_1497832133857_0330 State change from FINISHING to FINISHED" +253110:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO scheduler.AppSchedulingInfo: Application application_1497832133857_0330 requests cleared" +253111:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO capacity.LeafQueue: Application removed - appId: application_1497832133857_0330 user: hadoop queue: PROD #user-pending-applications: 0 #user-active-applications: 0 #queue-pending-applications: 0 #queue-active-applications: 0" +253112:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO capacity.ParentQueue: Application removed - appId: application_1497832133857_0330 user: hadoop leaf-queue of parent: root #applications: 2" +253113:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAuditLogger: USER=hadoop OPERATION=Application Finished - Succeeded TARGET=RMAppManager RESULT=SUCCESS APPID=application_1497832133857_0330" +253114:e,06/21/2017 16:18:35,yarn resourcemanager,DefaultTag,Pid="5156" Tid="19824" TS="0x01D2EAAA089569FD" String1="17/06/21 16:18:35 INFO resourcemanager.RMAppManager$ApplicationSummary: appId=application_1497832133857_0330,name=FraudDetection-1,user=hadoop, queue=PROD,state=FINISHED,trackingUrl=http://BY1PR00OC0019.namprd00.prod.outlook.com:8088/proxy/application_1497832133857_0330/,appMasterHost=by2pr08mb1799.namprd08.prod.outlook.com,startTime=1498061413073,finishTime=1498061905698,finalStatus=SUCCEEDED,memorySeconds=1330655,vcoreSeconds=704,preemptedAMContainers=0,preemptedNonAMContainers=0,preemptedResources=<memory:0\, vCores:0>,applicationType=MAPREDUCE" +e,06/20/2017 22:30:21,yarn resourcemanager,DefaultTag,Pid="5156" Tid="9896" TS="0x01D2EA14CDBCF8F1" String1="17/06/20 22:30:21 INFO capacity.CapacityScheduler: Application Attempt appattempt_1497832133857_0194_000001 is done. finalState=FAILED" http://git-wip-us.apache.org/repos/asf/hadoop/blob/9897538a/hadoop-tools/hadoop-tools-dist/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml index 61fdddb..7d57843 100644 --- a/hadoop-tools/hadoop-tools-dist/pom.xml +++ b/hadoop-tools/hadoop-tools-dist/pom.xml @@ -112,6 +112,12 @@ <scope>compile</scope> </dependency> <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-resourceestimator</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-azure-datalake</artifactId> <scope>compile</scope> http://git-wip-us.apache.org/repos/asf/hadoop/blob/9897538a/hadoop-tools/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml index 86d573a..db67639 100644 --- a/hadoop-tools/pom.xml +++ b/hadoop-tools/pom.xml @@ -44,6 +44,7 @@ <module>hadoop-pipes</module> <module>hadoop-openstack</module> <module>hadoop-sls</module> + <module>hadoop-resourceestimator</module> <module>hadoop-aws</module> <module>hadoop-azure</module> <module>hadoop-azure-datalake</module> --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
