Repository: hive Updated Branches: refs/heads/HIVE-19429 5b694b7be -> a2b2e125e
first commit dummy Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9765a43d Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9765a43d Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9765a43d Branch: refs/heads/HIVE-19429 Commit: 9765a43d0caa72c2a52eebb4b7d48a7b17034282 Parents: 5b694b7 Author: Vihang Karajgaonkar <vih...@cloudera.com> Authored: Sat Jun 2 09:21:46 2018 -0700 Committer: Vihang Karajgaonkar <vih...@cloudera.com> Committed: Mon Jun 4 11:33:12 2018 -0700 ---------------------------------------------------------------------- .../execution/DockerBasedExecutionPhase.java | 119 +++++++++++++++++++ .../ptest/execution/containers/DockerPTest.java | 40 +++++++ .../ptest/api/server/TestTestStartRequest.java | 40 +++++++ 3 files changed, 199 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/9765a43d/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/DockerBasedExecutionPhase.java ---------------------------------------------------------------------- diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/DockerBasedExecutionPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/DockerBasedExecutionPhase.java new file mode 100644 index 0000000..4b19ccc --- /dev/null +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/DockerBasedExecutionPhase.java @@ -0,0 +1,119 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.ptest.execution; + +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableMap; +import org.apache.hive.ptest.execution.conf.TestBatch; +import org.apache.hive.ptest.execution.context.ExecutionContext; +import org.slf4j.Logger; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +public class DockerBasedExecutionPhase extends ExecutionPhase { + + public DockerBasedExecutionPhase(List<HostExecutor> hostExecutors, + ExecutionContext executionContext, HostExecutorBuilder hostExecutorBuilder, + LocalCommandFactory localCommandFactory, ImmutableMap<String, String> templateDefaults, + File succeededLogDir, File failedLogDir, Supplier<List<TestBatch>> testBatchSupplier, + Set<String> executedTests, Set<String> failedTests, Logger logger) throws IOException { + super(hostExecutors, executionContext, hostExecutorBuilder, localCommandFactory, + templateDefaults, succeededLogDir, failedLogDir, testBatchSupplier, executedTests, + failedTests, logger); + } + + @Override + public void execute() throws Throwable { + long start = System.currentTimeMillis(); + /*List<TestBatch> testBatches = Lists.newArrayList(); + for(TestBatch batch : testBatchSupplier.get()) { + testBatches.add(batch); + if(batch.isParallel()) { + parallelWorkQueue.add(batch); + } else { + isolatedWorkQueue.add(batch); + } + } + logger.info("ParallelWorkQueueSize={}, IsolatedWorkQueueSize={}", parallelWorkQueue.size(), + isolatedWorkQueue.size()); + if (logger.isDebugEnabled()) { + for (TestBatch testBatch : parallelWorkQueue) { + logger.debug("PBatch: {}", testBatch); + } + for (TestBatch testBatch : isolatedWorkQueue) { + logger.debug("IBatch: {}", testBatch); + } + }*/ + try { + int expectedNumHosts = hostExecutors.size(); + initalizeHosts(); + resetPerfMetrics(); + /*do { + //replaceBadHosts(expectedNumHosts); + List<ListenableFuture<Void>> results = Lists.newArrayList(); + for(HostExecutor hostExecutor : ImmutableList.copyOf(hostExecutors)) { + results.add(hostExecutor.submitTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults)); + } + Futures.allAsList(results).get(); + } while(!(parallelWorkQueue.isEmpty() && isolatedWorkQueue.isEmpty())); + for(TestBatch batch : testBatches) { + File batchLogDir; + if(failedTestResults.contains(batch)) { + batchLogDir = new File(failedLogDir, batch.getName()); + } else { + batchLogDir = new File(succeededLogDir, batch.getName()); + } + JUnitReportParser parser = new JUnitReportParser(logger, batchLogDir); + executedTests.addAll(parser.getAllExecutedTests()); + for (String failedTest : parser.getAllFailedTests()) { + failedTests.add(failedTest + " (batchId=" + batch.getBatchId() + ")"); + } + + // if the TEST*.xml was not generated or was corrupt, let someone know + if (parser.getTestClassesWithReportAvailable().size() < batch.getTestClasses().size()) { + Set<String> expTestClasses = new HashSet<>(batch.getTestClasses()); + expTestClasses.removeAll(parser.getTestClassesWithReportAvailable()); + for (String testClass : expTestClasses) { + StringBuilder messageBuilder = new StringBuilder(); + messageBuilder.append(testClass).append(" - did not produce a TEST-*.xml file (likely timed out)") + .append(" (batchId=").append(batch.getBatchId()).append(")"); + if (batch instanceof QFileTestBatch) { + Collection<String> tests = ((QFileTestBatch)batch).getTests(); + if (tests.size() != 0) { + messageBuilder.append("\n\t["); + messageBuilder.append(Joiner.on(",").join(tests)); + messageBuilder.append("]"); + } + } + failedTests.add(messageBuilder.toString()); + } + } + }*/ + } finally { + long elapsed = System.currentTimeMillis() - start; + //addAggregatePerfMetrics(); + logger.info("PERF: exec phase " + TimeUnit.MINUTES.convert(elapsed, TimeUnit.MILLISECONDS) + " minutes"); + } + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/9765a43d/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPTest.java ---------------------------------------------------------------------- diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPTest.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPTest.java new file mode 100644 index 0000000..81867cc --- /dev/null +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPTest.java @@ -0,0 +1,40 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.ptest.execution.containers; + +import org.apache.hive.ptest.execution.LocalCommandFactory; +import org.apache.hive.ptest.execution.PTest; +import org.apache.hive.ptest.execution.conf.TestConfiguration; +import org.apache.hive.ptest.execution.context.ExecutionContext; +import org.apache.hive.ptest.execution.ssh.RSyncCommandExecutor; +import org.apache.hive.ptest.execution.ssh.SSHCommandExecutor; +import org.slf4j.Logger; + +import java.io.File; + +public class DockerPTest extends PTest { + public DockerPTest(TestConfiguration configuration, ExecutionContext executionContext, + String buildTag, File logDir, LocalCommandFactory localCommandFactory, + SSHCommandExecutor sshCommandExecutor, RSyncCommandExecutor rsyncCommandExecutor, + Logger logger) throws Exception { + super(configuration, executionContext, buildTag, logDir, localCommandFactory, + sshCommandExecutor, rsyncCommandExecutor, logger); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/9765a43d/testutils/ptest2/src/test/java/org/apache/hive/ptest/api/server/TestTestStartRequest.java ---------------------------------------------------------------------- diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/api/server/TestTestStartRequest.java b/testutils/ptest2/src/test/java/org/apache/hive/ptest/api/server/TestTestStartRequest.java new file mode 100644 index 0000000..d5282c9 --- /dev/null +++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/api/server/TestTestStartRequest.java @@ -0,0 +1,40 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.ptest.api.server; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.hive.ptest.api.request.TestStartRequest; +import org.junit.Test; + +public class TestTestStartRequest { + ObjectMapper mMapper = new ObjectMapper(); + @Test + public void testJson() throws JsonProcessingException { + String profile = "ptest"; + String testHandle = "Dummy-Precommit-Test-1"; + String jira = "HIVE-19425"; + String patch = "https://issues.apache.org/jira/secure/attachment/12923111/HIVE-19429.01-ptest.patch";; + boolean clearLibraryCache = false; + TestStartRequest startRequest = new TestStartRequest(profile, testHandle, jira, patch, clearLibraryCache); + String payloadString = mMapper.writeValueAsString(startRequest); + System.out.println(payloadString); + } +}
