Docker prep phase working
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/519b62e3 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/519b62e3 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/519b62e3 Branch: refs/heads/HIVE-19429 Commit: 519b62e3e28f26d8eb52c7dba486f8999f06b443 Parents: 5787e77 Author: Vihang Karajgaonkar <vih...@cloudera.com> Authored: Thu Jun 21 23:21:43 2018 -0700 Committer: Vihang Karajgaonkar <vih...@cloudera.com> Committed: Thu Jun 21 23:21:43 2018 -0700 ---------------------------------------------------------------------- .../execution/containers/DockerClient.java | 2 ++ .../execution/containers/DockerPrepPhase.java | 4 +-- .../src/main/resources/dockerfile-template.vm | 20 ++++++----- .../ptest/execution/TestPtestOnDockers.java | 34 +++++++++--------- .../ptest2/src/test/resources/DUMMY-001.patch | 38 ++++++++++++++------ 5 files changed, 59 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/519b62e3/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java ---------------------------------------------------------------------- diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java index 8e6e0cf..da54348 100644 --- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerClient.java @@ -64,6 +64,8 @@ public class DockerClient implements ContainerClient { logger.info("Building image"); String dockerBuildCommand = new StringBuilder("docker build") + .append(" --memory " + "2g") + .append(" --memory-swap " + "-1") .append(" --tag " + imageName()) .append(" --build-arg ") .append(" workingDir=$workingDir") http://git-wip-us.apache.org/repos/asf/hive/blob/519b62e3/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java ---------------------------------------------------------------------- diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java index f5455b5..80c1aef 100644 --- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/containers/DockerPrepPhase.java @@ -58,11 +58,11 @@ public class DockerPrepPhase extends PrepPhase { execLocally(getDockerBuildCommand()); elapsedTime = TimeUnit.MINUTES.convert((System.currentTimeMillis() - start), TimeUnit.MILLISECONDS); - logger.info("PERF: Docker source prep took " + elapsedTime + " minutes"); + logger.info("PERF: Docker build image took " + elapsedTime + " minutes"); } @VisibleForTesting - String getLocalScratchDir() { + public String getLocalScratchDir() { return mScratchDir.getAbsolutePath(); } http://git-wip-us.apache.org/repos/asf/hive/blob/519b62e3/testutils/ptest2/src/main/resources/dockerfile-template.vm ---------------------------------------------------------------------- diff --git a/testutils/ptest2/src/main/resources/dockerfile-template.vm b/testutils/ptest2/src/main/resources/dockerfile-template.vm index fe421f3..4f96ea7 100644 --- a/testutils/ptest2/src/main/resources/dockerfile-template.vm +++ b/testutils/ptest2/src/main/resources/dockerfile-template.vm @@ -35,16 +35,18 @@ ARG repository ARG branch ARG buildTag -RUN export MAVEN_OPTS="$mavenEnvOpts" +ENV MAVEN_OPTS="$mavenEnvOpts" +RUN /usr/bin/mvn -version +RUN echo $MAVEN_OPTS RUN /usr/bin/git clone $repository RUN cd hive WORKDIR /home/ptestuser/hive -RUN /usr/bin/git checkout $branch || checkout -b $branch origin/$branch -RUN /usr/bin/git reset --hard origin/$branch -RUN /usr/bin/git merge --ff-only origin/$branch -RUN /home/ptestuser/scratch/smart-apply-patch.sh /home/ptestuser/scratch/build.patch -RUN /usr/bin/mvn -B -T 4 -q install -Dtest=TestMetastoreConf -RUN cd itests -RUN /usr/bin/mvn -B -T 4 -q install -DskipSparkTests -DskipTests -RUN echo This build is labeled $buildTag +RUN /usr/bin/git checkout $branch || checkout -b $branch origin/$branch &&\ + /usr/bin/git reset --hard origin/$branch &&\ + /usr/bin/git merge --ff-only origin/$branch &&\ + /home/ptestuser/scratch/smart-apply-patch.sh /home/ptestuser/scratch/build.patch &&\ + /usr/bin/mvn -q -B install -Dtest=TestMetastoreConf &&\ + cd itests &&\ + /usr/bin/mvn -q -B install -DskipSparkTests -DskipTests &&\ + echo This build is labeled $buildTag http://git-wip-us.apache.org/repos/asf/hive/blob/519b62e3/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java ---------------------------------------------------------------------- diff --git a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java index 1109a8a..76716c4 100644 --- a/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java +++ b/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestPtestOnDockers.java @@ -23,9 +23,10 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; -import org.apache.commons.io.FileUtils; import org.apache.hive.ptest.execution.LocalCommand.CollectLogPolicy; import org.apache.hive.ptest.execution.conf.Host; +import org.apache.hive.ptest.execution.conf.TestBatch; +import org.apache.hive.ptest.execution.containers.DockerExecutionPhase; import org.apache.hive.ptest.execution.containers.DockerPrepPhase; import org.apache.hive.ptest.execution.containers.TestDockerPrepPhase; import org.apache.hive.ptest.execution.context.ExecutionContext; @@ -51,7 +52,8 @@ import static org.mockito.Mockito.spy; public class TestPtestOnDockers { //TODO add logic to ignore this test if docker is not found on the machine - private DockerPrepPhase phase; + private DockerPrepPhase prepPhase; + private DockerExecutionPhase execPhase; private static File dummyPatchFile; private static final Logger logger = LoggerFactory.getLogger(TestDockerPrepPhase.class); @@ -79,7 +81,7 @@ public class TestPtestOnDockers { private static final String REAL_BRANCH = "master"; private static final String REAL_REPOSITORY = "https://github.com/apache/hive.git";; private static final String REAL_REPOSITORY_NAME = "apache-hive"; - private static final String REAL_MAVEN_OPTS = "-Xmx1g"; + private static final String REAL_MAVEN_OPTS = "-Xmx2048m"; private MockSSHCommandExecutor sshCommandExecutor; private MockRSyncCommandExecutor rsyncCommandExecutor; private static final String BUILD_TAG = "docker-ptest-tag"; @@ -124,8 +126,13 @@ public class TestPtestOnDockers { public void setup() throws Exception { initialize(getClass().getSimpleName()); createHostExecutor(); - phase = new DockerPrepPhase(hostExecutors, localCommandFactory, + prepPhase = new DockerPrepPhase(hostExecutors, localCommandFactory, templateDefaults, baseDir, dummyPatchFile, logger); + /*execPhase = new DockerExecutionPhase(hostExecutors, executionContext, + hostExecutorBuilder, localCommandFactory, + templateDefaults, succeededLogDir, failedLogDir, + testBatchSupplier, executedTests, + failedTests, logger);*/ } private void createHostExecutor() { @@ -137,7 +144,7 @@ public class TestPtestOnDockers { @After public void teardown() { - phase = null; + prepPhase = null; //FileUtils.deleteQuietly(baseDir); } @@ -147,18 +154,9 @@ public class TestPtestOnDockers { */ @Test public void testDockerFile() throws Exception { - phase.execute(); - File dockerFile = new File(scratchDir, "Dockerfile"); - Assert.assertTrue("Docker file was not found" , dockerFile.exists()); - Assert.assertTrue("patch file not found", new File(scratchDir, "build.patch").exists()); - LocalCommandFactory localCommandFactory = new LocalCommandFactory(logger); - CollectLogPolicy localCollector = new CollectLogPolicy(logger); - localCommandFactory.create(localCollector, "cp -f " + dockerFile.getAbsolutePath() + " /tmp/myDockerFile"); - LocalCommand localCmd = localCommandFactory.create(localCollector, phase.getDockerBuildCommand()); - if(localCmd.getExitCode() != 0) { - throw new NonZeroExitCodeException(String.format( - "Command '%s' failed with exit status %d and output '%s'", - localCmd, localCmd.getExitCode(), localCollector.getOutput())); - } + prepPhase.execute(); + Assert.assertNotNull("Scratch directory needs to be set", prepPhase.getLocalScratchDir()); + File dockerFile = new File(prepPhase.getLocalScratchDir(), "Dockerfile"); + Assert.assertTrue("Docker file not found", dockerFile.exists()); } } http://git-wip-us.apache.org/repos/asf/hive/blob/519b62e3/testutils/ptest2/src/test/resources/DUMMY-001.patch ---------------------------------------------------------------------- diff --git a/testutils/ptest2/src/test/resources/DUMMY-001.patch b/testutils/ptest2/src/test/resources/DUMMY-001.patch index 6277ca9..97d7feb 100644 --- a/testutils/ptest2/src/test/resources/DUMMY-001.patch +++ b/testutils/ptest2/src/test/resources/DUMMY-001.patch @@ -1,12 +1,30 @@ -diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java -index 2868ff0..6a6a5ff 100644 ---- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java -+++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java -@@ -69,6 +69,7 @@ +diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java +index 1d7eceb1ef380fdb6f62d8b831b1e2bd23333c67..b78e7b2bef78b99a3cab9cc89bd8551649285417 100644 +--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java ++++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java +@@ -68,9 +68,7 @@ - public class PTest { + import com.google.common.annotations.VisibleForTesting; + import com.google.protobuf.CodedInputStream; +- +-import sun.misc.Cleaner; +- ++//import sun.misc.Cleaner; -+ // test patching - static { - Velocity.init(); - } + /** + * Encoded reader implementation. +@@ -1654,12 +1652,12 @@ private void releaseBuffer(ByteBuffer bb, boolean isFromDataReader) { + Field localCf = cleanerField; + if (!bb.isDirect() || localCf == null) return; + try { +- Cleaner cleaner = (Cleaner) localCf.get(bb); ++ /*Cleaner cleaner = (Cleaner) localCf.get(bb); + if (cleaner != null) { + cleaner.clean(); + } else { + LOG.debug("Unable to clean a buffer using cleaner - no cleaner"); +- } ++ }*/ + } catch (Exception e) { + // leave it for GC to clean up + LOG.warn("Unable to clean direct buffers using Cleaner.");
