This is an automated email from the ASF dual-hosted git repository. arp pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push: new 18fa439 MAPREDUCE-7298. Distcp doesn't close the job after the job is completed. Contributed by Aasha Medhi. 18fa439 is described below commit 18fa4397e6dc7663bcc7c7309126f45eb8d3fa17 Author: Arpit Agarwal <aagar...@cloudera.com> AuthorDate: Fri Oct 2 08:29:55 2020 -0700 MAPREDUCE-7298. Distcp doesn't close the job after the job is completed. Contributed by Aasha Medhi. Change-Id: I63d249bbb18ccedaeee9f10123a78e32f9e54ed2 --- .../main/java/org/apache/hadoop/tools/DistCp.java | 15 +++++++++-- .../org/apache/hadoop/tools/TestExternalCall.java | 31 ++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java index c36335a..6f8ab2b 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java @@ -127,6 +127,7 @@ public class DistCp extends Configured implements Tool { * to target location, by: * 1. Creating a list of files to be copied to target. * 2. Launching a Map-only job to copy the files. (Delegates to execute().) + * The MR job is not closed as part of run if its a blocking call to run * @param argv List of arguments passed to DistCp, from the ToolRunner. * @return On success, it returns 0. Else, -1. */ @@ -148,9 +149,10 @@ public class DistCp extends Configured implements Tool { OptionsParser.usage(); return DistCpConstants.INVALID_ARGUMENT; } - + + Job job = null; try { - execute(); + job = execute(); } catch (InvalidInputException e) { LOG.error("Invalid input: ", e); return DistCpConstants.INVALID_ARGUMENT; @@ -166,6 +168,15 @@ public class DistCp extends Configured implements Tool { } catch (Exception e) { LOG.error("Exception encountered ", e); return DistCpConstants.UNKNOWN_ERROR; + } finally { + //Blocking distcp so close the job after its done + if (job != null && context.shouldBlock()) { + try { + job.close(); + } catch (IOException e) { + LOG.error("Exception encountered while closing distcp job", e); + } + } } return DistCpConstants.SUCCESS; } diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestExternalCall.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestExternalCall.java index 06122e6..eba86a9 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestExternalCall.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestExternalCall.java @@ -18,6 +18,8 @@ package org.apache.hadoop.tools; +import org.apache.hadoop.mapreduce.Job; +import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -35,6 +37,8 @@ import java.io.IOException; import java.io.OutputStream; import java.security.Permission; +import static org.mockito.Mockito.*; + public class TestExternalCall { private static final Logger LOG = LoggerFactory.getLogger(TestExternalCall.class); @@ -134,6 +138,33 @@ public class TestExternalCall { } + /** + * test methods run end execute of DistCp class. distcp job should be cleaned up after completion + * @throws Exception + */ + @Test + public void testCleanupOfJob() throws Exception { + + Configuration conf = getConf(); + + Path stagingDir = JobSubmissionFiles.getStagingDir(new Cluster(conf), + conf); + stagingDir.getFileSystem(conf).mkdirs(stagingDir); + Path soure = createFile("tmp.txt"); + Path target = createFile("target.txt"); + + DistCp distcp = mock(DistCp.class); + Job job = spy(Job.class); + Mockito.when(distcp.getConf()).thenReturn(conf); + Mockito.when(distcp.execute()).thenReturn(job); + Mockito.when(distcp.run(Mockito.any())).thenCallRealMethod(); + String[] arg = { soure.toString(), target.toString() }; + + distcp.run(arg); + Mockito.verify(job, times(1)).close(); + } + + private SecurityManager securityManager; protected static class ExitException extends SecurityException { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org