Repository: hadoop Updated Branches: refs/heads/branch-2 e1d3b8e71 -> 5fa241daa
YARN-4491. yarn list command to support filtering by tags. Contributed by Varun Saxena (cherry picked from commit 143c59e4c5a811eb2c12cf6626d558f9b8796e03) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5fa241da Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5fa241da Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5fa241da Branch: refs/heads/branch-2 Commit: 5fa241daa055571ca331d09005f6ddf8c1b514c5 Parents: e1d3b8e Author: Naganarasimha <naganarasimha...@apache.org> Authored: Wed Aug 24 01:53:02 2016 +0530 Committer: Naganarasimha <naganarasimha...@apache.org> Committed: Wed Aug 24 01:55:24 2016 +0530 ---------------------------------------------------------------------- .../hadoop/mapred/ResourceMgrDelegate.java | 10 + .../hadoop/yarn/client/api/YarnClient.java | 25 ++ .../yarn/client/api/impl/YarnClientImpl.java | 11 + .../hadoop/yarn/client/cli/ApplicationCLI.java | 42 +++- .../hadoop/yarn/client/cli/TestYarnCLI.java | 241 ++++++++++++++++--- 5 files changed, 282 insertions(+), 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fa241da/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index cc164fd..159b518 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -349,6 +349,16 @@ public class ResourceMgrDelegate extends YarnClient { } @Override + public List<ApplicationReport> getApplications( + Set<String> applicationTypes, + EnumSet<YarnApplicationState> applicationStates, + Set<String> applicationTags) + throws YarnException, IOException { + return client.getApplications( + applicationTypes, applicationStates, applicationTags); + } + + @Override public List<ApplicationReport> getApplications(Set<String> queues, Set<String> users, Set<String> applicationTypes, EnumSet<YarnApplicationState> applicationStates) throws YarnException, http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fa241da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index 218bb34..619ea0d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -314,6 +314,31 @@ public abstract class YarnClient extends AbstractService { /** * <p> + * Get a report (ApplicationReport) of Applications matching the given + * application types, application states and application tags in the cluster. + * </p> + * + * <p> + * If the user does not have <code>VIEW_APP</code> access for an application + * then the corresponding report will be filtered as described in + * {@link #getApplicationReport(ApplicationId)}. + * </p> + * + * @param applicationTypes set of application types you are interested in + * @param applicationStates set of application states you are interested in + * @param applicationTags set of application tags you are interested in + * @return a list of reports of applications + * @throws YarnException + * @throws IOException + */ + public abstract List<ApplicationReport> getApplications( + Set<String> applicationTypes, + EnumSet<YarnApplicationState> applicationStates, + Set<String> applicationTags) throws YarnException, + IOException; + + /** + * <p> * Get a report (ApplicationReport) of Applications matching the given users, * queues, application types and application states in the cluster. If any of * the params is set to null, it is not used when filtering. http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fa241da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index dee31c5..b91fde0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -505,6 +505,17 @@ public class YarnClientImpl extends YarnClient { } @Override + public List<ApplicationReport> getApplications(Set<String> applicationTypes, + EnumSet<YarnApplicationState> applicationStates, + Set<String> applicationTags) throws YarnException, IOException { + GetApplicationsRequest request = + GetApplicationsRequest.newInstance(applicationTypes, applicationStates); + request.setApplicationTags(applicationTags); + GetApplicationsResponse response = rmClient.getApplications(request); + return response.getApplicationList(); + } + + @Override public List<ApplicationReport> getApplications(Set<String> queues, Set<String> users, Set<String> applicationTypes, EnumSet<YarnApplicationState> applicationStates) throws YarnException, http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fa241da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 865ce00..f0b1c47 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -73,6 +73,7 @@ public class ApplicationCLI extends YarnCLI { private static final String APP_TYPE_CMD = "appTypes"; private static final String APP_STATE_CMD = "appStates"; + private static final String APP_TAG_CMD = "appTags"; private static final String ALLSTATES_OPTION = "ALL"; private static final String QUEUE_CMD = "queue"; public static final String APPLICATION = "application"; @@ -102,8 +103,9 @@ public class ApplicationCLI extends YarnCLI { "Prints the status of the application."); opts.addOption(LIST_CMD, false, "List applications. " + "Supports optional use of -appTypes to filter applications " - + "based on application type, " - + "and -appStates to filter applications based on application state."); + + "based on application type, -appStates to filter applications " + + "based on application state and -appTags to filter applications " + + "based on application tag."); opts.addOption(MOVE_TO_QUEUE_CMD, true, "Moves the application to a " + "different queue."); opts.addOption(QUEUE_CMD, true, "Works with the movetoqueue command to" @@ -123,6 +125,13 @@ public class ApplicationCLI extends YarnCLI { appStateOpt.setArgs(Option.UNLIMITED_VALUES); appStateOpt.setArgName("States"); opts.addOption(appStateOpt); + Option appTagOpt = new Option(APP_TAG_CMD, true, "Works with -list to " + + "filter applications based on input comma-separated list of " + + "application tags."); + appTagOpt.setValueSeparator(','); + appTagOpt.setArgs(Option.UNLIMITED_VALUES); + appTagOpt.setArgName("Tags"); + opts.addOption(appTagOpt); opts.addOption(APP_ID, true, "Specify Application Id to be operated"); opts.addOption(UPDATE_PRIORITY, true, "update priority of an application. ApplicationId can be" @@ -229,7 +238,19 @@ public class ApplicationCLI extends YarnCLI { } } } - listApplications(appTypes, appStates); + + Set<String> appTags = new HashSet<String>(); + if (cliParser.hasOption(APP_TAG_CMD)) { + String[] tags = cliParser.getOptionValues(APP_TAG_CMD); + if (tags != null) { + for (String tag : tags) { + if (!tag.trim().isEmpty()) { + appTags.add(tag.trim()); + } + } + } + } + listApplications(appTypes, appStates, appTags); } else if (args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) { if (args.length != 3) { printUsage(title, opts); @@ -434,17 +455,18 @@ public class ApplicationCLI extends YarnCLI { } /** - * Lists the applications matching the given application Types And application - * States present in the Resource Manager + * Lists the applications matching the given application Types, application + * States and application Tags present in the Resource Manager. * * @param appTypes * @param appStates + * @param appTags * @throws YarnException * @throws IOException */ private void listApplications(Set<String> appTypes, - EnumSet<YarnApplicationState> appStates) throws YarnException, - IOException { + EnumSet<YarnApplicationState> appStates, Set<String> appTags) + throws YarnException, IOException { PrintWriter writer = new PrintWriter( new OutputStreamWriter(sysout, Charset.forName("UTF-8"))); if (allAppStates) { @@ -460,11 +482,11 @@ public class ApplicationCLI extends YarnCLI { } List<ApplicationReport> appsReport = client.getApplications(appTypes, - appStates); + appStates, appTags); writer.println("Total number of applications (application-types: " - + appTypes + " and states: " + appStates + ")" + ":" - + appsReport.size()); + + appTypes + ", states: " + appStates + " and tags: " + appTags + ")" + + ":" + appsReport.size()); writer.printf(APPLICATIONS_PATTERN, "Application-Id", "Application-Name", "Application-Type", "User", "Queue", "State", "Final-State", "Progress", "Tracking-URL"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fa241da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 6d7aaa7..1cd513b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -87,6 +87,7 @@ import org.junit.Test; import org.mortbay.log.Log; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; public class TestYarnCLI { @@ -373,8 +374,10 @@ public class TestYarnCLI { applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, - FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); - List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>(); + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, + Sets.newHashSet("tag1", "tag3"), false, Priority.UNDEFINED, "", ""); + List<ApplicationReport> applicationReports = + new ArrayList<ApplicationReport>(); applicationReports.add(newApplicationReport); ApplicationId applicationId2 = ApplicationId.newInstance(1234, 6); @@ -383,7 +386,8 @@ public class TestYarnCLI { "user2", "queue2", "appname2", "host2", 125, null, YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", - null); + null, Sets.newHashSet("tag2", "tag3"), false, Priority.UNDEFINED, + "", ""); applicationReports.add(newApplicationReport2); ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7); @@ -392,7 +396,8 @@ public class TestYarnCLI { "user3", "queue3", "appname3", "host3", 126, null, YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", - null); + null, Sets.newHashSet("tag1", "tag4"), false, Priority.UNDEFINED, + "", ""); applicationReports.add(newApplicationReport3); ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8); @@ -400,8 +405,9 @@ public class TestYarnCLI { applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4), "user4", "queue4", "appname4", "host4", 127, null, YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, - FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE", - null); + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, + "NON-MAPREDUCE", null, Sets.newHashSet("tag1"), false, + Priority.UNDEFINED, "", ""); applicationReports.add(newApplicationReport4); ApplicationId applicationId5 = ApplicationId.newInstance(1234, 9); @@ -409,8 +415,8 @@ public class TestYarnCLI { applicationId5, ApplicationAttemptId.newInstance(applicationId5, 5), "user5", "queue5", "appname5", "host5", 128, null, YarnApplicationState.ACCEPTED, "diagnostics5", "url5", 5, 5, - FinalApplicationStatus.KILLED, null, "N/A", 0.93789f, "HIVE", - null); + FinalApplicationStatus.KILLED, null, "N/A", 0.93789f, "HIVE", null, + Sets.newHashSet("tag2", "tag4"), false, Priority.UNDEFINED, "", ""); applicationReports.add(newApplicationReport5); ApplicationId applicationId6 = ApplicationId.newInstance(1234, 10); @@ -419,7 +425,7 @@ public class TestYarnCLI { "user6", "queue6", "appname6", "host6", 129, null, YarnApplicationState.SUBMITTED, "diagnostics6", "url6", 6, 6, FinalApplicationStatus.KILLED, null, "N/A", 0.99789f, "PIG", - null); + null, new HashSet<String>(), false, Priority.UNDEFINED, "", ""); applicationReports.add(newApplicationReport6); // Test command yarn application -list @@ -433,16 +439,18 @@ public class TestYarnCLI { appState1.add(YarnApplicationState.RUNNING); appState1.add(YarnApplicationState.ACCEPTED); appState1.add(YarnApplicationState.SUBMITTED); - when(client.getApplications(appType1, appState1)).thenReturn( - getApplicationReports(applicationReports, appType1, appState1, false)); + Set<String> appTag = new HashSet<String>(); + when(client.getApplications(appType1, appState1, appTag)).thenReturn( + getApplicationReports( + applicationReports, appType1, appState1, appTag, false)); int result = cli.run(new String[] { "application", "-list" }); assertEquals(0, result); - verify(client).getApplications(appType1, appState1); + verify(client).getApplications(appType1, appState1, appTag); ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter pw = new PrintWriter(baos); pw.println("Total number of applications (application-types: " + appType1 - + " and states: " + appState1 + ")" + ":" + 4); + + ", states: " + appState1 + " and tags: " + appTag + ")" + ":" + 4); pw.print(" Application-Id\t Application-Name"); pw.print("\t Application-Type"); pw.print("\t User\t Queue\t State\t "); @@ -487,17 +495,18 @@ public class TestYarnCLI { appState2.add(YarnApplicationState.RUNNING); appState2.add(YarnApplicationState.ACCEPTED); appState2.add(YarnApplicationState.SUBMITTED); - when(client.getApplications(appType2, appState2)).thenReturn( - getApplicationReports(applicationReports, appType2, appState2, false)); + when(client.getApplications(appType2, appState2, appTag)).thenReturn( + getApplicationReports( + applicationReports, appType2, appState2, appTag, false)); result = cli.run(new String[] { "application", "-list", "-appTypes", "YARN, ,, NON-YARN", " ,, ,," }); assertEquals(0, result); - verify(client).getApplications(appType2, appState2); + verify(client).getApplications(appType2, appState2, appTag); baos = new ByteArrayOutputStream(); pw = new PrintWriter(baos); pw.println("Total number of applications (application-types: " + appType2 - + " and states: " + appState2 + ")" + ":" + 1); + + ", states: " + appState2 + " and tags: " + appTag + ")" + ":" + 1); pw.print(" Application-Id\t Application-Name"); pw.print("\t Application-Type"); pw.print("\t User\t Queue\t State\t "); @@ -522,17 +531,18 @@ public class TestYarnCLI { appState3.add(YarnApplicationState.FINISHED); appState3.add(YarnApplicationState.FAILED); - when(client.getApplications(appType3, appState3)).thenReturn( - getApplicationReports(applicationReports, appType3, appState3, false)); + when(client.getApplications(appType3, appState3, appTag)).thenReturn( + getApplicationReports( + applicationReports, appType3, appState3, appTag, false)); result = cli.run(new String[] { "application", "-list", "--appStates", "FINISHED ,, , FAILED", ",,FINISHED" }); assertEquals(0, result); - verify(client).getApplications(appType3, appState3); + verify(client).getApplications(appType3, appState3, appTag); baos = new ByteArrayOutputStream(); pw = new PrintWriter(baos); pw.println("Total number of applications (application-types: " + appType3 - + " and states: " + appState3 + ")" + ":" + 2); + + ", states: " + appState3 + " and tags: " + appTag + ")" + ":" + 2); pw.print(" Application-Id\t Application-Name"); pw.print("\t Application-Type"); pw.print("\t User\t Queue\t State\t "); @@ -565,17 +575,18 @@ public class TestYarnCLI { appState4.add(YarnApplicationState.FINISHED); appState4.add(YarnApplicationState.FAILED); - when(client.getApplications(appType4, appState4)).thenReturn( - getApplicationReports(applicationReports, appType4, appState4, false)); + when(client.getApplications(appType4, appState4, appTag)).thenReturn( + getApplicationReports( + applicationReports, appType4, appState4, appTag, false)); result = cli.run(new String[] { "application", "-list", "--appTypes", "YARN,NON-YARN", "--appStates", "FINISHED ,, , FAILED" }); assertEquals(0, result); - verify(client).getApplications(appType2, appState2); + verify(client).getApplications(appType2, appState2, appTag); baos = new ByteArrayOutputStream(); pw = new PrintWriter(baos); pw.println("Total number of applications (application-types: " + appType4 - + " and states: " + appState4 + ")" + ":" + 1); + + ", states: " + appState4 + " and tags: " + appTag + ")" + ":" + 1); pw.print(" Application-Id\t Application-Name"); pw.print("\t Application-Type"); pw.print("\t User\t Queue\t State\t "); @@ -620,17 +631,18 @@ public class TestYarnCLI { EnumSet<YarnApplicationState> appState5 = EnumSet.noneOf(YarnApplicationState.class); appState5.add(YarnApplicationState.FINISHED); - when(client.getApplications(appType5, appState5)).thenReturn( - getApplicationReports(applicationReports, appType5, appState5, true)); + when(client.getApplications(appType5, appState5, appTag)).thenReturn( + getApplicationReports( + applicationReports, appType5, appState5, appTag, true)); result = cli.run(new String[] { "application", "-list", "--appStates", "FINISHED ,, , ALL" }); assertEquals(0, result); - verify(client).getApplications(appType5, appState5); + verify(client).getApplications(appType5, appState5, appTag); baos = new ByteArrayOutputStream(); pw = new PrintWriter(baos); pw.println("Total number of applications (application-types: " + appType5 - + " and states: " + appState5 + ")" + ":" + 6); + + ", states: " + appState5 + " and tags: " + appTag + ")" + ":" + 6); pw.print(" Application-Id\t Application-Name"); pw.print("\t Application-Type"); pw.print("\t User\t Queue\t State\t "); @@ -680,17 +692,18 @@ public class TestYarnCLI { EnumSet<YarnApplicationState> appState6 = EnumSet.noneOf(YarnApplicationState.class); appState6.add(YarnApplicationState.FINISHED); - when(client.getApplications(appType6, appState6)).thenReturn( - getApplicationReports(applicationReports, appType6, appState6, false)); + when(client.getApplications(appType6, appState6, appTag)).thenReturn( + getApplicationReports( + applicationReports, appType6, appState6, appTag, false)); result = cli.run(new String[] { "application", "-list", "-appTypes", "YARN, ,, NON-YARN", "--appStates", "finished" }); assertEquals(0, result); - verify(client).getApplications(appType6, appState6); + verify(client).getApplications(appType6, appState6, appTag); baos = new ByteArrayOutputStream(); pw = new PrintWriter(baos); pw.println("Total number of applications (application-types: " + appType6 - + " and states: " + appState6 + ")" + ":" + 1); + + ", states: " + appState6 + " and tags: " + appTag + ")" + ":" + 1); pw.print(" Application-Id\t Application-Name"); pw.print("\t Application-Type"); pw.print("\t User\t Queue\t State\t "); @@ -705,12 +718,146 @@ public class TestYarnCLI { appsReportStr = baos.toString("UTF-8"); Assert.assertEquals(appsReportStr, sysOutStream.toString()); verify(sysOut, times(6)).write(any(byte[].class), anyInt(), anyInt()); + + // Test command yarn application with tags. + sysOutStream.reset(); + Set<String> appTag1 = Sets.newHashSet("tag1"); + when(client.getApplications(appType1, appState1, appTag1)).thenReturn( + getApplicationReports( + applicationReports, appType1, appState1, appTag1, false)); + result = + cli.run(new String[] { "application", "-list", "-appTags", "tag1" }); + assertEquals(0, result); + verify(client).getApplications(appType1, appState1, appTag1); + baos = new ByteArrayOutputStream(); + pw = new PrintWriter(baos); + pw.println("Total number of applications (application-types: " + appType1 + + ", states: " + appState1 + " and tags: " + appTag1 + ")" + ":" + 2); + pw.print(" Application-Id\t Application-Name"); + pw.print("\t Application-Type"); + pw.print("\t User\t Queue\t State\t "); + pw.print("Final-State\t Progress"); + pw.println("\t Tracking-URL"); + pw.print(" application_1234_0005\t "); + pw.print("appname\t YARN\t user\t "); + pw.print("queue\t RUNNING\t "); + pw.print("SUCCEEDED\t 53.79%"); + pw.println("\t N/A"); + pw.print(" application_1234_0007\t "); + pw.print("appname3\t MAPREDUCE\t user3\t "); + pw.print("queue3\t RUNNING\t "); + pw.print("SUCCEEDED\t 73.79%"); + pw.println("\t N/A"); + pw.close(); + appsReportStr = baos.toString("UTF-8"); + Assert.assertEquals(appsReportStr, sysOutStream.toString()); + verify(sysOut, times(7)).write(any(byte[].class), anyInt(), anyInt()); + + sysOutStream.reset(); + EnumSet<YarnApplicationState> appState7 = + EnumSet.of(YarnApplicationState.RUNNING, YarnApplicationState.FAILED); + when(client.getApplications(appType1, appState7, appTag1)).thenReturn( + getApplicationReports( + applicationReports, appType1, appState7, appTag1, false)); + result = cli.run( + new String[] { "application", "-list", "-appStates", "RUNNING,FAILED", + "-appTags", "tag1" }); + assertEquals(0, result); + verify(client).getApplications(appType1, appState7, appTag1); + baos = new ByteArrayOutputStream(); + pw = new PrintWriter(baos); + pw.println("Total number of applications (application-types: " + appType1 + + ", states: " + appState7 + " and tags: " + appTag1 + ")" + ":" + 3); + pw.print(" Application-Id\t Application-Name"); + pw.print("\t Application-Type"); + pw.print("\t User\t Queue\t State\t "); + pw.print("Final-State\t Progress"); + pw.println("\t Tracking-URL"); + pw.print(" application_1234_0005\t "); + pw.print("appname\t YARN\t user\t "); + pw.print("queue\t RUNNING\t "); + pw.print("SUCCEEDED\t 53.79%"); + pw.println("\t N/A"); + pw.print(" application_1234_0007\t "); + pw.print("appname3\t MAPREDUCE\t user3\t "); + pw.print("queue3\t RUNNING\t "); + pw.print("SUCCEEDED\t 73.79%"); + pw.println("\t N/A"); + pw.print(" application_1234_0008\t "); + pw.print("appname4\t NON-MAPREDUCE\t user4\t "); + pw.print("queue4\t FAILED\t "); + pw.print("SUCCEEDED\t 83.79%"); + pw.println("\t N/A"); + pw.close(); + appsReportStr = baos.toString("UTF-8"); + Assert.assertEquals(appsReportStr, sysOutStream.toString()); + verify(sysOut, times(8)).write(any(byte[].class), anyInt(), anyInt()); + + sysOutStream.reset(); + Set<String> appType9 = Sets.newHashSet("YARN"); + Set<String> appTag2 = Sets.newHashSet("tag3"); + when(client.getApplications(appType9, appState1, appTag2)).thenReturn( + getApplicationReports( + applicationReports, appType9, appState1, appTag2, false)); + result = cli.run(new String[] { "application", "-list", "-appTypes", "YARN", + "-appTags", "tag3" }); + assertEquals(0, result); + verify(client).getApplications(appType9, appState1, appTag2); + baos = new ByteArrayOutputStream(); + pw = new PrintWriter(baos); + pw.println("Total number of applications (application-types: " + appType9 + + ", states: " + appState1 + " and tags: " + appTag2 + ")" + ":" + 1); + pw.print(" Application-Id\t Application-Name"); + pw.print("\t Application-Type"); + pw.print("\t User\t Queue\t State\t "); + pw.print("Final-State\t Progress"); + pw.println("\t Tracking-URL"); + pw.print(" application_1234_0005\t "); + pw.print("appname\t YARN\t user\t "); + pw.print("queue\t RUNNING\t "); + pw.print("SUCCEEDED\t 53.79%"); + pw.println("\t N/A"); + pw.close(); + appsReportStr = baos.toString("UTF-8"); + Assert.assertEquals(appsReportStr, sysOutStream.toString()); + verify(sysOut, times(9)).write(any(byte[].class), anyInt(), anyInt()); + + sysOutStream.reset(); + Set<String> appType10 = Sets.newHashSet("HIVE"); + Set<String> appTag3 = Sets.newHashSet("tag4"); + EnumSet<YarnApplicationState> appState10 = + EnumSet.of(YarnApplicationState.ACCEPTED); + when(client.getApplications(appType10, appState10, appTag3)).thenReturn( + getApplicationReports( + applicationReports, appType10, appState10, appTag3, false)); + result = cli.run(new String[] { "application", "-list", "-appTypes", "HIVE", + "-appStates", "ACCEPTED", "-appTags", "tag4" }); + assertEquals(0, result); + verify(client).getApplications(appType10, appState10, appTag3); + baos = new ByteArrayOutputStream(); + pw = new PrintWriter(baos); + pw.println("Total number of applications (application-types: " + appType10 + + ", states: " + appState10 + " and tags: " + appTag3 + ")" + ":" + 1); + pw.print(" Application-Id\t Application-Name"); + pw.print("\t Application-Type"); + pw.print("\t User\t Queue\t State\t "); + pw.print("Final-State\t Progress"); + pw.println("\t Tracking-URL"); + pw.print(" application_1234_0009\t "); + pw.print("appname5\t HIVE\t user5\t "); + pw.print("queue5\t ACCEPTED\t "); + pw.print("KILLED\t 93.79%"); + pw.println("\t N/A"); + pw.close(); + appsReportStr = baos.toString("UTF-8"); + Assert.assertEquals(appsReportStr, sysOutStream.toString()); + verify(sysOut, times(10)).write(any(byte[].class), anyInt(), anyInt()); } private List<ApplicationReport> getApplicationReports( List<ApplicationReport> applicationReports, Set<String> appTypes, EnumSet<YarnApplicationState> appStates, - boolean allStates) { + Set<String> appTags, boolean allStates) { List<ApplicationReport> appReports = new ArrayList<ApplicationReport>(); @@ -732,6 +879,22 @@ public class TestYarnCLI { } } + if (appTags != null && !appTags.isEmpty()) { + Set<String> tags = appReport.getApplicationTags(); + if (tags == null || tags.isEmpty()) { + continue; + } + boolean match = false; + for (String appTag : appTags) { + if (tags.contains(appTag)) { + match = true; + break; + } + } + if (!match) { + continue; + } + } appReports.add(appReport); } return appReports; @@ -1816,6 +1979,9 @@ public class TestYarnCLI { pw.println(" state can be one of the following:"); pw.println(" ALL,NEW,NEW_SAVING,SUBMITTED,ACCEPTED,RUN"); pw.println(" NING,FINISHED,FAILED,KILLED"); + pw.println(" -appTags <Tags> Works with -list to filter applications"); + pw.println(" based on input comma-separated list of"); + pw.println(" application tags."); pw.println(" -appTypes <Types> Works with -list to filter applications"); pw.println(" based on input comma-separated list of"); pw.println(" application types."); @@ -1825,9 +1991,10 @@ public class TestYarnCLI { pw.println(" with space"); pw.println(" -list List applications. Supports optional use"); pw.println(" of -appTypes to filter applications based"); - pw.println(" on application type, and -appStates to"); - pw.println(" filter applications based on application"); - pw.println(" state."); + pw.println(" on application type, -appStates to filter"); + pw.println(" applications based on application state"); + pw.println(" and -appTags to filter applications based"); + pw.println(" on application tag."); pw.println(" -movetoqueue <Application ID> Moves the application to a different"); pw.println(" queue."); pw.println(" -queue <Queue Name> Works with the movetoqueue command to"); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
