Repository: oozie Updated Branches: refs/heads/master cc12e8f17 -> f9b3746ae
OOZIE-2174 Add missing admin commands to OozieClient and OozieCLI (rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/f9b3746a Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/f9b3746a Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/f9b3746a Branch: refs/heads/master Commit: f9b3746ae53d2544ef20510c2aa091e22813b1bc Parents: cc12e8f Author: Robert Kanter <[email protected]> Authored: Tue Mar 31 10:33:31 2015 -0700 Committer: Robert Kanter <[email protected]> Committed: Tue Mar 31 10:33:31 2015 -0700 ---------------------------------------------------------------------- .../java/org/apache/oozie/cli/OozieCLI.java | 109 +++- .../org/apache/oozie/client/OozieClient.java | 499 ++++++++++++++++++- .../org/apache/oozie/client/TestOozieCLI.java | 184 ++++++- docs/src/site/twiki/DG_CommandLineTool.twiki | 222 ++++++++- release-log.txt | 1 + 5 files changed, 981 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/f9b3746a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java index 5feb360..5393f68 100644 --- a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java +++ b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java @@ -72,6 +72,7 @@ import java.util.Locale; import java.util.Map; import java.util.Properties; import java.util.TimeZone; +import java.util.TreeMap; import java.util.concurrent.Callable; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -154,7 +155,12 @@ public class OozieCLI { public static final String SLA_ENABLE_ALERT = "sla_enable"; public static final String SLA_CHANGE = "sla_change"; + public static final String SERVER_CONFIGURATION_OPTION = "configuration"; + public static final String SERVER_OS_ENV_OPTION = "osenv"; + public static final String SERVER_JAVA_SYSTEM_PROPERTIES_OPTION = "javasysprops"; + public static final String METRICS_OPTION = "metrics"; + public static final String INSTRUMENTATION_OPTION = "instrumentation"; public static final String AUTH_OPTION = "auth"; @@ -259,6 +265,11 @@ public class OozieCLI { Option availServers = new Option(AVAILABLE_SERVERS_OPTION, false, "list available Oozie servers" + " (more than one only if HA is enabled)"); Option sharelibUpdate = new Option(UPDATE_SHARELIB_OPTION, false, "Update server to use a newer version of sharelib"); + Option serverConfiguration = new Option(SERVER_CONFIGURATION_OPTION, false, "show Oozie system configuration"); + Option osEnv = new Option(SERVER_OS_ENV_OPTION, false, "show Oozie system OS environment"); + Option javaSysProps = new Option(SERVER_JAVA_SYSTEM_PROPERTIES_OPTION, false, "show Oozie Java system properties"); + Option metrics = new Option(METRICS_OPTION, false, "show Oozie system metrics"); + Option instrumentation = new Option(INSTRUMENTATION_OPTION, false, "show Oozie system instrumentation"); Option sharelib = new Option(LIST_SHARELIB_LIB_OPTION, false, "List available sharelib that can be specified in a workflow action"); @@ -275,6 +286,11 @@ public class OozieCLI { group.addOption(availServers); group.addOption(sharelibUpdate); group.addOption(sharelib); + group.addOption(serverConfiguration); + group.addOption(osEnv); + group.addOption(javaSysProps); + group.addOption(metrics); + group.addOption(instrumentation); adminOptions.addOptionGroup(group); addAuthOptions(adminOptions); return adminOptions; @@ -1816,9 +1832,38 @@ public class OozieCLI { } } else if (options.contains(AVAILABLE_SERVERS_OPTION)) { - Map<String, String> availableOozieServers = wc.getAvailableOozieServers(); - for (String key : availableOozieServers.keySet()) { - System.out.println(key + " : " + availableOozieServers.get(key)); + Map<String, String> availableOozieServers = new TreeMap<String, String>(wc.getAvailableOozieServers()); + for (Map.Entry<String, String> ent : availableOozieServers.entrySet()) { + System.out.println(ent.getKey() + " : " + ent.getValue()); + } + } else if (options.contains(SERVER_CONFIGURATION_OPTION)) { + Map<String, String> serverConfig = new TreeMap<String, String>(wc.getServerConfiguration()); + for (Map.Entry<String, String> ent : serverConfig.entrySet()) { + System.out.println(ent.getKey() + " : " + ent.getValue()); + } + } else if (options.contains(SERVER_OS_ENV_OPTION)) { + Map<String, String> osEnv = new TreeMap<String, String>(wc.getOSEnv()); + for (Map.Entry<String, String> ent : osEnv.entrySet()) { + System.out.println(ent.getKey() + " : " + ent.getValue()); + } + } else if (options.contains(SERVER_JAVA_SYSTEM_PROPERTIES_OPTION)) { + Map<String, String> javaSysProps = new TreeMap<String, String>(wc.getJavaSystemProperties()); + for (Map.Entry<String, String> ent : javaSysProps.entrySet()) { + System.out.println(ent.getKey() + " : " + ent.getValue()); + } + } else if (options.contains(METRICS_OPTION)) { + OozieClient.Metrics metrics = wc.getMetrics(); + if (metrics == null) { + System.out.println("Metrics are unavailable. Try Instrumentation (-" + INSTRUMENTATION_OPTION + ") instead"); + } else { + printMetrics(metrics); + } + } else if (options.contains(INSTRUMENTATION_OPTION)) { + OozieClient.Instrumentation instrumentation = wc.getInstrumentation(); + if (instrumentation == null) { + System.out.println("Instrumentation is unavailable. Try Metrics (-" + METRICS_OPTION + ") instead"); + } else { + printInstrumentation(instrumentation); } } } @@ -2186,4 +2231,62 @@ public class OozieCLI { } } + private void printMetrics(OozieClient.Metrics metrics) { + System.out.println("COUNTERS"); + System.out.println("--------"); + Map<String, Long> counters = new TreeMap<String, Long>(metrics.getCounters()); + for (Map.Entry<String, Long> ent : counters.entrySet()) { + System.out.println(ent.getKey() + " : " + ent.getValue()); + } + System.out.println("\nGAUGES"); + System.out.println("------"); + Map<String, Object> gauges = new TreeMap<String, Object>(metrics.getGauges()); + for (Map.Entry<String, Object> ent : gauges.entrySet()) { + System.out.println(ent.getKey() + " : " + ent.getValue()); + } + System.out.println("\nTIMERS"); + System.out.println("------"); + Map<String, OozieClient.Metrics.Timer> timers = new TreeMap<String, OozieClient.Metrics.Timer>(metrics.getTimers()); + for (Map.Entry<String, OozieClient.Metrics.Timer> ent : timers.entrySet()) { + System.out.println(ent.getKey()); + System.out.println(ent.getValue()); + } + System.out.println("\nHISTOGRAMS"); + System.out.println("----------"); + Map<String, OozieClient.Metrics.Histogram> histograms = + new TreeMap<String, OozieClient.Metrics.Histogram>(metrics.getHistograms()); + for (Map.Entry<String, OozieClient.Metrics.Histogram> ent : histograms.entrySet()) { + System.out.println(ent.getKey()); + System.out.println(ent.getValue()); + } + } + + private void printInstrumentation(OozieClient.Instrumentation instrumentation) { + System.out.println("COUNTERS"); + System.out.println("--------"); + Map<String, Long> counters = new TreeMap<String, Long>(instrumentation.getCounters()); + for (Map.Entry<String, Long> ent : counters.entrySet()) { + System.out.println(ent.getKey() + " : " + ent.getValue()); + } + System.out.println("\nVARIABLES"); + System.out.println("---------"); + Map<String, Object> variables = new TreeMap<String, Object>(instrumentation.getVariables()); + for (Map.Entry<String, Object> ent : variables.entrySet()) { + System.out.println(ent.getKey() + " : " + ent.getValue()); + } + System.out.println("\nSAMPLERS"); + System.out.println("---------"); + Map<String, Double> samplers = new TreeMap<String, Double>(instrumentation.getSamplers()); + for (Map.Entry<String, Double> ent : samplers.entrySet()) { + System.out.println(ent.getKey() + " : " + ent.getValue()); + } + System.out.println("\nTIMERS"); + System.out.println("---------"); + Map<String, OozieClient.Instrumentation.Timer> timers = + new TreeMap<String, OozieClient.Instrumentation.Timer>(instrumentation.getTimers()); + for (Map.Entry<String, OozieClient.Instrumentation.Timer> ent : timers.entrySet()) { + System.out.println(ent.getKey()); + System.out.println(ent.getValue()); + } + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/f9b3746a/client/src/main/java/org/apache/oozie/client/OozieClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/client/OozieClient.java b/client/src/main/java/org/apache/oozie/client/OozieClient.java index 416d066..61d7923 100644 --- a/client/src/main/java/org/apache/oozie/client/OozieClient.java +++ b/client/src/main/java/org/apache/oozie/client/OozieClient.java @@ -567,6 +567,30 @@ public class OozieClient { protected abstract T call(HttpURLConnection conn) throws IOException, OozieClientException; } + protected abstract class MapClientCallable extends ClientCallable<Map<String, String>> { + + MapClientCallable(String method, String collection, String resource, Map<String, String> params) { + super(method, collection, resource, params); + } + + @Override + protected Map<String, String> call(HttpURLConnection conn) throws IOException, OozieClientException { + if ((conn.getResponseCode() == HttpURLConnection.HTTP_OK)) { + Reader reader = new InputStreamReader(conn.getInputStream()); + JSONObject json = (JSONObject) JSONValue.parse(reader); + Map<String, String> map = new HashMap<String, String>(); + for (Object key : json.keySet()) { + map.put((String)key, (String)json.get(key)); + } + return map; + } + else { + handleError(conn); + } + return null; + } + } + static void handleError(HttpURLConnection conn) throws IOException, OozieClientException { int status = conn.getResponseCode(); String error = conn.getHeaderField(RestConstants.OOZIE_ERROR_CODE); @@ -2000,7 +2024,6 @@ public class OozieClient { } bf.append(System.getProperty("line.separator")); } - return bf.toString(); } else{ JSONObject obj = (JSONObject) ((JSONObject) sharelib).get(JsonTags.SHARELIB_LIB_UPDATE); @@ -2010,6 +2033,7 @@ public class OozieClient { } bf.append(System.getProperty("line.separator")); } + return bf.toString(); } else { handleError(conn); @@ -2245,22 +2269,91 @@ public class OozieClient { return new GetQueueDump().call(); } - private class GetAvailableOozieServers extends ClientCallable<Map<String, String>> { + private class GetAvailableOozieServers extends MapClientCallable { GetAvailableOozieServers() { super("GET", RestConstants.ADMIN, RestConstants.ADMIN_AVAILABLE_OOZIE_SERVERS_RESOURCE, prepareParams()); } + } + + /** + * Return the list of available Oozie servers. + * + * @return the list of available Oozie servers. + * @throws OozieClientException throw if it the list of available Oozie servers could not be retrieved. + */ + public Map<String, String> getAvailableOozieServers() throws OozieClientException { + return new GetAvailableOozieServers().call(); + } + + private class GetServerConfiguration extends MapClientCallable { + + GetServerConfiguration() { + super("GET", RestConstants.ADMIN, RestConstants.ADMIN_CONFIG_RESOURCE, prepareParams()); + } + } + + /** + * Return the Oozie system configuration. + * + * @return the Oozie system configuration. + * @throws OozieClientException throw if the system configuration could not be retrieved. + */ + public Map<String, String> getServerConfiguration() throws OozieClientException { + return new GetServerConfiguration().call(); + } + + private class GetJavaSystemProperties extends MapClientCallable { + + GetJavaSystemProperties() { + super("GET", RestConstants.ADMIN, RestConstants.ADMIN_JAVA_SYS_PROPS_RESOURCE, prepareParams()); + } + } + + /** + * Return the Oozie Java system properties. + * + * @return the Oozie Java system properties. + * @throws OozieClientException throw if the system properties could not be retrieved. + */ + public Map<String, String> getJavaSystemProperties() throws OozieClientException { + return new GetJavaSystemProperties().call(); + } + + private class GetOSEnv extends MapClientCallable { + + GetOSEnv() { + super("GET", RestConstants.ADMIN, RestConstants.ADMIN_OS_ENV_RESOURCE, prepareParams()); + } + } + + /** + * Return the Oozie system OS environment. + * + * @return the Oozie system OS environment. + * @throws OozieClientException throw if the system OS environment could not be retrieved. + */ + public Map<String, String> getOSEnv() throws OozieClientException { + return new GetOSEnv().call(); + } + + private class GetMetrics extends ClientCallable<Metrics> { + + GetMetrics() { + super("GET", RestConstants.ADMIN, RestConstants.ADMIN_METRICS_RESOURCE, prepareParams()); + } @Override - protected Map<String, String> call(HttpURLConnection conn) throws IOException, OozieClientException { + protected Metrics call(HttpURLConnection conn) throws IOException, OozieClientException { if ((conn.getResponseCode() == HttpURLConnection.HTTP_OK)) { Reader reader = new InputStreamReader(conn.getInputStream()); JSONObject json = (JSONObject) JSONValue.parse(reader); - Map<String, String> map = new HashMap<String, String>(); - for (Object key : json.keySet()) { - map.put((String)key, (String)json.get(key)); - } - return map; + Metrics metrics = new Metrics(json); + return metrics; + } + else if ((conn.getResponseCode() == HttpURLConnection.HTTP_UNAVAILABLE)) { + // Use Instrumentation endpoint + return null; } else { handleError(conn); @@ -2269,16 +2362,396 @@ public class OozieClient { } } + public class Metrics { + private Map<String, Long> counters; + private Map<String, Object> gauges; + private Map<String, Timer> timers; + private Map<String, Histogram> histograms; + + @SuppressWarnings("unchecked") + public Metrics(JSONObject json) { + JSONObject jCounters = (JSONObject) json.get("counters"); + counters = new HashMap<String, Long>(jCounters.size()); + for (Object entO : jCounters.entrySet()) { + Entry<String, JSONObject> ent = (Entry<String, JSONObject>) entO; + counters.put(ent.getKey(), (Long)ent.getValue().get("count")); + } + + JSONObject jGuages = (JSONObject) json.get("gauges"); + gauges = new HashMap<String, Object>(jGuages.size()); + for (Object entO : jGuages.entrySet()) { + Entry<String, JSONObject> ent = (Entry<String, JSONObject>) entO; + gauges.put(ent.getKey(), ent.getValue().get("value")); + } + + JSONObject jTimers = (JSONObject) json.get("timers"); + timers = new HashMap<String, Timer>(jTimers.size()); + for (Object entO : jTimers.entrySet()) { + Entry<String, JSONObject> ent = (Entry<String, JSONObject>) entO; + timers.put(ent.getKey(), new Timer(ent.getValue())); + } + + JSONObject jHistograms = (JSONObject) json.get("histograms"); + histograms = new HashMap<String, Histogram>(jHistograms.size()); + for (Object entO : jHistograms.entrySet()) { + Entry<String, JSONObject> ent = (Entry<String, JSONObject>) entO; + histograms.put(ent.getKey(), new Histogram(ent.getValue())); + } + } + + public Map<String, Long> getCounters() { + return counters; + } + + public Map<String, Object> getGauges() { + return gauges; + } + + public Map<String, Timer> getTimers() { + return timers; + } + + public Map<String, Histogram> getHistograms() { + return histograms; + } + + public class Timer extends Histogram { + private double m15Rate; + private double m5Rate; + private double m1Rate; + private double meanRate; + private String durationUnits; + private String rateUnits; + + public Timer(JSONObject json) { + super(json); + m15Rate = Double.valueOf(json.get("m15_rate").toString()); + m5Rate = Double.valueOf(json.get("m5_rate").toString()); + m1Rate = Double.valueOf(json.get("m1_rate").toString()); + meanRate = Double.valueOf(json.get("mean_rate").toString()); + durationUnits = json.get("duration_units").toString(); + rateUnits = json.get("rate_units").toString(); + } + + public double get15MinuteRate() { + return m15Rate; + } + + public double get5MinuteRate() { + return m5Rate; + } + + public double get1MinuteRate() { + return m1Rate; + } + + public double getMeanRate() { + return meanRate; + } + + public String getDurationUnits() { + return durationUnits; + } + + public String getRateUnits() { + return rateUnits; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(super.toString()); + sb.append("\n\t15 minute rate : ").append(m15Rate); + sb.append("\n\t5 minute rate : ").append(m5Rate); + sb.append("\n\t1 minute rate : ").append(m15Rate); + sb.append("\n\tmean rate : ").append(meanRate); + sb.append("\n\tduration units : ").append(durationUnits); + sb.append("\n\trate units : ").append(rateUnits); + return sb.toString(); + } + } + + public class Histogram { + private double p999; + private double p99; + private double p98; + private double p95; + private double p75; + private double p50; + private double mean; + private double min; + private double max; + private double stdDev; + private long count; + + public Histogram(JSONObject json) { + p999 = Double.valueOf(json.get("p999").toString()); + p99 = Double.valueOf(json.get("p99").toString()); + p98 = Double.valueOf(json.get("p98").toString()); + p95 = Double.valueOf(json.get("p95").toString()); + p75 = Double.valueOf(json.get("p75").toString()); + p50 = Double.valueOf(json.get("p50").toString()); + mean = Double.valueOf(json.get("mean").toString()); + min = Double.valueOf(json.get("min").toString()); + max = Double.valueOf(json.get("max").toString()); + stdDev = Double.valueOf(json.get("stddev").toString()); + count = Long.valueOf(json.get("count").toString()); + } + + public double get999thPercentile() { + return p999; + } + + public double get99thPercentile() { + return p99; + } + + public double get98thPercentile() { + return p98; + } + + public double get95thPercentile() { + return p95; + } + + public double get75thPercentile() { + return p75; + } + + public double get50thPercentile() { + return p50; + } + + public double getMean() { + return mean; + } + + public double getMin() { + return min; + } + + public double getMax() { + return max; + } + + public double getStandardDeviation() { + return stdDev; + } + + public long getCount() { + return count; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("\t999th percentile : ").append(p999); + sb.append("\n\t99th percentile : ").append(p99); + sb.append("\n\t98th percentile : ").append(p98); + sb.append("\n\t95th percentile : ").append(p95); + sb.append("\n\t75th percentile : ").append(p75); + sb.append("\n\t50th percentile : ").append(p50); + sb.append("\n\tmean : ").append(mean); + sb.append("\n\tmax : ").append(max); + sb.append("\n\tmin : ").append(min); + sb.append("\n\tcount : ").append(count); + sb.append("\n\tstandard deviation : ").append(stdDev); + return sb.toString(); + } + } + } + /** - * Return the list of available Oozie servers. + * Return the Oozie metrics. If null is returned, then try {@link #getInstrumentation()}. * - * @return the list of available Oozie servers. - * @throws OozieClientException throw if it the list of available Oozie servers could not be retrieved. + * @return the Oozie metrics or null. + * @throws OozieClientException throw if the metrics could not be retrieved. */ - public Map<String, String> getAvailableOozieServers() throws OozieClientException { - return new GetAvailableOozieServers().call(); + public Metrics getMetrics() throws OozieClientException { + return new GetMetrics().call(); + } + + private class GetInstrumentation extends ClientCallable<Instrumentation> { + + GetInstrumentation() { + super("GET", RestConstants.ADMIN, RestConstants.ADMIN_INSTRUMENTATION_RESOURCE, prepareParams()); + } + + @Override + protected Instrumentation call(HttpURLConnection conn) throws IOException, OozieClientException { + if ((conn.getResponseCode() == HttpURLConnection.HTTP_OK)) { + Reader reader = new InputStreamReader(conn.getInputStream()); + JSONObject json = (JSONObject) JSONValue.parse(reader); + Instrumentation instrumentation = new Instrumentation(json); + return instrumentation; + } + else if ((conn.getResponseCode() == HttpURLConnection.HTTP_UNAVAILABLE)) { + // Use Metrics endpoint + return null; + } + else { + handleError(conn); + } + return null; + } } + public class Instrumentation { + private Map<String, Long> counters; + private Map<String, Object> variables; + private Map<String, Double> samplers; + private Map<String, Timer> timers; + + public Instrumentation(JSONObject json) { + JSONArray jCounters = (JSONArray) json.get("counters"); + counters = new HashMap<String, Long>(jCounters.size()); + for (Object groupO : jCounters) { + JSONObject group = (JSONObject) groupO; + String groupName = group.get("group").toString() + "."; + JSONArray data = (JSONArray) group.get("data"); + for (Object datO : data) { + JSONObject dat = (JSONObject) datO; + counters.put(groupName + dat.get("name").toString(), Long.valueOf(dat.get("value").toString())); + } + } + + JSONArray jVariables = (JSONArray) json.get("variables"); + variables = new HashMap<String, Object>(jVariables.size()); + for (Object groupO : jVariables) { + JSONObject group = (JSONObject) groupO; + String groupName = group.get("group").toString() + "."; + JSONArray data = (JSONArray) group.get("data"); + for (Object datO : data) { + JSONObject dat = (JSONObject) datO; + variables.put(groupName + dat.get("name").toString(), dat.get("value")); + } + } + + JSONArray jSamplers = (JSONArray) json.get("samplers"); + samplers = new HashMap<String, Double>(jSamplers.size()); + for (Object groupO : jSamplers) { + JSONObject group = (JSONObject) groupO; + String groupName = group.get("group").toString() + "."; + JSONArray data = (JSONArray) group.get("data"); + for (Object datO : data) { + JSONObject dat = (JSONObject) datO; + samplers.put(groupName + dat.get("name").toString(), Double.valueOf(dat.get("value").toString())); + } + } + + JSONArray jTimers = (JSONArray) json.get("timers"); + timers = new HashMap<String, Timer>(jTimers.size()); + for (Object groupO : jTimers) { + JSONObject group = (JSONObject) groupO; + String groupName = group.get("group").toString() + "."; + JSONArray data = (JSONArray) group.get("data"); + for (Object datO : data) { + JSONObject dat = (JSONObject) datO; + timers.put(groupName + dat.get("name").toString(), new Timer(dat)); + } + } + } + + public class Timer { + private double ownTimeStdDev; + private long ownTimeAvg; + private long ownMaxTime; + private long ownMinTime; + private double totalTimeStdDev; + private long totalTimeAvg; + private long totalMaxTime; + private long totalMinTime; + private long ticks; + + public Timer(JSONObject json) { + ownTimeStdDev = Double.valueOf(json.get("ownTimeStdDev").toString()); + ownTimeAvg = Long.valueOf(json.get("ownTimeAvg").toString()); + ownMaxTime = Long.valueOf(json.get("ownMaxTime").toString()); + ownMinTime = Long.valueOf(json.get("ownMinTime").toString()); + totalTimeStdDev = Double.valueOf(json.get("totalTimeStdDev").toString()); + totalTimeAvg = Long.valueOf(json.get("totalTimeAvg").toString()); + totalMaxTime = Long.valueOf(json.get("totalMaxTime").toString()); + totalMinTime = Long.valueOf(json.get("totalMinTime").toString()); + ticks = Long.valueOf(json.get("ticks").toString()); + } + + public double getOwnTimeStandardDeviation() { + return ownTimeStdDev; + } + + public long getOwnTimeAverage() { + return ownTimeAvg; + } + + public long getOwnMaxTime() { + return ownMaxTime; + } + + public long getOwnMinTime() { + return ownMinTime; + } + + public double getTotalTimeStandardDeviation() { + return totalTimeStdDev; + } + + public long getTotalTimeAverage() { + return totalTimeAvg; + } + + public long getTotalMaxTime() { + return totalMaxTime; + } + + public long getTotalMinTime() { + return totalMinTime; + } + + public long getTicks() { + return ticks; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("\town time standard deviation : ").append(ownTimeStdDev); + sb.append("\n\town average time : ").append(ownTimeAvg); + sb.append("\n\town max time : ").append(ownMaxTime); + sb.append("\n\town min time : ").append(ownMinTime); + sb.append("\n\ttotal time standard deviation : ").append(totalTimeStdDev); + sb.append("\n\ttotal average time : ").append(totalTimeAvg); + sb.append("\n\ttotal max time : ").append(totalMaxTime); + sb.append("\n\ttotal min time : ").append(totalMinTime); + sb.append("\n\tticks : ").append(ticks); + return sb.toString(); + } + } + + public Map<String, Long> getCounters() { + return counters; + } + + public Map<String, Object> getVariables() { + return variables; + } + + public Map<String, Double> getSamplers() { + return samplers; + } + + public Map<String, Timer> getTimers() { + return timers; + } + } + + /** + * Return the Oozie instrumentation. If null is returned, then try {@link #getMetrics()}. + * + * @return the Oozie intstrumentation or null. + * @throws OozieClientException throw if the intstrumentation could not be retrieved. + */ + public Instrumentation getInstrumentation() throws OozieClientException { + return new GetInstrumentation().call(); + } /** * Check if the string is not null or not empty. http://git-wip-us.apache.org/repos/asf/oozie/blob/f9b3746a/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java b/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java index e55d5f6..83929f5 100644 --- a/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java +++ b/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java @@ -18,20 +18,27 @@ package org.apache.oozie.client; +import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.FileWriter; import java.io.OutputStream; +import java.io.PrintStream; import java.io.StringReader; import java.util.Properties; import java.util.concurrent.Callable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.oozie.BuildInfo; import org.apache.oozie.cli.CLIParser; import org.apache.oozie.cli.OozieCLI; import org.apache.oozie.client.rest.RestConstants; +import org.apache.oozie.service.InstrumentationService; +import org.apache.oozie.service.MetricsInstrumentationService; +import org.apache.oozie.service.Services; +import org.apache.oozie.service.ShareLibService; import org.apache.oozie.servlet.DagServletTestCase; import org.apache.oozie.servlet.MockCoordinatorEngineService; import org.apache.oozie.servlet.MockDagEngineService; @@ -906,10 +913,12 @@ public class TestOozieCLI extends DagServletTestCase { String oozieUrl = getContextURL(); String[] args = new String[]{"admin", "-status", "-oozie", oozieUrl}; - assertEquals(0, new OozieCLI().run(args)); + String out = runOozieCLIAndGetStdout(args); + assertEquals("System mode: NORMAL\n", out); args = new String[]{"admin", "-oozie", oozieUrl, "-systemmode", "NORMAL"}; - assertEquals(0, new OozieCLI().run(args)); + out = runOozieCLIAndGetStdout(args); + assertEquals("System mode: NORMAL\n", out); return null; } }); @@ -923,7 +932,9 @@ public class TestOozieCLI extends DagServletTestCase { String oozieUrl = getContextURL(); String[] args = new String[]{"admin", "-version", "-oozie", oozieUrl}; - assertEquals(0, new OozieCLI().run(args)); + String out = runOozieCLIAndGetStdout(args); + assertEquals("Oozie server build version: " + BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION)+ "\n", + out); return null; } @@ -932,7 +943,8 @@ public class TestOozieCLI extends DagServletTestCase { public void testClientBuildVersion() throws Exception { String[] args = new String[]{"version"}; - assertEquals(0, new OozieCLI().run(args)); + String out = runOozieCLIAndGetStdout(args); + assertEquals("Oozie client build version: " + BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION) + "\n", out); } public void testJobInfo() throws Exception { @@ -1101,7 +1113,8 @@ public class TestOozieCLI extends DagServletTestCase { String oozieUrl = getContextURL(); String[] args = new String[]{"admin", "-queuedump", "-oozie", oozieUrl}; - assertEquals(0, new OozieCLI().run(args)); + String out = runOozieCLIAndGetStdout(args); + assertTrue(out.contains("Server Queue Dump")); return null; } @@ -1113,7 +1126,8 @@ public class TestOozieCLI extends DagServletTestCase { assertEquals(0, new OozieCLI().run(args)); args = new String[]{"info", "-timezones"}; - assertEquals(0, new OozieCLI().run(args)); + String out = runOozieCLIAndGetStdout(args); + assertTrue(out.contains("Available Time Zones")); } public void testValidateWorkFlowCommand() throws Exception { @@ -1135,11 +1149,13 @@ public class TestOozieCLI extends DagServletTestCase { IOUtils.copyCharStream(new StringReader(validContent), new FileWriter(validfile)); String [] args = new String[] { "validate", validFileName }; - assertEquals(0, new OozieCLI().run(args)); + String out = runOozieCLIAndGetStdout(args); + assertTrue(out.contains("Valid")); IOUtils.copyCharStream(new StringReader(invalidContent), new FileWriter(invalidfile)); args = new String[] { "validate", invalidFileName }; - assertEquals(-1, new OozieCLI().run(args)); + out = runOozieCLIAndGetStderr(args); + assertTrue(out.contains("Invalid")); return null; } @@ -1183,7 +1199,8 @@ public class TestOozieCLI extends DagServletTestCase { String oozieUrl = getContextURL(); String[] args = new String[] {"sla", "-oozie", oozieUrl, "-len", "1" }; - assertEquals(-1, new OozieCLI().run(args)); + String out = runOozieCLIAndGetStderr(args); + assertTrue(out.contains("Could not authenticate, Authentication failed, status: 404, message: Not Found")); return null; } @@ -1195,10 +1212,12 @@ public class TestOozieCLI extends DagServletTestCase { @Override public Void call() throws Exception { HeaderTestingVersionServlet.OOZIE_HEADERS.clear(); + Services.get().setService(ShareLibService.class); String oozieUrl = getContextURL(); String[] args = new String[] { "admin", "-sharelibupdate", "-oozie", oozieUrl }; - assertEquals(0, new OozieCLI().run(args)); + String out = runOozieCLIAndGetStdout(args); + assertTrue(out.contains("ShareLib update status")); return null; } @@ -1213,7 +1232,8 @@ public class TestOozieCLI extends DagServletTestCase { String oozieUrl = getContextURL(); String[] args = new String[] { "admin", "-sharelibupdate", "-oozie", oozieUrl }; - assertEquals(-1, new OozieCLI().run(args)); + String out = runOozieCLIAndGetStderr(args); + assertEquals("Error: E0503 : E0503: User [test] does not have admin privileges\n", out); return null; } @@ -1225,10 +1245,12 @@ public class TestOozieCLI extends DagServletTestCase { @Override public Void call() throws Exception { HeaderTestingVersionServlet.OOZIE_HEADERS.clear(); + Services.get().setService(ShareLibService.class); String oozieUrl = getContextURL(); String[] args = new String[] { "admin", "-shareliblist", "-oozie", oozieUrl }; - assertEquals(0, new OozieCLI().run(args)); + String out = runOozieCLIAndGetStdout(args); + assertTrue(out.contains("Available ShareLib")); return null; } @@ -1240,10 +1262,12 @@ public class TestOozieCLI extends DagServletTestCase { @Override public Void call() throws Exception { HeaderTestingVersionServlet.OOZIE_HEADERS.clear(); + Services.get().setService(ShareLibService.class); String oozieUrl = getContextURL(); String[] args = new String[] { "admin", "-shareliblist", "pig", "-oozie", oozieUrl }; - assertEquals(0, new OozieCLI().run(args)); + String out = runOozieCLIAndGetStdout(args); + assertTrue(out.contains("Available ShareLib")); return null; } @@ -1311,7 +1335,8 @@ public class TestOozieCLI extends DagServletTestCase { String oozieUrl = getContextURL(); String[] args = new String[] { "job", "-oozie", oozieUrl }; - assertEquals(-1, new OozieCLI().run(args)); + String out = runOozieCLIAndGetStderr(args); + assertTrue(out.contains("Invalid sub-command")); return null; } }); @@ -1392,4 +1417,135 @@ public class TestOozieCLI extends DagServletTestCase { }); } + public void testAdminConfiguration() throws Exception { + runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable<Void>() { + @Override + public Void call() throws Exception { + HeaderTestingVersionServlet.OOZIE_HEADERS.clear(); + + String oozieUrl = getContextURL(); + String[] args = new String[]{"admin", "-configuration", "-oozie", oozieUrl}; + String out = runOozieCLIAndGetStdout(args); + assertTrue(out.contains("oozie.base.url")); + + return null; + } + }); + } + + public void testAdminOsEnv() throws Exception { + runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable<Void>() { + @Override + public Void call() throws Exception { + HeaderTestingVersionServlet.OOZIE_HEADERS.clear(); + + String oozieUrl = getContextURL(); + String[] args = new String[]{"admin", "-osenv", "-oozie", oozieUrl}; + String out = runOozieCLIAndGetStdout(args); + assertTrue(out.contains("JAVA_HOME")); + + return null; + } + }); + } + + public void testAdminJavaSystemProperties() throws Exception { + runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable<Void>() { + @Override + public Void call() throws Exception { + HeaderTestingVersionServlet.OOZIE_HEADERS.clear(); + + String oozieUrl = getContextURL(); + String[] args = new String[]{"admin", "-javasysprops", "-oozie", oozieUrl}; + String out = runOozieCLIAndGetStdout(args); + assertTrue(out.contains("java.vendor")); + + return null; + } + }); + } + + public void testAdminInstrumentation() throws Exception { + runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable<Void>() { + @Override + public Void call() throws Exception { + HeaderTestingVersionServlet.OOZIE_HEADERS.clear(); + Services.get().setService(InstrumentationService.class); + + String oozieUrl = getContextURL(); + String[] args = new String[]{"admin", "-instrumentation", "-oozie", oozieUrl}; + String out = runOozieCLIAndGetStdout(args); + assertTrue(out.contains("webservices.version-GET")); + + args = new String[]{"admin", "-metrics", "-oozie", oozieUrl}; + out = runOozieCLIAndGetStdout(args); + assertTrue(out.contains("Metrics are unavailable")); + + return null; + } + }); + } + + public void testAdminMetrics() throws Exception { + runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable<Void>() { + @Override + public Void call() throws Exception { + HeaderTestingVersionServlet.OOZIE_HEADERS.clear(); + Services.get().setService(MetricsInstrumentationService.class); + + String oozieUrl = getContextURL(); + String[] args = new String[]{"admin", "-metrics", "-oozie", oozieUrl}; + String out = runOozieCLIAndGetStdout(args); + assertTrue(out.contains("webservices.version-GET")); + + args = new String[]{"admin", "-instrumentation", "-oozie", oozieUrl}; + out = runOozieCLIAndGetStdout(args); + assertTrue(out.contains("Instrumentation is unavailable")); + + return null; + } + }); + } + + private String runOozieCLIAndGetStdout(String[] args) { + PrintStream original = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(baos); + String outStr = null; + System.out.flush(); + try { + System.setOut(ps); + assertEquals(0, new OozieCLI().run(args)); + System.out.flush(); + outStr = baos.toString(); + } finally { + System.setOut(original); + if (outStr != null) { + System.out.print(outStr); + } + System.out.flush(); + } + return outStr; + } + + private String runOozieCLIAndGetStderr(String[] args) { + PrintStream original = System.err; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(baos); + String outStr = null; + System.err.flush(); + try { + System.setErr(ps); + assertEquals(-1, new OozieCLI().run(args)); + System.err.flush(); + outStr = baos.toString(); + } finally { + System.setErr(original); + if (outStr != null) { + System.err.print(outStr); + } + System.err.flush(); + } + return outStr; + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/f9b3746a/docs/src/site/twiki/DG_CommandLineTool.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/DG_CommandLineTool.twiki b/docs/src/site/twiki/DG_CommandLineTool.twiki index 762e2f1..3c85eb6 100644 --- a/docs/src/site/twiki/DG_CommandLineTool.twiki +++ b/docs/src/site/twiki/DG_CommandLineTool.twiki @@ -118,17 +118,21 @@ usage: . oozie admin <OPTIONS> : admin operations -auth <arg> select authentication type [SIMPLE|KERBEROS] - -doas <arg> doAs user, impersonates as the specified user. + -configuration show Oozie system configuration + -doas <arg> doAs user, impersonates as the specified user + -instrumentation show Oozie system instrumentation + -javasysprops show Oozie Java system properties + -metrics show Oozie system metrics -oozie <arg> Oozie URL + -osenv show Oozie system OS environment -queuedump show Oozie server queue elements -servers list available Oozie servers (more than one only if HA is enabled) + -shareliblist List available sharelib that can be specified in a workflow action + -sharelibupdate Update server to use a newer version of sharelib -status show the current system status -systemmode <arg> Supported in Oozie-2.0 or later versions ONLY. Change oozie system mode [NORMAL|NOWEBSERVICE|SAFEMODE] -version show Oozie server build version - -shareliblist List available sharelib that can be specified in a workflow - action - -sharelibupdate Update server to use a newer version of sharelib . oozie validate <ARGS> : validate a workflow XML file . @@ -1191,6 +1195,216 @@ hostC : http://hostC:11000/oozie It returns a list of available Oozie Servers. This is useful when Oozie is configured for [[AG_Install#HA][High Availability]]; if not, it will simply return the one Oozie Server. +---+++ Displaying the Oozie server configuration + +Example: + +<verbatim> +$ oozie admin -oozie http://localhost:11000/oozie -configuration +local.realm : LOCALHOST +oozie.JobCommand.job.console.url : http://localhost:11000/oozie?job= +oozie.action.fs.glob.max : 1000 +oozie.action.jobinfo.enable : false +oozie.action.launcher.mapreduce.job.ubertask.enable : true +oozie.action.launcher.yarn.timeline-service.enabled : false +oozie.action.mapreduce.uber.jar.enable : false +oozie.action.max.output.data : 2048 +oozie.action.retries.max : 3 +... +</verbatim> + +It returns a list of the configuration properties and values from oozie-site.xml and oozie-default.xml being used by the Oozie +server. + +---+++ Displaying the Oozie server OS environment + +Example: + +<verbatim> +$ oozie admin -oozie http://localhost:11000/oozie -osenv +Apple_PubSub_Socket_Render : /private/tmp/com.apple.launchd.V2oxXoP55M/Render +CATALINA_BASE : /Users/rkanter/dev/oozie/oozie-server +CATALINA_OUT : /Users/rkanter/dev/oozie/logs/catalina.out +CATALINA_PID : /Users/rkanter/dev/oozie/oozie-server/temp/oozie.pid +HADOOP_HOME : /Users/rkanter/dev/hadoop +HADOOP_HOME_WARN_SUPPRESS : TRUE +HADOOP_OPTS : -Djava.security.krb5.realm=OX.AC.UK -Djava.security.krb5.kdc=kdc0.ox.ac.uk:kdc1.ox.ac.uk +... +</verbatim> + +It returns a list of OS environment variables in the Oozie server. + +---+++ Displaying the Oozie server Java system properties + +Example: + +<verbatim> +$ oozie admin -oozie http://localhost:11000/oozie -javasysprops +awt.toolkit : sun.lwawt.macosx.LWCToolkit +catalina.base : /Users/rkanter/dev/oozie/oozie-server +catalina.home : /Users/rkanter/dev/oozie/oozie-server +catalina.useNaming : true +common.loader : ${catalina.base}/lib,${catalina.base}/lib/*.jar,${catalina.home}/lib,${catalina.home}/lib/*.jar +derby.stream.error.file : /Users/rkanter/dev/oozie/logs/derby.log +file.encoding : UTF-8 +file.encoding.pkg : sun.io +file.separator : / +... +</verbatim> + +It returns a list of java system properties in the Oozie server. + +---+++ Displaying the Oozie server Instrumentation + +Example: + +<verbatim> +$ oozie admin -oozie http://localhost:11000/oozie -instrumentation +COUNTERS +-------- +callablequeue.executed : 1 +callablequeue.queued : 1 +commands.purge.executions : 1 +... + +VARIABLES +--------- +configuration.action.types : [hive, shell, :START:, :FORK:, switch, spark, ssh, hive2, pig, :END:, email, distcp, :KILL:, sub-workflow, fs, java, :JOIN:, sqoop, map-reduce] +configuration.config.dir : /Users/rkanter/dev/oozie/conf +configuration.config.file : /Users/rkanter/dev/oozie/conf/oozie-site.xml +... + +SAMPLERS +--------- +callablequeue.queue.size : 0.0 +callablequeue.threads.active : 0.0 +jdbc.connections.active : 0.0 +... + +TIMERS +--------- +callablequeue.time.in.queue + own time standard deviation : -1.0 + own average time : 1 + own max time : 1 + own min time : 1 + total time standard deviation : -1.0 + total average time : 1 + total max time : 1 + total min time : 1 + ticks : 1 +commands.purge.call + own time standard deviation : -1.0 + own average time : 222 + own max time : 222 + own min time : 222 + total time standard deviation : -1.0 + total average time : 222 + total max time : 222 + total min time : 222 + ticks : 1 +... +</verbatim> + +It returns the instrumentation from the Oozie server. Keep in mind that timers and counters that the Oozie server +hasn't incremented yet will not show up. + +*Note:* If Instrumentation is enabled, then Metrics is unavailable. + +---+++ Displaying the Oozie server Metrics + +Example: + +<verbatim> +$ oozie admin -oozie http://localhost:11000/oozie -metrics +COUNTERS +-------- +callablequeue.executed : 1 +callablequeue.queued : 1 +commands.purge.executions : 1 +... + +GAUGES +------ +configuration.action.types : [hive, shell, :START:, :FORK:, switch, spark, ssh, hive2, pig, :END:, email, distcp, :KILL:, sub-workflow, fs, java, :JOIN:, sqoop, map-reduce] +configuration.config.dir : /Users/rkanter/dev/oozie/conf +configuration.config.file : /Users/rkanter/dev/oozie/conf/oozie-site.xml +... + +TIMERS +------ +callablequeue.time.in.queue.timer + 999th percentile : 4.0 + 99th percentile : 4.0 + 98th percentile : 4.0 + 95th percentile : 4.0 + 75th percentile : 4.0 + 50th percentile : 4.0 + mean : 4.0 + max : 4.0 + min : 4.0 + count : 1 + standard deviation : 0.0 + 15 minute rate : 0.0 + 5 minute rate : 0.0 + 1 minute rate : 0.0 + mean rate : 0.0 + duration units : milliseconds + rate units : calls/millisecond +commands.purge.call.timer + 999th percentile : 260.0 + 99th percentile : 260.0 + 98th percentile : 260.0 + 95th percentile : 260.0 + 75th percentile : 260.0 + 50th percentile : 260.0 + mean : 260.0 + max : 260.0 + min : 260.0 + count : 1 + standard deviation : 0.0 + 15 minute rate : 0.0 + 5 minute rate : 0.0 + 1 minute rate : 0.0 + mean rate : 0.0 + duration units : milliseconds + rate units : calls/millisecond +... + +HISTOGRAMS +---------- +callablequeue.queue.size.histogram + 999th percentile : 0.0 + 99th percentile : 0.0 + 98th percentile : 0.0 + 95th percentile : 0.0 + 75th percentile : 0.0 + 50th percentile : 0.0 + mean : 0.0 + max : 0.0 + min : 0.0 + count : 13 + standard deviation : 0.0 +callablequeue.threads.active.histogram + 999th percentile : 10.0 + 99th percentile : 10.0 + 98th percentile : 10.0 + 95th percentile : 10.0 + 75th percentile : 0.0 + 50th percentile : 0.0 + mean : 0.8461538461538461 + max : 10.0 + min : 0.0 + count : 13 + standard deviation : 2.764240517940803 +... +</verbatim> + +It returns the metrics from the Oozie server. Keep in mind that timers and counters that the Oozie server +hasn't incremented yet will not show up. + +*Note:* If Metrics is enabled, then Instrumentation is unavailable. + ---++ Validate Operations ---+++ Validating a Workflow XML http://git-wip-us.apache.org/repos/asf/oozie/blob/f9b3746a/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 7e0010d..a6c5d4b 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.2.0 release (trunk - unreleased) +OOZIE-2174 Add missing admin commands to OozieClient and OozieCLI (rkanter) OOZIE-2186 Upgrade Tomcat to 6.0.43 (rkanter) OOZIE-2181 JsonToBean has some missing and incorrect mappings (rkanter) OOZIE-2184 Change default value of action tmp dir removal to true (Viji via harsh)
