avijayanhwx closed pull request #12: [AMBARI-25077] Grafana on a workload
cluster should show visualizatio…
URL: https://github.com/apache/ambari-metrics/pull/12
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/ambari-metrics-grafana/ambari-metrics/datasource.js
b/ambari-metrics-grafana/ambari-metrics/datasource.js
index 6e14f33..47c207b 100644
--- a/ambari-metrics-grafana/ambari-metrics/datasource.js
+++ b/ambari-metrics-grafana/ambari-metrics/datasource.js
@@ -59,7 +59,7 @@ define([
//We get a list of components and their associated metrics.
this.initMetricAppidMapping = function () {
- return this.doAmbariRequest({url: '/ws/v1/timeline/metrics/metadata'})
+ return this.doAmbariRequest({url: '/metadata'})
.then(function (items) {
items = items.data;
allMetrics = {};
@@ -227,7 +227,7 @@ define([
var metricTransform = !target.transform || target.transform ===
"none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator ||
target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' +
target.seriesAggregator;
return self.doAmbariRequest({
- url: '/ws/v1/timeline/metrics?metricNames=' + target.metric +
metricTransform +
+ url: '?metricNames=' + target.metric + metricTransform +
metricAggregator + "&hostname=" + target.hosts + '&appId=' +
target.app + instanceId + '&startTime=' + from +
'&endTime=' + to + precision + seriesAggregator
}).then(
@@ -268,7 +268,7 @@ define([
var metricTransform = !target.transform || target.transform ===
"none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator ||
target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' +
target.seriesAggregator;
return self.doAmbariRequest({
- url: '/ws/v1/timeline/metrics?metricNames=' + target.metric +
metricTransform
+ url: '?metricNames=' + target.metric + metricTransform
+ metricAggregator + '&hostname=' + tHost + '&appId=' + target.app
+ instanceId + '&startTime=' + from +
'&endTime=' + to + precision + seriesAggregator
}).then(
@@ -302,7 +302,7 @@ define([
var seriesAggregator = !target.seriesAggregator ||
target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' +
target.seriesAggregator;
var templatedComponent = (_.isEmpty(tComponent)) ? target.app :
tComponent;
return self.doAmbariRequest({
- url: '/ws/v1/timeline/metrics?metricNames=' + target.metric +
metricTransform
+ url: '?metricNames=' + target.metric + metricTransform
+ metricAggregator + '&hostname=' + target.templatedHost +
'&appId=' + templatedComponent + instanceId
+ '&startTime=' + from + '&endTime=' + to + precision + topN +
seriesAggregator
}).then(
@@ -317,7 +317,7 @@ define([
var metricTransform = !target.transform || target.transform ===
"none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator ||
target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' +
target.seriesAggregator;
return self.doAmbariRequest({
- url: '/ws/v1/timeline/metrics?metricNames=' +
encodeURIComponent(target.queue) + metricTransform
+ url: '?metricNames=' + encodeURIComponent(target.queue) +
metricTransform
+ metricAggregator + '&appId=resourcemanager' + instanceId +
'&startTime=' + from +
'&endTime=' + to + precision + seriesAggregator
}).then(
@@ -330,7 +330,7 @@ define([
+ target.precision;
var seriesAggregator = !target.seriesAggregator ||
target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' +
target.seriesAggregator;
return self.doAmbariRequest({
- url: '/ws/v1/timeline/metrics?metricNames=' + target.hbMetric +
instanceId + '&appId=hbase&startTime='
+ url: '?metricNames=' + target.hbMetric + instanceId +
'&appId=hbase&startTime='
+ from + '&endTime=' + to + precision + seriesAggregator
}).then(
allHostMetricsData(target)
@@ -345,7 +345,7 @@ define([
var metricTransform = !target.transform || target.transform ===
"none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator ||
target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' +
target.seriesAggregator;
return self.doAmbariRequest({
- url: '/ws/v1/timeline/metrics?metricNames=' + target.kbMetric +
metricTransform + instanceId
+ url: '?metricNames=' + target.kbMetric + metricTransform +
instanceId
+ metricAggregator + '&appId=kafka_broker&startTime=' + from +
'&endTime=' + to + precision + seriesAggregator
}).then(
@@ -361,7 +361,7 @@ define([
var metricTransform = !target.transform || target.transform ===
"none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator ||
target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' +
target.seriesAggregator;
return self.doAmbariRequest({
- url: '/ws/v1/timeline/metrics?metricNames=' + target.nnMetric +
metricTransform + instanceId
+ url: '?metricNames=' + target.nnMetric + metricTransform +
instanceId
+ metricAggregator + '&appId=namenode&startTime=' + from +
'&endTime=' + to + precision + seriesAggregator
}).then(
allHostMetricsData(target)
@@ -377,7 +377,7 @@ define([
var metricTransform = !target.transform || target.transform ===
"none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator ||
target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' +
target.seriesAggregator;
return self.doAmbariRequest({
- url: '/ws/v1/timeline/metrics?metricNames=' + target.sTopoMetric +
metricTransform + instanceId
+ url: '?metricNames=' + target.sTopoMetric + metricTransform +
instanceId
+ metricAggregator + '&appId=nimbus&startTime=' + from +
'&endTime=' + to + precision + seriesAggregator
}).then(
allHostMetricsData(target)
@@ -393,7 +393,7 @@ define([
var metricTransform = !target.transform || target.transform ===
"none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator ||
target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' +
target.seriesAggregator;
return self.doAmbariRequest({
- url: '/ws/v1/timeline/metrics?metricNames=' + target.sCoreMetric +
metricTransform + instanceId
+ url: '?metricNames=' + target.sCoreMetric + metricTransform +
instanceId
+ metricAggregator + '&appId=ambari-infra-solr&startTime=' + from
+ '&endTime=' + to + precision + seriesAggregator
}).then(
allHostMetricsData(target)
@@ -408,7 +408,7 @@ define([
var metricTransform = !target.transform || target.transform ===
"none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator ||
target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' +
target.seriesAggregator;
return self.doAmbariRequest({
- url: '/ws/v1/timeline/metrics?metricNames=' +
target.sCollectionMetric + metricTransform + instanceId
+ url: '?metricNames=' + target.sCollectionMetric + metricTransform
+ instanceId
+ metricAggregator + '&appId=ambari-infra-solr&startTime=' + from
+ '&endTime=' + to + precision + seriesAggregator
}).then(
allHostMetricsData(target)
@@ -424,7 +424,7 @@ define([
var metricTransform = !target.transform || target.transform ===
"none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator ||
target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' +
target.seriesAggregator;
return self.doAmbariRequest({
- url: '/ws/v1/timeline/metrics?metricNames=' +
target.sDataSourceMetric + metricTransform + instanceId
+ url: '?metricNames=' + target.sDataSourceMetric + metricTransform
+ instanceId
+ metricAggregator + '&appId=druid&startTime=' + from +
'&endTime=' + to + precision + seriesAggregator
}).then(
allHostMetricsData(target)
@@ -1116,7 +1116,7 @@ define([
*/
this.testDatasource = function () {
return this.doAmbariRequest({
- url: '/ws/v1/timeline/metrics/metadata',
+ url: '/metadata',
method: 'GET'
}).then(function (response) {
console.log(response);
@@ -1166,7 +1166,7 @@ define([
}
return this.doAmbariRequest({
method: 'GET',
- url: '/ws/v1/timeline/metrics/instances?' + 'appId=' + app
+ url: '/instance?' + 'appId=' + app
}).then(function (response) {
var clusters = [];
var data = response.data;
@@ -1193,7 +1193,7 @@ define([
}
return this.doAmbariRequest({
method: 'GET',
- url: '/ws/v1/timeline/metrics/instances?' + 'appId=' + app +
'&instanceId=' + cluster
+ url: '/instance?' + 'appId=' + app + '&instanceId=' + cluster
}).then(function (response) {
var hosts = [];
var data = response.data;
diff --git
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java
index c90c93b..3619af6 100644
---
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java
+++
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java
@@ -17,7 +17,6 @@
*/
package org.apache.ambari.metrics.core.timeline;
-import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.DEFAULT_INSTANCE_ID;
import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.USE_GROUPBY_AGGREGATOR_QUERIES;
import static
org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.ACTUAL_AGGREGATOR_NAMES;
@@ -87,6 +86,7 @@
private TimelineMetricMetadataManager metricMetadataManager;
private MetricCollectorHAController haController;
private boolean containerMetricsDisabled = false;
+ private String defaultInstanceId = "";
/**
* Construct the service.
@@ -218,6 +218,7 @@ private synchronized void initializeSubsystem() {
"delay = " + initDelay + ", delay = " + delay);
}
containerMetricsDisabled = configuration.isContainerMetricsDisabled();
+ defaultInstanceId = configuration.getDefaultInstanceId();
isInitialized = true;
}
@@ -258,7 +259,7 @@ public TimelineMetrics getTimelineMetrics(List<String>
metricNames,
List<String> transientMetricNames = new ArrayList<>();
if (configuration.getTimelineMetricsMultipleClusterSupport() &&
StringUtils.isEmpty(instanceId)) {
- instanceId = DEFAULT_INSTANCE_ID;
+ instanceId = this.defaultInstanceId;
}
List<byte[]> uuids =
metricMetadataManager.getUuidsForGetMetricQuery(metricFunctions.keySet(),
diff --git
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
index b6ff202..6f1e1c0 100644
---
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
+++
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
@@ -29,7 +29,6 @@
import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_SECOND_TABLE_TTL;
import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CONTAINER_METRICS_TTL;
import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.DATE_TIERED_COMPACTION_POLICY;
-import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.DEFAULT_INSTANCE_ID;
import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.FIFO_COMPACTION_POLICY_CLASS;
import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.GLOBAL_MAX_RETRIES;
import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.GLOBAL_RESULT_LIMIT;
@@ -203,6 +202,7 @@
private TimelineMetricMetadataManager metadataManagerInstance;
private Set<String> eventMetricPatterns = new HashSet<>();
private boolean supportMultipleClusterMetrics = false;
+ private String defaultInstanceId = "";
private Map<String, Integer> tableTTL = new HashMap<>();
@@ -263,6 +263,7 @@ public PhoenixHBaseAccessor(PhoenixConnectionProvider
dataSource) {
tableTTL.put(METRIC_TRANSIENT_TABLE_NAME,
metricsConf.getInt(METRICS_TRANSIENT_TABLE_TTL, 7 * 86400)); //7 days
this.supportMultipleClusterMetrics =
Boolean.valueOf(metricsConf.get(TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS,
"false"));
+ this.defaultInstanceId = configuration.getDefaultInstanceId();
if (cacheEnabled) {
LOG.debug("Initialising and starting metrics cache committer thread...");
@@ -903,6 +904,10 @@ public void
insertMetricRecordsWithMetadata(TimelineMetricMetadataManager metada
boolean acceptMetric = TimelineMetricsFilter.acceptMetric(tm);
+ if (supportMultipleClusterMetrics &&
StringUtils.isEmpty(tm.getInstanceId())) {
+ tm.setInstanceId(defaultInstanceId);
+ }
+
// Write to metadata cache on successful write to store
if (metadataManager != null) {
metadataManager.putIfModifiedTimelineMetricMetadata(
@@ -912,9 +917,6 @@ public void
insertMetricRecordsWithMetadata(TimelineMetricMetadataManager metada
tm.getHostName(), tm.getAppId());
if (!tm.getAppId().equals("FLUME_HANDLER")) {
- if (supportMultipleClusterMetrics &&
StringUtils.isEmpty(tm.getInstanceId())) {
- tm.setInstanceId(DEFAULT_INSTANCE_ID);
- }
metadataManager.putIfModifiedHostedInstanceMetadata(tm.getInstanceId(),
tm.getHostName());
}
}
diff --git
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java
index 9766f0c..f8f4bb0 100644
---
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java
+++
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java
@@ -292,6 +292,9 @@
public static final String TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS =
"timeline.metrics.support.multiple.clusters";
+ public static final String TIMELINE_METRICS_DEFAULT_INSTANCE_ID =
+ "timeline.metrics.default.instanceid";
+
public static final String TIMELINE_METRICS_EVENT_METRIC_PATTERNS =
"timeline.metrics.downsampler.event.metric.patterns";
@@ -512,6 +515,13 @@ public boolean getTimelineMetricsMultipleClusterSupport() {
return false;
}
+ public String getDefaultInstanceId() {
+ if (metricsConf != null) {
+ return metricsConf.get(TIMELINE_METRICS_DEFAULT_INSTANCE_ID,
DEFAULT_INSTANCE_ID);
+ }
+ return DEFAULT_INSTANCE_ID;
+ }
+
public String getTimelineServiceRpcAddress() {
String defaultRpcAddress = "0.0.0.0:60200";
if (metricsConf != null) {
diff --git
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixTransactSQL.java
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixTransactSQL.java
index ee50051..092da51 100644
---
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixTransactSQL.java
+++
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixTransactSQL.java
@@ -154,7 +154,7 @@
"START_TIME UNSIGNED_LONG, " +
"SUPPORTS_AGGREGATION BOOLEAN, " +
"IS_WHITELISTED BOOLEAN " +
- "CONSTRAINT pk PRIMARY KEY (METRIC_NAME, APP_ID)) " +
+ "CONSTRAINT pk PRIMARY KEY (METRIC_NAME, APP_ID, INSTANCE_ID)) " +
"DATA_BLOCK_ENCODING='%s', COMPRESSION='%s'";
public static final String CREATE_HOSTED_APPS_METADATA_TABLE_SQL =
diff --git
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/TimelineWebServices.java
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/TimelineWebServices.java
index b3378be..71477ec 100644
---
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/TimelineWebServices.java
+++
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/TimelineWebServices.java
@@ -36,6 +36,7 @@
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.WebApplicationException;
@@ -287,6 +288,31 @@ public TimelineMetrics getTimelineMetrics(
}
}
+ @GET
+ @Path("metrics/{instanceId}")
+ @Produces({ MediaType.APPLICATION_JSON })
+ public TimelineMetrics getTimelineMetricsForInstance(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("instanceId") String instanceId,
+ @QueryParam("metricNames") String metricNames,
+ @QueryParam("appId") String appId,
+ @QueryParam("hostname") String hostname,
+ @QueryParam("startTime") String startTime,
+ @QueryParam("endTime") String endTime,
+ @QueryParam("precision") String precision,
+ @QueryParam("limit") String limit,
+ @QueryParam("grouped") String grouped,
+ @QueryParam("topN") String topN,
+ @QueryParam("topNFunction") String topNFunction,
+ @QueryParam("isBottomN") String isBottomN,
+ @QueryParam("seriesAggregateFunction") String seriesAggregateFunction
+ ) {
+
+ return getTimelineMetrics(req, res, metricNames, appId, instanceId,
hostname, startTime, endTime, precision, limit,
+ grouped, topN, topNFunction, isBottomN, seriesAggregateFunction);
+ }
+
@GET
@Path("/metrics/summary")
@Produces({ MediaType.APPLICATION_JSON })
@@ -324,6 +350,28 @@ public TimelineMetricServiceSummary
getTimelineMetricServiceSummary(
}
}
+ @GET
+ @Path("/metrics/{instanceId}/metadata")
+ @Produces({ MediaType.APPLICATION_JSON })
+ public Map<String, List<TimelineMetricMetadata>>
getTimelineMetricMetadataForInstance(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @QueryParam("appId") String appId,
+ @QueryParam("metricName") String metricPattern,
+ @QueryParam("includeAll") String includeBlacklistedMetrics
+ ) {
+ init(res);
+
+ try {
+ return timelineMetricStore.getTimelineMetricMetadata(
+ parseStr(appId),
+ parseStr(metricPattern),
+ parseBoolean(includeBlacklistedMetrics));
+ } catch (Exception e) {
+ throw new WebApplicationException(e,
Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
@GET
@Path("/metrics/hosts")
@Produces({ MediaType.APPLICATION_JSON })
@@ -341,7 +389,7 @@ public TimelineMetricServiceSummary
getTimelineMetricServiceSummary(
}
@GET
- @Path("/metrics/instances")
+ @Path("/metrics/instance")
@Produces({ MediaType.APPLICATION_JSON })
public Map<String, Map<String, Set<String>>> getClusterHostsMetadata(
@Context HttpServletRequest req,
@@ -358,6 +406,24 @@ public TimelineMetricServiceSummary
getTimelineMetricServiceSummary(
}
}
+ @GET
+ @Path("/metrics/{instanceId}/instance")
+ @Produces({ MediaType.APPLICATION_JSON })
+ public Map<String, Map<String, Set<String>>>
getClusterHostsMetadataForInstance(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @QueryParam("appId") String appId,
+ @PathParam("instanceId") String instanceId
+ ) {
+ init(res);
+
+ try {
+ return timelineMetricStore.getInstanceHostsMetadata(instanceId, appId);
+ } catch (Exception e) {
+ throw new WebApplicationException(e,
Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
@GET
@Path("/metrics/uuid")
@Produces({ MediaType.APPLICATION_JSON })
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
