AMBARI-15902. Refactor Metadata manager for supporting distributed collector. (swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/798a70e7 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/798a70e7 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/798a70e7 Branch: refs/heads/branch-2.5 Commit: 798a70e76eebb71d5b62accbecbb8e5cdeda1d52 Parents: 14a4f97 Author: Aravindan Vijayan <[email protected]> Authored: Mon Nov 14 20:25:59 2016 -0800 Committer: Aravindan Vijayan <[email protected]> Committed: Tue Nov 15 11:02:12 2016 -0800 ---------------------------------------------------------------------- .../timeline/TimelineMetricConfiguration.java | 3 + .../TimelineMetricMetadataManager.java | 61 ++++++++-- .../discovery/TimelineMetricMetadataSync.java | 72 +++++++++++- .../timeline/discovery/TestMetadataManager.java | 4 +- .../timeline/discovery/TestMetadataSync.java | 116 +++++++++++++++++++ .../server/upgrade/UpgradeCatalog240.java | 26 +++-- .../0.1.0/configuration/ams-site.xml | 8 ++ 7 files changed, 266 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/798a70e7/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java index c4a3fee..9a4ed58 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java @@ -252,6 +252,9 @@ public class TimelineMetricConfiguration { public static final String TIMELINE_METRIC_AGGREGATION_SQL_FILTERS = "timeline.metrics.cluster.aggregation.sql.filters"; + public static final String TIMELINE_METRIC_METADATA_FILTERS = + "timeline.metrics.service.metadata.filters"; + public static final String HOST_APP_ID = "HOST"; public static final String DEFAULT_INSTANCE_PORT = "12001"; http://git-wip-us.apache.org/repos/asf/ambari/blob/798a70e7/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java index 3053252..3e4fc5f 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery; import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -27,8 +28,11 @@ import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata; import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -36,9 +40,11 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; + import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.DISABLE_METRIC_METADATA_MGMT; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.METRICS_METADATA_SYNC_INIT_DELAY; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.METRICS_METADATA_SYNC_SCHEDULE_DELAY; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_METADATA_FILTERS; public class TimelineMetricMetadataManager { private static final Log LOG = LogFactory.getLog(TimelineMetricMetadataManager.class); @@ -56,10 +62,19 @@ public class TimelineMetricMetadataManager { private PhoenixHBaseAccessor hBaseAccessor; private Configuration metricsConf; + TimelineMetricMetadataSync metricMetadataSync; + // Filter metrics names matching given patterns, from metadata + final List<String> metricNameFilters = new ArrayList<>(); + public TimelineMetricMetadataManager(PhoenixHBaseAccessor hBaseAccessor, Configuration metricsConf) { this.hBaseAccessor = hBaseAccessor; this.metricsConf = metricsConf; + + String patternStrings = metricsConf.get(TIMELINE_METRIC_METADATA_FILTERS); + if (!StringUtils.isEmpty(patternStrings)) { + metricNameFilters.addAll(Arrays.asList(patternStrings.split(","))); + } } /** @@ -69,21 +84,21 @@ public class TimelineMetricMetadataManager { if (metricsConf.getBoolean(DISABLE_METRIC_METADATA_MGMT, false)) { isDisabled = true; } else { + metricMetadataSync = new TimelineMetricMetadataSync(this); // Schedule the executor to sync to store - executorService.scheduleWithFixedDelay(new TimelineMetricMetadataSync(this), + executorService.scheduleWithFixedDelay(metricMetadataSync, metricsConf.getInt(METRICS_METADATA_SYNC_INIT_DELAY, 120), // 2 minutes metricsConf.getInt(METRICS_METADATA_SYNC_SCHEDULE_DELAY, 300), // 5 minutes TimeUnit.SECONDS); // Read from store and initialize map try { - Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadata = - hBaseAccessor.getTimelineMetricMetadata(); + Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadata = getMetadataFromStore(); LOG.info("Retrieved " + metadata.size() + ", metadata objects from store."); // Store in the cache METADATA_CACHE.putAll(metadata); - Map<String, Set<String>> hostedAppData = hBaseAccessor.getHostedAppsMetadata(); + Map<String, Set<String>> hostedAppData = getHostedAppsFromStore(); LOG.info("Retrieved " + hostedAppData.size() + " host objects from store."); HOSTED_APPS_MAP.putAll(hostedAppData); @@ -115,10 +130,26 @@ public class TimelineMetricMetadataManager { } /** + * Test metric name for valid patterns and return true/false + */ + boolean skipMetadataCache(String metricName) { + for (String pattern : metricNameFilters) { + if (metricName.contains(pattern)) { + return true; + } + } + return false; + } + + /** * Update value in metadata cache * @param metadata @TimelineMetricMetadata */ public void putIfModifiedTimelineMetricMetadata(TimelineMetricMetadata metadata) { + if (skipMetadataCache(metadata.getMetricName())) { + return; + } + TimelineMetricMetadataKey key = new TimelineMetricMetadataKey( metadata.getMetricName(), metadata.getAppId()); @@ -176,16 +207,28 @@ public class TimelineMetricMetadataManager { ); } + public boolean isDisabled() { + return isDisabled; + } + + boolean isDistributedModeEnabled() { + return metricsConf.get("timeline.metrics.service.operation.mode", "").equals("distributed"); + } + /** - * Fetch hosted apps from store + * Fetch metrics metadata from store * @throws SQLException */ - Map<String, Set<String>> getPersistedHostedAppsData() throws SQLException { - return hBaseAccessor.getHostedAppsMetadata(); + Map<TimelineMetricMetadataKey, TimelineMetricMetadata> getMetadataFromStore() throws SQLException { + return hBaseAccessor.getTimelineMetricMetadata(); } - public boolean isDisabled() { - return isDisabled; + /** + * Fetch hosted apps from store + * @throws SQLException + */ + Map<String, Set<String>> getHostedAppsFromStore() throws SQLException { + return hBaseAccessor.getHostedAppsMetadata(); } private boolean supportAggregates(TimelineMetric metric) { http://git-wip-us.apache.org/repos/asf/ambari/blob/798a70e7/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataSync.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataSync.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataSync.java index 54ea200..25b525a 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataSync.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataSync.java @@ -41,6 +41,22 @@ public class TimelineMetricMetadataSync implements Runnable { @Override public void run() { + LOG.debug("Persisting metric metadata..."); + persistMetricMetadata(); + LOG.debug("Persisting hosted apps metadata..."); + persistHostAppsMetadata(); + if (cacheManager.isDistributedModeEnabled()) { + LOG.debug("Refreshing metric metadata..."); + refreshMetricMetadata(); + LOG.debug("Refreshing hosted apps metadata..."); + refreshHostAppsMetadata(); + } + } + + /** + * Find metrics not persisted to store and persist them + */ + private void persistMetricMetadata() { List<TimelineMetricMetadata> metadataToPersist = new ArrayList<>(); // Find all entries to persist for (TimelineMetricMetadata metadata : cacheManager.getMetadataCache().values()) { @@ -70,11 +86,38 @@ public class TimelineMetricMetadataSync implements Runnable { cacheManager.getMetadataCache().put(key, metadata); } } - // Sync hosted apps data is needed + } + + /** + * Read all metric metadata and update cached values - HA mode + */ + private void refreshMetricMetadata() { + Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadataFromStore = null; + try { + metadataFromStore = cacheManager.getMetadataFromStore(); + } catch (SQLException e) { + LOG.warn("Error refreshing metadata from store.", e); + } + if (metadataFromStore != null) { + Map<TimelineMetricMetadataKey, TimelineMetricMetadata> cachedMetadata = + cacheManager.getMetadataCache(); + + for (Map.Entry<TimelineMetricMetadataKey, TimelineMetricMetadata> metadataEntry : metadataFromStore.entrySet()) { + if (!cachedMetadata.containsKey(metadataEntry.getKey())) { + cachedMetadata.put(metadataEntry.getKey(), metadataEntry.getValue()); + } + } + } + } + + /** + * Sync hosted apps data if needed + */ + private void persistHostAppsMetadata() { if (cacheManager.syncHostedAppsMetadata()) { Map<String, Set<String>> persistedData = null; try { - persistedData = cacheManager.getPersistedHostedAppsData(); + persistedData = cacheManager.getHostedAppsFromStore(); } catch (SQLException e) { LOG.warn("Failed on fetching hosted apps data from store.", e); return; // Something wrong with store @@ -86,8 +129,8 @@ public class TimelineMetricMetadataSync implements Runnable { for (Map.Entry<String, Set<String>> cacheEntry : cachedData.entrySet()) { // No persistence / stale data in store if (persistedData == null || persistedData.isEmpty() || - !persistedData.containsKey(cacheEntry.getKey()) || - !persistedData.get(cacheEntry.getKey()).containsAll(cacheEntry.getValue())) { + !persistedData.containsKey(cacheEntry.getKey()) || + !persistedData.get(cacheEntry.getKey()).containsAll(cacheEntry.getValue())) { dataToSync.put(cacheEntry.getKey(), cacheEntry.getValue()); } } @@ -102,4 +145,25 @@ public class TimelineMetricMetadataSync implements Runnable { } } + + /** + * Read all hosted apps metadata and update cached values - HA + */ + private void refreshHostAppsMetadata() { + Map<String, Set<String>> hostedAppsDataFromStore = null; + try { + hostedAppsDataFromStore = cacheManager.getHostedAppsFromStore(); + } catch (SQLException e) { + LOG.warn("Error refreshing metadata from store.", e); + } + if (hostedAppsDataFromStore != null) { + Map<String, Set<String>> cachedData = cacheManager.getHostedAppsCache(); + + for (Map.Entry<String, Set<String>> storeEntry : hostedAppsDataFromStore.entrySet()) { + if (!cachedData.containsKey(storeEntry.getKey())) { + cachedData.put(storeEntry.getKey(), storeEntry.getValue()); + } + } + } + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/798a70e7/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java index e99c001..2809640 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java @@ -122,13 +122,11 @@ public class TestMetadataManager extends AbstractMiniHBaseClusterTest { Assert.assertEquals(value2, savedData.get(key2)); Map<String, Set<String>> cachedHostData = metadataManager.getHostedAppsCache(); - Map<String, Set<String>> savedHostData = metadataManager.getPersistedHostedAppsData(); + Map<String, Set<String>> savedHostData = metadataManager.getHostedAppsFromStore(); Assert.assertEquals(cachedData.size(), savedData.size()); Assert.assertEquals("dummy_app1", cachedHostData.get("dummy_host1").iterator().next()); Assert.assertEquals("dummy_app2", cachedHostData.get("dummy_host2").iterator().next()); Assert.assertEquals("dummy_app1", savedHostData.get("dummy_host1").iterator().next()); Assert.assertEquals("dummy_app2", savedHostData.get("dummy_host2").iterator().next()); } - - } http://git-wip-us.apache.org/repos/asf/ambari/blob/798a70e7/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataSync.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataSync.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataSync.java new file mode 100644 index 0000000..78f2bfe --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataSync.java @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery; + +import junit.framework.Assert; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata; +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor; +import org.junit.Test; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import static org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata.MetricType.GAUGE; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_METADATA_FILTERS; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; + +public class TestMetadataSync { + @Test + public void testRefreshMetadataOnWrite() throws Exception { + Configuration configuration = createNiceMock(Configuration.class); + PhoenixHBaseAccessor hBaseAccessor = createNiceMock(PhoenixHBaseAccessor.class); + + final TimelineMetricMetadata testMetadata1 = new TimelineMetricMetadata( + "m1", "a1", "", GAUGE.name(), System.currentTimeMillis(), true); + final TimelineMetricMetadata testMetadata2 = new TimelineMetricMetadata( + "m2", "a2", "", GAUGE.name(), System.currentTimeMillis(), true); + + Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadata = + new HashMap<TimelineMetricMetadataKey, TimelineMetricMetadata>() {{ + put(new TimelineMetricMetadataKey("m1", "a1"), testMetadata1); + put(new TimelineMetricMetadataKey("m2", "a2"), testMetadata2); + }}; + + Map<String, Set<String>> hostedApps = new HashMap<String, Set<String>>() {{ + put("h1", new HashSet<>(Arrays.asList("a1"))); + put("h2", new HashSet<>(Arrays.asList("a1", "a2"))); + }}; + + expect(configuration.get("timeline.metrics.service.operation.mode", "")).andReturn("distributed"); + expect(hBaseAccessor.getTimelineMetricMetadata()).andReturn(metadata); + expect(hBaseAccessor.getHostedAppsMetadata()).andReturn(hostedApps); + + replay(configuration, hBaseAccessor); + + TimelineMetricMetadataManager metadataManager = new + TimelineMetricMetadataManager(hBaseAccessor, configuration); + + metadataManager.metricMetadataSync = new TimelineMetricMetadataSync(metadataManager); + + metadataManager.metricMetadataSync.run(); + + verify(configuration, hBaseAccessor); + + metadata = metadataManager.getMetadataCache(); + Assert.assertEquals(2, metadata.size()); + Assert.assertTrue(metadata.containsKey(new TimelineMetricMetadataKey("m1", "a1"))); + Assert.assertTrue(metadata.containsKey(new TimelineMetricMetadataKey("m2", "a2"))); + + hostedApps = metadataManager.getHostedAppsCache(); + Assert.assertEquals(2, hostedApps.size()); + Assert.assertEquals(1, hostedApps.get("h1").size()); + Assert.assertEquals(2, hostedApps.get("h2").size()); + } + + @Test + public void testFilterByRegexOnMetricName() throws Exception { + Configuration configuration = createNiceMock(Configuration.class); + PhoenixHBaseAccessor hBaseAccessor = createNiceMock(PhoenixHBaseAccessor.class); + + TimelineMetricMetadata metadata1 = new TimelineMetricMetadata( + "xxx.abc.yyy", "a1", "", GAUGE.name(), System.currentTimeMillis(), true); + TimelineMetricMetadata metadata2 = new TimelineMetricMetadata( + "xxx.cdef.yyy", "a2", "", GAUGE.name(), System.currentTimeMillis(), true); + TimelineMetricMetadata metadata3 = new TimelineMetricMetadata( + "xxx.pqr.zzz", "a3", "", GAUGE.name(), System.currentTimeMillis(), true); + + expect(configuration.get(TIMELINE_METRIC_METADATA_FILTERS)).andReturn("abc,cde"); + + replay(configuration, hBaseAccessor); + + TimelineMetricMetadataManager metadataManager = new + TimelineMetricMetadataManager(hBaseAccessor, configuration); + + metadataManager.putIfModifiedTimelineMetricMetadata(metadata1); + metadataManager.putIfModifiedTimelineMetricMetadata(metadata2); + metadataManager.putIfModifiedTimelineMetricMetadata(metadata3); + + verify(configuration, hBaseAccessor); + + Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadata = + metadataManager.getMetadataCache(); + + Assert.assertEquals(1, metadata.size()); + Assert.assertEquals("xxx.pqr.zzz", metadata.keySet().iterator().next().getMetricName()); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/798a70e7/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java index cfd25e7..25b6360 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java @@ -90,7 +90,7 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.jdbc.support.JdbcUtils; - +import org.apache.commons.lang.StringUtils; import javax.persistence.EntityManager; import javax.persistence.Query; import javax.persistence.TypedQuery; @@ -1441,10 +1441,10 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog { */ protected void updateAlertCurrentTable() throws SQLException { dbAccessor.addColumn(ALERT_CURRENT_TABLE, - new DBColumnInfo(ALERT_CURRENT_OCCURRENCES_COLUMN, Long.class, null, 1, false)); + new DBColumnInfo(ALERT_CURRENT_OCCURRENCES_COLUMN, Long.class, null, 1, false)); dbAccessor.addColumn(ALERT_CURRENT_TABLE, new DBColumnInfo(ALERT_CURRENT_FIRMNESS_COLUMN, - String.class, 255, AlertFirmness.HARD.name(), false)); + String.class, 255, AlertFirmness.HARD.name(), false)); } /** @@ -1469,15 +1469,15 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog { dbAccessor.executeUpdate(String.format(updateStatement, 2, PermissionEntity.CLUSTER_ADMINISTRATOR_PERMISSION_NAME)); dbAccessor.executeUpdate(String.format(updateStatement, - 3, PermissionEntity.CLUSTER_OPERATOR_PERMISSION_NAME)); + 3, PermissionEntity.CLUSTER_OPERATOR_PERMISSION_NAME)); dbAccessor.executeUpdate(String.format(updateStatement, - 4, PermissionEntity.SERVICE_ADMINISTRATOR_PERMISSION_NAME)); + 4, PermissionEntity.SERVICE_ADMINISTRATOR_PERMISSION_NAME)); dbAccessor.executeUpdate(String.format(updateStatement, - 5, PermissionEntity.SERVICE_OPERATOR_PERMISSION_NAME)); + 5, PermissionEntity.SERVICE_OPERATOR_PERMISSION_NAME)); dbAccessor.executeUpdate(String.format(updateStatement, - 6, PermissionEntity.CLUSTER_USER_PERMISSION_NAME)); + 6, PermissionEntity.CLUSTER_USER_PERMISSION_NAME)); dbAccessor.executeUpdate(String.format(updateStatement, - 7, PermissionEntity.VIEW_USER_PERMISSION_NAME)); + 7, PermissionEntity.VIEW_USER_PERMISSION_NAME)); } /** @@ -1911,6 +1911,16 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog { } } } + + Config amsSite = cluster.getDesiredConfigByType("ams-site"); + if (amsSite != null) { + String metadataFilters = amsSite.getProperties().get("timeline.metrics.service.metadata.filters"); + if (StringUtils.isEmpty(metadataFilters) || + !metadataFilters.contains("ContainerResource")) { + updateConfigurationProperties("ams-site", + Collections.singletonMap("timeline.metrics.service.metadata.filters", "ContainerResource"), true, false); + } + } } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/798a70e7/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml index a5860e0..e217011 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml @@ -708,4 +708,12 @@ </description> <on-ambari-upgrade add="true"/> </property> + <property> + <name>timeline.metrics.service.metadata.filters</name> + <value>ContainerResource</value> + <description> + Commas separated list of regular expressions that match metric names + which prevents certain metrics from ending up in metadata cache. + </description> + </property> </configuration>
