Repository: ambari Updated Branches: refs/heads/trunk fc37b7376 -> c2c72b9da
AMBARI-20056 : FIFO compaction resulting in too many store files on large cluster. (avijayan) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c2c72b9d Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c2c72b9d Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c2c72b9d Branch: refs/heads/trunk Commit: c2c72b9da289bf0d172ee103eb0c337a9ffda1df Parents: fc37b73 Author: Aravindan Vijayan <[email protected]> Authored: Thu Feb 23 15:39:36 2017 -0800 Committer: Aravindan Vijayan <[email protected]> Committed: Thu Feb 23 15:39:36 2017 -0800 ---------------------------------------------------------------------- .../timeline/HBaseTimelineMetricStore.java | 18 +- .../metrics/timeline/PhoenixHBaseAccessor.java | 199 +++++++++++-------- .../timeline/TimelineMetricConfiguration.java | 25 ++- .../timeline/ITPhoenixHBaseAccessor.java | 21 +- .../server/upgrade/UpgradeCatalog250.java | 7 + .../0.1.0/configuration/ams-site.xml | 8 - .../server/upgrade/UpgradeCatalog250Test.java | 20 +- 7 files changed, 189 insertions(+), 109 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c2c72b9d/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java index 575b8ea..17c58f0 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java @@ -104,13 +104,17 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin hBaseAccessor.initPoliciesAndTTL(); // Start HA service // Start the controller - haController = new MetricCollectorHAController(configuration); - try { - haController.initializeHAController(); - } catch (Exception e) { - LOG.error(e); - throw new MetricsSystemInitializationException("Unable to " + - "initialize HA controller", e); + if (!configuration.isDistributedCollectorModeDisabled()) { + haController = new MetricCollectorHAController(configuration); + try { + haController.initializeHAController(); + } catch (Exception e) { + LOG.error(e); + throw new MetricsSystemInitializationException("Unable to " + + "initialize HA controller", e); + } + } else { + LOG.info("Distributed collector mode disabled"); } //Initialize whitelisting & blacklisting if needed http://git-wip-us.apache.org/repos/asf/ambari/blob/c2c72b9d/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java index 04405f7..8b0d84b 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline; +import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; @@ -81,6 +82,12 @@ import java.util.concurrent.TimeUnit; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.AGGREGATE_TABLE_SPLIT_POINTS; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_PRECISION_TABLE_HBASE_BLOCKING_STORE_FILES; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_AGGREGATE_TABLE_HBASE_BLOCKING_STORE_FILES; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_CLASS; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_KEY; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_CLASS; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_KEY; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_PRECISION_TABLE_DURABILITY; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_AGGREGATE_TABLES_DURABILITY; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HBASE_BLOCKING_STORE_FILES; @@ -177,10 +184,12 @@ public class PhoenixHBaseAccessor { static final String HSTORE_COMPACTION_CLASS_KEY = "hbase.hstore.defaultengine.compactionpolicy.class"; + static final String HSTORE_ENGINE_CLASS = + "hbase.hstore.engine.class"; static final String FIFO_COMPACTION_POLICY_CLASS = "org.apache.hadoop.hbase.regionserver.compactions.FIFOCompactionPolicy"; - static final String DEFAULT_COMPACTION_POLICY_CLASS = - "org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy"; + static final String DATE_TIERED_COMPACTION_POLICY = + "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine"; static final String BLOCKING_STORE_FILES_KEY = "hbase.hstore.blockingStoreFiles"; @@ -503,8 +512,6 @@ public class PhoenixHBaseAccessor { } protected void initPoliciesAndTTL() { - boolean enableNormalizer = hbaseConf.getBoolean("hbase.normalizer.enabled", false); - boolean enableFifoCompaction = metricsConf.getBoolean("timeline.metrics.hbase.fifo.compaction.enabled", true); HBaseAdmin hBaseAdmin = null; try { @@ -519,90 +526,23 @@ public class PhoenixHBaseAccessor { boolean modifyTable = false; HTableDescriptor tableDescriptor = hBaseAdmin.getTableDescriptor(tableName.getBytes()); + //Set normalizer preferences + boolean enableNormalizer = hbaseConf.getBoolean("hbase.normalizer.enabled", false); if (enableNormalizer ^ tableDescriptor.isNormalizationEnabled()) { tableDescriptor.setNormalizationEnabled(enableNormalizer); LOG.info("Normalizer set to " + enableNormalizer + " for " + tableName); modifyTable = true; } - if (METRICS_RECORD_TABLE_NAME.equals(tableName)) { - if (!timelineMetricsPrecisionTableDurability.isEmpty()) { - LOG.info("Setting WAL option " + timelineMetricsPrecisionTableDurability + " for table : " + tableName); - boolean validDurability = true; - if ("SKIP_WAL".equals(timelineMetricsPrecisionTableDurability)) { - tableDescriptor.setDurability(Durability.SKIP_WAL); - } else if ("SYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) { - tableDescriptor.setDurability(Durability.SYNC_WAL); - } else if ("ASYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) { - tableDescriptor.setDurability(Durability.ASYNC_WAL); - } else if ("FSYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) { - tableDescriptor.setDurability(Durability.FSYNC_WAL); - } else { - LOG.info("Unknown value for " + TIMELINE_METRICS_PRECISION_TABLE_DURABILITY + " : " + timelineMetricsPrecisionTableDurability); - validDurability = false; - } - if (validDurability) { - modifyTable = true; - } - } - } else { - if (!timelineMetricsTablesDurability.isEmpty()) { - LOG.info("Setting WAL option " + timelineMetricsTablesDurability + " for table : " + tableName); - boolean validDurability = true; - if ("SKIP_WAL".equals(timelineMetricsTablesDurability)) { - tableDescriptor.setDurability(Durability.SKIP_WAL); - } else if ("SYNC_WAL".equals(timelineMetricsTablesDurability)) { - tableDescriptor.setDurability(Durability.SYNC_WAL); - } else if ("ASYNC_WAL".equals(timelineMetricsTablesDurability)) { - tableDescriptor.setDurability(Durability.ASYNC_WAL); - } else if ("FSYNC_WAL".equals(timelineMetricsTablesDurability)) { - tableDescriptor.setDurability(Durability.FSYNC_WAL); - } else { - LOG.info("Unknown value for " + TIMELINE_METRICS_AGGREGATE_TABLES_DURABILITY + " : " + timelineMetricsTablesDurability); - validDurability = false; - } - if (validDurability) { - modifyTable = true; - } - } - } + //Set durability preferences + boolean durabilitySettingsModified = setDurabilityForTable(tableName, tableDescriptor); + modifyTable = modifyTable || durabilitySettingsModified; + + //Set compaction policy preferences + boolean compactionPolicyModified = false; + compactionPolicyModified = setCompactionPolicyForTable(tableName, tableDescriptor); + modifyTable = modifyTable || compactionPolicyModified; - Map<String, String> config = tableDescriptor.getConfiguration(); - if (enableFifoCompaction && - !FIFO_COMPACTION_POLICY_CLASS.equals(config.get(HSTORE_COMPACTION_CLASS_KEY))) { - tableDescriptor.setConfiguration(HSTORE_COMPACTION_CLASS_KEY, - FIFO_COMPACTION_POLICY_CLASS); - LOG.info("Setting config property " + HSTORE_COMPACTION_CLASS_KEY + - " = " + FIFO_COMPACTION_POLICY_CLASS + " for " + tableName); - // Need to set blockingStoreFiles to 1000 for FIFO - int blockingStoreFiles = hbaseConf.getInt(HBASE_BLOCKING_STORE_FILES, 1000); - if (blockingStoreFiles < 1000) { - blockingStoreFiles = 1000; - } - tableDescriptor.setConfiguration(BLOCKING_STORE_FILES_KEY, String.valueOf(blockingStoreFiles)); - LOG.info("Setting config property " + BLOCKING_STORE_FILES_KEY + - " = " + blockingStoreFiles + " for " + tableName); - modifyTable = true; - } - // Set back original policy if fifo disabled - if (!enableFifoCompaction && - FIFO_COMPACTION_POLICY_CLASS.equals(config.get(HSTORE_COMPACTION_CLASS_KEY))) { - tableDescriptor.setConfiguration(HSTORE_COMPACTION_CLASS_KEY, - DEFAULT_COMPACTION_POLICY_CLASS); - LOG.info("Setting config property " + HSTORE_COMPACTION_CLASS_KEY + - " = " + DEFAULT_COMPACTION_POLICY_CLASS + " for " + tableName); - - int blockingStoreFiles = hbaseConf.getInt(HBASE_BLOCKING_STORE_FILES, 300); - if (blockingStoreFiles > 300) { - LOG.warn("HBase blocking store file set too high without FIFO " + - "Compaction policy enabled, restoring low value = 300."); - blockingStoreFiles = 300; - } - tableDescriptor.setConfiguration(BLOCKING_STORE_FILES_KEY, String.valueOf(blockingStoreFiles)); - LOG.info("Setting config property " + BLOCKING_STORE_FILES_KEY + - " = " + blockingStoreFiles + " for " + tableName); - modifyTable = true; - } // Change TTL setting to match user configuration HColumnDescriptor[] columnFamilies = tableDescriptor.getColumnFamilies(); if (columnFamilies != null) { @@ -635,6 +575,103 @@ public class PhoenixHBaseAccessor { } } + private boolean setDurabilityForTable(String tableName, HTableDescriptor tableDescriptor) { + + boolean modifyTable = false; + //Set WAL preferences + if (METRICS_RECORD_TABLE_NAME.equals(tableName)) { + if (!timelineMetricsPrecisionTableDurability.isEmpty()) { + LOG.info("Setting WAL option " + timelineMetricsPrecisionTableDurability + " for table : " + tableName); + boolean validDurability = true; + if ("SKIP_WAL".equals(timelineMetricsPrecisionTableDurability)) { + tableDescriptor.setDurability(Durability.SKIP_WAL); + } else if ("SYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) { + tableDescriptor.setDurability(Durability.SYNC_WAL); + } else if ("ASYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) { + tableDescriptor.setDurability(Durability.ASYNC_WAL); + } else if ("FSYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) { + tableDescriptor.setDurability(Durability.FSYNC_WAL); + } else { + LOG.info("Unknown value for " + TIMELINE_METRICS_PRECISION_TABLE_DURABILITY + " : " + timelineMetricsPrecisionTableDurability); + validDurability = false; + } + if (validDurability) { + modifyTable = true; + } + } + } else { + if (!timelineMetricsTablesDurability.isEmpty()) { + LOG.info("Setting WAL option " + timelineMetricsTablesDurability + " for table : " + tableName); + boolean validDurability = true; + if ("SKIP_WAL".equals(timelineMetricsTablesDurability)) { + tableDescriptor.setDurability(Durability.SKIP_WAL); + } else if ("SYNC_WAL".equals(timelineMetricsTablesDurability)) { + tableDescriptor.setDurability(Durability.SYNC_WAL); + } else if ("ASYNC_WAL".equals(timelineMetricsTablesDurability)) { + tableDescriptor.setDurability(Durability.ASYNC_WAL); + } else if ("FSYNC_WAL".equals(timelineMetricsTablesDurability)) { + tableDescriptor.setDurability(Durability.FSYNC_WAL); + } else { + LOG.info("Unknown value for " + TIMELINE_METRICS_AGGREGATE_TABLES_DURABILITY + " : " + timelineMetricsTablesDurability); + validDurability = false; + } + if (validDurability) { + modifyTable = true; + } + } + } + return modifyTable; + } + + private boolean setCompactionPolicyForTable(String tableName, HTableDescriptor tableDescriptor) { + + String compactionPolicyKey = metricsConf.get(TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_KEY, + HSTORE_ENGINE_CLASS); + String compactionPolicyClass = metricsConf.get(TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_CLASS, + DATE_TIERED_COMPACTION_POLICY); + int blockingStoreFiles = hbaseConf.getInt(TIMELINE_METRICS_AGGREGATE_TABLE_HBASE_BLOCKING_STORE_FILES, 60); + + if (tableName.equals(METRICS_RECORD_TABLE_NAME)) { + compactionPolicyKey = metricsConf.get(TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_KEY, + HSTORE_COMPACTION_CLASS_KEY); + compactionPolicyClass = metricsConf.get(TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_CLASS, + FIFO_COMPACTION_POLICY_CLASS); + blockingStoreFiles = hbaseConf.getInt(TIMELINE_METRICS_PRECISION_TABLE_HBASE_BLOCKING_STORE_FILES, 1000); + } + + Map<String, String> config = new HashMap(tableDescriptor.getConfiguration()); + + if (StringUtils.isEmpty(compactionPolicyKey) || StringUtils.isEmpty(compactionPolicyClass)) { + config.remove(HSTORE_COMPACTION_CLASS_KEY); + config.remove(HSTORE_ENGINE_CLASS); + //Default blockingStoreFiles = 300 + setHbaseBlockingStoreFiles(tableDescriptor, tableName, 300); + } else { + tableDescriptor.setConfiguration(compactionPolicyKey, compactionPolicyClass); + setHbaseBlockingStoreFiles(tableDescriptor, tableName, blockingStoreFiles); + } + + if (!compactionPolicyKey.equals(HSTORE_ENGINE_CLASS)) { + tableDescriptor.removeConfiguration(HSTORE_ENGINE_CLASS); + } + if (!compactionPolicyKey.equals(HSTORE_COMPACTION_CLASS_KEY)) { + tableDescriptor.removeConfiguration(HSTORE_COMPACTION_CLASS_KEY); + } + + Map<String, String> newConfig = tableDescriptor.getConfiguration(); + return !Maps.difference(config, newConfig).areEqual(); + } + + private void setHbaseBlockingStoreFiles(HTableDescriptor tableDescriptor, String tableName, int value) { + int blockingStoreFiles = hbaseConf.getInt(HBASE_BLOCKING_STORE_FILES, value); + if (blockingStoreFiles != value) { + blockingStoreFiles = value; + } + tableDescriptor.setConfiguration(BLOCKING_STORE_FILES_KEY, String.valueOf(value)); + LOG.info("Setting config property " + BLOCKING_STORE_FILES_KEY + + " = " + blockingStoreFiles + " for " + tableName); + } + protected String getSplitPointsStr(String splitPoints) { if (StringUtils.isEmpty(splitPoints.trim())) { return ""; http://git-wip-us.apache.org/repos/asf/ambari/blob/c2c72b9d/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java index b7b9968..72e895d 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java @@ -269,6 +269,24 @@ public class TimelineMetricConfiguration { public static final String TIMELINE_METRIC_AGGREGATION_SQL_FILTERS = "timeline.metrics.cluster.aggregation.sql.filters"; + public static final String TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_KEY = + "timeline.metrics.hbase.aggregate.table.compaction.policy.key"; + + public static final String TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_CLASS = + "timeline.metrics.hbase.aggregate.table.compaction.policy.class"; + + public static final String TIMELINE_METRICS_AGGREGATE_TABLE_HBASE_BLOCKING_STORE_FILES = + "timeline.metrics.aggregate.table.hbase.hstore.blockingStoreFiles"; + + public static final String TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_KEY = + "timeline.metrics.hbase.precision.table.compaction.policy.key"; + + public static final String TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_CLASS = + "timeline.metrics.hbase.precision.table.compaction.policy.class"; + + public static final String TIMELINE_METRICS_PRECISION_TABLE_HBASE_BLOCKING_STORE_FILES = + "timeline.metrics.precision.table.hbase.hstore.blockingStoreFiles"; + public static final String HOST_APP_ID = "HOST"; public static final String DEFAULT_INSTANCE_PORT = "12001"; @@ -425,9 +443,12 @@ public class TimelineMetricConfiguration { return defaultRpcAddress; } - public boolean isDistributedOperationModeEnabled() { + public boolean isDistributedCollectorModeDisabled() { try { - return getMetricsConf().get("timeline.metrics.service.operation.mode").equals("distributed"); + if (metricsConf != null) { + return Boolean.parseBoolean(metricsConf.get("timeline.metrics.service.distributed.collector.mode.disabled", "false")); + } + return false; } catch (Exception e) { return false; } http://git-wip-us.apache.org/repos/asf/ambari/blob/c2c72b9d/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java index 3bc1929..0087fd9 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java @@ -62,8 +62,10 @@ import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.ti import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricTestHelper.createEmptyTimelineMetric; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricTestHelper.createMetricHostAggregate; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricTestHelper.prepareSingleTimelineMetric; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor.DATE_TIERED_COMPACTION_POLICY; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor.FIFO_COMPACTION_POLICY_CLASS; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor.HSTORE_COMPACTION_CLASS_KEY; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor.HSTORE_ENGINE_CLASS; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.PHOENIX_TABLES; @@ -327,8 +329,6 @@ public class ITPhoenixHBaseAccessor extends AbstractMiniHBaseClusterTest { HTableDescriptor tableDescriptor = hBaseAdmin.getTableDescriptor(tableName.getBytes()); tableDescriptor.setNormalizationEnabled(true); Assert.assertTrue("Normalizer enabled.", tableDescriptor.isNormalizationEnabled()); - Assert.assertNull("Default compaction policy is null.", - tableDescriptor.getConfigurationValue(HSTORE_COMPACTION_CLASS_KEY)); for (HColumnDescriptor family : tableDescriptor.getColumnFamilies()) { if (tableName.equals(METRICS_RECORD_TABLE_NAME)) { @@ -352,7 +352,8 @@ public class ITPhoenixHBaseAccessor extends AbstractMiniHBaseClusterTest { // Verify expected policies are set boolean normalizerEnabled = false; - String compactionPolicy = null; + String precisionTableCompactionPolicy = null; + String aggregateTableCompactionPolicy = null; boolean tableDurabilitySet = false; for (int i = 0; i < 10; i++) { LOG.warn("Policy check retry : " + i); @@ -360,12 +361,15 @@ public class ITPhoenixHBaseAccessor extends AbstractMiniHBaseClusterTest { HTableDescriptor tableDescriptor = hBaseAdmin.getTableDescriptor(tableName.getBytes()); normalizerEnabled = tableDescriptor.isNormalizationEnabled(); tableDurabilitySet = (Durability.ASYNC_WAL.equals(tableDescriptor.getDurability())); - compactionPolicy = tableDescriptor.getConfigurationValue(HSTORE_COMPACTION_CLASS_KEY); + if (tableName.equals(METRICS_RECORD_TABLE_NAME)) { + precisionTableCompactionPolicy = tableDescriptor.getConfigurationValue(HSTORE_ENGINE_CLASS); + } else { + aggregateTableCompactionPolicy = tableDescriptor.getConfigurationValue(HSTORE_COMPACTION_CLASS_KEY); + } LOG.debug("Table: " + tableName + ", normalizerEnabled = " + normalizerEnabled); - LOG.debug("Table: " + tableName + ", compactionPolicy = " + compactionPolicy); // Best effort for 20 seconds - if (normalizerEnabled || compactionPolicy == null) { - Thread.sleep(2000l); + if (normalizerEnabled || (precisionTableCompactionPolicy == null && aggregateTableCompactionPolicy ==null)) { + Thread.sleep(20000l); } if (tableName.equals(METRICS_RECORD_TABLE_NAME)) { for (HColumnDescriptor family : tableDescriptor.getColumnFamilies()) { @@ -377,7 +381,8 @@ public class ITPhoenixHBaseAccessor extends AbstractMiniHBaseClusterTest { Assert.assertFalse("Normalizer disabled.", normalizerEnabled); Assert.assertTrue("Durability Set.", tableDurabilitySet); - Assert.assertEquals("FIFO compaction policy is set.", FIFO_COMPACTION_POLICY_CLASS, compactionPolicy); + Assert.assertEquals("FIFO compaction policy is set for METRIC_RECORD.", FIFO_COMPACTION_POLICY_CLASS, precisionTableCompactionPolicy); + Assert.assertEquals("FIFO compaction policy is set for aggregate tables", DATE_TIERED_COMPACTION_POLICY, aggregateTableCompactionPolicy); Assert.assertEquals("Precision TTL value not changed.", String.valueOf(2 * 86400), precisionTtl); hBaseAdmin.close(); http://git-wip-us.apache.org/repos/asf/ambari/blob/c2c72b9d/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java index bfab0fe..b0243b7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java @@ -381,6 +381,13 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog { if ("distributed".equals(amsSite.getProperties().get(AMS_MODE))) { isDistributed = true; } + + Map<String, String> amsSiteProperties = amsSite.getProperties(); + + if (amsSiteProperties != null && amsSiteProperties.containsKey("timeline.metrics.hbase.fifo.compaction.enabled")) { + LOG.info("Removing timeline.metrics.hbase.fifo.compaction.enabled from ams-site"); + removeConfigurationPropertiesFromCluster(cluster, AMS_SITE, Collections.singleton("timeline.metrics.hbase.fifo.compaction.enabled")); + } } if (isDistributed) { http://git-wip-us.apache.org/repos/asf/ambari/blob/c2c72b9d/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml index 45fe1a7..8e1671e 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml @@ -651,14 +651,6 @@ <on-ambari-upgrade add="true"/> </property> <property> - <name>timeline.metrics.hbase.fifo.compaction.enabled</name> - <value>true</value> - <description> - Enable Compaction policy for lower for Precision and Minute aggregate tables. - </description> - <on-ambari-upgrade add="true"/> - </property> - <property> <name>timeline.metrics.aggregators.skip.blockcache.enabled</name> <value>false</value> <description> http://git-wip-us.apache.org/repos/asf/ambari/blob/c2c72b9d/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java index a08b38b..274d7eb 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java @@ -67,6 +67,7 @@ import org.apache.ambari.server.state.Config; import org.apache.ambari.server.state.Service; import org.apache.ambari.server.state.stack.OsFamily; import org.easymock.Capture; +import org.easymock.CaptureType; import org.easymock.EasyMock; import org.easymock.EasyMockRunner; import org.easymock.EasyMockSupport; @@ -643,8 +644,16 @@ public class UpgradeCatalog250Test { Map<String, String> amsSite = new HashMap<String, String>() { { put("timeline.metrics.service.operation.mode", "distributed"); + put("timeline.metrics.hbase.fifo.compaction.enabled", "true"); } }; + + Map<String, String> newAmsSite = new HashMap<String, String>() { + { + put("timeline.metrics.service.operation.mode", "distributed"); + } + }; + EasyMockSupport easyMockSupport = new EasyMockSupport(); Config mockAmsHbaseSite = easyMockSupport.createNiceMock(Config.class); @@ -670,18 +679,23 @@ public class UpgradeCatalog250Test { .createNiceMock(); Injector injector2 = easyMockSupport.createNiceMock(Injector.class); - Capture<Map<String, String>> propertiesCapture = EasyMock.newCapture(); + Capture<Map<String, String>> propertiesCapture = EasyMock.newCapture(CaptureType.ALL); expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes(); expect(controller.getClusters()).andReturn(clusters).anyTimes(); expect(controller.createConfig(anyObject(Cluster.class), anyString(), capture(propertiesCapture), anyString(), - EasyMock.<Map<String, Map<String, String>>>anyObject())).andReturn(config).once(); + EasyMock.<Map<String, Map<String, String>>>anyObject())).andReturn(config).times(2); replay(controller, injector2); new UpgradeCatalog250(injector2).updateAMSConfigs(); easyMockSupport.verifyAll(); - Map<String, String> updatedProperties = propertiesCapture.getValue(); + assertTrue(propertiesCapture.getValues().size() == 2); + + Map<String, String> updatedProperties = propertiesCapture.getValues().get(0); + assertTrue(Maps.difference(newAmsSite, updatedProperties).areEqual()); + + updatedProperties = propertiesCapture.getValues().get(1); assertTrue(Maps.difference(newProperties, updatedProperties).areEqual()); }
