PHOENIX-4764 Cleanup metadata of child views for a base table that has been dropped (addendum)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/c1dce9cf Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/c1dce9cf Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/c1dce9cf Branch: refs/heads/4.x-cdh5.15 Commit: c1dce9cfaf78e316d223f84a50b28b7128449efa Parents: 26f6ee6 Author: Kadir <kozde...@salesforce.com> Authored: Wed Oct 31 00:47:26 2018 +0000 Committer: pboado <pedro.bo...@gmail.com> Committed: Sun Nov 25 22:09:23 2018 +0000 ---------------------------------------------------------------------- .../end2end/MigrateSystemTablesToSystemNamespaceIT.java | 4 ++-- .../phoenix/end2end/SystemCatalogCreationOnConnectionIT.java | 4 ++-- .../org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java | 4 +++- .../org/apache/phoenix/coprocessor/TaskRegionObserver.java | 7 +++++-- .../org/apache/phoenix/query/ConnectionQueryServicesImpl.java | 2 +- .../src/main/java/org/apache/phoenix/query/QueryServices.java | 2 ++ .../java/org/apache/phoenix/query/QueryServicesOptions.java | 3 ++- 7 files changed, 17 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1dce9cf/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java index 3f2d6f3..a26cfe5 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java @@ -64,10 +64,10 @@ public class MigrateSystemTablesToSystemNamespaceIT extends BaseTest { private static final Set<String> PHOENIX_SYSTEM_TABLES = new HashSet<>(Arrays.asList( "SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION", - "SYSTEM.MUTEX","SYSTEM.LOG", "SYSTEM.CHILD_LINK")); + "SYSTEM.MUTEX","SYSTEM.LOG", "SYSTEM.CHILD_LINK", "SYSTEM.TASK")); private static final Set<String> PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>( Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION", - "SYSTEM:MUTEX","SYSTEM:LOG", "SYSTEM:CHILD_LINK")); + "SYSTEM:MUTEX","SYSTEM:LOG", "SYSTEM:CHILD_LINK", "SYSTEM:TASK")); private static final String SCHEMA_NAME = "MIGRATETEST"; private static final String TABLE_NAME = SCHEMA_NAME + "." + MigrateSystemTablesToSystemNamespaceIT.class.getSimpleName().toUpperCase(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1dce9cf/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java index aa2d971..a1685c44 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java @@ -72,11 +72,11 @@ public class SystemCatalogCreationOnConnectionIT { private static final Set<String> PHOENIX_SYSTEM_TABLES = new HashSet<>(Arrays.asList( "SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION", - "SYSTEM.MUTEX", "SYSTEM.LOG", "SYSTEM.CHILD_LINK")); + "SYSTEM.MUTEX", "SYSTEM.LOG", "SYSTEM.CHILD_LINK", "SYSTEM.TASK")); private static final Set<String> PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>( Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION", - "SYSTEM:MUTEX", "SYSTEM:LOG", "SYSTEM:CHILD_LINK")); + "SYSTEM:MUTEX", "SYSTEM:LOG", "SYSTEM:CHILD_LINK", "SYSTEM:TASK")); private static class PhoenixSysCatCreationServices extends ConnectionQueryServicesImpl { http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1dce9cf/phoenix-core/src/it/java/org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java index 5c016f6..9c2b763 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java @@ -19,6 +19,7 @@ package org.apache.phoenix.monitoring; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.phoenix.coprocessor.TaskRegionObserver; import org.apache.phoenix.end2end.BaseUniqueNamesOwnClusterIT; import org.apache.phoenix.jdbc.PhoenixDriver; import org.apache.phoenix.query.QueryServices; @@ -53,7 +54,8 @@ public class BasePhoenixMetricsIT extends BaseUniqueNamesOwnClusterIT { @BeforeClass public static void doSetup() throws Exception { - Map<String, String> props = Maps.newHashMapWithExpectedSize(1); + Map<String, String> props = Maps.newHashMapWithExpectedSize(3); + props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE)); // Phoenix Global client metrics are enabled by default // Enable request metric collection at the driver level props.put(QueryServices.COLLECT_REQUEST_LEVEL_METRICS, String.valueOf(true)); http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1dce9cf/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java index ca71961..87f7414 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java @@ -68,8 +68,7 @@ public class TaskRegionObserver extends BaseRegionObserver { private long timeInterval = QueryServicesOptions.DEFAULT_TASK_HANDLING_INTERVAL_MS; private long timeMaxInterval = QueryServicesOptions.DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS; @GuardedBy("TaskRegionObserver.class") - // initial delay before the first task is handled - private static final long initialDelay = 10000; // 10 secs + private long initialDelay; @Override public void preClose(final ObserverContext<RegionCoprocessorEnvironment> c, @@ -88,6 +87,10 @@ public class TaskRegionObserver extends BaseRegionObserver { config.getLong( QueryServices.TASK_HANDLING_MAX_INTERVAL_MS_ATTRIB, QueryServicesOptions.DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS); + initialDelay = + config.getLong( + QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, + QueryServicesOptions.DEFAULT_TASK_HANDLING_INITIAL_DELAY_MS); } @Override http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1dce9cf/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index 6d098e6..eddaf14 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -1177,7 +1177,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement splits, isNamespaceMapped); if (!tableExist) { - if (isMetaTable && !isUpgradeRequired() && (!isAutoUpgradeEnabled || isDoNotUpgradePropSet)) { + if (SchemaUtil.isSystemTable(physicalTableName) && !isUpgradeRequired() && (!isAutoUpgradeEnabled || isDoNotUpgradePropSet)) { // Disallow creating the SYSTEM.CATALOG or SYSTEM:CATALOG HBase table throw new UpgradeRequiredException(); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1dce9cf/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java index d06f07c..a8bbc22 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java @@ -325,6 +325,8 @@ public interface QueryServices extends SQLCloseable { public static final String TASK_HANDLING_INTERVAL_MS_ATTRIB = "phoenix.task.handling.interval.ms"; // The maximum time for a task to stay in table SYSTEM.TASK public static final String TASK_HANDLING_MAX_INTERVAL_MS_ATTRIB = "phoenix.task.handling.maxInterval.ms"; + // The initial delay before the first task from table SYSTEM.TASK is handled + public static final String TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB = "phoenix.task.handling.initial.delay.ms"; /** * Get executor service used for parallel scans http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1dce9cf/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java index a6a73f7..d2f5bce 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java @@ -342,7 +342,8 @@ public class QueryServicesOptions { // default system task handling interval in milliseconds public static final long DEFAULT_TASK_HANDLING_INTERVAL_MS = 60*1000; // 1 min - public static final long DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS = 30*60*1000; // 30 minutes + public static final long DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS = 30*60*1000; // 30 min + public static final long DEFAULT_TASK_HANDLING_INITIAL_DELAY_MS = 10*1000; // 10 sec @SuppressWarnings("serial") public static final Set<String> DEFAULT_QUERY_SERVER_SKIP_WORDS = new HashSet<String>() {