HIVE-18715: Remove index support from metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich <k...@rxd.hu> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7c22d74c Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7c22d74c Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7c22d74c Branch: refs/heads/master Commit: 7c22d74c8d0eb0650adf6e84e0536127c103e46c Parents: 31e36f0 Author: Zoltan Haindrich <k...@rxd.hu> Authored: Wed Mar 7 08:51:03 2018 +0100 Committer: Zoltan Haindrich <k...@rxd.hu> Committed: Wed Mar 7 08:51:48 2018 +0100 ---------------------------------------------------------------------- .../hcatalog/mapreduce/HCatOutputFormat.java | 9 - .../listener/DbNotificationListener.java | 54 +- .../hcatalog/messaging/AlterIndexMessage.java | 35 - .../hcatalog/messaging/CreateIndexMessage.java | 35 - .../hcatalog/messaging/DropIndexMessage.java | 35 - .../hcatalog/messaging/HCatEventMessage.java | 14 +- .../hcatalog/messaging/MessageDeserializer.java | 23 - .../hive/hcatalog/messaging/MessageFactory.java | 25 - .../messaging/json/JSONAlterIndexMessage.java | 89 - .../messaging/json/JSONCreateIndexMessage.java | 82 - .../messaging/json/JSONDropIndexMessage.java | 82 - .../messaging/json/JSONMessageDeserializer.java | 33 - .../messaging/json/JSONMessageFactory.java | 48 +- .../listener/DummyRawStoreFailEvent.java | 48 - .../listener/TestDbNotificationListener.java | 253 +- .../org/apache/hadoop/hive/ql/QTestUtil.java | 5 - .../upgrade/hive/hive-schema-3.0.0.hive.sql | 48 - .../org/apache/hadoop/hive/ql/ErrorMsg.java | 1 - .../apache/hadoop/hive/ql/metadata/Table.java | 25 +- .../hive/ql/parse/DDLSemanticAnalyzer.java | 4 - .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 7 +- .../AuthorizationPreEventListener.java | 2 - .../hadoop/hive/ql/metadata/TestHive.java | 2 - .../clientpositive/llap/resourceplan.q.out | 104 - .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 9982 ++++++---------- .../gen/thrift/gen-cpp/ThriftHiveMetastore.h | 1318 +-- .../ThriftHiveMetastore_server.skeleton.cpp | 30 - .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 4871 ++++---- .../gen/thrift/gen-cpp/hive_metastore_types.h | 152 - .../hive/metastore/api/AbortTxnsRequest.java | 32 +- .../api/AddDefaultConstraintRequest.java | 36 +- .../metastore/api/AddDynamicPartitions.java | 32 +- .../metastore/api/AddForeignKeyRequest.java | 36 +- .../api/AddNotNullConstraintRequest.java | 36 +- .../metastore/api/AddPartitionsRequest.java | 36 +- .../hive/metastore/api/AddPartitionsResult.java | 36 +- .../metastore/api/AddPrimaryKeyRequest.java | 36 +- .../api/AddUniqueConstraintRequest.java | 36 +- .../hadoop/hive/metastore/api/AggrStats.java | 36 +- .../api/AllocateTableWriteIdsRequest.java | 32 +- .../api/AllocateTableWriteIdsResponse.java | 36 +- .../metastore/api/ClearFileMetadataRequest.java | 32 +- .../hive/metastore/api/ClientCapabilities.java | 32 +- .../hive/metastore/api/ColumnStatistics.java | 36 +- .../hive/metastore/api/CompactionRequest.java | 44 +- .../hive/metastore/api/CreationMetadata.java | 32 +- .../api/DefaultConstraintsResponse.java | 36 +- .../metastore/api/DropPartitionsResult.java | 36 +- .../hive/metastore/api/EnvironmentContext.java | 44 +- .../hive/metastore/api/FireEventRequest.java | 32 +- .../hive/metastore/api/ForeignKeysResponse.java | 36 +- .../hadoop/hive/metastore/api/Function.java | 36 +- .../metastore/api/GetAllFunctionsResponse.java | 36 +- .../api/GetFileMetadataByExprRequest.java | 32 +- .../api/GetFileMetadataByExprResult.java | 48 +- .../metastore/api/GetFileMetadataRequest.java | 32 +- .../metastore/api/GetFileMetadataResult.java | 44 +- .../metastore/api/GetOpenTxnsInfoResponse.java | 36 +- .../hive/metastore/api/GetOpenTxnsResponse.java | 32 +- .../hive/metastore/api/GetTablesRequest.java | 32 +- .../hive/metastore/api/GetTablesResult.java | 36 +- .../metastore/api/GetValidWriteIdsRequest.java | 32 +- .../metastore/api/GetValidWriteIdsResponse.java | 36 +- .../api/HeartbeatTxnRangeResponse.java | 64 +- .../apache/hadoop/hive/metastore/api/Index.java | 1374 --- .../api/IndexAlreadyExistsException.java | 395 - .../metastore/api/InsertEventRequestData.java | 64 +- .../hadoop/hive/metastore/api/LockRequest.java | 36 +- .../hive/metastore/api/Materialization.java | 32 +- .../api/NotNullConstraintsResponse.java | 36 +- .../api/NotificationEventResponse.java | 36 +- .../hive/metastore/api/OpenTxnsResponse.java | 32 +- .../metastore/api/PartitionValuesRequest.java | 72 +- .../metastore/api/PartitionValuesResponse.java | 36 +- .../hive/metastore/api/PartitionValuesRow.java | 32 +- .../metastore/api/PartitionsByExprResult.java | 36 +- .../metastore/api/PartitionsStatsRequest.java | 64 +- .../metastore/api/PartitionsStatsResult.java | 76 +- .../hive/metastore/api/PrimaryKeysResponse.java | 36 +- .../metastore/api/PutFileMetadataRequest.java | 64 +- .../hive/metastore/api/RequestPartsSpec.java | 68 +- .../hadoop/hive/metastore/api/Schema.java | 80 +- .../api/SetPartitionsStatsRequest.java | 36 +- .../hive/metastore/api/ShowCompactResponse.java | 36 +- .../hive/metastore/api/ShowLocksResponse.java | 36 +- .../hive/metastore/api/TableStatsRequest.java | 32 +- .../hive/metastore/api/TableStatsResult.java | 36 +- .../hive/metastore/api/TableValidWriteIds.java | 32 +- .../hive/metastore/api/ThriftHiveMetastore.java | 10346 ++--------------- .../api/UniqueConstraintsResponse.java | 36 +- .../hive/metastore/api/WMFullResourcePlan.java | 144 +- .../api/WMGetAllResourcePlanResponse.java | 36 +- .../WMGetTriggersForResourePlanResponse.java | 36 +- .../api/WMValidateResourcePlanResponse.java | 64 +- .../gen-php/metastore/ThriftHiveMetastore.php | 4545 ++------ .../src/gen/thrift/gen-php/metastore/Types.php | 1890 ++- .../hive_metastore/ThriftHiveMetastore-remote | 42 - .../hive_metastore/ThriftHiveMetastore.py | 3882 ++----- .../gen/thrift/gen-py/hive_metastore/ttypes.py | 1255 +- .../gen/thrift/gen-rb/hive_metastore_types.rb | 55 - .../gen/thrift/gen-rb/thrift_hive_metastore.rb | 419 - .../DefaultMetaStoreFilterHookImpl.java | 16 - .../hadoop/hive/metastore/HiveMetaStore.java | 420 +- .../hive/metastore/HiveMetaStoreClient.java | 154 +- .../hadoop/hive/metastore/IMetaStoreClient.java | 75 +- .../hive/metastore/MetaStoreEventListener.java | 24 - .../hive/metastore/MetaStoreFilterHook.java | 18 - .../metastore/MetaStoreListenerNotifier.java | 21 - .../hadoop/hive/metastore/ObjectStore.java | 223 +- .../apache/hadoop/hive/metastore/RawStore.java | 17 - .../apache/hadoop/hive/metastore/TableType.java | 2 +- .../hive/metastore/cache/CachedStore.java | 117 +- .../metastore/client/builder/IndexBuilder.java | 109 - .../metastore/client/builder/TableBuilder.java | 21 +- .../hive/metastore/events/AddIndexEvent.java | 40 - .../hive/metastore/events/AlterIndexEvent.java | 46 - .../hive/metastore/events/DropIndexEvent.java | 40 - .../hive/metastore/events/PreAddIndexEvent.java | 41 - .../metastore/events/PreAlterIndexEvent.java | 47 - .../metastore/events/PreDropIndexEvent.java | 40 - .../hive/metastore/events/PreEventContext.java | 3 - .../metastore/messaging/AlterIndexMessage.java | 33 - .../metastore/messaging/CreateIndexMessage.java | 46 - .../metastore/messaging/DropIndexMessage.java | 46 - .../hive/metastore/messaging/EventMessage.java | 13 +- .../messaging/MessageDeserializer.java | 21 - .../metastore/messaging/MessageFactory.java | 26 - .../messaging/json/JSONAlterIndexMessage.java | 98 - .../messaging/json/JSONCreateIndexMessage.java | 87 - .../messaging/json/JSONDropIndexMessage.java | 90 - .../messaging/json/JSONMessageDeserializer.java | 33 - .../messaging/json/JSONMessageFactory.java | 27 +- .../hive/metastore/utils/MetaStoreUtils.java | 43 +- .../src/main/thrift/hive_metastore.thrift | 32 - .../hadoop/hive/metastore/DummyListener.java | 18 - .../DummyRawStoreControlledCommit.java | 36 - .../DummyRawStoreForJdoConnection.java | 49 +- .../hadoop/hive/metastore/TestFilterHooks.java | 51 +- .../metastore/TestMetaStoreEventListener.java | 85 - .../TestMetaStoreEventListenerWithOldConf.java | 54 +- .../hadoop/hive/metastore/TestObjectStore.java | 7 - .../client/TestAddAlterDropIndexes.java | 845 -- .../hive/metastore/client/TestDatabases.java | 70 - .../metastore/client/TestGetListIndexes.java | 442 - 144 files changed, 12422 insertions(+), 35523 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatOutputFormat.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatOutputFormat.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatOutputFormat.java index 166b565..8b75782 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatOutputFormat.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatOutputFormat.java @@ -35,7 +35,6 @@ import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.io.WritableComparable; @@ -92,14 +91,6 @@ public class HCatOutputFormat extends HCatBaseOutputFormat { Table table = HCatUtil.getTable(client, outputJobInfo.getDatabaseName(), outputJobInfo.getTableName()); - List<String> indexList = client.listIndexNames(outputJobInfo.getDatabaseName(), outputJobInfo.getTableName(), Short.MAX_VALUE); - - for (String indexName : indexList) { - Index index = client.getIndex(outputJobInfo.getDatabaseName(), outputJobInfo.getTableName(), indexName); - if (!index.isDeferredRebuild()) { - throw new HCatException(ErrorType.ERROR_NOT_SUPPORTED, "Store into a table with an automatic index from Pig/Mapreduce is not supported"); - } - } StorageDescriptor sd = table.getTTable().getSd(); if (sd.isCompressed()) { http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java index 39d97fc..d647181 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java @@ -36,7 +36,6 @@ import org.apache.hadoop.hive.metastore.TransactionalMetaStoreEventListener; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Function; -import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NotificationEvent; import org.apache.hadoop.hive.metastore.api.Partition; @@ -48,13 +47,11 @@ import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hadoop.hive.metastore.events.AddForeignKeyEvent; -import org.apache.hadoop.hive.metastore.events.AddIndexEvent; import org.apache.hadoop.hive.metastore.events.AddNotNullConstraintEvent; import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; import org.apache.hadoop.hive.metastore.events.AddPrimaryKeyEvent; import org.apache.hadoop.hive.metastore.events.AddUniqueConstraintEvent; import org.apache.hadoop.hive.metastore.events.AlterDatabaseEvent; -import org.apache.hadoop.hive.metastore.events.AlterIndexEvent; import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; import org.apache.hadoop.hive.metastore.events.AlterTableEvent; import org.apache.hadoop.hive.metastore.events.ConfigChangeEvent; @@ -64,7 +61,6 @@ import org.apache.hadoop.hive.metastore.events.CreateTableEvent; import org.apache.hadoop.hive.metastore.events.DropConstraintEvent; import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; import org.apache.hadoop.hive.metastore.events.DropFunctionEvent; -import org.apache.hadoop.hive.metastore.events.DropIndexEvent; import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; import org.apache.hadoop.hive.metastore.events.InsertEvent; @@ -390,49 +386,6 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener process(event, fnEvent); } - /** - * @param indexEvent index event - * @throws MetaException - */ - @Override - public void onAddIndex(AddIndexEvent indexEvent) throws MetaException { - Index index = indexEvent.getIndex(); - NotificationEvent event = - new NotificationEvent(0, now(), EventType.CREATE_INDEX.toString(), msgFactory - .buildCreateIndexMessage(index).toString()); - event.setDbName(index.getDbName()); - process(event, indexEvent); - } - - /** - * @param indexEvent index event - * @throws MetaException - */ - @Override - public void onDropIndex(DropIndexEvent indexEvent) throws MetaException { - Index index = indexEvent.getIndex(); - NotificationEvent event = - new NotificationEvent(0, now(), EventType.DROP_INDEX.toString(), msgFactory - .buildDropIndexMessage(index).toString()); - event.setDbName(index.getDbName()); - process(event, indexEvent); - } - - /** - * @param indexEvent index event - * @throws MetaException - */ - @Override - public void onAlterIndex(AlterIndexEvent indexEvent) throws MetaException { - Index before = indexEvent.getOldIndex(); - Index after = indexEvent.getNewIndex(); - NotificationEvent event = - new NotificationEvent(0, now(), EventType.ALTER_INDEX.toString(), msgFactory - .buildAlterIndexMessage(before, after).toString()); - event.setDbName(before.getDbName()); - process(event, indexEvent); - } - class FileChksumIterator implements Iterator<String> { private List<String> files; private List<String> chksums; @@ -625,8 +578,11 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener } public void setTimeToLive(long configTtl) { - if (configTtl > Integer.MAX_VALUE) ttl = Integer.MAX_VALUE; - else ttl = (int)configTtl; + if (configTtl > Integer.MAX_VALUE) { + ttl = Integer.MAX_VALUE; + } else { + ttl = (int)configTtl; + } } } http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterIndexMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterIndexMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterIndexMessage.java deleted file mode 100644 index d3b0cc6..0000000 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/AlterIndexMessage.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.messaging; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; - -/** - * HCat message sent when a Index is altered in HCatalog. - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public abstract class AlterIndexMessage extends HCatEventMessage { - - protected AlterIndexMessage() { - super(EventType.ALTER_INDEX); - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/CreateIndexMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/CreateIndexMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/CreateIndexMessage.java deleted file mode 100644 index 79608ea..0000000 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/CreateIndexMessage.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.messaging; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; - -/** - * HCat message sent when a Index is created in HCatalog. - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public abstract class CreateIndexMessage extends HCatEventMessage { - - protected CreateIndexMessage() { - super(EventType.CREATE_INDEX); - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/DropIndexMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/DropIndexMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/DropIndexMessage.java deleted file mode 100644 index 2be1d07..0000000 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/DropIndexMessage.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.messaging; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; - -/** - * HCat message sent when a Index is dropped in HCatalog. - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public abstract class DropIndexMessage extends HCatEventMessage { - - protected DropIndexMessage() { - super(EventType.DROP_INDEX); - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/HCatEventMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/HCatEventMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/HCatEventMessage.java index ca2253c..8bfdf3d 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/HCatEventMessage.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/HCatEventMessage.java @@ -46,10 +46,7 @@ public abstract class HCatEventMessage { ALTER_PARTITION(HCatConstants.HCAT_ALTER_PARTITION_EVENT), INSERT(HCatConstants.HCAT_INSERT_EVENT), CREATE_FUNCTION(HCatConstants.HCAT_CREATE_FUNCTION_EVENT), - DROP_FUNCTION(HCatConstants.HCAT_DROP_FUNCTION_EVENT), - CREATE_INDEX(HCatConstants.HCAT_CREATE_INDEX_EVENT), - DROP_INDEX(HCatConstants.HCAT_DROP_INDEX_EVENT), - ALTER_INDEX(HCatConstants.HCAT_ALTER_INDEX_EVENT); + DROP_FUNCTION(HCatConstants.HCAT_DROP_FUNCTION_EVENT); private String typeString; @@ -100,12 +97,15 @@ public abstract class HCatEventMessage { * Class invariant. Checked after construction or deserialization. */ public HCatEventMessage checkValid() { - if (getServer() == null || getServicePrincipal() == null) + if (getServer() == null || getServicePrincipal() == null) { throw new IllegalStateException("Server-URL/Service-Principal shouldn't be null."); - if (getEventType() == null) + } + if (getEventType() == null) { throw new IllegalStateException("Event-type unset."); - if (getDB() == null) + } + if (getDB() == null) { throw new IllegalArgumentException("DB-name unset."); + } return this; } } http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageDeserializer.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageDeserializer.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageDeserializer.java index 61178cb..857b89f 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageDeserializer.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageDeserializer.java @@ -21,8 +21,6 @@ package org.apache.hive.hcatalog.messaging; import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hive.hcatalog.messaging.json.JSONCreateFunctionMessage; -import org.apache.hive.hcatalog.messaging.json.JSONDropFunctionMessage; /** * Interface for converting HCat events from String-form back to HCatEventMessage instances. @@ -57,12 +55,6 @@ public abstract class MessageDeserializer { return getCreateFunctionMessage(messageBody); case DROP_FUNCTION: return getDropFunctionMessage(messageBody); - case CREATE_INDEX: - return getCreateIndexMessage(messageBody); - case DROP_INDEX: - return getDropIndexMessage(messageBody); - case ALTER_INDEX: - return getAlterIndexMessage(messageBody); case INSERT: return getInsertMessage(messageBody); default: @@ -125,21 +117,6 @@ public abstract class MessageDeserializer { public abstract DropFunctionMessage getDropFunctionMessage(String messageBody); /** - * Method to de-serialize CreateIndexMessage instance. - */ - public abstract CreateIndexMessage getCreateIndexMessage(String messageBody); - - /** - * Method to de-serialize DropIndexMessage instance. - */ - public abstract DropIndexMessage getDropIndexMessage(String messageBody); - - /** - * Method to de-serialize AlterIndexMessage instance. - */ - public abstract AlterIndexMessage getAlterIndexMessage(String messageBody); - - /** * Method to deserialize InsertMessage * @param messageBody the message in serialized form * @return message in object form http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java index 6480c5e..a248bbf 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java @@ -25,12 +25,9 @@ import org.apache.hadoop.hive.common.classification.InterfaceStability; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Function; -import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.util.ReflectionUtils; -import org.apache.hive.hcatalog.messaging.json.JSONMessageFactory; - import java.util.Iterator; import java.util.List; import java.util.Map; @@ -183,28 +180,6 @@ public abstract class MessageFactory { public abstract DropFunctionMessage buildDropFunctionMessage(Function fn); /** - * Factory method for CreateIndexMessage. - * @param idx The Index being added. - * @return CreateIndexMessage instance. - */ - public abstract CreateIndexMessage buildCreateIndexMessage(Index idx); - - /** - * Factory method for DropIndexMessage. - * @param idx The Index being dropped. - * @return DropIndexMessage instance. - */ - public abstract DropIndexMessage buildDropIndexMessage(Index idx); - - /** - * Factory method for AlterIndexMessage. - * @param before The index before the alter - * @param after The index after the alter - * @return AlterIndexMessage - */ - public abstract AlterIndexMessage buildAlterIndexMessage(Index before, Index after); - - /** * Factory method for building insert message * @param db Name of the database the insert occurred in * @param table Name of the table the insert occurred in http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterIndexMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterIndexMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterIndexMessage.java deleted file mode 100644 index d7da100..0000000 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONAlterIndexMessage.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.messaging.json; - -import org.apache.hadoop.hive.metastore.api.Index; -import org.apache.hive.hcatalog.messaging.AlterIndexMessage; -import org.apache.thrift.TException; - -import org.codehaus.jackson.annotate.JsonProperty; - -/** - * JSON Implementation of AlterIndexMessage. - */ -public class JSONAlterIndexMessage extends AlterIndexMessage { - - @JsonProperty - String server, servicePrincipal, db, beforeIndexObjJson, afterIndexObjJson; - - @JsonProperty - Long timestamp; - - /** - * Default constructor, required for Jackson. - */ - public JSONAlterIndexMessage() {} - - public JSONAlterIndexMessage(String server, String servicePrincipal, Index before, Index after, - Long timestamp) { - this.server = server; - this.servicePrincipal = servicePrincipal; - this.db = after.getDbName(); - this.timestamp = timestamp; - try { - this.beforeIndexObjJson = JSONMessageFactory.createIndexObjJson(before); - this.afterIndexObjJson = JSONMessageFactory.createIndexObjJson(after); - } catch (TException ex) { - throw new IllegalArgumentException("Could not serialize Index object", ex); - } - - checkValid(); - } - - @Override - public String getDB() { return db; } - - @Override - public String getServer() { return server; } - - @Override - public String getServicePrincipal() { return servicePrincipal; } - - @Override - public Long getTimestamp() { return timestamp; } - - public String getBeforeIndexObjJson() { - return beforeIndexObjJson; - } - - public String getAfterIndexObjJson() { - return afterIndexObjJson; - } - - @Override - public String toString() { - try { - return JSONMessageDeserializer.mapper.writeValueAsString(this); - } - catch (Exception exception) { - throw new IllegalArgumentException("Could not serialize: ", exception); - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONCreateIndexMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONCreateIndexMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONCreateIndexMessage.java deleted file mode 100644 index 094913b..0000000 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONCreateIndexMessage.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.messaging.json; - -import org.apache.hadoop.hive.metastore.api.Index; -import org.apache.hive.hcatalog.messaging.CreateIndexMessage; -import org.apache.thrift.TException; -import org.codehaus.jackson.annotate.JsonProperty; - -/** - * JSON Implementation of CreateIndexMessage. - */ -public class JSONCreateIndexMessage extends CreateIndexMessage { - - @JsonProperty - String server, servicePrincipal, db, indexObjJson; - - @JsonProperty - Long timestamp; - - /** - * Default constructor, required for Jackson. - */ - public JSONCreateIndexMessage() {} - - public JSONCreateIndexMessage(String server, String servicePrincipal, Index index, Long timestamp) { - this.server = server; - this.servicePrincipal = servicePrincipal; - this.db = index.getDbName(); - try { - this.indexObjJson = JSONMessageFactory.createIndexObjJson(index); - } catch (TException ex) { - throw new IllegalArgumentException("Could not serialize Index object", ex); - } - - this.timestamp = timestamp; - checkValid(); - } - - @Override - public String getDB() { return db; } - - @Override - public String getServer() { return server; } - - @Override - public String getServicePrincipal() { return servicePrincipal; } - - @Override - public Long getTimestamp() { return timestamp; } - - public String getIndexObjJson() { - return indexObjJson; - } - - @Override - public String toString() { - try { - return JSONMessageDeserializer.mapper.writeValueAsString(this); - } - catch (Exception exception) { - throw new IllegalArgumentException("Could not serialize: ", exception); - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropIndexMessage.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropIndexMessage.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropIndexMessage.java deleted file mode 100644 index c80bd29..0000000 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONDropIndexMessage.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.hcatalog.messaging.json; - -import org.apache.hadoop.hive.metastore.api.Index; -import org.apache.hive.hcatalog.messaging.DropIndexMessage; -import org.apache.thrift.TException; -import org.codehaus.jackson.annotate.JsonProperty; - -/** - * JSON Implementation of DropIndexMessage. - */ -public class JSONDropIndexMessage extends DropIndexMessage { - - @JsonProperty - String server, servicePrincipal, db, indexObjJson; - - @JsonProperty - Long timestamp; - - /** - * Default constructor, required for Jackson. - */ - public JSONDropIndexMessage() {} - - public JSONDropIndexMessage(String server, String servicePrincipal, Index index, Long timestamp) { - this.server = server; - this.servicePrincipal = servicePrincipal; - this.db = index.getDbName(); - try { - this.indexObjJson = JSONMessageFactory.createIndexObjJson(index); - } catch (TException ex) { - throw new IllegalArgumentException("Could not serialize Index object", ex); - } - - this.timestamp = timestamp; - checkValid(); - } - - @Override - public String getDB() { return db; } - - @Override - public String getServer() { return server; } - - @Override - public String getServicePrincipal() { return servicePrincipal; } - - @Override - public Long getTimestamp() { return timestamp; } - - public String getIndexObjJson() { - return indexObjJson; - } - - @Override - public String toString() { - try { - return JSONMessageDeserializer.mapper.writeValueAsString(this); - } - catch (Exception exception) { - throw new IllegalArgumentException("Could not serialize: ", exception); - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageDeserializer.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageDeserializer.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageDeserializer.java index bb8c849..ab26a14 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageDeserializer.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageDeserializer.java @@ -20,16 +20,13 @@ package org.apache.hive.hcatalog.messaging.json; import org.apache.hive.hcatalog.messaging.AddPartitionMessage; -import org.apache.hive.hcatalog.messaging.AlterIndexMessage; import org.apache.hive.hcatalog.messaging.AlterPartitionMessage; import org.apache.hive.hcatalog.messaging.AlterTableMessage; import org.apache.hive.hcatalog.messaging.CreateDatabaseMessage; import org.apache.hive.hcatalog.messaging.CreateFunctionMessage; -import org.apache.hive.hcatalog.messaging.CreateIndexMessage; import org.apache.hive.hcatalog.messaging.CreateTableMessage; import org.apache.hive.hcatalog.messaging.DropDatabaseMessage; import org.apache.hive.hcatalog.messaging.DropFunctionMessage; -import org.apache.hive.hcatalog.messaging.DropIndexMessage; import org.apache.hive.hcatalog.messaging.DropPartitionMessage; import org.apache.hive.hcatalog.messaging.DropTableMessage; import org.apache.hive.hcatalog.messaging.InsertMessage; @@ -149,36 +146,6 @@ public class JSONMessageDeserializer extends MessageDeserializer { } @Override - public CreateIndexMessage getCreateIndexMessage(String messageBody) { - try { - return mapper.readValue(messageBody, JSONCreateIndexMessage.class); - } - catch (Exception exception) { - throw new IllegalArgumentException("Could not construct JSONCreateIndexMessage.", exception); - } - } - - @Override - public DropIndexMessage getDropIndexMessage(String messageBody) { - try { - return mapper.readValue(messageBody, JSONDropIndexMessage.class); - } - catch (Exception exception) { - throw new IllegalArgumentException("Could not construct JSONDropIndexMessage.", exception); - } - } - - @Override - public AlterIndexMessage getAlterIndexMessage(String messageBody) { - try { - return mapper.readValue(messageBody, JSONAlterIndexMessage.class); - } - catch (Exception exception) { - throw new IllegalArgumentException("Could not construct JSONAlterIndexMessage.", exception); - } - } - - @Override public InsertMessage getInsertMessage(String messageBody) { try { return mapper.readValue(messageBody, JSONInsertMessage.class); http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java index 345934b..0973c18 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java @@ -19,15 +19,13 @@ package org.apache.hive.hcatalog.messaging.json; -import com.google.common.collect.Iterators; -import com.google.common.collect.Lists; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nullable; -import org.apache.hadoop.hive.metastore.api.Index; -import org.apache.hive.hcatalog.messaging.AlterIndexMessage; -import org.apache.hive.hcatalog.messaging.CreateFunctionMessage; -import org.apache.hive.hcatalog.messaging.CreateIndexMessage; -import org.apache.hive.hcatalog.messaging.DropFunctionMessage; -import org.apache.hive.hcatalog.messaging.DropIndexMessage; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.api.Partition; @@ -36,8 +34,10 @@ import org.apache.hive.hcatalog.messaging.AddPartitionMessage; import org.apache.hive.hcatalog.messaging.AlterPartitionMessage; import org.apache.hive.hcatalog.messaging.AlterTableMessage; import org.apache.hive.hcatalog.messaging.CreateDatabaseMessage; +import org.apache.hive.hcatalog.messaging.CreateFunctionMessage; import org.apache.hive.hcatalog.messaging.CreateTableMessage; import org.apache.hive.hcatalog.messaging.DropDatabaseMessage; +import org.apache.hive.hcatalog.messaging.DropFunctionMessage; import org.apache.hive.hcatalog.messaging.DropPartitionMessage; import org.apache.hive.hcatalog.messaging.DropTableMessage; import org.apache.hive.hcatalog.messaging.InsertMessage; @@ -49,11 +49,8 @@ import org.apache.thrift.protocol.TJSONProtocol; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nullable; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; +import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; /** * The JSON implementation of the MessageFactory. Constructs JSON implementations of @@ -144,24 +141,6 @@ public class JSONMessageFactory extends MessageFactory { } @Override - public CreateIndexMessage buildCreateIndexMessage(Index idx) { - return new JSONCreateIndexMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, idx, - now()); - } - - @Override - public DropIndexMessage buildDropIndexMessage(Index idx) { - return new JSONDropIndexMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, idx, - now()); - } - - @Override - public AlterIndexMessage buildAlterIndexMessage(Index before, Index after) { - return new JSONAlterIndexMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before, after, - now()); - } - - @Override public InsertMessage buildInsertMessage(String db, String table, Map<String,String> partKeyVals, List<String> files) { return new JSONInsertMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db, table, null, @@ -181,9 +160,10 @@ public class JSONMessageFactory extends MessageFactory { private static Map<String, String> getPartitionKeyValues(Table table, Partition partition) { Map<String, String> partitionKeys = new LinkedHashMap<String, String>(); - for (int i=0; i<table.getPartitionKeysSize(); ++i) + for (int i=0; i<table.getPartitionKeysSize(); ++i) { partitionKeys.put(table.getPartitionKeys().get(i).getName(), partition.getValues().get(i)); + } return partitionKeys; } @@ -201,8 +181,4 @@ public class JSONMessageFactory extends MessageFactory { return serializer.toString(functionObj, "UTF-8"); } - static String createIndexObjJson(Index indexObj) throws TException { - TSerializer serializer = new TSerializer(new TJSONProtocol.Factory()); - return serializer.toString(indexObj, "UTF-8"); - } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java ---------------------------------------------------------------------- diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java index d0ff871..4ecd89c 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java @@ -40,7 +40,6 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.FileMetadataExprType; import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege; -import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.InvalidInputException; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; @@ -339,53 +338,6 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable { } @Override - public boolean addIndex(Index index) throws InvalidObjectException, MetaException { - if (shouldEventSucceed) { - return objectStore.addIndex(index); - } else { - throw new RuntimeException("Event failed."); - } - } - - @Override - public Index getIndex(String dbName, String origTableName, String indexName) - throws MetaException { - return objectStore.getIndex(dbName, origTableName, indexName); - } - - @Override - public boolean dropIndex(String dbName, String origTableName, String indexName) - throws MetaException { - if (shouldEventSucceed) { - return objectStore.dropIndex(dbName, origTableName, indexName); - } else { - throw new RuntimeException("Event failed."); - } - } - - @Override - public List<Index> getIndexes(String dbName, String origTableName, int max) - throws MetaException { - return objectStore.getIndexes(dbName, origTableName, max); - } - - @Override - public List<String> listIndexNames(String dbName, String origTableName, short max) - throws MetaException { - return objectStore.listIndexNames(dbName, origTableName, max); - } - - @Override - public void alterIndex(String dbName, String baseTblName, String name, Index newIndex) - throws InvalidObjectException, MetaException { - if (shouldEventSucceed) { - objectStore.alterIndex(dbName, baseTblName, name, newIndex); - } else { - throw new RuntimeException("Event failed."); - } - } - - @Override public List<Partition> getPartitionsByFilter(String dbName, String tblName, String filter, short maxParts) throws MetaException, NoSuchObjectException { return objectStore.getPartitionsByFilter(dbName, tblName, filter, maxParts); http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java ---------------------------------------------------------------------- diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java index 7aadf18..e0e2965 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java @@ -28,7 +28,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Stack; @@ -49,12 +48,10 @@ import org.apache.hadoop.hive.metastore.api.FireEventRequest; import org.apache.hadoop.hive.metastore.api.FireEventRequestData; import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.api.FunctionType; -import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.InsertEventRequestData; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NotificationEvent; import org.apache.hadoop.hive.metastore.api.NotificationEventResponse; -import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.ResourceType; @@ -62,9 +59,7 @@ import org.apache.hadoop.hive.metastore.api.ResourceUri; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.events.AddIndexEvent; import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; -import org.apache.hadoop.hive.metastore.events.AlterIndexEvent; import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; import org.apache.hadoop.hive.metastore.events.AlterTableEvent; import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; @@ -72,22 +67,18 @@ import org.apache.hadoop.hive.metastore.events.CreateFunctionEvent; import org.apache.hadoop.hive.metastore.events.CreateTableEvent; import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; import org.apache.hadoop.hive.metastore.events.DropFunctionEvent; -import org.apache.hadoop.hive.metastore.events.DropIndexEvent; import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; import org.apache.hadoop.hive.metastore.events.InsertEvent; import org.apache.hadoop.hive.metastore.events.ListenerEvent; import org.apache.hadoop.hive.metastore.messaging.AddPartitionMessage; -import org.apache.hadoop.hive.metastore.messaging.AlterIndexMessage; import org.apache.hadoop.hive.metastore.messaging.AlterPartitionMessage; import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage; import org.apache.hadoop.hive.metastore.messaging.CreateDatabaseMessage; import org.apache.hadoop.hive.metastore.messaging.CreateFunctionMessage; -import org.apache.hadoop.hive.metastore.messaging.CreateIndexMessage; import org.apache.hadoop.hive.metastore.messaging.CreateTableMessage; import org.apache.hadoop.hive.metastore.messaging.DropDatabaseMessage; import org.apache.hadoop.hive.metastore.messaging.DropFunctionMessage; -import org.apache.hadoop.hive.metastore.messaging.DropIndexMessage; import org.apache.hadoop.hive.metastore.messaging.DropPartitionMessage; import org.apache.hadoop.hive.metastore.messaging.DropTableMessage; import org.apache.hadoop.hive.metastore.messaging.EventMessage.EventType; @@ -102,7 +93,6 @@ import org.apache.hive.hcatalog.data.Pair; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestRule; @@ -174,58 +164,57 @@ public class TestDbNotificationListener { super(config); } + @Override public void onCreateTable (CreateTableEvent tableEvent) throws MetaException { pushEventId(EventType.CREATE_TABLE, tableEvent); } + @Override public void onDropTable (DropTableEvent tableEvent) throws MetaException { pushEventId(EventType.DROP_TABLE, tableEvent); } + @Override public void onAlterTable (AlterTableEvent tableEvent) throws MetaException { pushEventId(EventType.ALTER_TABLE, tableEvent); } + @Override public void onAddPartition (AddPartitionEvent partitionEvent) throws MetaException { pushEventId(EventType.ADD_PARTITION, partitionEvent); } + @Override public void onDropPartition (DropPartitionEvent partitionEvent) throws MetaException { pushEventId(EventType.DROP_PARTITION, partitionEvent); } + @Override public void onAlterPartition (AlterPartitionEvent partitionEvent) throws MetaException { pushEventId(EventType.ALTER_PARTITION, partitionEvent); } + @Override public void onCreateDatabase (CreateDatabaseEvent dbEvent) throws MetaException { pushEventId(EventType.CREATE_DATABASE, dbEvent); } + @Override public void onDropDatabase (DropDatabaseEvent dbEvent) throws MetaException { pushEventId(EventType.DROP_DATABASE, dbEvent); } - public void onAddIndex(AddIndexEvent indexEvent) throws MetaException { - pushEventId(EventType.CREATE_INDEX, indexEvent); - } - - public void onDropIndex(DropIndexEvent indexEvent) throws MetaException { - pushEventId(EventType.DROP_INDEX, indexEvent); - } - - public void onAlterIndex(AlterIndexEvent indexEvent) throws MetaException { - pushEventId(EventType.ALTER_INDEX, indexEvent); - } - + @Override public void onCreateFunction (CreateFunctionEvent fnEvent) throws MetaException { pushEventId(EventType.CREATE_FUNCTION, fnEvent); } + @Override public void onDropFunction (DropFunctionEvent fnEvent) throws MetaException { pushEventId(EventType.DROP_FUNCTION, fnEvent); } + @Override public void onInsert(InsertEvent insertEvent) throws MetaException { pushEventId(EventType.INSERT, insertEvent); } @@ -963,226 +952,6 @@ public class TestDbNotificationListener { } @Test - @Ignore("HIVE-18715") - public void createIndex() throws Exception { - String indexName = "createIndex"; - String dbName = "default"; - String tableName = "createIndexTable"; - String indexTableName = tableName + "__" + indexName + "__"; - int startTime = (int) (System.currentTimeMillis() / 1000); - List<FieldSchema> cols = new ArrayList<FieldSchema>(); - cols.add(new FieldSchema("col1", "int", "")); - SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); - Map<String, String> params = new HashMap<String, String>(); - params.put("key", "value"); - StorageDescriptor sd = - new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 17, serde, - Arrays.asList("bucketcol"), Arrays.asList(new Order("sortcol", 1)), params); - Table table = - new Table(tableName, dbName, "me", startTime, startTime, 0, sd, null, emptyParameters, - null, null, null); - // Event 1 - msClient.createTable(table); - Index index = - new Index(indexName, null, "default", tableName, startTime, startTime, indexTableName, sd, - emptyParameters, false); - Table indexTable = - new Table(indexTableName, dbName, "me", startTime, startTime, 0, sd, null, emptyParameters, - null, null, null); - // Event 2, 3 (index table and index) - msClient.createIndex(index, indexTable); - - // Get notifications from metastore - NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null); - assertEquals(3, rsp.getEventsSize()); - NotificationEvent event = rsp.getEvents().get(2); - assertEquals(firstEventId + 3, event.getEventId()); - assertTrue(event.getEventTime() >= startTime); - assertEquals(EventType.CREATE_INDEX.toString(), event.getEventType()); - assertEquals(dbName, event.getDbName()); - - // Parse the message field - CreateIndexMessage createIdxMessage = md.getCreateIndexMessage(event.getMessage()); - assertEquals(dbName, createIdxMessage.getDB()); - Index indexObj = createIdxMessage.getIndexObj(); - assertEquals(dbName, indexObj.getDbName()); - assertEquals(indexName, indexObj.getIndexName()); - assertEquals(tableName, indexObj.getOrigTableName()); - assertEquals(indexTableName, indexObj.getIndexTableName()); - - // Verify the eventID was passed to the non-transactional listener - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_INDEX, firstEventId + 3); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 2); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); - - // When hive.metastore.transactional.event.listeners is set, - // a failed event should not create a new notification - DummyRawStoreFailEvent.setEventSucceed(false); - index = - new Index("createIndexTable2", null, "default", tableName, startTime, startTime, - "createIndexTable2__createIndexTable2__", sd, emptyParameters, false); - Table indexTable2 = - new Table("createIndexTable2__createIndexTable2__", dbName, "me", startTime, startTime, 0, - sd, null, emptyParameters, null, null, null); - try { - msClient.createIndex(index, indexTable2); - fail("Error: create index should've failed"); - } catch (Exception ex) { - // expected - } - rsp = msClient.getNextNotification(firstEventId, 0, null); - assertEquals(3, rsp.getEventsSize()); - } - - @Test - @Ignore("HIVE-18715") - public void dropIndex() throws Exception { - String indexName = "dropIndex"; - String dbName = "default"; - String tableName = "dropIndexTable"; - String indexTableName = tableName + "__" + indexName + "__"; - int startTime = (int) (System.currentTimeMillis() / 1000); - List<FieldSchema> cols = new ArrayList<FieldSchema>(); - cols.add(new FieldSchema("col1", "int", "")); - SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); - Map<String, String> params = new HashMap<String, String>(); - params.put("key", "value"); - StorageDescriptor sd = - new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 17, serde, - Arrays.asList("bucketcol"), Arrays.asList(new Order("sortcol", 1)), params); - Table table = - new Table(tableName, dbName, "me", startTime, startTime, 0, sd, null, emptyParameters, - null, null, null); - // Event 1 - msClient.createTable(table); - Index index = - new Index(indexName, null, "default", tableName, startTime, startTime, indexTableName, sd, - emptyParameters, false); - Table indexTable = - new Table(indexTableName, dbName, "me", startTime, startTime, 0, sd, null, emptyParameters, - null, null, null); - // Event 2, 3 (index table and index) - msClient.createIndex(index, indexTable); - // Event 4 (drops index and indexTable) - msClient.dropIndex(dbName, tableName, indexName, true); - - // Get notifications from metastore - NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null); - assertEquals(4, rsp.getEventsSize()); - NotificationEvent event = rsp.getEvents().get(3); - assertEquals(firstEventId + 4, event.getEventId()); - assertTrue(event.getEventTime() >= startTime); - assertEquals(EventType.DROP_INDEX.toString(), event.getEventType()); - assertEquals(dbName, event.getDbName()); - - // Parse the message field - DropIndexMessage dropIdxMsg = md.getDropIndexMessage(event.getMessage()); - assertEquals(dbName, dropIdxMsg.getDB()); - assertEquals(indexName.toLowerCase(), dropIdxMsg.getIndexName()); - assertEquals(indexTableName.toLowerCase(), dropIdxMsg.getIndexTableName()); - assertEquals(tableName.toLowerCase(), dropIdxMsg.getOrigTableName()); - - // Verify the eventID was passed to the non-transactional listener - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.DROP_INDEX, firstEventId + 4); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_INDEX, firstEventId + 3); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 2); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); - - // When hive.metastore.transactional.event.listeners is set, - // a failed event should not create a new notification - index = - new Index("dropIndexTable2", null, "default", tableName, startTime, startTime, - "dropIndexTable__dropIndexTable2__", sd, emptyParameters, false); - Table indexTable2 = - new Table("dropIndexTable__dropIndexTable2__", dbName, "me", startTime, startTime, 0, sd, - null, emptyParameters, null, null, null); - msClient.createIndex(index, indexTable2); - DummyRawStoreFailEvent.setEventSucceed(false); - try { - // drops index and indexTable - msClient.dropIndex(dbName, tableName, "dropIndex2", true); - fail("Error: drop index should've failed"); - } catch (Exception ex) { - // expected - } - - rsp = msClient.getNextNotification(firstEventId, 0, null); - assertEquals(6, rsp.getEventsSize()); - } - - @Test - @Ignore("HIVE-18715") - public void alterIndex() throws Exception { - String indexName = "alterIndex"; - String dbName = "default"; - String tableName = "alterIndexTable"; - String indexTableName = tableName + "__" + indexName + "__"; - int startTime = (int) (System.currentTimeMillis() / 1000); - List<FieldSchema> cols = new ArrayList<FieldSchema>(); - cols.add(new FieldSchema("col1", "int", "")); - SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); - Map<String, String> params = new HashMap<String, String>(); - params.put("key", "value"); - StorageDescriptor sd = - new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 17, serde, - Arrays.asList("bucketcol"), Arrays.asList(new Order("sortcol", 1)), params); - Table table = - new Table(tableName, dbName, "me", startTime, startTime, 0, sd, null, emptyParameters, - null, null, null); - // Event 1 - msClient.createTable(table); - Index oldIndex = - new Index(indexName, null, "default", tableName, startTime, startTime, indexTableName, sd, - emptyParameters, false); - Table oldIndexTable = - new Table(indexTableName, dbName, "me", startTime, startTime, 0, sd, null, emptyParameters, - null, null, null); - // Event 2, 3 - msClient.createIndex(oldIndex, oldIndexTable); // creates index and index table - Index newIndex = - new Index(indexName, null, "default", tableName, startTime, startTime + 1, indexTableName, - sd, emptyParameters, false); - // Event 4 - msClient.alter_index(dbName, tableName, indexName, newIndex); - - // Get notifications from metastore - NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null); - assertEquals(4, rsp.getEventsSize()); - NotificationEvent event = rsp.getEvents().get(3); - assertEquals(firstEventId + 4, event.getEventId()); - assertTrue(event.getEventTime() >= startTime); - assertEquals(EventType.ALTER_INDEX.toString(), event.getEventType()); - assertEquals(dbName, event.getDbName()); - - // Parse the message field - AlterIndexMessage alterIdxMsg = md.getAlterIndexMessage(event.getMessage()); - Index indexObj = alterIdxMsg.getIndexObjAfter(); - assertEquals(dbName, indexObj.getDbName()); - assertEquals(indexName, indexObj.getIndexName()); - assertEquals(tableName, indexObj.getOrigTableName()); - assertEquals(indexTableName, indexObj.getIndexTableName()); - assertTrue(indexObj.getCreateTime() < indexObj.getLastAccessTime()); - - // Verify the eventID was passed to the non-transactional listener - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ALTER_INDEX, firstEventId + 4); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_INDEX, firstEventId + 3); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 2); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); - - // When hive.metastore.transactional.event.listeners is set, - // a failed event should not create a new notification - DummyRawStoreFailEvent.setEventSucceed(false); - try { - msClient.alter_index(dbName, tableName, indexName, newIndex); - fail("Error: alter index should've failed"); - } catch (Exception ex) { - // expected - } - rsp = msClient.getNextNotification(firstEventId, 0, null); - assertEquals(4, rsp.getEventsSize()); - } - - @Test public void insertTable() throws Exception { String defaultDbName = "default"; String tblName = "inserttbl"; http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index f0f23ca..82590ce 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -1004,11 +1004,6 @@ public class QTestUtil { LOG.warn("Trying to drop table " + e.getTableName() + ". But it does not exist."); continue; } - // dropping index table can not be dropped directly. Dropping the base - // table will automatically drop all its index table - if(tblObj.isIndexTable()) { - continue; - } db.dropTable(dbName, tblName, true, true, fsType == FsType.encrypted_hdfs); } } http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/metastore/scripts/upgrade/hive/hive-schema-3.0.0.hive.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/hive/hive-schema-3.0.0.hive.sql b/metastore/scripts/upgrade/hive/hive-schema-3.0.0.hive.sql index f4b2a2b..3444439 100644 --- a/metastore/scripts/upgrade/hive/hive-schema-3.0.0.hive.sql +++ b/metastore/scripts/upgrade/hive/hive-schema-3.0.0.hive.sql @@ -157,54 +157,6 @@ FROM \"GLOBAL_PRIVS\"" ); -CREATE TABLE IF NOT EXISTS `IDXS` ( - `INDEX_ID` bigint, - `CREATE_TIME` int, - `DEFERRED_REBUILD` boolean, - `INDEX_HANDLER_CLASS` string, - `INDEX_NAME` string, - `INDEX_TBL_ID` bigint, - `LAST_ACCESS_TIME` int, - `ORIG_TBL_ID` bigint, - `SD_ID` bigint, - CONSTRAINT `SYS_PK_IDXS` PRIMARY KEY (`INDEX_ID`) DISABLE -) -STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' -TBLPROPERTIES ( -"hive.sql.database.type" = "METASTORE", -"hive.sql.query" = -"SELECT - \"INDEX_ID\", - \"CREATE_TIME\", - \"DEFERRED_REBUILD\", - \"INDEX_HANDLER_CLASS\", - \"INDEX_NAME\", - \"INDEX_TBL_ID\", - \"LAST_ACCESS_TIME\", - \"ORIG_TBL_ID\", - \"SD_ID\" -FROM - \"IDXS\"" -); - -CREATE TABLE IF NOT EXISTS `INDEX_PARAMS` ( - `INDEX_ID` bigint, - `PARAM_KEY` string, - `PARAM_VALUE` string, - CONSTRAINT `SYS_PK_INDEX_PARAMS` PRIMARY KEY (`INDEX_ID`,`PARAM_KEY`) DISABLE -) -STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' -TBLPROPERTIES ( -"hive.sql.database.type" = "METASTORE", -"hive.sql.query" = -"SELECT - \"INDEX_ID\", - \"PARAM_KEY\", - \"PARAM_VALUE\" -FROM - \"INDEX_PARAMS\"" -); - CREATE TABLE IF NOT EXISTS `PARTITIONS` ( `PART_ID` bigint, `CREATE_TIME` int, http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index d3546d4..1faa50a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -238,7 +238,6 @@ public enum ErrorMsg { INVALID_JDO_FILTER_EXPRESSION(10143, "Invalid expression for JDO filter"), - SHOW_CREATETABLE_INDEX(10144, "SHOW CREATE TABLE does not support tables of type INDEX_TABLE."), ALTER_BUCKETNUM_NONBUCKETIZED_TBL(10145, "Table is not bucketized."), TRUNCATE_FOR_NON_MANAGED_TABLE(10146, "Cannot truncate non-managed table {0}.", true), http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index 632a213..a51fdd3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -42,7 +42,6 @@ import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.HiveMetaStoreUtils; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.Warehouse; -import org.apache.hadoop.hive.metastore.api.BasicTxnInfo; import org.apache.hadoop.hive.metastore.api.CreationMetadata; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; @@ -383,7 +382,9 @@ public class Table implements Serializable { if (spec.containsKey(fs.getName())) { ++columnsFound; } - if (columnsFound == spec.size()) break; + if (columnsFound == spec.size()) { + break; + } } if (columnsFound < spec.size()) { throw new ValidationFailureSemanticException("Partition spec " + spec + " contains non-partition columns"); @@ -875,13 +876,6 @@ public class Table implements Serializable { } /** - * @return whether this table is actually an index table - */ - public boolean isIndexTable() { - return TableType.INDEX_TABLE.equals(getTableType()); - } - - /** * Creates a partition name -> value spec map object * * @param tp @@ -989,13 +983,20 @@ public class Table implements Serializable { public static boolean shouldStoreFieldsInMetastore( HiveConf conf, String serdeLib, Map<String, String> tableParams) { - if (hasMetastoreBasedSchema(conf, serdeLib)) return true; - if (HiveConf.getBoolVar(conf, ConfVars.HIVE_LEGACY_SCHEMA_FOR_ALL_SERDES)) return true; + if (hasMetastoreBasedSchema(conf, serdeLib)) { + return true; + } + if (HiveConf.getBoolVar(conf, ConfVars.HIVE_LEGACY_SCHEMA_FOR_ALL_SERDES)) { + return true; + } // Table may or may not be using metastore. Only the SerDe can tell us. AbstractSerDe deserializer = null; try { Class<?> clazz = conf.getClassByName(serdeLib); - if (!AbstractSerDe.class.isAssignableFrom(clazz)) return true; // The default. + if (!AbstractSerDe.class.isAssignableFrom(clazz)) + { + return true; // The default. + } deserializer = ReflectionUtil.newInstance( conf.getClassByName(serdeLib).asSubclass(AbstractSerDe.class), conf); } catch (Exception ex) { http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index 1793402..5761795 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -2517,10 +2517,6 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { showCreateTblDesc = new ShowCreateTableDesc(tableName, ctx.getResFile().toString()); Table tab = getTable(tableName); - if (tab.getTableType() == org.apache.hadoop.hive.metastore.TableType.INDEX_TABLE) { - throw new SemanticException(ErrorMsg.SHOW_CREATETABLE_INDEX.getMsg(tableName - + " has table type INDEX_TABLE")); - } inputs.add(new ReadEntity(tab)); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), showCreateTblDesc), conf)); http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 7f446ca..d99df80 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -43,7 +43,6 @@ import java.util.UUID; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; -import com.google.common.collect.ImmutableSet; import org.antlr.runtime.ClassicToken; import org.antlr.runtime.CommonToken; import org.antlr.runtime.TokenRewriteStream; @@ -64,14 +63,12 @@ import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.StatsSetupConst.StatDB; -import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.conf.HiveConf.StrictChecks; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.TransactionalValidationListener; import org.apache.hadoop.hive.metastore.Warehouse; -import org.apache.hadoop.hive.metastore.api.CreationMetadata; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; @@ -6610,7 +6607,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { (dest_tab.getSortCols().size() > 0)) { sortCols = getSortCols(dest, qb, dest_tab, table_desc, input, true); sortOrders = getSortOrders(dest, qb, dest_tab, input); - if (!enforceBucketing && !dest_tab.isIndexTable()) { + if (!enforceBucketing) { throw new SemanticException(ErrorMsg.TBL_SORTED_NOT_BUCKETED.getErrorCodedMsg(dest_tab.getCompleteName())); } else { if (!enforceBucketing) { @@ -6677,7 +6674,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { if ((dest_tab.getSortCols() != null) && (dest_tab.getSortCols().size() > 0)) { - if (!enforceBucketing && !dest_tab.isIndexTable()) { + if (!enforceBucketing) { throw new SemanticException(ErrorMsg.TBL_SORTED_NOT_BUCKETED.getErrorCodedMsg(dest_tab.getCompleteName())); } else { http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java index 19815a9..16efb72 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java @@ -441,8 +441,6 @@ public class AuthorizationPreEventListener extends MetaStorePreEventListener { // TableType specified was null, we need to figure out what type it was. if (MetaStoreUtils.isExternalTable(wrapperApiTable)){ wrapperApiTable.setTableType(TableType.EXTERNAL_TABLE.toString()); - } else if (MetaStoreUtils.isIndexTable(wrapperApiTable)) { - wrapperApiTable.setTableType(TableType.INDEX_TABLE.toString()); } else if (MetaStoreUtils.isMaterializedViewTable(wrapperApiTable)) { wrapperApiTable.setTableType(TableType.MATERIALIZED_VIEW.toString()); } else if ((wrapperApiTable.getSd() == null) || (wrapperApiTable.getSd().getLocation() == null)) { http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java index d982555..a24b642 100755 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java @@ -35,10 +35,8 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.PartitionDropOptions; import org.apache.hadoop.hive.metastore.Warehouse; -import org.apache.hadoop.hive.metastore.api.BasicTxnInfo; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/ql/src/test/results/clientpositive/llap/resourceplan.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/resourceplan.q.out b/ql/src/test/results/clientpositive/llap/resourceplan.q.out index 6c4627b..96ed8c6 100644 --- a/ql/src/test/results/clientpositive/llap/resourceplan.q.out +++ b/ql/src/test/results/clientpositive/llap/resourceplan.q.out @@ -414,110 +414,6 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@GLOBAL_PRIVS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `IDXS` ( - `INDEX_ID` bigint, - `CREATE_TIME` int, - `DEFERRED_REBUILD` boolean, - `INDEX_HANDLER_CLASS` string, - `INDEX_NAME` string, - `INDEX_TBL_ID` bigint, - `LAST_ACCESS_TIME` int, - `ORIG_TBL_ID` bigint, - `SD_ID` bigint, - CONSTRAINT `SYS_PK_IDXS` PRIMARY KEY (`INDEX_ID`) DISABLE -) -STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' -TBLPROPERTIES ( -"hive.sql.database.type" = "METASTORE", -"hive.sql.query" = -"SELECT - \"INDEX_ID\", - \"CREATE_TIME\", - \"DEFERRED_REBUILD\", - \"INDEX_HANDLER_CLASS\", - \"INDEX_NAME\", - \"INDEX_TBL_ID\", - \"LAST_ACCESS_TIME\", - \"ORIG_TBL_ID\", - \"SD_ID\" -FROM - \"IDXS\"" -) -PREHOOK: type: CREATETABLE -PREHOOK: Output: SYS@IDXS -PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `IDXS` ( - `INDEX_ID` bigint, - `CREATE_TIME` int, - `DEFERRED_REBUILD` boolean, - `INDEX_HANDLER_CLASS` string, - `INDEX_NAME` string, - `INDEX_TBL_ID` bigint, - `LAST_ACCESS_TIME` int, - `ORIG_TBL_ID` bigint, - `SD_ID` bigint, - CONSTRAINT `SYS_PK_IDXS` PRIMARY KEY (`INDEX_ID`) DISABLE -) -STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' -TBLPROPERTIES ( -"hive.sql.database.type" = "METASTORE", -"hive.sql.query" = -"SELECT - \"INDEX_ID\", - \"CREATE_TIME\", - \"DEFERRED_REBUILD\", - \"INDEX_HANDLER_CLASS\", - \"INDEX_NAME\", - \"INDEX_TBL_ID\", - \"LAST_ACCESS_TIME\", - \"ORIG_TBL_ID\", - \"SD_ID\" -FROM - \"IDXS\"" -) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: SYS@IDXS -POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `INDEX_PARAMS` ( - `INDEX_ID` bigint, - `PARAM_KEY` string, - `PARAM_VALUE` string, - CONSTRAINT `SYS_PK_INDEX_PARAMS` PRIMARY KEY (`INDEX_ID`,`PARAM_KEY`) DISABLE -) -STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' -TBLPROPERTIES ( -"hive.sql.database.type" = "METASTORE", -"hive.sql.query" = -"SELECT - \"INDEX_ID\", - \"PARAM_KEY\", - \"PARAM_VALUE\" -FROM - \"INDEX_PARAMS\"" -) -PREHOOK: type: CREATETABLE -PREHOOK: Output: SYS@INDEX_PARAMS -PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `INDEX_PARAMS` ( - `INDEX_ID` bigint, - `PARAM_KEY` string, - `PARAM_VALUE` string, - CONSTRAINT `SYS_PK_INDEX_PARAMS` PRIMARY KEY (`INDEX_ID`,`PARAM_KEY`) DISABLE -) -STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' -TBLPROPERTIES ( -"hive.sql.database.type" = "METASTORE", -"hive.sql.query" = -"SELECT - \"INDEX_ID\", - \"PARAM_KEY\", - \"PARAM_VALUE\" -FROM - \"INDEX_PARAMS\"" -) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: SYS@INDEX_PARAMS -POSTHOOK: Output: database:sys PREHOOK: query: CREATE TABLE IF NOT EXISTS `PARTITIONS` ( `PART_ID` bigint, `CREATE_TIME` int,
