Repository: hive Updated Branches: refs/heads/branch-3 d558b3cc3 -> 774a8ef7a
http://git-wip-us.apache.org/repos/asf/hive/blob/774a8ef7/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java index ef447e1..2c47ee4 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java @@ -21,6 +21,7 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.common.classification.RetrySemantics; import org.apache.hadoop.hive.metastore.api.*; @@ -28,6 +29,7 @@ import org.apache.hadoop.hive.metastore.api.*; import java.sql.SQLException; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; /** @@ -40,7 +42,7 @@ public interface TxnStore extends Configurable { enum MUTEX_KEY { Initiator, Cleaner, HouseKeeper, CompactionHistory, CheckLock, - WriteSetCleaner, CompactionScheduler, WriteIdAllocator + WriteSetCleaner, CompactionScheduler, WriteIdAllocator, MaterializationRebuild } // Compactor states (Should really be enum) String INITIATED_RESPONSE = "initiated"; @@ -124,21 +126,33 @@ public interface TxnStore extends Configurable { void replTableWriteIdState(ReplTblWriteIdStateRequest rqst) throws MetaException; /** - * Get the first transaction corresponding to given database and table after transactions - * referenced in the transaction snapshot. - * @return + * Get invalidation info for the materialization. Currently, the materialization information + * only contains information about whether there was update/delete operations on the source + * tables used by the materialization since it was created. + * @param cm creation metadata for the materialization + * @param validTxnList valid transaction list for snapshot taken for current query * @throws MetaException */ @RetrySemantics.Idempotent - BasicTxnInfo getFirstCompletedTransactionForTableAfterCommit( - String inputDbName, String inputTableName, ValidWriteIdList txnList) + Materialization getMaterializationInvalidationInfo( + final CreationMetadata cm, final String validTxnList) throws MetaException; - /** - * Gets the list of valid write ids for the given table wrt to current txn - * @param rqst info on transaction and list of table names associated with given transaction - * @throws NoSuchTxnException - * @throws MetaException - */ + + LockResponse lockMaterializationRebuild(String dbName, String tableName, long txnId) + throws MetaException; + + boolean heartbeatLockMaterializationRebuild(String dbName, String tableName, long txnId) + throws MetaException; + + long cleanupMaterializationRebuildLocks(ValidTxnList validTxnList, long timeout) + throws MetaException; + + /** + * Gets the list of valid write ids for the given table wrt to current txn + * @param rqst info on transaction and list of table names associated with given transaction + * @throws NoSuchTxnException + * @throws MetaException + */ @RetrySemantics.ReadOnly GetValidWriteIdsResponse getValidWriteIds(GetValidWriteIdsRequest rqst) throws NoSuchTxnException, MetaException; http://git-wip-us.apache.org/repos/asf/hive/blob/774a8ef7/standalone-metastore/src/main/sql/derby/hive-schema-3.1.0.derby.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/derby/hive-schema-3.1.0.derby.sql b/standalone-metastore/src/main/sql/derby/hive-schema-3.1.0.derby.sql index 352b43e..043cf07 100644 --- a/standalone-metastore/src/main/sql/derby/hive-schema-3.1.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/hive-schema-3.1.0.derby.sql @@ -197,7 +197,8 @@ CREATE TABLE "APP"."MV_CREATION_METADATA" ( "CAT_NAME" VARCHAR(256) NOT NULL, "DB_NAME" VARCHAR(128) NOT NULL, "TBL_NAME" VARCHAR(256) NOT NULL, - "TXN_LIST" CLOB + "TXN_LIST" CLOB, + "MATERIALIZATION_TIME" BIGINT NOT NULL ); CREATE TABLE "APP"."MV_TABLES_USED" ( @@ -526,7 +527,8 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS ( CTC_TABLE varchar(256), CTC_PARTITION varchar(767), CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID bigint + CTC_WRITEID bigint, + CTC_UPDATE_DELETE char(1) NOT NULL ); CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); @@ -645,6 +647,14 @@ CREATE TABLE MIN_HISTORY_LEVEL ( CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID); +CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( + MRL_TXN_ID BIGINT NOT NULL, + MRL_DB_NAME VARCHAR(128) NOT NULL, + MRL_TBL_NAME VARCHAR(256) NOT NULL, + MRL_LAST_HEARTBEAT BIGINT NOT NULL, + PRIMARY KEY(MRL_TXN_ID) +); + CREATE TABLE "APP"."I_SCHEMA" ( "SCHEMA_ID" bigint primary key, "SCHEMA_TYPE" integer not null, http://git-wip-us.apache.org/repos/asf/hive/blob/774a8ef7/standalone-metastore/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql b/standalone-metastore/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql index e45fb64..8ab35c3 100644 --- a/standalone-metastore/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql @@ -29,5 +29,24 @@ CREATE INDEX "APP"."TAB_COL_STATS_IDX" ON "APP"."TAB_COL_STATS" ("CAT_NAME", "DB -- HIVE-19340 ALTER TABLE TXNS ADD COLUMN TXN_TYPE integer; +-- HIVE-19027 +-- add column MATERIALIZATION_TIME (bigint) to MV_CREATION_METADATA table +ALTER TABLE "APP"."MV_CREATION_METADATA" ADD COLUMN "MATERIALIZATION_TIME" BIGINT; +UPDATE "APP"."MV_CREATION_METADATA" SET "MATERIALIZATION_TIME" = 0; +ALTER TABLE "APP"."MV_CREATION_METADATA" ALTER COLUMN "MATERIALIZATION_TIME" NOT NULL; + +-- add column CTC_UPDATE_DELETE (char) to COMPLETED_TXN_COMPONENTS table +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD COLUMN CTC_UPDATE_DELETE char(1); +UPDATE COMPLETED_TXN_COMPONENTS SET CTC_UPDATE_DELETE = 'N'; +ALTER TABLE COMPLETED_TXN_COMPONENTS ALTER COLUMN CTC_UPDATE_DELETE NOT NULL; + +CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( + MRL_TXN_ID BIGINT NOT NULL, + MRL_DB_NAME VARCHAR(128) NOT NULL, + MRL_TBL_NAME VARCHAR(256) NOT NULL, + MRL_LAST_HEARTBEAT BIGINT NOT NULL, + PRIMARY KEY(MRL_TXN_ID) +); + -- This needs to be the last thing done. Insert any changes above this line. UPDATE "APP".VERSION SET SCHEMA_VERSION='3.1.0', VERSION_COMMENT='Hive release version 3.1.0' where VER_ID=1; http://git-wip-us.apache.org/repos/asf/hive/blob/774a8ef7/standalone-metastore/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql index bc11b40..fef6ab9 100644 --- a/standalone-metastore/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql @@ -388,7 +388,8 @@ CREATE TABLE MV_CREATION_METADATA CAT_NAME nvarchar(256) NOT NULL, DB_NAME nvarchar(128) NOT NULL, TBL_NAME nvarchar(256) NOT NULL, - TXN_LIST text NULL + TXN_LIST text NULL, + MATERIALIZATION_TIME bigint NOT NULL ); ALTER TABLE MV_CREATION_METADATA ADD CONSTRAINT MV_CREATION_METADATA_PK PRIMARY KEY (MV_CREATION_METADATA_ID); @@ -1034,7 +1035,8 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS( CTC_TABLE nvarchar(128) NULL, CTC_PARTITION nvarchar(767) NULL, CTC_TIMESTAMP datetime2 DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID bigint + CTC_WRITEID bigint, + CTC_UPDATE_DELETE char(1) NOT NULL ); CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); @@ -1191,6 +1193,17 @@ PRIMARY KEY CLUSTERED CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID); +CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( + MRL_TXN_ID bigint NOT NULL, + MRL_DB_NAME nvarchar(128) NOT NULL, + MRL_TBL_NAME nvarchar(256) NOT NULL, + MRL_LAST_HEARTBEAT bigint NOT NULL, +PRIMARY KEY CLUSTERED +( + MRL_TXN_ID ASC +) +); + CREATE TABLE "I_SCHEMA" ( "SCHEMA_ID" bigint primary key, "SCHEMA_TYPE" int not null, http://git-wip-us.apache.org/repos/asf/hive/blob/774a8ef7/standalone-metastore/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql index 3480edb..6967319 100644 --- a/standalone-metastore/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql @@ -30,6 +30,24 @@ CREATE INDEX TAB_COL_STATS_IDX ON TAB_COL_STATS (CAT_NAME, DB_NAME, TABLE_NAME, -- HIVE-19340 ALTER TABLE TXNS ADD TXN_TYPE int NULL; +-- HIVE-19027 +-- add column MATERIALIZATION_TIME (bigint) to MV_CREATION_METADATA table +ALTER TABLE MV_CREATION_METADATA ADD MATERIALIZATION_TIME bigint NOT NULL DEFAULT(0); + +-- add column CTC_UPDATE_DELETE (char) to COMPLETED_TXN_COMPONENTS table +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_UPDATE_DELETE char(1) NOT NULL DEFAULT('N'); + +CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( + MRL_TXN_ID bigint NOT NULL, + MRL_DB_NAME nvarchar(128) NOT NULL, + MRL_TBL_NAME nvarchar(256) NOT NULL, + MRL_LAST_HEARTBEAT bigint NOT NULL, +PRIMARY KEY CLUSTERED +( + MRL_TXN_ID ASC +) +); + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.1.0', VERSION_COMMENT='Hive release version 3.1.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0' AS MESSAGE; http://git-wip-us.apache.org/repos/asf/hive/blob/774a8ef7/standalone-metastore/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql index af955dc..8510ceb 100644 --- a/standalone-metastore/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql @@ -603,6 +603,7 @@ CREATE TABLE IF NOT EXISTS `MV_CREATION_METADATA` ( `DB_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, `TBL_NAME` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, `TXN_LIST` TEXT DEFAULT NULL, + `MATERIALIZATION_TIME` bigint(20) NOT NULL, PRIMARY KEY (`MV_CREATION_METADATA_ID`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; /*!40101 SET character_set_client = @saved_cs_client */; @@ -1006,7 +1007,8 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS ( CTC_TABLE varchar(256), CTC_PARTITION varchar(767), CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID bigint + CTC_WRITEID bigint, + CTC_UPDATE_DELETE char(1) NOT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1; CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION) USING BTREE; @@ -1124,6 +1126,14 @@ CREATE TABLE MIN_HISTORY_LEVEL ( CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID); +CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( + MRL_TXN_ID bigint NOT NULL, + MRL_DB_NAME VARCHAR(128) NOT NULL, + MRL_TBL_NAME VARCHAR(256) NOT NULL, + MRL_LAST_HEARTBEAT bigint NOT NULL, + PRIMARY KEY(MRL_TXN_ID) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + CREATE TABLE `I_SCHEMA` ( `SCHEMA_ID` BIGINT PRIMARY KEY, `SCHEMA_TYPE` INTEGER NOT NULL, http://git-wip-us.apache.org/repos/asf/hive/blob/774a8ef7/standalone-metastore/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql index 61d50d2..1ac6db1 100644 --- a/standalone-metastore/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql @@ -30,6 +30,25 @@ CREATE INDEX TAB_COL_STATS_IDX ON TAB_COL_STATS (CAT_NAME, DB_NAME, TABLE_NAME, -- HIVE-19340 ALTER TABLE TXNS ADD COLUMN TXN_TYPE int DEFAULT NULL; +-- HIVE-19027 +-- add column MATERIALIZATION_TIME (bigint) to MV_CREATION_METADATA table +ALTER TABLE `MV_CREATION_METADATA` ADD `MATERIALIZATION_TIME` BIGINT; +UPDATE `MV_CREATION_METADATA` SET `MATERIALIZATION_TIME` = 0; +ALTER TABLE `MV_CREATION_METADATA` MODIFY COLUMN `MATERIALIZATION_TIME` BIGINT NOT NULL; + +-- add column CTC_UPDATE_DELETE (char) to COMPLETED_TXN_COMPONENTS table +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_UPDATE_DELETE char(1); +UPDATE COMPLETED_TXN_COMPONENTS SET CTC_UPDATE_DELETE = 'N'; +ALTER TABLE COMPLETED_TXN_COMPONENTS MODIFY COLUMN CTC_UPDATE_DELETE char(1) NOT NULL; + +CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( + MRL_TXN_ID BIGINT NOT NULL, + MRL_DB_NAME VARCHAR(128) NOT NULL, + MRL_TBL_NAME VARCHAR(256) NOT NULL, + MRL_LAST_HEARTBEAT BIGINT NOT NULL, + PRIMARY KEY(MRL_TXN_ID) +); + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.1.0', VERSION_COMMENT='Hive release version 3.1.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0' AS ' '; http://git-wip-us.apache.org/repos/asf/hive/blob/774a8ef7/standalone-metastore/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql index bc13703..90388e7 100644 --- a/standalone-metastore/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql @@ -410,7 +410,8 @@ CREATE TABLE MV_CREATION_METADATA CAT_NAME VARCHAR2(256) NOT NULL, DB_NAME VARCHAR2(128) NOT NULL, TBL_NAME VARCHAR2(256) NOT NULL, - TXN_LIST CLOB NULL + TXN_LIST CLOB NULL, + MATERIALIZATION_TIME NUMBER NOT NULL ); ALTER TABLE MV_CREATION_METADATA ADD CONSTRAINT MV_CREATION_METADATA_PK PRIMARY KEY (MV_CREATION_METADATA_ID); @@ -983,7 +984,8 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS ( CTC_TABLE VARCHAR2(256), CTC_PARTITION VARCHAR2(767), CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID NUMBER(19) + CTC_WRITEID NUMBER(19), + CTC_UPDATE_DELETE CHAR(1) NOT NULL ) ROWDEPENDENCIES; CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); @@ -1100,6 +1102,14 @@ CREATE TABLE MIN_HISTORY_LEVEL ( CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID); +CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( + MRL_TXN_ID NUMBER NOT NULL, + MRL_DB_NAME VARCHAR(128) NOT NULL, + MRL_TBL_NAME VARCHAR(256) NOT NULL, + MRL_LAST_HEARTBEAT NUMBER NOT NULL, + PRIMARY KEY(MRL_TXN_ID) +); + CREATE TABLE "I_SCHEMA" ( "SCHEMA_ID" number primary key, "SCHEMA_TYPE" number not null, http://git-wip-us.apache.org/repos/asf/hive/blob/774a8ef7/standalone-metastore/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql index a971258..7bf0260 100644 --- a/standalone-metastore/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql @@ -30,6 +30,25 @@ CREATE INDEX TAB_COL_STATS_IDX ON TAB_COL_STATS (CAT_NAME, DB_NAME, TABLE_NAME, -- HIVE-19340 ALTER TABLE TXNS ADD TXN_TYPE number(10) NULL; +-- HIVE-19027 +-- add column MATERIALIZATION_TIME (bigint) to MV_CREATION_METADATA table +ALTER TABLE MV_CREATION_METADATA ADD MATERIALIZATION_TIME NUMBER NULL; +UPDATE MV_CREATION_METADATA SET MATERIALIZATION_TIME = 0; +ALTER TABLE MV_CREATION_METADATA MODIFY(MATERIALIZATION_TIME NOT NULL); + +-- add column CTC_UPDATE_DELETE (char) to COMPLETED_TXN_COMPONENTS table +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_UPDATE_DELETE char(1) NULL; +UPDATE COMPLETED_TXN_COMPONENTS SET CTC_UPDATE_DELETE = 'N'; +ALTER TABLE COMPLETED_TXN_COMPONENTS MODIFY(CTC_UPDATE_DELETE NOT NULL); + +CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( + MRL_TXN_ID NUMBER NOT NULL, + MRL_DB_NAME VARCHAR(128) NOT NULL, + MRL_TBL_NAME VARCHAR(256) NOT NULL, + MRL_LAST_HEARTBEAT NUMBER NOT NULL, + PRIMARY KEY(MRL_TXN_ID) +); + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.1.0', VERSION_COMMENT='Hive release version 3.1.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0' AS Status from dual; http://git-wip-us.apache.org/repos/asf/hive/blob/774a8ef7/standalone-metastore/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql index 36bab70..028d091 100644 --- a/standalone-metastore/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql @@ -404,7 +404,8 @@ CREATE TABLE "MV_CREATION_METADATA" ( "CAT_NAME" character varying(256) NOT NULL, "DB_NAME" character varying(128) NOT NULL, "TBL_NAME" character varying(256) NOT NULL, - "TXN_LIST" text + "TXN_LIST" text, + "MATERIALIZATION_TIME" bigint NOT NULL ); -- @@ -1673,7 +1674,8 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS ( CTC_TABLE varchar(256), CTC_PARTITION varchar(767), CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID bigint + CTC_WRITEID bigint, + CTC_UPDATE_DELETE char(1) NOT NULL ); CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON COMPLETED_TXN_COMPONENTS USING btree (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); @@ -1790,6 +1792,14 @@ CREATE TABLE MIN_HISTORY_LEVEL ( CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID); +CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( + MRL_TXN_ID bigint NOT NULL, + MRL_DB_NAME varchar(128) NOT NULL, + MRL_TBL_NAME varchar(256) NOT NULL, + MRL_LAST_HEARTBEAT bigint NOT NULL, + PRIMARY KEY(MRL_TXN_ID) +); + CREATE TABLE "I_SCHEMA" ( "SCHEMA_ID" bigint primary key, "SCHEMA_TYPE" integer not null, http://git-wip-us.apache.org/repos/asf/hive/blob/774a8ef7/standalone-metastore/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql index fe96047..f16915f 100644 --- a/standalone-metastore/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql @@ -32,6 +32,25 @@ CREATE INDEX "TAB_COL_STATS_IDX" ON "TAB_COL_STATS" USING btree ("CAT_NAME", "DB -- HIVE-19340 ALTER TABLE TXNS ADD COLUMN TXN_TYPE integer DEFAULT NULL; +-- HIVE-19027 +-- add column MATERIALIZATION_TIME (bigint) to MV_CREATION_METADATA table +ALTER TABLE "MV_CREATION_METADATA" ADD COLUMN "MATERIALIZATION_TIME" bigint NULL; +UPDATE "MV_CREATION_METADATA" SET "MATERIALIZATION_TIME" = 0; +ALTER TABLE "MV_CREATION_METADATA" ALTER COLUMN "MATERIALIZATION_TIME" SET NOT NULL; + +-- add column CTC_UPDATE_DELETE (char) to COMPLETED_TXN_COMPONENTS table +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD COLUMN CTC_UPDATE_DELETE char(1) NULL; +UPDATE COMPLETED_TXN_COMPONENTS SET CTC_UPDATE_DELETE = 'N'; +ALTER TABLE COMPLETED_TXN_COMPONENTS ALTER COLUMN CTC_UPDATE_DELETE SET NOT NULL; + +CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( + MRL_TXN_ID bigint NOT NULL, + MRL_DB_NAME varchar(128) NOT NULL, + MRL_TBL_NAME varchar(256) NOT NULL, + MRL_LAST_HEARTBEAT bigint NOT NULL, + PRIMARY KEY(MRL_TXN_ID) +); + -- These lines need to be last. Insert any changes above. UPDATE "VERSION" SET "SCHEMA_VERSION"='3.1.0', "VERSION_COMMENT"='Hive release version 3.1.0' where "VER_ID"=1; SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0'; http://git-wip-us.apache.org/repos/asf/hive/blob/774a8ef7/standalone-metastore/src/main/thrift/hive_metastore.thrift ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/thrift/hive_metastore.thrift b/standalone-metastore/src/main/thrift/hive_metastore.thrift index 6e503eb..ad1dc1f 100644 --- a/standalone-metastore/src/main/thrift/hive_metastore.thrift +++ b/standalone-metastore/src/main/thrift/hive_metastore.thrift @@ -1061,6 +1061,7 @@ struct CreationMetadata { 3: required string tblName, 4: required set<string> tablesUsed, 5: optional string validTxnList, + 6: optional i64 materializationTime } struct NotificationEventRequest { @@ -1246,10 +1247,7 @@ struct TableMeta { } struct Materialization { - 1: required set<string> tablesUsed; - 2: optional string validTxnList - 3: optional i64 invalidationTime; - 4: optional bool sourceTablesUpdateDeleteModified; + 1: required bool sourceTablesUpdateDeleteModified; } // Data types for workload management. @@ -1701,7 +1699,7 @@ service ThriftHiveMetastore extends fb303.FacebookService GetTableResult get_table_req(1:GetTableRequest req) throws (1:MetaException o1, 2:NoSuchObjectException o2) GetTablesResult get_table_objects_by_name_req(1:GetTablesRequest req) throws (1:MetaException o1, 2:InvalidOperationException o2, 3:UnknownDBException o3) - map<string, Materialization> get_materialization_invalidation_info(1:string dbname, 2:list<string> tbl_names) + Materialization get_materialization_invalidation_info(1:CreationMetadata creation_metadata, 2:string validTxnList) throws (1:MetaException o1, 2:InvalidOperationException o2, 3:UnknownDBException o3) void update_creation_metadata(1: string catName, 2:string dbname, 3:string tbl_name, 4:CreationMetadata creation_metadata) throws (1:MetaException o1, 2:InvalidOperationException o2, 3:UnknownDBException o3) http://git-wip-us.apache.org/repos/asf/hive/blob/774a8ef7/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java index 2d87a2f..2d57cfb 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java @@ -164,8 +164,6 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos // instantiate the metastore server handler directly instead of connecting // through the network client = HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true); - // Initialize materializations invalidation cache (only for local metastore) - MaterializationsInvalidationCache.get().init(conf, (IHMSHandler) client); isConnected = true; snapshotActiveConf(); return; @@ -1442,10 +1440,9 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos /** {@inheritDoc} */ @Override - public Map<String, Materialization> getMaterializationsInvalidationInfo(String dbName, List<String> viewNames) + public Materialization getMaterializationInvalidationInfo(CreationMetadata cm, String validTxnList) throws MetaException, InvalidOperationException, UnknownDBException, TException { - return client.get_materialization_invalidation_info( - dbName, filterHook.filterTableNames(null, dbName, viewNames)); + return client.get_materialization_invalidation_info(cm, validTxnList); } /** {@inheritDoc} */ http://git-wip-us.apache.org/repos/asf/hive/blob/774a8ef7/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMaterializationsCacheCleaner.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMaterializationsCacheCleaner.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMaterializationsCacheCleaner.java deleted file mode 100644 index 8debcce..0000000 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMaterializationsCacheCleaner.java +++ /dev/null @@ -1,328 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.metastore; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.api.BasicTxnInfo; -import org.apache.hadoop.hive.metastore.api.CreationMetadata; -import org.apache.hadoop.hive.metastore.api.Materialization; -import org.apache.hadoop.hive.metastore.api.Table; -import org.junit.Assert; -import org.junit.FixMethodOrder; -import org.junit.Test; -import org.junit.runners.MethodSorters; - -import java.util.Map; - -import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * Unit tests for {@link org.apache.hadoop.hive.metastore.MaterializationsInvalidationCache}. - * The tests focus on arrival of notifications (possibly out of order) and the logic - * to clean up the materializations cache. Tests need to be executed in a certain order - * to avoid interactions among them, as the invalidation cache is a singleton. - */ -@FixMethodOrder(MethodSorters.NAME_ASCENDING) -public class TestMetaStoreMaterializationsCacheCleaner { - - private static final String DB_NAME = "hive3252"; - private static final String TBL_NAME_1 = "tmptbl1"; - private static final String TBL_NAME_2 = "tmptbl2"; - private static final String TBL_NAME_3 = "tmptbl3"; - private static final String MV_NAME_1 = "mv1"; - private static final String MV_NAME_2 = "mv2"; - - - @Test - public void testCleanerScenario1() throws Exception { - // create mock raw store - Configuration conf = new Configuration(); - conf.set("metastore.materializations.invalidation.impl", "DISABLE"); - // create mock handler - final IHMSHandler handler = mock(IHMSHandler.class); - // initialize invalidation cache (set conf to disable) - MaterializationsInvalidationCache.get().init(conf, handler); - - // This is a dummy test, invalidation cache is not supposed to - // record any information. - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, 1, 1, false); - int id = 2; - BasicTxnInfo txn2 = createTxnInfo(DB_NAME, TBL_NAME_1, id); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, id, id, false); - // Create tbl2 (nothing to do) - id = 3; - BasicTxnInfo txn3 = createTxnInfo(DB_NAME, TBL_NAME_1, id); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, id, id, false); - // Cleanup (current = 4, duration = 4) -> Does nothing - long removed = MaterializationsInvalidationCache.get().cleanup(0L); - Assert.assertEquals(0L, removed); - // Create mv1 - Table mv1 = mock(Table.class); - when(mv1.getDbName()).thenReturn(DB_NAME); - when(mv1.getTableName()).thenReturn(MV_NAME_1); - CreationMetadata mockCM1 = new CreationMetadata( - DEFAULT_CATALOG_NAME, DB_NAME, MV_NAME_1, - ImmutableSet.of( - DB_NAME + "." + TBL_NAME_1, - DB_NAME + "." + TBL_NAME_2)); - // Create txn list (highWatermark=4;minOpenTxn=Long.MAX_VALUE) - mockCM1.setValidTxnList("3:" + Long.MAX_VALUE + "::"); - when(mv1.getCreationMetadata()).thenReturn(mockCM1); - MaterializationsInvalidationCache.get().createMaterializedView(mockCM1.getDbName(), mockCM1.getTblName(), - mockCM1.getTablesUsed(), mockCM1.getValidTxnList()); - // Format <txnId>$<table_name>:<hwm>:<minOpenWriteId>:<open_writeids>:<abort_writeids>$<table_name> - Map<String, Materialization> invalidationInfos = - MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( - DB_NAME, ImmutableList.of(MV_NAME_1)); - Assert.assertTrue(invalidationInfos.isEmpty()); - id = 10; - BasicTxnInfo txn10 = createTxnInfo(DB_NAME, TBL_NAME_2, id); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, id, id, false); - id = 9; - BasicTxnInfo txn9 = createTxnInfo(DB_NAME, TBL_NAME_1, id); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, id, id, false); - // Cleanup (current = 12, duration = 4) -> Removes txn1, txn2, txn3 - removed = MaterializationsInvalidationCache.get().cleanup(8L); - Assert.assertEquals(0L, removed); - invalidationInfos = - MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( - DB_NAME, ImmutableList.of(MV_NAME_1)); - Assert.assertTrue(invalidationInfos.isEmpty()); - // Create mv2 - Table mv2 = mock(Table.class); - when(mv2.getDbName()).thenReturn(DB_NAME); - when(mv2.getTableName()).thenReturn(MV_NAME_2); - CreationMetadata mockCM2 = new CreationMetadata( - DEFAULT_CATALOG_NAME, DB_NAME, MV_NAME_2, - ImmutableSet.of( - DB_NAME + "." + TBL_NAME_1, - DB_NAME + "." + TBL_NAME_2)); - // Create txn list (highWatermark=10;minOpenTxn=Long.MAX_VALUE) - mockCM2.setValidTxnList("10:" + Long.MAX_VALUE + "::"); - when(mv2.getCreationMetadata()).thenReturn(mockCM2); - MaterializationsInvalidationCache.get().createMaterializedView(mockCM2.getDbName(), mockCM2.getTblName(), - mockCM2.getTablesUsed(), mockCM2.getValidTxnList()); - when(mv2.getCreationMetadata()).thenReturn(mockCM2); - invalidationInfos = - MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( - DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2)); - Assert.assertTrue(invalidationInfos.isEmpty()); - // Create tbl3 (nothing to do) - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_3, 11, 11, false); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_3, 18, 18, false); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, 14, 14, false); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, 17, 17, false); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, 16, 16, false); - // Cleanup (current = 20, duration = 4) -> Removes txn10, txn11 - removed = MaterializationsInvalidationCache.get().cleanup(16L); - Assert.assertEquals(0L, removed); - invalidationInfos = - MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( - DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2)); - Assert.assertTrue(invalidationInfos.isEmpty()); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, 12, 12, false); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, 15, 15, false); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, 7, 7, false); - invalidationInfos = - MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( - DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2)); - Assert.assertTrue(invalidationInfos.isEmpty()); - // Cleanup (current = 24, duration = 4) -> Removes txn9, txn14, txn15, txn16, txn17, txn18 - removed = MaterializationsInvalidationCache.get().cleanup(20L); - Assert.assertEquals(0L, removed); - invalidationInfos = - MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( - DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2)); - Assert.assertTrue(invalidationInfos.isEmpty()); - // Cleanup (current = 28, duration = 4) -> Removes txn9 - removed = MaterializationsInvalidationCache.get().cleanup(24L); - Assert.assertEquals(0L, removed); - } - - @Test - public void testCleanerScenario2() throws Exception { - // create mock raw store - Configuration conf = new Configuration(); - conf.set("metastore.materializations.invalidation.impl", "DEFAULT"); - // create mock handler - final IHMSHandler handler = mock(IHMSHandler.class); - // initialize invalidation cache (set conf to default) - MaterializationsInvalidationCache.get().init(conf, handler); - - // Scenario consists of the following steps: - // Create tbl1 - // (t = 1) Insert row in tbl1 - // (t = 2) Insert row in tbl1 - // Create tbl2 - // (t = 3) Insert row in tbl2 - // Cleanup (current = 4, duration = 4) -> Does nothing - // Create mv1 - // (t = 10) Insert row in tbl2 - // (t = 9) Insert row in tbl1 (out of order) - // Cleanup (current = 12, duration = 4) -> Removes txn1, txn2, txn3 - // Create mv2 - // Create tbl3 - // (t = 11) Insert row in tbl3 - // (t = 18) Insert row in tbl3 - // (t = 14) Insert row in tbl1 - // (t = 17) Insert row in tbl1 - // (t = 16) Insert row in tbl2 - // Cleanup (current = 20, duration = 4) -> Removes txn10, txn11 - // (t = 12) Insert row in tbl1 - // (t = 15) Insert row in tbl2 - // (t = 7) Insert row in tbl2 - // Cleanup (current = 24, duration = 4) -> Removes txn9, txn14, txn15, txn16, txn17, txn18 - // Create tbl1 (nothing to do) - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, 1, 1, false); - int id = 2; - BasicTxnInfo txn2 = createTxnInfo(DB_NAME, TBL_NAME_1, id); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, id, id, false); - // Create tbl2 (nothing to do) - id = 3; - BasicTxnInfo txn3 = createTxnInfo(DB_NAME, TBL_NAME_1, id); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, id, id, false); - // Cleanup (current = 4, duration = 4) -> Does nothing - long removed = MaterializationsInvalidationCache.get().cleanup(0L); - Assert.assertEquals(0L, removed); - // Create mv1 - Table mv1 = mock(Table.class); - when(mv1.getDbName()).thenReturn(DB_NAME); - when(mv1.getTableName()).thenReturn(MV_NAME_1); - CreationMetadata mockCM1 = new CreationMetadata( - DEFAULT_CATALOG_NAME, DB_NAME, MV_NAME_1, - ImmutableSet.of( - DB_NAME + "." + TBL_NAME_1, - DB_NAME + "." + TBL_NAME_2)); - // Create txn list (highWatermark=4;minOpenTxn=Long.MAX_VALUE) - mockCM1.setValidTxnList("3$" + DB_NAME + "." + TBL_NAME_1 + ":3:" + Long.MAX_VALUE + "::" + - "$" + DB_NAME + "." + TBL_NAME_2 + ":3:" + Long.MAX_VALUE + "::"); - when(mv1.getCreationMetadata()).thenReturn(mockCM1); - MaterializationsInvalidationCache.get().createMaterializedView(mockCM1.getDbName(), mockCM1.getTblName(), - mockCM1.getTablesUsed(), mockCM1.getValidTxnList()); - Map<String, Materialization> invalidationInfos = - MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( - DB_NAME, ImmutableList.of(MV_NAME_1)); - Assert.assertEquals(0L, invalidationInfos.get(MV_NAME_1).getInvalidationTime()); - id = 10; - BasicTxnInfo txn10 = createTxnInfo(DB_NAME, TBL_NAME_2, id); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, id, id, false); - id = 9; - BasicTxnInfo txn9 = createTxnInfo(DB_NAME, TBL_NAME_1, id); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, id, id, false); - // Cleanup (current = 12, duration = 4) -> Removes txn1, txn2, txn3 - removed = MaterializationsInvalidationCache.get().cleanup(8L); - Assert.assertEquals(3L, removed); - invalidationInfos = - MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( - DB_NAME, ImmutableList.of(MV_NAME_1)); - Assert.assertEquals(9L, invalidationInfos.get(MV_NAME_1).getInvalidationTime()); - // Create mv2 - Table mv2 = mock(Table.class); - when(mv2.getDbName()).thenReturn(DB_NAME); - when(mv2.getTableName()).thenReturn(MV_NAME_2); - CreationMetadata mockCM2 = new CreationMetadata( - DEFAULT_CATALOG_NAME, DB_NAME, MV_NAME_2, - ImmutableSet.of( - DB_NAME + "." + TBL_NAME_1, - DB_NAME + "." + TBL_NAME_2)); - // Create txn list (highWatermark=10;minOpenTxn=Long.MAX_VALUE) - mockCM2.setValidTxnList("10$" + DB_NAME + "." + TBL_NAME_1 + ":10:" + Long.MAX_VALUE + "::" + - "$" + DB_NAME + "." + TBL_NAME_2 + ":10:" + Long.MAX_VALUE + "::"); - when(mv2.getCreationMetadata()).thenReturn(mockCM2); - MaterializationsInvalidationCache.get().createMaterializedView(mockCM2.getDbName(), mockCM2.getTblName(), - mockCM2.getTablesUsed(), mockCM2.getValidTxnList()); - when(mv2.getCreationMetadata()).thenReturn(mockCM2); - invalidationInfos = - MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( - DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2)); - Assert.assertEquals(9L, invalidationInfos.get(MV_NAME_1).getInvalidationTime()); - Assert.assertEquals(0L, invalidationInfos.get(MV_NAME_2).getInvalidationTime()); - // Create tbl3 (nothing to do) - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_3, 11, 11, false); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_3, 18, 18, false); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, 14, 14, false); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, 17, 17, false); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, 16, 16, false); - // Cleanup (current = 20, duration = 4) -> Removes txn10, txn11 - removed = MaterializationsInvalidationCache.get().cleanup(16L); - Assert.assertEquals(2L, removed); - invalidationInfos = - MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( - DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2)); - Assert.assertEquals(9L, invalidationInfos.get(MV_NAME_1).getInvalidationTime()); - Assert.assertEquals(14L, invalidationInfos.get(MV_NAME_2).getInvalidationTime()); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, 12, 12, false); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, 15, 15, false); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, 7, 7, false); - invalidationInfos = - MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( - DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2)); - Assert.assertEquals(7L, invalidationInfos.get(MV_NAME_1).getInvalidationTime()); - Assert.assertEquals(12L, invalidationInfos.get(MV_NAME_2).getInvalidationTime()); - // Cleanup (current = 24, duration = 4) -> Removes txn9, txn14, txn15, txn16, txn17, txn18 - removed = MaterializationsInvalidationCache.get().cleanup(20L); - Assert.assertEquals(6L, removed); - invalidationInfos = - MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( - DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2)); - Assert.assertEquals(7L, invalidationInfos.get(MV_NAME_1).getInvalidationTime()); - Assert.assertEquals(12L, invalidationInfos.get(MV_NAME_2).getInvalidationTime()); - // Cleanup (current = 28, duration = 4) -> Removes txn9 - removed = MaterializationsInvalidationCache.get().cleanup(24L); - Assert.assertEquals(0L, removed); - } - - private static BasicTxnInfo createTxnInfo(String dbName, String tableName, int i) { - BasicTxnInfo r = new BasicTxnInfo(); - r.setDbname(dbName); - r.setTablename(tableName); - r.setTxnid(i); - r.setTime(i); - return r; - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/774a8ef7/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java index 0281182..c1eff55 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java @@ -1193,6 +1193,7 @@ public class TestTablesCreateDropAlterTruncate extends MetaStoreClientTest { // Update the metadata for the materialized view CreationMetadata cm = client.getTable(catName, dbName, tableNames[3]).getCreationMetadata(); cm.addToTablesUsed(dbName + "." + tableNames[1]); + cm.unsetMaterializationTime(); client.updateCreationMetadata(catName, dbName, tableNames[3], cm); List<String> partNames = new ArrayList<>();
