http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index 3785f89..9dd3787 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -26,10 +26,10 @@ import java.sql.SQLException; import java.sql.SQLFeatureNotSupportedException; import java.sql.Savepoint; import java.sql.Statement; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; -import java.util.Calendar; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -40,7 +40,6 @@ import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.SortedSet; -import java.util.TimeZone; import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Semaphore; @@ -59,11 +58,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.ValidReadTxnList; import org.apache.hadoop.hive.common.ValidReaderWriteIdList; import org.apache.hadoop.hive.common.ValidTxnList; +import org.apache.hadoop.hive.common.ValidTxnWriteIdList; import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.common.classification.RetrySemantics; import org.apache.hadoop.hive.metastore.DatabaseProduct; -import org.apache.hadoop.hive.metastore.MaterializationsInvalidationCache; -import org.apache.hadoop.hive.metastore.MaterializationsRebuildLockHandler; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.MetaStoreListenerNotifier; import org.apache.hadoop.hive.metastore.TransactionalMetaStoreEventListener; @@ -869,10 +867,7 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { @RetrySemantics.Idempotent("No-op if already committed") public void commitTxn(CommitTxnRequest rqst) throws NoSuchTxnException, TxnAbortedException, MetaException { - MaterializationsRebuildLockHandler materializationsRebuildLockHandler = - MaterializationsRebuildLockHandler.get(); - List<TransactionRegistryInfo> txnComponents = new ArrayList<>(); - boolean isUpdateDelete = false; + char isUpdateDelete = 'N'; long txnid = rqst.getTxnid(); long sourceTxnId = -1; @@ -936,7 +931,7 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { "tc_operation_type " + conflictSQLSuffix)); } if (rs != null && rs.next()) { - isUpdateDelete = true; + isUpdateDelete = 'Y'; close(rs); //if here it means currently committing txn performed update/delete and we should check WW conflict /** @@ -1033,8 +1028,8 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { // Move the record from txn_components into completed_txn_components so that the compactor // knows where to look to compact. s = "insert into COMPLETED_TXN_COMPONENTS (ctc_txnid, ctc_database, " + - "ctc_table, ctc_partition, ctc_writeid) select tc_txnid, tc_database, tc_table, " + - "tc_partition, tc_writeid from TXN_COMPONENTS where tc_txnid = " + txnid; + "ctc_table, ctc_partition, ctc_writeid, ctc_update_delete) select tc_txnid, tc_database, tc_table, " + + "tc_partition, tc_writeid, '" + isUpdateDelete + "' from TXN_COMPONENTS where tc_txnid = " + txnid; LOG.debug("Going to execute insert <" + s + ">"); if ((stmt.executeUpdate(s)) < 1) { @@ -1050,10 +1045,11 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { rows.add(txnid + "," + quoteString(writeEventInfo.getDatabase()) + "," + quoteString(writeEventInfo.getTable()) + "," + quoteString(writeEventInfo.getPartition()) + "," + - writeEventInfo.getWriteId()); + writeEventInfo.getWriteId() + "," + + "'" + isUpdateDelete + "'"); } List<String> queries = sqlGenerator.createInsertValuesStmt("COMPLETED_TXN_COMPONENTS " + - "(ctc_txnid," + " ctc_database, ctc_table, ctc_partition, ctc_writeid)", rows); + "(ctc_txnid," + " ctc_database, ctc_table, ctc_partition, ctc_writeid, ctc_update_delete)", rows); for (String q : queries) { LOG.debug("Going to execute insert <" + q + "> "); stmt.execute(q); @@ -1066,18 +1062,6 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { stmt.executeUpdate(s); } - // Obtain information that we need to update registry - s = "select ctc_database, ctc_table, ctc_writeid, ctc_timestamp from COMPLETED_TXN_COMPONENTS" + - " where ctc_txnid = " + txnid; - - LOG.debug("Going to extract table modification information for invalidation cache <" + s + ">"); - rs = stmt.executeQuery(s); - while (rs.next()) { - // We only enter in this loop if the transaction actually affected any table - txnComponents.add(new TransactionRegistryInfo(rs.getString(1), rs.getString(2), - rs.getLong(3), rs.getTimestamp(4, Calendar.getInstance(TimeZone.getTimeZone("UTC"))).getTime())); - } - // cleanup all txn related metadata s = "delete from TXN_COMPONENTS where tc_txnid = " + txnid; LOG.debug("Going to execute update <" + s + ">"); @@ -1092,29 +1076,19 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { LOG.debug("Going to execute update <" + s + ">"); stmt.executeUpdate(s); LOG.info("Removed committed transaction: (" + txnid + ") from MIN_HISTORY_LEVEL"); + + s = "delete from MATERIALIZATION_REBUILD_LOCKS where mrl_txn_id = " + txnid; + LOG.debug("Going to execute update <" + s + ">"); + stmt.executeUpdate(s); + if (transactionalListeners != null) { MetaStoreListenerNotifier.notifyEventWithDirectSql(transactionalListeners, EventMessage.EventType.COMMIT_TXN, new CommitTxnEvent(txnid, null), dbConn, sqlGenerator); } - MaterializationsInvalidationCache materializationsInvalidationCache = - MaterializationsInvalidationCache.get(); - for (TransactionRegistryInfo info : txnComponents) { - if (materializationsInvalidationCache.containsMaterialization(info.dbName, info.tblName) && - !materializationsRebuildLockHandler.readyToCommitResource(info.dbName, info.tblName, txnid)) { - throw new MetaException( - "Another process is rebuilding the materialized view " + info.fullyQualifiedName); - } - } LOG.debug("Going to commit"); close(rs); dbConn.commit(); - - // Update registry with modifications - for (TransactionRegistryInfo info : txnComponents) { - materializationsInvalidationCache.notifyTableModification( - info.dbName, info.tblName, info.writeId, info.timestamp, isUpdateDelete); - } } catch (SQLException e) { LOG.debug("Going to rollback"); rollbackDBConn(dbConn); @@ -1125,9 +1099,6 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { close(commitIdRs); close(lockHandle, stmt, dbConn); unlockInternal(); - for (TransactionRegistryInfo info : txnComponents) { - materializationsRebuildLockHandler.unlockResource(info.dbName, info.tblName, txnid); - } } } catch (RetryException e) { commitTxn(rqst); @@ -1694,16 +1665,30 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { } /** - * Gets the information of the first transaction for the given table - * after the transaction with the input id was committed (if any). + * Get invalidation info for the materialization. Currently, the materialization information + * only contains information about whether there was update/delete operations on the source + * tables used by the materialization since it was created. */ @Override @RetrySemantics.ReadOnly - public BasicTxnInfo getFirstCompletedTransactionForTableAfterCommit( - String inputDbName, String inputTableName, ValidWriteIdList txnList) - throws MetaException { - final List<Long> openTxns = Arrays.asList(ArrayUtils.toObject(txnList.getInvalidWriteIds())); + public Materialization getMaterializationInvalidationInfo( + CreationMetadata creationMetadata, String validTxnListStr) throws MetaException { + if (creationMetadata.getTablesUsed().isEmpty()) { + // Bail out + LOG.warn("Materialization creation metadata does not contain any table"); + return null; + } + + // Parse validTxnList + final ValidReadTxnList validTxnList = + new ValidReadTxnList(validTxnListStr); + + // Parse validReaderWriteIdList from creation metadata + final ValidTxnWriteIdList validReaderWriteIdList = + new ValidTxnWriteIdList(creationMetadata.getValidTxnList()); + // We are composing a query that returns a single row if an update happened after + // the materialization was created. Otherwise, query returns 0 rows. Connection dbConn = null; Statement stmt = null; ResultSet rs = null; @@ -1711,32 +1696,207 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); stmt = dbConn.createStatement(); stmt.setMaxRows(1); - String s = "select ctc_timestamp, ctc_writeid, ctc_database, ctc_table " - + "from COMPLETED_TXN_COMPONENTS " - + "where ctc_database=" + quoteString(inputDbName) + " and ctc_table=" + quoteString(inputTableName) - + " and ctc_writeid > " + txnList.getHighWatermark() - + (txnList.getInvalidWriteIds().length == 0 ? - " " : " or ctc_writeid IN(" + StringUtils.join(",", openTxns) + ") ") - + "order by ctc_timestamp asc"; + StringBuilder query = new StringBuilder(); + // compose a query that select transactions containing an update... + query.append("select ctc_update_delete from COMPLETED_TXN_COMPONENTS where ctc_update_delete='Y' AND ("); + int i = 0; + for (String fullyQualifiedName : creationMetadata.getTablesUsed()) { + // ...for each of the tables that are part of the materialized view, + // where the transaction had to be committed after the materialization was created... + if (i != 0) { + query.append("OR"); + } + String[] names = TxnUtils.getDbTableName(fullyQualifiedName); + query.append(" (ctc_database=" + quoteString(names[0]) + " AND ctc_table=" + quoteString(names[1])); + ValidWriteIdList tblValidWriteIdList = + validReaderWriteIdList.getTableValidWriteIdList(fullyQualifiedName); + if (tblValidWriteIdList == null) { + LOG.warn("ValidWriteIdList for table {} not present in creation metadata, this should not happen"); + return null; + } + query.append(" AND (ctc_writeid > " + tblValidWriteIdList.getHighWatermark()); + query.append(tblValidWriteIdList.getInvalidWriteIds().length == 0 ? ") " : + " OR ctc_writeid IN(" + StringUtils.join(",", + Arrays.asList(ArrayUtils.toObject(tblValidWriteIdList.getInvalidWriteIds()))) + ") "); + query.append(") "); + i++; + } + // ... and where the transaction has already been committed as per snapshot taken + // when we are running current query + query.append(") AND ctc_txnid <= " + validTxnList.getHighWatermark()); + query.append(validTxnList.getInvalidTransactions().length == 0 ? " " : + " AND ctc_txnid NOT IN(" + StringUtils.join(",", + Arrays.asList(ArrayUtils.toObject(validTxnList.getInvalidTransactions()))) + ") "); + + // Execute query + String s = query.toString(); if (LOG.isDebugEnabled()) { LOG.debug("Going to execute query <" + s + ">"); } rs = stmt.executeQuery(s); - if(!rs.next()) { - return new BasicTxnInfo(true); + return new Materialization(rs.next()); + } catch (SQLException ex) { + LOG.warn("getMaterializationInvalidationInfo failed due to " + getMessage(ex), ex); + throw new MetaException("Unable to retrieve materialization invalidation information due to " + + StringUtils.stringifyException(ex)); + } finally { + close(rs, stmt, dbConn); + } + } + + @Override + public LockResponse lockMaterializationRebuild(String dbName, String tableName, long txnId) + throws MetaException { + + if (LOG.isDebugEnabled()) { + LOG.debug("Acquiring lock for materialization rebuild with txnId={} for {}", txnId, Warehouse.getQualifiedName(dbName,tableName)); + } + + TxnStore.MutexAPI.LockHandle handle = null; + Connection dbConn = null; + Statement stmt = null; + ResultSet rs = null; + try { + lockInternal(); + /** + * MUTEX_KEY.MaterializationRebuild lock ensures that there is only 1 entry in + * Initiated/Working state for any resource. This ensures we do not run concurrent + * rebuild operations on any materialization. + */ + handle = getMutexAPI().acquireLock(MUTEX_KEY.MaterializationRebuild.name()); + dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); + stmt = dbConn.createStatement(); + + String selectQ = "select mrl_txn_id from MATERIALIZATION_REBUILD_LOCKS where" + + " mrl_db_name =" + quoteString(dbName) + + " AND mrl_tbl_name=" + quoteString(tableName); + LOG.debug("Going to execute query <" + selectQ + ">"); + rs = stmt.executeQuery(selectQ); + if(rs.next()) { + LOG.info("Ignoring request to rebuild " + dbName + "/" + tableName + + " since it is already being rebuilt"); + return new LockResponse(txnId, LockState.NOT_ACQUIRED); } - final BasicTxnInfo txnInfo = new BasicTxnInfo(false); - txnInfo.setTime(rs.getTimestamp(1, Calendar.getInstance(TimeZone.getTimeZone("UTC"))).getTime()); - txnInfo.setTxnid(rs.getLong(2)); - txnInfo.setDbname(rs.getString(3)); - txnInfo.setTablename(rs.getString(4)); - return txnInfo; + String insertQ = "insert into MATERIALIZATION_REBUILD_LOCKS " + + "(mrl_txn_id, mrl_db_name, mrl_tbl_name, mrl_last_heartbeat) values (" + txnId + + ", '" + dbName + "', '" + tableName + "', " + Instant.now().toEpochMilli() + ")"; + LOG.debug("Going to execute update <" + insertQ + ">"); + stmt.executeUpdate(insertQ); + LOG.debug("Going to commit"); + dbConn.commit(); + return new LockResponse(txnId, LockState.ACQUIRED); } catch (SQLException ex) { - LOG.warn("getLastCompletedTransactionForTable failed due to " + getMessage(ex), ex); - throw new MetaException("Unable to retrieve commits information due to " + StringUtils.stringifyException(ex)); + LOG.warn("lockMaterializationRebuild failed due to " + getMessage(ex), ex); + throw new MetaException("Unable to retrieve materialization invalidation information due to " + + StringUtils.stringifyException(ex)); } finally { close(rs, stmt, dbConn); + if(handle != null) { + handle.releaseLocks(); + } + unlockInternal(); + } + } + + @Override + public boolean heartbeatLockMaterializationRebuild(String dbName, String tableName, long txnId) + throws MetaException { + try { + Connection dbConn = null; + Statement stmt = null; + ResultSet rs = null; + try { + lockInternal(); + dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); + stmt = dbConn.createStatement(); + String s = "update MATERIALIZATION_REBUILD_LOCKS" + + " set mrl_last_heartbeat = " + Instant.now().toEpochMilli() + + " where mrl_txn_id = " + txnId + + " AND mrl_db_name =" + quoteString(dbName) + + " AND mrl_tbl_name=" + quoteString(tableName); + LOG.debug("Going to execute update <" + s + ">"); + int rc = stmt.executeUpdate(s); + if (rc < 1) { + LOG.debug("Going to rollback"); + dbConn.rollback(); + LOG.info("No lock found for rebuild of " + Warehouse.getQualifiedName(dbName, tableName) + + " when trying to heartbeat"); + // It could not be renewed, return that information + return false; + } + LOG.debug("Going to commit"); + dbConn.commit(); + // It could be renewed, return that information + return true; + } catch (SQLException e) { + LOG.debug("Going to rollback"); + rollbackDBConn(dbConn); + checkRetryable(dbConn, e, + "heartbeatLockMaterializationRebuild(" + Warehouse.getQualifiedName(dbName, tableName) + ", " + txnId + ")"); + throw new MetaException("Unable to heartbeat rebuild lock due to " + + StringUtils.stringifyException(e)); + } finally { + close(rs, stmt, dbConn); + unlockInternal(); + } + } catch (RetryException e) { + return heartbeatLockMaterializationRebuild(dbName, tableName ,txnId); + } + } + + @Override + public long cleanupMaterializationRebuildLocks(ValidTxnList validTxnList, long timeout) throws MetaException { + try { + // Aux values + long cnt = 0L; + List<Long> txnIds = new ArrayList<>(); + long timeoutTime = Instant.now().toEpochMilli() - timeout; + + Connection dbConn = null; + Statement stmt = null; + ResultSet rs = null; + try { + lockInternal(); + dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); + stmt = dbConn.createStatement(); + + String selectQ = "select mrl_txn_id, mrl_last_heartbeat from MATERIALIZATION_REBUILD_LOCKS"; + LOG.debug("Going to execute query <" + selectQ + ">"); + rs = stmt.executeQuery(selectQ); + while(rs.next()) { + long lastHeartbeat = rs.getLong(2); + if (lastHeartbeat < timeoutTime) { + // The heartbeat has timeout, double check whether we can remove it + long txnId = rs.getLong(1); + if (validTxnList.isTxnValid(txnId) || validTxnList.isTxnAborted(txnId)) { + // Txn was committed (but notification was not received) or it was aborted. + // Either case, we can clean it up + txnIds.add(txnId); + } + } + } + if (!txnIds.isEmpty()) { + String deleteQ = "delete from MATERIALIZATION_REBUILD_LOCKS where" + + " mrl_txn_id IN(" + StringUtils.join(",", txnIds) + ") "; + LOG.debug("Going to execute update <" + deleteQ + ">"); + cnt = stmt.executeUpdate(deleteQ); + } + LOG.debug("Going to commit"); + dbConn.commit(); + return cnt; + } catch (SQLException e) { + LOG.debug("Going to rollback"); + rollbackDBConn(dbConn); + checkRetryable(dbConn, e, "cleanupMaterializationRebuildLocks"); + throw new MetaException("Unable to clean rebuild locks due to " + + StringUtils.stringifyException(e)); + } finally { + close(rs, stmt, dbConn); + unlockInternal(); + } + } catch (RetryException e) { + return cleanupMaterializationRebuildLocks(validTxnList, timeout); } } @@ -2009,6 +2169,7 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { private static String normalizeCase(String s) { return s == null ? null : s.toLowerCase(); } + private LockResponse checkLockWithRetry(Connection dbConn, long extLockId, long txnId) throws NoSuchLockException, NoSuchTxnException, TxnAbortedException, MetaException { try { @@ -4887,20 +5048,4 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { } }; - private class TransactionRegistryInfo { - final String dbName; - final String tblName; - final String fullyQualifiedName; - final long writeId; - final long timestamp; - - public TransactionRegistryInfo (String dbName, String tblName, long writeId, long timestamp) { - this.dbName = dbName; - this.tblName = tblName; - this.fullyQualifiedName = Warehouse.getQualifiedName(dbName, tblName); - this.writeId = writeId; - this.timestamp = timestamp; - } - } - }
http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java index d972d10..33f24fb 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java @@ -21,6 +21,7 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.common.classification.RetrySemantics; import org.apache.hadoop.hive.metastore.api.*; @@ -29,6 +30,7 @@ import org.apache.hadoop.hive.metastore.events.AcidWriteEvent; import java.sql.SQLException; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; /** @@ -41,7 +43,7 @@ public interface TxnStore extends Configurable { enum MUTEX_KEY { Initiator, Cleaner, HouseKeeper, CompactionHistory, CheckLock, - WriteSetCleaner, CompactionScheduler, WriteIdAllocator + WriteSetCleaner, CompactionScheduler, WriteIdAllocator, MaterializationRebuild } // Compactor states (Should really be enum) String INITIATED_RESPONSE = "initiated"; @@ -128,21 +130,33 @@ public interface TxnStore extends Configurable { void replTableWriteIdState(ReplTblWriteIdStateRequest rqst) throws MetaException; /** - * Get the first transaction corresponding to given database and table after transactions - * referenced in the transaction snapshot. - * @return + * Get invalidation info for the materialization. Currently, the materialization information + * only contains information about whether there was update/delete operations on the source + * tables used by the materialization since it was created. + * @param cm creation metadata for the materialization + * @param validTxnList valid transaction list for snapshot taken for current query * @throws MetaException */ @RetrySemantics.Idempotent - BasicTxnInfo getFirstCompletedTransactionForTableAfterCommit( - String inputDbName, String inputTableName, ValidWriteIdList txnList) + Materialization getMaterializationInvalidationInfo( + final CreationMetadata cm, final String validTxnList) throws MetaException; - /** - * Gets the list of valid write ids for the given table wrt to current txn - * @param rqst info on transaction and list of table names associated with given transaction - * @throws NoSuchTxnException - * @throws MetaException - */ + + LockResponse lockMaterializationRebuild(String dbName, String tableName, long txnId) + throws MetaException; + + boolean heartbeatLockMaterializationRebuild(String dbName, String tableName, long txnId) + throws MetaException; + + long cleanupMaterializationRebuildLocks(ValidTxnList validTxnList, long timeout) + throws MetaException; + + /** + * Gets the list of valid write ids for the given table wrt to current txn + * @param rqst info on transaction and list of table names associated with given transaction + * @throws NoSuchTxnException + * @throws MetaException + */ @RetrySemantics.ReadOnly GetValidWriteIdsResponse getValidWriteIds(GetValidWriteIdsRequest rqst) throws NoSuchTxnException, MetaException; http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/derby/hive-schema-3.1.0.derby.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/sql/derby/hive-schema-3.1.0.derby.sql b/standalone-metastore/metastore-common/src/main/sql/derby/hive-schema-3.1.0.derby.sql index a696d06..5e8693e 100644 --- a/standalone-metastore/metastore-common/src/main/sql/derby/hive-schema-3.1.0.derby.sql +++ b/standalone-metastore/metastore-common/src/main/sql/derby/hive-schema-3.1.0.derby.sql @@ -197,7 +197,8 @@ CREATE TABLE "APP"."MV_CREATION_METADATA" ( "CAT_NAME" VARCHAR(256) NOT NULL, "DB_NAME" VARCHAR(128) NOT NULL, "TBL_NAME" VARCHAR(256) NOT NULL, - "TXN_LIST" CLOB + "TXN_LIST" CLOB, + "MATERIALIZATION_TIME" BIGINT NOT NULL ); CREATE TABLE "APP"."MV_TABLES_USED" ( @@ -526,7 +527,8 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS ( CTC_TABLE varchar(256), CTC_PARTITION varchar(767), CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID bigint + CTC_WRITEID bigint, + CTC_UPDATE_DELETE char(1) NOT NULL ); CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); @@ -645,6 +647,14 @@ CREATE TABLE MIN_HISTORY_LEVEL ( CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID); +CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( + MRL_TXN_ID BIGINT NOT NULL, + MRL_DB_NAME VARCHAR(128) NOT NULL, + MRL_TBL_NAME VARCHAR(256) NOT NULL, + MRL_LAST_HEARTBEAT BIGINT NOT NULL, + PRIMARY KEY(MRL_TXN_ID) +); + CREATE TABLE "APP"."I_SCHEMA" ( "SCHEMA_ID" bigint primary key, "SCHEMA_TYPE" integer not null, http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/derby/hive-schema-4.0.0.derby.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/sql/derby/hive-schema-4.0.0.derby.sql b/standalone-metastore/metastore-common/src/main/sql/derby/hive-schema-4.0.0.derby.sql index 7cab4fb..5ba71c4 100644 --- a/standalone-metastore/metastore-common/src/main/sql/derby/hive-schema-4.0.0.derby.sql +++ b/standalone-metastore/metastore-common/src/main/sql/derby/hive-schema-4.0.0.derby.sql @@ -197,7 +197,8 @@ CREATE TABLE "APP"."MV_CREATION_METADATA" ( "CAT_NAME" VARCHAR(256) NOT NULL, "DB_NAME" VARCHAR(128) NOT NULL, "TBL_NAME" VARCHAR(256) NOT NULL, - "TXN_LIST" CLOB + "TXN_LIST" CLOB, + "MATERIALIZATION_TIME" BIGINT NOT NULL ); CREATE TABLE "APP"."MV_TABLES_USED" ( @@ -526,7 +527,8 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS ( CTC_TABLE varchar(256), CTC_PARTITION varchar(767), CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID bigint + CTC_WRITEID bigint, + CTC_UPDATE_DELETE char(1) NOT NULL ); CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); @@ -645,6 +647,14 @@ CREATE TABLE MIN_HISTORY_LEVEL ( CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID); +CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( + MRL_TXN_ID BIGINT NOT NULL, + MRL_DB_NAME VARCHAR(128) NOT NULL, + MRL_TBL_NAME VARCHAR(256) NOT NULL, + MRL_LAST_HEARTBEAT BIGINT NOT NULL, + PRIMARY KEY(MRL_TXN_ID) +); + CREATE TABLE "APP"."I_SCHEMA" ( "SCHEMA_ID" bigint primary key, "SCHEMA_TYPE" integer not null, http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql b/standalone-metastore/metastore-common/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql index 7058ab0..2b200f2 100644 --- a/standalone-metastore/metastore-common/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql +++ b/standalone-metastore/metastore-common/src/main/sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql @@ -45,5 +45,24 @@ CREATE TABLE TXN_WRITE_NOTIFICATION_LOG ( ); INSERT INTO SEQUENCE_TABLE (SEQUENCE_NAME, NEXT_VAL) VALUES ('org.apache.hadoop.hive.metastore.model.MTxnWriteNotificationLog', 1); +-- HIVE-19027 +-- add column MATERIALIZATION_TIME (bigint) to MV_CREATION_METADATA table +ALTER TABLE "APP"."MV_CREATION_METADATA" ADD COLUMN "MATERIALIZATION_TIME" BIGINT; +UPDATE "APP"."MV_CREATION_METADATA" SET "MATERIALIZATION_TIME" = 0; +ALTER TABLE "APP"."MV_CREATION_METADATA" ALTER COLUMN "MATERIALIZATION_TIME" NOT NULL; + +-- add column CTC_UPDATE_DELETE (char) to COMPLETED_TXN_COMPONENTS table +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD COLUMN CTC_UPDATE_DELETE char(1); +UPDATE COMPLETED_TXN_COMPONENTS SET CTC_UPDATE_DELETE = 'N'; +ALTER TABLE COMPLETED_TXN_COMPONENTS ALTER COLUMN CTC_UPDATE_DELETE NOT NULL; + +CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( + MRL_TXN_ID BIGINT NOT NULL, + MRL_DB_NAME VARCHAR(128) NOT NULL, + MRL_TBL_NAME VARCHAR(256) NOT NULL, + MRL_LAST_HEARTBEAT BIGINT NOT NULL, + PRIMARY KEY(MRL_TXN_ID) +); + -- This needs to be the last thing done. Insert any changes above this line. UPDATE "APP".VERSION SET SCHEMA_VERSION='3.1.0', VERSION_COMMENT='Hive release version 3.1.0' where VER_ID=1; http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql b/standalone-metastore/metastore-common/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql index d7722dc..446ee6e 100644 --- a/standalone-metastore/metastore-common/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql +++ b/standalone-metastore/metastore-common/src/main/sql/mssql/hive-schema-3.1.0.mssql.sql @@ -388,7 +388,8 @@ CREATE TABLE MV_CREATION_METADATA CAT_NAME nvarchar(256) NOT NULL, DB_NAME nvarchar(128) NOT NULL, TBL_NAME nvarchar(256) NOT NULL, - TXN_LIST text NULL + TXN_LIST text NULL, + MATERIALIZATION_TIME bigint NOT NULL ); ALTER TABLE MV_CREATION_METADATA ADD CONSTRAINT MV_CREATION_METADATA_PK PRIMARY KEY (MV_CREATION_METADATA_ID); @@ -1034,7 +1035,8 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS( CTC_TABLE nvarchar(128) NULL, CTC_PARTITION nvarchar(767) NULL, CTC_TIMESTAMP datetime2 DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID bigint + CTC_WRITEID bigint, + CTC_UPDATE_DELETE char(1) NOT NULL ); CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); @@ -1191,6 +1193,17 @@ PRIMARY KEY CLUSTERED CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID); +CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( + MRL_TXN_ID bigint NOT NULL, + MRL_DB_NAME nvarchar(128) NOT NULL, + MRL_TBL_NAME nvarchar(256) NOT NULL, + MRL_LAST_HEARTBEAT bigint NOT NULL, +PRIMARY KEY CLUSTERED +( + MRL_TXN_ID ASC +) +); + CREATE TABLE "I_SCHEMA" ( "SCHEMA_ID" bigint primary key, "SCHEMA_TYPE" int not null, http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql b/standalone-metastore/metastore-common/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql index a81fc40..bbc8ea2 100644 --- a/standalone-metastore/metastore-common/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql +++ b/standalone-metastore/metastore-common/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql @@ -248,7 +248,6 @@ CREATE TABLE TAB_COL_STATS ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_PK PRIMARY KEY (CS_ID); CREATE INDEX TAB_COL_STATS_IDX ON TAB_COL_STATS (CAT_NAME, DB_NAME, TABLE_NAME, COLUMN_NAME); - -- Table TYPES for classes [org.apache.hadoop.hive.metastore.model.MType] CREATE TABLE TYPES ( @@ -389,7 +388,8 @@ CREATE TABLE MV_CREATION_METADATA CAT_NAME nvarchar(256) NOT NULL, DB_NAME nvarchar(128) NOT NULL, TBL_NAME nvarchar(256) NOT NULL, - TXN_LIST text NULL + TXN_LIST text NULL, + MATERIALIZATION_TIME bigint NOT NULL ); ALTER TABLE MV_CREATION_METADATA ADD CONSTRAINT MV_CREATION_METADATA_PK PRIMARY KEY (MV_CREATION_METADATA_ID); @@ -1035,7 +1035,8 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS( CTC_TABLE nvarchar(128) NULL, CTC_PARTITION nvarchar(767) NULL, CTC_TIMESTAMP datetime2 DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID bigint + CTC_WRITEID bigint, + CTC_UPDATE_DELETE char(1) NOT NULL ); CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); @@ -1192,6 +1193,17 @@ PRIMARY KEY CLUSTERED CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID); +CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( + MRL_TXN_ID bigint NOT NULL, + MRL_DB_NAME nvarchar(128) NOT NULL, + MRL_TBL_NAME nvarchar(256) NOT NULL, + MRL_LAST_HEARTBEAT bigint NOT NULL, +PRIMARY KEY CLUSTERED +( + MRL_TXN_ID ASC +) +); + CREATE TABLE "I_SCHEMA" ( "SCHEMA_ID" bigint primary key, "SCHEMA_TYPE" int not null, http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql b/standalone-metastore/metastore-common/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql index 41f23f7..d44cfdb 100644 --- a/standalone-metastore/metastore-common/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql +++ b/standalone-metastore/metastore-common/src/main/sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql @@ -46,6 +46,25 @@ CREATE TABLE TXN_WRITE_NOTIFICATION_LOG ( ALTER TABLE TXN_WRITE_NOTIFICATION_LOG ADD CONSTRAINT TXN_WRITE_NOTIFICATION_LOG_PK PRIMARY KEY (WNL_TXNID, WNL_DATABASE, WNL_TABLE, WNL_PARTITION); INSERT INTO SEQUENCE_TABLE (SEQUENCE_NAME, NEXT_VAL) VALUES ('org.apache.hadoop.hive.metastore.model.MTxnWriteNotificationLog', 1); +-- HIVE-19027 +-- add column MATERIALIZATION_TIME (bigint) to MV_CREATION_METADATA table +ALTER TABLE MV_CREATION_METADATA ADD MATERIALIZATION_TIME bigint NOT NULL DEFAULT(0); + +-- add column CTC_UPDATE_DELETE (char) to COMPLETED_TXN_COMPONENTS table +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_UPDATE_DELETE char(1) NOT NULL DEFAULT('N'); + +CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( + MRL_TXN_ID bigint NOT NULL, + MRL_DB_NAME nvarchar(128) NOT NULL, + MRL_TBL_NAME nvarchar(256) NOT NULL, + MRL_LAST_HEARTBEAT bigint NOT NULL, +PRIMARY KEY CLUSTERED +( + MRL_TXN_ID ASC +) +); + + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.1.0', VERSION_COMMENT='Hive release version 3.1.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0' AS MESSAGE; http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql b/standalone-metastore/metastore-common/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql index 29d4a43..75612a7 100644 --- a/standalone-metastore/metastore-common/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql +++ b/standalone-metastore/metastore-common/src/main/sql/mysql/hive-schema-3.1.0.mysql.sql @@ -603,6 +603,7 @@ CREATE TABLE IF NOT EXISTS `MV_CREATION_METADATA` ( `DB_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, `TBL_NAME` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, `TXN_LIST` TEXT DEFAULT NULL, + `MATERIALIZATION_TIME` bigint(20) NOT NULL, PRIMARY KEY (`MV_CREATION_METADATA_ID`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; /*!40101 SET character_set_client = @saved_cs_client */; @@ -1006,7 +1007,8 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS ( CTC_TABLE varchar(256), CTC_PARTITION varchar(767), CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID bigint + CTC_WRITEID bigint, + CTC_UPDATE_DELETE char(1) NOT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1; CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION) USING BTREE; @@ -1124,6 +1126,14 @@ CREATE TABLE MIN_HISTORY_LEVEL ( CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID); +CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( + MRL_TXN_ID bigint NOT NULL, + MRL_DB_NAME VARCHAR(128) NOT NULL, + MRL_TBL_NAME VARCHAR(256) NOT NULL, + MRL_LAST_HEARTBEAT bigint NOT NULL, + PRIMARY KEY(MRL_TXN_ID) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + CREATE TABLE `I_SCHEMA` ( `SCHEMA_ID` BIGINT PRIMARY KEY, `SCHEMA_TYPE` INTEGER NOT NULL, http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql b/standalone-metastore/metastore-common/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql index 968f4a4..d53e7fc 100644 --- a/standalone-metastore/metastore-common/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql +++ b/standalone-metastore/metastore-common/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql @@ -603,6 +603,7 @@ CREATE TABLE IF NOT EXISTS `MV_CREATION_METADATA` ( `DB_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, `TBL_NAME` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, `TXN_LIST` TEXT DEFAULT NULL, + `MATERIALIZATION_TIME` bigint(20) NOT NULL, PRIMARY KEY (`MV_CREATION_METADATA_ID`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; /*!40101 SET character_set_client = @saved_cs_client */; @@ -1006,7 +1007,8 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS ( CTC_TABLE varchar(256), CTC_PARTITION varchar(767), CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID bigint + CTC_WRITEID bigint, + CTC_UPDATE_DELETE char(1) NOT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1; CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION) USING BTREE; @@ -1124,6 +1126,14 @@ CREATE TABLE MIN_HISTORY_LEVEL ( CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID); +CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( + MRL_TXN_ID bigint NOT NULL, + MRL_DB_NAME VARCHAR(128) NOT NULL, + MRL_TBL_NAME VARCHAR(256) NOT NULL, + MRL_LAST_HEARTBEAT bigint NOT NULL, + PRIMARY KEY(MRL_TXN_ID) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + CREATE TABLE `I_SCHEMA` ( `SCHEMA_ID` BIGINT PRIMARY KEY, `SCHEMA_TYPE` INTEGER NOT NULL, http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql b/standalone-metastore/metastore-common/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql index e103bef..7752e89 100644 --- a/standalone-metastore/metastore-common/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql +++ b/standalone-metastore/metastore-common/src/main/sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql @@ -46,6 +46,26 @@ CREATE TABLE TXN_WRITE_NOTIFICATION_LOG ( ) ENGINE=InnoDB DEFAULT CHARSET=latin1; INSERT INTO `SEQUENCE_TABLE` (`SEQUENCE_NAME`, `NEXT_VAL`) VALUES ('org.apache.hadoop.hive.metastore.model.MTxnWriteNotificationLog', 1); +-- HIVE-19027 +-- add column MATERIALIZATION_TIME (bigint) to MV_CREATION_METADATA table +ALTER TABLE `MV_CREATION_METADATA` ADD `MATERIALIZATION_TIME` BIGINT; +UPDATE `MV_CREATION_METADATA` SET `MATERIALIZATION_TIME` = 0; +ALTER TABLE `MV_CREATION_METADATA` MODIFY COLUMN `MATERIALIZATION_TIME` BIGINT NOT NULL; + +-- add column CTC_UPDATE_DELETE (char) to COMPLETED_TXN_COMPONENTS table +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_UPDATE_DELETE char(1); +UPDATE COMPLETED_TXN_COMPONENTS SET CTC_UPDATE_DELETE = 'N'; +ALTER TABLE COMPLETED_TXN_COMPONENTS MODIFY COLUMN CTC_UPDATE_DELETE char(1) NOT NULL; + +CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( + MRL_TXN_ID BIGINT NOT NULL, + MRL_DB_NAME VARCHAR(128) NOT NULL, + MRL_TBL_NAME VARCHAR(256) NOT NULL, + MRL_LAST_HEARTBEAT BIGINT NOT NULL, + PRIMARY KEY(MRL_TXN_ID) +); + + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.1.0', VERSION_COMMENT='Hive release version 3.1.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0' AS ' '; http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql b/standalone-metastore/metastore-common/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql index 9adea31..a4720c8 100644 --- a/standalone-metastore/metastore-common/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql +++ b/standalone-metastore/metastore-common/src/main/sql/oracle/hive-schema-3.1.0.oracle.sql @@ -410,7 +410,8 @@ CREATE TABLE MV_CREATION_METADATA CAT_NAME VARCHAR2(256) NOT NULL, DB_NAME VARCHAR2(128) NOT NULL, TBL_NAME VARCHAR2(256) NOT NULL, - TXN_LIST CLOB NULL + TXN_LIST CLOB NULL, + MATERIALIZATION_TIME NUMBER NOT NULL ); ALTER TABLE MV_CREATION_METADATA ADD CONSTRAINT MV_CREATION_METADATA_PK PRIMARY KEY (MV_CREATION_METADATA_ID); @@ -983,7 +984,8 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS ( CTC_TABLE VARCHAR2(256), CTC_PARTITION VARCHAR2(767), CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID NUMBER(19) + CTC_WRITEID NUMBER(19), + CTC_UPDATE_DELETE CHAR(1) NOT NULL ) ROWDEPENDENCIES; CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); @@ -1100,6 +1102,14 @@ CREATE TABLE MIN_HISTORY_LEVEL ( CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID); +CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( + MRL_TXN_ID NUMBER NOT NULL, + MRL_DB_NAME VARCHAR(128) NOT NULL, + MRL_TBL_NAME VARCHAR(256) NOT NULL, + MRL_LAST_HEARTBEAT NUMBER NOT NULL, + PRIMARY KEY(MRL_TXN_ID) +); + CREATE TABLE "I_SCHEMA" ( "SCHEMA_ID" number primary key, "SCHEMA_TYPE" number not null, http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql b/standalone-metastore/metastore-common/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql index faca669..e58ee33 100644 --- a/standalone-metastore/metastore-common/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql +++ b/standalone-metastore/metastore-common/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql @@ -410,7 +410,8 @@ CREATE TABLE MV_CREATION_METADATA CAT_NAME VARCHAR2(256) NOT NULL, DB_NAME VARCHAR2(128) NOT NULL, TBL_NAME VARCHAR2(256) NOT NULL, - TXN_LIST CLOB NULL + TXN_LIST CLOB NULL, + MATERIALIZATION_TIME NUMBER NOT NULL ); ALTER TABLE MV_CREATION_METADATA ADD CONSTRAINT MV_CREATION_METADATA_PK PRIMARY KEY (MV_CREATION_METADATA_ID); @@ -983,7 +984,8 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS ( CTC_TABLE VARCHAR2(256), CTC_PARTITION VARCHAR2(767), CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID NUMBER(19) + CTC_WRITEID NUMBER(19), + CTC_UPDATE_DELETE CHAR(1) NOT NULL ) ROWDEPENDENCIES; CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); @@ -1100,6 +1102,14 @@ CREATE TABLE MIN_HISTORY_LEVEL ( CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID); +CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( + MRL_TXN_ID NUMBER NOT NULL, + MRL_DB_NAME VARCHAR(128) NOT NULL, + MRL_TBL_NAME VARCHAR(256) NOT NULL, + MRL_LAST_HEARTBEAT NUMBER NOT NULL, + PRIMARY KEY(MRL_TXN_ID) +); + CREATE TABLE "I_SCHEMA" ( "SCHEMA_ID" number primary key, "SCHEMA_TYPE" number not null, http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql b/standalone-metastore/metastore-common/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql index cf8699b..e4efe4d 100644 --- a/standalone-metastore/metastore-common/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql +++ b/standalone-metastore/metastore-common/src/main/sql/oracle/upgrade-3.0.0-to-3.1.0.oracle.sql @@ -46,6 +46,25 @@ CREATE TABLE TXN_WRITE_NOTIFICATION_LOG ( ); INSERT INTO SEQUENCE_TABLE (SEQUENCE_NAME, NEXT_VAL) VALUES ('org.apache.hadoop.hive.metastore.model.MTxnWriteNotificationLog', 1); +-- HIVE-19027 +-- add column MATERIALIZATION_TIME (bigint) to MV_CREATION_METADATA table +ALTER TABLE MV_CREATION_METADATA ADD MATERIALIZATION_TIME NUMBER NULL; +UPDATE MV_CREATION_METADATA SET MATERIALIZATION_TIME = 0; +ALTER TABLE MV_CREATION_METADATA MODIFY(MATERIALIZATION_TIME NOT NULL); + +-- add column CTC_UPDATE_DELETE (char) to COMPLETED_TXN_COMPONENTS table +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_UPDATE_DELETE char(1) NULL; +UPDATE COMPLETED_TXN_COMPONENTS SET CTC_UPDATE_DELETE = 'N'; +ALTER TABLE COMPLETED_TXN_COMPONENTS MODIFY(CTC_UPDATE_DELETE NOT NULL); + +CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( + MRL_TXN_ID NUMBER NOT NULL, + MRL_DB_NAME VARCHAR(128) NOT NULL, + MRL_TBL_NAME VARCHAR(256) NOT NULL, + MRL_LAST_HEARTBEAT NUMBER NOT NULL, + PRIMARY KEY(MRL_TXN_ID) +); + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.1.0', VERSION_COMMENT='Hive release version 3.1.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0' AS Status from dual; http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql b/standalone-metastore/metastore-common/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql index 7a8a419..a74c388 100644 --- a/standalone-metastore/metastore-common/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql +++ b/standalone-metastore/metastore-common/src/main/sql/postgres/hive-schema-3.1.0.postgres.sql @@ -404,7 +404,8 @@ CREATE TABLE "MV_CREATION_METADATA" ( "CAT_NAME" character varying(256) NOT NULL, "DB_NAME" character varying(128) NOT NULL, "TBL_NAME" character varying(256) NOT NULL, - "TXN_LIST" text + "TXN_LIST" text, + "MATERIALIZATION_TIME" bigint NOT NULL ); -- @@ -1673,7 +1674,8 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS ( CTC_TABLE varchar(256), CTC_PARTITION varchar(767), CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID bigint + CTC_WRITEID bigint, + CTC_UPDATE_DELETE char(1) NOT NULL ); CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON COMPLETED_TXN_COMPONENTS USING btree (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); @@ -1790,6 +1792,14 @@ CREATE TABLE MIN_HISTORY_LEVEL ( CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID); +CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( + MRL_TXN_ID bigint NOT NULL, + MRL_DB_NAME varchar(128) NOT NULL, + MRL_TBL_NAME varchar(256) NOT NULL, + MRL_LAST_HEARTBEAT bigint NOT NULL, + PRIMARY KEY(MRL_TXN_ID) +); + CREATE TABLE "I_SCHEMA" ( "SCHEMA_ID" bigint primary key, "SCHEMA_TYPE" integer not null, http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql b/standalone-metastore/metastore-common/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql index 2e7ac5a..5d1a525 100644 --- a/standalone-metastore/metastore-common/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql +++ b/standalone-metastore/metastore-common/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql @@ -404,7 +404,8 @@ CREATE TABLE "MV_CREATION_METADATA" ( "CAT_NAME" character varying(256) NOT NULL, "DB_NAME" character varying(128) NOT NULL, "TBL_NAME" character varying(256) NOT NULL, - "TXN_LIST" text + "TXN_LIST" text, + "MATERIALIZATION_TIME" bigint NOT NULL ); -- @@ -1282,6 +1283,11 @@ CREATE INDEX "TBLS_N50" ON "TBLS" USING btree ("SD_ID"); CREATE INDEX "TBL_COL_PRIVS_N49" ON "TBL_COL_PRIVS" USING btree ("TBL_ID"); +-- +-- Name: TAB_COL_STATS_IDX; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "TAB_COL_STATS_IDX" ON "TAB_COL_STATS" USING btree ("CAT_NAME", "DB_NAME","TABLE_NAME","COLUMN_NAME"); -- -- Name: TBL_PRIVS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: @@ -1303,13 +1309,6 @@ CREATE INDEX "TYPE_FIELDS_N49" ON "TYPE_FIELDS" USING btree ("TYPE_NAME"); CREATE INDEX "TAB_COL_STATS_N49" ON "TAB_COL_STATS" USING btree ("TBL_ID"); -- --- Name: TAB_COL_STATS_IDX; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: --- - -CREATE INDEX "TAB_COL_STATS_IDX" ON "TAB_COL_STATS" USING btree ("CAT_NAME", "DB_NAME","TABLE_NAME","COLUMN_NAME"); - - --- -- Name: PART_COL_STATS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: -- @@ -1675,7 +1674,8 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS ( CTC_TABLE varchar(256), CTC_PARTITION varchar(767), CTC_TIMESTAMP timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL, - CTC_WRITEID bigint + CTC_WRITEID bigint, + CTC_UPDATE_DELETE char(1) NOT NULL ); CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON COMPLETED_TXN_COMPONENTS USING btree (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); @@ -1792,6 +1792,14 @@ CREATE TABLE MIN_HISTORY_LEVEL ( CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID); +CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( + MRL_TXN_ID bigint NOT NULL, + MRL_DB_NAME varchar(128) NOT NULL, + MRL_TBL_NAME varchar(256) NOT NULL, + MRL_LAST_HEARTBEAT bigint NOT NULL, + PRIMARY KEY(MRL_TXN_ID) +); + CREATE TABLE "I_SCHEMA" ( "SCHEMA_ID" bigint primary key, "SCHEMA_TYPE" integer not null, http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql b/standalone-metastore/metastore-common/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql index 445c3a2..dadf065 100644 --- a/standalone-metastore/metastore-common/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql +++ b/standalone-metastore/metastore-common/src/main/sql/postgres/upgrade-3.0.0-to-3.1.0.postgres.sql @@ -48,6 +48,25 @@ CREATE TABLE "TXN_WRITE_NOTIFICATION_LOG" ( ); INSERT INTO "SEQUENCE_TABLE" ("SEQUENCE_NAME", "NEXT_VAL") VALUES ('org.apache.hadoop.hive.metastore.model.MTxnWriteNotificationLog', 1); +-- HIVE-19027 +-- add column MATERIALIZATION_TIME (bigint) to MV_CREATION_METADATA table +ALTER TABLE "MV_CREATION_METADATA" ADD COLUMN "MATERIALIZATION_TIME" bigint NULL; +UPDATE "MV_CREATION_METADATA" SET "MATERIALIZATION_TIME" = 0; +ALTER TABLE "MV_CREATION_METADATA" ALTER COLUMN "MATERIALIZATION_TIME" SET NOT NULL; + +-- add column CTC_UPDATE_DELETE (char) to COMPLETED_TXN_COMPONENTS table +ALTER TABLE COMPLETED_TXN_COMPONENTS ADD COLUMN CTC_UPDATE_DELETE char(1) NULL; +UPDATE COMPLETED_TXN_COMPONENTS SET CTC_UPDATE_DELETE = 'N'; +ALTER TABLE COMPLETED_TXN_COMPONENTS ALTER COLUMN CTC_UPDATE_DELETE SET NOT NULL; + +CREATE TABLE MATERIALIZATION_REBUILD_LOCKS ( + MRL_TXN_ID bigint NOT NULL, + MRL_DB_NAME varchar(128) NOT NULL, + MRL_TBL_NAME varchar(256) NOT NULL, + MRL_LAST_HEARTBEAT bigint NOT NULL, + PRIMARY KEY(MRL_TXN_ID) +); + -- These lines need to be last. Insert any changes above. UPDATE "VERSION" SET "SCHEMA_VERSION"='3.1.0', "VERSION_COMMENT"='Hive release version 3.1.0' where "VER_ID"=1; SELECT 'Finished upgrading MetaStore schema from 3.0.0 to 3.1.0'; http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift index 1ca6454..8965059 100644 --- a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift +++ b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift @@ -1073,6 +1073,7 @@ struct CreationMetadata { 3: required string tblName, 4: required set<string> tablesUsed, 5: optional string validTxnList, + 6: optional i64 materializationTime } struct NotificationEventRequest { @@ -1273,10 +1274,7 @@ struct TableMeta { } struct Materialization { - 1: required set<string> tablesUsed; - 2: optional string validTxnList - 3: optional i64 invalidationTime; - 4: optional bool sourceTablesUpdateDeleteModified; + 1: required bool sourceTablesUpdateDeleteModified; } // Data types for workload management. @@ -1728,7 +1726,7 @@ service ThriftHiveMetastore extends fb303.FacebookService GetTableResult get_table_req(1:GetTableRequest req) throws (1:MetaException o1, 2:NoSuchObjectException o2) GetTablesResult get_table_objects_by_name_req(1:GetTablesRequest req) throws (1:MetaException o1, 2:InvalidOperationException o2, 3:UnknownDBException o3) - map<string, Materialization> get_materialization_invalidation_info(1:string dbname, 2:list<string> tbl_names) + Materialization get_materialization_invalidation_info(1:CreationMetadata creation_metadata, 2:string validTxnList) throws (1:MetaException o1, 2:InvalidOperationException o2, 3:UnknownDBException o3) void update_creation_metadata(1: string catName, 2:string dbname, 3:string tbl_name, 4:CreationMetadata creation_metadata) throws (1:MetaException o1, 2:InvalidOperationException o2, 3:UnknownDBException o3) http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java index 53c4d24..d91f737 100644 --- a/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java +++ b/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java @@ -164,8 +164,6 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos // instantiate the metastore server handler directly instead of connecting // through the network client = HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true); - // Initialize materializations invalidation cache (only for local metastore) - MaterializationsInvalidationCache.get().init(conf, (IHMSHandler) client); isConnected = true; snapshotActiveConf(); return; @@ -1442,10 +1440,9 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos /** {@inheritDoc} */ @Override - public Map<String, Materialization> getMaterializationsInvalidationInfo(String dbName, List<String> viewNames) + public Materialization getMaterializationInvalidationInfo(CreationMetadata cm, String validTxnList) throws MetaException, InvalidOperationException, UnknownDBException, TException { - return client.get_materialization_invalidation_info( - dbName, filterHook.filterTableNames(null, dbName, viewNames)); + return client.get_materialization_invalidation_info(cm, validTxnList); } /** {@inheritDoc} */ http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMaterializationsCacheCleaner.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMaterializationsCacheCleaner.java b/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMaterializationsCacheCleaner.java deleted file mode 100644 index 8debcce..0000000 --- a/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMaterializationsCacheCleaner.java +++ /dev/null @@ -1,328 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.metastore; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.api.BasicTxnInfo; -import org.apache.hadoop.hive.metastore.api.CreationMetadata; -import org.apache.hadoop.hive.metastore.api.Materialization; -import org.apache.hadoop.hive.metastore.api.Table; -import org.junit.Assert; -import org.junit.FixMethodOrder; -import org.junit.Test; -import org.junit.runners.MethodSorters; - -import java.util.Map; - -import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * Unit tests for {@link org.apache.hadoop.hive.metastore.MaterializationsInvalidationCache}. - * The tests focus on arrival of notifications (possibly out of order) and the logic - * to clean up the materializations cache. Tests need to be executed in a certain order - * to avoid interactions among them, as the invalidation cache is a singleton. - */ -@FixMethodOrder(MethodSorters.NAME_ASCENDING) -public class TestMetaStoreMaterializationsCacheCleaner { - - private static final String DB_NAME = "hive3252"; - private static final String TBL_NAME_1 = "tmptbl1"; - private static final String TBL_NAME_2 = "tmptbl2"; - private static final String TBL_NAME_3 = "tmptbl3"; - private static final String MV_NAME_1 = "mv1"; - private static final String MV_NAME_2 = "mv2"; - - - @Test - public void testCleanerScenario1() throws Exception { - // create mock raw store - Configuration conf = new Configuration(); - conf.set("metastore.materializations.invalidation.impl", "DISABLE"); - // create mock handler - final IHMSHandler handler = mock(IHMSHandler.class); - // initialize invalidation cache (set conf to disable) - MaterializationsInvalidationCache.get().init(conf, handler); - - // This is a dummy test, invalidation cache is not supposed to - // record any information. - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, 1, 1, false); - int id = 2; - BasicTxnInfo txn2 = createTxnInfo(DB_NAME, TBL_NAME_1, id); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, id, id, false); - // Create tbl2 (nothing to do) - id = 3; - BasicTxnInfo txn3 = createTxnInfo(DB_NAME, TBL_NAME_1, id); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, id, id, false); - // Cleanup (current = 4, duration = 4) -> Does nothing - long removed = MaterializationsInvalidationCache.get().cleanup(0L); - Assert.assertEquals(0L, removed); - // Create mv1 - Table mv1 = mock(Table.class); - when(mv1.getDbName()).thenReturn(DB_NAME); - when(mv1.getTableName()).thenReturn(MV_NAME_1); - CreationMetadata mockCM1 = new CreationMetadata( - DEFAULT_CATALOG_NAME, DB_NAME, MV_NAME_1, - ImmutableSet.of( - DB_NAME + "." + TBL_NAME_1, - DB_NAME + "." + TBL_NAME_2)); - // Create txn list (highWatermark=4;minOpenTxn=Long.MAX_VALUE) - mockCM1.setValidTxnList("3:" + Long.MAX_VALUE + "::"); - when(mv1.getCreationMetadata()).thenReturn(mockCM1); - MaterializationsInvalidationCache.get().createMaterializedView(mockCM1.getDbName(), mockCM1.getTblName(), - mockCM1.getTablesUsed(), mockCM1.getValidTxnList()); - // Format <txnId>$<table_name>:<hwm>:<minOpenWriteId>:<open_writeids>:<abort_writeids>$<table_name> - Map<String, Materialization> invalidationInfos = - MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( - DB_NAME, ImmutableList.of(MV_NAME_1)); - Assert.assertTrue(invalidationInfos.isEmpty()); - id = 10; - BasicTxnInfo txn10 = createTxnInfo(DB_NAME, TBL_NAME_2, id); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, id, id, false); - id = 9; - BasicTxnInfo txn9 = createTxnInfo(DB_NAME, TBL_NAME_1, id); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, id, id, false); - // Cleanup (current = 12, duration = 4) -> Removes txn1, txn2, txn3 - removed = MaterializationsInvalidationCache.get().cleanup(8L); - Assert.assertEquals(0L, removed); - invalidationInfos = - MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( - DB_NAME, ImmutableList.of(MV_NAME_1)); - Assert.assertTrue(invalidationInfos.isEmpty()); - // Create mv2 - Table mv2 = mock(Table.class); - when(mv2.getDbName()).thenReturn(DB_NAME); - when(mv2.getTableName()).thenReturn(MV_NAME_2); - CreationMetadata mockCM2 = new CreationMetadata( - DEFAULT_CATALOG_NAME, DB_NAME, MV_NAME_2, - ImmutableSet.of( - DB_NAME + "." + TBL_NAME_1, - DB_NAME + "." + TBL_NAME_2)); - // Create txn list (highWatermark=10;minOpenTxn=Long.MAX_VALUE) - mockCM2.setValidTxnList("10:" + Long.MAX_VALUE + "::"); - when(mv2.getCreationMetadata()).thenReturn(mockCM2); - MaterializationsInvalidationCache.get().createMaterializedView(mockCM2.getDbName(), mockCM2.getTblName(), - mockCM2.getTablesUsed(), mockCM2.getValidTxnList()); - when(mv2.getCreationMetadata()).thenReturn(mockCM2); - invalidationInfos = - MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( - DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2)); - Assert.assertTrue(invalidationInfos.isEmpty()); - // Create tbl3 (nothing to do) - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_3, 11, 11, false); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_3, 18, 18, false); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, 14, 14, false); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, 17, 17, false); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, 16, 16, false); - // Cleanup (current = 20, duration = 4) -> Removes txn10, txn11 - removed = MaterializationsInvalidationCache.get().cleanup(16L); - Assert.assertEquals(0L, removed); - invalidationInfos = - MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( - DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2)); - Assert.assertTrue(invalidationInfos.isEmpty()); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, 12, 12, false); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, 15, 15, false); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, 7, 7, false); - invalidationInfos = - MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( - DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2)); - Assert.assertTrue(invalidationInfos.isEmpty()); - // Cleanup (current = 24, duration = 4) -> Removes txn9, txn14, txn15, txn16, txn17, txn18 - removed = MaterializationsInvalidationCache.get().cleanup(20L); - Assert.assertEquals(0L, removed); - invalidationInfos = - MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( - DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2)); - Assert.assertTrue(invalidationInfos.isEmpty()); - // Cleanup (current = 28, duration = 4) -> Removes txn9 - removed = MaterializationsInvalidationCache.get().cleanup(24L); - Assert.assertEquals(0L, removed); - } - - @Test - public void testCleanerScenario2() throws Exception { - // create mock raw store - Configuration conf = new Configuration(); - conf.set("metastore.materializations.invalidation.impl", "DEFAULT"); - // create mock handler - final IHMSHandler handler = mock(IHMSHandler.class); - // initialize invalidation cache (set conf to default) - MaterializationsInvalidationCache.get().init(conf, handler); - - // Scenario consists of the following steps: - // Create tbl1 - // (t = 1) Insert row in tbl1 - // (t = 2) Insert row in tbl1 - // Create tbl2 - // (t = 3) Insert row in tbl2 - // Cleanup (current = 4, duration = 4) -> Does nothing - // Create mv1 - // (t = 10) Insert row in tbl2 - // (t = 9) Insert row in tbl1 (out of order) - // Cleanup (current = 12, duration = 4) -> Removes txn1, txn2, txn3 - // Create mv2 - // Create tbl3 - // (t = 11) Insert row in tbl3 - // (t = 18) Insert row in tbl3 - // (t = 14) Insert row in tbl1 - // (t = 17) Insert row in tbl1 - // (t = 16) Insert row in tbl2 - // Cleanup (current = 20, duration = 4) -> Removes txn10, txn11 - // (t = 12) Insert row in tbl1 - // (t = 15) Insert row in tbl2 - // (t = 7) Insert row in tbl2 - // Cleanup (current = 24, duration = 4) -> Removes txn9, txn14, txn15, txn16, txn17, txn18 - // Create tbl1 (nothing to do) - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, 1, 1, false); - int id = 2; - BasicTxnInfo txn2 = createTxnInfo(DB_NAME, TBL_NAME_1, id); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, id, id, false); - // Create tbl2 (nothing to do) - id = 3; - BasicTxnInfo txn3 = createTxnInfo(DB_NAME, TBL_NAME_1, id); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, id, id, false); - // Cleanup (current = 4, duration = 4) -> Does nothing - long removed = MaterializationsInvalidationCache.get().cleanup(0L); - Assert.assertEquals(0L, removed); - // Create mv1 - Table mv1 = mock(Table.class); - when(mv1.getDbName()).thenReturn(DB_NAME); - when(mv1.getTableName()).thenReturn(MV_NAME_1); - CreationMetadata mockCM1 = new CreationMetadata( - DEFAULT_CATALOG_NAME, DB_NAME, MV_NAME_1, - ImmutableSet.of( - DB_NAME + "." + TBL_NAME_1, - DB_NAME + "." + TBL_NAME_2)); - // Create txn list (highWatermark=4;minOpenTxn=Long.MAX_VALUE) - mockCM1.setValidTxnList("3$" + DB_NAME + "." + TBL_NAME_1 + ":3:" + Long.MAX_VALUE + "::" + - "$" + DB_NAME + "." + TBL_NAME_2 + ":3:" + Long.MAX_VALUE + "::"); - when(mv1.getCreationMetadata()).thenReturn(mockCM1); - MaterializationsInvalidationCache.get().createMaterializedView(mockCM1.getDbName(), mockCM1.getTblName(), - mockCM1.getTablesUsed(), mockCM1.getValidTxnList()); - Map<String, Materialization> invalidationInfos = - MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( - DB_NAME, ImmutableList.of(MV_NAME_1)); - Assert.assertEquals(0L, invalidationInfos.get(MV_NAME_1).getInvalidationTime()); - id = 10; - BasicTxnInfo txn10 = createTxnInfo(DB_NAME, TBL_NAME_2, id); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, id, id, false); - id = 9; - BasicTxnInfo txn9 = createTxnInfo(DB_NAME, TBL_NAME_1, id); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, id, id, false); - // Cleanup (current = 12, duration = 4) -> Removes txn1, txn2, txn3 - removed = MaterializationsInvalidationCache.get().cleanup(8L); - Assert.assertEquals(3L, removed); - invalidationInfos = - MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( - DB_NAME, ImmutableList.of(MV_NAME_1)); - Assert.assertEquals(9L, invalidationInfos.get(MV_NAME_1).getInvalidationTime()); - // Create mv2 - Table mv2 = mock(Table.class); - when(mv2.getDbName()).thenReturn(DB_NAME); - when(mv2.getTableName()).thenReturn(MV_NAME_2); - CreationMetadata mockCM2 = new CreationMetadata( - DEFAULT_CATALOG_NAME, DB_NAME, MV_NAME_2, - ImmutableSet.of( - DB_NAME + "." + TBL_NAME_1, - DB_NAME + "." + TBL_NAME_2)); - // Create txn list (highWatermark=10;minOpenTxn=Long.MAX_VALUE) - mockCM2.setValidTxnList("10$" + DB_NAME + "." + TBL_NAME_1 + ":10:" + Long.MAX_VALUE + "::" + - "$" + DB_NAME + "." + TBL_NAME_2 + ":10:" + Long.MAX_VALUE + "::"); - when(mv2.getCreationMetadata()).thenReturn(mockCM2); - MaterializationsInvalidationCache.get().createMaterializedView(mockCM2.getDbName(), mockCM2.getTblName(), - mockCM2.getTablesUsed(), mockCM2.getValidTxnList()); - when(mv2.getCreationMetadata()).thenReturn(mockCM2); - invalidationInfos = - MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( - DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2)); - Assert.assertEquals(9L, invalidationInfos.get(MV_NAME_1).getInvalidationTime()); - Assert.assertEquals(0L, invalidationInfos.get(MV_NAME_2).getInvalidationTime()); - // Create tbl3 (nothing to do) - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_3, 11, 11, false); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_3, 18, 18, false); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, 14, 14, false); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, 17, 17, false); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, 16, 16, false); - // Cleanup (current = 20, duration = 4) -> Removes txn10, txn11 - removed = MaterializationsInvalidationCache.get().cleanup(16L); - Assert.assertEquals(2L, removed); - invalidationInfos = - MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( - DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2)); - Assert.assertEquals(9L, invalidationInfos.get(MV_NAME_1).getInvalidationTime()); - Assert.assertEquals(14L, invalidationInfos.get(MV_NAME_2).getInvalidationTime()); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_1, 12, 12, false); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, 15, 15, false); - MaterializationsInvalidationCache.get().notifyTableModification( - DB_NAME, TBL_NAME_2, 7, 7, false); - invalidationInfos = - MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( - DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2)); - Assert.assertEquals(7L, invalidationInfos.get(MV_NAME_1).getInvalidationTime()); - Assert.assertEquals(12L, invalidationInfos.get(MV_NAME_2).getInvalidationTime()); - // Cleanup (current = 24, duration = 4) -> Removes txn9, txn14, txn15, txn16, txn17, txn18 - removed = MaterializationsInvalidationCache.get().cleanup(20L); - Assert.assertEquals(6L, removed); - invalidationInfos = - MaterializationsInvalidationCache.get().getMaterializationInvalidationInfo( - DB_NAME, ImmutableList.of(MV_NAME_1, MV_NAME_2)); - Assert.assertEquals(7L, invalidationInfos.get(MV_NAME_1).getInvalidationTime()); - Assert.assertEquals(12L, invalidationInfos.get(MV_NAME_2).getInvalidationTime()); - // Cleanup (current = 28, duration = 4) -> Removes txn9 - removed = MaterializationsInvalidationCache.get().cleanup(24L); - Assert.assertEquals(0L, removed); - } - - private static BasicTxnInfo createTxnInfo(String dbName, String tableName, int i) { - BasicTxnInfo r = new BasicTxnInfo(); - r.setDbname(dbName); - r.setTablename(tableName); - r.setTxnid(i); - r.setTime(i); - return r; - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/1b5903b0/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java b/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java index efa3e7c..816a735 100644 --- a/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java +++ b/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java @@ -1211,6 +1211,7 @@ public class TestTablesCreateDropAlterTruncate extends MetaStoreClientTest { // Update the metadata for the materialized view CreationMetadata cm = client.getTable(catName, dbName, tableNames[3]).getCreationMetadata(); cm.addToTablesUsed(dbName + "." + tableNames[1]); + cm.unsetMaterializationTime(); client.updateCreationMetadata(catName, dbName, tableNames[3], cm); List<String> partNames = new ArrayList<>();