This is an automated email from the ASF dual-hosted git repository.
krisztiankasa pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 4146d220bfc HIVE-26375: Invalid materialized view after rebuild if
source table was compacted (Krisztian Kasa, reviewed by Aman Sinha, Laszlo
Pinter)
4146d220bfc is described below
commit 4146d220bfc0a29d117676844a4c7021fcc6a30d
Author: Krisztian Kasa <[email protected]>
AuthorDate: Fri Jul 22 12:18:27 2022 +0200
HIVE-26375: Invalid materialized view after rebuild if source table was
compacted (Krisztian Kasa, reviewed by Aman Sinha, Laszlo Pinter)
---
.../txn/compactor/TestMaterializedViewRebuild.java | 100 ++++++++++++++------
.../org/apache/hadoop/hive/ql/metadata/Hive.java | 5 +-
.../hadoop/hive/metastore/txn/TestTxnHandler.java | 10 +-
.../hadoop/hive/metastore/HiveMetaStoreClient.java | 8 +-
.../hadoop/hive/metastore/IMetaStoreClient.java | 9 +-
.../apache/hadoop/hive/metastore/HMSHandler.java | 5 +-
.../apache/hadoop/hive/metastore/ObjectStore.java | 52 +++++++++--
.../hadoop/hive/metastore/txn/TxnHandler.java | 103 ---------------------
.../apache/hadoop/hive/metastore/txn/TxnStore.java | 11 +--
.../metastore/HiveMetaStoreClientPreCatalog.java | 12 +--
10 files changed, 133 insertions(+), 182 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestMaterializedViewRebuild.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestMaterializedViewRebuild.java
index 1ddf22e9201..a0bf2608bfb 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestMaterializedViewRebuild.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestMaterializedViewRebuild.java
@@ -38,42 +38,37 @@ public class TestMaterializedViewRebuild extends
CompactorOnTezTest {
private static final String MV1 = "mat1";
private static final List<String> FULL_REBUILD_PLAN = Arrays.asList(
- "CBO PLAN:",
- "HiveProject(a=[$0], b=[$1], c=[$2])",
- " HiveFilter(condition=[OR(IS NULL($0), >($0, 0))])",
- " HiveTableScan(table=[[default, t1]], table:alias=[t1])",
- ""
+ "CBO PLAN:",
+ "HiveProject(a=[$0], b=[$1], c=[$2])",
+ " HiveFilter(condition=[OR(IS NULL($0), >($0, 0))])",
+ " HiveTableScan(table=[[default, t1]], table:alias=[t1])",
+ ""
);
private static final List<String> INCREMENTAL_REBUILD_PLAN = Arrays.asList(
- "CBO PLAN:",
- "HiveProject(a=[$0], b=[$1], c=[$2])",
- " HiveFilter(condition=[AND(<(1, $5.writeid), OR(>($0, 0), IS
NULL($0)))])",
- " HiveTableScan(table=[[default, t1]], table:alias=[t1])",
- ""
+ "CBO PLAN:",
+ "HiveProject(a=[$0], b=[$1], c=[$2])",
+ " HiveFilter(condition=[AND(<(2, $5.writeid), OR(>($0, 0), IS
NULL($0)))])",
+ " HiveTableScan(table=[[default, t1]], table:alias=[t1])",
+ ""
);
private static final List<String> EXPECTED_RESULT = Arrays.asList(
- "1\tone\t1.1",
- "2\ttwo\t2.2",
- "3\tthree\t3.3",
- "NULL\tNULL\tNULL"
+ "1\tone\t1.1",
+ "2\ttwo\t2.2",
+ "3\tthree\t3.3",
+ "NULL\tNULL\tNULL"
);
@Override
public void setup() throws Exception {
super.setup();
- }
- private void createTestSchema(String tblproperties) throws Exception {
- executeStatementOnDriver("create table " + TABLE1 + "(a int, b
varchar(128), c float) stored as orc " +
- "TBLPROPERTIES ('transactional'='true'" + tblproperties + ")", driver);
- executeStatementOnDriver("insert into " + TABLE1 + "(a,b, c) values " +
- "(1, 'one', 1.1), (2, 'two', 2.2), (NULL, NULL, NULL)", driver);
- executeStatementOnDriver("create materialized view " + MV1 + " stored as
orc " +
- "TBLPROPERTIES ('transactional'='true') as " +
- "select a,b,c from " + TABLE1 + " where a > 0 or a is null", driver);
+ executeStatementOnDriver("create table " + TABLE1 + "(a int, b
varchar(128), c float) stored as orc TBLPROPERTIES ('transactional'='true')",
driver);
+ executeStatementOnDriver("insert into " + TABLE1 + "(a,b, c) values (1,
'one', 1.1), (2, 'two', 2.2), (NULL, NULL, NULL)", driver);
+ executeStatementOnDriver("create materialized view " + MV1 + " stored as
orc TBLPROPERTIES ('transactional'='true') as " +
+ "select a,b,c from " + TABLE1 + " where a > 0 or a is null",
driver);
}
@Override
@@ -85,8 +80,7 @@ public class TestMaterializedViewRebuild extends
CompactorOnTezTest {
}
@Test
- public void
testWhenMajorCompactionThenIncrementalMVRebuildIsStillAvailable() throws
Exception {
- createTestSchema("");
+ public void testWhenMajorCompactionThenIncrementalMVRebuildNotUsed() throws
Exception {
executeStatementOnDriver("insert into " + TABLE1 + "(a,b,c) values (3,
'three', 3.3)", driver);
@@ -97,7 +91,7 @@ public class TestMaterializedViewRebuild extends
CompactorOnTezTest {
txnHandler.cleanTxnToWriteIdTable();
List<String> result = execSelectAndDumpData("explain cbo alter
materialized view " + MV1 + " rebuild", driver, "");
- Assert.assertEquals(INCREMENTAL_REBUILD_PLAN, result);
+ Assert.assertEquals(FULL_REBUILD_PLAN, result);
executeStatementOnDriver("alter materialized view " + MV1 + " rebuild",
driver);
result = execSelectAndDumpData("select * from " + MV1 , driver, "");
@@ -108,8 +102,7 @@ public class TestMaterializedViewRebuild extends
CompactorOnTezTest {
}
@Test
- public void
testWhenCompactInsertOnlySourceTableThenIncrementalMVRebuildIsNotAvailable()
throws Exception {
- createTestSchema(", 'transactional_properties'='insert_only'");
+ public void testSecondRebuildCanBeIncrementalAfterMajorCompaction() throws
Exception {
executeStatementOnDriver("insert into " + TABLE1 + "(a,b,c) values (3,
'three', 3.3)", driver);
@@ -119,6 +112,30 @@ public class TestMaterializedViewRebuild extends
CompactorOnTezTest {
TxnStore txnHandler = TxnUtils.getTxnStore(conf);
txnHandler.cleanTxnToWriteIdTable();
+ executeStatementOnDriver("alter materialized view " + MV1 + " rebuild",
driver);
+
+ // Insert after first rebuild.
+ executeStatementOnDriver("insert into " + TABLE1 + "(a,b,c) values (4,
'four', 4.4)", driver);
+
+ List<String> result = execSelectAndDumpData("explain cbo alter
materialized view " + MV1 + " rebuild", driver, "");
+ Assert.assertEquals(INCREMENTAL_REBUILD_PLAN, result);
+ executeStatementOnDriver("alter materialized view " + MV1 + " rebuild",
driver);
+
+ result = execSelectAndDumpData("select * from " + MV1 , driver, "");
+ List<String> expected = new ArrayList(EXPECTED_RESULT);
+ expected.add("4\tfour\t4.4");
+ assertResult(expected, result);
+
+ result = execSelectAndDumpData("explain cbo select a,b,c from " + TABLE1 +
" where a > 0 or a is null", driver, "");
+ Assert.assertEquals(Arrays.asList("CBO PLAN:",
"HiveTableScan(table=[[default, " + MV1 + "]], table:alias=[default." + MV1 +
"])", ""), result);
+ }
+
+ @Test
+ public void
testWhenCleanUpOfMajorCompactionHasNotFinishedIncrementalMVRebuildNotUsed()
throws Exception {
+ executeStatementOnDriver("insert into " + TABLE1 + "(a,b,c) values (3,
'three', 3.3)", driver);
+
+ CompactorTestUtil.runCompaction(conf, "default", TABLE1 ,
CompactionType.MAJOR, true);
+
List<String> result = execSelectAndDumpData("explain cbo alter
materialized view " + MV1 + " rebuild", driver, "");
Assert.assertEquals(FULL_REBUILD_PLAN, result);
executeStatementOnDriver("alter materialized view " + MV1 + " rebuild",
driver);
@@ -130,6 +147,33 @@ public class TestMaterializedViewRebuild extends
CompactorOnTezTest {
Assert.assertEquals(Arrays.asList("CBO PLAN:",
"HiveTableScan(table=[[default, " + MV1 + "]], table:alias=[default." + MV1 +
"])", ""), result);
}
+ private static final List<String> EXPECTED_RESULT_AFTER_UPDATE =
Arrays.asList(
+ "1\tChanged\t1.1",
+ "2\ttwo\t2.2",
+ "NULL\tNULL\tNULL"
+ );
+
+ @Test
+ public void
testWhenMajorCompactionThenIncrementalMVRebuildNotUsedInPresenceOfUpdate()
throws Exception {
+ executeStatementOnDriver("update " + TABLE1 + " set b = 'Changed' where a
= 1", driver);
+
+ CompactorTestUtil.runCompaction(conf, "default", TABLE1 ,
CompactionType.MAJOR, true);
+ CompactorTestUtil.runCleaner(conf);
+ verifySuccessfulCompaction(1);
+ TxnStore txnHandler = TxnUtils.getTxnStore(conf);
+ txnHandler.cleanTxnToWriteIdTable();
+
+ List<String> result = execSelectAndDumpData("explain cbo alter
materialized view " + MV1 + " rebuild", driver, "");
+ Assert.assertEquals(FULL_REBUILD_PLAN, result);
+ executeStatementOnDriver("alter materialized view " + MV1 + " rebuild",
driver);
+
+ result = execSelectAndDumpData("select * from " + MV1 , driver, "");
+ assertResult(EXPECTED_RESULT_AFTER_UPDATE, result);
+
+ result = execSelectAndDumpData("explain cbo select a,b,c from " + TABLE1 +
" where a > 0 or a is null", driver, "");
+ Assert.assertEquals(Arrays.asList("CBO PLAN:",
"HiveTableScan(table=[[default, " + MV1 + "]], table:alias=[default." + MV1 +
"])", ""), result);
+ }
+
private void assertResult(List<String> expected, List<String> actual) {
Assert.assertEquals(expected.size(), actual.size());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index d011736bcf1..f640b9778e7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -2080,7 +2080,7 @@ public class Hive {
// Obtain additional information if we should try incremental
rewriting / rebuild
// We will not try partial rewriting if there were
update/delete/compaction operations on source tables
Materialization invalidationInfo =
getMSC().getMaterializationInvalidationInfo(
- materializedViewTable.getMVMetadata().creationMetadata);
+ materializedViewTable.getMVMetadata().creationMetadata,
conf.get(ValidTxnList.VALID_TXNS_KEY));
if (invalidationInfo == null ||
invalidationInfo.isSourceTablesUpdateDeleteModified() ||
invalidationInfo.isSourceTablesCompacted()) {
// We ignore (as it did not meet the requirements), but we do
not need to update it in the
@@ -2177,7 +2177,8 @@ public class Hive {
} else {
// Obtain additional information if we should try incremental
rewriting / rebuild
// We will not try partial rewriting if there were
update/delete/compaction operations on source tables
- invalidationInfo =
getMSC().getMaterializationInvalidationInfo(metadata.creationMetadata);
+ invalidationInfo = getMSC().getMaterializationInvalidationInfo(
+ metadata.creationMetadata,
conf.get(ValidTxnList.VALID_TXNS_KEY));
ignore = invalidationInfo == null ||
invalidationInfo.isSourceTablesCompacted();
}
if (ignore) {
diff --git
a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
index 969f0a504de..43fa5410c39 100644
--- a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
+++ b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
@@ -1879,6 +1879,7 @@ public class TestTxnHandler {
@Test
public void testGetMaterializationInvalidationInfo() throws MetaException {
testGetMaterializationInvalidationInfo(
+ new ValidReadTxnList(new long[] {6, 11}, new BitSet(), 10L, 12L),
new ValidReaderWriteIdList(TableName.getDbTable("default", "t1"),
new long[] { 2 }, new BitSet(), 1)
);
}
@@ -1886,6 +1887,7 @@ public class TestTxnHandler {
@Test
public void testGetMaterializationInvalidationInfoWhenTableHasNoException()
throws MetaException {
testGetMaterializationInvalidationInfo(
+ new ValidReadTxnList(new long[] {6, 11}, new BitSet(), 10L, 12L),
new ValidReaderWriteIdList(TableName.getDbTable("default", "t1"),
new long[0], new BitSet(), 1)
);
}
@@ -1893,12 +1895,13 @@ public class TestTxnHandler {
@Test
public void
testGetMaterializationInvalidationInfoWhenCurrentTxnListHasNoException() throws
MetaException {
testGetMaterializationInvalidationInfo(
+ new ValidReadTxnList(new long[0], new BitSet(), 10L, 12L),
new ValidReaderWriteIdList(TableName.getDbTable("default", "t1"),
new long[] { 2 }, new BitSet(), 1)
);
}
private void testGetMaterializationInvalidationInfo(
- ValidReaderWriteIdList... tableWriteIdList) throws MetaException {
+ ValidReadTxnList currentValidTxnList, ValidReaderWriteIdList...
tableWriteIdList) throws MetaException {
ValidTxnWriteIdList validTxnWriteIdList = new ValidTxnWriteIdList(5L);
for (ValidReaderWriteIdList tableWriteId : tableWriteIdList) {
validTxnWriteIdList.addTableValidWriteIdList(tableWriteId);
@@ -1917,10 +1920,11 @@ public class TestTxnHandler {
CreationMetadata creationMetadata = new CreationMetadata();
creationMetadata.setDbName("default");
creationMetadata.setTblName("mat1");
- creationMetadata.setSourceTables(Collections.singletonList(sourceTable));
+ creationMetadata.setTablesUsed(new HashSet<String>() {{ add("default.t1");
}});
creationMetadata.setValidTxnList(validTxnWriteIdList.toString());
- Materialization materialization =
txnHandler.getMaterializationInvalidationInfo(creationMetadata);
+ Materialization materialization =
txnHandler.getMaterializationInvalidationInfo(
+ creationMetadata, currentValidTxnList.toString());
assertFalse(materialization.isSourceTablesUpdateDeleteModified());
}
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 08d92f1ef25..ecf85fb817a 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -2704,13 +2704,9 @@ public class HiveMetaStoreClient implements
IMetaStoreClient, AutoCloseable {
return
deepCopyTables(FilterUtils.filterTablesIfEnabled(isClientFilterEnabled,
filterHook, tabs));
}
- public Materialization getMaterializationInvalidationInfo(CreationMetadata
cm)
- throws MetaException, InvalidOperationException, UnknownDBException,
TException {
- return client.get_materialization_invalidation_info(cm, null);
- }
-
+ @Override
public Materialization getMaterializationInvalidationInfo(CreationMetadata
cm, String validTxnList)
- throws MetaException, InvalidOperationException, UnknownDBException,
TException {
+ throws MetaException, InvalidOperationException, UnknownDBException,
TException {
return client.get_materialization_invalidation_info(cm, validTxnList);
}
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
index dfd06f96a2a..75e1b2c8bc8 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
@@ -808,15 +808,8 @@ public interface IMetaStoreClient {
/**
* Returns the invalidation information for the materialized views given as
input.
*/
- Materialization getMaterializationInvalidationInfo(CreationMetadata cm)
- throws MetaException, InvalidOperationException, UnknownDBException,
TException;
-
- @Deprecated
- /**
- * Use {@link
IMetaStoreClient#getMaterializationInvalidationInfo(CreationMetadata)} instead.
- */
Materialization getMaterializationInvalidationInfo(CreationMetadata cm,
String validTxnList)
- throws MetaException, InvalidOperationException, UnknownDBException,
TException;
+ throws MetaException, InvalidOperationException, UnknownDBException,
TException;
/**
* Updates the creation metadata for the materialized view.
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
index 2cde211c297..6b6f5e463c8 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
@@ -3848,10 +3848,7 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
}
@Override
- public Materialization get_materialization_invalidation_info(final
CreationMetadata cm, String validTxnList) throws MetaException {
- if (validTxnList == null) {
- return getTxnHandler().getMaterializationInvalidationInfo(cm);
- }
+ public Materialization get_materialization_invalidation_info(final
CreationMetadata cm, final String validTxnList) throws MetaException {
return getTxnHandler().getMaterializationInvalidationInfo(cm,
validTxnList);
}
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 83afdce22ea..435ac4fc6bd 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -2527,19 +2527,51 @@ public class ObjectStore implements RawStore,
Configurable {
}
assert !m.isSetMaterializationTime();
Set<MMVSource> tablesUsed = new HashSet<>();
- for (SourceTable sourceTable : m.getSourceTables()) {
- Table table = sourceTable.getTable();
- MTable mtbl = getMTable(m.getCatName(), table.getDbName(),
table.getTableName(), false).mtbl;
- MMVSource source = new MMVSource();
- source.setTable(mtbl);
- source.setInsertedCount(sourceTable.getInsertedCount());
- source.setUpdatedCount(sourceTable.getUpdatedCount());
- source.setDeletedCount(sourceTable.getDeletedCount());
- tablesUsed.add(source);
+ if (m.isSetSourceTables()) {
+ for (SourceTable sourceTable : m.getSourceTables()) {
+ tablesUsed.add(convertToSourceTable(m.getCatName(), sourceTable));
+ }
+ } else {
+ for (String fullyQualifiedName : m.getTablesUsed()) {
+ tablesUsed.add(convertToSourceTable(m.getCatName(),
fullyQualifiedName));
+ }
}
return new MCreationMetadata(normalizeIdentifier(m.getCatName()),
normalizeIdentifier(m.getDbName()),
normalizeIdentifier(m.getTblName()),
- tablesUsed, m.getValidTxnList(), System.currentTimeMillis());
+ tablesUsed, m.getValidTxnList(), System.currentTimeMillis());
+ }
+
+ private MMVSource convertToSourceTable(String catalog, SourceTable
sourceTable) {
+ Table table = sourceTable.getTable();
+ MTable mtbl = getMTable(catalog, table.getDbName(), table.getTableName(),
false).mtbl;
+ MMVSource source = new MMVSource();
+ source.setTable(mtbl);
+ source.setInsertedCount(sourceTable.getInsertedCount());
+ source.setUpdatedCount(sourceTable.getUpdatedCount());
+ source.setDeletedCount(sourceTable.getDeletedCount());
+ return source;
+ }
+
+ /**
+ * This method resets the stats to 0 and supports only backward
compatibility with clients does not
+ * send {@link SourceTable} instances.
+ *
+ * Use {@link ObjectStore#convertToSourceTable(String, SourceTable)} instead.
+ *
+ * @param catalog Catalog name where source table is located
+ * @param fullyQualifiedTableName fully qualified name of source table
+ * @return {@link MMVSource} instance represents this source table.
+ */
+ @Deprecated
+ private MMVSource convertToSourceTable(String catalog, String
fullyQualifiedTableName) {
+ String[] names = fullyQualifiedTableName.split("\\.");
+ MTable mtbl = getMTable(catalog, names[0], names[1], false).mtbl;
+ MMVSource source = new MMVSource();
+ source.setTable(mtbl);
+ source.setInsertedCount(0L);
+ source.setUpdatedCount(0L);
+ source.setDeletedCount(0L);
+ return source;
}
private CreationMetadata convertToCreationMetadata(MCreationMetadata s)
throws MetaException {
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
index 9e270fc731d..5ab11b1c6bc 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
@@ -2544,109 +2544,6 @@ abstract class TxnHandler implements TxnStore,
TxnStore.MutexAPI {
*/
@Override
@RetrySemantics.ReadOnly
- public Materialization getMaterializationInvalidationInfo(CreationMetadata
creationMetadata) throws MetaException {
- if (creationMetadata.getSourceTables().isEmpty()) {
- // Bail out
- LOG.warn("Materialization creation metadata does not contain any table");
- return null;
- }
-
- boolean sourceTablesUpdateDeleteModified = false;
- for (SourceTable sourceTable : creationMetadata.getSourceTables()) {
- if (sourceTable.getDeletedCount() > 0 || sourceTable.getUpdatedCount() >
0) {
- sourceTablesUpdateDeleteModified = true;
- break;
- }
- }
-
- Boolean sourceTablesCompacted = wasCompacted(creationMetadata);
- if (sourceTablesCompacted == null) {
- return null;
- }
- return new Materialization(sourceTablesUpdateDeleteModified,
sourceTablesCompacted);
- }
-
- private Boolean wasCompacted(CreationMetadata creationMetadata) throws
MetaException {
- Set<String> insertOnlyTables = new HashSet<>();
- for (SourceTable sourceTable : creationMetadata.getSourceTables()) {
- Table table = sourceTable.getTable();
- String transactionalProp =
table.getParameters().get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
- if (!"insert_only".equalsIgnoreCase(transactionalProp)) {
- continue;
- }
-
- insertOnlyTables.add(
- TableName.getDbTable(sourceTable.getTable().getDbName(),
sourceTable.getTable().getTableName()));
- }
-
- if (insertOnlyTables.isEmpty()) {
- return false;
- }
-
- // We are composing a query that returns a single row if a compaction
happened after
- // the materialization was created. Otherwise, query returns 0 rows.
-
- // Parse validReaderWriteIdList from creation metadata
- final ValidTxnWriteIdList validReaderWriteIdList =
- new ValidTxnWriteIdList(creationMetadata.getValidTxnList());
-
-
- List<String> params = new ArrayList<>();
- StringBuilder queryCompletedCompactions = new StringBuilder();
- StringBuilder queryCompactionQueue = new StringBuilder();
- // compose a query that select transactions containing an update...
- queryCompletedCompactions.append("SELECT 1 FROM \"COMPLETED_COMPACTIONS\"
WHERE (");
- queryCompactionQueue.append("SELECT 1 FROM \"COMPACTION_QUEUE\" WHERE (");
- int i = 0;
- for (String fullyQualifiedName : insertOnlyTables) {
- ValidWriteIdList tblValidWriteIdList =
-
validReaderWriteIdList.getTableValidWriteIdList(fullyQualifiedName);
- if (tblValidWriteIdList == null) {
- LOG.warn("ValidWriteIdList for table {} not present in creation
metadata, this should not happen", fullyQualifiedName);
- return null;
- }
-
- // ...for each of the tables that are part of the materialized view,
- // where the transaction had to be committed after the materialization
was created...
- if (i != 0) {
- queryCompletedCompactions.append("OR");
- queryCompactionQueue.append("OR");
- }
- String[] names = TxnUtils.getDbTableName(fullyQualifiedName);
- assert (names.length == 2);
- queryCompletedCompactions.append(" (\"CC_DATABASE\"=? AND
\"CC_TABLE\"=?");
- queryCompactionQueue.append(" (\"CQ_DATABASE\"=? AND \"CQ_TABLE\"=?");
- params.add(names[0]);
- params.add(names[1]);
- queryCompletedCompactions.append(" AND (\"CC_HIGHEST_WRITE_ID\" > ");
- queryCompletedCompactions.append(tblValidWriteIdList.getHighWatermark());
-
queryCompletedCompactions.append(tblValidWriteIdList.getInvalidWriteIds().length
== 0 ? ") " :
- " OR \"CC_HIGHEST_WRITE_ID\" IN(" + StringUtils.join(",",
-
Arrays.asList(ArrayUtils.toObject(tblValidWriteIdList.getInvalidWriteIds()))) +
") ) ");
- queryCompletedCompactions.append(") ");
- queryCompactionQueue.append(") ");
- i++;
- }
- // ... and where the transaction has already been committed as per
snapshot taken
- // when we are running current query
- queryCompletedCompactions.append(")");
- queryCompactionQueue.append(") ");
-
- // Execute query
- queryCompletedCompactions.append(" UNION ");
- queryCompletedCompactions.append(queryCompactionQueue);
- List<String> paramsTwice = new ArrayList<>(params);
- paramsTwice.addAll(params);
- return executeBoolean(queryCompletedCompactions.toString(), paramsTwice,
- "Unable to retrieve materialization invalidation information:
compactions");
- }
-
- /**
- * Use {@link
TxnHandler#getMaterializationInvalidationInfo(CreationMetadata)} instead.
- */
- @Override
- @Deprecated
- @RetrySemantics.ReadOnly
public Materialization getMaterializationInvalidationInfo(
CreationMetadata creationMetadata, String validTxnListStr) throws
MetaException {
if (creationMetadata.getTablesUsed().isEmpty()) {
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java
index cdfd95263da..34d77f90742 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java
@@ -197,17 +197,14 @@ public interface TxnStore extends Configurable {
/**
* Get invalidation info for the materialization. Currently, the
materialization information
* only contains information about whether there was update/delete
operations on the source
- * tables and were any of the insert-only source tables compacted used by
the materialization
- * since it was created.
+ * tables used by the materialization since it was created.
* @param cm creation metadata for the materialization
+ * @param validTxnList valid transaction list for snapshot taken for current
query
* @throws MetaException
*/
@RetrySemantics.Idempotent
- Materialization getMaterializationInvalidationInfo(final CreationMetadata cm)
- throws MetaException;
-
- @RetrySemantics.Idempotent
- Materialization getMaterializationInvalidationInfo(final CreationMetadata
cm, String validTxnList)
+ Materialization getMaterializationInvalidationInfo(
+ final CreationMetadata cm, final String validTxnList)
throws MetaException;
@RetrySemantics.ReadOnly
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
index 222203c63bc..2a92c09c580 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
@@ -1587,18 +1587,8 @@ public class HiveMetaStoreClientPreCatalog implements
IMetaStoreClient, AutoClos
/** {@inheritDoc} */
@Override
- public Materialization getMaterializationInvalidationInfo(CreationMetadata
cm)
- throws MetaException, InvalidOperationException, UnknownDBException,
TException {
- return client.get_materialization_invalidation_info(cm, null);
- }
-
- @Deprecated
- /**
- * Use {@link
HiveMetaStoreClientPreCatalog#getMaterializationInvalidationInfo(CreationMetadata)}
instead.
- */
- @Override
public Materialization getMaterializationInvalidationInfo(CreationMetadata
cm, String validTxnList)
- throws MetaException, InvalidOperationException, UnknownDBException,
TException {
+ throws MetaException, InvalidOperationException, UnknownDBException,
TException {
return client.get_materialization_invalidation_info(cm, validTxnList);
}
