jcamachor commented on a change in pull request #749: HIVE-21344
URL: https://github.com/apache/hive/pull/749#discussion_r333763093
##
File path: ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
##
@@ -1612,6 +1612,150 @@ public Table
apply(org.apache.hadoop.hive.metastore.api.Table table) {
}
}
+ /**
+ * Get the materialized views that have been enabled for rewriting from the
+ * cache (registry). It will preprocess them to discard those that are
+ * outdated and augment those that need to be augmented, e.g., if incremental
+ * rewriting is enabled.
+ *
+ * @return the list of materialized views available for rewriting from the
registry
+ * @throws HiveException
+ */
+ public List
getPreprocessedMaterializedViewsFromRegistry(
+ List tablesUsed, HiveTxnManager txnMgr) throws HiveException {
+// From cache
+List materializedViews =
+HiveMaterializedViewsRegistry.get().getRewritingMaterializedViews();
+if (materializedViews.isEmpty()) {
+ // Bail out: empty list
+ return new ArrayList<>();
+}
+// Add to final result
+return filterAugmentMaterializedViews(materializedViews, tablesUsed,
txnMgr);
+ }
+
+ private List
filterAugmentMaterializedViews(List materializedViews,
+List tablesUsed, HiveTxnManager txnMgr) throws HiveException {
+final String validTxnsList = conf.get(ValidTxnList.VALID_TXNS_KEY);
+final ValidTxnWriteIdList currentTxnWriteIds =
txnMgr.getValidWriteIds(tablesUsed, validTxnsList);
+final boolean tryIncrementalRewriting =
+HiveConf.getBoolVar(conf,
HiveConf.ConfVars.HIVE_MATERIALIZED_VIEW_REWRITING_INCREMENTAL);
+final long defaultTimeWindow =
+HiveConf.getTimeVar(conf,
HiveConf.ConfVars.HIVE_MATERIALIZED_VIEW_REWRITING_TIME_WINDOW,
+TimeUnit.MILLISECONDS);
+try {
+ // Final result
+ List result = new ArrayList<>();
+ for (RelOptMaterialization materialization : materializedViews) {
+final RelNode viewScan = materialization.tableRel;
+final Table materializedViewTable;
+if (viewScan instanceof Project) {
+ // There is a Project on top (due to nullability)
+ materializedViewTable = ((RelOptHiveTable)
viewScan.getInput(0).getTable()).getHiveTableMD();
+} else {
+ materializedViewTable = ((RelOptHiveTable)
viewScan.getTable()).getHiveTableMD();
+}
+final Boolean outdated =
isOutdatedMaterializedView(materializedViewTable, currentTxnWriteIds,
+defaultTimeWindow, tablesUsed, false);
+if (outdated == null) {
+ continue;
+}
+
+final CreationMetadata creationMetadata =
materializedViewTable.getCreationMetadata();
+if (outdated) {
+ // The MV is outdated, see whether we should consider it for
rewriting or not
+ if (!tryIncrementalRewriting) {
+LOG.debug("Materialized view " +
materializedViewTable.getFullyQualifiedName() +
+" ignored for rewriting as its contents are outdated");
+continue;
+ }
+ // We will rewrite it to include the filters on transaction list
+ // so we can produce partial rewritings.
+ // This would be costly since we are doing it for every materialized
view
+ // that is outdated, but it only happens for more than one
materialized view
+ // if rewriting with outdated materialized views is enabled
(currently
+ // disabled by default).
+ materialization = augmentMaterializationWithTimeInformation(
+ materialization, validTxnsList, new ValidTxnWriteIdList(
+ creationMetadata.getValidTxnList()));
+}
+result.add(materialization);
+ }
+ return result;
+} catch (Exception e) {
+ throw new HiveException(e);
+}
+ }
+
+ /**
+ * Validate that the materialized views retrieved from registry are still
up-to-date.
+ * For those that are not, the method loads them from the metastore into the
registry.
+ *
+ * @return true if they are up-to-date, otherwise false
+ * @throws HiveException
+ */
+ public boolean validateMaterializedViewsFromRegistry(List
cachedMaterializedViewTables,
+ List tablesUsed, HiveTxnManager txnMgr) throws HiveException {
+final long defaultTimeWindow =
+HiveConf.getTimeVar(conf,
HiveConf.ConfVars.HIVE_MATERIALIZED_VIEW_REWRITING_TIME_WINDOW,
+TimeUnit.MILLISECONDS);
+final String validTxnsList = conf.get(ValidTxnList.VALID_TXNS_KEY);
+final ValidTxnWriteIdList currentTxnWriteIds =
txnMgr.getValidWriteIds(tablesUsed, validTxnsList);
+try {
+ // Final result
+ boolean result = true;
+ for (Table cachedMaterializedViewTable : cachedMaterializedViewTables) {
+// Retrieve the materialized view table from the metastore
+final Table materializedViewTable = getTable(
+
