deniskuzZ commented on code in PR #6159:
URL: https://github.com/apache/hive/pull/6159#discussion_r2523557189
##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java:
##########
@@ -9165,98 +9155,97 @@ private void writeMPartitionColumnStatistics(Table
table, Partition partition,
}
}
- /**
- * Get table's column stats
- *
- * @return Map of column name and its stats
- */
- private Map<String, MTableColumnStatistics> getPartitionColStats(Table
table, List<String> colNames, String engine)
- throws MetaException {
- Map<String, MTableColumnStatistics> statsMap = Maps.newHashMap();
- List<MTableColumnStatistics> stats = getMTableColumnStatistics(table,
colNames, engine);
- for (MTableColumnStatistics cStat : stats) {
- statsMap.put(cStat.getColName(), cStat);
- }
- return statsMap;
- }
-
@Override
public Map<String, String> updateTableColumnStatistics(ColumnStatistics
colStats, String validWriteIds, long writeId)
throws NoSuchObjectException, MetaException, InvalidObjectException,
InvalidInputException {
boolean committed = false;
-
List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
-
- Lock tableLock = getTableLockFor(statsDesc.getDbName(),
statsDesc.getTableName());
- tableLock.lock();
+ long start = System.currentTimeMillis();
+ String catName = statsDesc.isSetCatName() ? statsDesc.getCatName() :
getDefaultCatalog(conf);
try {
openTransaction();
// DataNucleus objects get detached all over the place for no (real)
reason.
// So let's not use them anywhere unless absolutely necessary.
- String catName = statsDesc.isSetCatName() ? statsDesc.getCatName() :
getDefaultCatalog(conf);
MTable mTable = ensureGetMTable(catName, statsDesc.getDbName(),
statsDesc.getTableName());
- Table table = convertToTable(mTable);
- List<String> colNames = new ArrayList<>();
- for (ColumnStatisticsObj statsObj : statsObjs) {
- colNames.add(statsObj.getColName());
- }
-
- Map<String, MTableColumnStatistics> oldStats =
getPartitionColStats(table, colNames, colStats.getEngine());
-
- for (ColumnStatisticsObj statsObj : statsObjs) {
- MTableColumnStatistics mStatsObj =
StatObjectConverter.convertToMTableColumnStatistics(
- mTable, statsDesc,
- statsObj, colStats.getEngine());
- writeMTableColumnStatistics(table, mStatsObj,
oldStats.get(statsObj.getColName()));
- // There is no need to add colname again, otherwise we will get
duplicate colNames.
- }
-
- // TODO: (HIVE-20109) ideally the col stats stats should be in colstats,
not in the table!
- // Set the table properties
- // No need to check again if it exists.
- String dbname = table.getDbName();
- String name = table.getTableName();
- MTable oldt = mTable;
- Map<String, String> newParams = new HashMap<>(table.getParameters());
- StatsSetupConst.setColumnStatsState(newParams, colNames);
- boolean isTxn = TxnUtils.isTransactionalTable(oldt.getParameters());
- if (isTxn) {
- if (!areTxnStatsSupported) {
- StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE);
- } else {
- String errorMsg = verifyStatsChangeCtx(TableName.getDbTable(dbname,
name),
- oldt.getParameters(), newParams, writeId, validWriteIds, true);
- if (errorMsg != null) {
- throw new MetaException(errorMsg);
- }
- if (!isCurrentStatsValidForTheQuery(oldt, validWriteIds, true)) {
- // Make sure we set the flag to invalid regardless of the current
value.
+ int maxRetries = MetastoreConf.getIntVar(conf,
ConfVars.METASTORE_S4U_NOWAIT_MAX_RETRIES);
+ long sleepInterval = MetastoreConf.getTimeVar(conf,
+ ConfVars.METASTORE_S4U_NOWAIT_RETRY_SLEEP_INTERVAL,
TimeUnit.MILLISECONDS);
+ Map<String, String> result = new RetryingExecutor<>(maxRetries, () -> {
+ Ref<Exception> exceptionRef = new Ref<>();
+ String savePoint = "uts_" + ThreadLocalRandom.current().nextInt(10000)
+ "_" + System.nanoTime();
+ setTransactionSavePoint(savePoint);
+ executePlainSQL(
+ sqlGenerator.addForUpdateNoWait("SELECT \"TBL_ID\" FROM \"TBLS\"
WHERE \"TBL_ID\" = " + mTable.getId()),
+ exception -> {
+ rollbackTransactionToSavePoint(savePoint);
+ exceptionRef.t = exception;
+ });
+ if (exceptionRef.t != null) {
+ throw new RetryingExecutor.RetryException(exceptionRef.t);
+ }
+ pm.refresh(mTable);
+ Table table = convertToTable(mTable);
+ List<String> colNames = new ArrayList<>();
+ for (ColumnStatisticsObj statsObj : statsObjs) {
+ colNames.add(statsObj.getColName());
+ }
+
+ Map<String, MTableColumnStatistics> oldStats = Maps.newHashMap();
+ List<MTableColumnStatistics> stats = getMTableColumnStatistics(table,
colNames, colStats.getEngine());
+ for (MTableColumnStatistics cStat : stats) {
+ oldStats.put(cStat.getColName(), cStat);
+ }
+
+ for (ColumnStatisticsObj statsObj : statsObjs) {
+ MTableColumnStatistics mStatsObj =
StatObjectConverter.convertToMTableColumnStatistics(mTable, statsDesc,
+ statsObj, colStats.getEngine());
+ writeMTableColumnStatistics(table, mStatsObj,
oldStats.get(statsObj.getColName()));
+ // There is no need to add colname again, otherwise we will get
duplicate colNames.
+ }
+
+ // TODO: (HIVE-20109) ideally the col stats stats should be in
colstats, not in the table!
+ // Set the table properties
+ // No need to check again if it exists.
+ String dbname = table.getDbName();
+ String name = table.getTableName();
+ MTable oldt = mTable;
+ Map<String, String> newParams = new HashMap<>(table.getParameters());
+ StatsSetupConst.setColumnStatsState(newParams, colNames);
+ boolean isTxn = TxnUtils.isTransactionalTable(oldt.getParameters());
+ if (isTxn) {
+ if (!areTxnStatsSupported) {
StatsSetupConst.setBasicStatsState(newParams,
StatsSetupConst.FALSE);
- LOG.info("Removed COLUMN_STATS_ACCURATE from the parameters of the
table "
- + dbname + "." + name);
+ } else {
+ String errorMsg =
verifyStatsChangeCtx(TableName.getDbTable(dbname, name), oldt.getParameters(),
newParams,
+ writeId, validWriteIds, true);
+ if (errorMsg != null) {
+ throw new MetaException(errorMsg);
+ }
+ if (!isCurrentStatsValidForTheQuery(oldt, validWriteIds, true)) {
+ // Make sure we set the flag to invalid regardless of the
current value.
+ StatsSetupConst.setBasicStatsState(newParams,
StatsSetupConst.FALSE);
+ LOG.info("Removed COLUMN_STATS_ACCURATE from the parameters of
the table " + dbname + "." + name);
+ }
+ oldt.setWriteId(writeId);
}
- oldt.setWriteId(writeId);
}
- }
- oldt.setParameters(newParams);
-
+ oldt.setParameters(newParams);
+ return newParams;
+ }).onRetry(e -> e instanceof RetryingExecutor.RetryException)
+
.commandName("updateTableColumnStatistics").sleepInterval(sleepInterval,
interval ->
+ ThreadLocalRandom.current().nextLong(sleepInterval) + 30).run();
committed = commitTransaction();
// TODO: similar to update...Part, this used to do "return committed;";
makes little sense.
- return committed ? newParams : null;
+ return committed ? result : null;
} finally {
- try {
- rollbackAndCleanup(committed, null);
- } finally {
- tableLock.unlock();
- }
+ LOG.info("{} updateTableColumnStatistics took {}ms, success: {}",
Review Comment:
do we need this under info? maybe debug or trace
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
