veghlaci05 commented on code in PR #4384: URL: https://github.com/apache/hive/pull/4384#discussion_r1310013878
########## standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/retryhandling/RetryHandler.java: ########## @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.txn.retryhandling; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.txn.MetaWrapperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.dao.DataAccessException; +import org.springframework.jdbc.UncategorizedSQLException; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.DefaultTransactionDefinition; + +import java.sql.SQLException; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; +import java.util.regex.Pattern; + +/** + * Supports building resilient APIs by automatically retrying failed function calls. + */ +public class RetryHandler { + + private static final Logger LOG = LoggerFactory.getLogger(RetryHandler.class); + + private static final int ALLOWED_REPEATED_DEADLOCKS = 10; + private static final String MANUAL_RETRY = "ManualRetry"; + + /** + * Derby specific concurrency control + */ + private static final ReentrantLock derbyLock = new ReentrantLock(true); + + + private DataSourceWrapper jdbcTemplate; + private long deadlockRetryInterval; + private long retryInterval; + private int retryLimit; + + private Configuration conf; + + public static String getMessage(Exception ex) { + StringBuilder sb = new StringBuilder(); + sb.append(ex.getMessage()); + if (ex instanceof SQLException) { + SQLException sqlEx = (SQLException)ex; + sb.append(" (SQLState=").append(sqlEx.getSQLState()).append(", ErrorCode=").append(sqlEx.getErrorCode()).append(")"); + } + return sb.toString(); + } + + public void init(Configuration conf, DataSourceWrapper jdbcTemplate){ + this.conf = conf; + this.jdbcTemplate = jdbcTemplate; + + retryInterval = MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.HMS_HANDLER_INTERVAL, + TimeUnit.MILLISECONDS); + retryLimit = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.HMS_HANDLER_ATTEMPTS); + deadlockRetryInterval = retryInterval / 10; + } + + /** + * Establishes a {@link DataSourceWrapper.RetryContext} and executes the passed {@link TransactionalFunction}. + * Automatically retries the execution in case of failure. + * @param properties {@link RetryCallProperties} instance used to fine-tune/alter the execution-mechanism. + * @param function The {@link TransactionalFunction} to execute. + * @return Returns with the result of the execution of the passed {@link TransactionalFunction}. + * @param <Result> Type of the result + * @throws MetaException Thrown in case of execution error. + */ + public <Result> Result executeWithRetry(RetryCallProperties properties, TransactionalFunction<Result> function) + throws MetaException { + return executeWithRetryInternal(properties, function, retryLimit, ALLOWED_REPEATED_DEADLOCKS); + } + + /** + * Executes the passed {@link TransactionalFunction}, without retry in case of failure. + * @param properties {@link RetryCallProperties} instance used to fine-tune/alter the execution-mechanism. + * @param function The {@link TransactionalFunction} to execute. + * @return Returns with the result of the execution of the passed {@link TransactionalFunction}. + * @param <Result> Type of the result + * @throws MetaException Thrown in case of execution error. + */ + public <Result> Result executeWithoutRetry(RetryCallProperties properties, TransactionalFunction<Result> function) + throws MetaException { + return executeWithRetryInternal(properties, function, 0, 0); + } + + private <Result> Result executeWithRetryInternal(RetryCallProperties properties, TransactionalFunction<Result> function + , int retryCount, int deadlockCount) throws MetaException { + Objects.requireNonNull(function, "RetryFunction<Result> cannot be null!"); + Objects.requireNonNull(properties, "RetryCallProperties cannot be null!"); + + if (jdbcTemplate.hasRetryContext() && properties.getRetryPropagation().canJoinContext()) { + /* + If there is a RetryState in the ThreadLocal and we are allowed to join it, we can skip establishing a nested + retry-call (and transaction). + */ + LOG.info("Already established retry-context detected, current retry-call will join it. The passed CallProperties " + + "instance will be ignored, using the original one."); + try { + return function.execute(jdbcTemplate); + } catch (SQLException e) { + throw new UncategorizedSQLException(null, null, e); + } + } + + if (!properties.getRetryPropagation().canCreateContext()) { + throw new MetaException("The current RetryPropagation mode (" + properties.getRetryPropagation().name() + + ") allows only to join an existing retry-context, but there is no context to join!"); + } + + LOG.debug("Running retry function:" + properties); + + if (org.apache.commons.lang3.StringUtils.isEmpty(properties.getCaller())) { + properties.withCallerId(function.toString()); + } + + try { + if (properties.isLockInternally()) { + lockInternal(); + } + try { + DefaultTransactionDefinition transactionDefinition = new DefaultTransactionDefinition(TransactionDefinition.PROPAGATION_REQUIRED); Review Comment: Created the propagation in RetryCallProperties. The idea was to tie together the retry and transaction handling: In case of failure rollback AND retry. In case of success commit AND do no retry. Handling them independently would allow more fine grained control, but would also introduce extra complexity: cases like doing a retry with the old connection and txn, or committing the txn, and still doing a retry. I'm not sure it's worth allowing these, as currently I cannot see places where such cases are necessary. ########## standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/retryhandling/RetryHandler.java: ########## @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.txn.retryhandling; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.txn.MetaWrapperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.dao.DataAccessException; +import org.springframework.jdbc.UncategorizedSQLException; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.DefaultTransactionDefinition; + +import java.sql.SQLException; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; +import java.util.regex.Pattern; + +/** + * Supports building resilient APIs by automatically retrying failed function calls. + */ +public class RetryHandler { + + private static final Logger LOG = LoggerFactory.getLogger(RetryHandler.class); + + private static final int ALLOWED_REPEATED_DEADLOCKS = 10; + private static final String MANUAL_RETRY = "ManualRetry"; + + /** + * Derby specific concurrency control + */ + private static final ReentrantLock derbyLock = new ReentrantLock(true); + + + private DataSourceWrapper jdbcTemplate; + private long deadlockRetryInterval; + private long retryInterval; + private int retryLimit; + + private Configuration conf; + + public static String getMessage(Exception ex) { + StringBuilder sb = new StringBuilder(); + sb.append(ex.getMessage()); + if (ex instanceof SQLException) { + SQLException sqlEx = (SQLException)ex; + sb.append(" (SQLState=").append(sqlEx.getSQLState()).append(", ErrorCode=").append(sqlEx.getErrorCode()).append(")"); + } + return sb.toString(); + } + + public void init(Configuration conf, DataSourceWrapper jdbcTemplate){ + this.conf = conf; + this.jdbcTemplate = jdbcTemplate; + + retryInterval = MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.HMS_HANDLER_INTERVAL, + TimeUnit.MILLISECONDS); + retryLimit = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.HMS_HANDLER_ATTEMPTS); + deadlockRetryInterval = retryInterval / 10; + } + + /** + * Establishes a {@link DataSourceWrapper.RetryContext} and executes the passed {@link TransactionalFunction}. + * Automatically retries the execution in case of failure. + * @param properties {@link RetryCallProperties} instance used to fine-tune/alter the execution-mechanism. + * @param function The {@link TransactionalFunction} to execute. + * @return Returns with the result of the execution of the passed {@link TransactionalFunction}. + * @param <Result> Type of the result + * @throws MetaException Thrown in case of execution error. + */ + public <Result> Result executeWithRetry(RetryCallProperties properties, TransactionalFunction<Result> function) + throws MetaException { + return executeWithRetryInternal(properties, function, retryLimit, ALLOWED_REPEATED_DEADLOCKS); + } + + /** + * Executes the passed {@link TransactionalFunction}, without retry in case of failure. + * @param properties {@link RetryCallProperties} instance used to fine-tune/alter the execution-mechanism. + * @param function The {@link TransactionalFunction} to execute. + * @return Returns with the result of the execution of the passed {@link TransactionalFunction}. + * @param <Result> Type of the result + * @throws MetaException Thrown in case of execution error. + */ + public <Result> Result executeWithoutRetry(RetryCallProperties properties, TransactionalFunction<Result> function) + throws MetaException { + return executeWithRetryInternal(properties, function, 0, 0); + } + + private <Result> Result executeWithRetryInternal(RetryCallProperties properties, TransactionalFunction<Result> function + , int retryCount, int deadlockCount) throws MetaException { + Objects.requireNonNull(function, "RetryFunction<Result> cannot be null!"); + Objects.requireNonNull(properties, "RetryCallProperties cannot be null!"); + + if (jdbcTemplate.hasRetryContext() && properties.getRetryPropagation().canJoinContext()) { + /* + If there is a RetryState in the ThreadLocal and we are allowed to join it, we can skip establishing a nested + retry-call (and transaction). + */ + LOG.info("Already established retry-context detected, current retry-call will join it. The passed CallProperties " + + "instance will be ignored, using the original one."); + try { + return function.execute(jdbcTemplate); + } catch (SQLException e) { + throw new UncategorizedSQLException(null, null, e); + } + } + + if (!properties.getRetryPropagation().canCreateContext()) { + throw new MetaException("The current RetryPropagation mode (" + properties.getRetryPropagation().name() + + ") allows only to join an existing retry-context, but there is no context to join!"); + } + + LOG.debug("Running retry function:" + properties); + + if (org.apache.commons.lang3.StringUtils.isEmpty(properties.getCaller())) { + properties.withCallerId(function.toString()); + } + + try { + if (properties.isLockInternally()) { + lockInternal(); + } + try { + DefaultTransactionDefinition transactionDefinition = new DefaultTransactionDefinition(TransactionDefinition.PROPAGATION_REQUIRED); Review Comment: Created the propagation in `RetryCallProperties`. The idea was to tie together the retry and transaction handling: In case of failure rollback AND retry. In case of success commit AND do no retry. Handling them independently would allow more fine grained control, but would also introduce extra complexity: cases like doing a retry with the old connection and txn, or committing the txn, and still doing a retry. I'm not sure it's worth allowing these, as currently I cannot see places where such cases are necessary. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
