This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 408a12a Core: Add check-status config to TableProperties (#2596)
408a12a is described below
commit 408a12aaad25779eba7d1232bd41fd97d6a0409f
Author: Peidian li <[email protected]>
AuthorDate: Sat May 22 06:02:32 2021 +0800
Core: Add check-status config to TableProperties (#2596)
---
.../org/apache/iceberg/BaseMetastoreTableOperations.java | 16 +++++++++++++---
.../main/java/org/apache/iceberg/TableProperties.java | 11 ++++++++++-
2 files changed, 23 insertions(+), 4 deletions(-)
diff --git
a/core/src/main/java/org/apache/iceberg/BaseMetastoreTableOperations.java
b/core/src/main/java/org/apache/iceberg/BaseMetastoreTableOperations.java
index d8e6e93..49a08a7 100644
--- a/core/src/main/java/org/apache/iceberg/BaseMetastoreTableOperations.java
+++ b/core/src/main/java/org/apache/iceberg/BaseMetastoreTableOperations.java
@@ -39,6 +39,12 @@ import org.slf4j.LoggerFactory;
import static org.apache.iceberg.TableProperties.COMMIT_NUM_STATUS_CHECKS;
import static
org.apache.iceberg.TableProperties.COMMIT_NUM_STATUS_CHECKS_DEFAULT;
+import static
org.apache.iceberg.TableProperties.COMMIT_STATUS_CHECKS_MAX_WAIT_MS;
+import static
org.apache.iceberg.TableProperties.COMMIT_STATUS_CHECKS_MAX_WAIT_MS_DEFAULT;
+import static
org.apache.iceberg.TableProperties.COMMIT_STATUS_CHECKS_MIN_WAIT_MS;
+import static
org.apache.iceberg.TableProperties.COMMIT_STATUS_CHECKS_MIN_WAIT_MS_DEFAULT;
+import static
org.apache.iceberg.TableProperties.COMMIT_STATUS_CHECKS_TOTAL_WAIT_MS;
+import static
org.apache.iceberg.TableProperties.COMMIT_STATUS_CHECKS_TOTAL_WAIT_MS_DEFAULT;
public abstract class BaseMetastoreTableOperations implements TableOperations {
private static final Logger LOG =
LoggerFactory.getLogger(BaseMetastoreTableOperations.class);
@@ -50,8 +56,6 @@ public abstract class BaseMetastoreTableOperations implements
TableOperations {
private static final String METADATA_FOLDER_NAME = "metadata";
- private static final int COMMIT_STATUS_CHECK_WAIT_MS = 1000;
-
private TableMetadata currentMetadata = null;
private String currentMetadataLocation = null;
private boolean shouldRefresh = true;
@@ -270,13 +274,19 @@ public abstract class BaseMetastoreTableOperations
implements TableOperations {
protected CommitStatus checkCommitStatus(String newMetadataLocation,
TableMetadata config) {
int maxAttempts = PropertyUtil.propertyAsInt(config.properties(),
COMMIT_NUM_STATUS_CHECKS,
COMMIT_NUM_STATUS_CHECKS_DEFAULT);
+ long minWaitMs = PropertyUtil.propertyAsLong(config.properties(),
COMMIT_STATUS_CHECKS_MIN_WAIT_MS,
+ COMMIT_STATUS_CHECKS_MIN_WAIT_MS_DEFAULT);
+ long maxWaitMs = PropertyUtil.propertyAsLong(config.properties(),
COMMIT_STATUS_CHECKS_MAX_WAIT_MS,
+ COMMIT_STATUS_CHECKS_MAX_WAIT_MS_DEFAULT);
+ long totalRetryMs = PropertyUtil.propertyAsLong(config.properties(),
COMMIT_STATUS_CHECKS_TOTAL_WAIT_MS,
+ COMMIT_STATUS_CHECKS_TOTAL_WAIT_MS_DEFAULT);
AtomicReference<CommitStatus> status = new
AtomicReference<>(CommitStatus.UNKNOWN);
Tasks.foreach(newMetadataLocation)
.retry(maxAttempts)
.suppressFailureWhenFinished()
- .exponentialBackoff(COMMIT_STATUS_CHECK_WAIT_MS,
COMMIT_STATUS_CHECK_WAIT_MS, Long.MAX_VALUE, 2.0)
+ .exponentialBackoff(minWaitMs, maxWaitMs, totalRetryMs, 2.0)
.onFailure((location, checkException) ->
LOG.error("Cannot check if commit to {} exists.", tableName(),
checkException))
.run(location -> {
diff --git a/core/src/main/java/org/apache/iceberg/TableProperties.java
b/core/src/main/java/org/apache/iceberg/TableProperties.java
index bdc632b..56d921b 100644
--- a/core/src/main/java/org/apache/iceberg/TableProperties.java
+++ b/core/src/main/java/org/apache/iceberg/TableProperties.java
@@ -36,9 +36,18 @@ public class TableProperties {
public static final String COMMIT_TOTAL_RETRY_TIME_MS =
"commit.retry.total-timeout-ms";
public static final int COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT = 1800000; // 30
minutes
- public static final String COMMIT_NUM_STATUS_CHECKS =
"commit.num-status-checks";
+ public static final String COMMIT_NUM_STATUS_CHECKS =
"commit.status-check.num-retries";
public static final int COMMIT_NUM_STATUS_CHECKS_DEFAULT = 3;
+ public static final String COMMIT_STATUS_CHECKS_MIN_WAIT_MS =
"commit.status-check.min-wait-ms";
+ public static final long COMMIT_STATUS_CHECKS_MIN_WAIT_MS_DEFAULT = 1000L;
// 1s
+
+ public static final String COMMIT_STATUS_CHECKS_MAX_WAIT_MS =
"commit.status-check.max-wait-ms";
+ public static final long COMMIT_STATUS_CHECKS_MAX_WAIT_MS_DEFAULT = 60000L;
// 1 minute
+
+ public static final String COMMIT_STATUS_CHECKS_TOTAL_WAIT_MS =
"commit.status-check.total-timeout-ms";
+ public static final long COMMIT_STATUS_CHECKS_TOTAL_WAIT_MS_DEFAULT =
1800000; // 30 minutes
+
public static final String MANIFEST_TARGET_SIZE_BYTES =
"commit.manifest.target-size-bytes";
public static final long MANIFEST_TARGET_SIZE_BYTES_DEFAULT = 8388608; // 8
MB
