This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new eeddac702ed [HUDI-1623][Tests] Fix test TestWaitBasedTimeGenerator
(#9972)
eeddac702ed is described below
commit eeddac702ed3a97d6b08a699c506a1898de4af16
Author: Rex(Hui) An <[email protected]>
AuthorDate: Fri Nov 3 08:18:44 2023 +0800
[HUDI-1623][Tests] Fix test TestWaitBasedTimeGenerator (#9972)
---
.../java/org/apache/hudi/config/DynamoDbBasedLockConfig.java | 2 +-
.../main/java/org/apache/hudi/config/HoodieLockConfig.java | 3 ++-
.../hudi/client/transaction/lock/InProcessLockProvider.java | 2 +-
.../org/apache/hudi/common/config/LockConfiguration.java | 4 +---
.../org/apache/hudi/common/table/timeline/TimeGenerator.java | 8 ++++----
.../apache/hudi/common/table/timeline/TimeGeneratorBase.java | 2 +-
.../hudi/common/table/timeline/WaitBasedTimeGenerator.java | 12 ++++++------
.../common/table/timeline/TestWaitBasedTimeGenerator.java | 6 ++++--
8 files changed, 20 insertions(+), 19 deletions(-)
diff --git
a/hudi-aws/src/main/java/org/apache/hudi/config/DynamoDbBasedLockConfig.java
b/hudi-aws/src/main/java/org/apache/hudi/config/DynamoDbBasedLockConfig.java
index 5639db02582..0e884a6797f 100644
--- a/hudi-aws/src/main/java/org/apache/hudi/config/DynamoDbBasedLockConfig.java
+++ b/hudi-aws/src/main/java/org/apache/hudi/config/DynamoDbBasedLockConfig.java
@@ -127,7 +127,7 @@ public class DynamoDbBasedLockConfig extends HoodieConfig {
public static final ConfigProperty<Integer>
LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY = ConfigProperty
.key(LockConfiguration.LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY)
- .defaultValue(LockConfiguration.DEFAULT_ACQUIRE_LOCK_WAIT_TIMEOUT_MS)
+ .defaultValue(LockConfiguration.DEFAULT_LOCK_ACQUIRE_WAIT_TIMEOUT_MS)
.markAdvanced()
.sinceVersion("0.10.0")
.withDocumentation("Lock Acquire Wait Timeout in milliseconds");
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieLockConfig.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieLockConfig.java
index b24aecf46c1..fa38da8f8ab 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieLockConfig.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieLockConfig.java
@@ -36,6 +36,7 @@ import java.util.Properties;
import static
org.apache.hudi.common.config.LockConfiguration.DEFAULT_LOCK_ACQUIRE_NUM_RETRIES;
import static
org.apache.hudi.common.config.LockConfiguration.DEFAULT_LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS;
+import static
org.apache.hudi.common.config.LockConfiguration.DEFAULT_LOCK_ACQUIRE_WAIT_TIMEOUT_MS;
import static
org.apache.hudi.common.config.LockConfiguration.DEFAULT_ZK_CONNECTION_TIMEOUT_MS;
import static
org.apache.hudi.common.config.LockConfiguration.DEFAULT_ZK_SESSION_TIMEOUT_MS;
import static
org.apache.hudi.common.config.LockConfiguration.FILESYSTEM_LOCK_EXPIRE_PROP_KEY;
@@ -106,7 +107,7 @@ public class HoodieLockConfig extends HoodieConfig {
public static final ConfigProperty<Integer> LOCK_ACQUIRE_WAIT_TIMEOUT_MS =
ConfigProperty
.key(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY)
- .defaultValue(60 * 1000)
+ .defaultValue(DEFAULT_LOCK_ACQUIRE_WAIT_TIMEOUT_MS)
.markAdvanced()
.sinceVersion("0.8.0")
.withDocumentation("Timeout in ms, to wait on an individual lock
acquire() call, at the lock provider.");
diff --git
a/hudi-common/src/main/java/org/apache/hudi/client/transaction/lock/InProcessLockProvider.java
b/hudi-common/src/main/java/org/apache/hudi/client/transaction/lock/InProcessLockProvider.java
index c3437f91c8c..c2edb1864b0 100644
---
a/hudi-common/src/main/java/org/apache/hudi/client/transaction/lock/InProcessLockProvider.java
+++
b/hudi-common/src/main/java/org/apache/hudi/client/transaction/lock/InProcessLockProvider.java
@@ -61,7 +61,7 @@ public class InProcessLockProvider implements
LockProvider<ReentrantReadWriteLoc
ValidationUtils.checkArgument(basePath != null);
lock = LOCK_INSTANCE_PER_BASEPATH.computeIfAbsent(basePath, (ignore) ->
new ReentrantReadWriteLock());
maxWaitTimeMillis =
typedProperties.getLong(LockConfiguration.LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY,
- LockConfiguration.DEFAULT_ACQUIRE_LOCK_WAIT_TIMEOUT_MS);
+ LockConfiguration.DEFAULT_LOCK_ACQUIRE_WAIT_TIMEOUT_MS);
}
@Override
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/config/LockConfiguration.java
b/hudi-common/src/main/java/org/apache/hudi/common/config/LockConfiguration.java
index 9e652c64efe..1171dcf3fce 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/config/LockConfiguration.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/config/LockConfiguration.java
@@ -43,7 +43,7 @@ public class LockConfiguration implements Serializable {
public static final String LOCK_ACQUIRE_CLIENT_NUM_RETRIES_PROP_KEY =
LOCK_PREFIX + "client.num_retries";
public static final String LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY =
LOCK_PREFIX + "wait_time_ms";
- public static final String DEFAULT_LOCK_ACQUIRE_WAIT_TIMEOUT_MS =
String.valueOf(60 * 1000);
+ public static final int DEFAULT_LOCK_ACQUIRE_WAIT_TIMEOUT_MS = 60 * 1000;
// configs for file system based locks. NOTE: This only works for DFS with
atomic create/delete operation
public static final String FILESYSTEM_BASED_LOCK_PROPERTY_PREFIX =
LOCK_PREFIX + "filesystem.";
@@ -100,8 +100,6 @@ public class LockConfiguration implements Serializable {
/** @deprecated Use {@link #LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY} */
@Deprecated
public static final String LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP =
LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY;
- @Deprecated
- public static final int DEFAULT_ACQUIRE_LOCK_WAIT_TIMEOUT_MS = 60 * 1000;
/** @deprecated Use {@link #HIVE_DATABASE_NAME_PROP_KEY} */
@Deprecated
public static final String HIVE_DATABASE_NAME_PROP =
HIVE_DATABASE_NAME_PROP_KEY;
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimeGenerator.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimeGenerator.java
index 7d3b8d71341..5a124c453dc 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimeGenerator.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimeGenerator.java
@@ -35,17 +35,17 @@ public interface TimeGenerator {
* a new generated timestamp T is guaranteed to be greater than
* any timestamp generated by the preceding calls.
*
- * @param locked Whether the caller holds the lock itself.
+ * @param skipLocking If this is triggered by another parent transaction,
locking can be skipped.
* @return Current true time as milliseconds.
*/
- long currentTimeMillis(boolean locked);
+ long currentTimeMillis(boolean skipLocking);
/**
* Passes an auto generated timestamp to the given function {@code func}.
The implementations
* need to ensure timestamp generation and executing func are atomic.
*
- * @param locked Whether the caller holds the lock itself.
+ * @param skipLocking If this is triggered by another parent transaction,
locking can be skipped.
* @param func A consumer that takes a monotonically increasing timestamp.
*/
- void consumeTimestamp(boolean locked, Consumer<Long> func);
+ void consumeTimestamp(boolean skipLocking, Consumer<Long> func);
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimeGeneratorBase.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimeGeneratorBase.java
index 8b37b21c59b..4acb8d2af54 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimeGeneratorBase.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimeGeneratorBase.java
@@ -82,7 +82,7 @@ public abstract class TimeGeneratorBase implements
TimeGenerator, Serializable {
maxRetries =
lockConfiguration.getConfig().getInteger(LOCK_ACQUIRE_CLIENT_NUM_RETRIES_PROP_KEY,
Integer.parseInt(DEFAULT_LOCK_ACQUIRE_NUM_RETRIES));
lockAcquireWaitTimeInMs =
lockConfiguration.getConfig().getInteger(LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY,
- Integer.parseInt(DEFAULT_LOCK_ACQUIRE_WAIT_TIMEOUT_MS));
+ DEFAULT_LOCK_ACQUIRE_WAIT_TIMEOUT_MS);
maxWaitTimeInMs =
lockConfiguration.getConfig().getLong(LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS_PROP_KEY,
Long.parseLong(DEFAULT_LOCK_ACQUIRE_CLIENT_RETRY_WAIT_TIME_IN_MILLIS));
lockRetryHelper = new RetryHelper<>(maxWaitTimeInMs, maxRetries,
maxWaitTimeInMs,
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/WaitBasedTimeGenerator.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/WaitBasedTimeGenerator.java
index 8728629e983..2cf389ab991 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/WaitBasedTimeGenerator.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/WaitBasedTimeGenerator.java
@@ -36,9 +36,9 @@ public class WaitBasedTimeGenerator extends TimeGeneratorBase
{
}
@Override
- public long currentTimeMillis(boolean locked) {
+ public long currentTimeMillis(boolean skipLocking) {
try {
- if (!locked) {
+ if (!skipLocking) {
lock();
}
long ts = System.currentTimeMillis();
@@ -47,22 +47,22 @@ public class WaitBasedTimeGenerator extends
TimeGeneratorBase {
} catch (InterruptedException e) {
throw new HoodieException("Interrupted when get the current time", e);
} finally {
- if (!locked) {
+ if (!skipLocking) {
unlock();
}
}
}
@Override
- public void consumeTimestamp(boolean locked, Consumer<Long> func) {
+ public void consumeTimestamp(boolean skipLocking, Consumer<Long> func) {
try {
- if (!locked) {
+ if (!skipLocking) {
lock();
}
long currentTimeMillis = currentTimeMillis(true);
func.accept(currentTimeMillis);
} finally {
- if (!locked) {
+ if (!skipLocking) {
unlock();
}
}
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestWaitBasedTimeGenerator.java
b/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestWaitBasedTimeGenerator.java
index a31b8e50dd8..ba6c88fdae1 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestWaitBasedTimeGenerator.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestWaitBasedTimeGenerator.java
@@ -64,7 +64,9 @@ public class TestWaitBasedTimeGenerator {
}
}
boolean locked = super.tryLock(time, unit);
- SIGNAL.countDown();
+ if (locked) {
+ SIGNAL.countDown();
+ }
return locked;
}
}
@@ -79,7 +81,7 @@ public class TestWaitBasedTimeGenerator {
@BeforeEach
public void initialize() {
timeGeneratorConfig = HoodieTimeGeneratorConfig.newBuilder()
- .withPath("")
+ .withPath("test_wait_based")
.withMaxExpectedClockSkewMs(25L)
.withTimeGeneratorType(TimeGeneratorType.WAIT_TO_ADJUST_SKEW)
.build();
