This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new ddc11b57de9 [HUDI-6088] Do not set the optmistic concurrency mode when
MDT is enabled (#8477)
ddc11b57de9 is described below
commit ddc11b57de928cf9c20c98a7f2703620bb0711c2
Author: Danny Chan <[email protected]>
AuthorDate: Tue Apr 18 15:41:43 2023 +0800
[HUDI-6088] Do not set the optmistic concurrency mode when MDT is enabled
(#8477)
---
.../org/apache/hudi/util/FlinkWriteClients.java | 7 +--
.../org/apache/hudi/sink/TestWriteCopyOnWrite.java | 4 +-
.../apache/hudi/utils/TestFlinkWriteClients.java | 61 ++++++++++++++++++++++
3 files changed, 65 insertions(+), 7 deletions(-)
diff --git
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/FlinkWriteClients.java
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/FlinkWriteClients.java
index fa324952ed8..34d326f843f 100644
---
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/FlinkWriteClients.java
+++
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/FlinkWriteClients.java
@@ -27,7 +27,6 @@ import
org.apache.hudi.common.config.SerializableConfiguration;
import org.apache.hudi.common.config.HoodieStorageConfig;
import org.apache.hudi.common.engine.EngineType;
import org.apache.hudi.common.model.HoodieCleaningPolicy;
-import org.apache.hudi.common.model.WriteConcurrencyMode;
import org.apache.hudi.common.model.WriteOperationType;
import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.config.HoodieArchivalConfig;
@@ -225,11 +224,7 @@ public class FlinkWriteClients {
.withProps(flinkConf2TypedProperties(conf))
.withSchema(getSourceSchema(conf).toString());
- if (conf.getBoolean(FlinkOptions.METADATA_ENABLED)) {
-
builder.withWriteConcurrencyMode(WriteConcurrencyMode.OPTIMISTIC_CONCURRENCY_CONTROL);
- }
-
- if (!conf.containsKey(HoodieLockConfig.LOCK_PROVIDER_CLASS_NAME.key()) &&
OptionsResolver.needsGuardByLock(conf)) {
+ if (OptionsResolver.needsGuardByLock(conf) &&
!conf.containsKey(HoodieLockConfig.LOCK_PROVIDER_CLASS_NAME.key())) {
// configure the fs lock provider by default
builder.withLockConfig(HoodieLockConfig.newBuilder()
.withConflictResolutionStrategy(OptionsResolver.getConflictResolutionStrategy(conf))
diff --git
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java
index 9f88dd24140..efecc0d4c9a 100644
---
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java
+++
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java
@@ -148,7 +148,9 @@ public class TestWriteCopyOnWrite extends TestWriteBase {
.checkpointThrows(4, "Timeout(1000ms) while waiting for instant
initialize")
.assertEmptyEvent()
.subTaskFails(0, 3)
- .assertEmptyEvent()
+ // the last checkpoint instant was rolled back by subTaskFails(0, 2)
+ // with EAGER cleaning strategy
+ .assertNoEvent()
.end();
}
diff --git
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestFlinkWriteClients.java
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestFlinkWriteClients.java
new file mode 100644
index 00000000000..262eea1f329
--- /dev/null
+++
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestFlinkWriteClients.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.utils;
+
+import org.apache.hudi.client.transaction.lock.FileSystemBasedLockProvider;
+import org.apache.hudi.common.model.WriteConcurrencyMode;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.configuration.FlinkOptions;
+import org.apache.hudi.util.FlinkWriteClients;
+import org.apache.hudi.util.StreamerUtil;
+
+import org.apache.flink.configuration.Configuration;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.File;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+/**
+ * Test cases for {@link org.apache.hudi.util.FlinkWriteClients}.
+ */
+public class TestFlinkWriteClients {
+ @TempDir
+ File tempFile;
+
+ private Configuration conf;
+
+ @BeforeEach
+ public void before() throws Exception {
+ this.conf = TestConfigurations.getDefaultConf(tempFile.getAbsolutePath());
+ }
+
+ @Test
+ void testAutoSetupLockProvider() throws Exception {
+ conf.setBoolean(FlinkOptions.METADATA_ENABLED, true);
+ StreamerUtil.initTableIfNotExists(conf);
+ HoodieWriteConfig writeConfig =
FlinkWriteClients.getHoodieClientConfig(conf, false, false);
+ assertThat(writeConfig.getLockProviderClass(),
is(FileSystemBasedLockProvider.class.getName()));
+ assertThat(writeConfig.getWriteConcurrencyMode(),
is(WriteConcurrencyMode.SINGLE_WRITER));
+ }
+}
