stream2000 commented on code in PR #9651:
URL: https://github.com/apache/hudi/pull/9651#discussion_r1329665072
##########
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/meta/TestCkpMetadata.java:
##########
@@ -94,9 +107,17 @@ void testBootstrap() throws Exception {
metadata1.getInstantCache(), is(Collections.singletonList("4")));
}
- private CkpMetadata getCkpMetadata(String uniqueId) {
- String basePath = tempFile.getAbsolutePath();
- FileSystem fs = FSUtils.getFs(basePath,
HadoopConfigurations.getHadoopConf(new Configuration()));
- return CkpMetadata.getInstance(fs, basePath, uniqueId);
+ protected CkpMetadata getCkpMetadata(String uniqueId) {
+ conf.set(FlinkOptions.WRITE_CLIENT_ID, uniqueId);
+ return
CkpMetadataFactory.getCkpMetadata(writeClient.getHoodieTable().getMetaClient(),
writeClient.getConfig(), conf);
+ }
Review Comment:
Changed to use metaClient created in `setup` method instead
##########
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/TestWriteBase.java:
##########
@@ -137,14 +140,17 @@ public static TestHarness instance() {
private String lastPending;
private String lastComplete;
+ private HoodieFlinkWriteClient writeClient;
+
public TestHarness preparePipeline(File basePath, Configuration conf)
throws Exception {
this.baseFile = basePath;
this.basePath = this.baseFile.getAbsolutePath();
this.conf = conf;
this.pipeline = TestData.getWritePipeline(this.basePath, conf);
// open the function and ingest data
this.pipeline.openFunction();
- this.ckpMetadata = CkpMetadata.getInstance(conf);
+ this.writeClient = FlinkWriteClients.createWriteClient(conf);
+ this.ckpMetadata =
CkpMetadataFactory.getCkpMetadata(StreamerUtil.createMetaClient(conf),
writeClient.getConfig(), conf);
Review Comment:
We maintain a ckpMetadata in `TestWriteBase` for `checkInflightInstant` and
get `lastPendingInstant`. This metadata is not from coordinator so we need a
timeline server for it.
##########
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/meta/TestCkpMetadata.java:
##########
@@ -49,11 +52,21 @@ public class TestCkpMetadata {
@TempDir
File tempFile;
+ protected Configuration conf;
+
+ protected HoodieFlinkWriteClient writeClient;
+
@BeforeEach
public void beforeEach() throws Exception {
+ setup();
+ }
+
+ protected void setup() throws IOException {
String basePath = tempFile.getAbsolutePath();
- Configuration conf = TestConfigurations.getDefaultConf(basePath);
+ this.conf = TestConfigurations.getDefaultConf(basePath);
+
conf.setString(HoodieWriteConfig.INSTANT_STATE_TIMELINE_SERVER_BASED.key(),
"false");
StreamerUtil.initTableIfNotExists(conf);
+ this.writeClient = FlinkWriteClients.createWriteClient(conf);
}
Review Comment:
In `TestCkpMetadata` there is no coordinator so we need to create a write
client to start the timeline server.
##########
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/meta/TestCkpMetadata.java:
##########
@@ -94,9 +107,17 @@ void testBootstrap() throws Exception {
metadata1.getInstantCache(), is(Collections.singletonList("4")));
}
- private CkpMetadata getCkpMetadata(String uniqueId) {
- String basePath = tempFile.getAbsolutePath();
- FileSystem fs = FSUtils.getFs(basePath,
HadoopConfigurations.getHadoopConf(new Configuration()));
- return CkpMetadata.getInstance(fs, basePath, uniqueId);
+ protected CkpMetadata getCkpMetadata(String uniqueId) {
+ conf.set(FlinkOptions.WRITE_CLIENT_ID, uniqueId);
+ return
CkpMetadataFactory.getCkpMetadata(writeClient.getHoodieTable().getMetaClient(),
writeClient.getConfig(), conf);
+ }
+
+ @AfterEach
+ public void cleanup() {
+ if (writeClient != null) {
+ writeClient.close();
+ writeClient = null;
+ }
}
+
Review Comment:
removed it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
