This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 4a3f118e6109 test(common): hoist FileCreateUtils duplicates into base
(#20052)
4a3f118e6109 is described below
commit 4a3f118e61093afafea023b7a550d8851f7f8867
Author: voonhous <[email protected]>
AuthorDate: Fri Sep 25 17:27:12 2026 +0800
test(common): hoist FileCreateUtils duplicates into base (#20052)
FileCreateUtils and FileCreateUtilsLegacy both redeclared
markerFileName(String, IOType) and isBaseOrLogFilename, which
FileCreateUtilsBase already defines, and carried byte-identical
copies of getPartitionPaths, getBaseFilePath, renameFileToTemp and
renameTempToMetaFile. Delete the shadows and move the four copies up
to the base. Callers use the subclass names, which still resolve to
the inherited statics.
---
.../hudi/common/testutils/FileCreateUtils.java | 43 ----------------------
.../hudi/common/testutils/FileCreateUtilsBase.java | 27 ++++++++++++++
.../common/testutils/FileCreateUtilsLegacy.java | 39 --------------------
3 files changed, 27 insertions(+), 82 deletions(-)
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java
b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java
index 676fe95d3aa5..8f2abf675a75 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java
@@ -29,9 +29,7 @@ import org.apache.hudi.avro.model.HoodieRollbackPlan;
import org.apache.hudi.avro.model.HoodieSavepointMetadata;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieCommitMetadata;
-import org.apache.hudi.common.model.HoodieFileFormat;
import org.apache.hudi.common.model.HoodieLogFile;
-import org.apache.hudi.common.model.HoodiePartitionMetadata;
import org.apache.hudi.common.model.HoodieReplaceCommitMetadata;
import org.apache.hudi.common.model.IOType;
import org.apache.hudi.common.table.HoodieTableMetaClient;
@@ -58,12 +56,9 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.FileTime;
import java.time.Instant;
-import java.util.Collections;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
-import java.util.stream.Collectors;
/**
* Utils for creating dummy Hudi files in testing.
@@ -97,10 +92,6 @@ public class FileCreateUtils extends FileCreateUtilsBase {
metadata.flatMap(m ->
metaClient.getCommitMetadataSerDe().getInstantWriter(m)));
}
- public static String markerFileName(String fileName, IOType ioType) {
- return String.format("%s%s.%s", fileName,
HoodieTableMetaClient.MARKER_EXTN, ioType.name());
- }
-
public static String dataFileMarkerFileName(String instantTime, String
fileId, IOType ioType, String fileExtension, String writeToken) {
return markerFileName(FSUtils.makeBaseFileName(instantTime, writeToken,
fileId, fileExtension), ioType);
}
@@ -394,11 +385,6 @@ public class FileCreateUtils extends FileCreateUtilsBase {
return baseFilePath.toString();
}
- public static Path getBaseFilePath(String basePath, String partitionPath,
String instantTime, String fileId) {
- Path parentPath = Paths.get(basePath, partitionPath);
- return parentPath.resolve(baseFileName(instantTime, fileId));
- }
-
public static String createLogFile(HoodieTableMetaClient metaClient, String
partitionPath, String instantTime, String fileId, int version)
throws Exception {
return createLogFile(metaClient, partitionPath, instantTime, fileId,
version, 0);
@@ -520,16 +506,6 @@ public class FileCreateUtils extends FileCreateUtilsBase {
removeMetaFileInTimelinePath(metaClient.getTimelinePath().toUri().getPath(),
instantTime, HoodieTimeline.ROLLBACK_EXTENSION);
}
- public static Path renameFileToTemp(Path sourcePath, String instantTime)
throws IOException {
- Path dummyFilePath = sourcePath.getParent().resolve(instantTime + ".temp");
- Files.move(sourcePath, dummyFilePath);
- return dummyFilePath;
- }
-
- public static void renameTempToMetaFile(Path tempFilePath, Path destPath)
throws IOException {
- Files.move(tempFilePath, destPath);
- }
-
public static long getTotalMarkerFileCount(HoodieTableMetaClient metaClient,
String partitionPath, String instantTime, IOType ioType) throws IOException {
Path parentPath = Paths.get(metaClient.getTempFolderPath(), instantTime,
partitionPath);
if (Files.notExists(parentPath)) {
@@ -539,25 +515,6 @@ public class FileCreateUtils extends FileCreateUtilsBase {
.endsWith(String.format("%s.%s", HoodieTableMetaClient.MARKER_EXTN,
ioType))).count();
}
- public static List<Path> getPartitionPaths(Path basePath) throws IOException
{
- if (Files.notExists(basePath)) {
- return Collections.emptyList();
- }
- return Files.list(basePath).filter(entry ->
!entry.getFileName().toString().equals(HoodieTableMetaClient.METAFOLDER_NAME)
- && !isBaseOrLogFilename(entry.getFileName().toString())
- &&
!entry.getFileName().toString().startsWith(HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE_PREFIX))
- .collect(Collectors.toList());
- }
-
- public static boolean isBaseOrLogFilename(String filename) {
- for (HoodieFileFormat format : HoodieFileFormat.values()) {
- if (filename.contains(format.getFileExtension())) {
- return true;
- }
- }
- return false;
- }
-
/**
* Find total basefiles for passed in paths.
*/
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtilsBase.java
b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtilsBase.java
index d4721602a660..683af14f14e9 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtilsBase.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtilsBase.java
@@ -43,10 +43,12 @@ import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
/**
* Utils for creating dummy Hudi files in testing.
@@ -102,6 +104,31 @@ public class FileCreateUtilsBase {
return false;
}
+ public static List<Path> getPartitionPaths(Path basePath) throws IOException
{
+ if (Files.notExists(basePath)) {
+ return Collections.emptyList();
+ }
+ return Files.list(basePath).filter(entry ->
!entry.getFileName().toString().equals(HoodieTableMetaClient.METAFOLDER_NAME)
+ && !isBaseOrLogFilename(entry.getFileName().toString())
+ &&
!entry.getFileName().toString().startsWith(HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE_PREFIX))
+ .collect(Collectors.toList());
+ }
+
+ public static Path getBaseFilePath(String basePath, String partitionPath,
String instantTime, String fileId) {
+ Path parentPath = Paths.get(basePath, partitionPath);
+ return parentPath.resolve(baseFileName(instantTime, fileId));
+ }
+
+ public static Path renameFileToTemp(Path sourcePath, String instantTime)
throws IOException {
+ Path dummyFilePath = sourcePath.getParent().resolve(instantTime + ".temp");
+ Files.move(sourcePath, dummyFilePath);
+ return dummyFilePath;
+ }
+
+ public static void renameTempToMetaFile(Path tempFilePath, Path destPath)
throws IOException {
+ Files.move(tempFilePath, destPath);
+ }
+
public static URI createPartitionMetaFile(String basePath, String
partitionPath) throws IOException {
Path metaFilePath;
try {
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtilsLegacy.java
b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtilsLegacy.java
index cf28d9fb2b78..2ee5ea84dbf9 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtilsLegacy.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtilsLegacy.java
@@ -24,9 +24,7 @@ import
org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata;
import org.apache.hudi.avro.model.HoodieRollbackPlan;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieCommitMetadata;
-import org.apache.hudi.common.model.HoodieFileFormat;
import org.apache.hudi.common.model.HoodieLogFile;
-import org.apache.hudi.common.model.HoodiePartitionMetadata;
import org.apache.hudi.common.model.HoodieReplaceCommitMetadata;
import org.apache.hudi.common.model.IOType;
import org.apache.hudi.common.table.HoodieTableMetaClient;
@@ -49,10 +47,7 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.FileTime;
import java.time.Instant;
-import java.util.Collections;
-import java.util.List;
import java.util.function.Supplier;
-import java.util.stream.Collectors;
import static
org.apache.hudi.common.testutils.HoodieTestUtils.COMMIT_METADATA_SER_DE;
@@ -202,11 +197,6 @@ public class FileCreateUtilsLegacy extends
FileCreateUtilsBase {
return baseFilePath.toString();
}
- public static Path getBaseFilePath(String basePath, String partitionPath,
String instantTime, String fileId) {
- Path parentPath = Paths.get(basePath, partitionPath);
- return parentPath.resolve(baseFileName(instantTime, fileId));
- }
-
public static String createLogFile(String basePath, String partitionPath,
String instantTime, String fileId, int version)
throws Exception {
return createLogFile(basePath, partitionPath, instantTime, fileId,
version, 0);
@@ -305,16 +295,6 @@ public class FileCreateUtilsLegacy extends
FileCreateUtilsBase {
removeMetaFileInTimelinePath(getTimelinePath(new
StoragePath(basePath)).toUri().getPath(), instantTime,
HoodieTimeline.ROLLBACK_EXTENSION);
}
- public static Path renameFileToTemp(Path sourcePath, String instantTime)
throws IOException {
- Path dummyFilePath = sourcePath.getParent().resolve(instantTime + ".temp");
- Files.move(sourcePath, dummyFilePath);
- return dummyFilePath;
- }
-
- public static void renameTempToMetaFile(Path tempFilePath, Path destPath)
throws IOException {
- Files.move(tempFilePath, destPath);
- }
-
public static long getTotalMarkerFileCount(String basePath, String
partitionPath, String instantTime, IOType ioType) throws IOException {
Path parentPath = Paths.get(basePath,
HoodieTableMetaClient.TEMPFOLDER_NAME, instantTime, partitionPath);
if (Files.notExists(parentPath)) {
@@ -324,25 +304,6 @@ public class FileCreateUtilsLegacy extends
FileCreateUtilsBase {
.endsWith(String.format("%s.%s", HoodieTableMetaClient.MARKER_EXTN,
ioType))).count();
}
- public static List<Path> getPartitionPaths(Path basePath) throws IOException
{
- if (Files.notExists(basePath)) {
- return Collections.emptyList();
- }
- return Files.list(basePath).filter(entry ->
!entry.getFileName().toString().equals(HoodieTableMetaClient.METAFOLDER_NAME)
- && !isBaseOrLogFilename(entry.getFileName().toString())
- &&
!entry.getFileName().toString().startsWith(HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE_PREFIX))
- .collect(Collectors.toList());
- }
-
- public static boolean isBaseOrLogFilename(String filename) {
- for (HoodieFileFormat format : HoodieFileFormat.values()) {
- if (filename.contains(format.getFileExtension())) {
- return true;
- }
- }
- return false;
- }
-
public static void deleteDeltaCommit(String basePath, String instantTime,
HoodieStorage storage) throws
IOException {
deleteMetaFileInTimeline(getTimelinePath(new
StoragePath(basePath)).toUri().getPath(), instantTime,
HoodieTimeline.DELTA_COMMIT_EXTENSION, storage);
