This is an automated email from the ASF dual-hosted git repository.
jonvex pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 7f0a88ce905 [MINOR] Remove unused util methods in LogReaderUtils
(#11086)
7f0a88ce905 is described below
commit 7f0a88ce905f8cc0872778f87bce9fa5b03f1f0c
Author: Y Ethan Guo <[email protected]>
AuthorDate: Wed Apr 24 14:41:57 2024 -0700
[MINOR] Remove unused util methods in LogReaderUtils (#11086)
---
.../hudi/common/table/log/LogReaderUtils.java | 58 ----------------------
1 file changed, 58 deletions(-)
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/log/LogReaderUtils.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/log/LogReaderUtils.java
index 68568417627..a837bc9b88b 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/log/LogReaderUtils.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/log/LogReaderUtils.java
@@ -18,21 +18,9 @@
package org.apache.hudi.common.table.log;
-import org.apache.hudi.common.model.HoodieLogFile;
-import org.apache.hudi.common.table.HoodieTableMetaClient;
-import org.apache.hudi.common.table.log.HoodieLogFormat.Reader;
-import org.apache.hudi.common.table.log.block.HoodieDataBlock;
-import org.apache.hudi.common.table.log.block.HoodieLogBlock;
import
org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType;
-import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
-import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.util.Base64CodecUtil;
-import org.apache.hudi.common.util.collection.Pair;
-import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.storage.HoodieStorageUtils;
-import org.apache.avro.Schema;
-import org.apache.hadoop.conf.Configuration;
import org.roaringbitmap.longlong.Roaring64NavigableMap;
import java.io.ByteArrayInputStream;
@@ -40,58 +28,12 @@ import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
-import java.util.List;
-import java.util.Map;
import java.util.Set;
-import java.util.stream.Collectors;
/**
* Utils class for performing various log file reading operations.
*/
public class LogReaderUtils {
-
- private static Schema readSchemaFromLogFileInReverse(HoodieStorage storage,
HoodieActiveTimeline activeTimeline, HoodieLogFile hoodieLogFile)
- throws IOException {
- // set length for the HoodieLogFile as it will be leveraged by
HoodieLogFormat.Reader with reverseReading enabled
- Schema writerSchema = null;
- try (Reader reader = HoodieLogFormat.newReader(storage, hoodieLogFile,
null, true)) {
- HoodieTimeline completedTimeline =
activeTimeline.getCommitsTimeline().filterCompletedInstants();
- while (reader.hasPrev()) {
- HoodieLogBlock block = reader.prev();
- if (block instanceof HoodieDataBlock) {
- HoodieDataBlock lastBlock = (HoodieDataBlock) block;
- if (completedTimeline
-
.containsOrBeforeTimelineStarts(lastBlock.getLogBlockHeader().get(HeaderMetadataType.INSTANT_TIME)))
{
- writerSchema = new
Schema.Parser().parse(lastBlock.getLogBlockHeader().get(HeaderMetadataType.SCHEMA));
- break;
- }
- }
- }
- }
- return writerSchema;
- }
-
- public static Schema readLatestSchemaFromLogFiles(String basePath,
List<HoodieLogFile> logFiles, Configuration config)
- throws IOException {
- HoodieTableMetaClient metaClient =
HoodieTableMetaClient.builder().setConf(config).setBasePath(basePath).build();
- List<String> deltaPaths =
logFiles.stream().sorted(HoodieLogFile.getReverseLogFileComparator()).map(s ->
s.getPath().toString())
- .collect(Collectors.toList());
- if (deltaPaths.size() > 0) {
- Map<String, HoodieLogFile> deltaFilePathToFileStatus =
logFiles.stream().map(entry -> Pair.of(entry.getPath().toString(), entry))
- .collect(Collectors.toMap(Pair::getKey, Pair::getValue));
- for (String logPath : deltaPaths) {
- HoodieStorage storage = HoodieStorageUtils.getStorage(logPath, config);
- Schema schemaFromLogFile =
- readSchemaFromLogFileInReverse(storage,
metaClient.getActiveTimeline(),
- deltaFilePathToFileStatus.get(logPath));
- if (schemaFromLogFile != null) {
- return schemaFromLogFile;
- }
- }
- }
- return null;
- }
-
/**
* Encodes a list of record positions in long type.
* <p>
