This is an automated email from the ASF dual-hosted git repository. leesf pushed a commit to branch revert-2116-patch-4 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 218e0eea27a15f0d69ee1aa5f3793d5536e3bacd Author: leesf <[email protected]> AuthorDate: Sat Sep 26 21:32:55 2020 +0800 Revert "[MINOR] fix typos (#2116)" This reverts commit ae68b2b355a9dd23ff9fd1dd3f5637039b02c14a. --- .../apache/hudi/cli/commands/BootstrapCommand.java | 2 +- .../java/org/apache/hudi/table/TestCleaner.java | 12 ++--- .../bootstrap/index/HFileBootstrapIndex.java | 8 +-- .../hudi/common/model/BootstrapFileMapping.java | 16 +++--- .../table/view/AbstractTableFileSystemView.java | 4 +- .../common/util/queue/BoundedInMemoryQueue.java | 62 ++++++++-------------- .../util/queue/BoundedInMemoryQueueProducer.java | 2 +- .../hudi/common/bootstrap/TestBootstrapIndex.java | 2 +- .../java/org/apache/hudi/client/TestBootstrap.java | 4 +- 9 files changed, 46 insertions(+), 66 deletions(-) diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/BootstrapCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/BootstrapCommand.java index 015743d..2db49a2 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/BootstrapCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/BootstrapCommand.java @@ -179,7 +179,7 @@ public class BootstrapCommand implements CommandMarker { final List<Comparable[]> rows = new ArrayList<>(); for (BootstrapFileMapping mapping : mappingList) { rows.add(new Comparable[] {mapping.getPartitionPath(), mapping.getFileId(), - mapping.getBootstrapBasePath(), mapping.getBootstrapPartitionPath(), mapping.getBootstrapFileStatus().getPath().getUri()}); + mapping.getBootstrapBasePath(), mapping.getBootstrapPartitionPath(), mapping.getBoostrapFileStatus().getPath().getUri()}); } return rows; } diff --git a/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java b/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java index a4c7b5b..c1b5296 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java @@ -559,13 +559,13 @@ public class TestCleaner extends HoodieClientTestBase { : cleanStat.getSuccessDeleteBootstrapBaseFiles().size()), "Must clean at least 1 file"); if (enableBootstrapSourceClean) { HoodieFileStatus fstatus = - bootstrapMapping.get(p0).get(0).getBootstrapFileStatus(); + bootstrapMapping.get(p0).get(0).getBoostrapFileStatus(); // This ensures full path is recorded in metadata. assertTrue(cleanStat.getSuccessDeleteBootstrapBaseFiles().contains(fstatus.getPath().getUri()), "Successful delete files were " + cleanStat.getSuccessDeleteBootstrapBaseFiles() + " but did not contain " + fstatus.getPath().getUri()); assertFalse(Files.exists(Paths.get(bootstrapMapping.get( - p0).get(0).getBootstrapFileStatus().getPath().getUri()))); + p0).get(0).getBoostrapFileStatus().getPath().getUri()))); } cleanStat = getCleanStat(hoodieCleanStatsTwo, p1); String file2P0C1 = partitionAndFileId002.get(p0); @@ -579,13 +579,13 @@ public class TestCleaner extends HoodieClientTestBase { : cleanStat.getSuccessDeleteBootstrapBaseFiles().size()), "Must clean at least 1 file"); if (enableBootstrapSourceClean) { HoodieFileStatus fstatus = - bootstrapMapping.get(p1).get(0).getBootstrapFileStatus(); + bootstrapMapping.get(p1).get(0).getBoostrapFileStatus(); // This ensures full path is recorded in metadata. assertTrue(cleanStat.getSuccessDeleteBootstrapBaseFiles().contains(fstatus.getPath().getUri()), "Successful delete files were " + cleanStat.getSuccessDeleteBootstrapBaseFiles() + " but did not contain " + fstatus.getPath().getUri()); assertFalse(Files.exists(Paths.get(bootstrapMapping.get( - p1).get(0).getBootstrapFileStatus().getPath().getUri()))); + p1).get(0).getBoostrapFileStatus().getPath().getUri()))); } // make next commit, with 2 updates to existing files, and 1 insert @@ -928,7 +928,7 @@ public class TestCleaner extends HoodieClientTestBase { assertTrue(testTable.baseFileExists(p0, "00000000000004", file4P0C3)); if (enableBootstrapSourceClean) { assertFalse(Files.exists(Paths.get(bootstrapMapping.get( - p0).get(0).getBootstrapFileStatus().getPath().getUri()))); + p0).get(0).getBoostrapFileStatus().getPath().getUri()))); } // No cleaning on partially written file, with no commit. @@ -968,7 +968,7 @@ public class TestCleaner extends HoodieClientTestBase { for (Map.Entry<String, List<BootstrapFileMapping>> entry : bootstrapMapping.entrySet()) { new File(sourcePath.toString() + "/" + entry.getKey()).mkdirs(); - assertTrue(new File(entry.getValue().get(0).getBootstrapFileStatus().getPath().getUri()).createNewFile()); + assertTrue(new File(entry.getValue().get(0).getBoostrapFileStatus().getPath().getUri()).createNewFile()); } return bootstrapMapping; } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/HFileBootstrapIndex.java b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/HFileBootstrapIndex.java index d4a77b0..7dc0f69 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/HFileBootstrapIndex.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/HFileBootstrapIndex.java @@ -391,7 +391,7 @@ public class HFileBootstrapIndex extends BootstrapIndex { } /** - * Bootstrap Index Writer to build bootstrap index. + * Boostrap Index Writer to build bootstrap index. */ public static class HFileBootstrapIndexWriter extends BootstrapIndex.IndexWriter { @@ -443,7 +443,7 @@ public class HFileBootstrapIndex extends BootstrapIndex { bootstrapPartitionMetadata.setPartitionPath(partitionPath); bootstrapPartitionMetadata.setFileIdToBootstrapFile( bootstrapFileMappings.stream().map(m -> Pair.of(m.getFileId(), - m.getBootstrapFileStatus())).collect(Collectors.toMap(Pair::getKey, Pair::getValue))); + m.getBoostrapFileStatus())).collect(Collectors.toMap(Pair::getKey, Pair::getValue))); Option<byte[]> bytes = TimelineMetadataUtils.serializeAvroMetadata(bootstrapPartitionMetadata, HoodieBootstrapPartitionMetadata.class); if (bytes.isPresent()) { indexByPartitionWriter @@ -459,14 +459,14 @@ public class HFileBootstrapIndex extends BootstrapIndex { /** * Write next source file to hudi file-id. Entries are expected to be appended in hudi file-group id * order. - * @param mapping bootstrap source file mapping. + * @param mapping boostrap source file mapping. */ private void writeNextSourceFileMapping(BootstrapFileMapping mapping) { try { HoodieBootstrapFilePartitionInfo srcFilePartitionInfo = new HoodieBootstrapFilePartitionInfo(); srcFilePartitionInfo.setPartitionPath(mapping.getPartitionPath()); srcFilePartitionInfo.setBootstrapPartitionPath(mapping.getBootstrapPartitionPath()); - srcFilePartitionInfo.setBootstrapFileStatus(mapping.getBootstrapFileStatus()); + srcFilePartitionInfo.setBootstrapFileStatus(mapping.getBoostrapFileStatus()); KeyValue kv = new KeyValue(getFileGroupKey(mapping.getFileGroupId()).getBytes(), new byte[0], new byte[0], HConstants.LATEST_TIMESTAMP, KeyValue.Type.Put, TimelineMetadataUtils.serializeAvroMetadata(srcFilePartitionInfo, diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/BootstrapFileMapping.java b/hudi-common/src/main/java/org/apache/hudi/common/model/BootstrapFileMapping.java index 428d411..a9642c7 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/BootstrapFileMapping.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/BootstrapFileMapping.java @@ -29,17 +29,17 @@ public class BootstrapFileMapping implements Serializable, Comparable<BootstrapF private final String bootstrapBasePath; private final String bootstrapPartitionPath; - private final HoodieFileStatus bootstrapFileStatus; + private final HoodieFileStatus boostrapFileStatus; private final String partitionPath; private final String fileId; public BootstrapFileMapping(String bootstrapBasePath, String bootstrapPartitionPath, String partitionPath, - HoodieFileStatus bootstrapFileStatus, String fileId) { + HoodieFileStatus boostrapFileStatus, String fileId) { this.bootstrapBasePath = bootstrapBasePath; this.bootstrapPartitionPath = bootstrapPartitionPath; this.partitionPath = partitionPath; - this.bootstrapFileStatus = bootstrapFileStatus; + this.boostrapFileStatus = boostrapFileStatus; this.fileId = fileId; } @@ -48,7 +48,7 @@ public class BootstrapFileMapping implements Serializable, Comparable<BootstrapF return "BootstrapFileMapping{" + "bootstrapBasePath='" + bootstrapBasePath + '\'' + ", bootstrapPartitionPath='" + bootstrapPartitionPath + '\'' - + ", bootstrapFileStatus=" + bootstrapFileStatus + + ", boostrapFileStatus=" + boostrapFileStatus + ", partitionPath='" + partitionPath + '\'' + ", fileId='" + fileId + '\'' + '}'; @@ -66,13 +66,13 @@ public class BootstrapFileMapping implements Serializable, Comparable<BootstrapF return Objects.equals(bootstrapBasePath, mapping.bootstrapBasePath) && Objects.equals(bootstrapPartitionPath, mapping.bootstrapPartitionPath) && Objects.equals(partitionPath, mapping.partitionPath) - && Objects.equals(bootstrapFileStatus, mapping.bootstrapFileStatus) + && Objects.equals(boostrapFileStatus, mapping.boostrapFileStatus) && Objects.equals(fileId, mapping.fileId); } @Override public int hashCode() { - return Objects.hash(bootstrapBasePath, bootstrapPartitionPath, partitionPath, bootstrapFileStatus, fileId); + return Objects.hash(bootstrapBasePath, bootstrapPartitionPath, partitionPath, boostrapFileStatus, fileId); } public String getBootstrapBasePath() { @@ -87,8 +87,8 @@ public class BootstrapFileMapping implements Serializable, Comparable<BootstrapF return partitionPath; } - public HoodieFileStatus getBootstrapFileStatus() { - return bootstrapFileStatus; + public HoodieFileStatus getBoostrapFileStatus() { + return boostrapFileStatus; } public String getFileId() { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java index 9fa4472..77fcd16 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java @@ -126,12 +126,12 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV if (!isPartitionAvailableInStore(partition)) { if (bootstrapIndex.useIndex()) { try (BootstrapIndex.IndexReader reader = bootstrapIndex.createReader()) { - LOG.info("Bootstrap Index available for partition " + partition); + LOG.info("Boostrap Index available for partition " + partition); List<BootstrapFileMapping> sourceFileMappings = reader.getSourceFileMappingForPartition(partition); addBootstrapBaseFileMapping(sourceFileMappings.stream() .map(s -> new BootstrapBaseFileMapping(new HoodieFileGroupId(s.getPartitionPath(), - s.getFileId()), s.getBootstrapFileStatus()))); + s.getFileId()), s.getBoostrapFileStatus()))); } } storePartitionView(partition, value); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueue.java b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueue.java index 756a48a..2c2f919 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueue.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueue.java @@ -51,63 +51,43 @@ import java.util.function.Function; */ public class BoundedInMemoryQueue<I, O> implements Iterable<O> { - /** Interval used for polling records in the queue. **/ + // interval used for polling records in the queue. public static final int RECORD_POLL_INTERVAL_SEC = 1; - - /** Rate used for sampling records to determine avg record size in bytes. **/ + // rate used for sampling records to determine avg record size in bytes. public static final int RECORD_SAMPLING_RATE = 64; - - /** Maximum records that will be cached **/ + // maximum records that will be cached private static final int RECORD_CACHING_LIMIT = 128 * 1024; - private static final Logger LOG = LogManager.getLogger(BoundedInMemoryQueue.class); - - /** - * It indicates number of records to cache. We will be using sampled record's average size to - * determine how many records we should cache and will change (increase/decrease) permits accordingly. - */ + // It indicates number of records to cache. We will be using sampled record's average size to + // determine how many + // records we should cache and will change (increase/decrease) permits accordingly. public final Semaphore rateLimiter = new Semaphore(1); - - /** Used for sampling records with "RECORD_SAMPLING_RATE" frequency. **/ + // used for sampling records with "RECORD_SAMPLING_RATE" frequency. public final AtomicLong samplingRecordCounter = new AtomicLong(-1); - - /** Internal queue for records. **/ + // internal queue for records. private final LinkedBlockingQueue<Option<O>> queue = new LinkedBlockingQueue<>(); - - /** Maximum amount of memory to be used for queueing records. **/ + // maximum amount of memory to be used for queueing records. private final long memoryLimit; - - /** - * it holds the root cause of the exception in case either queueing records - * (consuming from inputIterator) fails or thread reading records from queue fails. - */ + // it holds the root cause of the exception in case either queueing records (consuming from + // inputIterator) fails or + // thread reading records from queue fails. private final AtomicReference<Exception> hasFailed = new AtomicReference<>(null); - - /** Used for indicating that all the records from queue are read successfully. **/ + // used for indicating that all the records from queue are read successfully. private final AtomicBoolean isReadDone = new AtomicBoolean(false); - - /** used for indicating that all records have been enqueued. **/ + // used for indicating that all records have been enqueued private final AtomicBoolean isWriteDone = new AtomicBoolean(false); - - /** Function to transform the input payload to the expected output payload. **/ + // Function to transform the input payload to the expected output payload private final Function<I, O> transformFunction; - - /** Payload Size Estimator. **/ + // Payload Size Estimator private final SizeEstimator<O> payloadSizeEstimator; - - /** Singleton (w.r.t this instance) Iterator for this queue. **/ + // Singleton (w.r.t this instance) Iterator for this queue private final QueueIterator iterator; - - /** - * indicates rate limit (number of records to cache). it is updated - * whenever there is a change in avg record size. - */ + // indicates rate limit (number of records to cache). it is updated whenever there is a change + // in avg record size. public int currentRateLimit = 1; - - /** Indicates avg record size in bytes. It is updated whenever a new record is sampled. **/ + // indicates avg record size in bytes. It is updated whenever a new record is sampled. public long avgRecordSizeInBytes = 0; - - /** Indicates number of samples collected so far. **/ + // indicates number of samples collected so far. private long numSamples = 0; /** diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueueProducer.java b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueueProducer.java index ecea9f2..31c3a3c 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueueProducer.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueueProducer.java @@ -19,7 +19,7 @@ package org.apache.hudi.common.util.queue; /** - * Producer for {@link BoundedInMemoryQueue}. Memory Bounded Buffer supports multiple producers single consumer pattern. + * Producer for BoundedInMemoryQueue. Memory Bounded Buffer supports multiple producers single consumer pattern. * * @param <I> Input type for buffer items produced */ diff --git a/hudi-common/src/test/java/org/apache/hudi/common/bootstrap/TestBootstrapIndex.java b/hudi-common/src/test/java/org/apache/hudi/common/bootstrap/TestBootstrapIndex.java index bbe75cf..ecfb59d 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/bootstrap/TestBootstrapIndex.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/bootstrap/TestBootstrapIndex.java @@ -156,7 +156,7 @@ public class TestBootstrapIndex extends HoodieCommonTestHarness { assertEquals(x.getFileId(), res.getFileId()); assertEquals(x.getPartitionPath(), res.getPartitionPath()); assertEquals(BOOTSTRAP_BASE_PATH, res.getBootstrapBasePath()); - assertEquals(x.getBootstrapFileStatus(), res.getBootstrapFileStatus()); + assertEquals(x.getBoostrapFileStatus(), res.getBoostrapFileStatus()); assertEquals(x.getBootstrapPartitionPath(), res.getBootstrapPartitionPath()); }); }); diff --git a/hudi-spark/src/test/java/org/apache/hudi/client/TestBootstrap.java b/hudi-spark/src/test/java/org/apache/hudi/client/TestBootstrap.java index 14f36d4..ad42443 100644 --- a/hudi-spark/src/test/java/org/apache/hudi/client/TestBootstrap.java +++ b/hudi-spark/src/test/java/org/apache/hudi/client/TestBootstrap.java @@ -309,7 +309,7 @@ public class TestBootstrap extends HoodieClientTestBase { } @Test - public void testFullBootstrapOnlyCOW() throws Exception { + public void testFullBoostrapOnlyCOW() throws Exception { testBootstrapCommon(true, false, EffectiveMode.FULL_BOOTSTRAP_MODE); } @@ -319,7 +319,7 @@ public class TestBootstrap extends HoodieClientTestBase { } @Test - public void testMetaAndFullBootstrapCOW() throws Exception { + public void testMetaAndFullBoostrapCOW() throws Exception { testBootstrapCommon(true, false, EffectiveMode.MIXED_BOOTSTRAP_MODE); }
