This is an automated email from the ASF dual-hosted git repository. yihua pushed a commit to branch branch-0.x in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 4680cb453334e545b3de553f9d98cd3bce829173 Author: harshal <[email protected]> AuthorDate: Thu Mar 7 13:52:40 2024 +0530 [HUDI-7197] Adding mis fixes related with table services testing (#10280) --- .../apache/hudi/table/action/clean/CleanActionExecutor.java | 2 +- .../hudi/table/action/commit/BaseCommitActionExecutor.java | 1 + .../main/java/org/apache/hudi/utilities/UtilHelpers.java | 1 + .../java/org/apache/hudi/utilities/TestHoodieIndexer.java | 13 +++++++++++-- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java index 0b5b3dfa42f..f84dac5fe6f 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java @@ -234,7 +234,7 @@ public class CleanActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I, K, throw new HoodieIOException("Failed to clean up after commit", e); } finally { if (!skipLocking) { - this.txnManager.endTransaction(Option.of(inflightInstant)); + this.txnManager.endTransaction(Option.ofNullable(inflightInstant)); } } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java index 4f4cc7d9bc7..8def1bf3e8a 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java @@ -259,6 +259,7 @@ public abstract class BaseCommitActionExecutor<T, I, K, O, R> writeMetadata.setPartitionToReplaceFileIds(getPartitionToReplacedFileIds(clusteringPlan, writeMetadata)); commitOnAutoCommit(writeMetadata); if (!writeMetadata.getCommitMetadata().isPresent()) { + LOG.info("Found empty commit metadata for clustering with instant time " + instantTime); HoodieCommitMetadata commitMetadata = CommitUtils.buildMetadata(writeMetadata.getWriteStats().get(), writeMetadata.getPartitionToReplaceFileIds(), extraMetadata, operationType, getSchemaToStoreInCommit(), getCommitActionType()); writeMetadata.setCommitMetadata(Option.of(commitMetadata)); diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java index d0781849755..35904fb2055 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java @@ -620,6 +620,7 @@ public class UtilHelpers { } while (ret != 0 && maxRetryCount-- > 0); } catch (Throwable t) { LOG.error(errorMessage, t); + throw new RuntimeException("Failed in retry", t); } return ret; } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieIndexer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieIndexer.java index e853d0ca366..9614dd28c1e 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieIndexer.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieIndexer.java @@ -39,6 +39,8 @@ import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieMetadataException; import org.apache.hudi.metadata.HoodieBackedTableMetadata; import org.apache.hudi.metadata.HoodieTableMetadataUtil; import org.apache.hudi.metadata.MetadataPartitionType; @@ -77,6 +79,7 @@ import static org.apache.hudi.utilities.UtilHelpers.SCHEDULE; import static org.apache.hudi.utilities.UtilHelpers.SCHEDULE_AND_EXECUTE; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; public class TestHoodieIndexer extends SparkClientFunctionalTestHarness implements SparkProvider { @@ -289,7 +292,10 @@ public class TestHoodieIndexer extends SparkClientFunctionalTestHarness implemen // start the indexer and validate files index is completely built out HoodieIndexer indexer = new HoodieIndexer(jsc(), config); // The catchup won't finish due to inflight delta commit, and this is expected - assertEquals(-1, indexer.start(0)); + Throwable cause = assertThrows(RuntimeException.class, () -> indexer.start(0)) + .getCause(); + assertTrue(cause instanceof HoodieMetadataException); + assertTrue(cause.getMessage().contains("Failed to index partition")); // Now, make sure that the inflight delta commit happened before the async indexer // is intact @@ -365,7 +371,10 @@ public class TestHoodieIndexer extends SparkClientFunctionalTestHarness implemen config.propsFilePath = propsPath; // start the indexer and validate index building fails HoodieIndexer indexer = new HoodieIndexer(jsc(), config); - assertEquals(-1, indexer.start(0)); + Throwable cause = assertThrows(RuntimeException.class, () -> indexer.start(0)) + .getCause(); + assertTrue(cause instanceof HoodieException); + assertTrue(cause.getMessage().contains("Metadata table is not yet initialized")); // validate table config metaClient = reload(metaClient);
