[
https://issues.apache.org/jira/browse/HUDI-3647?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Ethan Guo updated HUDI-3647:
----------------------------
Description:
HoodieMetadataTableValidator throws the following exceptions when the metadata
table is not fully initialized. These can be ignored and there could be a
fallback mechanism if metadata table is not ready for read.
{code:java}
org.apache.hudi.exception.HoodieException: Unable to do hoodie metadata table
validation in
file:/Users/ethan/Work/scripts/mt_rollout_testing/deploy_c_multi_writer/c5_mor_09mt_011mt/test_table
at
org.apache.hudi.utilities.HoodieMetadataTableValidator.run(HoodieMetadataTableValidator.java:364)
at
org.apache.hudi.utilities.HoodieMetadataTableValidator.main(HoodieMetadataTableValidator.java:345)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
at
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:955)
at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)
at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)
at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)
at
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1043)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1052)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Caused by: java.lang.IllegalArgumentException: Positive number of partitions
required
at
org.apache.spark.rdd.ParallelCollectionRDD$.slice(ParallelCollectionRDD.scala:118)
at
org.apache.spark.rdd.ParallelCollectionRDD.getPartitions(ParallelCollectionRDD.scala:96)
at org.apache.spark.rdd.RDD.$anonfun$partitions$2(RDD.scala:300)
at scala.Option.getOrElse(Option.scala:189)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:296)
at
org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:49)
at org.apache.spark.rdd.RDD.$anonfun$partitions$2(RDD.scala:300)
at scala.Option.getOrElse(Option.scala:189)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:296)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2279)
at org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030)
at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:414)
at org.apache.spark.rdd.RDD.collect(RDD.scala:1029)
at org.apache.spark.api.java.JavaRDDLike.collect(JavaRDDLike.scala:362)
at org.apache.spark.api.java.JavaRDDLike.collect$(JavaRDDLike.scala:361)
at
org.apache.spark.api.java.AbstractJavaRDDLike.collect(JavaRDDLike.scala:45)
at org.apache.hudi.data.HoodieJavaRDD.collectAsList(HoodieJavaRDD.java:157)
at
org.apache.hudi.utilities.HoodieMetadataTableValidator.doMetadataTableValidation(HoodieMetadataTableValidator.java:451)
at
org.apache.hudi.utilities.HoodieMetadataTableValidator.doHoodieMetadataTableValidationOnce(HoodieMetadataTableValidator.java:375)
at
org.apache.hudi.utilities.HoodieMetadataTableValidator.run(HoodieMetadataTableValidator.java:361)
... 13 more {code}
{code:java}
org.apache.hudi.exception.HoodieIOException: Could not load Hoodie properties
from
file:/Users/ethan/Work/scripts/mt_rollout_testing/deploy_b_single_writer_async_services/b2_ds_mor_010nomt_011mt_conf/test_table/.hoodie/metadata/.hoodie/hoodie.properties
at
org.apache.hudi.common.table.HoodieTableConfig.<init>(HoodieTableConfig.java:226)
at
org.apache.hudi.common.table.HoodieTableMetaClient.<init>(HoodieTableMetaClient.java:120)
at
org.apache.hudi.common.table.HoodieTableMetaClient.<init>(HoodieTableMetaClient.java:77)
at
org.apache.hudi.common.table.HoodieTableMetaClient$Builder.build(HoodieTableMetaClient.java:657)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.initIfNeeded(HoodieBackedTableMetadata.java:108)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.<init>(HoodieBackedTableMetadata.java:97)
at
org.apache.hudi.metadata.HoodieTableMetadata.create(HoodieTableMetadata.java:111)
at
org.apache.hudi.metadata.HoodieTableMetadata.create(HoodieTableMetadata.java:105)
at org.apache.hudi.common.fs.FSUtils.getAllPartitionPaths(FSUtils.java:296)
at
org.apache.hudi.utilities.HoodieMetadataTableValidator.validatePartitions(HoodieMetadataTableValidator.java:386)
at
org.apache.hudi.utilities.HoodieMetadataTableValidator.doMetadataTableValidation(HoodieMetadataTableValidator.java:349)
at
org.apache.hudi.utilities.HoodieMetadataTableValidator.doHoodieMetadataTableValidationOnce(HoodieMetadataTableValidator.java:324)
at
org.apache.hudi.utilities.HoodieMetadataTableValidator.run(HoodieMetadataTableValidator.java:310)
at
org.apache.hudi.utilities.HoodieMetadataTableValidator.main(HoodieMetadataTableValidator.java:294)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
at
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:955)
at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)
at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)
at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)
at
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1043)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1052)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Caused by: java.io.FileNotFoundException: File
file:/Users/ethan/Work/scripts/mt_rollout_testing/deploy_b_single_writer_async_services/b2_ds_mor_010nomt_011mt_conf/test_table/.hoodie/metadata/.hoodie/hoodie.properties.backup
does not exist
at
org.apache.hadoop.fs.RawLocalFileSystem.deprecatedGetFileStatus(RawLocalFileSystem.java:779)
at
org.apache.hadoop.fs.RawLocalFileSystem.getFileLinkStatusInternal(RawLocalFileSystem.java:1100)
at
org.apache.hadoop.fs.RawLocalFileSystem.getFileStatus(RawLocalFileSystem.java:769)
at
org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:462)
at
org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSInputChecker.<init>(ChecksumFileSystem.java:160)
at org.apache.hadoop.fs.ChecksumFileSystem.open(ChecksumFileSystem.java:372)
at org.apache.hadoop.fs.FileSystem.open(FileSystem.java:976)
at
org.apache.hudi.common.fs.HoodieWrapperFileSystem.open(HoodieWrapperFileSystem.java:460)
at
org.apache.hudi.common.table.HoodieTableConfig.fetchConfigs(HoodieTableConfig.java:287)
at
org.apache.hudi.common.table.HoodieTableConfig.<init>(HoodieTableConfig.java:216)
... 25 more {code}
{code:java}
org.apache.hudi.exception.HoodieMetadataException: Failed to retrieve files in
partition
file:/Users/ethan/Work/scripts/mt_rollout_testing/deploy_b_single_writer_async_services/b2_ds_mor_010nomt_011mt_conf/test_table/2022/1/24
from metadata at
org.apache.hudi.metadata.BaseTableMetadata.getAllFilesInPartition(BaseTableMetadata.java:137)
at
org.apache.hudi.metadata.HoodieMetadataFileSystemView.listPartition(HoodieMetadataFileSystemView.java:65)
at
org.apache.hudi.common.table.view.AbstractTableFileSystemView.lambda$ensurePartitionLoadedCorrectly$9(AbstractTableFileSystemView.java:304)
at
java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1660)
at
org.apache.hudi.common.table.view.AbstractTableFileSystemView.ensurePartitionLoadedCorrectly(AbstractTableFileSystemView.java:295)
at
org.apache.hudi.common.table.view.AbstractTableFileSystemView.getLatestFileSlices(AbstractTableFileSystemView.java:591)
at
org.apache.hudi.utilities.HoodieMetadataTableValidator$HoodieMetadataValidationContext.getSortedLatestFileSliceList(HoodieMetadataTableValidator.java:649)
at
org.apache.hudi.utilities.HoodieMetadataTableValidator.validateLatestFileSlices(HoodieMetadataTableValidator.java:488)
at
org.apache.hudi.utilities.HoodieMetadataTableValidator.validateFilesInPartition(HoodieMetadataTableValidator.java:418)
at
org.apache.hudi.utilities.HoodieMetadataTableValidator.lambda$doMetadataTableValidation$f2a6c13f$1(HoodieMetadataTableValidator.java:357)
at
org.apache.spark.api.java.JavaPairRDD$.$anonfun$toScalaFunction$1(JavaPairRDD.scala:1070)
at scala.collection.Iterator$$anon$10.next(Iterator.scala:461) at
scala.collection.Iterator.foreach(Iterator.scala:943) at
scala.collection.Iterator.foreach$(Iterator.scala:943) at
scala.collection.AbstractIterator.foreach(Iterator.scala:1431) at
scala.collection.generic.Growable.$plus$plus$eq(Growable.scala:62) at
scala.collection.generic.Growable.$plus$plus$eq$(Growable.scala:53) at
scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:105)
at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:49)
at scala.collection.TraversableOnce.to(TraversableOnce.scala:366) at
scala.collection.TraversableOnce.to$(TraversableOnce.scala:364) at
scala.collection.AbstractIterator.to(Iterator.scala:1431) at
scala.collection.TraversableOnce.toBuffer(TraversableOnce.scala:358) at
scala.collection.TraversableOnce.toBuffer$(TraversableOnce.scala:358) at
scala.collection.AbstractIterator.toBuffer(Iterator.scala:1431) at
scala.collection.TraversableOnce.toArray(TraversableOnce.scala:345) at
scala.collection.TraversableOnce.toArray$(TraversableOnce.scala:339) at
scala.collection.AbstractIterator.toArray(Iterator.scala:1431) at
org.apache.spark.rdd.RDD.$anonfun$collect$2(RDD.scala:1030) at
org.apache.spark.SparkContext.$anonfun$runJob$5(SparkContext.scala:2254) at
org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) at
org.apache.spark.scheduler.Task.run(Task.scala:131) at
org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:506)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1462)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:509) at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748) Caused by:
java.util.NoSuchElementException: No value present in Option at
org.apache.hudi.common.util.Option.get(Option.java:88) at
org.apache.hudi.metadata.HoodieTableMetadataUtil.getPartitionFileSlices(HoodieTableMetadataUtil.java:831)
at
org.apache.hudi.metadata.HoodieTableMetadataUtil.getPartitionLatestMergedFileSlices(HoodieTableMetadataUtil.java:775)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.getPartitionFileSliceToKeysMapping(HoodieBackedTableMetadata.java:262)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.getRecordsByKeys(HoodieBackedTableMetadata.java:135)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.getRecordByKey(HoodieBackedTableMetadata.java:128)
at
org.apache.hudi.metadata.BaseTableMetadata.fetchAllFilesInPartition(BaseTableMetadata.java:312)
at
org.apache.hudi.metadata.BaseTableMetadata.getAllFilesInPartition(BaseTableMetadata.java:135)
... 37 more
{code}
was:
HoodieMetadataTableValidator throws the following exceptions when the metadata
table is not fully initialized. These can be ignored and there could be a
fallback mechanism if metadata table is not ready for read.
{code:java}
org.apache.hudi.exception.HoodieIOException: Could not load Hoodie properties
from
file:/Users/ethan/Work/scripts/mt_rollout_testing/deploy_b_single_writer_async_services/b2_ds_mor_010nomt_011mt_conf/test_table/.hoodie/metadata/.hoodie/hoodie.properties
at
org.apache.hudi.common.table.HoodieTableConfig.<init>(HoodieTableConfig.java:226)
at
org.apache.hudi.common.table.HoodieTableMetaClient.<init>(HoodieTableMetaClient.java:120)
at
org.apache.hudi.common.table.HoodieTableMetaClient.<init>(HoodieTableMetaClient.java:77)
at
org.apache.hudi.common.table.HoodieTableMetaClient$Builder.build(HoodieTableMetaClient.java:657)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.initIfNeeded(HoodieBackedTableMetadata.java:108)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.<init>(HoodieBackedTableMetadata.java:97)
at
org.apache.hudi.metadata.HoodieTableMetadata.create(HoodieTableMetadata.java:111)
at
org.apache.hudi.metadata.HoodieTableMetadata.create(HoodieTableMetadata.java:105)
at org.apache.hudi.common.fs.FSUtils.getAllPartitionPaths(FSUtils.java:296)
at
org.apache.hudi.utilities.HoodieMetadataTableValidator.validatePartitions(HoodieMetadataTableValidator.java:386)
at
org.apache.hudi.utilities.HoodieMetadataTableValidator.doMetadataTableValidation(HoodieMetadataTableValidator.java:349)
at
org.apache.hudi.utilities.HoodieMetadataTableValidator.doHoodieMetadataTableValidationOnce(HoodieMetadataTableValidator.java:324)
at
org.apache.hudi.utilities.HoodieMetadataTableValidator.run(HoodieMetadataTableValidator.java:310)
at
org.apache.hudi.utilities.HoodieMetadataTableValidator.main(HoodieMetadataTableValidator.java:294)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
at
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:955)
at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)
at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)
at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)
at
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1043)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1052)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Caused by: java.io.FileNotFoundException: File
file:/Users/ethan/Work/scripts/mt_rollout_testing/deploy_b_single_writer_async_services/b2_ds_mor_010nomt_011mt_conf/test_table/.hoodie/metadata/.hoodie/hoodie.properties.backup
does not exist
at
org.apache.hadoop.fs.RawLocalFileSystem.deprecatedGetFileStatus(RawLocalFileSystem.java:779)
at
org.apache.hadoop.fs.RawLocalFileSystem.getFileLinkStatusInternal(RawLocalFileSystem.java:1100)
at
org.apache.hadoop.fs.RawLocalFileSystem.getFileStatus(RawLocalFileSystem.java:769)
at
org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:462)
at
org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSInputChecker.<init>(ChecksumFileSystem.java:160)
at org.apache.hadoop.fs.ChecksumFileSystem.open(ChecksumFileSystem.java:372)
at org.apache.hadoop.fs.FileSystem.open(FileSystem.java:976)
at
org.apache.hudi.common.fs.HoodieWrapperFileSystem.open(HoodieWrapperFileSystem.java:460)
at
org.apache.hudi.common.table.HoodieTableConfig.fetchConfigs(HoodieTableConfig.java:287)
at
org.apache.hudi.common.table.HoodieTableConfig.<init>(HoodieTableConfig.java:216)
... 25 more {code}
{code:java}
org.apache.hudi.exception.HoodieMetadataException: Failed to retrieve files in
partition
file:/Users/ethan/Work/scripts/mt_rollout_testing/deploy_b_single_writer_async_services/b2_ds_mor_010nomt_011mt_conf/test_table/2022/1/24
from metadata at
org.apache.hudi.metadata.BaseTableMetadata.getAllFilesInPartition(BaseTableMetadata.java:137)
at
org.apache.hudi.metadata.HoodieMetadataFileSystemView.listPartition(HoodieMetadataFileSystemView.java:65)
at
org.apache.hudi.common.table.view.AbstractTableFileSystemView.lambda$ensurePartitionLoadedCorrectly$9(AbstractTableFileSystemView.java:304)
at
java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1660)
at
org.apache.hudi.common.table.view.AbstractTableFileSystemView.ensurePartitionLoadedCorrectly(AbstractTableFileSystemView.java:295)
at
org.apache.hudi.common.table.view.AbstractTableFileSystemView.getLatestFileSlices(AbstractTableFileSystemView.java:591)
at
org.apache.hudi.utilities.HoodieMetadataTableValidator$HoodieMetadataValidationContext.getSortedLatestFileSliceList(HoodieMetadataTableValidator.java:649)
at
org.apache.hudi.utilities.HoodieMetadataTableValidator.validateLatestFileSlices(HoodieMetadataTableValidator.java:488)
at
org.apache.hudi.utilities.HoodieMetadataTableValidator.validateFilesInPartition(HoodieMetadataTableValidator.java:418)
at
org.apache.hudi.utilities.HoodieMetadataTableValidator.lambda$doMetadataTableValidation$f2a6c13f$1(HoodieMetadataTableValidator.java:357)
at
org.apache.spark.api.java.JavaPairRDD$.$anonfun$toScalaFunction$1(JavaPairRDD.scala:1070)
at scala.collection.Iterator$$anon$10.next(Iterator.scala:461) at
scala.collection.Iterator.foreach(Iterator.scala:943) at
scala.collection.Iterator.foreach$(Iterator.scala:943) at
scala.collection.AbstractIterator.foreach(Iterator.scala:1431) at
scala.collection.generic.Growable.$plus$plus$eq(Growable.scala:62) at
scala.collection.generic.Growable.$plus$plus$eq$(Growable.scala:53) at
scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:105)
at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:49)
at scala.collection.TraversableOnce.to(TraversableOnce.scala:366) at
scala.collection.TraversableOnce.to$(TraversableOnce.scala:364) at
scala.collection.AbstractIterator.to(Iterator.scala:1431) at
scala.collection.TraversableOnce.toBuffer(TraversableOnce.scala:358) at
scala.collection.TraversableOnce.toBuffer$(TraversableOnce.scala:358) at
scala.collection.AbstractIterator.toBuffer(Iterator.scala:1431) at
scala.collection.TraversableOnce.toArray(TraversableOnce.scala:345) at
scala.collection.TraversableOnce.toArray$(TraversableOnce.scala:339) at
scala.collection.AbstractIterator.toArray(Iterator.scala:1431) at
org.apache.spark.rdd.RDD.$anonfun$collect$2(RDD.scala:1030) at
org.apache.spark.SparkContext.$anonfun$runJob$5(SparkContext.scala:2254) at
org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) at
org.apache.spark.scheduler.Task.run(Task.scala:131) at
org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:506)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1462)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:509) at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748) Caused by:
java.util.NoSuchElementException: No value present in Option at
org.apache.hudi.common.util.Option.get(Option.java:88) at
org.apache.hudi.metadata.HoodieTableMetadataUtil.getPartitionFileSlices(HoodieTableMetadataUtil.java:831)
at
org.apache.hudi.metadata.HoodieTableMetadataUtil.getPartitionLatestMergedFileSlices(HoodieTableMetadataUtil.java:775)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.getPartitionFileSliceToKeysMapping(HoodieBackedTableMetadata.java:262)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.getRecordsByKeys(HoodieBackedTableMetadata.java:135)
at
org.apache.hudi.metadata.HoodieBackedTableMetadata.getRecordByKey(HoodieBackedTableMetadata.java:128)
at
org.apache.hudi.metadata.BaseTableMetadata.fetchAllFilesInPartition(BaseTableMetadata.java:312)
at
org.apache.hudi.metadata.BaseTableMetadata.getAllFilesInPartition(BaseTableMetadata.java:135)
... 37 more
{code}
> Ignore errors if metadata table has not been initialized fully
> --------------------------------------------------------------
>
> Key: HUDI-3647
> URL: https://issues.apache.org/jira/browse/HUDI-3647
> Project: Apache Hudi
> Issue Type: Bug
> Reporter: Ethan Guo
> Assignee: Yue Zhang
> Priority: Blocker
> Labels: pull-request-available
> Fix For: 0.11.0
>
>
> HoodieMetadataTableValidator throws the following exceptions when the
> metadata table is not fully initialized. These can be ignored and there
> could be a fallback mechanism if metadata table is not ready for read.
> {code:java}
> org.apache.hudi.exception.HoodieException: Unable to do hoodie metadata table
> validation in
> file:/Users/ethan/Work/scripts/mt_rollout_testing/deploy_c_multi_writer/c5_mor_09mt_011mt/test_table
> at
> org.apache.hudi.utilities.HoodieMetadataTableValidator.run(HoodieMetadataTableValidator.java:364)
> at
> org.apache.hudi.utilities.HoodieMetadataTableValidator.main(HoodieMetadataTableValidator.java:345)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
> org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
> at
> org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:955)
> at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)
> at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)
> at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)
> at
> org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1043)
> at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1052)
> at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
> Caused by: java.lang.IllegalArgumentException: Positive number of partitions
> required
> at
> org.apache.spark.rdd.ParallelCollectionRDD$.slice(ParallelCollectionRDD.scala:118)
> at
> org.apache.spark.rdd.ParallelCollectionRDD.getPartitions(ParallelCollectionRDD.scala:96)
> at org.apache.spark.rdd.RDD.$anonfun$partitions$2(RDD.scala:300)
> at scala.Option.getOrElse(Option.scala:189)
> at org.apache.spark.rdd.RDD.partitions(RDD.scala:296)
> at
> org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:49)
> at org.apache.spark.rdd.RDD.$anonfun$partitions$2(RDD.scala:300)
> at scala.Option.getOrElse(Option.scala:189)
> at org.apache.spark.rdd.RDD.partitions(RDD.scala:296)
> at org.apache.spark.SparkContext.runJob(SparkContext.scala:2279)
> at org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030)
> at
> org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
> at
> org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
> at org.apache.spark.rdd.RDD.withScope(RDD.scala:414)
> at org.apache.spark.rdd.RDD.collect(RDD.scala:1029)
> at org.apache.spark.api.java.JavaRDDLike.collect(JavaRDDLike.scala:362)
> at org.apache.spark.api.java.JavaRDDLike.collect$(JavaRDDLike.scala:361)
> at
> org.apache.spark.api.java.AbstractJavaRDDLike.collect(JavaRDDLike.scala:45)
> at
> org.apache.hudi.data.HoodieJavaRDD.collectAsList(HoodieJavaRDD.java:157)
> at
> org.apache.hudi.utilities.HoodieMetadataTableValidator.doMetadataTableValidation(HoodieMetadataTableValidator.java:451)
> at
> org.apache.hudi.utilities.HoodieMetadataTableValidator.doHoodieMetadataTableValidationOnce(HoodieMetadataTableValidator.java:375)
> at
> org.apache.hudi.utilities.HoodieMetadataTableValidator.run(HoodieMetadataTableValidator.java:361)
> ... 13 more {code}
>
> {code:java}
> org.apache.hudi.exception.HoodieIOException: Could not load Hoodie properties
> from
> file:/Users/ethan/Work/scripts/mt_rollout_testing/deploy_b_single_writer_async_services/b2_ds_mor_010nomt_011mt_conf/test_table/.hoodie/metadata/.hoodie/hoodie.properties
> at
> org.apache.hudi.common.table.HoodieTableConfig.<init>(HoodieTableConfig.java:226)
> at
> org.apache.hudi.common.table.HoodieTableMetaClient.<init>(HoodieTableMetaClient.java:120)
> at
> org.apache.hudi.common.table.HoodieTableMetaClient.<init>(HoodieTableMetaClient.java:77)
> at
> org.apache.hudi.common.table.HoodieTableMetaClient$Builder.build(HoodieTableMetaClient.java:657)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadata.initIfNeeded(HoodieBackedTableMetadata.java:108)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadata.<init>(HoodieBackedTableMetadata.java:97)
> at
> org.apache.hudi.metadata.HoodieTableMetadata.create(HoodieTableMetadata.java:111)
> at
> org.apache.hudi.metadata.HoodieTableMetadata.create(HoodieTableMetadata.java:105)
> at
> org.apache.hudi.common.fs.FSUtils.getAllPartitionPaths(FSUtils.java:296)
> at
> org.apache.hudi.utilities.HoodieMetadataTableValidator.validatePartitions(HoodieMetadataTableValidator.java:386)
> at
> org.apache.hudi.utilities.HoodieMetadataTableValidator.doMetadataTableValidation(HoodieMetadataTableValidator.java:349)
> at
> org.apache.hudi.utilities.HoodieMetadataTableValidator.doHoodieMetadataTableValidationOnce(HoodieMetadataTableValidator.java:324)
> at
> org.apache.hudi.utilities.HoodieMetadataTableValidator.run(HoodieMetadataTableValidator.java:310)
> at
> org.apache.hudi.utilities.HoodieMetadataTableValidator.main(HoodieMetadataTableValidator.java:294)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
> org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
> at
> org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:955)
> at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)
> at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)
> at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)
> at
> org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1043)
> at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1052)
> at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
> Caused by: java.io.FileNotFoundException: File
> file:/Users/ethan/Work/scripts/mt_rollout_testing/deploy_b_single_writer_async_services/b2_ds_mor_010nomt_011mt_conf/test_table/.hoodie/metadata/.hoodie/hoodie.properties.backup
> does not exist
> at
> org.apache.hadoop.fs.RawLocalFileSystem.deprecatedGetFileStatus(RawLocalFileSystem.java:779)
> at
> org.apache.hadoop.fs.RawLocalFileSystem.getFileLinkStatusInternal(RawLocalFileSystem.java:1100)
> at
> org.apache.hadoop.fs.RawLocalFileSystem.getFileStatus(RawLocalFileSystem.java:769)
> at
> org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:462)
> at
> org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSInputChecker.<init>(ChecksumFileSystem.java:160)
> at
> org.apache.hadoop.fs.ChecksumFileSystem.open(ChecksumFileSystem.java:372)
> at org.apache.hadoop.fs.FileSystem.open(FileSystem.java:976)
> at
> org.apache.hudi.common.fs.HoodieWrapperFileSystem.open(HoodieWrapperFileSystem.java:460)
> at
> org.apache.hudi.common.table.HoodieTableConfig.fetchConfigs(HoodieTableConfig.java:287)
> at
> org.apache.hudi.common.table.HoodieTableConfig.<init>(HoodieTableConfig.java:216)
> ... 25 more {code}
> {code:java}
> org.apache.hudi.exception.HoodieMetadataException: Failed to retrieve files
> in partition
> file:/Users/ethan/Work/scripts/mt_rollout_testing/deploy_b_single_writer_async_services/b2_ds_mor_010nomt_011mt_conf/test_table/2022/1/24
> from metadata at
> org.apache.hudi.metadata.BaseTableMetadata.getAllFilesInPartition(BaseTableMetadata.java:137)
> at
> org.apache.hudi.metadata.HoodieMetadataFileSystemView.listPartition(HoodieMetadataFileSystemView.java:65)
> at
> org.apache.hudi.common.table.view.AbstractTableFileSystemView.lambda$ensurePartitionLoadedCorrectly$9(AbstractTableFileSystemView.java:304)
> at
> java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1660)
> at
> org.apache.hudi.common.table.view.AbstractTableFileSystemView.ensurePartitionLoadedCorrectly(AbstractTableFileSystemView.java:295)
> at
> org.apache.hudi.common.table.view.AbstractTableFileSystemView.getLatestFileSlices(AbstractTableFileSystemView.java:591)
> at
> org.apache.hudi.utilities.HoodieMetadataTableValidator$HoodieMetadataValidationContext.getSortedLatestFileSliceList(HoodieMetadataTableValidator.java:649)
> at
> org.apache.hudi.utilities.HoodieMetadataTableValidator.validateLatestFileSlices(HoodieMetadataTableValidator.java:488)
> at
> org.apache.hudi.utilities.HoodieMetadataTableValidator.validateFilesInPartition(HoodieMetadataTableValidator.java:418)
> at
> org.apache.hudi.utilities.HoodieMetadataTableValidator.lambda$doMetadataTableValidation$f2a6c13f$1(HoodieMetadataTableValidator.java:357)
> at
> org.apache.spark.api.java.JavaPairRDD$.$anonfun$toScalaFunction$1(JavaPairRDD.scala:1070)
> at scala.collection.Iterator$$anon$10.next(Iterator.scala:461) at
> scala.collection.Iterator.foreach(Iterator.scala:943) at
> scala.collection.Iterator.foreach$(Iterator.scala:943) at
> scala.collection.AbstractIterator.foreach(Iterator.scala:1431) at
> scala.collection.generic.Growable.$plus$plus$eq(Growable.scala:62) at
> scala.collection.generic.Growable.$plus$plus$eq$(Growable.scala:53) at
> scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:105)
> at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:49)
> at scala.collection.TraversableOnce.to(TraversableOnce.scala:366) at
> scala.collection.TraversableOnce.to$(TraversableOnce.scala:364) at
> scala.collection.AbstractIterator.to(Iterator.scala:1431) at
> scala.collection.TraversableOnce.toBuffer(TraversableOnce.scala:358) at
> scala.collection.TraversableOnce.toBuffer$(TraversableOnce.scala:358) at
> scala.collection.AbstractIterator.toBuffer(Iterator.scala:1431) at
> scala.collection.TraversableOnce.toArray(TraversableOnce.scala:345) at
> scala.collection.TraversableOnce.toArray$(TraversableOnce.scala:339) at
> scala.collection.AbstractIterator.toArray(Iterator.scala:1431) at
> org.apache.spark.rdd.RDD.$anonfun$collect$2(RDD.scala:1030) at
> org.apache.spark.SparkContext.$anonfun$runJob$5(SparkContext.scala:2254)
> at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) at
> org.apache.spark.scheduler.Task.run(Task.scala:131) at
> org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:506)
> at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1462)
> at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:509)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748) Caused by:
> java.util.NoSuchElementException: No value present in Option at
> org.apache.hudi.common.util.Option.get(Option.java:88) at
> org.apache.hudi.metadata.HoodieTableMetadataUtil.getPartitionFileSlices(HoodieTableMetadataUtil.java:831)
> at
> org.apache.hudi.metadata.HoodieTableMetadataUtil.getPartitionLatestMergedFileSlices(HoodieTableMetadataUtil.java:775)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadata.getPartitionFileSliceToKeysMapping(HoodieBackedTableMetadata.java:262)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadata.getRecordsByKeys(HoodieBackedTableMetadata.java:135)
> at
> org.apache.hudi.metadata.HoodieBackedTableMetadata.getRecordByKey(HoodieBackedTableMetadata.java:128)
> at
> org.apache.hudi.metadata.BaseTableMetadata.fetchAllFilesInPartition(BaseTableMetadata.java:312)
> at
> org.apache.hudi.metadata.BaseTableMetadata.getAllFilesInPartition(BaseTableMetadata.java:135)
> ... 37 more
> {code}
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
