[
https://issues.apache.org/jira/browse/HUDI-5014?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
sivabalan narayanan updated HUDI-5014:
--------------------------------------
Description:
sample logs. Search for keyword "Reverting to complete sync" below.
{code:java}
13345 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
Rocks DB : [db/column_family.cc:475] --------------- Options for column family
[hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]:13345
[main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks
DB : [db/db_impl.cc:1546] Created column family
[hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]
(ID 1)
13349 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
Rocks DB : [db/column_family.cc:475] --------------- Options for column family
[hudi_pending_compaction__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]:13350
[main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks
DB : [db/db_impl.cc:1546] Created column family
[hudi_pending_compaction__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]
(ID 2)
13355 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
Rocks DB : [db/column_family.cc:475] --------------- Options for column family
[hudi_pending_log_compaction__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]:13355
[main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks
DB : [db/db_impl.cc:1546] Created column family
[hudi_pending_log_compaction__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]
(ID 3)
13361 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
Rocks DB : [db/column_family.cc:475] --------------- Options for column family
[hudi_bootstrap_basefile__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]:13361
[main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks
DB : [db/db_impl.cc:1546] Created column family
[hudi_bootstrap_basefile__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]
(ID 4)
13368 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
Rocks DB : [db/column_family.cc:475] --------------- Options for column family
[hudi_partitions__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]:13368
[main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks
DB : [db/db_impl.cc:1546] Created column family
[hudi_partitions__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]
(ID 5)
13377 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
Rocks DB : [db/column_family.cc:475] --------------- Options for column family
[hudi_replaced_fg_var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]:13377
[main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks
DB : [db/db_impl.cc:1546] Created column family
[hudi_replaced_fg_var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]
(ID 6)
13387 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
Rocks DB : [db/column_family.cc:475] --------------- Options for column family
[hudi_pending_clustering_fg_var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]:13387
[main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks
DB : [db/db_impl.cc:1546] Created column family
[hudi_pending_clustering_fg_var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]
(ID 7)
13396 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Resetting replacedFileGroups to ROCKSDB based file-system view at
/var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411,
Total file-groups=0
13396 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix DELETE (query=part=) on
hudi_replaced_fg_var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747
13396 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Resetting replacedFileGroups to ROCKSDB based file-system view complete
13396 [main] INFO
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - Took 0 ms to
read 0 instants, 0 replaced file groups
13396 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Initializing pending compaction operations. Count=0
13398 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Initializing pending Log compaction operations. Count=30
13398 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Initializing external data file mapping. Count=0
13398 [main] INFO org.apache.hudi.common.util.ClusteringUtils [] - Found 0
files in pending clustering operations
13398 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Resetting file groups in pending clustering to ROCKSDB based file-system
view at
/var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411,
Total file-groups=0
13398 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix DELETE (query=part=) on
hudi_pending_clustering_fg_var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747
13398 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Resetting replacedFileGroups to ROCKSDB based file-system view complete
13398 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Created ROCKSDB based file-system view at
/var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411
13399 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=type=slice,part=2018/01/01,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=1. Serialization Time taken(micro)=834, num entries=10
13400 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=type=slice,part=2018/01/02,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=1. Serialization Time taken(micro)=812, num entries=10
13401 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=type=slice,part=2019/03/01,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=1. Serialization Time taken(micro)=845, num entries=10
13401 [main] INFO
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - Building
file system view for partition (2018/01/01)
13402 [main] DEBUG
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - #files found
in partition (2018/01/01) =30, Time taken =1
13404 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Resetting and adding new partition (2018/01/01) to ROCKSDB based
file-system view at
/var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411,
Total file-groups=10
13404 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix DELETE (query=type=slice,part=2018/01/01,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747
13404 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix DELETE (query=type=df,part=2018/01/01,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747
13404 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Finished adding new partition (2018/01/01) to ROCKSDB based file-system
view at
/var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411,
Total file-groups=10
13404 [main] INFO
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] -
addFilesToView: NumFiles=30, NumFileGroups=10, FileGroupsCreationTime=2,
StoreTimeTaken=0
13404 [main] DEBUG
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - Time to load
partition (2018/01/01) =3
13405 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=type=slice,part=2018/01/01,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=1. Serialization Time taken(micro)=857, num entries=10
13405 [main] INFO
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - Building
file system view for partition (2018/01/02)
13406 [main] DEBUG
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - #files found
in partition (2018/01/02) =30, Time taken =1
13408 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Resetting and adding new partition (2018/01/02) to ROCKSDB based
file-system view at
/var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411,
Total file-groups=10
13408 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix DELETE (query=type=slice,part=2018/01/02,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747
13408 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix DELETE (query=type=df,part=2018/01/02,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747
13408 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Finished adding new partition (2018/01/02) to ROCKSDB based file-system
view at
/var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411,
Total file-groups=10
13408 [main] INFO
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] -
addFilesToView: NumFiles=30, NumFileGroups=10, FileGroupsCreationTime=2,
StoreTimeTaken=0
13408 [main] DEBUG
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - Time to load
partition (2018/01/02) =3
13409 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=type=slice,part=2018/01/02,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=1. Serialization Time taken(micro)=821, num entries=10
13409 [main] INFO
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - Building
file system view for partition (2019/03/01)
13411 [main] DEBUG
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - #files found
in partition (2019/03/01) =30, Time taken =2
13412 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Resetting and adding new partition (2019/03/01) to ROCKSDB based
file-system view at
/var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411,
Total file-groups=10
13412 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix DELETE (query=type=slice,part=2019/03/01,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747
13412 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix DELETE (query=type=df,part=2019/03/01,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747
13412 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Finished adding new partition (2019/03/01) to ROCKSDB based file-system
view at
/var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411,
Total file-groups=10
13412 [main] INFO
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] -
addFilesToView: NumFiles=30, NumFileGroups=10, FileGroupsCreationTime=1,
StoreTimeTaken=0
13412 [main] DEBUG
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - Time to load
partition (2019/03/01) =3
13413 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=type=slice,part=2019/03/01,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=1. Serialization Time taken(micro)=858, num entries=10
13414 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=) on
hudi_pending_compaction__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=0. Serialization Time taken(micro)=0, num entries=0
13414 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=) on
hudi_pending_compaction__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=0. Serialization Time taken(micro)=0, num entries=0
13414 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=) on
hudi_pending_log_compaction__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=0. Serialization Time taken(micro)=82, num entries=30
13414 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=) on
hudi_pending_log_compaction__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=0. Serialization Time taken(micro)=62, num entries=30
13415 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=type=slice,part=2018/01/01,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=1. Serialization Time taken(micro)=848, num entries=10
13416 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=type=slice,part=2018/01/02,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=1. Serialization Time taken(micro)=840, num entries=10
13417 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=type=slice,part=2019/03/01,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=1. Serialization Time taken(micro)=810, num entries=10
13418 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=type=slice,part=2018/01/01,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=1. Serialization Time taken(micro)=810, num entries=10
13419 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=type=slice,part=2018/01/02,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=1. Serialization Time taken(micro)=809, num entries=10
13420 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=type=slice,part=2019/03/01,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=1. Serialization Time taken(micro)=808, num entries=10
13431 [main] INFO org.apache.hudi.common.util.FileIOUtils [] - Created a new
file in meta path:
/var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/.hoodie/.aux/15.compaction.requested
13455 [main] INFO org.apache.hudi.common.table.timeline.HoodieActiveTimeline
[] - Loaded instants upto : Option{val=[==>15__compaction__REQUESTED]}
13456 [main] INFO
org.apache.hudi.common.table.view.IncrementalTimelineSyncFileSystemView [] -
Doing incremental sync
13456 [main] INFO
org.apache.hudi.common.table.view.IncrementalTimelineSyncFileSystemView [] -
Timeline Diff Result is
:TimelineDiffResult{newlySeenInstants=[[==>14__logcompaction__REQUESTED],
[==>15__compaction__REQUESTED]], finishedCompactionInstants=[],
finishedOrRemovedLogCompactionInstants=[], canSyncIncrementally=true}
13456 [main] INFO
org.apache.hudi.common.table.view.IncrementalTimelineSyncFileSystemView [] -
Syncing pending log compaction instant ([==>14__logcompaction__REQUESTED])
13457 [main] ERROR
org.apache.hudi.common.table.view.IncrementalTimelineSyncFileSystemView [] -
Got exception trying to perform incremental sync. Reverting to complete sync
java.lang.IllegalArgumentException: Duplicate FileGroupId found in pending log
compaction operations. FgId :HoodieFileGroupId{partitionPath='2018/01/01',
fileId='fbec26a6-59bf-4b1b-8b35-bda65858186e'}
at
org.apache.hudi.common.util.ValidationUtils.checkArgument(ValidationUtils.java:40)
~[classes/:?]
at
org.apache.hudi.common.table.view.RocksDbBasedFileSystemView.lambda$null$8(RocksDbBasedFileSystemView.java:158)
~[classes/:?]
at
java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1382)
~[?:1.8.0_192]
at
java.util.stream.ReferencePipeline$Head.forEach(ReferencePipeline.java:580)
~[?:1.8.0_192]
at
org.apache.hudi.common.table.view.RocksDbBasedFileSystemView.lambda$addPendingLogCompactionOperations$9(RocksDbBasedFileSystemView.java:157)
~[classes/:?]
at
org.apache.hudi.common.util.collection.RocksDBDAO.writeBatch(RocksDBDAO.java:157)
~[classes/:?]
at
org.apache.hudi.common.table.view.RocksDbBasedFileSystemView.addPendingLogCompactionOperations(RocksDbBasedFileSystemView.java:156)
~[classes/:?]
at
org.apache.hudi.common.table.view.IncrementalTimelineSyncFileSystemView.addPendingLogCompactionInstant(IncrementalTimelineSyncFileSystemView.java:232)
~[classes/:?]
at
org.apache.hudi.common.table.view.IncrementalTimelineSyncFileSystemView.lambda$runIncrementalSync$3(IncrementalTimelineSyncFileSystemView.java:145)
~[classes/:?]
at java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:184)
~[?:1.8.0_192]
at
java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:175)
~[?:1.8.0_192]
at
java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1382)
~[?:1.8.0_192]
at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:481)
~[?:1.8.0_192]
at
java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471)
~[?:1.8.0_192]
at
java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:151)
~[?:1.8.0_192]
at
java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:174)
~[?:1.8.0_192]
at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
~[?:1.8.0_192]
at java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:418)
~[?:1.8.0_192]
at
org.apache.hudi.common.table.view.IncrementalTimelineSyncFileSystemView.runIncrementalSync(IncrementalTimelineSyncFileSystemView.java:133)
~[classes/:?]
at
org.apache.hudi.common.table.view.IncrementalTimelineSyncFileSystemView.runSync(IncrementalTimelineSyncFileSystemView.java:86)
~[classes/:?]
at
org.apache.hudi.common.table.view.AbstractTableFileSystemView.sync(AbstractTableFileSystemView.java:1260)
~[classes/:?]
at
org.apache.hudi.common.table.view.TestIncrementalFSViewSync.scheduleCompaction(TestIncrementalFSViewSync.java:690)
~[test-classes/:?]
at
org.apache.hudi.common.table.view.TestIncrementalFSViewSync.testAsyncMajorAndMinorCompaction(TestIncrementalFSViewSync.java:183)
~[test-classes/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
~[?:1.8.0_192]
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
~[?:1.8.0_192]
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
~[?:1.8.0_192]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_192]
at
org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
~[junit-platform-commons-1.7.2.jar:1.7.2]
at
org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestableMethod(TimeoutExtension.java:140)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestMethod(TimeoutExtension.java:84)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeTestMethod$6(TestMethodTestDescriptor.java:210)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeTestMethod(TestMethodTestDescriptor.java:206)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:131)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:65)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at java.util.ArrayList.forEach(ArrayList.java:1257) ~[?:1.8.0_192]
at
org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at java.util.ArrayList.forEach(ArrayList.java:1257) ~[?:1.8.0_192]
at
org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:108)
~[junit-platform-launcher-1.7.2.jar:1.7.2]
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:88)
~[junit-platform-launcher-1.7.2.jar:1.7.2]
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:54)
~[junit-platform-launcher-1.7.2.jar:1.7.2]
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:67)
~[junit-platform-launcher-1.7.2.jar:1.7.2]
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:52)
~[junit-platform-launcher-1.7.2.jar:1.7.2]
at
org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:96)
~[junit-platform-launcher-1.7.2.jar:1.7.2]
at
org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:75)
~[junit-platform-launcher-1.7.2.jar:1.7.2]
at
com.intellij.junit5.JUnit5IdeaTestRunner.startRunnerWithArgs(JUnit5IdeaTestRunner.java:71)
~[junit5-rt.jar:?]
at
com.intellij.rt.junit.IdeaTestRunner$Repeater$1.execute(IdeaTestRunner.java:38)
~[junit-rt.jar:?]
at
com.intellij.rt.execution.junit.TestsRepeater.repeat(TestsRepeater.java:11)
~[idea_rt.jar:?]
at
com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:35)
~[junit-rt.jar:?]
at
com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:235)
~[junit-rt.jar:?]
at com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:54)
~[junit-rt.jar:?]
13462 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Deleting all rocksdb data associated with table filesystem view
13462 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
Rocks DB : [db/db_impl.cc:365] Shutdown: canceling all background work
13462 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
Rocks DB : [db/db_impl.cc:521] Shutdown complete
13463 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
DELETING RocksDB persisted at
/var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/8960336680386206726/_var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747/41f9a94a-6c76-4d28-995a-ddf83cc84fab
13464 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - No
column family found. Loading default
13464 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
Rocks DB : [db/db_impl_open.cc:230] Creating manifest 1 13466 [main] INFO
org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks DB :
[db/version_set.cc:3406] Recovering from manifest file: MANIFEST-00000113466
[main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks
DB : [db/column_family.cc:475] --------------- Options for column family
[default]:13466 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO
[] - From Rocks DB : [db/version_set.cc:3610] Recovered from manifest
file:/var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/8960336680386206726/_var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747/41f9a94a-6c76-4d28-995a-ddf83cc84fab/MANIFEST-000001
succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is
0, log_number is 0,prev_log_number is 0,max_column_family is
0,min_log_number_to_keep is 013466 [main] INFO
org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks DB :
[db/version_set.cc:3618] Column family [default] (ID 0), log number is 013470
[main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks
DB : [db/db_impl_open.cc:1287] DB pointer 0x7fec34e5d600
13470 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
Rocks DB : [db/version_set.cc:2936] Creating manifest 613471 [main] INFO
org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks DB :
[db/column_family.cc:475] --------------- Options for column family
[hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]:
{code}
was:
sample logs
{code:java}
13345 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
Rocks DB : [db/column_family.cc:475] --------------- Options for column family
[hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]:13345
[main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks
DB : [db/db_impl.cc:1546] Created column family
[hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]
(ID 1)
13349 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
Rocks DB : [db/column_family.cc:475] --------------- Options for column family
[hudi_pending_compaction__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]:13350
[main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks
DB : [db/db_impl.cc:1546] Created column family
[hudi_pending_compaction__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]
(ID 2)
13355 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
Rocks DB : [db/column_family.cc:475] --------------- Options for column family
[hudi_pending_log_compaction__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]:13355
[main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks
DB : [db/db_impl.cc:1546] Created column family
[hudi_pending_log_compaction__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]
(ID 3)
13361 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
Rocks DB : [db/column_family.cc:475] --------------- Options for column family
[hudi_bootstrap_basefile__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]:13361
[main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks
DB : [db/db_impl.cc:1546] Created column family
[hudi_bootstrap_basefile__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]
(ID 4)
13368 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
Rocks DB : [db/column_family.cc:475] --------------- Options for column family
[hudi_partitions__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]:13368
[main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks
DB : [db/db_impl.cc:1546] Created column family
[hudi_partitions__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]
(ID 5)
13377 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
Rocks DB : [db/column_family.cc:475] --------------- Options for column family
[hudi_replaced_fg_var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]:13377
[main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks
DB : [db/db_impl.cc:1546] Created column family
[hudi_replaced_fg_var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]
(ID 6)
13387 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
Rocks DB : [db/column_family.cc:475] --------------- Options for column family
[hudi_pending_clustering_fg_var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]:13387
[main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks
DB : [db/db_impl.cc:1546] Created column family
[hudi_pending_clustering_fg_var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]
(ID 7)
13396 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Resetting replacedFileGroups to ROCKSDB based file-system view at
/var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411,
Total file-groups=0
13396 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix DELETE (query=part=) on
hudi_replaced_fg_var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747
13396 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Resetting replacedFileGroups to ROCKSDB based file-system view complete
13396 [main] INFO
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - Took 0 ms to
read 0 instants, 0 replaced file groups
13396 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Initializing pending compaction operations. Count=0
13398 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Initializing pending Log compaction operations. Count=30
13398 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Initializing external data file mapping. Count=0
13398 [main] INFO org.apache.hudi.common.util.ClusteringUtils [] - Found 0
files in pending clustering operations
13398 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Resetting file groups in pending clustering to ROCKSDB based file-system
view at
/var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411,
Total file-groups=0
13398 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix DELETE (query=part=) on
hudi_pending_clustering_fg_var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747
13398 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Resetting replacedFileGroups to ROCKSDB based file-system view complete
13398 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Created ROCKSDB based file-system view at
/var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411
13399 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=type=slice,part=2018/01/01,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=1. Serialization Time taken(micro)=834, num entries=10
13400 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=type=slice,part=2018/01/02,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=1. Serialization Time taken(micro)=812, num entries=10
13401 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=type=slice,part=2019/03/01,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=1. Serialization Time taken(micro)=845, num entries=10
13401 [main] INFO
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - Building
file system view for partition (2018/01/01)
13402 [main] DEBUG
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - #files found
in partition (2018/01/01) =30, Time taken =1
13404 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Resetting and adding new partition (2018/01/01) to ROCKSDB based
file-system view at
/var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411,
Total file-groups=10
13404 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix DELETE (query=type=slice,part=2018/01/01,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747
13404 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix DELETE (query=type=df,part=2018/01/01,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747
13404 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Finished adding new partition (2018/01/01) to ROCKSDB based file-system
view at
/var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411,
Total file-groups=10
13404 [main] INFO
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] -
addFilesToView: NumFiles=30, NumFileGroups=10, FileGroupsCreationTime=2,
StoreTimeTaken=0
13404 [main] DEBUG
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - Time to load
partition (2018/01/01) =3
13405 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=type=slice,part=2018/01/01,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=1. Serialization Time taken(micro)=857, num entries=10
13405 [main] INFO
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - Building
file system view for partition (2018/01/02)
13406 [main] DEBUG
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - #files found
in partition (2018/01/02) =30, Time taken =1
13408 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Resetting and adding new partition (2018/01/02) to ROCKSDB based
file-system view at
/var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411,
Total file-groups=10
13408 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix DELETE (query=type=slice,part=2018/01/02,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747
13408 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix DELETE (query=type=df,part=2018/01/02,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747
13408 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Finished adding new partition (2018/01/02) to ROCKSDB based file-system
view at
/var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411,
Total file-groups=10
13408 [main] INFO
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] -
addFilesToView: NumFiles=30, NumFileGroups=10, FileGroupsCreationTime=2,
StoreTimeTaken=0
13408 [main] DEBUG
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - Time to load
partition (2018/01/02) =3
13409 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=type=slice,part=2018/01/02,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=1. Serialization Time taken(micro)=821, num entries=10
13409 [main] INFO
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - Building
file system view for partition (2019/03/01)
13411 [main] DEBUG
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - #files found
in partition (2019/03/01) =30, Time taken =2
13412 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Resetting and adding new partition (2019/03/01) to ROCKSDB based
file-system view at
/var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411,
Total file-groups=10
13412 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix DELETE (query=type=slice,part=2019/03/01,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747
13412 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix DELETE (query=type=df,part=2019/03/01,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747
13412 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Finished adding new partition (2019/03/01) to ROCKSDB based file-system
view at
/var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411,
Total file-groups=10
13412 [main] INFO
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] -
addFilesToView: NumFiles=30, NumFileGroups=10, FileGroupsCreationTime=1,
StoreTimeTaken=0
13412 [main] DEBUG
org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - Time to load
partition (2019/03/01) =3
13413 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=type=slice,part=2019/03/01,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=1. Serialization Time taken(micro)=858, num entries=10
13414 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=) on
hudi_pending_compaction__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=0. Serialization Time taken(micro)=0, num entries=0
13414 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=) on
hudi_pending_compaction__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=0. Serialization Time taken(micro)=0, num entries=0
13414 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=) on
hudi_pending_log_compaction__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=0. Serialization Time taken(micro)=82, num entries=30
13414 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=) on
hudi_pending_log_compaction__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=0. Serialization Time taken(micro)=62, num entries=30
13415 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=type=slice,part=2018/01/01,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=1. Serialization Time taken(micro)=848, num entries=10
13416 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=type=slice,part=2018/01/02,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=1. Serialization Time taken(micro)=840, num entries=10
13417 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=type=slice,part=2019/03/01,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=1. Serialization Time taken(micro)=810, num entries=10
13418 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=type=slice,part=2018/01/01,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=1. Serialization Time taken(micro)=810, num entries=10
13419 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=type=slice,part=2018/01/02,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=1. Serialization Time taken(micro)=809, num entries=10
13420 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
Prefix Search for (query=type=slice,part=2019/03/01,id=) on
hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
Total Time Taken (msec)=1. Serialization Time taken(micro)=808, num entries=10
13431 [main] INFO org.apache.hudi.common.util.FileIOUtils [] - Created a new
file in meta path:
/var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/.hoodie/.aux/15.compaction.requested
13455 [main] INFO org.apache.hudi.common.table.timeline.HoodieActiveTimeline
[] - Loaded instants upto : Option{val=[==>15__compaction__REQUESTED]}
13456 [main] INFO
org.apache.hudi.common.table.view.IncrementalTimelineSyncFileSystemView [] -
Doing incremental sync
13456 [main] INFO
org.apache.hudi.common.table.view.IncrementalTimelineSyncFileSystemView [] -
Timeline Diff Result is
:TimelineDiffResult{newlySeenInstants=[[==>14__logcompaction__REQUESTED],
[==>15__compaction__REQUESTED]], finishedCompactionInstants=[],
finishedOrRemovedLogCompactionInstants=[], canSyncIncrementally=true}
13456 [main] INFO
org.apache.hudi.common.table.view.IncrementalTimelineSyncFileSystemView [] -
Syncing pending log compaction instant ([==>14__logcompaction__REQUESTED])
13457 [main] ERROR
org.apache.hudi.common.table.view.IncrementalTimelineSyncFileSystemView [] -
Got exception trying to perform incremental sync. Reverting to complete sync
java.lang.IllegalArgumentException: Duplicate FileGroupId found in pending log
compaction operations. FgId :HoodieFileGroupId{partitionPath='2018/01/01',
fileId='fbec26a6-59bf-4b1b-8b35-bda65858186e'}
at
org.apache.hudi.common.util.ValidationUtils.checkArgument(ValidationUtils.java:40)
~[classes/:?]
at
org.apache.hudi.common.table.view.RocksDbBasedFileSystemView.lambda$null$8(RocksDbBasedFileSystemView.java:158)
~[classes/:?]
at
java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1382)
~[?:1.8.0_192]
at
java.util.stream.ReferencePipeline$Head.forEach(ReferencePipeline.java:580)
~[?:1.8.0_192]
at
org.apache.hudi.common.table.view.RocksDbBasedFileSystemView.lambda$addPendingLogCompactionOperations$9(RocksDbBasedFileSystemView.java:157)
~[classes/:?]
at
org.apache.hudi.common.util.collection.RocksDBDAO.writeBatch(RocksDBDAO.java:157)
~[classes/:?]
at
org.apache.hudi.common.table.view.RocksDbBasedFileSystemView.addPendingLogCompactionOperations(RocksDbBasedFileSystemView.java:156)
~[classes/:?]
at
org.apache.hudi.common.table.view.IncrementalTimelineSyncFileSystemView.addPendingLogCompactionInstant(IncrementalTimelineSyncFileSystemView.java:232)
~[classes/:?]
at
org.apache.hudi.common.table.view.IncrementalTimelineSyncFileSystemView.lambda$runIncrementalSync$3(IncrementalTimelineSyncFileSystemView.java:145)
~[classes/:?]
at java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:184)
~[?:1.8.0_192]
at
java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:175)
~[?:1.8.0_192]
at
java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1382)
~[?:1.8.0_192]
at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:481)
~[?:1.8.0_192]
at
java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471)
~[?:1.8.0_192]
at
java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:151)
~[?:1.8.0_192]
at
java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:174)
~[?:1.8.0_192]
at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
~[?:1.8.0_192]
at java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:418)
~[?:1.8.0_192]
at
org.apache.hudi.common.table.view.IncrementalTimelineSyncFileSystemView.runIncrementalSync(IncrementalTimelineSyncFileSystemView.java:133)
~[classes/:?]
at
org.apache.hudi.common.table.view.IncrementalTimelineSyncFileSystemView.runSync(IncrementalTimelineSyncFileSystemView.java:86)
~[classes/:?]
at
org.apache.hudi.common.table.view.AbstractTableFileSystemView.sync(AbstractTableFileSystemView.java:1260)
~[classes/:?]
at
org.apache.hudi.common.table.view.TestIncrementalFSViewSync.scheduleCompaction(TestIncrementalFSViewSync.java:690)
~[test-classes/:?]
at
org.apache.hudi.common.table.view.TestIncrementalFSViewSync.testAsyncMajorAndMinorCompaction(TestIncrementalFSViewSync.java:183)
~[test-classes/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
~[?:1.8.0_192]
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
~[?:1.8.0_192]
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
~[?:1.8.0_192]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_192]
at
org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
~[junit-platform-commons-1.7.2.jar:1.7.2]
at
org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestableMethod(TimeoutExtension.java:140)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestMethod(TimeoutExtension.java:84)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeTestMethod$6(TestMethodTestDescriptor.java:210)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeTestMethod(TestMethodTestDescriptor.java:206)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:131)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:65)
~[junit-jupiter-engine-5.7.2.jar:5.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at java.util.ArrayList.forEach(ArrayList.java:1257) ~[?:1.8.0_192]
at
org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at java.util.ArrayList.forEach(ArrayList.java:1257) ~[?:1.8.0_192]
at
org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51)
~[junit-platform-engine-1.7.2.jar:1.7.2]
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:108)
~[junit-platform-launcher-1.7.2.jar:1.7.2]
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:88)
~[junit-platform-launcher-1.7.2.jar:1.7.2]
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:54)
~[junit-platform-launcher-1.7.2.jar:1.7.2]
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:67)
~[junit-platform-launcher-1.7.2.jar:1.7.2]
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:52)
~[junit-platform-launcher-1.7.2.jar:1.7.2]
at
org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:96)
~[junit-platform-launcher-1.7.2.jar:1.7.2]
at
org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:75)
~[junit-platform-launcher-1.7.2.jar:1.7.2]
at
com.intellij.junit5.JUnit5IdeaTestRunner.startRunnerWithArgs(JUnit5IdeaTestRunner.java:71)
~[junit5-rt.jar:?]
at
com.intellij.rt.junit.IdeaTestRunner$Repeater$1.execute(IdeaTestRunner.java:38)
~[junit-rt.jar:?]
at
com.intellij.rt.execution.junit.TestsRepeater.repeat(TestsRepeater.java:11)
~[idea_rt.jar:?]
at
com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:35)
~[junit-rt.jar:?]
at
com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:235)
~[junit-rt.jar:?]
at com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:54)
~[junit-rt.jar:?]
13462 [main] INFO org.apache.hudi.common.table.view.RocksDbBasedFileSystemView
[] - Deleting all rocksdb data associated with table filesystem view
13462 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
Rocks DB : [db/db_impl.cc:365] Shutdown: canceling all background work
13462 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
Rocks DB : [db/db_impl.cc:521] Shutdown complete
13463 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
DELETING RocksDB persisted at
/var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/8960336680386206726/_var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747/41f9a94a-6c76-4d28-995a-ddf83cc84fab
13464 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - No
column family found. Loading default
13464 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
Rocks DB : [db/db_impl_open.cc:230] Creating manifest 1 13466 [main] INFO
org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks DB :
[db/version_set.cc:3406] Recovering from manifest file: MANIFEST-00000113466
[main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks
DB : [db/column_family.cc:475] --------------- Options for column family
[default]:13466 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO
[] - From Rocks DB : [db/version_set.cc:3610] Recovered from manifest
file:/var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/8960336680386206726/_var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747/41f9a94a-6c76-4d28-995a-ddf83cc84fab/MANIFEST-000001
succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is
0, log_number is 0,prev_log_number is 0,max_column_family is
0,min_log_number_to_keep is 013466 [main] INFO
org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks DB :
[db/version_set.cc:3618] Column family [default] (ID 0), log number is 013470
[main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks
DB : [db/db_impl_open.cc:1287] DB pointer 0x7fec34e5d600
13470 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
Rocks DB : [db/version_set.cc:2936] Creating manifest 613471 [main] INFO
org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks DB :
[db/column_family.cc:475] --------------- Options for column family
[hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]:
{code}
> TestRocksDBBasedIncrementalFSViewSync.testAsyncMajorAndMinorCompaction is
> having some errors but still the test succeeds
> ------------------------------------------------------------------------------------------------------------------------
>
> Key: HUDI-5014
> URL: https://issues.apache.org/jira/browse/HUDI-5014
> Project: Apache Hudi
> Issue Type: Test
> Components: tests-ci
> Reporter: sivabalan narayanan
> Assignee: Surya Prasanna Yalla
> Priority: Critical
> Fix For: 0.13.0
>
>
> sample logs. Search for keyword "Reverting to complete sync" below.
>
> {code:java}
> 13345 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> From Rocks DB : [db/column_family.cc:475] --------------- Options for column
> family
> [hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]:13345
> [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
> Rocks DB : [db/db_impl.cc:1546] Created column family
> [hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]
> (ID 1)
> 13349 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> From Rocks DB : [db/column_family.cc:475] --------------- Options for column
> family
> [hudi_pending_compaction__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]:13350
> [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
> Rocks DB : [db/db_impl.cc:1546] Created column family
> [hudi_pending_compaction__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]
> (ID 2)
> 13355 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> From Rocks DB : [db/column_family.cc:475] --------------- Options for column
> family
> [hudi_pending_log_compaction__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]:13355
> [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
> Rocks DB : [db/db_impl.cc:1546] Created column family
> [hudi_pending_log_compaction__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]
> (ID 3)
> 13361 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> From Rocks DB : [db/column_family.cc:475] --------------- Options for column
> family
> [hudi_bootstrap_basefile__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]:13361
> [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
> Rocks DB : [db/db_impl.cc:1546] Created column family
> [hudi_bootstrap_basefile__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]
> (ID 4)
> 13368 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> From Rocks DB : [db/column_family.cc:475] --------------- Options for column
> family
> [hudi_partitions__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]:13368
> [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
> Rocks DB : [db/db_impl.cc:1546] Created column family
> [hudi_partitions__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]
> (ID 5)
> 13377 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> From Rocks DB : [db/column_family.cc:475] --------------- Options for column
> family
> [hudi_replaced_fg_var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]:13377
> [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
> Rocks DB : [db/db_impl.cc:1546] Created column family
> [hudi_replaced_fg_var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]
> (ID 6)
> 13387 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> From Rocks DB : [db/column_family.cc:475] --------------- Options for column
> family
> [hudi_pending_clustering_fg_var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]:13387
> [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
> Rocks DB : [db/db_impl.cc:1546] Created column family
> [hudi_pending_clustering_fg_var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]
> (ID 7)
> 13396 [main] INFO
> org.apache.hudi.common.table.view.RocksDbBasedFileSystemView [] - Resetting
> replacedFileGroups to ROCKSDB based file-system view at
> /var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411,
> Total file-groups=0
> 13396 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> Prefix DELETE (query=part=) on
> hudi_replaced_fg_var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747
> 13396 [main] INFO
> org.apache.hudi.common.table.view.RocksDbBasedFileSystemView [] - Resetting
> replacedFileGroups to ROCKSDB based file-system view complete
> 13396 [main] INFO
> org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - Took 0 ms
> to read 0 instants, 0 replaced file groups
> 13396 [main] INFO
> org.apache.hudi.common.table.view.RocksDbBasedFileSystemView [] -
> Initializing pending compaction operations. Count=0
> 13398 [main] INFO
> org.apache.hudi.common.table.view.RocksDbBasedFileSystemView [] -
> Initializing pending Log compaction operations. Count=30
> 13398 [main] INFO
> org.apache.hudi.common.table.view.RocksDbBasedFileSystemView [] -
> Initializing external data file mapping. Count=0
> 13398 [main] INFO org.apache.hudi.common.util.ClusteringUtils [] - Found 0
> files in pending clustering operations
> 13398 [main] INFO
> org.apache.hudi.common.table.view.RocksDbBasedFileSystemView [] - Resetting
> file groups in pending clustering to ROCKSDB based file-system view at
> /var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411,
> Total file-groups=0
> 13398 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> Prefix DELETE (query=part=) on
> hudi_pending_clustering_fg_var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747
> 13398 [main] INFO
> org.apache.hudi.common.table.view.RocksDbBasedFileSystemView [] - Resetting
> replacedFileGroups to ROCKSDB based file-system view complete
> 13398 [main] INFO
> org.apache.hudi.common.table.view.RocksDbBasedFileSystemView [] - Created
> ROCKSDB based file-system view at
> /var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411
> 13399 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> Prefix Search for (query=type=slice,part=2018/01/01,id=) on
> hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
> Total Time Taken (msec)=1. Serialization Time taken(micro)=834, num
> entries=10
> 13400 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> Prefix Search for (query=type=slice,part=2018/01/02,id=) on
> hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
> Total Time Taken (msec)=1. Serialization Time taken(micro)=812, num
> entries=10
> 13401 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> Prefix Search for (query=type=slice,part=2019/03/01,id=) on
> hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
> Total Time Taken (msec)=1. Serialization Time taken(micro)=845, num
> entries=10
> 13401 [main] INFO
> org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - Building
> file system view for partition (2018/01/01)
> 13402 [main] DEBUG
> org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - #files
> found in partition (2018/01/01) =30, Time taken =1
> 13404 [main] INFO
> org.apache.hudi.common.table.view.RocksDbBasedFileSystemView [] - Resetting
> and adding new partition (2018/01/01) to ROCKSDB based file-system view at
> /var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411,
> Total file-groups=10
> 13404 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> Prefix DELETE (query=type=slice,part=2018/01/01,id=) on
> hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747
> 13404 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> Prefix DELETE (query=type=df,part=2018/01/01,id=) on
> hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747
> 13404 [main] INFO
> org.apache.hudi.common.table.view.RocksDbBasedFileSystemView [] - Finished
> adding new partition (2018/01/01) to ROCKSDB based file-system view at
> /var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411,
> Total file-groups=10
> 13404 [main] INFO
> org.apache.hudi.common.table.view.AbstractTableFileSystemView [] -
> addFilesToView: NumFiles=30, NumFileGroups=10, FileGroupsCreationTime=2,
> StoreTimeTaken=0
> 13404 [main] DEBUG
> org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - Time to
> load partition (2018/01/01) =3
> 13405 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> Prefix Search for (query=type=slice,part=2018/01/01,id=) on
> hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
> Total Time Taken (msec)=1. Serialization Time taken(micro)=857, num
> entries=10
> 13405 [main] INFO
> org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - Building
> file system view for partition (2018/01/02)
> 13406 [main] DEBUG
> org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - #files
> found in partition (2018/01/02) =30, Time taken =1
> 13408 [main] INFO
> org.apache.hudi.common.table.view.RocksDbBasedFileSystemView [] - Resetting
> and adding new partition (2018/01/02) to ROCKSDB based file-system view at
> /var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411,
> Total file-groups=10
> 13408 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> Prefix DELETE (query=type=slice,part=2018/01/02,id=) on
> hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747
> 13408 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> Prefix DELETE (query=type=df,part=2018/01/02,id=) on
> hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747
> 13408 [main] INFO
> org.apache.hudi.common.table.view.RocksDbBasedFileSystemView [] - Finished
> adding new partition (2018/01/02) to ROCKSDB based file-system view at
> /var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411,
> Total file-groups=10
> 13408 [main] INFO
> org.apache.hudi.common.table.view.AbstractTableFileSystemView [] -
> addFilesToView: NumFiles=30, NumFileGroups=10, FileGroupsCreationTime=2,
> StoreTimeTaken=0
> 13408 [main] DEBUG
> org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - Time to
> load partition (2018/01/02) =3
> 13409 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> Prefix Search for (query=type=slice,part=2018/01/02,id=) on
> hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
> Total Time Taken (msec)=1. Serialization Time taken(micro)=821, num
> entries=10
> 13409 [main] INFO
> org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - Building
> file system view for partition (2019/03/01)
> 13411 [main] DEBUG
> org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - #files
> found in partition (2019/03/01) =30, Time taken =2
> 13412 [main] INFO
> org.apache.hudi.common.table.view.RocksDbBasedFileSystemView [] - Resetting
> and adding new partition (2019/03/01) to ROCKSDB based file-system view at
> /var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411,
> Total file-groups=10
> 13412 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> Prefix DELETE (query=type=slice,part=2019/03/01,id=) on
> hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747
> 13412 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> Prefix DELETE (query=type=df,part=2019/03/01,id=) on
> hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747
> 13412 [main] INFO
> org.apache.hudi.common.table.view.RocksDbBasedFileSystemView [] - Finished
> adding new partition (2019/03/01) to ROCKSDB based file-system view at
> /var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/7036287458630409411,
> Total file-groups=10
> 13412 [main] INFO
> org.apache.hudi.common.table.view.AbstractTableFileSystemView [] -
> addFilesToView: NumFiles=30, NumFileGroups=10, FileGroupsCreationTime=1,
> StoreTimeTaken=0
> 13412 [main] DEBUG
> org.apache.hudi.common.table.view.AbstractTableFileSystemView [] - Time to
> load partition (2019/03/01) =3
> 13413 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> Prefix Search for (query=type=slice,part=2019/03/01,id=) on
> hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
> Total Time Taken (msec)=1. Serialization Time taken(micro)=858, num
> entries=10
> 13414 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> Prefix Search for (query=) on
> hudi_pending_compaction__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
> Total Time Taken (msec)=0. Serialization Time taken(micro)=0, num entries=0
> 13414 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> Prefix Search for (query=) on
> hudi_pending_compaction__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
> Total Time Taken (msec)=0. Serialization Time taken(micro)=0, num entries=0
> 13414 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> Prefix Search for (query=) on
> hudi_pending_log_compaction__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
> Total Time Taken (msec)=0. Serialization Time taken(micro)=82, num entries=30
> 13414 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> Prefix Search for (query=) on
> hudi_pending_log_compaction__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
> Total Time Taken (msec)=0. Serialization Time taken(micro)=62, num entries=30
> 13415 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> Prefix Search for (query=type=slice,part=2018/01/01,id=) on
> hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
> Total Time Taken (msec)=1. Serialization Time taken(micro)=848, num
> entries=10
> 13416 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> Prefix Search for (query=type=slice,part=2018/01/02,id=) on
> hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
> Total Time Taken (msec)=1. Serialization Time taken(micro)=840, num
> entries=10
> 13417 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> Prefix Search for (query=type=slice,part=2019/03/01,id=) on
> hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
> Total Time Taken (msec)=1. Serialization Time taken(micro)=810, num
> entries=10
> 13418 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> Prefix Search for (query=type=slice,part=2018/01/01,id=) on
> hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
> Total Time Taken (msec)=1. Serialization Time taken(micro)=810, num
> entries=10
> 13419 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> Prefix Search for (query=type=slice,part=2018/01/02,id=) on
> hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
> Total Time Taken (msec)=1. Serialization Time taken(micro)=809, num
> entries=10
> 13420 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> Prefix Search for (query=type=slice,part=2019/03/01,id=) on
> hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747.
> Total Time Taken (msec)=1. Serialization Time taken(micro)=808, num
> entries=10
> 13431 [main] INFO org.apache.hudi.common.util.FileIOUtils [] - Created a new
> file in meta path:
> /var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/.hoodie/.aux/15.compaction.requested
> 13455 [main] INFO org.apache.hudi.common.table.timeline.HoodieActiveTimeline
> [] - Loaded instants upto : Option{val=[==>15__compaction__REQUESTED]}
> 13456 [main] INFO
> org.apache.hudi.common.table.view.IncrementalTimelineSyncFileSystemView [] -
> Doing incremental sync
> 13456 [main] INFO
> org.apache.hudi.common.table.view.IncrementalTimelineSyncFileSystemView [] -
> Timeline Diff Result is
> :TimelineDiffResult{newlySeenInstants=[[==>14__logcompaction__REQUESTED],
> [==>15__compaction__REQUESTED]], finishedCompactionInstants=[],
> finishedOrRemovedLogCompactionInstants=[], canSyncIncrementally=true}
> 13456 [main] INFO
> org.apache.hudi.common.table.view.IncrementalTimelineSyncFileSystemView [] -
> Syncing pending log compaction instant ([==>14__logcompaction__REQUESTED])
> 13457 [main] ERROR
> org.apache.hudi.common.table.view.IncrementalTimelineSyncFileSystemView [] -
> Got exception trying to perform incremental sync. Reverting to complete sync
> java.lang.IllegalArgumentException: Duplicate FileGroupId found in pending
> log compaction operations. FgId
> :HoodieFileGroupId{partitionPath='2018/01/01',
> fileId='fbec26a6-59bf-4b1b-8b35-bda65858186e'}
> at
> org.apache.hudi.common.util.ValidationUtils.checkArgument(ValidationUtils.java:40)
> ~[classes/:?]
> at
> org.apache.hudi.common.table.view.RocksDbBasedFileSystemView.lambda$null$8(RocksDbBasedFileSystemView.java:158)
> ~[classes/:?]
> at
> java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1382)
> ~[?:1.8.0_192]
> at
> java.util.stream.ReferencePipeline$Head.forEach(ReferencePipeline.java:580)
> ~[?:1.8.0_192]
> at
> org.apache.hudi.common.table.view.RocksDbBasedFileSystemView.lambda$addPendingLogCompactionOperations$9(RocksDbBasedFileSystemView.java:157)
> ~[classes/:?]
> at
> org.apache.hudi.common.util.collection.RocksDBDAO.writeBatch(RocksDBDAO.java:157)
> ~[classes/:?]
> at
> org.apache.hudi.common.table.view.RocksDbBasedFileSystemView.addPendingLogCompactionOperations(RocksDbBasedFileSystemView.java:156)
> ~[classes/:?]
> at
> org.apache.hudi.common.table.view.IncrementalTimelineSyncFileSystemView.addPendingLogCompactionInstant(IncrementalTimelineSyncFileSystemView.java:232)
> ~[classes/:?]
> at
> org.apache.hudi.common.table.view.IncrementalTimelineSyncFileSystemView.lambda$runIncrementalSync$3(IncrementalTimelineSyncFileSystemView.java:145)
> ~[classes/:?]
> at
> java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:184)
> ~[?:1.8.0_192]
> at
> java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:175)
> ~[?:1.8.0_192]
> at
> java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1382)
> ~[?:1.8.0_192]
> at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:481)
> ~[?:1.8.0_192]
> at
> java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471)
> ~[?:1.8.0_192]
> at
> java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:151)
> ~[?:1.8.0_192]
> at
> java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:174)
> ~[?:1.8.0_192]
> at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
> ~[?:1.8.0_192]
> at java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:418)
> ~[?:1.8.0_192]
> at
> org.apache.hudi.common.table.view.IncrementalTimelineSyncFileSystemView.runIncrementalSync(IncrementalTimelineSyncFileSystemView.java:133)
> ~[classes/:?]
> at
> org.apache.hudi.common.table.view.IncrementalTimelineSyncFileSystemView.runSync(IncrementalTimelineSyncFileSystemView.java:86)
> ~[classes/:?]
> at
> org.apache.hudi.common.table.view.AbstractTableFileSystemView.sync(AbstractTableFileSystemView.java:1260)
> ~[classes/:?]
> at
> org.apache.hudi.common.table.view.TestIncrementalFSViewSync.scheduleCompaction(TestIncrementalFSViewSync.java:690)
> ~[test-classes/:?]
> at
> org.apache.hudi.common.table.view.TestIncrementalFSViewSync.testAsyncMajorAndMinorCompaction(TestIncrementalFSViewSync.java:183)
> ~[test-classes/:?]
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> ~[?:1.8.0_192]
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> ~[?:1.8.0_192]
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> ~[?:1.8.0_192]
> at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_192]
> at
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
> ~[junit-platform-commons-1.7.2.jar:1.7.2]
> at
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
> ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
> at
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
> ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
> at
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
> ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
> at
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestableMethod(TimeoutExtension.java:140)
> ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
> at
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestMethod(TimeoutExtension.java:84)
> ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
> at
> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
> ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
> at
> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
> ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
> at
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
> ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
> at
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
> ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
> at
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
> ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
> at
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
> ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
> at
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
> ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
> at
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
> ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
> at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeTestMethod$6(TestMethodTestDescriptor.java:210)
> ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
> at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeTestMethod(TestMethodTestDescriptor.java:206)
> ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
> at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:131)
> ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
> at
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:65)
> ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at java.util.ArrayList.forEach(ArrayList.java:1257) ~[?:1.8.0_192]
> at
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at java.util.ArrayList.forEach(ArrayList.java:1257) ~[?:1.8.0_192]
> at
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51)
> ~[junit-platform-engine-1.7.2.jar:1.7.2]
> at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:108)
> ~[junit-platform-launcher-1.7.2.jar:1.7.2]
> at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:88)
> ~[junit-platform-launcher-1.7.2.jar:1.7.2]
> at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:54)
> ~[junit-platform-launcher-1.7.2.jar:1.7.2]
> at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:67)
> ~[junit-platform-launcher-1.7.2.jar:1.7.2]
> at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:52)
> ~[junit-platform-launcher-1.7.2.jar:1.7.2]
> at
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:96)
> ~[junit-platform-launcher-1.7.2.jar:1.7.2]
> at
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:75)
> ~[junit-platform-launcher-1.7.2.jar:1.7.2]
> at
> com.intellij.junit5.JUnit5IdeaTestRunner.startRunnerWithArgs(JUnit5IdeaTestRunner.java:71)
> ~[junit5-rt.jar:?]
> at
> com.intellij.rt.junit.IdeaTestRunner$Repeater$1.execute(IdeaTestRunner.java:38)
> ~[junit-rt.jar:?]
> at
> com.intellij.rt.execution.junit.TestsRepeater.repeat(TestsRepeater.java:11)
> ~[idea_rt.jar:?]
> at
> com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:35)
> ~[junit-rt.jar:?]
> at
> com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:235)
> ~[junit-rt.jar:?]
> at com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:54)
> ~[junit-rt.jar:?]
> 13462 [main] INFO
> org.apache.hudi.common.table.view.RocksDbBasedFileSystemView [] - Deleting
> all rocksdb data associated with table filesystem view
> 13462 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> From Rocks DB : [db/db_impl.cc:365] Shutdown: canceling all background work
> 13462 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> From Rocks DB : [db/db_impl.cc:521] Shutdown complete
> 13463 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> DELETING RocksDB persisted at
> /var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/8960336680386206726/_var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747/41f9a94a-6c76-4d28-995a-ddf83cc84fab
> 13464 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - No
> column family found. Loading default
> 13464 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> From Rocks DB : [db/db_impl_open.cc:230] Creating manifest 1 13466 [main]
> INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks DB :
> [db/version_set.cc:3406] Recovering from manifest file: MANIFEST-00000113466
> [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
> Rocks DB : [db/column_family.cc:475] --------------- Options for column
> family [default]:13466 [main] INFO
> org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks DB :
> [db/version_set.cc:3610] Recovered from manifest
> file:/var/folders/ym/8yjkm3n90kq8tk4gfmvk7y140000gn/T/junit2749486101886638747/8960336680386206726/_var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747/41f9a94a-6c76-4d28-995a-ddf83cc84fab/MANIFEST-000001
> succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is
> 0, log_number is 0,prev_log_number is 0,max_column_family is
> 0,min_log_number_to_keep is 013466 [main] INFO
> org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks DB :
> [db/version_set.cc:3618] Column family [default] (ID 0), log number is 013470
> [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] - From
> Rocks DB : [db/db_impl_open.cc:1287] DB pointer 0x7fec34e5d600
> 13470 [main] INFO org.apache.hudi.common.util.collection.RocksDBDAO [] -
> From Rocks DB : [db/version_set.cc:2936] Creating manifest 613471 [main] INFO
> org.apache.hudi.common.util.collection.RocksDBDAO [] - From Rocks DB :
> [db/column_family.cc:475] --------------- Options for column family
> [hudi_view__var_folders_ym_8yjkm3n90kq8tk4gfmvk7y140000gn_T_junit2749486101886638747]:
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
