yihua commented on code in PR #12647:
URL: https://github.com/apache/hudi/pull/12647#discussion_r1960714944
##########
hudi-common/src/main/java/org/apache/hudi/BaseHoodieTableFileIndex.java:
##########
@@ -433,6 +444,21 @@ private void doRefresh() {
// Reset it to null to trigger re-loading of all partition path
this.cachedAllPartitionPaths = null;
// Reset to force reload file slices inside partitions
+ try {
+ if (ConfigUtils.getBooleanWithAltKeys(configProperties,
HOODIE_FILE_INDEX_USE_SPILLABLE_MAP)) {
+ long spillableMemory =
ConfigUtils.getLongWithAltKeys(configProperties,
HOODIE_FILE_INDEX_SPILLABLE_MEMORY);
+ String spillablePath =
ConfigUtils.getStringWithAltKeys(configProperties,
HoodieMemoryConfig.SPILLABLE_MAP_BASE_PATH, true);
+ DiskMapType diskMapType =
ConfigUtils.getRawValueWithAltKeys(configProperties, SPILLABLE_DISK_MAP_TYPE,
true);
+ boolean isBitCaskCompressionEnabled =
ConfigUtils.getBooleanWithAltKeys(configProperties,
DISK_MAP_BITCASK_COMPRESSION_ENABLED);
+ this.cachedAllInputFileSlices = new ExternalSpillableMap<>(
+ spillableMemory, spillablePath, new DefaultSizeEstimator<>(), new
DefaultSizeEstimator<>(), diskMapType,
+ new HoodieFileSliceSerializer(), isBitCaskCompressionEnabled);
+ } else {
+ this.cachedAllInputFileSlices = new HashMap<>();
+ }
+ } catch (IOException e) {
+ throw new HoodieIOException("Failed to refresh file index", e);
+ }
this.cachedAllInputFileSlices = new HashMap<>();
Review Comment:
L462 should be removed?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
