This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 6192cfb0e95 [MINOR] Reduce logging volume (#11505)
6192cfb0e95 is described below
commit 6192cfb0e95d23da462d01bb5b4c849dfbaa1f2a
Author: Tim Brown <[email protected]>
AuthorDate: Mon Jun 24 19:16:53 2024 -0500
[MINOR] Reduce logging volume (#11505)
---
.../apache/hudi/client/timeline/HoodieTimelineArchiver.java | 10 ++++------
.../plan/generators/BaseHoodieCompactionPlanGenerator.java | 4 +++-
.../common/table/log/BaseHoodieMergedLogRecordScanner.java | 9 +++------
.../apache/hudi/common/table/view/FileSystemViewManager.java | 10 +++++-----
.../src/main/java/org/apache/hudi/hive/HiveSyncTool.java | 2 +-
5 files changed, 16 insertions(+), 19 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/HoodieTimelineArchiver.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/HoodieTimelineArchiver.java
index 2f5ecb2816d..817c3f650d9 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/HoodieTimelineArchiver.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/HoodieTimelineArchiver.java
@@ -112,14 +112,14 @@ public class HoodieTimelineArchiver<T extends
HoodieAvroPayload, I, K, O> {
// Sort again because the cleaning and rollback instants could break the
sequence.
List<ActiveAction> instantsToArchive =
getInstantsToArchive().sorted().collect(Collectors.toList());
if (!instantsToArchive.isEmpty()) {
- LOG.info("Archiving instants " + instantsToArchive);
+ LOG.info("Archiving and deleting instants {}", instantsToArchive);
Consumer<Exception> exceptionHandler = e -> {
if (this.config.isFailOnTimelineArchivingEnabled()) {
throw new HoodieException(e);
}
};
this.timelineWriter.write(instantsToArchive, Option.of(action ->
deleteAnyLeftOverMarkers(context, action)), Option.of(exceptionHandler));
- LOG.info("Deleting archived instants " + instantsToArchive);
+ LOG.debug("Deleting archived instants");
deleteArchivedInstants(instantsToArchive, context);
// triggers compaction and cleaning only after archiving action
this.timelineWriter.compactAndClean(context);
@@ -221,7 +221,7 @@ public class HoodieTimelineArchiver<T extends
HoodieAvroPayload, I, K, O> {
LOG.info("Not archiving as there is no compaction yet on the
metadata table");
return Collections.emptyList();
} else {
- LOG.info("Limiting archiving of instants to latest compaction on
metadata table at " + latestCompactionTime.get());
+ LOG.info("Limiting archiving of instants to latest compaction on
metadata table at {}", latestCompactionTime.get());
earliestInstantToRetainCandidates.add(
completedCommitsTimeline.findInstantsModifiedAfterByCompletionTime(latestCompactionTime.get()).firstInstant());
}
@@ -324,8 +324,6 @@ public class HoodieTimelineArchiver<T extends
HoodieAvroPayload, I, K, O> {
}
private boolean deleteArchivedInstants(List<ActiveAction> activeActions,
HoodieEngineContext context) {
- LOG.info("Deleting instants " + activeActions);
-
List<HoodieInstant> pendingInstants = new ArrayList<>();
List<HoodieInstant> completedInstants = new ArrayList<>();
@@ -365,7 +363,7 @@ public class HoodieTimelineArchiver<T extends
HoodieAvroPayload, I, K, O> {
private void deleteAnyLeftOverMarkers(HoodieEngineContext context,
ActiveAction activeAction) {
WriteMarkers writeMarkers =
WriteMarkersFactory.get(config.getMarkersType(), table,
activeAction.getInstantTime());
if (writeMarkers.deleteMarkerDir(context,
config.getMarkersDeleteParallelism())) {
- LOG.info("Cleaned up left over marker directory for instant :" +
activeAction);
+ LOG.info("Cleaned up left over marker directory for instant: {}",
activeAction);
}
}
}
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/BaseHoodieCompactionPlanGenerator.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/BaseHoodieCompactionPlanGenerator.java
index f768004cbce..e5ac5af9f64 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/BaseHoodieCompactionPlanGenerator.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/plan/generators/BaseHoodieCompactionPlanGenerator.java
@@ -91,7 +91,9 @@ public abstract class BaseHoodieCompactionPlanGenerator<T
extends HoodieRecordPa
// In case no partitions could be picked, return no compaction plan
return null;
}
- LOG.info("Looking for files to compact in " + partitionPaths + "
partitions");
+ // avoid logging all partitions in table by default
+ LOG.info("Looking for files to compact in {} partitions",
partitionPaths.size());
+ LOG.debug("Partitions scanned for compaction: {}", partitionPaths);
engineContext.setJobStatus(this.getClass().getSimpleName(), "Looking for
files to compact: " + writeConfig.getTableName());
SyncableFileSystemView fileSystemView = (SyncableFileSystemView)
this.hoodieTable.getSliceView();
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/log/BaseHoodieMergedLogRecordScanner.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/log/BaseHoodieMergedLogRecordScanner.java
index 17a6284d20e..8b6a4a73e8c 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/log/BaseHoodieMergedLogRecordScanner.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/log/BaseHoodieMergedLogRecordScanner.java
@@ -182,12 +182,9 @@ public abstract class BaseHoodieMergedLogRecordScanner<K
extends Serializable> e
this.numMergedRecordsInLog = records.size();
if (LOG.isInfoEnabled()) {
- LOG.info("Number of log files scanned => {}", logFilePaths.size());
- LOG.info("MaxMemoryInBytes allowed for compaction => {}",
maxMemorySizeInBytes);
- LOG.info("Number of entries in MemoryBasedMap in ExternalSpillableMap =>
{}", records.getInMemoryMapNumEntries());
- LOG.info("Total size in bytes of MemoryBasedMap in ExternalSpillableMap
=> {}", records.getCurrentInMemoryMapSize());
- LOG.info("Number of entries in DiskBasedMap in ExternalSpillableMap =>
{}", records.getDiskBasedMapNumEntries());
- LOG.info("Size of file spilled to disk => {}",
records.getSizeOfFileOnDiskInBytes());
+ LOG.info("Scanned {} log files with stats: MaxMemoryInBytes => {},
MemoryBasedMap => {} entries, {} total bytes, DiskBasedMap => {} entries, {}
total bytes",
+ logFilePaths.size(), maxMemorySizeInBytes,
records.getInMemoryMapNumEntries(), records.getCurrentInMemoryMapSize(),
+ records.getDiskBasedMapNumEntries(),
records.getSizeOfFileOnDiskInBytes());
}
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
index 122081d4cfd..7b729dacac4 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
@@ -240,23 +240,23 @@ public class FileSystemViewManager {
LOG.info("Creating View Manager with storage type {}.",
config.getStorageType());
switch (config.getStorageType()) {
case EMBEDDED_KV_STORE:
- LOG.info("Creating embedded rocks-db based Table View");
+ LOG.debug("Creating embedded rocks-db based Table View");
return new FileSystemViewManager(context, config,
(metaClient, viewConf) ->
createRocksDBBasedFileSystemView(viewConf, metaClient));
case SPILLABLE_DISK:
- LOG.info("Creating Spillable Disk based Table View");
+ LOG.debug("Creating Spillable Disk based Table View");
return new FileSystemViewManager(context, config,
(metaClient, viewConf) ->
createSpillableMapBasedFileSystemView(viewConf, metaClient, commonConfig));
case MEMORY:
- LOG.info("Creating in-memory based Table View");
+ LOG.debug("Creating in-memory based Table View");
return new FileSystemViewManager(context, config,
(metaClient, viewConfig) ->
createInMemoryFileSystemView(viewConfig, metaClient, metadataCreator));
case REMOTE_ONLY:
- LOG.info("Creating remote only table view");
+ LOG.debug("Creating remote only table view");
return new FileSystemViewManager(context, config, (metaClient,
viewConfig) -> createRemoteFileSystemView(viewConfig,
metaClient));
case REMOTE_FIRST:
- LOG.info("Creating remote first table view");
+ LOG.debug("Creating remote first table view");
return new FileSystemViewManager(context, config, (metaClient,
viewConfig) -> {
RemoteHoodieTableFileSystemView remoteFileSystemView =
createRemoteFileSystemView(viewConfig, metaClient);
diff --git
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java
index 7bb20cd1d48..2c32e2a6ca1 100644
---
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java
+++
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java
@@ -384,7 +384,7 @@ public class HiveSyncTool extends HoodieSyncTool implements
AutoCloseable {
config.getSplitStrings(META_SYNC_PARTITION_FIELDS),
config.getBooleanOrDefault(HIVE_SUPPORT_TIMESTAMP_TYPE));
if (schemaDiff.isEmpty()) {
- LOG.info("No Schema difference for {}\nMessageType: {}", tableName,
schema);
+ LOG.info("No Schema difference for {}.", tableName);
} else {
LOG.info("Schema difference found for {}. Updated schema: {}",
tableName, schema);
syncClient.updateTableSchema(tableName, schema);
