This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-20768-ab0c92675f91302d80e4667d766e783e1989ae10 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 665b36eccf80b63f8d391e3b67fafe040e1b33c8 Author: voonhous <[email protected]> AuthorDate: Fri Nov 20 02:22:50 2026 +0000 refactor(utilities): use Lombok @Slf4j for loggers (#20768) Replace hand-declared static final SLF4J loggers in hudi-utilities with Lombok @Slf4j: add the annotation, drop the field and the org.slf4j imports, and rename LOG to log. Migrated (8 files): HudiHiveSyncJob, S3EventsSource, WriteErrorReporter, SparkStreamerValidatorUtils, SparkWriteErrorValidator (main) and TestHoodieSnapshotExporter, HoodieOfflineJobTestBase, KafkaTestUtils (test). Topic preserved: HoodieOfflineJobTestBase logged under its superclass UtilitiesTestBase, so it uses @Slf4j(topic = "org.apache.hudi.utilities.testutils.UtilitiesTestBase"). No category changes. No cross-module touches. The package-private TestHoodieSnapshotExporter.LOG had no references outside its class. Skipped (left hand-declared): - CloudObjectsSelector: public static volatile `log`, reassigned by TestCloudObjectsSelector to inject a capturing logger. - DFSPathSelector: protected static volatile `log` (volatile field, inherited by DatePartitionPathSelector and DFSTestSuitePathSelector). --- .../java/org/apache/hudi/utilities/HudiHiveSyncJob.java | 16 +++++++--------- .../apache/hudi/utilities/sources/S3EventsSource.java | 8 +++----- .../hudi/utilities/streamer/WriteErrorReporter.java | 12 +++++------- .../streamer/validator/SparkStreamerValidatorUtils.java | 16 +++++++--------- .../streamer/validator/SparkWriteErrorValidator.java | 12 +++++------- .../utilities/functional/TestHoodieSnapshotExporter.java | 7 +++---- .../utilities/offlinejob/HoodieOfflineJobTestBase.java | 11 +++++------ .../apache/hudi/utilities/testutils/KafkaTestUtils.java | 7 +++---- 8 files changed, 38 insertions(+), 51 deletions(-) diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HudiHiveSyncJob.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HudiHiveSyncJob.java index 59d92658f4f8..4f7e731df02e 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HudiHiveSyncJob.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HudiHiveSyncJob.java @@ -26,11 +26,10 @@ import org.apache.hudi.hive.HiveSyncTool; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.spark.api.java.JavaSparkContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.Serializable; @@ -59,10 +58,9 @@ import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_BASE_PATH; * --hoodie-conf hoodie.datasource.hive_sync.table=my_table * </pre> */ +@Slf4j public class HudiHiveSyncJob { - private static final Logger LOG = LoggerFactory.getLogger(HudiHiveSyncJob.class); - private final Config cfg; private final Configuration hadoopConf; private final TypedProperties props; @@ -76,7 +74,7 @@ public class HudiHiveSyncJob { public static void main(String[] args) throws IOException { final Config cfg = new Config(); new JCommander(cfg, null, args); - LOG.info("Cfg received: {}", cfg); + log.info("Cfg received: {}", cfg); JavaSparkContext jsc; if (StringUtils.isNullOrEmpty(cfg.sparkMaster)) { jsc = UtilHelpers.buildSparkContext("HudiHiveSyncJob", "local[2]", true); @@ -91,24 +89,24 @@ public class HudiHiveSyncJob { } public void run() throws IOException { - LOG.info("Starting hive sync for {}", cfg.basePath); + log.info("Starting hive sync for {}", cfg.basePath); HoodieTimer timer = HoodieTimer.start(); HiveSyncTool syncTool = null; try { props.put(META_SYNC_BASE_PATH.key(), cfg.basePath); props.put(META_SYNC_BASE_FILE_FORMAT.key(), cfg.baseFileFormat); - LOG.info("HiveSyncConfig props used to sync data {}", props); + log.info("HiveSyncConfig props used to sync data {}", props); syncTool = new HiveSyncTool(props, new HiveConf(hadoopConf, HiveConf.class)); syncTool.syncHoodieTable(); } catch (Exception e) { - LOG.error("Exception in running hive-sync", e); + log.error("Exception in running hive-sync", e); throw new HoodieException("Hive sync failed", e); } finally { if (syncTool != null) { syncTool.close(); } - LOG.info("Hive-sync duration in ms {}", timer.endTimer()); + log.info("Hive-sync duration in ms {}", timer.endTimer()); } } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/S3EventsSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/S3EventsSource.java index b6b66d3fb418..92f426a1900a 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/S3EventsSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/S3EventsSource.java @@ -29,14 +29,13 @@ import org.apache.hudi.utilities.schema.SchemaProvider; import org.apache.hudi.utilities.sources.helpers.CloudObjectsSelector; import org.apache.hudi.utilities.sources.helpers.S3EventsMetaSelector; +import lombok.extern.slf4j.Slf4j; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.types.StructType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.sqs.SqsClient; import java.io.Closeable; @@ -52,10 +51,9 @@ import static org.apache.hudi.utilities.config.CloudSourceConfig.META_EVENTS_PER * to check S3 files activity over time. The hudi table created by this source is consumed by * {@link S3EventsHoodieIncrSource} to apply changes to the hudi table corresponding to user data. */ +@Slf4j public class S3EventsSource extends RowSource implements Closeable { - private static final Logger LOG = LoggerFactory.getLogger(S3EventsSource.class); - private final S3EventsMetaSelector pathSelector; private final SchemaProvider schemaProvider; private final List<CloudObjectsSelector.MessageTracker> processedMessages = new ArrayList<>(); @@ -113,7 +111,7 @@ public class S3EventsSource extends RowSource implements Closeable { @Override public void onCommit(String lastCkptStr) { - LOG.info("Deleting {} processed messages from SQS queue, checkpoint={}.", + log.info("Deleting {} processed messages from SQS queue, checkpoint={}.", processedMessages.size(), lastCkptStr); pathSelector.deleteProcessedMessages(sqs, pathSelector.queueUrl, processedMessages); processedMessages.clear(); diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/WriteErrorReporter.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/WriteErrorReporter.java index 2cb7c2fe6050..516cefc283ef 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/WriteErrorReporter.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/WriteErrorReporter.java @@ -21,8 +21,7 @@ package org.apache.hudi.utilities.streamer; import org.apache.hudi.client.WriteStatus; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.util.List; @@ -31,10 +30,9 @@ import java.util.List; * * <p>Extracted from {@code HoodieStreamerWriteStatusValidator} (#18750).</p> */ +@Slf4j public final class WriteErrorReporter { - private static final Logger LOG = LoggerFactory.getLogger(WriteErrorReporter.class); - private static final int DEFAULT_MAX_ERRORS = 100; private WriteErrorReporter() { @@ -53,7 +51,7 @@ public final class WriteErrorReporter { if (writeStatuses == null || maxErrors <= 0) { return; } - LOG.info("Printing out the top {} errored write statuses", maxErrors); + log.info("Printing out the top {} errored write statuses", maxErrors); writeStatuses.stream() .filter(WriteStatus::hasErrors) .limit(maxErrors) @@ -61,9 +59,9 @@ public final class WriteErrorReporter { } private static void logOne(WriteStatus writeStatus) { - LOG.error("Global error: {}", writeStatus.getGlobalError()); + log.error("Global error: {}", writeStatus.getGlobalError()); if (!writeStatus.getErrors().isEmpty()) { - writeStatus.getErrors().forEach((k, v) -> LOG.trace("Error for key {} : {}", k, v)); + writeStatus.getErrors().forEach((k, v) -> log.trace("Error for key {} : {}", k, v)); } } } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkStreamerValidatorUtils.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkStreamerValidatorUtils.java index 97b6dac2f273..13700910367f 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkStreamerValidatorUtils.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkStreamerValidatorUtils.java @@ -35,8 +35,7 @@ import org.apache.hudi.core.validator.ValidationContext; import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.exception.HoodieValidationException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; import java.util.Arrays; @@ -65,10 +64,9 @@ import java.util.stream.Collectors; * are NOT compatible with {@code SparkValidatorUtils}. Do not mix them under the same * {@code hoodie.precommit.validators} config if both paths are active.</p> */ +@Slf4j public class SparkStreamerValidatorUtils { - private static final Logger LOG = LoggerFactory.getLogger(SparkStreamerValidatorUtils.class); - /** * Run all configured pre-commit validators. * @@ -121,21 +119,21 @@ public class SparkStreamerValidatorUtils { try { Class<?> clazz = Class.forName(className); if (!BasePreCommitValidator.class.isAssignableFrom(clazz)) { - LOG.warn("Skipping validator {} in HoodieStreamer path — it does not extend BasePreCommitValidator. " + log.warn("Skipping validator {} in HoodieStreamer path — it does not extend BasePreCommitValidator. " + "If this is a SparkPreCommitValidator (e.g. SqlQueryEqualityPreCommitValidator), " + "it must be invoked via SparkValidatorUtils in the standard Spark write path instead.", className); continue; } BasePreCommitValidator validator = (BasePreCommitValidator) ReflectionUtils.loadClass(className, new Class<?>[] {TypedProperties.class}, props); - LOG.info("Running pre-commit validator: {} for instant: {}", className, instantTime); + log.info("Running pre-commit validator: {} for instant: {}", className, instantTime); validator.validateWithMetadata(context); - LOG.info("Pre-commit validator {} passed for instant: {}", className, instantTime); + log.info("Pre-commit validator {} passed for instant: {}", className, instantTime); } catch (HoodieValidationException e) { - LOG.error("Pre-commit validator {} failed for instant: {}", className, instantTime, e); + log.error("Pre-commit validator {} failed for instant: {}", className, instantTime, e); throw e; } catch (Exception e) { - LOG.error("Failed to instantiate or run validator: {}", className, e); + log.error("Failed to instantiate or run validator: {}", className, e); throw new HoodieValidationException( "Failed to run pre-commit validator: " + className, e); } diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkWriteErrorValidator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkWriteErrorValidator.java index c061b47021d1..e7125c3fd8fd 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkWriteErrorValidator.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkWriteErrorValidator.java @@ -26,8 +26,7 @@ import org.apache.hudi.core.validator.BasePreCommitValidator; import org.apache.hudi.core.validator.ValidationContext; import org.apache.hudi.exception.HoodieValidationException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.util.Arrays; @@ -75,10 +74,9 @@ import java.util.Arrays; * and must be invoked via {@link SparkStreamerValidatorUtils} — not {@code SparkValidatorUtils}, * which expects a different constructor signature.</p> */ +@Slf4j public class SparkWriteErrorValidator extends BasePreCommitValidator { - private static final Logger LOG = LoggerFactory.getLogger(SparkWriteErrorValidator.class); - private final ValidationFailurePolicy failurePolicy; public SparkWriteErrorValidator(TypedProperties config) { @@ -108,13 +106,13 @@ public class SparkWriteErrorValidator extends BasePreCommitValidator { if (totalRecords == 0) { // Empty commit (mirrors HSWSV "No new data, perform empty commit."). - LOG.info("Empty commit (no records written, no errors). Skipping write-error validation " + log.info("Empty commit (no records written, no errors). Skipping write-error validation " + "for instant {}.", context.getInstantTime()); return; } if (totalErrors == 0) { - LOG.info("Write-error validation passed for instant {}: 0 errors out of {} records.", + log.info("Write-error validation passed for instant {}: 0 errors out of {} records.", context.getInstantTime(), totalRecords); return; } @@ -128,7 +126,7 @@ public class SparkWriteErrorValidator extends BasePreCommitValidator { HoodiePreCommitValidatorConfig.VALIDATION_FAILURE_POLICY.key()); if (failurePolicy == ValidationFailurePolicy.WARN_LOG) { - LOG.warn("{} (failure policy is WARN_LOG, commit will proceed)", errorMsg); + log.warn("{} (failure policy is WARN_LOG, commit will proceed)", errorMsg); } else { throw new HoodieValidationException(errorMsg); } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotExporter.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotExporter.java index 8cf47f03c745..b280cf5410b9 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotExporter.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotExporter.java @@ -46,6 +46,7 @@ import org.apache.hudi.utilities.config.SqlTransformerConfig; import org.apache.hudi.utilities.exception.HoodieSnapshotExporterException; import org.apache.hudi.utilities.testutils.UtilitiesTestBase; +import lombok.extern.slf4j.Slf4j; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.function.ForeachFunction; import org.apache.spark.sql.Column; @@ -59,8 +60,6 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.file.Paths; @@ -75,10 +74,10 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +@Slf4j @Tag("functional") public class TestHoodieSnapshotExporter extends SparkClientFunctionalTestHarness { - static final Logger LOG = LoggerFactory.getLogger(TestHoodieSnapshotExporter.class); static final int NUM_RECORDS = 100; static final String COMMIT_TIME = "20200101000000"; static final String PARTITION_PATH = "2020"; @@ -112,7 +111,7 @@ public class TestHoodieSnapshotExporter extends SparkClientFunctionalTestHarness } List<StoragePathInfo> pathInfoList = storage.listFiles(new StoragePath(sourcePath)); for (StoragePathInfo pathInfo : pathInfoList) { - LOG.info(">>> Prepared test file: {}", pathInfo.getPath()); + log.info(">>> Prepared test file: {}", pathInfo.getPath()); } } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/offlinejob/HoodieOfflineJobTestBase.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/offlinejob/HoodieOfflineJobTestBase.java index 634c434d3845..b6e3bdd9d368 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/offlinejob/HoodieOfflineJobTestBase.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/offlinejob/HoodieOfflineJobTestBase.java @@ -29,13 +29,12 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; import org.apache.hudi.utilities.testutils.UtilitiesTestBase; +import lombok.extern.slf4j.Slf4j; import org.apache.spark.api.java.JavaRDD; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.List; import java.util.Properties; @@ -44,8 +43,8 @@ import java.util.stream.Collectors; import static org.apache.hudi.common.testutils.HoodieTestUtils.createMetaClient; import static org.junit.jupiter.api.Assertions.assertEquals; +@Slf4j(topic = "org.apache.hudi.utilities.testutils.UtilitiesTestBase") public class HoodieOfflineJobTestBase extends UtilitiesTestBase { - private static final Logger LOG = LoggerFactory.getLogger(UtilitiesTestBase.class); protected HoodieTestDataGenerator dataGen; protected SparkRDDWriteClient client; protected HoodieTableMetaClient metaClient; @@ -110,7 +109,7 @@ public class HoodieOfflineJobTestBase extends UtilitiesTestBase { static void assertNCompletedCommits(int expected, String tablePath) { HoodieTableMetaClient meta = createMetaClient(storage, tablePath); HoodieTimeline timeline = meta.getActiveTimeline().getWriteTimeline().filterCompletedInstants(); - LOG.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); + log.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); int numCommits = timeline.countInstants(); assertEquals(expected, numCommits, "Got=" + numCommits + ", exp =" + expected); } @@ -118,7 +117,7 @@ public class HoodieOfflineJobTestBase extends UtilitiesTestBase { static void assertNCleanCommits(int expected, String tablePath) { HoodieTableMetaClient meta = createMetaClient(storage, tablePath); HoodieTimeline timeline = meta.getActiveTimeline().getCleanerTimeline().filterCompletedInstants(); - LOG.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); + log.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); int numCleanCommits = timeline.countInstants(); assertEquals(expected, numCleanCommits, "Got=" + numCleanCommits + ", exp =" + expected); } @@ -126,7 +125,7 @@ public class HoodieOfflineJobTestBase extends UtilitiesTestBase { static void assertNClusteringCommits(int expected, String tablePath) { HoodieTableMetaClient meta = createMetaClient(storage, tablePath); HoodieTimeline timeline = meta.getActiveTimeline().getCompletedReplaceTimeline(); - LOG.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); + log.info("Timeline Instants={}", meta.getActiveTimeline().getInstants()); int numCommits = timeline.countInstants(); assertEquals(expected, numCommits, "Got=" + numCommits + ", exp =" + expected); } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/KafkaTestUtils.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/KafkaTestUtils.java index d99188e9ec2b..7fe6f6527a48 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/KafkaTestUtils.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/KafkaTestUtils.java @@ -20,6 +20,7 @@ package org.apache.hudi.utilities.testutils; import org.apache.hudi.common.util.StringUtils; +import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.AdminClientConfig; import org.apache.kafka.clients.admin.Config; @@ -31,8 +32,6 @@ import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.config.ConfigResource; import org.apache.kafka.common.errors.TopicExistsException; import org.apache.kafka.common.serialization.StringSerializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.testcontainers.kafka.ConfluentKafkaContainer; import org.testcontainers.utility.DockerImageName; @@ -44,8 +43,8 @@ import java.util.stream.Collectors; import scala.Tuple2; +@Slf4j public class KafkaTestUtils { - private static final Logger LOG = LoggerFactory.getLogger(KafkaTestUtils.class); private static final DockerImageName KAFKA_IMAGE = DockerImageName.parse("confluentinc/cp-kafka:7.7.1"); private final List<String> createdTopics = new ArrayList<>(); private ConfluentKafkaContainer kafkaContainer; @@ -110,7 +109,7 @@ public class KafkaTestUtils { adminClient.deleteTopics(createdTopics).all().get(); createdTopics.clear(); } catch (Exception e) { - LOG.warn("Failed to delete topics: {}", StringUtils.join(createdTopics, ","), e); + log.warn("Failed to delete topics: {}", StringUtils.join(createdTopics, ","), e); } }
