This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 202578e8c2b [fix](hudi) Keep storage credentials out of scanner logs
(branch-4.2) (#69479)
202578e8c2b is described below
commit 202578e8c2b8a23400f1e3047c23609a6b5afb6d
Author: Oliveira <[email protected]>
AuthorDate: Fri Oct 23 15:04:06 2026 +0800
[fix](hudi) Keep storage credentials out of scanner logs (branch-4.2)
(#69479)
### What problem does this PR solve?
Issue Number: N/A
Related PR: #69478 (backport to branch-4.2)
Problem Summary: When a Hudi JNI scan uses explicit S3A credentials,
enabling scanner DEBUG logging prints every parameter value, including
access keys, secret keys and session tokens. Log parameter names without
values and record the query ID separately so scans remain traceable. The
Hadoop configuration used to read data is unchanged.
Add a regression test that captures the real scanner's DEBUG output,
checks that parameter names and the query ID remain visible, rejects
ordinary and bucket-specific credentials and custom tokens, and verifies
that Hadoop properties retain their original values.
### Release note
Hudi JNI scanner debug logs no longer include storage credential values
and continue to include query IDs for diagnostics.
### Check List (For Author)
- Test
- [ ] Regression test
- [x] Unit Test
- [ ] Manual test
- [ ] No need to test or manual test.
Validation with JDK 17:
- `EXTRA_FE_MODULES=hudi=be-java-extensions/hadoop-hudi-scanner
./run-fe-ut.sh --run
'org.apache.doris.hudi.HadoopHudiJniScannerLogTest,org.apache.doris.hudi.HadoopHudiColumnValueTest,org.apache.doris.hudi.HudiUuidTransportTest'`:
9 tests passed.
- The log regression test verifies query ID retention, credential-free
DEBUG output and unchanged Hadoop properties.
- The test captures this branch's Log4j/reload4j backends.
- Maven `validate` / Checkstyle for
`be-java-extensions/hadoop-hudi-scanner` and its dependencies, plus `git
diff --check`: passed.
- The logging regression is covered directly by JUnit. FE/BE + S3
end-to-end regression was not run locally; SQL planning, credential
propagation and data-reading logic are unchanged.
- Behavior changed:
- [ ] No.
- [x] Yes. DEBUG logs retain parameter names and query IDs without
printing other parameter values.
- Does this need documentation?
- [x] No.
- [ ] Yes.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../apache/doris/hudi/HadoopHudiJniScanner.java | 3 +-
.../doris/hudi/HadoopHudiJniScannerLogTest.java | 137 +++++++++++++++++++++
2 files changed, 139 insertions(+), 1 deletion(-)
diff --git
a/fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiJniScanner.java
b/fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiJniScanner.java
index a49f61e9f06..2a084c4419f 100644
---
a/fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiJniScanner.java
+++
b/fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiJniScanner.java
@@ -123,12 +123,13 @@ public class HadoopHudiJniScanner extends JniScanner {
this.fieldInspectors = new ObjectInspector[requiredFields.length];
this.structFields = new StructField[requiredFields.length];
this.fsOptionsProps = Maps.newHashMap();
+ LOG.debug("initialize hudi scanner for query {}",
params.get("query_id"));
for (Map.Entry<String, String> entry : params.entrySet()) {
if (entry.getKey().startsWith(HADOOP_CONF_PREFIX)) {
fsOptionsProps.put(entry.getKey().substring(HADOOP_CONF_PREFIX.length()),
entry.getValue());
}
if (LOG.isDebugEnabled()) {
- LOG.debug("get hudi params {}: {}", entry.getKey(),
entry.getValue());
+ LOG.debug("get hudi parameter: {}", entry.getKey());
}
}
this.preExecutionAuthenticator =
PreExecutionAuthenticatorCache.getAuthenticator(fsOptionsProps);
diff --git
a/fe/be-java-extensions/hadoop-hudi-scanner/src/test/java/org/apache/doris/hudi/HadoopHudiJniScannerLogTest.java
b/fe/be-java-extensions/hadoop-hudi-scanner/src/test/java/org/apache/doris/hudi/HadoopHudiJniScannerLogTest.java
new file mode 100644
index 00000000000..656c8928e61
--- /dev/null
+++
b/fe/be-java-extensions/hadoop-hudi-scanner/src/test/java/org/apache/doris/hudi/HadoopHudiJniScannerLogTest.java
@@ -0,0 +1,137 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.hudi;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.Logger;
+import org.apache.logging.log4j.core.appender.AbstractAppender;
+import org.apache.logging.log4j.core.config.Property;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+public class HadoopHudiJniScannerLogTest {
+ @Test
+ public void logsQueryIdAndParameterNamesWithoutCredentialValues() throws
Exception {
+ Map<String, String> hadoopProperties = new HashMap<>();
+ hadoopProperties.put("fs.s3a.access.key", "synthetic-access-key");
+ hadoopProperties.put("fs.s3a.secret.key", "synthetic-secret-key");
+ hadoopProperties.put("fs.s3a.session.token",
"synthetic-session-token");
+ hadoopProperties.put("fs.s3a.bucket.test-bucket.access.key",
"synthetic-bucket-access-key");
+ hadoopProperties.put("fs.s3a.bucket.test-bucket.secret.key",
"synthetic-bucket-secret-key");
+ hadoopProperties.put("fs.s3a.bucket.test-bucket.session.token",
"synthetic-bucket-session-token");
+ hadoopProperties.put("custom.setting", "synthetic-custom-value");
+
+ Map<String, String> params = new HashMap<>();
+ params.put("query_id", "0123456789abcdef-0123456789abcdef");
+ params.put("base_path", "file:///unused");
+ params.put("data_file_path", "file:///unused/data.parquet");
+ params.put("data_file_length", "0");
+ params.put("hudi_column_names", "id");
+ params.put("hudi_column_types", "int");
+ params.put("columns_types", "int");
+ params.put("required_fields", "id");
+ params.put("time_zone", "UTC");
+ params.put("hoodie.custom.token", "synthetic-hudi-token");
+ hadoopProperties.forEach((key, value) -> params.put("hadoop_conf." +
key, value));
+
+ try (CapturedLog capture = new CapturedLog()) {
+ Assert.assertTrue("The test must exercise DEBUG parameter logging",
+
LoggerFactory.getLogger(HadoopHudiJniScanner.class).isDebugEnabled());
+ HadoopHudiJniScanner scanner = new HadoopHudiJniScanner(16,
params);
+ String text = String.join("\n", capture.messages);
+ Assert.assertTrue("Keep the query ID for diagnostics",
text.contains(params.get("query_id")));
+ for (String key : params.keySet()) {
+ Assert.assertTrue("Missing parameter name: " + key,
text.contains(key));
+ }
+ for (String value : hadoopProperties.values()) {
+ Assert.assertFalse("Hadoop parameter values must not be
logged", text.contains(value));
+ }
+ Assert.assertFalse("Non-Hadoop credentials must not be logged
either",
+ text.contains(params.get("hoodie.custom.token")));
+
+ Field properties =
HadoopHudiJniScanner.class.getDeclaredField("fsOptionsProps");
+ properties.setAccessible(true);
+ Assert.assertEquals("Removing values from logs must preserve the
actual Hadoop configuration",
+ hadoopProperties, properties.get(scanner));
+ }
+ }
+
+ private static class CapturedLog implements AutoCloseable {
+ private final List<String> messages = new CopyOnWriteArrayList<>();
+ private final Logger logger = (Logger)
LogManager.getLogger(HadoopHudiJniScanner.class);
+ private final org.apache.log4j.Logger reload4j =
+ org.apache.log4j.Logger.getLogger(HadoopHudiJniScanner.class);
+ private final org.apache.log4j.Level previousReload4jLevel =
reload4j.getLevel();
+ private final boolean previousReload4jAdditive =
reload4j.getAdditivity();
+ private final org.apache.log4j.AppenderSkeleton reload4jAppender = new
org.apache.log4j.AppenderSkeleton() {
+ @Override
+ protected void append(org.apache.log4j.spi.LoggingEvent event) {
+ messages.add(event.getRenderedMessage());
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public boolean requiresLayout() {
+ return false;
+ }
+ };
+ private final Level previousLevel = logger.getLevel();
+ private final boolean previousAdditive = logger.isAdditive();
+ private final AbstractAppender appender = new
AbstractAppender("hudi-parameter-test", null,
+ null, true, Property.EMPTY_ARRAY) {
+ @Override
+ public void append(LogEvent event) {
+ messages.add(event.getMessage().getFormattedMessage());
+ }
+ };
+
+ CapturedLog() {
+ reload4j.setLevel(org.apache.log4j.Level.DEBUG);
+ reload4j.setAdditivity(false);
+ reload4j.addAppender(reload4jAppender);
+ appender.start();
+ logger.setLevel(Level.DEBUG);
+ logger.setAdditive(false);
+ logger.addAppender(appender);
+ }
+
+ @Override
+ public void close() {
+ logger.removeAppender(appender);
+ logger.setLevel(previousLevel);
+ logger.setAdditive(previousAdditive);
+ appender.stop();
+ reload4j.removeAppender(reload4jAppender);
+ reload4j.setLevel(previousReload4jLevel);
+ reload4j.setAdditivity(previousReload4jAdditive);
+ reload4jAppender.close();
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
