This is an automated email from the ASF dual-hosted git repository.
diqiu50 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 9cc9084c14 [#13857] fix(server): Mask secrets in --updater-options and
--spark-conf job arguments (#13858)
9cc9084c14 is described below
commit 9cc9084c14629af3f563b35a6482dbd678213430
Author: MaSai <[email protected]>
AuthorDate: Fri Oct 16 21:40:11 2026 +0800
[#13857] fix(server): Mask secrets in --updater-options and --spark-conf
job arguments (#13858)
### What changes were proposed in this pull request?
Mask sensitive-named keys inside the JSON values of two built-in Iceberg
job arguments, on the job get/list/run/cancel response and the job event
path:
- `--updater-options` (credentials such as `password` and OAuth fields)
- `--spark-conf` (Spark catalog passwords such as `*.password`)
The whole JSON value stays one argument. Only sensitive-named keys
inside it are replaced with `******`. Stored runtime JSON and the
plaintext used to run the job are unchanged.
`RunJobPreEvent`, `RunJobEvent`, and `RunJobFailureEvent` carry a masked
copy of `jobConf`. The `updater_options` and `spark_conf` entries in
that copy are masked the same way. The map passed to job execution stays
plaintext.
Out of scope: other argument forms (`--password`, `-D`, `--conf`, shell
`-c`) and map fields (`environments`, `customFields`, `configs`).
### Why are the changes needed?
`builtin-iceberg-update-stats` substitutes `updater_options` and
`spark_conf` from `jobConf` into `runtimeJobTemplate.arguments`. Job
APIs and the run page then return those credentials in clear text. The
same JSON is also present on the run-job events' `jobConf`.
Fix: #13857
### Does this PR introduce _any_ user-facing change?
Yes. Job API responses and job events that include a resolved
`--updater-options` or `--spark-conf` argument now replace
sensitive-named values inside that JSON with `******`. `RunJobPreEvent`,
`RunJobEvent`, and `RunJobFailureEvent` do the same for the
`updater_options` and `spark_conf` entries of `jobConf`. Other arguments
and map fields are unchanged.
### How was this patch tested?
- Unit tests for `--updater-options` and `--spark-conf` JSON arguments:
sensitive keys are masked, non-sensitive keys in the same JSON are kept,
and an empty value stays empty.
- Unit tests for run-job events: the `jobConf` copy masks
`updater_options` and `spark_conf`, and the map passed to job execution
stays plaintext.
- Job get/list response test: a runtime template with those two
arguments returns masked JSON, and the stored runtime JSON stays
plaintext.
---------
Co-authored-by: Cursor <[email protected]>
---
.../connector/HiddenPropertyMaskUtils.java | 133 +++++++++++
.../gravitino/job/JobRuntimeTemplateMaskUtils.java | 214 +++++++++++++++++
.../gravitino/listener/JobEventDispatcher.java | 10 +-
.../gravitino/listener/api/info/JobInfo.java | 24 +-
.../connector/TestHiddenPropertyMaskUtils.java | 43 ++++
.../job/TestJobRuntimeTemplateMaskUtils.java | 264 +++++++++++++++++++++
.../listener/api/event/TestJobEventDispatcher.java | 54 +++++
.../gravitino/listener/api/info/TestJobInfo.java | 49 ++++
.../gravitino/server/web/rest/JobOperations.java | 12 +-
.../server/web/rest/TestJobOperations.java | 53 +++++
10 files changed, 842 insertions(+), 14 deletions(-)
diff --git
a/core/src/main/java/org/apache/gravitino/connector/HiddenPropertyMaskUtils.java
b/core/src/main/java/org/apache/gravitino/connector/HiddenPropertyMaskUtils.java
index d3e1c11c81..8301e73639 100644
---
a/core/src/main/java/org/apache/gravitino/connector/HiddenPropertyMaskUtils.java
+++
b/core/src/main/java/org/apache/gravitino/connector/HiddenPropertyMaskUtils.java
@@ -18,14 +18,25 @@
*/
package org.apache.gravitino.connector;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import javax.annotation.Nullable;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.gravitino.json.JsonUtils;
import org.apache.gravitino.secret.SecretPropertyUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Read/write helpers for sensitive and system-managed entity properties.
@@ -56,9 +67,18 @@ import org.apache.gravitino.secret.SecretPropertyUtils;
*/
public final class HiddenPropertyMaskUtils {
+ private static final Logger LOG =
LoggerFactory.getLogger(HiddenPropertyMaskUtils.class);
+
/** Placeholder returned instead of editable hidden / credential / secret
property values. */
public static final String MASKED_VALUE = "******";
+ /**
+ * Mapper for nest-JSON masking. Uses {@link JsonNode} with big-decimal
floats so rewritten JSON
+ * keeps numeric fidelity better than {@code Map}/{@code List} round-trips.
+ */
+ private static final ObjectMapper NEST_JSON_MAPPER =
+
JsonUtils.anyFieldMapper().copy().enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS);
+
private HiddenPropertyMaskUtils() {}
/** Returns true when {@code value} is the read-path masked placeholder. */
@@ -176,4 +196,117 @@ public final class HiddenPropertyMaskUtils {
Map<String, String> properties, PropertiesMetadata metadata) {
return maskHiddenProperties(properties,
classifyHiddenProperties(properties, metadata));
}
+
+ /**
+ * Parses {@code value} as JSON and masks sensitive-named keys in objects
(recursively through
+ * nested objects and arrays). Blank strings and non-JSON plain text
(trimmed value does not start
+ * with '{' or '[') are returned unchanged. Values that look like a JSON
object/array but fail to
+ * parse are replaced with {@link #MASKED_VALUE}. Scalar JSON roots are
returned unchanged.
+ *
+ * <p>Useful when a caller already knows a string is a nested JSON map (for
example a CLI argument
+ * following {@code --spark-conf}).
+ *
+ * @param value JSON string to mask; may be null
+ * @return masked JSON, {@link #MASKED_VALUE} on object/array parse failure,
or the original value
+ * when blank, plain non-JSON, scalar JSON, or null
+ */
+ @Nullable
+ public static String maskSensitiveJsonValue(@Nullable String value) {
+ if (value == null) {
+ return null;
+ }
+ return maskNestedJsonValue(value);
+ }
+
+ /**
+ * Parses {@code value} as JSON and masks sensitive-named keys in objects
(recursively through
+ * nested objects and arrays). Blank / plain non-JSON strings are returned
unchanged. Malformed
+ * object/array-looking JSON is replaced with {@link #MASKED_VALUE}.
+ */
+ private static String maskNestedJsonValue(String value) {
+ // Empty / whitespace-only values carry no secret (common for unresolved
{{spark_conf:-}}).
+ if (StringUtils.isBlank(value)) {
+ return value;
+ }
+ // Bare words (e.g. --options=verbose) are not JSON maps; do not treat
parse failure as secret.
+ String trimmed = value.trim();
+ if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) {
+ return value;
+ }
+ try {
+ JsonNode root = NEST_JSON_MAPPER.readTree(value);
+ if (!root.isObject() && !root.isArray()) {
+ // Scalar JSON (string/number/boolean/null) — nothing to walk by key
name.
+ return value;
+ }
+ MaskedJson masked = maskJsonNode(root);
+ return masked.changed ?
NEST_JSON_MAPPER.writeValueAsString(masked.value) : value;
+ } catch (Exception e) {
+ // Malformed object/array JSON on a caller-declared nest key — do not
leave a secret blob.
+ // Do not log the value; it may contain credentials.
+ LOG.warn(
+ "Replacing malformed nest JSON with masked placeholder after parse
failure: {}",
+ e.toString());
+ return MASKED_VALUE;
+ }
+ }
+
+ private static MaskedJson maskJsonNode(JsonNode node) {
+ if (node.isObject()) {
+ ObjectNode out = JsonNodeFactory.instance.objectNode();
+ boolean changed = false;
+ Iterator<Map.Entry<String, JsonNode>> fields = node.fields();
+ while (fields.hasNext()) {
+ Map.Entry<String, JsonNode> entry = fields.next();
+ String nestedKey = entry.getKey();
+ JsonNode nestedRaw = entry.getValue();
+ if (nestedRaw.isObject() || nestedRaw.isArray()) {
+ if (SecretPropertyUtils.isSensitivePropertyKey(nestedKey)) {
+ out.put(nestedKey, MASKED_VALUE);
+ changed = true;
+ } else {
+ MaskedJson child = maskJsonNode(nestedRaw);
+ out.set(nestedKey, child.value);
+ changed |= child.changed;
+ }
+ } else {
+ String nestedValue = nestedRaw.isNull() ? "" : nestedRaw.asText();
+ if (SecretPropertyUtils.isSensitivePropertyKey(nestedKey)
+ || SecretPropertyUtils.isSecretProperty(nestedKey, nestedValue))
{
+ out.put(nestedKey, MASKED_VALUE);
+ changed = true;
+ } else {
+ out.set(nestedKey, nestedRaw);
+ }
+ }
+ }
+ return new MaskedJson(out, changed);
+ }
+ if (node.isArray()) {
+ ArrayNode out = JsonNodeFactory.instance.arrayNode();
+ boolean changed = false;
+ for (JsonNode item : node) {
+ if (item.isObject() || item.isArray()) {
+ MaskedJson child = maskJsonNode(item);
+ out.add(child.value);
+ changed |= child.changed;
+ } else {
+ out.add(item);
+ }
+ }
+ return new MaskedJson(out, changed);
+ }
+ return new MaskedJson(node, false);
+ }
+
+ /** Result of walking a JSON tree for sensitive-key masking. */
+ private static final class MaskedJson {
+ private final JsonNode value;
+ private final boolean changed;
+
+ private MaskedJson(JsonNode value, boolean changed) {
+ this.value = value;
+ this.changed = changed;
+ }
+ }
}
diff --git
a/core/src/main/java/org/apache/gravitino/job/JobRuntimeTemplateMaskUtils.java
b/core/src/main/java/org/apache/gravitino/job/JobRuntimeTemplateMaskUtils.java
new file mode 100644
index 0000000000..71ae5dcfed
--- /dev/null
+++
b/core/src/main/java/org/apache/gravitino/job/JobRuntimeTemplateMaskUtils.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.job;
+
+import com.google.common.collect.ImmutableSet;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import javax.annotation.Nullable;
+import org.apache.gravitino.connector.HiddenPropertyMaskUtils;
+import org.apache.gravitino.dto.job.JobTemplateDTO;
+import org.apache.gravitino.dto.job.ShellJobTemplateDTO;
+import org.apache.gravitino.dto.job.SparkJobTemplateDTO;
+
+/**
+ * Display-masks credentials in a Spark job's {@code updater_options} and
{@code spark_conf} JSON
+ * before that JSON is returned by an API or placed on an event.
Sensitive-named keys inside the
+ * JSON are replaced with {@link HiddenPropertyMaskUtils#MASKED_VALUE}. This
is not encryption: the
+ * stored runtime template and the map passed to job execution stay plaintext.
+ *
+ * <p>Only those two JSON bags are masked.
+ *
+ * <ul>
+ * <li>Spark template arguments {@code --updater-options} and {@code
--spark-conf}. One list
+ * element is one argument, so the JSON is the whole next element (or
the text after {@code =}
+ * in {@code --flag=json}). It is not split on whitespace.
+ * <li>{@code jobConf} entries {@code updater_options} and {@code
spark_conf} on the run-job
+ * events. Those names are the Spark template placeholders {@code
{{updater_options}}} and
+ * {@code {{spark_conf}}}, not the hyphenated CLI flags.
+ * </ul>
+ *
+ * <p>Shell jobs have neither bag. Other arguments ({@code --password}, {@code
--conf}, {@code -c},
+ * {@code --options}) and map fields ({@code environments}, {@code
customFields}, Spark {@code
+ * configs}) are copied unchanged. A blank value, which is what {@code
{{spark_conf:-}}} becomes
+ * when unset, is left unchanged.
+ */
+public final class JobRuntimeTemplateMaskUtils {
+
+ /**
+ * Argument flags whose following value is a JSON map that may contain
credentials.
+ *
+ * <ul>
+ * <li>{@code --updater-options} — Gravitino / updater credentials
+ * <li>{@code --spark-conf} — Spark configs, including catalog passwords
+ * </ul>
+ */
+ static final Set<String> JSON_ARGUMENT_FLAGS =
+ ImmutableSet.of("--updater-options", "--spark-conf");
+
+ /**
+ * {@code jobConf} keys for the Spark placeholders {@code
{{updater_options}}} and {@code
+ * {{spark_conf}}}. The CLI flags in {@link #JSON_ARGUMENT_FLAGS} use
hyphens and are a different
+ * spelling of the same two JSON bags.
+ */
+ static final Set<String> JSON_JOB_CONF_KEYS =
ImmutableSet.of("updater_options", "spark_conf");
+
+ private JobRuntimeTemplateMaskUtils() {}
+
+ /**
+ * Returns a copy of {@code jobConf} for run-job events. Only the Spark
placeholder values {@code
+ * updater_options} and {@code spark_conf} are masked. A top-level key such
as {@code password} is
+ * copied unchanged, because this method does not scan every entry. Does not
mutate the input;
+ * callers must pass the original map to job execution.
+ *
+ * @param jobConf runtime configuration from the runJob caller; may be null
+ * @return a masked copy, or null when {@code jobConf} is null
+ */
+ @Nullable
+ public static Map<String, String> maskSensitiveJobConf(@Nullable Map<String,
String> jobConf) {
+ if (jobConf == null) {
+ return null;
+ }
+ Map<String, String> masked = new LinkedHashMap<>();
+ for (Map.Entry<String, String> entry : jobConf.entrySet()) {
+ String key = entry.getKey();
+ String value = entry.getValue();
+ if (key != null && JSON_JOB_CONF_KEYS.contains(key)) {
+ masked.put(key, HiddenPropertyMaskUtils.maskSensitiveJsonValue(value));
+ } else {
+ masked.put(key, value);
+ }
+ }
+ return masked;
+ }
+
+ /**
+ * Returns a copy of a Spark runtime template with {@code --updater-options}
and {@code
+ * --spark-conf} JSON masked, or {@code null} when the input is null. Spark
{@code configs},
+ * {@code environments}, and {@code customFields} are not masked. A Shell
template is rebuilt with
+ * the same argument walk so a hand-written template that uses these flags
does not leak; built-in
+ * credential JSON lives on the Spark template.
+ *
+ * <p>The DTO is rebuilt field by field (there is no DTO copy helper). When
adding a field to
+ * {@link SparkJobTemplateDTO} or {@link ShellJobTemplateDTO}, update this
method and the
+ * field-carry-over tests in {@code TestJobRuntimeTemplateMaskUtils}.
+ *
+ * @param jobTemplateDTO the resolved runtime job template DTO
+ * @return a masked copy, or null
+ */
+ @Nullable
+ public static JobTemplateDTO maskSensitiveFields(@Nullable JobTemplateDTO
jobTemplateDTO) {
+ if (jobTemplateDTO == null) {
+ return null;
+ }
+ List<String> maskedArguments =
maskSensitiveArguments(jobTemplateDTO.arguments());
+ if (jobTemplateDTO instanceof ShellJobTemplateDTO) {
+ ShellJobTemplateDTO shell = (ShellJobTemplateDTO) jobTemplateDTO;
+ return ShellJobTemplateDTO.builder()
+ .withName(shell.name())
+ .withComment(shell.comment())
+ .withJobType(shell.jobType())
+ .withExecutable(shell.executable())
+ .withArguments(maskedArguments)
+ .withEnvironments(shell.environments())
+ .withCustomFields(shell.customFields())
+ .withScripts(shell.scripts())
+ .withAudit(shell.audit())
+ .build();
+ }
+ if (jobTemplateDTO instanceof SparkJobTemplateDTO) {
+ SparkJobTemplateDTO spark = (SparkJobTemplateDTO) jobTemplateDTO;
+ return SparkJobTemplateDTO.builder()
+ .withName(spark.name())
+ .withComment(spark.comment())
+ .withJobType(spark.jobType())
+ .withExecutable(spark.executable())
+ .withArguments(maskedArguments)
+ .withEnvironments(spark.environments())
+ .withCustomFields(spark.customFields())
+ .withClassName(spark.className())
+ .withJars(spark.jars())
+ .withFiles(spark.files())
+ .withArchives(spark.archives())
+ .withConfigs(spark.configs())
+ .withAudit(spark.audit())
+ .build();
+ }
+ throw new IllegalArgumentException(
+ "Unsupported job template type: " + jobTemplateDTO.jobType());
+ }
+
+ /**
+ * Masks the JSON value of {@link #JSON_ARGUMENT_FLAGS}. The value is one
list element, including
+ * any spaces inside the JSON. Other arguments are copied unchanged.
+ */
+ @Nullable
+ private static List<String> maskSensitiveArguments(@Nullable List<String>
arguments) {
+ if (arguments == null) {
+ return null;
+ }
+ if (arguments.isEmpty()) {
+ return arguments;
+ }
+ List<String> result = new ArrayList<>(arguments.size());
+ boolean maskNextJson = false;
+ for (String arg : arguments) {
+ if (maskNextJson) {
+ maskNextJson = false;
+ if (arg != null && !arg.startsWith("--")) {
+ result.add(HiddenPropertyMaskUtils.maskSensitiveJsonValue(arg));
+ continue;
+ }
+ // Missing value for the previous flag; this element may itself be a
flag.
+ }
+ String equalsForm = maskEqualsForm(arg);
+ if (equalsForm != null) {
+ result.add(equalsForm);
+ continue;
+ }
+ result.add(arg);
+ if (arg != null && JSON_ARGUMENT_FLAGS.contains(arg)) {
+ maskNextJson = true;
+ }
+ }
+ return result;
+ }
+
+ /**
+ * @return the argument with its JSON payload masked, or {@code null} when
{@code arg} is not
+ * {@code --flag=json} for {@link #JSON_ARGUMENT_FLAGS}
+ */
+ @Nullable
+ private static String maskEqualsForm(@Nullable String arg) {
+ if (arg == null) {
+ return null;
+ }
+ for (String flag : JSON_ARGUMENT_FLAGS) {
+ String prefix = flag + "=";
+ if (arg.startsWith(prefix)) {
+ return prefix
+ +
HiddenPropertyMaskUtils.maskSensitiveJsonValue(arg.substring(prefix.length()));
+ }
+ }
+ return null;
+ }
+}
diff --git
a/core/src/main/java/org/apache/gravitino/listener/JobEventDispatcher.java
b/core/src/main/java/org/apache/gravitino/listener/JobEventDispatcher.java
index f8f4896fcd..4a54b583ab 100644
--- a/core/src/main/java/org/apache/gravitino/listener/JobEventDispatcher.java
+++ b/core/src/main/java/org/apache/gravitino/listener/JobEventDispatcher.java
@@ -27,6 +27,7 @@ import
org.apache.gravitino.exceptions.JobTemplateAlreadyExistsException;
import org.apache.gravitino.exceptions.NoSuchJobException;
import org.apache.gravitino.exceptions.NoSuchJobTemplateException;
import org.apache.gravitino.job.JobOperationDispatcher;
+import org.apache.gravitino.job.JobRuntimeTemplateMaskUtils;
import org.apache.gravitino.job.JobTemplateChange;
import org.apache.gravitino.listener.api.event.job.AlterJobTemplateEvent;
import
org.apache.gravitino.listener.api.event.job.AlterJobTemplateFailureEvent;
@@ -224,9 +225,12 @@ public class JobEventDispatcher implements
JobOperationDispatcher {
@Override
public JobEntity runJob(String metalake, String jobTemplateName, Map<String,
String> jobConf)
throws NoSuchJobTemplateException {
+ // Events carry a masked copy of the Spark updater_options / spark_conf
JSON. Execution below
+ // still uses the caller's plaintext jobConf.
+ Map<String, String> maskedJobConf =
JobRuntimeTemplateMaskUtils.maskSensitiveJobConf(jobConf);
eventBus.dispatchEvent(
new RunJobPreEvent(
- PrincipalUtils.getCurrentUserName(), metalake, jobTemplateName,
jobConf));
+ PrincipalUtils.getCurrentUserName(), metalake, jobTemplateName,
maskedJobConf));
try {
JobEntity job = jobOperationDispatcher.runJob(metalake, jobTemplateName,
jobConf);
@@ -235,13 +239,13 @@ public class JobEventDispatcher implements
JobOperationDispatcher {
PrincipalUtils.getCurrentUserName(),
metalake,
jobTemplateName,
- jobConf,
+ maskedJobConf,
JobInfo.fromJobEntity(job)));
return job;
} catch (Exception e) {
eventBus.dispatchEvent(
new RunJobFailureEvent(
- PrincipalUtils.getCurrentUserName(), metalake, jobTemplateName,
jobConf, e));
+ PrincipalUtils.getCurrentUserName(), metalake, jobTemplateName,
maskedJobConf, e));
throw e;
}
}
diff --git
a/core/src/main/java/org/apache/gravitino/listener/api/info/JobInfo.java
b/core/src/main/java/org/apache/gravitino/listener/api/info/JobInfo.java
index 5771324269..4d489b99fe 100644
--- a/core/src/main/java/org/apache/gravitino/listener/api/info/JobInfo.java
+++ b/core/src/main/java/org/apache/gravitino/listener/api/info/JobInfo.java
@@ -26,6 +26,7 @@ import org.apache.gravitino.annotation.DeveloperApi;
import org.apache.gravitino.dto.job.JobTemplateDTO;
import org.apache.gravitino.dto.util.DTOConverters;
import org.apache.gravitino.job.JobHandle;
+import org.apache.gravitino.job.JobRuntimeTemplateMaskUtils;
import org.apache.gravitino.job.JobTemplate;
import org.apache.gravitino.meta.JobEntity;
import org.slf4j.Logger;
@@ -89,18 +90,22 @@ public final class JobInfo {
/**
* Deserializes the job entity's stored runtime job template JSON, if any,
back into a {@link
- * JobTemplate}. This is called from inside the event dispatcher's try block
for getJob/runJob/
- * cancelJob, after the underlying operation has already succeeded (or, for
cancelJob, after the
- * external cancel call and entity update have already happened) - a
malformed or
- * forward-incompatible stored value (e.g. a job type unknown to this server
version) must not
- * turn that already-completed operation into a 500, so failures here are
logged and swallowed
- * rather than propagated.
+ * JobTemplate}. For a Spark runtime template, sensitive-named keys inside
{@code
+ * --updater-options} and {@code --spark-conf} JSON arguments are masked via
{@link
+ * JobRuntimeTemplateMaskUtils} before the template is exposed on the event
path. This does not
+ * mask the separate {@code jobConf} map on run-job events. This is called
from inside the event
+ * dispatcher's try block for getJob/runJob/cancelJob, after the underlying
operation has already
+ * succeeded (or, for cancelJob, after the external cancel call and entity
update have already
+ * happened) - a malformed or forward-incompatible stored value (e.g. a job
type unknown to this
+ * server version) must not turn that already-completed operation into a
500, so failures here are
+ * logged and swallowed rather than propagated.
*/
private static JobTemplate toRuntimeJobTemplate(JobEntity jobEntity) {
try {
JobTemplateDTO runtimeJobTemplateDTO =
- DTOConverters.fromRuntimeJobTemplateJson(
- jobEntity.runtimeJobTemplate(), jobEntity.name());
+ JobRuntimeTemplateMaskUtils.maskSensitiveFields(
+ DTOConverters.fromRuntimeJobTemplateJson(
+ jobEntity.runtimeJobTemplate(), jobEntity.name()));
return runtimeJobTemplateDTO == null ? null :
DTOConverters.fromDTO(runtimeJobTemplateDTO);
} catch (Exception e) {
LOG.warn(
@@ -183,7 +188,8 @@ public final class JobInfo {
/**
* Returns the resolved job template that was actually submitted for
execution, with placeholders
- * replaced. Its executable, scripts, jars, files and archives are the URIs
from the template.
+ * replaced. For a Spark template, sensitive-named keys inside {@code
--updater-options} and
+ * {@code --spark-conf} JSON arguments are masked for the event path.
*
* @return the runtime job template, or null for jobs run before this field
was introduced
*/
diff --git
a/core/src/test/java/org/apache/gravitino/connector/TestHiddenPropertyMaskUtils.java
b/core/src/test/java/org/apache/gravitino/connector/TestHiddenPropertyMaskUtils.java
index 913d9ef358..c646e47fc1 100644
---
a/core/src/test/java/org/apache/gravitino/connector/TestHiddenPropertyMaskUtils.java
+++
b/core/src/test/java/org/apache/gravitino/connector/TestHiddenPropertyMaskUtils.java
@@ -18,12 +18,15 @@
*/
package org.apache.gravitino.connector;
+import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+import java.math.BigDecimal;
import java.util.List;
import java.util.Map;
import org.apache.gravitino.Config;
import org.apache.gravitino.Configs;
+import org.apache.gravitino.json.JsonUtils;
import org.apache.gravitino.secret.SecretPropertyUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
@@ -203,4 +206,44 @@ public class TestHiddenPropertyMaskUtils {
Assertions.assertEquals("root", masked.get("jdbc-user"));
Assertions.assertEquals(HiddenPropertyMaskUtils.MASKED_VALUE,
masked.get("jdbc-passwrod"));
}
+
+ @Test
+ void testMaskSensitiveJsonValueLeavesPlainNonJsonUnchanged() {
+ Assertions.assertEquals("verbose",
HiddenPropertyMaskUtils.maskSensitiveJsonValue("verbose"));
+ Assertions.assertEquals("5",
HiddenPropertyMaskUtils.maskSensitiveJsonValue("5"));
+ Assertions.assertEquals(
+ HiddenPropertyMaskUtils.MASKED_VALUE,
+
HiddenPropertyMaskUtils.maskSensitiveJsonValue("{not-json,\"password\":\"x\"}"));
+ }
+
+ @Test
+ void testMaskSensitiveJsonValuePreservesNumericLiteralWhenMaskingSibling()
throws Exception {
+ String masked =
+ HiddenPropertyMaskUtils.maskSensitiveJsonValue(
+ "{\"count\":1e2,\"password\":\"pw\",\"flag\":true}");
+ Assertions.assertFalse(masked.contains("\"pw\""));
+
Assertions.assertTrue(masked.contains(HiddenPropertyMaskUtils.MASKED_VALUE));
+ Assertions.assertTrue(masked.contains("true"));
+ // JsonNode + USE_BIG_DECIMAL_FOR_FLOATS preserves the numeric value (not
a leaked secret).
+ JsonNode root = JsonUtils.anyFieldMapper().readTree(masked);
+ Assertions.assertEquals(0, root.get("count").decimalValue().compareTo(new
BigDecimal("100")));
+ }
+
+ @Test
+ void testMaskSensitiveJsonValueMasksSensitiveKeys() {
+ String masked =
+ HiddenPropertyMaskUtils.maskSensitiveJsonValue(
+
"{\"gravitino_uri\":\"http://localhost\",\"password\":\"pw\",\"oauth_token\":\"tok\"}";);
+ Assertions.assertTrue(masked.contains("http://localhost";));
+ Assertions.assertFalse(masked.contains("\"pw\""));
+ Assertions.assertFalse(masked.contains("\"tok\""));
+
Assertions.assertTrue(masked.contains(HiddenPropertyMaskUtils.MASKED_VALUE));
+ }
+
+ @Test
+ void testMaskSensitiveJsonValueLeavesBlankUnchanged() {
+ Assertions.assertEquals("",
HiddenPropertyMaskUtils.maskSensitiveJsonValue(""));
+ Assertions.assertEquals(" ",
HiddenPropertyMaskUtils.maskSensitiveJsonValue(" "));
+
Assertions.assertNull(HiddenPropertyMaskUtils.maskSensitiveJsonValue(null));
+ }
}
diff --git
a/core/src/test/java/org/apache/gravitino/job/TestJobRuntimeTemplateMaskUtils.java
b/core/src/test/java/org/apache/gravitino/job/TestJobRuntimeTemplateMaskUtils.java
new file mode 100644
index 0000000000..9a4bd1f396
--- /dev/null
+++
b/core/src/test/java/org/apache/gravitino/job/TestJobRuntimeTemplateMaskUtils.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.job;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.gravitino.connector.HiddenPropertyMaskUtils;
+import org.apache.gravitino.dto.AuditDTO;
+import org.apache.gravitino.dto.job.ShellJobTemplateDTO;
+import org.apache.gravitino.dto.job.SparkJobTemplateDTO;
+import org.apache.gravitino.job.JobTemplate.JobType;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class TestJobRuntimeTemplateMaskUtils {
+
+ @Test
+ void testMaskSensitiveFieldsMasksUpdaterOptionsAndSparkConf() {
+ SparkJobTemplateDTO input =
+ SparkJobTemplateDTO.builder()
+ .withName("spark_job")
+ .withComment("c")
+ .withJobType(JobType.SPARK)
+ .withExecutable("local:///app.jar")
+ .withArguments(
+ ImmutableList.of(
+ "--catalog",
+ "ice",
+ "--updater-options",
+
"{\"gravitino_uri\":\"http://localhost\",\"password\":\"arg-pw\",";
+ + "\"oauth_token\":\"tok\"}",
+ "--spark-conf",
+ "{\"spark.executor.memory\":\"1g\","
+ +
"\"spark.sql.catalog.ice.rest.auth.basic.password\":\"jdbc-pw\"}",
+ "--table",
+ "db.t"))
+ .withEnvironments(ImmutableMap.of("AUTH_PASSWORD", "env-secret",
"CATALOG", "ice"))
+ .withClassName("org.example.App")
+ .withConfigs(ImmutableMap.of("spark.hadoop.fs.s3a.secret.key",
"s3-secret"))
+ .build();
+
+ SparkJobTemplateDTO spark =
+ (SparkJobTemplateDTO)
JobRuntimeTemplateMaskUtils.maskSensitiveFields(input);
+ List<String> args = spark.arguments();
+ Assertions.assertEquals("--updater-options", args.get(2));
+ Assertions.assertFalse(args.get(3).contains("arg-pw"));
+ Assertions.assertFalse(args.get(3).contains("\"tok\""));
+ Assertions.assertTrue(args.get(3).contains("http://localhost";));
+
Assertions.assertTrue(args.get(3).contains(HiddenPropertyMaskUtils.MASKED_VALUE));
+ Assertions.assertEquals("--spark-conf", args.get(4));
+ Assertions.assertFalse(args.get(5).contains("jdbc-pw"));
+ Assertions.assertTrue(args.get(5).contains("1g"));
+ Assertions.assertEquals("--table", args.get(6));
+ Assertions.assertEquals("db.t", args.get(7));
+ // Map fields are outside this fix.
+ Assertions.assertEquals("env-secret",
spark.environments().get("AUTH_PASSWORD"));
+ Assertions.assertEquals("s3-secret",
spark.configs().get("spark.hadoop.fs.s3a.secret.key"));
+ }
+
+ @Test
+ void testMaskSensitiveArgumentsLeavesEmptyTokenUnchanged() {
+ SparkJobTemplateDTO input =
+ sparkWithArguments(ImmutableList.of("--spark-conf", "",
"--updater-options", " "));
+
+ SparkJobTemplateDTO spark =
+ (SparkJobTemplateDTO)
JobRuntimeTemplateMaskUtils.maskSensitiveFields(input);
+ Assertions.assertEquals(
+ ImmutableList.of("--spark-conf", "", "--updater-options", " "),
spark.arguments());
+ }
+
+ @Test
+ void testMaskSensitiveArgumentsSkipsMissingValue() {
+ SparkJobTemplateDTO input =
+ sparkWithArguments(ImmutableList.of("--spark-conf", "--catalog",
"ice"));
+
+ SparkJobTemplateDTO spark =
+ (SparkJobTemplateDTO)
JobRuntimeTemplateMaskUtils.maskSensitiveFields(input);
+ Assertions.assertEquals(
+ ImmutableList.of("--spark-conf", "--catalog", "ice"),
spark.arguments());
+ }
+
+ @Test
+ void testMaskSensitiveArgumentsMasksEqualsForm() {
+ SparkJobTemplateDTO input =
+ sparkWithArguments(
+ ImmutableList.of(
+
"--spark-conf={\"jdbc.password\":\"arg-jdbc\",\"spark.executor.memory\":\"1g\"}",
+ "--updater-options="));
+
+ SparkJobTemplateDTO spark =
+ (SparkJobTemplateDTO)
JobRuntimeTemplateMaskUtils.maskSensitiveFields(input);
+ List<String> args = spark.arguments();
+ Assertions.assertTrue(args.get(0).startsWith("--spark-conf="));
+ Assertions.assertFalse(args.get(0).contains("arg-jdbc"));
+ Assertions.assertTrue(args.get(0).contains("1g"));
+ Assertions.assertEquals("--updater-options=", args.get(1));
+ }
+
+ @Test
+ void testMaskSensitiveArgumentsLeavesOtherFlagsUnchanged() {
+ String where = "account_id = 5 and dt = '2024-01-01'";
+ String command = "export db_password=pd && run.sh";
+ ShellJobTemplateDTO input =
+ ShellJobTemplateDTO.builder()
+ .withName("shell_job")
+ .withJobType(JobType.SHELL)
+ .withExecutable("/bin/sh")
+ .withArguments(
+ ImmutableList.of(
+ "-c",
+ command,
+ "--password",
+ "my secret pass",
+ "--conf",
+ "spark.jdbc.password=my pass",
+ "--where",
+ where))
+ .build();
+
+ ShellJobTemplateDTO shell =
+ (ShellJobTemplateDTO)
JobRuntimeTemplateMaskUtils.maskSensitiveFields(input);
+ Assertions.assertEquals(
+ ImmutableList.of(
+ "-c",
+ command,
+ "--password",
+ "my secret pass",
+ "--conf",
+ "spark.jdbc.password=my pass",
+ "--where",
+ where),
+ shell.arguments());
+ }
+
+ @Test
+ void testMaskSensitiveFieldsNull() {
+
Assertions.assertNull(JobRuntimeTemplateMaskUtils.maskSensitiveFields(null));
+ }
+
+ @Test
+ void testMaskSensitiveFieldsCarriesAllSparkDtoFields() {
+ AuditDTO audit = AuditDTO.builder().withCreator("alice").build();
+ SparkJobTemplateDTO input =
+ SparkJobTemplateDTO.builder()
+ .withName("spark_job")
+ .withComment("keep-comment")
+ .withJobType(JobType.SPARK)
+ .withExecutable("local:///app.jar")
+ .withArguments(
+ ImmutableList.of("--updater-options", "{\"password\":\"pw\"}",
"--table", "db.t"))
+ .withEnvironments(ImmutableMap.of("K", "v"))
+ .withCustomFields(ImmutableMap.of("cf", "1"))
+ .withClassName("org.example.App")
+ .withJars(ImmutableList.of("a.jar"))
+ .withFiles(ImmutableList.of("f.txt"))
+ .withArchives(ImmutableList.of("a.zip"))
+ .withConfigs(ImmutableMap.of("spark.executor.memory", "1g"))
+ .withAudit(audit)
+ .build();
+
+ SparkJobTemplateDTO spark =
+ (SparkJobTemplateDTO)
JobRuntimeTemplateMaskUtils.maskSensitiveFields(input);
+
+ Assertions.assertEquals(input.name(), spark.name());
+ Assertions.assertEquals(input.comment(), spark.comment());
+ Assertions.assertEquals(input.jobType(), spark.jobType());
+ Assertions.assertEquals(input.executable(), spark.executable());
+ Assertions.assertEquals(input.environments(), spark.environments());
+ Assertions.assertEquals(input.customFields(), spark.customFields());
+ Assertions.assertEquals(input.className(), spark.className());
+ Assertions.assertEquals(input.jars(), spark.jars());
+ Assertions.assertEquals(input.files(), spark.files());
+ Assertions.assertEquals(input.archives(), spark.archives());
+ Assertions.assertEquals(input.configs(), spark.configs());
+ Assertions.assertEquals(input.audit(), spark.audit());
+ Assertions.assertNotEquals(input.arguments(), spark.arguments());
+ Assertions.assertFalse(spark.arguments().get(1).contains("pw"));
+ }
+
+ @Test
+ void testMaskSensitiveFieldsCarriesAllShellDtoFields() {
+ AuditDTO audit = AuditDTO.builder().withCreator("bob").build();
+ ShellJobTemplateDTO input =
+ ShellJobTemplateDTO.builder()
+ .withName("shell_job")
+ .withComment("shell-comment")
+ .withJobType(JobType.SHELL)
+ .withExecutable("/bin/sh")
+ .withArguments(ImmutableList.of("-c", "echo ok"))
+ .withEnvironments(ImmutableMap.of("HOME", "/tmp"))
+ .withCustomFields(ImmutableMap.of("owner", "tms"))
+ .withScripts(ImmutableList.of("a.sh"))
+ .withAudit(audit)
+ .build();
+
+ ShellJobTemplateDTO shell =
+ (ShellJobTemplateDTO)
JobRuntimeTemplateMaskUtils.maskSensitiveFields(input);
+
+ Assertions.assertEquals(input.name(), shell.name());
+ Assertions.assertEquals(input.comment(), shell.comment());
+ Assertions.assertEquals(input.jobType(), shell.jobType());
+ Assertions.assertEquals(input.executable(), shell.executable());
+ Assertions.assertEquals(input.arguments(), shell.arguments());
+ Assertions.assertEquals(input.environments(), shell.environments());
+ Assertions.assertEquals(input.customFields(), shell.customFields());
+ Assertions.assertEquals(input.scripts(), shell.scripts());
+ Assertions.assertEquals(input.audit(), shell.audit());
+ }
+
+ @Test
+ void testMaskSensitiveJobConfMasksUpdaterOptionsAndSparkConf() {
+ Map<String, String> jobConf = new LinkedHashMap<>();
+ jobConf.put("catalog_name", "ice");
+ jobConf.put(
+ "updater_options",
+
"{\"gravitino_uri\":\"http://localhost\",\"password\":\"arg-pw\",\"oauth_token\":\"tok\"}";);
+ jobConf.put(
+ "spark_conf",
+ "{\"spark.executor.memory\":\"1g\","
+ +
"\"spark.sql.catalog.ice.rest.auth.basic.password\":\"jdbc-pw\"}");
+ jobConf.put("AUTH_PASSWORD", "env-secret");
+
+ Map<String, String> masked =
JobRuntimeTemplateMaskUtils.maskSensitiveJobConf(jobConf);
+
+ Assertions.assertEquals("ice", masked.get("catalog_name"));
+ Assertions.assertEquals("env-secret", masked.get("AUTH_PASSWORD"));
+ Assertions.assertFalse(masked.get("updater_options").contains("arg-pw"));
+ Assertions.assertFalse(masked.get("updater_options").contains("\"tok\""));
+
Assertions.assertTrue(masked.get("updater_options").contains("http://localhost";));
+ Assertions.assertFalse(masked.get("spark_conf").contains("jdbc-pw"));
+ Assertions.assertTrue(masked.get("spark_conf").contains("1g"));
+ Assertions.assertTrue(jobConf.get("updater_options").contains("arg-pw"));
+
Assertions.assertNull(JobRuntimeTemplateMaskUtils.maskSensitiveJobConf(null));
+ }
+
+ private static SparkJobTemplateDTO sparkWithArguments(List<String>
arguments) {
+ return SparkJobTemplateDTO.builder()
+ .withName("spark_job")
+ .withJobType(JobType.SPARK)
+ .withExecutable("local:///app.jar")
+ .withArguments(arguments)
+ .withClassName("org.example.App")
+ .build();
+ }
+}
diff --git
a/core/src/test/java/org/apache/gravitino/listener/api/event/TestJobEventDispatcher.java
b/core/src/test/java/org/apache/gravitino/listener/api/event/TestJobEventDispatcher.java
index e43b85dba6..11029484d7 100644
---
a/core/src/test/java/org/apache/gravitino/listener/api/event/TestJobEventDispatcher.java
+++
b/core/src/test/java/org/apache/gravitino/listener/api/event/TestJobEventDispatcher.java
@@ -21,9 +21,12 @@ package org.apache.gravitino.listener.api.event;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import com.google.common.collect.ImmutableMap;
import java.time.Instant;
import java.util.Collections;
import java.util.Map;
@@ -466,6 +469,57 @@ public class TestJobEventDispatcher {
Assertions.assertEquals(jobConf, eventJobConf);
}
+ @Test
+ void testRunJobEventMasksSensitiveJobConf() throws
NoSuchJobTemplateException {
+ DummyEventListener listener = new DummyEventListener();
+ EventBus eventBus = new EventBus(Collections.singletonList(listener));
+ JobOperationDispatcher delegate = mock(JobOperationDispatcher.class);
+ when(delegate.runJob(any(String.class), any(String.class), any(Map.class)))
+ .thenReturn(jobEntity);
+ JobEventDispatcher maskingDispatcher = new JobEventDispatcher(eventBus,
delegate);
+ Map<String, String> conf =
+ ImmutableMap.of(
+ "catalog_name",
+ "ice",
+ "updater_options",
+ "{\"password\":\"arg-pw\",\"gravitino_uri\":\"http://localhost\"}";,
+ "spark_conf",
+
"{\"spark.sql.catalog.ice.rest.auth.basic.password\":\"jdbc-pw\"}");
+
+ String templateName = jobTemplateEntity.name();
+ maskingDispatcher.runJob("metalake", templateName, conf);
+
+ verify(delegate).runJob(eq("metalake"), eq(templateName), eq(conf));
+ Map<String, String> preJobConf = ((RunJobPreEvent)
listener.popPreEvent()).jobConf();
+ Map<String, String> postJobConf = ((RunJobEvent)
listener.popPostEvent()).jobConf();
+ Assertions.assertEquals("ice", preJobConf.get("catalog_name"));
+
Assertions.assertFalse(preJobConf.get("updater_options").contains("arg-pw"));
+ Assertions.assertFalse(preJobConf.get("spark_conf").contains("jdbc-pw"));
+
Assertions.assertFalse(postJobConf.get("updater_options").contains("arg-pw"));
+ Assertions.assertTrue(conf.get("updater_options").contains("arg-pw"));
+ }
+
+ @Test
+ void testRunJobFailureEventMasksSensitiveJobConf() {
+ DummyEventListener listener = new DummyEventListener();
+ EventBus eventBus = new EventBus(Collections.singletonList(listener));
+ JobOperationDispatcher delegate = mock(JobOperationDispatcher.class);
+ when(delegate.runJob(any(String.class), any(String.class), any(Map.class)))
+ .thenThrow(new GravitinoRuntimeException("failed"));
+ JobEventDispatcher maskingDispatcher = new JobEventDispatcher(eventBus,
delegate);
+ Map<String, String> conf =
+ ImmutableMap.of("updater_options", "{\"password\":\"arg-pw\"}",
"catalog_name", "ice");
+
+ Assertions.assertThrowsExactly(
+ GravitinoRuntimeException.class,
+ () -> maskingDispatcher.runJob("metalake", jobTemplateEntity.name(),
conf));
+
+ Map<String, String> eventJobConf = ((RunJobFailureEvent)
listener.popPostEvent()).jobConf();
+ Assertions.assertEquals("ice", eventJobConf.get("catalog_name"));
+
Assertions.assertFalse(eventJobConf.get("updater_options").contains("arg-pw"));
+ Assertions.assertTrue(conf.get("updater_options").contains("arg-pw"));
+ }
+
@Test
void testCancelJobFailureEvent() {
Assertions.assertThrowsExactly(
diff --git
a/core/src/test/java/org/apache/gravitino/listener/api/info/TestJobInfo.java
b/core/src/test/java/org/apache/gravitino/listener/api/info/TestJobInfo.java
index ef16fe4de9..18ab884695 100644
--- a/core/src/test/java/org/apache/gravitino/listener/api/info/TestJobInfo.java
+++ b/core/src/test/java/org/apache/gravitino/listener/api/info/TestJobInfo.java
@@ -21,10 +21,12 @@ package org.apache.gravitino.listener.api.info;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import java.time.Instant;
+import org.apache.gravitino.connector.HiddenPropertyMaskUtils;
import org.apache.gravitino.dto.util.DTOConverters;
import org.apache.gravitino.job.JobHandle;
import org.apache.gravitino.job.JobTemplate;
import org.apache.gravitino.job.ShellJobTemplate;
+import org.apache.gravitino.job.SparkJobTemplate;
import org.apache.gravitino.json.JsonUtils;
import org.apache.gravitino.meta.AuditInfo;
import org.apache.gravitino.meta.JobEntity;
@@ -140,6 +142,53 @@ public class TestJobInfo {
Assertions.assertEquals(resolvedTemplate, jobInfo.runtimeJobTemplate());
}
+ @Test
+ public void testFromJobEntityMasksSensitiveRuntimeJobTemplateFields() throws
Exception {
+ JobTemplate resolvedTemplate =
+ SparkJobTemplate.builder()
+ .withName("test-job-template")
+ .withComment("resolved")
+ .withExecutable("local:///app.jar")
+ .withClassName("org.example.App")
+ .withArguments(
+ Lists.newArrayList(
+ "--updater-options",
+
"{\"gravitino_uri\":\"http://localhost\",\"password\":\"nested-pw\"}";,
+ "--spark-conf",
+
"{\"spark.executor.memory\":\"1g\",\"jdbc.password\":\"jdbc-pw\"}"))
+ .withEnvironments(ImmutableMap.of("AUTH_PASSWORD", "env-secret",
"CATALOG", "iceberg"))
+ .build();
+ AuditInfo auditInfo =
+
AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build();
+ String runtimeJobTemplateJson =
+ JsonUtils.anyFieldMapper()
+ .writeValueAsString(
+ DTOConverters.toDTO(resolvedTemplate,
DTOConverters.toDTO(auditInfo)));
+
+ JobEntity jobEntity =
+ JobEntity.builder()
+ .withId(1L)
+ .withJobExecutionId("job-execution-1")
+ .withJobTemplateName("test-job-template")
+ .withStatus(JobHandle.Status.QUEUED)
+ .withNamespace(NamespaceUtil.ofJob("test"))
+ .withAuditInfo(auditInfo)
+ .withStartedAt(0L)
+ .withFinishedAt(0L)
+ .withRuntimeJobTemplate(runtimeJobTemplateJson)
+ .build();
+
+ JobInfo jobInfo = JobInfo.fromJobEntity(jobEntity);
+ JobTemplate masked = jobInfo.runtimeJobTemplate();
+ Assertions.assertNotNull(masked);
+ Assertions.assertEquals("env-secret",
masked.environments().get("AUTH_PASSWORD"));
+ Assertions.assertFalse(masked.arguments().get(1).contains("nested-pw"));
+
Assertions.assertTrue(masked.arguments().get(1).contains(HiddenPropertyMaskUtils.MASKED_VALUE));
+ Assertions.assertFalse(masked.arguments().get(3).contains("jdbc-pw"));
+ Assertions.assertTrue(masked.arguments().get(3).contains("1g"));
+
Assertions.assertTrue(jobEntity.runtimeJobTemplate().contains("nested-pw"));
+ }
+
@Test
public void testFromJobEntityWithMalformedRuntimeJobTemplateDoesNotThrow() {
// fromJobEntity() is called inside JobEventDispatcher's try block for
getJob/runJob/
diff --git
a/server/src/main/java/org/apache/gravitino/server/web/rest/JobOperations.java
b/server/src/main/java/org/apache/gravitino/server/web/rest/JobOperations.java
index 4a185ab4f6..d040cf97c5 100644
---
a/server/src/main/java/org/apache/gravitino/server/web/rest/JobOperations.java
+++
b/server/src/main/java/org/apache/gravitino/server/web/rest/JobOperations.java
@@ -68,6 +68,7 @@ import org.apache.gravitino.dto.responses.NameListResponse;
import org.apache.gravitino.dto.util.DTOConverters;
import org.apache.gravitino.job.JobHandle;
import org.apache.gravitino.job.JobOperationDispatcher;
+import org.apache.gravitino.job.JobRuntimeTemplateMaskUtils;
import org.apache.gravitino.job.JobTemplateChange;
import org.apache.gravitino.meta.AuditInfo;
import org.apache.gravitino.meta.JobEntity;
@@ -560,11 +561,18 @@ public class JobOperations {
* make the job unreadable or uncancellable through the API - failures here
are logged and
* swallowed rather than propagated, so callers of {@link #toDTO(JobEntity)}
(get/run/cancel/list
* job) always get a usable response with just the runtime job template
omitted.
+ *
+ * <p>For a Spark runtime template, sensitive-named keys inside {@code
--updater-options} and
+ * {@code --spark-conf} JSON arguments are masked before the DTO is returned
via {@link
+ * JobRuntimeTemplateMaskUtils}. Other arguments and map fields stay as
stored. This DTO has no
+ * {@code jobConf}; event {@code jobConf} is masked separately in the job
event dispatcher. Stored
+ * plaintext is unchanged.
*/
private static JobTemplateDTO toRuntimeJobTemplateDTO(JobEntity jobEntity) {
try {
- return DTOConverters.fromRuntimeJobTemplateJson(
- jobEntity.runtimeJobTemplate(), jobEntity.name());
+ return JobRuntimeTemplateMaskUtils.maskSensitiveFields(
+ DTOConverters.fromRuntimeJobTemplateJson(
+ jobEntity.runtimeJobTemplate(), jobEntity.name()));
} catch (Exception e) {
LOG.warn(
"Failed to deserialize the runtime job template for job {}, omitting
it from the "
diff --git
a/server/src/test/java/org/apache/gravitino/server/web/rest/TestJobOperations.java
b/server/src/test/java/org/apache/gravitino/server/web/rest/TestJobOperations.java
index da19b642fa..04f057ab5d 100644
---
a/server/src/test/java/org/apache/gravitino/server/web/rest/TestJobOperations.java
+++
b/server/src/test/java/org/apache/gravitino/server/web/rest/TestJobOperations.java
@@ -51,10 +51,12 @@ import javax.ws.rs.core.Response;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.gravitino.Config;
import org.apache.gravitino.GravitinoEnv;
+import org.apache.gravitino.connector.HiddenPropertyMaskUtils;
import org.apache.gravitino.dto.job.JobDTO;
import org.apache.gravitino.dto.job.JobTemplateDTO;
import org.apache.gravitino.dto.job.ShellJobTemplateDTO;
import org.apache.gravitino.dto.job.ShellTemplateUpdateDTO;
+import org.apache.gravitino.dto.job.SparkJobTemplateDTO;
import org.apache.gravitino.dto.requests.JobRunRequest;
import org.apache.gravitino.dto.requests.JobTemplateRegisterRequest;
import org.apache.gravitino.dto.requests.JobTemplateUpdateRequest;
@@ -1465,6 +1467,57 @@ public class TestJobOperations extends JerseyTest {
Assertions.assertEquals(Lists.newArrayList("resolved-arg"),
runtimeJobTemplateDTO.arguments());
}
+ @Test
+ public void testToDTORuntimeJobTemplateMasksSensitiveFields() {
+ String runtimeJobTemplateJson =
+ "{"
+ + "\"jobType\":\"spark\","
+ + "\"name\":\"spark_template_1\","
+ + "\"comment\":\"resolved\","
+ + "\"executable\":\"local:///app.jar\","
+ + "\"arguments\":["
+ + "\"--updater-options\","
+ +
"\"{\\\"gravitino_uri\\\":\\\"http://localhost\\\",\\\"password\\\":\\\"nested-pw\\\"}\",";
+ + "\"--spark-conf\","
+ +
"\"{\\\"spark.executor.memory\\\":\\\"1g\\\",\\\"jdbc.password\\\":\\\"jdbc-pw\\\"}\""
+ + "],"
+ +
"\"environments\":{\"AUTH_PASSWORD\":\"env-secret\",\"CATALOG\":\"ice\"},"
+ + "\"className\":\"org.example.App\","
+ + "\"configs\":{\"spark.hadoop.fs.s3a.secret.key\":\"s3-secret\"}"
+ + "}";
+ JobEntity jobWithSecrets =
+ JobEntity.builder()
+ .withId(new Random().nextLong())
+ .withJobExecutionId("job-execution-with-secrets")
+ .withNamespace(NamespaceUtil.ofJob(metalake))
+ .withJobTemplateName("spark_template_1")
+ .withStatus(JobHandle.Status.QUEUED)
+ .withStartedAt(0L)
+ .withFinishedAt(0L)
+ .withRuntimeJobTemplate(runtimeJobTemplateJson)
+ .withAuditInfo(
+
AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build())
+ .build();
+
+ JobDTO jobDTO = JobOperations.toDTO(jobWithSecrets);
+ Assertions.assertInstanceOf(SparkJobTemplateDTO.class,
jobDTO.runtimeJobTemplate());
+ SparkJobTemplateDTO spark = (SparkJobTemplateDTO)
jobDTO.runtimeJobTemplate();
+
+ Assertions.assertEquals("env-secret",
spark.environments().get("AUTH_PASSWORD"));
+ Assertions.assertEquals("s3-secret",
spark.configs().get("spark.hadoop.fs.s3a.secret.key"));
+ Assertions.assertEquals("--updater-options", spark.arguments().get(0));
+ Assertions.assertFalse(spark.arguments().get(1).contains("nested-pw"));
+
Assertions.assertTrue(spark.arguments().get(1).contains("http://localhost";));
+
Assertions.assertTrue(spark.arguments().get(1).contains(HiddenPropertyMaskUtils.MASKED_VALUE));
+ Assertions.assertEquals("--spark-conf", spark.arguments().get(2));
+ Assertions.assertFalse(spark.arguments().get(3).contains("jdbc-pw"));
+ Assertions.assertTrue(spark.arguments().get(3).contains("1g"));
+
+ // Stored entity JSON must remain unmasked (masking is response-only).
+
Assertions.assertTrue(jobWithSecrets.runtimeJobTemplate().contains("nested-pw"));
+
Assertions.assertTrue(jobWithSecrets.runtimeJobTemplate().contains("jdbc-pw"));
+ }
+
@Test
public void
testListJobsWithMalformedRuntimeJobTemplateDoesNotFailWholeList() {
// A single job whose stored runtime job template fails to deserialize
(e.g. corrupted or
