This is an automated email from the ASF dual-hosted git repository.
stankiewicz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 7cc35440762 Resolve inline Secret placeholders in OpenTelemetry
properties (#40579)
7cc35440762 is described below
commit 7cc354407622964509dada9612267dc5050e8fab
Author: Radosław Stankiewicz <[email protected]>
AuthorDate: Thu Oct 15 20:15:17 2026 +0200
Resolve inline Secret placeholders in OpenTelemetry properties (#40579)
---
.../apache/beam/sdk/options/SdkHarnessOptions.java | 46 ++++++++++++-
.../beam/sdk/options/SdkHarnessOptionsTest.java | 79 ++++++++++++++++++++++
2 files changed, 122 insertions(+), 3 deletions(-)
diff --git
a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/SdkHarnessOptions.java
b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/SdkHarnessOptions.java
index 0fcd770c1d1..bd1724688ea 100644
---
a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/SdkHarnessOptions.java
+++
b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/SdkHarnessOptions.java
@@ -34,7 +34,10 @@ import java.util.UUID;
import java.util.logging.Level;
import java.util.logging.LogManager;
import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.beam.sdk.util.InstanceBuilder;
+import org.apache.beam.sdk.util.Secret;
import
org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting;
import
org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap;
import org.checkerframework.checker.index.qual.NonNegative;
@@ -467,11 +470,19 @@ public interface SdkHarnessOptions extends
PipelineOptions, MemoryMonitorOptions
/**
* The OpenTelemetry properties that will be appended to the set of system
properties for SDK
- * harness instances. Property names must be specified without the 'otel.'
prefix.
+ * harness instances.
+ *
+ * <p>Property values may contain inline secret placeholders formatted as
{@code
+ * ${type:<secret_type>;<secret_param>:<value>}} (see {@link
Secret#parseSecretOption(String)}),
+ * which are resolved on the worker before setting system properties. For
example: {@code
+ *
otel.exporter.otlp.metrics.headers=api-key=${type:GcpSecret;version_name:projects/my-project/secrets/my-secret/versions/latest}}.
+ * Secret parameter values may contain {@code ':'} (such as ARNs or URLs),
but cannot contain
+ * {@code ';'} or {@code '}'}.
*/
@Description(
"The OpenTelemetry properties that will be appended to the set of system
properties for SDK "
- + "harness instances. Property names must be specified without the
'otel.' prefix.")
+ + "harness instances. Property values may reference secrets using "
+ + "${type:<secret_type>;<secret_param>:<value>} (parameter values
cannot contain ';' or '}').")
Map<String, String> getOpenTelemetryProperties();
void setOpenTelemetryProperties(Map<String, String> value);
@@ -483,7 +494,7 @@ public interface SdkHarnessOptions extends PipelineOptions,
MemoryMonitorOptions
openTelemetryProperties.forEach(
(k, v) -> {
if (k != null && v != null) {
- System.setProperty(k, v);
+ System.setProperty(k, resolveOpenTelemetryPropertyValue(v));
}
});
if (System.getProperty("otel.service.instance.id") == null) {
@@ -499,6 +510,33 @@ public interface SdkHarnessOptions extends
PipelineOptions, MemoryMonitorOptions
}
}
+ /**
+ * Resolves any inline {@code ${...}} secret placeholders in an
OpenTelemetry property value using
+ * {@link Secret#parseSecretOption(String)}.
+ */
+ @VisibleForTesting
+ static String resolveOpenTelemetryPropertyValue(String value) {
+ Matcher matcher =
GlobalOpenTelemetryFactory.SECRET_PLACEHOLDER_PATTERN.matcher(value);
+ if (!matcher.find()) {
+ return value;
+ }
+ // Use StringBuffer for Java 8 compatibility with
Matcher#appendReplacement.
+ StringBuffer resolved = new StringBuffer();
+ do {
+ String secretOption = checkStateNotNull(matcher.group(1));
+ String secretValue =
Secret.parseSecretOption(secretOption).getString(true);
+ if (secretValue == null) {
+ throw new IllegalStateException(
+ "Resolved secret value cannot be null for secret option: " +
secretOption);
+ }
+ // Trim surrounding whitespace/newlines commonly stored in Secret
Manager payloads and quote
+ // regex replacement metacharacters ('$' and '\') so they are not
interpreted as group refs.
+ matcher.appendReplacement(resolved,
Matcher.quoteReplacement(secretValue.trim()));
+ } while (matcher.find());
+ matcher.appendTail(resolved);
+ return resolved.toString();
+ }
+
@JsonIgnore
@Hidden
@Default.InstanceFactory(GlobalOpenTelemetryFactory.class)
@@ -507,6 +545,8 @@ public interface SdkHarnessOptions extends PipelineOptions,
MemoryMonitorOptions
void setOpenTelemetry(OpenTelemetry value);
class GlobalOpenTelemetryFactory implements
DefaultValueFactory<OpenTelemetry> {
+ private static final Pattern SECRET_PLACEHOLDER_PATTERN =
Pattern.compile("\\$\\{([^}]+)\\}");
+
@Override
public OpenTelemetry create(PipelineOptions options) {
return GlobalOpenTelemetry.get();
diff --git
a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/SdkHarnessOptionsTest.java
b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/SdkHarnessOptionsTest.java
index 80701773c5f..265189b3f15 100644
---
a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/SdkHarnessOptionsTest.java
+++
b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/SdkHarnessOptionsTest.java
@@ -19,20 +19,26 @@ package org.apache.beam.sdk.options;
import static org.apache.beam.sdk.options.SdkHarnessOptions.LogLevel.WARN;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThrows;
import com.fasterxml.jackson.databind.ObjectMapper;
import
org.apache.beam.sdk.options.SdkHarnessOptions.DefaultMaxCacheMemoryUsageMb;
import
org.apache.beam.sdk.options.SdkHarnessOptions.SdkHarnessLogLevelOverrides;
+import org.apache.beam.sdk.testing.RestoreSystemProperties;
import org.apache.beam.sdk.util.common.ReflectHelpers;
import
org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TestRule;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Tests for {@link SdkHarnessOptions}. */
@RunWith(JUnit4.class)
public class SdkHarnessOptionsTest {
+ @Rule public TestRule restoreSystemProperties = new
RestoreSystemProperties();
+
private static final ObjectMapper MAPPER =
new ObjectMapper()
.registerModules(ObjectMapper.findModules(ReflectHelpers.findClassLoader()));
@@ -112,4 +118,77 @@ public class SdkHarnessOptionsTest {
testValue,
MAPPER.writeValueAsString(MAPPER.readValue(testValue,
SdkHarnessLogLevelOverrides.class)));
}
+
+ @Test
+ public void testResolveOpenTelemetryPropertyValuePlain() {
+ assertEquals(
+ "https://otlp.example.com/v1/metrics";,
+
SdkHarnessOptions.resolveOpenTelemetryPropertyValue("https://otlp.example.com/v1/metrics";));
+ }
+
+ @Test
+ public void testResolveOpenTelemetryPropertyValueWithSecretPlaceholder() {
+ assertEquals(
+ "api-key=my-secret-api-key,env=prod",
+ SdkHarnessOptions.resolveOpenTelemetryPropertyValue(
+ "api-key=${type:TestSecret;secret:my-secret-api-key},env=prod"));
+ assertEquals(
+ "api-key=my-secret-api-key,env=prod",
+ SdkHarnessOptions.resolveOpenTelemetryPropertyValue(
+ "api-key=${secret:my-secret-api-key;type:TestSecret},env=prod"));
+ assertEquals(
+ "endpoint=https://otlp.example.com:4318";,
+ SdkHarnessOptions.resolveOpenTelemetryPropertyValue(
+
"endpoint=${type:TestSecret;secret:https://otlp.example.com:4318}";));
+ }
+
+ @Test
+ public void
testResolveOpenTelemetryPropertyValueTrimsAndQuotesSpecialChars() {
+ assertEquals(
+ "api-key=sec$1\\path,token=tok2",
+ SdkHarnessOptions.resolveOpenTelemetryPropertyValue(
+ "api-key=${type:TestSecret;secret:
sec$1\\path\n},token=${type:testsecret;secret:tok2\r\n}"));
+ }
+
+ @Test
+ public void testResolveOpenTelemetryPropertyValueWithInvalidSecretThrows() {
+ assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ SdkHarnessOptions.resolveOpenTelemetryPropertyValue(
+ "api-key=${type:UnknownSecretType;secret:foo}"));
+ assertThrows(
+ IllegalArgumentException.class,
+ () ->
SdkHarnessOptions.resolveOpenTelemetryPropertyValue("api-key=${secret:foo}"));
+ }
+
+ @Test
+ public void
testConfigureOpenTelemetrySetsResolvedPropertiesAndDoesNotMutateOptions() {
+ System.clearProperty("otel.exporter.otlp.metrics.headers");
+ System.clearProperty("otel.service.instance.id");
+
+ SdkHarnessOptions options =
PipelineOptionsFactory.as(SdkHarnessOptions.class);
+ String rawHeaderValue = "api-key=${type:TestSecret;secret:secret-api-key}";
+ options.setOpenTelemetryProperties(
+ ImmutableMap.of("otel.exporter.otlp.metrics.headers", rawHeaderValue));
+
+ SdkHarnessOptions.configureOpenTelemetry(options);
+
+ assertEquals(
+ "api-key=secret-api-key",
System.getProperty("otel.exporter.otlp.metrics.headers"));
+ assertNotNull(System.getProperty("otel.service.instance.id"));
+ assertEquals(
+ rawHeaderValue,
+
options.getOpenTelemetryProperties().get("otel.exporter.otlp.metrics.headers"));
+ }
+
+ @Test
+ public void testConfigureOpenTelemetryWhenEmptyDisablesAuthTargetSignals() {
+ System.clearProperty("google.otel.auth.target.signals");
+ SdkHarnessOptions options =
PipelineOptionsFactory.as(SdkHarnessOptions.class);
+
+ SdkHarnessOptions.configureOpenTelemetry(options);
+
+ assertEquals("none",
System.getProperty("google.otel.auth.target.signals"));
+ }
}
