This is an automated email from the ASF dual-hosted git repository.

cloud-fan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 5ce57a0ef3dd [SPARK-58504][CORE] Simplify FileTokenIngestor cache and 
detect rotation by content
5ce57a0ef3dd is described below

commit 5ce57a0ef3ddcbdcb4698b0c27edf8abf13cb2de
Author: Wenchen Fan <[email protected]>
AuthorDate: Mon Aug 3 22:24:15 2026 +0800

    [SPARK-58504][CORE] Simplify FileTokenIngestor cache and detect rotation by 
content
    
    ### What changes were proposed in this pull request?
    
    This follow-up to #57262 changes `FileTokenIngestor` to detect token 
rotation by comparing
    file content instead of modification timestamps. It also publishes the 
cached token content and
    parsed `UserContext` together through one immutable volatile snapshot.
    
    The change documents the `TokenIngestor` concurrency and refresh contract, 
removes an unsupported
    claim that every token is re-verified by a downstream STS exchange, and 
fixes two Javadoc issues.
    
    ### Why are the changes needed?
    
    The previous cache used the file modification time to decide whether to 
reuse a parsed token.
    That can miss token replacement when the replacement preserves the 
timestamp or occurs within the
    filesystem's timestamp granularity.
    
    Comparing the small token file's content detects those rotations reliably 
while preserving the
    optimization that unchanged tokens are not reparsed. Publishing the token 
content and parsed
    context as one immutable snapshot also removes the need to reason about the 
write ordering of two
    volatile cache fields.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No. This fixes unreleased OIDC credential propagation code on `master`.
    
    ### How was this patch tested?
    
    Added a regression test that replaces a token while preserving its mtime 
and verifies that the
    new identity is loaded.
    
    ```text
    build/sbt 'core/testOnly *FileTokenIngestorSuite'
    ```
    
    All 14 tests passed.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Codex (GPT-5)
    
    Closes #57706 from cloud-fan/fix-token-ingestor-cache.
    
    Authored-by: Wenchen Fan <[email protected]>
    Signed-off-by: Wenchen Fan <[email protected]>
---
 .../apache/spark/security/FileTokenIngestor.java   | 37 ++++++++++++----------
 .../org/apache/spark/security/TokenIngestor.java   |  5 ++-
 .../spark/security/FileTokenIngestorSuite.java     | 20 ++++++++++++
 3 files changed, 44 insertions(+), 18 deletions(-)

diff --git 
a/core/src/main/java/org/apache/spark/security/FileTokenIngestor.java 
b/core/src/main/java/org/apache/spark/security/FileTokenIngestor.java
index f99e957ad6e7..b51dcc7f0b43 100644
--- a/core/src/main/java/org/apache/spark/security/FileTokenIngestor.java
+++ b/core/src/main/java/org/apache/spark/security/FileTokenIngestor.java
@@ -36,13 +36,13 @@ import org.apache.spark.internal.SparkLoggerFactory;
 /**
  * A {@link TokenIngestor} that reads an OIDC identity token from a file.
  * <p>
- * The file path is typically a Kubernetes projected service account token
+ * The file path typically points to a Kubernetes projected service account 
token
  * (e.g., {@code /var/run/secrets/tokens/spark-identity}) or a path configured 
via
  * {@code spark.security.oidc.identityToken.file}.
  * <p>
  * This implementation:
  * <ul>
- *   <li>Detects file rotation via mtime change (only re-parses when the file 
changes)</li>
+ *   <li>Detects file rotation by comparing file content (only re-parses when 
it changes)</li>
  *   <li>Parses JWT claims by Base64-decoding the payload segment directly, 
without
  *       signature verification, since the token is trusted from the local 
filesystem
  *       and works with both signed (RS256, ES256) and unsigned tokens</li>
@@ -60,11 +60,17 @@ public class FileTokenIngestor implements TokenIngestor {
 
   private final Path tokenPath;
 
-  // Cached state for rotation detection.
-  // Write order matters for thread-safety: cachedContext must be visible 
before
-  // lastMtime, so a concurrent reader never sees a new mtime with a stale 
context.
-  private volatile UserContext cachedContext = null;
-  private volatile long lastMtime = -1L;
+  private volatile CachedToken cachedToken;
+
+  private static final class CachedToken {
+    private final String content;
+    private final UserContext context;
+
+    private CachedToken(String content, UserContext context) {
+      this.content = content;
+      this.context = context;
+    }
+  }
 
   /**
    * Construct a new FileTokenIngestor.
@@ -83,24 +89,22 @@ public class FileTokenIngestor implements TokenIngestor {
         return Optional.empty();
       }
 
-      // File did not change since last successful parse
-      long currentMtime = Files.getLastModifiedTime(tokenPath).toMillis();
-      if (currentMtime == lastMtime && cachedContext != null) {
-        return Optional.of(cachedContext);
-      }
-
       String content = new String(Files.readAllBytes(tokenPath), 
StandardCharsets.UTF_8).trim();
       if (content.isEmpty()) {
         LOG.warn("Token file is empty: {}", MDC.of(LogKeys.PATH, tokenPath));
         return Optional.empty();
       }
 
+      CachedToken currentCache = cachedToken;
+      if (currentCache != null && content.equals(currentCache.content)) {
+        return Optional.of(currentCache.context);
+      }
+
       Optional<UserContext> userContext = parseJwt(content);
       if (userContext.isPresent()) {
         // If the new file has invalid content, return empty and do NOT
         // fall back to the previously cached context. The caller will retry 
on next poll.
-        cachedContext = userContext.get();
-        lastMtime = currentMtime;
+        cachedToken = new CachedToken(content, userContext.get());
       }
       return userContext;
     } catch (Exception e) {
@@ -114,8 +118,7 @@ public class FileTokenIngestor implements TokenIngestor {
   /**
    * Parse a JWT token string into a UserContext by Base64-decoding the 
payload segment.
    * This works with both signed (RS256, ES256) and unsigned (alg:none) tokens 
since
-   * we never verify the signature - the token is trusted from the local 
filesystem and
-   * will be re-verified downstream at the STS token exchange.
+   * we never verify the signature - the token is trusted from the local 
filesystem.
    */
   private Optional<UserContext> parseJwt(String token) {
     try {
diff --git a/core/src/main/java/org/apache/spark/security/TokenIngestor.java 
b/core/src/main/java/org/apache/spark/security/TokenIngestor.java
index ad4a208e663f..0c25fc59d69e 100644
--- a/core/src/main/java/org/apache/spark/security/TokenIngestor.java
+++ b/core/src/main/java/org/apache/spark/security/TokenIngestor.java
@@ -23,10 +23,11 @@ import org.apache.spark.annotation.DeveloperApi;
 
 /**
  * :: DeveloperApi ::
- * Read an OIDC identity token and produces a {@link UserContext}.
+ * Reads an OIDC identity token and produces a {@link UserContext}.
  * <p>
  * Implementation should be stateless with respect to Spark configuration;
  * configuration is passed at construction time.
+ * Implementations must be thread-safe because {@link #load()} may be called 
concurrently.
  *
  * @since 4.3.0
  */
@@ -35,6 +36,8 @@ public interface TokenIngestor {
 
   /**
    * Attempt to load the current identity token and parse it into a 
UserContext.
+   * This method may be called repeatedly. Implementations may cache parsed 
tokens, but must
+   * detect changes to the underlying token source and return the current 
identity.
    *
    * @return a present Optional containing the UserContext if a valid token is 
available,
    * or empty if unavailable (e.g. empty content / missing file).
diff --git 
a/core/src/test/java/org/apache/spark/security/FileTokenIngestorSuite.java 
b/core/src/test/java/org/apache/spark/security/FileTokenIngestorSuite.java
index f6f8ea2c3636..9929e23964b3 100644
--- a/core/src/test/java/org/apache/spark/security/FileTokenIngestorSuite.java
+++ b/core/src/test/java/org/apache/spark/security/FileTokenIngestorSuite.java
@@ -19,6 +19,7 @@ package org.apache.spark.security;
 
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.attribute.FileTime;
 import java.security.KeyPair;
 import java.security.KeyPairGenerator;
 import java.util.Comparator;
@@ -143,6 +144,25 @@ public class FileTokenIngestorSuite {
     assertEquals(token2, result2.get().getRawToken());
   }
 
+  @Test
+  public void loadDetectsFileRotationWhenMtimeIsUnchanged() throws Exception {
+    Path tokenFile = tempDir.resolve("token");
+    String token1 = createUnsignedJwt("user1", "https://issuer.example.com";);
+    writeToken(tokenFile, token1);
+
+    FileTokenIngestor ingestor = new FileTokenIngestor(tokenFile);
+    assertEquals("user1", ingestor.load().get().getPrincipal());
+    FileTime originalMtime = Files.getLastModifiedTime(tokenFile);
+
+    String token2 = createUnsignedJwt("user2", "https://issuer.example.com";);
+    writeToken(tokenFile, token2);
+    Files.setLastModifiedTime(tokenFile, originalMtime);
+
+    Optional<UserContext> result = ingestor.load();
+    assertEquals("user2", result.get().getPrincipal());
+    assertEquals(token2, result.get().getRawToken());
+  }
+
   @Test
   public void loadReturnsEmptyForMissingFile() {
     Path tokenFile = tempDir.resolve("nonexistent");


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to