This is an automated email from the ASF dual-hosted git repository.
david-streamlio pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new a0f0f054640e [improve][misc] Replace MD5 NAR/archive checksums with
SHA-256 (#26152)
a0f0f054640e is described below
commit a0f0f054640e40190b1622857ef7b1e5a8deaa38
Author: David Kjerrumgaard <[email protected]>
AuthorDate: Thu Jul 9 07:52:39 2026 -0700
[improve][misc] Replace MD5 NAR/archive checksums with SHA-256 (#26152)
---
.../java/org/apache/pulsar/common/nar/FileUtils.java | 20 +++++++++++---------
.../org/apache/pulsar/common/nar/NarUnpacker.java | 8 ++++----
.../functions/utils/functions/FunctionArchive.java | 20 ++++++++++----------
.../functions/utils/functions/FunctionUtils.java | 16 ++++++----------
.../apache/pulsar/functions/utils/io/Connector.java | 20 ++++++++++----------
.../pulsar/functions/utils/io/ConnectorUtils.java | 16 ++++++----------
6 files changed, 47 insertions(+), 53 deletions(-)
diff --git
a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/FileUtils.java
b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/FileUtils.java
index a613d9a55f41..3955f9dba648 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/FileUtils.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/FileUtils.java
@@ -47,28 +47,30 @@ public class FileUtils {
public static final long MILLIS_BETWEEN_ATTEMPTS = 50L;
/**
- * Calculates an md5 sum of the specified file.
+ * Calculates a SHA-256 checksum of the specified file. The checksum is
used for
+ * change detection, not as a security control; SHA-256 is chosen over
MD5/SHA-1 so
+ * that the call also works on JVMs whose security provider rejects weak
digests
+ * (e.g. in FIPS deployments).
*
* @param file
- * to calculate the md5sum of
- * @return the md5sum bytes
+ * to calculate the checksum of
+ * @return the checksum bytes
* @throws IOException
* if cannot read file
*/
- public static byte[] calculateMd5sum(final File file) throws IOException {
+ public static byte[] calculateSha256sum(final File file) throws
IOException {
try (final FileInputStream inputStream = new FileInputStream(file)) {
- // codeql[java/weak-cryptographic-algorithm] - md5 is sufficient
for this use case
- final MessageDigest md5 = MessageDigest.getInstance("md5");
+ final MessageDigest digest = MessageDigest.getInstance("SHA-256");
- final byte[] buffer = new byte[1024];
+ final byte[] buffer = new byte[8192];
int read = inputStream.read(buffer);
while (read > -1) {
- md5.update(buffer, 0, read);
+ digest.update(buffer, 0, read);
read = inputStream.read(buffer);
}
- return md5.digest();
+ return digest.digest();
} catch (NoSuchAlgorithmException nsae) {
throw new IllegalArgumentException(nsae);
}
diff --git
a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java
b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java
index 6b71cc0018be..893c1fc19486 100644
--- a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java
@@ -75,9 +75,9 @@ public class NarUnpacker {
throw new IOException("Cannot create " + parentDirectory);
}
}
- String md5Sum =
Base64.getUrlEncoder().withoutPadding().encodeToString(FileUtils.calculateMd5sum(nar));
+ String checksum =
Base64.getUrlEncoder().withoutPadding().encodeToString(FileUtils.calculateSha256sum(nar));
// ensure that one process can extract the files
- File lockFile = new File(parentDirectory, "." + md5Sum + ".lock");
+ File lockFile = new File(parentDirectory, "." + checksum + ".lock");
// prevent OverlappingFileLockException by ensuring that one thread
tries to create a lock in this JVM
Object localLock =
CURRENT_JVM_FILE_LOCKS.computeIfAbsent(lockFile.getAbsolutePath(), key -> new
Object());
synchronized (localLock) {
@@ -85,9 +85,9 @@ public class NarUnpacker {
// using the same lock file don't execute concurrently
try (FileChannel channel = new RandomAccessFile(lockFile,
"rw").getChannel();
FileLock lock = channel.lock()) {
- File narWorkingDirectory = new File(parentDirectory, md5Sum);
+ File narWorkingDirectory = new File(parentDirectory, checksum);
if (!narWorkingDirectory.exists()) {
- File narExtractionTempDirectory = new
File(parentDirectory, md5Sum + ".tmp");
+ File narExtractionTempDirectory = new
File(parentDirectory, checksum + ".tmp");
if (narExtractionTempDirectory.exists()) {
FileUtils.deleteFile(narExtractionTempDirectory, true);
}
diff --git
a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionArchive.java
b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionArchive.java
index 70b452eba39c..95857d4fcd80 100644
---
a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionArchive.java
+++
b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionArchive.java
@@ -27,8 +27,8 @@ import
org.apache.pulsar.functions.utils.ValidatableFunctionPackage;
public class FunctionArchive implements AutoCloseable {
private final Path archivePath;
- /** MD5 hex of archive file contents; empty when {@link #archivePath} is
null (test doubles). */
- private final String archiveMd5Hex;
+ /** SHA-256 hex of archive file contents; empty when {@link #archivePath}
is null (test doubles). */
+ private final String archiveChecksumHex;
private final FunctionDefinition functionDefinition;
private final String narExtractionDirectory;
private final boolean enableClassloading;
@@ -41,25 +41,25 @@ public class FunctionArchive implements AutoCloseable {
}
/**
- * @param precomputedArchiveMd5Hex MD5 hex of {@code archivePath}
contents; if null and path is non-null,
+ * @param precomputedArchiveChecksumHex SHA-256 hex of {@code archivePath}
contents; if null and path is non-null,
* the hash is computed once at
construction time.
*/
public FunctionArchive(Path archivePath, FunctionDefinition
functionDefinition, String narExtractionDirectory,
- boolean enableClassloading, String
precomputedArchiveMd5Hex) {
+ boolean enableClassloading, String
precomputedArchiveChecksumHex) {
this.archivePath = archivePath;
this.functionDefinition = functionDefinition;
this.narExtractionDirectory = narExtractionDirectory;
this.enableClassloading = enableClassloading;
if (archivePath != null) {
try {
- this.archiveMd5Hex = precomputedArchiveMd5Hex != null
- ? precomputedArchiveMd5Hex
- : FunctionUtils.computeArchiveMd5Hex(archivePath);
+ this.archiveChecksumHex = precomputedArchiveChecksumHex != null
+ ? precomputedArchiveChecksumHex
+ : FunctionUtils.computeArchiveChecksumHex(archivePath);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
} else {
- this.archiveMd5Hex = "";
+ this.archiveChecksumHex = "";
}
}
@@ -67,8 +67,8 @@ public class FunctionArchive implements AutoCloseable {
return archivePath;
}
- public String getArchiveMd5Hex() {
- return archiveMd5Hex;
+ public String getArchiveChecksumHex() {
+ return archiveChecksumHex;
}
public synchronized ValidatableFunctionPackage getFunctionPackage() {
diff --git
a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java
b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java
index 0244490a6d03..5789fe174c47 100644
---
a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java
+++
b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java
@@ -47,14 +47,10 @@ public class FunctionUtils {
private static final String PULSAR_IO_SERVICE_NAME = "pulsar-io.yaml";
/**
- * Computes MD5 digest of a file as lower-case hex (for function archive
identity on reload).
+ * Computes a SHA-256 digest of a file as lower-case hex (for function
archive identity on reload).
*/
- public static String computeArchiveMd5Hex(Path path) throws IOException {
- return calculateMd5Hex(path.toAbsolutePath().normalize().toFile());
- }
-
- private static String calculateMd5Hex(File file) throws IOException {
- return HexFormat.of().formatHex(FileUtils.calculateMd5sum(file));
+ public static String computeArchiveChecksumHex(Path path) throws
IOException {
+ return
HexFormat.of().formatHex(FileUtils.calculateSha256sum(path.toAbsolutePath().normalize().toFile()));
}
/**
@@ -161,12 +157,12 @@ public class FunctionUtils {
FunctionDefinition funcDef =
FunctionUtils.getFunctionDefinition(archive.toFile());
if (!StringUtils.isEmpty(funcDef.getFunctionClass())) {
String name = funcDef.getName();
- String md5Hex = computeArchiveMd5Hex(archive);
+ String checksumHex =
computeArchiveChecksumHex(archive);
FunctionArchive prev = remaining.remove(name);
if (prev != null
&& prev.getArchivePath() != null
&& archive.equals(prev.getArchivePath())
- && md5Hex.equals(prev.getArchiveMd5Hex())) {
+ &&
checksumHex.equals(prev.getArchiveChecksumHex())) {
next.put(name, prev);
} else {
if (prev != null) {
@@ -178,7 +174,7 @@ public class FunctionUtils {
toClose.add(prev);
}
next.put(name, new FunctionArchive(archive,
funcDef, narExtractionDirectory,
- enableClassloading, md5Hex));
+ enableClassloading, checksumHex));
}
}
} catch (Throwable t) {
diff --git
a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/Connector.java
b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/Connector.java
index bff477b40cda..5ac21e5dc6fa 100644
---
a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/Connector.java
+++
b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/Connector.java
@@ -28,8 +28,8 @@ import
org.apache.pulsar.functions.utils.ValidatableFunctionPackage;
public class Connector implements AutoCloseable {
private final Path archivePath;
- /** MD5 hex of archive file contents; empty when {@link #archivePath} is
null (test doubles). */
- private final String archiveMd5Hex;
+ /** SHA-256 hex of archive file contents; empty when {@link #archivePath}
is null (test doubles). */
+ private final String archiveChecksumHex;
private final String narExtractionDirectory;
private final boolean enableClassloading;
private ValidatableFunctionPackage connectorFunctionPackage;
@@ -44,25 +44,25 @@ public class Connector implements AutoCloseable {
}
/**
- * @param precomputedArchiveMd5Hex MD5 hex of {@code archivePath}
contents; if null and path is non-null,
+ * @param precomputedArchiveChecksumHex SHA-256 hex of {@code archivePath}
contents; if null and path is non-null,
* the hash is computed once at
construction time.
*/
public Connector(Path archivePath, ConnectorDefinition
connectorDefinition, String narExtractionDirectory,
- boolean enableClassloading, String
precomputedArchiveMd5Hex) {
+ boolean enableClassloading, String
precomputedArchiveChecksumHex) {
this.archivePath = archivePath;
this.connectorDefinition = connectorDefinition;
this.narExtractionDirectory = narExtractionDirectory;
this.enableClassloading = enableClassloading;
if (archivePath != null) {
try {
- this.archiveMd5Hex = precomputedArchiveMd5Hex != null
- ? precomputedArchiveMd5Hex
- : ConnectorUtils.computeArchiveMd5Hex(archivePath);
+ this.archiveChecksumHex = precomputedArchiveChecksumHex != null
+ ? precomputedArchiveChecksumHex
+ :
ConnectorUtils.computeArchiveChecksumHex(archivePath);
} catch (java.io.IOException e) {
throw new java.io.UncheckedIOException(e);
}
} else {
- this.archiveMd5Hex = "";
+ this.archiveChecksumHex = "";
}
}
@@ -70,8 +70,8 @@ public class Connector implements AutoCloseable {
return archivePath;
}
- public String getArchiveMd5Hex() {
- return archiveMd5Hex;
+ public String getArchiveChecksumHex() {
+ return archiveChecksumHex;
}
public synchronized ValidatableFunctionPackage
getConnectorFunctionPackage() {
diff --git
a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java
b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java
index c263c99088e5..0bf9f87bb741 100644
---
a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java
+++
b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java
@@ -56,14 +56,10 @@ import org.apache.pulsar.io.core.annotations.FieldDoc;
public class ConnectorUtils {
/**
- * Computes MD5 digest of a file as lower-case hex (for connector archive
identity on reload).
+ * Computes a SHA-256 digest of a file as lower-case hex (for connector
archive identity on reload).
*/
- public static String computeArchiveMd5Hex(Path path) throws IOException {
- return calculateMd5Hex(path.toAbsolutePath().normalize().toFile());
- }
-
- private static String calculateMd5Hex(File file) throws IOException {
- return HexFormat.of().formatHex(FileUtils.calculateMd5sum(file));
+ public static String computeArchiveChecksumHex(Path path) throws
IOException {
+ return
HexFormat.of().formatHex(FileUtils.calculateSha256sum(path.toAbsolutePath().normalize().toFile()));
}
/**
@@ -236,12 +232,12 @@ public class ConnectorUtils {
try {
ConnectorDefinition cntDef =
ConnectorUtils.getConnectorDefinition(archive.toFile());
String name = cntDef.getName();
- String md5Hex = computeArchiveMd5Hex(archive);
+ String checksumHex = computeArchiveChecksumHex(archive);
Connector prev = remaining.remove(name);
if (prev != null
&& prev.getArchivePath() != null
&& archive.equals(prev.getArchivePath())
- && md5Hex.equals(prev.getArchiveMd5Hex())) {
+ &&
checksumHex.equals(prev.getArchiveChecksumHex())) {
next.put(name, prev);
} else {
if (prev != null) {
@@ -253,7 +249,7 @@ public class ConnectorUtils {
toClose.add(prev);
}
next.put(name, new Connector(archive, cntDef,
narExtractionDirectory, enableClassloading,
- md5Hex));
+ checksumHex));
}
} catch (Throwable t) {
log.warn()
