This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 12a58eb6d9dd Fetch avro version from pom.properties to fix NPE (#19596)
12a58eb6d9dd is described below
commit 12a58eb6d9ddf6d93c03018e99b2fe25667274c3
Author: Trivedhi <[email protected]>
AuthorDate: Fri Aug 14 13:44:28 2026 +0530
Fetch avro version from pom.properties to fix NPE (#19596)
---
.../apache/hudi/common/avro/HoodieAvroUtils.java | 59 ++++++++++++++++++++--
.../hudi/common/avro/TestHoodieAvroUtils.java | 11 ++++
2 files changed, 66 insertions(+), 4 deletions(-)
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroUtils.java
b/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroUtils.java
index 15f76a0fbc7f..de449eab8db1 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/avro/HoodieAvroUtils.java
@@ -38,6 +38,7 @@ import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.SchemaCompatibilityException;
+import lombok.extern.slf4j.Slf4j;
import org.apache.avro.AvroRuntimeException;
import org.apache.avro.Conversions;
import org.apache.avro.Conversions.DecimalConversion;
@@ -69,10 +70,12 @@ import javax.annotation.Nullable;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.math.MathContext;
import java.math.RoundingMode;
+import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.sql.Date;
@@ -89,6 +92,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Deque;
+import java.util.Enumeration;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
@@ -110,9 +114,10 @@ import static
org.apache.hudi.common.util.ValidationUtils.checkState;
/**
* Helper class to do common stuff across Avro.
*/
+@Slf4j
public class HoodieAvroUtils {
- public static final String AVRO_VERSION =
Schema.class.getPackage().getImplementationVersion();
+ public static final String AVRO_VERSION = resolveAvroVersion();
private static final ThreadLocal<BinaryEncoder> BINARY_ENCODER =
ThreadLocal.withInitial(() -> null);
private static final ThreadLocal<BinaryDecoder> BINARY_DECODER =
ThreadLocal.withInitial(() -> null);
@@ -124,6 +129,52 @@ public class HoodieAvroUtils {
private static final Properties PROPERTIES = new Properties();
+ /**
+ * Resolves the Avro library version, preferring Maven's generated
pom.properties over
+ * {@link Package#getImplementationVersion()}. The latter comes from
whatever manifest happens to
+ * seal the package, which is only avro's own manifest when avro ships as a
standalone jar. But once
+ * its classes get merged or relocated into a shaded/fat jar, that lookup
silently returns the
+ * assembling jar's version instead of avro's or nothing at all. So better
to resolve with pom.properties
+ * followed by manifest version.
+ */
+ private static String resolveAvroVersion() {
+ final String path = "META-INF/maven/org.apache.avro/avro/pom.properties";
+ try {
+ URL schemaClassUrl = Schema.class.getResource("Schema.class");
+ String schemaArchive = schemaClassUrl == null ? null :
archiveOf(schemaClassUrl);
+ Enumeration<URL> candidates =
Schema.class.getClassLoader().getResources(path);
+ while (candidates.hasMoreElements()) {
+ URL candidate = candidates.nextElement();
+ // only use the pom.properties that ships in the same archive as the
loaded Schema class
+ if (schemaArchive != null &&
!schemaArchive.equals(archiveOf(candidate))) {
+ continue;
+ }
+ Properties avroProperties = new Properties();
+ try (InputStream in = candidate.openStream()) {
+ avroProperties.load(in);
+ }
+ String version = avroProperties.getProperty("version");
+ if (version != null) {
+ return version;
+ }
+ }
+ } catch (Exception e) {
+ log.warn("Failed to resolve the avro version from {}, falling back to
the jar manifest", path, e);
+ }
+ String manifestVersion = Schema.class.getPackage() == null ? null :
Schema.class.getPackage().getImplementationVersion();
+ if (manifestVersion == null) {
+ log.warn("Could not resolve the avro version from {} nor from the jar
manifest, "
+ + "avro version checks will fall back to pre-1.9 behaviour.
Check that avro jar is on the classpath.", path);
+ }
+ return manifestVersion;
+ }
+
+ private static String archiveOf(URL url) {
+ String s = url.toString();
+ int separatorIdx = s.indexOf("!/");
+ return separatorIdx < 0 ? s : s.substring(0, separatorIdx);
+ }
+
/**
* Convert a given avro record to bytes.
*/
@@ -1531,15 +1582,15 @@ public class HoodieAvroUtils {
}
public static boolean gteqAvro1_9() {
- return StringUtils.compareVersions(AVRO_VERSION, "1.9") >= 0;
+ return AVRO_VERSION != null && StringUtils.compareVersions(AVRO_VERSION,
"1.9") >= 0;
}
public static boolean gteqAvro1_10() {
- return StringUtils.compareVersions(AVRO_VERSION, "1.10") >= 0;
+ return AVRO_VERSION != null && StringUtils.compareVersions(AVRO_VERSION,
"1.10") >= 0;
}
static boolean gteqAvro1_12() {
- return StringUtils.compareVersions(AVRO_VERSION, "1.12") >= 0;
+ return AVRO_VERSION != null && StringUtils.compareVersions(AVRO_VERSION,
"1.12") >= 0;
}
private static Object convertDefaultValueForAvroCompatibility(Object
defaultValue) {
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/avro/TestHoodieAvroUtils.java
b/hudi-common/src/test/java/org/apache/hudi/common/avro/TestHoodieAvroUtils.java
index fd5934616e65..30c726c094a8 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/avro/TestHoodieAvroUtils.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/avro/TestHoodieAvroUtils.java
@@ -124,6 +124,7 @@ import static
org.apache.hudi.common.schema.HoodieSchemaUtils.sanitizeName;
import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -1191,4 +1192,14 @@ public class TestHoodieAvroUtils {
assertEquals(NUM_FIELDS_IN_EXAMPLE_SCHEMA,
stripped.getSchema().getFields().size());
assertEquals("key1", stripped.get("_row_key").toString());
}
+
+ /**
+ * Verifies whether loaded avro version from pom.properties matches with
implementation version from manifest
+ */
+ @Test
+ void testAvroVersionMatchesLoadedAvroJar() {
+ assertNotNull(HoodieAvroUtils.AVRO_VERSION);
+ // the pom.properties lookup must agree with the jar that actually defines
Schema
+ assertEquals(Schema.class.getPackage().getImplementationVersion(),
HoodieAvroUtils.AVRO_VERSION);
+ }
}
