geyanggang commented on code in PR #14532:
URL: https://github.com/apache/gravitino/pull/14532#discussion_r4450924570
##########
build.gradle.kts:
##########
@@ -305,6 +306,95 @@ class LegalFilesTransformer(
}
}
+/**
+ * Removes the Apache Avro copy that hadoop-client-runtime shades under
+ * org/apache/hadoop/shaded/org/apache/avro when it is older than 1.11.4.
+ *
+ * Hadoop 3.3.6 shades Avro 1.7.7, and a dependency constraint cannot replace
classes that live inside
+ * another jar. Within Hadoop the shaded Avro only backs Avro serialization
(SequenceFile, MapReduce job
+ * history, `hadoop fs -text`), which Gravitino does not use; the FileSystem,
HDFS and Kerberos code
+ * paths do not load it. Other jars pass through unchanged. Remove this
transform once Hadoop is
+ * upgraded to a release that shades Avro 1.11.4 or later (Hadoop 3.4.2+).
+ */
+@CacheableTransform
+abstract class StripShadedAvroTransform :
TransformAction<TransformParameters.None> {
+ companion object {
+ val STRIPPED: Attribute<Boolean> =
+ Attribute.of("org.apache.gravitino.shaded-avro-stripped",
Boolean::class.javaObjectType)
+ const val SHADED_AVRO_PREFIX = "org/apache/hadoop/shaded/org/apache/avro/"
+ const val AVRO_METADATA_PREFIX = "META-INF/maven/org.apache.avro/"
+ const val AVRO_POM_PROPERTIES =
"META-INF/maven/org.apache.avro/avro/pom.properties"
+ private val MIN_AVRO_VERSION = listOf(1, 11, 4)
+
+ /** Returns true if the dotted [version] is lower than 1.11.4. */
+ fun isOutdatedAvro(version: String): Boolean {
+ val parts = version.split('.', '-').map { it.toIntOrNull() ?: 0 }
+ for (i in MIN_AVRO_VERSION.indices) {
+ val part = parts.getOrElse(i) { 0 }
+ if (part != MIN_AVRO_VERSION[i]) {
+ return part < MIN_AVRO_VERSION[i]
+ }
+ }
+ return false
+ }
+ }
+
+ @get:InputArtifact
+ @get:PathSensitive(PathSensitivity.NAME_ONLY)
+ abstract val inputArtifact: Provider<FileSystemLocation>
+
+ override fun transform(outputs: TransformOutputs) {
+ val input = inputArtifact.get().asFile
+ if (!input.isFile || !input.name.startsWith("hadoop-client-runtime-") ||
!input.name.endsWith(".jar")) {
+ outputs.file(inputArtifact)
+ return
+ }
+
+ val avroVersion = ZipFile(input).use { jar ->
+ jar.getEntry(AVRO_POM_PROPERTIES)?.let { entry ->
+ Properties().apply { jar.getInputStream(entry).use { load(it) }
}.getProperty("version")
+ }
+ }
+ if (avroVersion == null || !isOutdatedAvro(avroVersion)) {
+ outputs.file(inputArtifact)
+ return
+ }
+
+ // Keep the original file name so that the distribution layout and license
reports do not change.
+ val output = outputs.file(input.name)
+ var removed = 0
+ ZipFile(input).use { jar ->
+ ZipOutputStream(output).use { out ->
+ jar.entries().asSequence().forEach { source ->
+ if (source.name.startsWith(SHADED_AVRO_PREFIX) ||
source.name.startsWith(AVRO_METADATA_PREFIX)) {
+ removed++
+ return@forEach
+ }
+ val entry = ZipEntry(source.name)
+ entry.time = source.time
+ out.putNextEntry(entry)
+ if (!source.isDirectory) {
+ jar.getInputStream(source).use { it.copyTo(out) }
+ }
+ out.closeEntry()
+ }
+ }
+ }
+ if (removed == 0) {
+ throw GradleException("Expected shaded Avro $avroVersion entries in
${input.name}, found none")
+ }
+ }
+}
+
+// Projects whose runtime libraries ship hadoop-client-runtime in the
distribution.
+val shadedAvroStrippedProjects = setOf(
Review Comment:
The fat bundles are separately published artifacts that are also used
client-side (e.g. GVFS in non-Hadoop environments), where stripping Avro could
break user code that relies on Hadoop's Avro/SequenceFile APIs. Since this
issue targets distribution/package, I'd prefer to handle the bundles in a
follow-up issue with its own compatibility check.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
