(gravitino) branch main updated: [#4545] improvement(paimon-catalog): reduce catalog-lakehouse-paimon libs size from 222MB to 75MB (#4547)

[jshao]

This is an automated email from the ASF dual-hosted git repository.

jshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git


The following commit(s) were added to refs/heads/main by this push:
     new 3c20d9506 [#4545] improvement(paimon-catalog): reduce 
catalog-lakehouse-paimon libs size from 222MB to 75MB (#4547)
3c20d9506 is described below

commit 3c20d95065ea5d0139f6413b457221673e1d55ce
Author: Qiang-Liu <liuqiang9...@gmail.com>
AuthorDate: Fri Sep 27 15:49:28 2024 +0800

    [#4545] improvement(paimon-catalog): reduce catalog-lakehouse-paimon libs 
size from 222MB to 75MB (#4547)
    
    ### What changes were proposed in this pull request?
    
    remove some unnecessary dependencies
    
    ### Why are the changes needed?
    
    reduce catalog-lakehouse-paimon libs
    
    Fix: #4545
    
    ### Does this PR introduce _any_ user-facing change?
    
    No
    
    ### How was this patch tested?
    
    CI passed
    
    ---------
    
    Co-authored-by: fanng <xiaoj...@datastrato.com>
---
 catalogs/catalog-lakehouse-paimon/build.gradle.kts | 62 ++++++++++++++++++----
 1 file changed, 53 insertions(+), 9 deletions(-)

diff --git a/catalogs/catalog-lakehouse-paimon/build.gradle.kts 
b/catalogs/catalog-lakehouse-paimon/build.gradle.kts
index 3974fba61..f30fd8880 100644
--- a/catalogs/catalog-lakehouse-paimon/build.gradle.kts
+++ b/catalogs/catalog-lakehouse-paimon/build.gradle.kts
@@ -30,40 +30,80 @@ val sparkMajorVersion: String = 
sparkVersion.substringBeforeLast(".")
 val paimonVersion: String = libs.versions.paimon.get()
 
 dependencies {
-  implementation(project(":api"))
-  implementation(project(":common"))
-  implementation(project(":core"))
+  implementation(project(":api")) {
+    exclude("*")
+  }
+  implementation(project(":common")) {
+    exclude("*")
+  }
+  implementation(project(":core")) {
+    exclude("*")
+  }
   implementation(libs.bundles.paimon) {
     exclude("com.sun.jersey")
     exclude("javax.servlet")
+    exclude("org.apache.curator")
+    exclude("org.apache.hive")
+    exclude("org.apache.hbase")
+    exclude("org.apache.zookeeper")
+    exclude("org.eclipse.jetty.aggregate:jetty-all")
+    exclude("org.mortbay.jetty")
+    exclude("org.mortbay.jetty:jetty")
+    exclude("org.mortbay.jetty:jetty-util")
+    exclude("org.mortbay.jetty:jetty-sslengine")
+    exclude("it.unimi.dsi")
+    exclude("com.ververica")
+    exclude("org.apache.hadoop")
+    exclude("org.apache.commons")
+    exclude("org.xerial.snappy")
+    exclude("com.github.luben")
+    exclude("com.google.protobuf")
+    exclude("joda-time")
+    exclude("org.apache.parquet:parquet-jackson")
+    exclude("org.apache.parquet:parquet-format-structures")
+    exclude("org.apache.parquet:parquet-encoding")
+    exclude("org.apache.parquet:parquet-common")
+    exclude("org.apache.parquet:parquet-hadoop")
+    exclude("org.apache.paimon:paimon-codegen-loader")
+    exclude("org.apache.paimon:paimon-shade-caffeine-2")
+    exclude("org.apache.paimon:paimon-shade-guava-30")
   }
   implementation(libs.bundles.log4j)
   implementation(libs.commons.lang3)
-  implementation(libs.caffeine)
   implementation(libs.guava)
   implementation(libs.hadoop2.common) {
     exclude("com.github.spotbugs")
     exclude("com.sun.jersey")
     exclude("javax.servlet")
+    exclude("org.apache.curator")
+    exclude("org.apache.zookeeper")
+    exclude("org.mortbay.jetty")
   }
   implementation(libs.hadoop2.hdfs) {
+    exclude("*")
+  }
+  implementation(libs.hadoop2.hdfs.client) {
     exclude("com.sun.jersey")
     exclude("javax.servlet")
+    exclude("org.fusesource.leveldbjni")
+    exclude("org.mortbay.jetty")
   }
   implementation(libs.hadoop2.mapreduce.client.core) {
-    exclude("com.sun.jersey")
-    exclude("javax.servlet")
+    exclude("*")
   }
-
   annotationProcessor(libs.lombok)
   compileOnly(libs.lombok)
 
   testImplementation(project(":clients:client-java"))
   testImplementation(project(":integration-test-common", "testArtifacts"))
   testImplementation(project(":server"))
-  testImplementation(project(":server-common"))
+  testImplementation(project(":server-common")) {
+    exclude("org.mortbay.jetty")
+    exclude("com.sun.jersey.contribs")
+  }
   
testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion") {
     exclude("org.apache.hadoop")
+    exclude("org.rocksdb")
   }
   testImplementation("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion") 
{
     exclude("org.apache.avro")
@@ -94,7 +134,11 @@ tasks {
 
   val copyCatalogLibs by registering(Copy::class) {
     dependsOn("jar", "runtimeJars")
-    from("build/libs")
+    from("build/libs") {
+      exclude("guava-*.jar")
+      exclude("log4j-*.jar")
+      exclude("slf4j-*.jar")
+    }
     into("$rootDir/distribution/package/catalogs/lakehouse-paimon/libs")
   }