This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new ba26a891e8fc fix: include Google Cloud Storage runtime dependencies in
GCP bundle (#20020)
ba26a891e8fc is described below
commit ba26a891e8fc87704767606f11d2cc4eaf3a4950
Author: Xinyao Zhang <[email protected]>
AuthorDate: Tue Sep 29 22:38:41 2026 -0400
fix: include Google Cloud Storage runtime dependencies in GCP bundle
(#20020)
---
packaging/hudi-gcp-bundle/pom.xml | 162 ++++++++++++++++++++-
.../hudi/gcp/bundle/ITGcpBundleCompatibility.java | 155 ++++++++++++++++++++
2 files changed, 316 insertions(+), 1 deletion(-)
diff --git a/packaging/hudi-gcp-bundle/pom.xml
b/packaging/hudi-gcp-bundle/pom.xml
index 37113bdee69e..fb0f16827c3f 100644
--- a/packaging/hudi-gcp-bundle/pom.xml
+++ b/packaging/hudi-gcp-bundle/pom.xml
@@ -32,7 +32,6 @@
<properties>
<checkstyle.skip>true</checkstyle.skip>
<main.basedir>${project.parent.basedir}</main.basedir>
- <skipTests>true</skipTests>
</properties>
<dependencyManagement>
<dependencies>
@@ -99,7 +98,76 @@
<include>org.apache.hudi:hudi-hive-sync</include>
<include>org.apache.hudi:hudi-gcp</include>
<include>org.apache.parquet:parquet-avro</include>
+ <!-- GCS lock client and BigQuery sync SDK dependencies -->
<include>com.google.cloud:google-cloud-bigquery</include>
+
<include>com.google.cloud:google-cloud-bigquerystorage</include>
+ <include>com.google.cloud:google-cloud-core</include>
+ <include>com.google.cloud:google-cloud-core-http</include>
+ <include>com.google.cloud:google-cloud-core-grpc</include>
+ <include>com.google.cloud:google-cloud-storage</include>
+ <include>com.google.cloud:google-cloud-monitoring</include>
+ <include>com.google.api:api-common</include>
+ <include>com.google.api:gax</include>
+ <include>com.google.api:gax-grpc</include>
+ <include>com.google.api:gax-httpjson</include>
+
<include>com.google.apis:google-api-services-bigquery</include>
+
<include>com.google.apis:google-api-services-storage</include>
+ <include>com.google.api-client:google-api-client</include>
+
<include>com.google.api.grpc:proto-google-common-protos</include>
+ <include>com.google.api.grpc:proto-google-iam-v1</include>
+
<include>com.google.api.grpc:proto-google-cloud-storage-v2</include>
+
<include>com.google.api.grpc:grpc-google-cloud-storage-v2</include>
+
<include>com.google.api.grpc:gapic-google-cloud-storage-v2</include>
+
<include>com.google.api.grpc:proto-google-cloud-bigquerystorage-v1</include>
+
<include>com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1</include>
+
<include>com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2</include>
+
<include>com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1</include>
+
<include>com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1</include>
+
<include>com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2</include>
+
<include>com.google.api.grpc:proto-google-cloud-monitoring-v3</include>
+
<include>com.google.auth:google-auth-library-credentials</include>
+
<include>com.google.auth:google-auth-library-oauth2-http</include>
+ <include>com.google.code.gson:gson</include>
+ <include>com.google.http-client:google-http-client</include>
+
<include>com.google.http-client:google-http-client-apache-v2</include>
+
<include>com.google.http-client:google-http-client-appengine</include>
+
<include>com.google.http-client:google-http-client-gson</include>
+
<include>com.google.http-client:google-http-client-jackson2</include>
+
<include>com.google.oauth-client:google-oauth-client</include>
+ <include>com.google.protobuf:protobuf-java</include>
+ <include>com.google.protobuf:protobuf-java-util</include>
+ <include>io.grpc:grpc-api</include>
+ <include>io.grpc:grpc-context</include>
+ <include>io.grpc:grpc-core</include>
+ <include>io.grpc:grpc-stub</include>
+ <include>io.grpc:grpc-protobuf</include>
+ <include>io.grpc:grpc-protobuf-lite</include>
+ <include>io.grpc:grpc-auth</include>
+ <include>io.grpc:grpc-alts</include>
+ <include>io.grpc:grpc-grpclb</include>
+ <include>io.grpc:grpc-netty-shaded</include>
+ <include>io.grpc:grpc-googleapis</include>
+ <include>io.grpc:grpc-xds</include>
+ <include>io.grpc:grpc-services</include>
+ <include>io.grpc:grpc-rls</include>
+ <!-- Keep the OpenCensus implementation with its exporter
and Disruptor dependencies. -->
+ <include>io.opencensus:opencensus-api</include>
+ <include>io.opencensus:opencensus-contrib-http-util</include>
+ <include>io.opencensus:opencensus-impl</include>
+ <include>io.opencensus:opencensus-impl-core</include>
+
<include>io.opencensus:opencensus-exporter-stats-stackdriver</include>
+
<include>io.opencensus:opencensus-exporter-metrics-util</include>
+
<include>io.opencensus:opencensus-contrib-exemplar-util</include>
+
<include>io.opencensus:opencensus-contrib-resource-util</include>
+
<include>io.opencensus:opencensus-contrib-grpc-metrics</include>
+ <include>io.opencensus:opencensus-proto</include>
+ <include>io.perfmark:perfmark-api</include>
+ <include>com.lmax:disruptor</include>
+ <include>org.threeten:threetenbp</include>
+ <include>org.threeten:threeten-extra</include>
+ <include>com.google.guava:guava</include>
+ <include>com.google.guava:failureaccess</include>
+ <include>com.google.guava:listenablefuture</include>
<include>com.beust:jcommander</include>
<include>commons-io:commons-io</include>
<include>org.openjdk.jol:jol-core</include>
@@ -114,6 +182,24 @@
<pattern>org.openjdk.jol.</pattern>
<shadedPattern>org.apache.hudi.org.openjdk.jol.</shadedPattern>
</relocation>
+ <relocation>
+ <pattern>com.google.common.</pattern>
+
<shadedPattern>org.apache.hudi.com.google.common.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google.thirdparty.</pattern>
+
<shadedPattern>org.apache.hudi.com.google.thirdparty.</shadedPattern>
+ </relocation>
+ <!-- hudi-io already uses org.apache.hudi.com.google.protobuf,
so keep the GCP
+ protobuf and gRPC copies in a separate namespace. -->
+ <relocation>
+ <pattern>com.google.protobuf.</pattern>
+
<shadedPattern>org.apache.hudi.gcp.shaded.com.google.protobuf.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>io.grpc.</pattern>
+
<shadedPattern>org.apache.hudi.gcp.shaded.io.grpc.</shadedPattern>
+ </relocation>
</relocations>
<createDependencyReducedPom>true</createDependencyReducedPom>
<!-- Keep dependencies that are not absorbed into the shaded
jar, so the reduced POM still
@@ -136,6 +222,67 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-compatibility-fixtures</id>
+ <phase>pre-integration-test</phase>
+ <goals>
+ <goal>copy</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>2.5.0</version>
+
<outputDirectory>${project.build.directory}/compatibility-libs</outputDirectory>
+ </artifactItem>
+ <artifactItem>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>27.0-jre</version>
+
<outputDirectory>${project.build.directory}/compatibility-libs</outputDirectory>
+ </artifactItem>
+ <!-- Jackson Core is supplied by the host in this deployment
scenario. -->
+ <artifactItem>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>${fasterxml.version}</version>
+
<outputDirectory>${project.build.directory}/compatibility-libs</outputDirectory>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <configuration>
+ <skipTests>${skipITs}</skipTests>
+ <includes>
+ <include>**/ITGcpBundleCompatibility.java</include>
+ </includes>
+ <systemPropertyVariables>
+
<gcp.bundle.jar>${project.build.directory}/${project.build.finalName}.jar</gcp.bundle.jar>
+
<gcp.test.classes>${project.build.testOutputDirectory}</gcp.test.classes>
+
<host.protobuf.jar>${project.build.directory}/compatibility-libs/protobuf-java-2.5.0.jar</host.protobuf.jar>
+
<host.guava.jar>${project.build.directory}/compatibility-libs/guava-27.0-jre.jar</host.guava.jar>
+
<host.jackson.core.jar>${project.build.directory}/compatibility-libs/jackson-core-${fasterxml.version}.jar</host.jackson.core.jar>
+ </systemPropertyVariables>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
<resources>
<resource>
@@ -175,6 +322,19 @@
<artifactId>google-cloud-bigquery</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.platform</groupId>
+ <artifactId>junit-platform-launcher</artifactId>
+ <version>${junit.platform.version}</version>
+ <scope>test</scope>
+ </dependency>
+
<!-- Parquet -->
<dependency>
<groupId>org.apache.parquet</groupId>
diff --git
a/packaging/hudi-gcp-bundle/src/test/java/org/apache/hudi/gcp/bundle/ITGcpBundleCompatibility.java
b/packaging/hudi-gcp-bundle/src/test/java/org/apache/hudi/gcp/bundle/ITGcpBundleCompatibility.java
new file mode 100644
index 000000000000..45a2ff18b53b
--- /dev/null
+++
b/packaging/hudi-gcp-bundle/src/test/java/org/apache/hudi/gcp/bundle/ITGcpBundleCompatibility.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.gcp.bundle;
+
+import com.google.cloud.NoCredentials;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.storage.StorageOptions;
+import org.junit.jupiter.api.Test;
+
+import java.io.File;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+class ITGcpBundleCompatibility {
+
+ private enum ClasspathOrder {
+ HOST_FIRST,
+ BUNDLE_FIRST
+ }
+
+ @Test
+ void testHostDependenciesBeforeBundle() throws Exception {
+ runSmokeTest(ClasspathOrder.HOST_FIRST);
+ }
+
+ @Test
+ void testBundleBeforeHostDependencies() throws Exception {
+ runSmokeTest(ClasspathOrder.BUNDLE_FIRST);
+ }
+
+ private static void runSmokeTest(ClasspathOrder order) throws Exception {
+ String bundleJar = System.getProperty("gcp.bundle.jar");
+ String testClasses = System.getProperty("gcp.test.classes");
+ String hostProtobuf = System.getProperty("host.protobuf.jar");
+ String hostGuava = System.getProperty("host.guava.jar");
+ String hostJacksonCore = System.getProperty("host.jackson.core.jar");
+
+ List<String> orderedClasspath = new ArrayList<>();
+ orderedClasspath.add(testClasses);
+ if (order == ClasspathOrder.HOST_FIRST) {
+ orderedClasspath.add(hostProtobuf);
+ orderedClasspath.add(hostGuava);
+ }
+ orderedClasspath.add(bundleJar);
+ if (order == ClasspathOrder.BUNDLE_FIRST) {
+ orderedClasspath.add(hostProtobuf);
+ orderedClasspath.add(hostGuava);
+ }
+ orderedClasspath.add(hostJacksonCore);
+
+ Process process = new ProcessBuilder(
+ new File(System.getProperty("java.home"),
"bin/java").getAbsolutePath(),
+ "-cp",
+ String.join(File.pathSeparator, orderedClasspath),
+ ITGcpBundleCompatibility.class.getName(),
+ "smoke")
+ .inheritIO()
+ .start();
+ assertEquals(0, process.waitFor(), "GCP bundle compatibility smoke test
failed");
+ }
+
+ public static void main(String[] args) throws Exception {
+ if (args.length != 1 || !"smoke".equals(args[0])) {
+ throw new IllegalArgumentException("Expected smoke mode");
+ }
+
+ verifyShadedProtobuf();
+ verifyShadedGuava();
+ verifyHostGuava();
+ verifyStorageClient();
+ verifyBigQueryClient();
+ verifyThreeTenExtra();
+ }
+
+ private static void verifyShadedProtobuf() throws Exception {
+ Class<?> timestampClass = Class.forName(
+ "org.apache.hudi.gcp.shaded.com.google.protobuf.Timestamp");
+ Object timestampBuilder =
timestampClass.getMethod("newBuilder").invoke(null);
+ Method setSeconds = timestampBuilder.getClass().getMethod("setSeconds",
long.class);
+ Object timestamp = timestampBuilder.getClass().getMethod("build")
+ .invoke(setSeconds.invoke(timestampBuilder, 1L));
+ byte[] timestampBytes = (byte[])
timestampClass.getMethod("toByteArray").invoke(timestamp);
+ if (timestampBytes.length == 0) {
+ throw new AssertionError("Protobuf serialization returned no bytes");
+ }
+ }
+
+ private static void verifyShadedGuava() throws Exception {
+ Class<?> domainNameClass = Class.forName(
+ "org.apache.hudi.com.google.common.net.InternetDomainName");
+ Object domainName = domainNameClass.getMethod("from",
String.class).invoke(null, "example.com");
+ Object publicSuffix =
domainNameClass.getMethod("publicSuffix").invoke(domainName);
+ if (!"com".equals(publicSuffix.toString())) {
+ throw new AssertionError("Unexpected public suffix: " + publicSuffix);
+ }
+ }
+
+ private static void verifyHostGuava() throws Exception {
+ Class<?> hostDomainNameClass =
Class.forName("com.google.common.net.InternetDomainName");
+ Object hostDomainName = hostDomainNameClass.getMethod("from", String.class)
+ .invoke(null, "example.com");
+ Object hostPublicSuffix = hostDomainNameClass.getMethod("publicSuffix")
+ .invoke(hostDomainName);
+ if (!"com".equals(hostPublicSuffix.toString())) {
+ throw new AssertionError("Unexpected host public suffix: " +
hostPublicSuffix);
+ }
+ }
+
+ private static void verifyStorageClient() {
+ StorageOptions options = StorageOptions.newBuilder()
+ .setProjectId("review")
+ .setCredentials(NoCredentials.getInstance())
+ .setRetrySettings(StorageOptions.getDefaultRetrySettings().toBuilder()
+ .setMaxAttempts(1)
+ .build())
+ .build();
+ options.getService();
+ }
+
+ private static void verifyBigQueryClient() {
+ BigQueryOptions.newBuilder()
+ .setProjectId("review")
+ .setCredentials(NoCredentials.getInstance())
+ .build()
+ .getService();
+ }
+
+ private static void verifyThreeTenExtra() throws Exception {
+ Class<?> periodDurationClass =
Class.forName("org.threeten.extra.PeriodDuration");
+ Object periodDuration = periodDurationClass.getMethod("parse",
CharSequence.class)
+ .invoke(null, "P1D");
+ if (!"P1D".equals(periodDuration.toString())) {
+ throw new AssertionError("Unexpected period duration: " +
periodDuration);
+ }
+ }
+}
