diqiu50 commented on code in PR #13740: URL: https://github.com/apache/gravitino/pull/13740#discussion_r4237299858
########## flink-connector/v2.2/flink-runtime/build.gradle.kts: ########## @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar + +plugins { + `maven-publish` + id("java") + id("idea") + alias(libs.plugins.shadow) +} + +repositories { + mavenCentral() +} + +val flinkVersion: String = libs.versions.flink22.get() +val flinkMajorVersion: String = flinkVersion.substringBeforeLast(".") +// The connector uses Flink's Java APIs and does not need Scala-version-specific artifacts. +val artifactName = "gravitino-${project.name}" +val baseName = "${rootProject.name}-flink-connector-runtime-$flinkMajorVersion" + +configurations.all { + resolutionStrategy.eachDependency { + if (requested.group == "org.apache.logging.log4j") { + throw GradleException("Dependency 'org.apache.logging.log4j' is not allowed.") + } + } +} + +dependencies { + implementation(project(":clients:client-java-runtime", configuration = "shadow")) + implementation(project(":flink-connector:flink-2.2")) + // flink-2.2's own openlineageSqlJava21 constraint only applies to that module's compileOnly and + // testImplementation configurations, so it never reaches this module's runtimeClasspath (which + // shadowJar packages). Depend on it directly here so the shaded runtime jar ships the patched + // 1.52.0 release instead of the vulnerable 1.32.0 pulled transitively by flink-connector-jdbc. + implementation(libs.openlineageSqlJava21) + + testImplementation(libs.junit.jupiter.api) + testRuntimeOnly(libs.junit.jupiter.engine) +} + +val shadowJarTask = tasks.named<ShadowJar>("shadowJar") + +shadowJarTask.configure { + isZip64 = true + configurations = listOf(project.configurations.runtimeClasspath.get()) + archiveFileName.set("$baseName-$version.jar") + archiveClassifier.set("") + mergeServiceFiles() + + exclude("org/slf4j/**") + exclude("META-INF/maven/org.slf4j/**") + + relocate("com.google", "org.apache.gravitino.shaded.com.google") + relocate("google", "org.apache.gravitino.shaded.google") + relocate("org.apache.hc", "org.apache.gravitino.shaded.org.apache.hc") +} + +tasks.test { + useJUnitPlatform() + dependsOn(shadowJarTask) + inputs.file(shadowJarTask.flatMap { it.archiveFile }) Review Comment: `inputs.file(shadowJarTask.flatMap { it.archiveFile })` is only added for v2.2. Should v2.1's runtime module get the same change so its test task is re-run when the shadow jar changes? Same for the stricter factory assertions in `TestRuntimeJarDependencies` and the build comment wording at line 34 — it'd be good to keep v2.1 and v2.2 consistent. ########## flink-connector/v2.2/flink/build.gradle.kts: ########## @@ -0,0 +1,279 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +plugins { + `maven-publish` + id("java") + id("idea") +} + +repositories { + mavenCentral() +} + +// flink-common's sources are compiled directly into this module's own sourceSet (in addition +// to flink-common's own jar) so that they are re-checked against this module's own +// Flink/Iceberg/Paimon dependency versions (this matters once versions diverge in incompatible +// ways, e.g. Flink 2.x removing APIs that are still present in the 1.x line). +val commonProject = project(":flink-connector:flink-common") + +sourceSets { + main { + java.srcDir(commonProject.file("src/main/java")) + } + test { + java.srcDir(commonProject.file("src/test/java")) + resources.srcDir(commonProject.file("src/test/resources")) + } +} + +// Spotless can't format files outside the project dir; flink-common's own sources are linted +// by the flink-common project itself, so restrict this project's spotless target to its own. +plugins.withType<com.diffplug.gradle.spotless.SpotlessPlugin>().configureEach { + configure<com.diffplug.gradle.spotless.SpotlessExtension> { + java { + target("src/**/*.java") + } + } +} + +val flinkVersion: String = libs.versions.flink22.get() +val flinkMajorVersion: String = flinkVersion.substringBeforeLast(".") +val icebergVersion: String = libs.versions.iceberg4flink22.get() +val paimonVersion: String = libs.versions.paimon4flink22.get() +// This connector uses Flink Java APIs, so its artifacts need no Scala-version suffix. +// Flink internals still use Scala. +val artifactName = "${rootProject.name}-flink-$flinkMajorVersion" + +dependencies { + constraints { + // Force upgrade for outdated transitive libthrift pulled by Hive Metastore + compileOnly(libs.thrift) + testImplementation(libs.thrift) + // flink-connector-jdbc-core transitively pulls openlineage-sql-java:1.32.0, whose bundled, + // unrelocated org.apache.commons.lang3.SystemProperties predates the getUserName(String) + // overload and shadows the real commons-lang3 on the classpath (NoSuchMethodError). This + // can't simply be excluded: the connector's lineage extraction actually calls into it at + // runtime (JdbcSource.getLineageVertex). Force this newer release instead, which ships an + // up-to-date bundled copy. + compileOnly(libs.openlineageSqlJava21) Review Comment: nit: `openlineageSqlJava21` in `libs.versions.toml` is now also used by the Flink 2.2 module. Consider renaming it to something version-neutral like `openlineageSqlJava`. ########## docs/flink-connector/flink-catalog-jdbc.md: ########## @@ -31,8 +31,14 @@ Place the following JAR files in the lib directory of your Flink installation: | 1.19 | `3.3.0-1.19` | `gravitino-flink-connector-runtime-1.19_2.12-${gravitino-version}.jar` | | 1.20 | `3.3.0-1.20` | `gravitino-flink-connector-runtime-1.20_2.12-${gravitino-version}.jar` | | 2.1 | `4.1.0-2.1` | `gravitino-flink-connector-runtime-2.1-${gravitino-version}.jar` | +| 2.2 | `4.1.0-2.2` | `gravitino-flink-connector-runtime-2.2-${gravitino-version}.jar` | -Starting with Flink 2.1, `flink-connector-jdbc` is split into per-database artifacts: place `flink-connector-jdbc-core-4.1.0-2.1.jar`, `flink-connector-jdbc-mysql-4.1.0-2.1.jar`, and `flink-connector-jdbc-postgres-4.1.0-2.1.jar` together in the classpath instead of a single combined JAR. +For Flink 2.x, `flink-connector-jdbc` is split into core and per-database artifacts. Place all three JARs from the matching row in the classpath instead of a single combined JAR: + +| Flink version | Core artifact | MySQL artifact | PostgreSQL artifact | Review Comment: nit: the column widths of this table are not aligned (the header and separator rows are narrower than the data rows). -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
