RustedBones commented on code in PR #26320:
URL: https://github.com/apache/beam/pull/26320#discussion_r1214471057
##########
sdks/java/extensions/avro/build.gradle:
##########
@@ -63,67 +74,91 @@ dependencies {
testImplementation library.java.junit
testRuntimeOnly project(path: ":runners:direct-java", configuration:
"shadow")
testRuntimeOnly library.java.slf4j_jdk14
- avroVersions.each {
- "avroVersion$it.key" "org.apache.avro:avro:$it.value"
- "avroVersion$it.key" "org.apache.avro:avro-tools:$it.value"
+ avroVersions.each { k,v ->
+ "avroVersion$k"(project(path: ":sdks:java:core", configuration:
"shadowTest")) {
+ // Exclude Avro dependencies from "core" since Avro support moved to
this extension
+ exclude group: "org.apache.avro", module: "avro"
+ }
+ "avroVersion$k" library.java.junit
+ "avroVersion$k" project(path: ":runners:direct-java", configuration:
"shadow")
+ "avroVersion$k" library.java.slf4j_jdk14
+ "avroVersion$k" "org.tukaani:xz:1.9" // makes as provided since 1.9
+ "avroVersion$k" library.java.zstd_jni // makes as provided since 1.9
Review Comment:
Avro version `1.9` and above mark `xz` as optional. We can't run the
compression tests otherwise.
Also include `zstd` and run tests for avro `1.9` and above
##########
sdks/java/extensions/avro/build.gradle:
##########
@@ -35,7 +41,12 @@ def avroVersions = [
'1111': "1.11.1",
]
-avroVersions.each { k, v -> configurations.create("avroVersion$k") }
+avroVersions.each { k, v ->
+ configurations {
+ create("avroVersion$k").extendsFrom(implementation)
Review Comment:
extends the implementation configuration.
`avroVersion$k` should not extends test as we do not want to include code
generated by the `1.8.2` version used in test
##########
sdks/java/extensions/avro/build.gradle:
##########
@@ -63,67 +74,91 @@ dependencies {
testImplementation library.java.junit
testRuntimeOnly project(path: ":runners:direct-java", configuration:
"shadow")
testRuntimeOnly library.java.slf4j_jdk14
- avroVersions.each {
- "avroVersion$it.key" "org.apache.avro:avro:$it.value"
- "avroVersion$it.key" "org.apache.avro:avro-tools:$it.value"
+ avroVersions.each { k,v ->
+ "avroVersion$k"(project(path: ":sdks:java:core", configuration:
"shadowTest")) {
+ // Exclude Avro dependencies from "core" since Avro support moved to
this extension
+ exclude group: "org.apache.avro", module: "avro"
+ }
+ "avroVersion$k" library.java.junit
+ "avroVersion$k" project(path: ":runners:direct-java", configuration:
"shadow")
+ "avroVersion$k" library.java.slf4j_jdk14
+ "avroVersion$k" "org.tukaani:xz:1.9" // makes as provided since 1.9
+ "avroVersion$k" library.java.zstd_jni // makes as provided since 1.9
+ "avroVersion${k}Generate" "org.apache.avro:avro-tools:$v"
}
}
-avroVersions.each { kv ->
- configurations."avroVersion$kv.key" {
- resolutionStrategy {
- force "org.apache.avro:avro:$kv.value"
+task printSourceSetInformation(){
+
+ doLast{
+ sourceSets.each { srcSet ->
+ println "["+srcSet.name+"]"
+ print "-->Source directories: "+srcSet.allJava.srcDirs+"\n"
+ print "-->Output directories: "+srcSet.output.classesDirs.files+"\n"
+ print "-->Compile classpath:\n"
+ srcSet.runtimeClasspath.files.each {
+ print " "+it.path+"\n"
+ }
+ println ""
}
}
+}
Review Comment:
This was a debug leftover. will clean
##########
sdks/java/extensions/avro/build.gradle:
##########
@@ -63,67 +74,91 @@ dependencies {
testImplementation library.java.junit
testRuntimeOnly project(path: ":runners:direct-java", configuration:
"shadow")
testRuntimeOnly library.java.slf4j_jdk14
- avroVersions.each {
- "avroVersion$it.key" "org.apache.avro:avro:$it.value"
- "avroVersion$it.key" "org.apache.avro:avro-tools:$it.value"
+ avroVersions.each { k,v ->
+ "avroVersion$k"(project(path: ":sdks:java:core", configuration:
"shadowTest")) {
+ // Exclude Avro dependencies from "core" since Avro support moved to
this extension
+ exclude group: "org.apache.avro", module: "avro"
+ }
+ "avroVersion$k" library.java.junit
+ "avroVersion$k" project(path: ":runners:direct-java", configuration:
"shadow")
+ "avroVersion$k" library.java.slf4j_jdk14
+ "avroVersion$k" "org.tukaani:xz:1.9" // makes as provided since 1.9
+ "avroVersion$k" library.java.zstd_jni // makes as provided since 1.9
+ "avroVersion${k}Generate" "org.apache.avro:avro-tools:$v"
}
}
-avroVersions.each { kv ->
- configurations."avroVersion$kv.key" {
- resolutionStrategy {
- force "org.apache.avro:avro:$kv.value"
+task printSourceSetInformation(){
+
+ doLast{
+ sourceSets.each { srcSet ->
+ println "["+srcSet.name+"]"
+ print "-->Source directories: "+srcSet.allJava.srcDirs+"\n"
+ print "-->Output directories: "+srcSet.output.classesDirs.files+"\n"
+ print "-->Compile classpath:\n"
+ srcSet.runtimeClasspath.files.each {
+ print " "+it.path+"\n"
+ }
+ println ""
}
}
+}
+
+avroVersions.each { k, v ->
+ configurations."avroVersion$k" {
+ resolutionStrategy.force "org.apache.avro:avro:$v",
"org.apache.avro:avro:$v:tests", library.java.joda_time
Review Comment:
Appending `joda_time` too as `force` is transitive and downgrades the lib
when using `1.9`
##########
sdks/java/extensions/avro/build.gradle:
##########
@@ -63,67 +74,91 @@ dependencies {
testImplementation library.java.junit
testRuntimeOnly project(path: ":runners:direct-java", configuration:
"shadow")
testRuntimeOnly library.java.slf4j_jdk14
- avroVersions.each {
- "avroVersion$it.key" "org.apache.avro:avro:$it.value"
- "avroVersion$it.key" "org.apache.avro:avro-tools:$it.value"
+ avroVersions.each { k,v ->
+ "avroVersion$k"(project(path: ":sdks:java:core", configuration:
"shadowTest")) {
+ // Exclude Avro dependencies from "core" since Avro support moved to
this extension
+ exclude group: "org.apache.avro", module: "avro"
+ }
+ "avroVersion$k" library.java.junit
+ "avroVersion$k" project(path: ":runners:direct-java", configuration:
"shadow")
+ "avroVersion$k" library.java.slf4j_jdk14
+ "avroVersion$k" "org.tukaani:xz:1.9" // makes as provided since 1.9
+ "avroVersion$k" library.java.zstd_jni // makes as provided since 1.9
+ "avroVersion${k}Generate" "org.apache.avro:avro-tools:$v"
}
}
-avroVersions.each { kv ->
- configurations."avroVersion$kv.key" {
- resolutionStrategy {
- force "org.apache.avro:avro:$kv.value"
+task printSourceSetInformation(){
+
+ doLast{
+ sourceSets.each { srcSet ->
+ println "["+srcSet.name+"]"
+ print "-->Source directories: "+srcSet.allJava.srcDirs+"\n"
+ print "-->Output directories: "+srcSet.output.classesDirs.files+"\n"
+ print "-->Compile classpath:\n"
+ srcSet.runtimeClasspath.files.each {
+ print " "+it.path+"\n"
+ }
+ println ""
}
}
+}
+
+avroVersions.each { k, v ->
+ configurations."avroVersion$k" {
+ resolutionStrategy.force "org.apache.avro:avro:$v",
"org.apache.avro:avro:$v:tests", library.java.joda_time
+ }
sourceSets {
- "avro${kv.key}" {
+ "avro$k" {
java {
- srcDirs "build/generated/sources/avro${kv.key}/test/java"
+ srcDirs "src/test/java", "build/generated/sources/avro$k/test/java"
Review Comment:
Recompile tests with generated coder
##########
sdks/java/extensions/avro/build.gradle:
##########
@@ -35,7 +41,12 @@ def avroVersions = [
'1111': "1.11.1",
]
-avroVersions.each { k, v -> configurations.create("avroVersion$k") }
+avroVersions.each { k, v ->
Review Comment:
Related to this comment:
https://github.com/apache/beam/pull/26320#discussion_r1214111539. Will put more
detailed comments.
##########
sdks/java/extensions/avro/build.gradle:
##########
@@ -63,67 +74,91 @@ dependencies {
testImplementation library.java.junit
testRuntimeOnly project(path: ":runners:direct-java", configuration:
"shadow")
testRuntimeOnly library.java.slf4j_jdk14
- avroVersions.each {
- "avroVersion$it.key" "org.apache.avro:avro:$it.value"
- "avroVersion$it.key" "org.apache.avro:avro-tools:$it.value"
+ avroVersions.each { k,v ->
+ "avroVersion$k"(project(path: ":sdks:java:core", configuration:
"shadowTest")) {
+ // Exclude Avro dependencies from "core" since Avro support moved to
this extension
+ exclude group: "org.apache.avro", module: "avro"
+ }
+ "avroVersion$k" library.java.junit
+ "avroVersion$k" project(path: ":runners:direct-java", configuration:
"shadow")
+ "avroVersion$k" library.java.slf4j_jdk14
Review Comment:
we have to re-declare test deps for the `avroVersion$k` configuration
##########
sdks/java/extensions/avro/build.gradle:
##########
@@ -35,7 +41,12 @@ def avroVersions = [
'1111': "1.11.1",
]
-avroVersions.each { k, v -> configurations.create("avroVersion$k") }
+avroVersions.each { k, v ->
+ configurations {
+ create("avroVersion$k").extendsFrom(implementation)
+ create("avroVersion${k}Generate")
Review Comment:
Create a config for the `"generateAvroClasses$k"` (`avro-tools` and
dependencies `avro-compiler` are not required for the tests)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
