jason810496 commented on code in PR #69527:
URL: https://github.com/apache/airflow/pull/69527#discussion_r3534237701
##########
java-sdk/sdk/build.gradle.kts:
##########
@@ -150,14 +155,67 @@ abstract class GenerateDiscriminatorTask : DefaultTask() {
}
}
+abstract class SyncSupervisorSchemaTask : DefaultTask() {
+ @get:Input
+ abstract val schemaVersion: Property<String>
+
+ @get:Input
+ abstract val baseUrl: Property<String>
+
+ @get:Internal
+ abstract val schemaFile: RegularFileProperty
+
+ private fun apiVersionOf(file: File): String =
+ if (file.exists()) {
+ com.fasterxml.jackson.databind
+ .ObjectMapper()
+ .readTree(file)
+ .path("api_version")
+ .asText()
+ } else {
+ ""
+ }
+
+ @TaskAction
+ fun sync() {
+ val file = schemaFile.get().asFile
+ val version = schemaVersion.get()
+ if (apiVersionOf(file) == version) {
+ logger.lifecycle("Supervisor Schema is up-to-date
(api_version=$version).")
+ return
+ }
+ val url = "${baseUrl.get()}/$version.json"
+ logger.lifecycle("Refreshing Supervisor Schema with $url")
+ file.parentFile.mkdirs()
+ URI(url).toURL().openStream().use { input ->
+ Files.copy(input, file.toPath(),
StandardCopyOption.REPLACE_EXISTING)
+ }
Review Comment:
nit from CC:
No download timeouts. URI(url).toURL().openStream() can hang the build
indefinitely; set connect/read timeouts on the URLConnection. Also, a 404
surfaces as a bare FileNotFoundException — wrapping it in a GradleException
hinting "version not published yet?" would help whoever bumps the property.
##########
java-sdk/sdk/schema/schema.json:
##########
@@ -0,0 +1,4925 @@
+{
Review Comment:
Just to double check: Is it intentional to keep a copy instead of symbolic
link to bump the schema version manually?
##########
java-sdk/sdk/build.gradle.kts:
##########
@@ -150,16 +155,69 @@ abstract class GenerateDiscriminatorTask : DefaultTask() {
}
}
+abstract class SyncSupervisorSchemaTask : DefaultTask() {
+ @get:Input
+ abstract val schemaVersion: Property<String>
+
+ @get:Input
+ abstract val baseUrl: Property<String>
+
+ @get:Internal
+ abstract val schemaFile: RegularFileProperty
+
+ private fun apiVersionOf(file: File): String =
+ if (file.exists()) {
+ com.fasterxml.jackson.databind
+ .ObjectMapper()
+ .readTree(file)
+ .path("api_version")
+ .asText()
+ } else {
+ ""
+ }
+
+ @TaskAction
+ fun sync() {
+ val file = schemaFile.get().asFile
+ val version = schemaVersion.get()
+ if (apiVersionOf(file) == version) {
+ logger.lifecycle("Supervisor Schema is up-to-date
(api_version=$version).")
+ return
+ }
Review Comment:
How about providing a flag to force regenerate (to not rely on the date)? We
might want to bump java-sdk side schema.json manually while both files have the
same date but there're drifts (e.g. schema change before `3.4.0` release and we
want to catchup on Java-SDK side).
##########
java-sdk/sdk/build.gradle.kts:
##########
@@ -150,14 +155,67 @@ abstract class GenerateDiscriminatorTask : DefaultTask() {
}
}
+abstract class SyncSupervisorSchemaTask : DefaultTask() {
+ @get:Input
+ abstract val schemaVersion: Property<String>
+
+ @get:Input
+ abstract val baseUrl: Property<String>
+
+ @get:Internal
+ abstract val schemaFile: RegularFileProperty
+
+ private fun apiVersionOf(file: File): String =
+ if (file.exists()) {
+ com.fasterxml.jackson.databind
+ .ObjectMapper()
+ .readTree(file)
+ .path("api_version")
+ .asText()
+ } else {
+ ""
+ }
+
+ @TaskAction
+ fun sync() {
+ val file = schemaFile.get().asFile
+ val version = schemaVersion.get()
+ if (apiVersionOf(file) == version) {
+ logger.lifecycle("Supervisor Schema is up-to-date
(api_version=$version).")
+ return
+ }
+ val url = "${baseUrl.get()}/$version.json"
+ logger.lifecycle("Refreshing Supervisor Schema with $url")
+ file.parentFile.mkdirs()
+ URI(url).toURL().openStream().use { input ->
+ Files.copy(input, file.toPath(),
StandardCopyOption.REPLACE_EXISTING)
+ }
+ val downloaded = apiVersionOf(file)
+ if (downloaded != version) {
+ throw GradleException(
+ "Downloaded schema declares api_version='$downloaded' but
expected '$version' ($url)",
+ )
+ }
+ }
+}
+
+val syncSupervisorSchema by tasks.registering(SyncSupervisorSchemaTask::class)
{
+ description = "Ensure the bundled Supervisor Schema is up-to-date with the
Gradle property."
+ schemaVersion = airflowSupervisorSchemaVersion
+ baseUrl = schemaBaseUrl
+ schemaFile = layout.projectDirectory.file("schema/schema.json")
Review Comment:
nit: the `layout.projectDirectory.file("schema/schema.json")` is duplicated
twice.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
