hudi-agent commented on code in PR #19161: URL: https://github.com/apache/hudi/pull/19161#discussion_r3521934153
########## hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestHoodieProcedureFilterUtils.scala: ########## @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hudi.procedure + +import org.apache.spark.sql.Row +import org.apache.spark.sql.hudi.command.procedures.HoodieProcedureFilterUtils +import org.apache.spark.sql.types._ + +import java.sql.{Date, Timestamp} + +/** + * Direct unit tests for [[HoodieProcedureFilterUtils]] which evaluates SQL filter + * expressions against procedure output rows. Covers primitive/date/decimal/complex + * type conversion, the function-resolution table, numeric type coercion, validation + * and error handling. + */ +class TestHoodieProcedureFilterUtils extends HoodieSparkProcedureTestBase { + + private def schemaOf(fields: (String, DataType)*): StructType = + StructType(fields.map { case (n, dt) => StructField(n, dt, nullable = true) }) + + private def keep(rows: Seq[Row], expr: String, s: StructType): Seq[Row] = Review Comment: 🤖 nit: could you rename `s` to `schema`? The parameter is referenced in every `keep(...)` call throughout the file, and a one-letter name makes each call site slightly harder to read at a glance. <sub><i>⚠️ AI-generated; verify before applying. React 👍/👎 to flag quality.</i></sub> ########## hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestValidateHoodieSyncProcedure.scala: ########## @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hudi.procedure + +/** + * Tests for [[org.apache.spark.sql.hudi.command.procedures.ValidateHoodieSyncProcedure]]. + * + * The "complete" / "latestPartitions" modes require a live Hive/JDBC endpoint, which is not + * available in the unit-test environment. Passing any other mode short-circuits the record + * counting (record counts stay 0) while still exercising the timeline comparison, the + * catch-up-commit computation and the result formatting, which is the bulk of the procedure. + */ +class TestValidateHoodieSyncProcedure extends HoodieSparkProcedureTestBase { + + private def createTable(tableName: String, path: String): Unit = { + spark.sql( + s""" + |create table $tableName ( + | id int, + | name string, + | price double, + | ts long + |) using hudi + | location '$path' + | tblproperties ( + | primaryKey = 'id', + | orderingFields = 'ts' + | ) + """.stripMargin) + } + + test("Test Call sync_validate when the target table is ahead (catch-up commits)") { + withTempDir { tmp => + val srcTable = generateTableName + val dstTable = generateTableName + createTable(srcTable, s"${tmp.getCanonicalPath}/$srcTable") + // The source table has a single, earlier commit. + spark.sql(s"insert into $srcTable select 1, 'a1', 10, 1000") + + createTable(dstTable, s"${tmp.getCanonicalPath}/$dstTable") + // The destination table receives later commits, so it is ahead of the source. + spark.sql(s"insert into $dstTable select 1, 'a1', 10, 1000") + spark.sql(s"insert into $dstTable select 2, 'a2', 20, 2000") + + val result = spark.sql( + s"""call sync_validate(src_table => '$srcTable', dst_table => '$dstTable', + | mode => 'noop', hive_server_url => 'jdbc:hive2://unused', hive_pass => 'x')""" + .stripMargin).collect() + + assertResult(1)(result.length) + val message = result.head.getString(0) + assert(message.startsWith("Count difference now is"), + s"Unexpected message: $message") + // The destination has commits after the source's latest, so a catch-up count is reported. + assert(message.contains("Catach up count is"), s"Unexpected message: $message") Review Comment: 🤖 nit: "Catach up count is" looks like a typo for "Catch up count is" — if this is intentionally matching a typo in `ValidateHoodieSyncProcedure`'s output message, a short inline comment (`// matches existing typo in procedure output`) would save the next reader from wondering whether the test itself is wrong (same string also appears on line 91). <sub><i>⚠️ AI-generated; verify before applying. React 👍/👎 to flag quality.</i></sub> -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
