This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-8765-e0fe77acf7506c2b6e185b9849cb71a5d5b4fdc6 in repository https://gitbox.apache.org/repos/asf/texera.git
commit 5d2bd21debb15c89effe47e68e6c87bc3661b4fc Author: Kary Zheng <[email protected]> AuthorDate: Sat Oct 3 07:43:57 2026 +0000 feat(operator): reject label types the ML Scorer cannot score (#8765) ### What changes were proposed in this PR? The Machine Learning Scorer's schema check let any two columns of the same type through as a classification pair. For BINARY and LARGE_BINARY pairs, every metric then failed inside scikit-learn. For TIMESTAMP pairs, Precision, Recall and F1 failed the same way. The error did not name either column. The check now rejects these pairs and names the two columns and the metrics that cannot score them. A TIMESTAMP pair scored by Accuracy alone is still accepted, since scikit-learn scores it (it only counts equal rows). ### Any related issues, documentation, discussions? Closes #8756. ### How was this PR tested? Three new cases in `MachineLearningScorerOpDescSpec`: - BINARY and LARGE_BINARY pairs are rejected. - A TIMESTAMP pair is rejected for F1 and not for Accuracy. - A TIMESTAMP pair with Accuracy alone passes. The first two fail without the change. The runtime failures were reproduced first by running the operator's generated metric code under scikit-learn 1.7.2 on each column type. ### Was this PR authored or co-authored using generative AI tooling? Generated-by: Claude Code (Claude Opus 5.5) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 <[email protected]> --- .../Scorer/MachineLearningScorerOpDesc.scala | 15 ++++++++++ .../Scorer/MachineLearningScorerOpDescSpec.scala | 34 +++++++++++++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/Scorer/MachineLearningScorerOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/Scorer/MachineLearningScorerOpDesc.scala index 7ad96e3b3c..4c78f3493a 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/Scorer/MachineLearningScorerOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/Scorer/MachineLearningScorerOpDesc.scala @@ -171,6 +171,21 @@ class MachineLearningScorerOpDesc extends PythonOperatorDescriptor with Standalo s"'$predictValueColumn' (${predictType.getName}) hold different kinds of label, " + "so a classification metric cannot compare them" ) + // A matching pair can still be one sklearn refuses as labels. Bytes fail + // every metric. Timestamps only fail the per-class ones, since Accuracy + // just counts equal rows. + val perClass = classificationMetrics.filterNot(_ == classificationMetricsFnc.accuracy) + val refusedBy = + if (Set(AttributeType.BINARY, AttributeType.LARGE_BINARY).contains(actualType)) + classificationMetrics + else if (actualType == AttributeType.TIMESTAMP) perClass + else List() + if (refusedBy.nonEmpty) + throw new RuntimeException( + s"Actual Value '$actualValueColumn' and Predicted Value '$predictValueColumn' are " + + s"${actualType.getName} columns, which ${refusedBy.map(_.getName).mkString(", ")} " + + "cannot score as labels" + ) } } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/Scorer/MachineLearningScorerOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/Scorer/MachineLearningScorerOpDescSpec.scala index c21ff4c439..046f72ddfe 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/Scorer/MachineLearningScorerOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/Scorer/MachineLearningScorerOpDescSpec.scala @@ -139,7 +139,13 @@ class MachineLearningScorerOpDescSpec extends AnyFlatSpec with Matchers { new Attribute("y_int", AttributeType.INTEGER), new Attribute("pred_long", AttributeType.LONG), new Attribute("label_str", AttributeType.STRING), - new Attribute("pred_str", AttributeType.STRING) + new Attribute("pred_str", AttributeType.STRING), + new Attribute("ts_a", AttributeType.TIMESTAMP), + new Attribute("ts_b", AttributeType.TIMESTAMP), + new Attribute("bin_a", AttributeType.BINARY), + new Attribute("bin_b", AttributeType.BINARY), + new Attribute("lb_a", AttributeType.LARGE_BINARY), + new Attribute("lb_b", AttributeType.LARGE_BINARY) ) ) ) @@ -172,6 +178,32 @@ class MachineLearningScorerOpDescSpec extends AnyFlatSpec with Matchers { "kinds of label, so a classification metric cannot compare them" } + it should "reject a matching BINARY or LARGE_BINARY pair for every metric" in { + Seq("bin" -> "binary", "lb" -> "large_binary").foreach { + case (prefix, typeName) => + val d = scorer(regression = false, s"${prefix}_a", s"${prefix}_b") + d.classificationMetrics = List(classificationMetricsFnc.accuracy) + the[RuntimeException] thrownBy d.getOutputSchemas(inputSchemas(d)) should have message + s"Actual Value '${prefix}_a' and Predicted Value '${prefix}_b' are $typeName columns, " + + "which Accuracy cannot score as labels" + } + } + + it should "reject a matching TIMESTAMP pair for the per-class metrics only" in { + val d = scorer(regression = false, "ts_a", "ts_b") + d.classificationMetrics = + List(classificationMetricsFnc.accuracy, classificationMetricsFnc.f1Score) + the[RuntimeException] thrownBy d.getOutputSchemas(inputSchemas(d)) should have message + "Actual Value 'ts_a' and Predicted Value 'ts_b' are timestamp columns, which F1 Score " + + "cannot score as labels" + } + + it should "accept a matching TIMESTAMP pair scored by Accuracy alone" in { + val d = scorer(regression = false, "ts_a", "ts_b") + d.classificationMetrics = List(classificationMetricsFnc.accuracy) + noException should be thrownBy d.getOutputSchemas(inputSchemas(d)) + } + it should "reject a non-numeric column once the task is regression" in { val d = scorer(regression = true, "label_str", "pred_str") d.regressionMetrics = List(regressionMetricsFnc.mse)
