This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-7567-2f77d5ab93f94c8e9b33195f805a35e63c52f970 in repository https://gitbox.apache.org/repos/asf/texera.git
commit 490e57f2cc384581401f56aa9eafb5cd7af03d7d Author: Kary Zheng <[email protected]> AuthorDate: Fri Aug 14 09:41:00 2026 +0000 feat(visualization): drop rows with missing values before laying out a Dumbbell Plot (#7567) ### What changes were proposed in this PR? Dumbbell Plot took the distinct values of the compared column and sorted them. An empty cell reaches the operator as `None`, so `unique()` returned a list with a `None` in it and `sorted` compared `None` against a string, ending the run with a TypeError. An empty value is ordinary input here. A blank CSV cell arrives as null, since univocity returns null for an empty field and `AttributeTypeUtils.parseField` passes it through by design. It now drops rows missing any of the three columns it reads, which is what the other visualization operators do: twenty-four of them open their generated Python with `dropna(subset=[...]) #remove missing values`, and Dumbbell Plot was the one that did not. A table left empty by the drop renders the operator's own error rather than an exception, matching how it already reports an empty input. ### Any related issues, documentation, discussions? Closes #7562 ### How was this PR tested? `DumbbellPlotOpDescSpec` gains a case asserting the generated Python drops on all three configured columns before it sorts. It fails on the previous behavior, 7 passed / 1 failed before the change and 8 / 0 after. ### Was this PR authored or co-authored using generative AI tooling? Generated-by: Claude Code (Claude Opus 5) Co-authored-by: Claude Opus 5 (1M context) <[email protected]> Co-authored-by: Xinyuan Lin <[email protected]> --- .../dumbbellPlot/DumbbellPlotOpDesc.scala | 4 ++++ .../dumbbellPlot/DumbbellPlotOpDescSpec.scala | 27 ++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/dumbbellPlot/DumbbellPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/dumbbellPlot/DumbbellPlotOpDesc.scala index 9d62310e85..e855507e56 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/dumbbellPlot/DumbbellPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/dumbbellPlot/DumbbellPlotOpDesc.scala @@ -183,6 +183,10 @@ class DumbbellPlotOpDesc extends PythonOperatorDescriptor { | if table.empty: | yield {'html-content': self.render_error("input table is empty.")} | return + | table = table.dropna(subset=[$comparedColumnName, $categoryColumnName, $measurementColumnName]) #remove missing values + | if table.empty: + | yield {'html-content': self.render_error("input table has no rows with all of the configured columns filled in.")} + | return | ${createPlotlyDumbbellLineFigure()} | ${addPlotlyDots()} | # convert fig to html content diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/dumbbellPlot/DumbbellPlotOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/dumbbellPlot/DumbbellPlotOpDescSpec.scala index 5f32abb409..b9841984eb 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/dumbbellPlot/DumbbellPlotOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/dumbbellPlot/DumbbellPlotOpDescSpec.scala @@ -26,8 +26,14 @@ import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.nio.charset.StandardCharsets +import java.util.Base64 + class DumbbellPlotOpDescSpec extends AnyFlatSpec with Matchers { + private def b64(s: String): String = + Base64.getEncoder.encodeToString(s.getBytes(StandardCharsets.UTF_8)) + "DumbbellPlotOpDesc.operatorInfo" should "advertise the name and Basic visualization group" in { val info = (new DumbbellPlotOpDesc).operatorInfo @@ -67,6 +73,27 @@ class DumbbellPlotOpDescSpec extends AnyFlatSpec with Matchers { code should include("go.Scatter(") } + it should "drop rows missing any configured column before laying the plot out" in { + val d = new DumbbellPlotOpDesc + d.categoryColumnName = "entity" + d.measurementColumnName = "metric" + d.comparedColumnName = "phase" + d.dumbbellStartValue = "before" + d.dumbbellEndValue = "after" + val code = d.generatePythonCode() + + // Without this, an empty cell in the compared column reaches `sorted` as None + // and raises a TypeError against the other, string, entity names. + val dropna = code.linesIterator + .find(_.contains("dropna")) + .getOrElse(fail("generated code no longer drops rows with missing values")) + // The pyb macro base64-encodes interpolated column names, so the plain text + // never appears in the template. + Seq("phase", "entity", "metric").foreach(col => dropna should include(b64(col))) + + code.indexOf("dropna") should be < code.indexOf("sorted(") + } + "DumbbellPlotOpDesc.createPlotlyDumbbellLineFigure" should "select the showlegend flag from showLegends" in { val on = new DumbbellPlotOpDesc
