This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-8365-2351cb856d8f67b3da3ee53f1e72381122ee7cf6 in repository https://gitbox.apache.org/repos/asf/texera.git
commit 6a101ad4623dcc81030e298a58341ec4e0d12b16 Author: Kary Zheng <[email protected]> AuthorDate: Thu Oct 15 01:21:40 2026 +0000 feat(visualization): export the charts that draw over a table's columns (#8365) ### What changes were proposed in this PR? Sixteen plots whose shape comes from the columns they are pointed at: the bar, line and pie, the histograms, the scatter family, the box and violin, the strip and the ECDF, the heat map, the candlestick, the funnel and the dot plot. A line's series configuration and an error band's are one type extending the other, so they travel together. ### Any related issues, documentation, discussions? Part of #8325, 14 of 27; that issue lists the set in order. It needs #8327 for the trait, so it does not compile until that lands, and the rows these operators add to the verification runner follow with the harness rather than as whole new files here. Closes #8417, the task this change is the whole of. ### How was this PR tested? Each operator asserts the block it emits in its own spec. Once the harness lands, each is also run through the engine and through its generated script, on every configuration its schema offers, and the two answers compared. ### Was this PR authored or co-authored using generative AI tooling? Generated-by: Claude Code (Opus 5) --------- Co-authored-by: Claude Opus 5 (1M context) <[email protected]> --- .../visualization/DotPlot/DotPlotOpDesc.scala | 45 ++++++++++- .../ScatterMatrixChartOpDesc.scala | 24 +++++- .../visualization/barChart/BarChartOpDesc.scala | 60 ++++++++++++++- .../boxViolinPlot/BoxViolinPlotOpDesc.scala | 60 ++++++++++++++- .../bubbleChart/BubbleChartOpDesc.scala | 60 ++++++++++++++- .../candlestickChart/CandlestickChartOpDesc.scala | 30 +++++++- .../continuousErrorBands/BandConfig.scala | 3 +- .../ContinuousErrorBandsOpDesc.scala | 87 +++++++++++++++++++++- .../visualization/ecdfPlot/ECDFPlotOpDesc.scala | 55 +++++++++++++- .../funnelPlot/FunnelPlotOpDesc.scala | 37 ++++++++- .../visualization/heatMap/HeatMapOpDesc.scala | 31 +++++++- .../histogram/HistogramChartOpDesc.scala | 36 ++++++++- .../histogram2d/Histogram2DOpDesc.scala | 46 +++++++++++- .../visualization/lineChart/LineChartOpDesc.scala | 58 ++++++++++++++- .../visualization/lineChart/LineConfig.scala | 3 +- .../visualization/pieChart/PieChartOpDesc.scala | 41 +++++++++- .../scatterplot/ScatterplotOpDesc.scala | 43 ++++++++++- .../stripChart/StripChartOpDesc.scala | 36 ++++++++- .../histogram2d/Histogram2DOpDescSpec.scala | 79 ++++++++++++++++++++ 19 files changed, 800 insertions(+), 34 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/DotPlot/DotPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/DotPlot/DotPlotOpDesc.scala index 42e23cbece..7ca9592844 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/DotPlot/DotPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/DotPlot/DotPlotOpDesc.scala @@ -22,17 +22,21 @@ package org.apache.texera.amber.operator.visualization.DotPlot import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder import javax.validation.constraints.NotNull -class DotPlotOpDesc extends PythonOperatorDescriptor { +class DotPlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "Count Attribute", required = true) @JsonSchemaTitle("Count Attribute") @@ -100,4 +104,41 @@ class DotPlotOpDesc extends PythonOperatorDescriptor { finalCode.encode } + // Output is an HTML chart, not a tabular DataFrame. + // The translator skips it in the leaf-DataFrame print block. + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val countLit = pyStringLiteral(countAttribute) + // The error page is written to the same file a plotted chart lands in, so a + // reason for "no chart" is where the reader looks for the chart — printing it + // to the terminal alone left that file absent. render_error's continuation + // line keeps the runtime path's indentation, since the HTML is triple-quoted + // and those spaces reach the browser. + s"""def render_error(error_msg): + | return '''<h1>DotPlot is not available.</h1> + | <p>Reasons are: {} </p> + | '''.format(error_msg) + | + |def fail(error_msg): + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error(error_msg)) + | print(f"Dot plot error: {error_msg}") + | + |if in1df.empty: + | fail("Input table is empty.") + |else: + | df = in1df.groupby([$countLit])[$countLit].count().reset_index(name='counts') + | if df.empty: + | fail("No valid rows left (every row has at least 1 missing value).") + | else: + | fig = px.strip(df, x='counts', y=$countLit, orientation='h', color=$countLit, + | color_discrete_sequence=px.colors.qualitative.Dark2) + | fig.update_traces(marker=dict(size=12, line=dict(width=2, color='DarkSlateGrey'))) + | fig.update_layout(margin=dict(t=0, b=0, l=0, r=0)) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Dot plot saved to " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ScatterMatrixChart/ScatterMatrixChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ScatterMatrixChart/ScatterMatrixChartOpDesc.scala index f30b5cee90..65ae629b95 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ScatterMatrixChart/ScatterMatrixChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ScatterMatrixChart/ScatterMatrixChartOpDesc.scala @@ -22,10 +22,14 @@ package org.apache.texera.amber.operator.visualization.ScatterMatrixChart import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.{ AutofillAttributeName, AutofillAttributeNameList @@ -46,7 +50,7 @@ import javax.validation.constraints.{NotEmpty, NotNull} } } """) -class ScatterMatrixChartOpDesc extends PythonOperatorDescriptor { +class ScatterMatrixChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "Selected Attributes", required = true) @JsonSchemaTitle("Selected Attributes") @@ -114,4 +118,20 @@ class ScatterMatrixChartOpDesc extends PythonOperatorDescriptor { finalcode.encode } + // Output is an HTML visualization, not a tabular DataFrame. + // The translator skips it in the leaf-DataFrame print block. + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val dimensions = selectedAttributes.map(pyStringLiteral).mkString(", ") + // No empty-input guard, matching generatePythonCode: it goes straight to + // the figure, so an empty table raises out of px.scatter_matrix on both + // paths. A guard here would only be a divergence — and the one removed + // here printed to stdout without writing any output at all. + s"""fig = px.scatter_matrix(in1df, dimensions=[$dimensions], color=${pyStringLiteral(color)}) + |fig.update_layout(margin=dict(t=0, b=0, l=0, r=0)) + |fig.write_json(outputJson) + |fig.write_html(outputHtml) + |print("Scatter matrix saved to " + outputHtml)""".stripMargin + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/barChart/BarChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/barChart/BarChartOpDesc.scala index 440437c9c0..f701351321 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/barChart/BarChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/barChart/BarChartOpDesc.scala @@ -22,10 +22,14 @@ package org.apache.texera.amber.operator.visualization.barChart import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder @@ -42,7 +46,7 @@ import javax.validation.constraints.NotNull } } """) -class BarChartOpDesc extends PythonOperatorDescriptor { +class BarChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "value", required = true) @JsonSchemaTitle("Value Column") @@ -155,4 +159,56 @@ class BarChartOpDesc extends PythonOperatorDescriptor { finalCode.encode } + // Output is an HTML chart, not a tabular DataFrame. + // The translator skips it in the leaf-DataFrame print block. + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val hasCategory = categoryColumn.nonEmpty && categoryColumn != "No Selection" + val colorArg = if (hasCategory) pyStringLiteral(categoryColumn) else "None" + val patternArg = if (pattern.nonEmpty) pyStringLiteral(pattern) else "None" + val fieldsLit = pyStringLiteral(fields) + val valueLit = pyStringLiteral(value) + + val barArgs = + if (horizontalOrientation) + s"""y=$fieldsLit, x=$valueLit, color=$colorArg, pattern_shape=$patternArg, orientation='h'""" + else + s"""y=$valueLit, x=$fieldsLit, color=$colorArg, pattern_shape=$patternArg""" + + // The error page is written to the same file a plotted chart lands in, so a + // reason for "no chart" is where the reader looks for the chart — printing it + // to the terminal alone left that file absent. render_error's continuation + // line keeps the runtime path's indentation, since the HTML is triple-quoted + // and those spaces reach the browser. + s"""def render_error(error_msg): + | return '''<h1>Bar chart is not available.</h1> + | <p>Reason is: {} </p> + | '''.format(error_msg) + | + |def fail(error_msg): + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error(error_msg)) + | print(f"Bar chart error: {error_msg}") + | + |if $fieldsLit == $valueLit: + | fail("Fields should not have the same value.") + |elif in1df.empty: + | # Checked before the dropna, unlike the runtime path: an empty table read + | # back from JSONL carries no columns at all, so naming them in `subset` + | # would raise a KeyError instead of reporting the empty table. + | fail("Table should not have any empty/null values or fields.") + |else: + | # Bound to a name of its own: the same frame can feed another branch + | # of the plan, which must still see every row. + | chart_df = in1df.dropna(subset=[$valueLit, $fieldsLit]) + | if chart_df.empty: + | fail("Table should not have any empty/null values or fields.") + | else: + | fig = go.Figure(px.bar(chart_df, $barArgs)) + | fig.update_layout(margin=dict(l=0, r=0, t=0, b=0)) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Bar chart saved to " + outputJson + " and " + outputHtml)""".stripMargin + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/boxViolinPlot/BoxViolinPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/boxViolinPlot/BoxViolinPlotOpDesc.scala index ec92ea8445..1b1200b0e8 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/boxViolinPlot/BoxViolinPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/boxViolinPlot/BoxViolinPlotOpDesc.scala @@ -22,10 +22,14 @@ package org.apache.texera.amber.operator.visualization.boxViolinPlot import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription, JsonPropertyOrder} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder @@ -42,7 +46,7 @@ import javax.validation.constraints.NotNull } } """) -class BoxViolinPlotOpDesc extends PythonOperatorDescriptor { +class BoxViolinPlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "value", required = true) @JsonSchemaTitle("Value Column") @@ -152,4 +156,56 @@ class BoxViolinPlotOpDesc extends PythonOperatorDescriptor { finalCode.encode } + // Output is a Plotly visualization, not a tabular DataFrame. + // The translator skips it in the leaf-DataFrame print block. + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val horizontal = if (horizontalOrientation) "True" else "False" + val violin = if (violinPlot) "True" else "False" + val quartileMethod = + if (quartileType == null) "linear" else quartileType.getQuartiletype + val valueLit = pyStringLiteral(value) + + // The error page is written to the same file a plotted chart lands in, so a + // reason for "no chart" is where the reader looks for the chart — printing it + // to the terminal alone left that file absent. render_error's continuation + // line keeps the runtime path's indentation, since the HTML is triple-quoted + // and those spaces reach the browser. + s"""def render_error(error_msg): + | return '''<h1>Box/Violin Plot is not available.</h1> + | <p>Reason is: {} </p> + | '''.format(error_msg) + | + |def fail(error_msg): + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error(error_msg)) + | print(f"Box/Violin Plot error: {error_msg}") + | + |if in1df.empty: + | fail("input table is empty.") + |else: + | # Bound to a name of its own: the same frame can feed another branch + | # of the plan, which must still see every row. + | chart_df = in1df.dropna(subset=[$valueLit]) + | if chart_df.empty: + | fail("value column contains only non-positive numbers or nulls.") + | else: + | if $violin: + | if $horizontal: + | fig = px.violin(chart_df, x=$valueLit, box=True, points='all') + | else: + | fig = px.violin(chart_df, y=$valueLit, box=True, points='all') + | else: + | if $horizontal: + | fig = px.box(chart_df, x=$valueLit, boxmode="overlay", points='all') + | else: + | fig = px.box(chart_df, y=$valueLit, boxmode="overlay", points='all') + | fig.update_traces(quartilemethod=${pyStringLiteral(quartileMethod)}, col=1) + | fig.update_layout(margin=dict(t=0, b=0, l=0, r=0)) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Box/Violin Plot saved to " + outputJson + " and " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/bubbleChart/BubbleChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/bubbleChart/BubbleChartOpDesc.scala index edb83c2bbb..be05f4dfec 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/bubbleChart/BubbleChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/bubbleChart/BubbleChartOpDesc.scala @@ -22,10 +22,14 @@ package org.apache.texera.amber.operator.visualization.bubbleChart import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder @@ -49,7 +53,7 @@ import javax.validation.constraints.NotNull } } """) -class BubbleChartOpDesc extends PythonOperatorDescriptor { +class BubbleChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "xValue", required = true) @JsonSchemaTitle("X-Column") @@ -157,4 +161,56 @@ class BubbleChartOpDesc extends PythonOperatorDescriptor { |""" finalCode.encode } + + // Output is an HTML chart, not a tabular DataFrame. + // The translator skips it in the leaf-DataFrame print block. + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + // Same rule as the platform path: an unset column is "no color" even with the + // toggle on, else px.scatter(color='') fails. + val colorArg = + if (enableColor && colorCategory.nonEmpty) s""", color=${pyStringLiteral(colorCategory)}""" + else "" + val xLit = pyStringLiteral(xValue) + val yLit = pyStringLiteral(yValue) + val zLit = pyStringLiteral(zValue) + + // The error page is written to the same file a plotted chart lands in, so a + // reason for "no chart" is where the reader looks for the chart — printing it + // to the terminal alone left that file absent. The heading is the + // runtime path's, TreeMap and all, so both paths say the same thing. + // render_error's continuation line keeps the runtime path's indentation, since + // the HTML is triple-quoted and those spaces reach the browser. + s"""def render_error(error_msg): + | return '''<h1>TreeMap is not available.</h1> + | <p>Reasons are: {} </p> + | '''.format(error_msg) + | + |def fail(error_msg): + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error(error_msg)) + | print(f"Bubble chart error: {error_msg}") + | + |if in1df.empty: + | fail("Input table is empty.") + |else: + | # Bound to a name of its own: the same frame can feed another branch + | # of the plan, which must still see every row. + | chart_df = in1df.dropna(subset=[$xLit, $yLit, $zLit]) + | if chart_df.empty: + | fail("No valid rows left (every row has at least 1 missing value).") + | else: + | fig = go.Figure(px.scatter( + | chart_df, + | x=$xLit, + | y=$yLit, + | size=$zLit, + | size_max=100$colorArg + | )) + | fig.update_layout(margin=dict(l=0, r=0, b=0, t=0)) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Bubble chart saved to " + outputHtml)""".stripMargin + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/candlestickChart/CandlestickChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/candlestickChart/CandlestickChartOpDesc.scala index ccf0592576..138253f866 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/candlestickChart/CandlestickChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/candlestickChart/CandlestickChartOpDesc.scala @@ -22,21 +22,26 @@ package org.apache.texera.amber.operator.visualization.candlestickChart import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor -import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode +import org.apache.texera.amber.operator.metadata.annotations.{AutofillAttributeName, SampleColumn} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import javax.validation.constraints.NotNull -class CandlestickChartOpDesc extends PythonOperatorDescriptor { +class CandlestickChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "date", required = true) @JsonSchemaTitle("Date Column") @JsonPropertyDescription("the date of the candlestick") @AutofillAttributeName + @SampleColumn("trade_date") @NotNull(message = "Date Column cannot be empty") var date: EncodableString = "" @@ -44,6 +49,7 @@ class CandlestickChartOpDesc extends PythonOperatorDescriptor { @JsonSchemaTitle("Opening Price Column") @JsonPropertyDescription("the opening price of the candlestick") @AutofillAttributeName + @SampleColumn("open") @NotNull(message = "Opening Price Column cannot be empty") var open: EncodableString = "" @@ -51,6 +57,7 @@ class CandlestickChartOpDesc extends PythonOperatorDescriptor { @JsonSchemaTitle("Highest Price Column") @JsonPropertyDescription("the highest price of the candlestick") @AutofillAttributeName + @SampleColumn("high") @NotNull(message = "Highest Price Column cannot be empty") var high: EncodableString = "" @@ -58,6 +65,7 @@ class CandlestickChartOpDesc extends PythonOperatorDescriptor { @JsonSchemaTitle("Lowest Price Column") @JsonPropertyDescription("the lowest price of the candlestick") @AutofillAttributeName + @SampleColumn("low") @NotNull(message = "Lowest Price Column cannot be empty") var low: EncodableString = "" @@ -65,6 +73,7 @@ class CandlestickChartOpDesc extends PythonOperatorDescriptor { @JsonSchemaTitle("Closing Price Column") @JsonPropertyDescription("the closing price of the candlestick") @AutofillAttributeName + @SampleColumn("close") @NotNull(message = "Closing Price Column cannot be empty") var close: EncodableString = "" @@ -113,4 +122,19 @@ class CandlestickChartOpDesc extends PythonOperatorDescriptor { |""".encode } + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = + s"""fig = go.Figure(data=[go.Candlestick( + | x=in1df[${pyStringLiteral(date)}], + | open=in1df[${pyStringLiteral(open)}], + | high=in1df[${pyStringLiteral(high)}], + | low=in1df[${pyStringLiteral(low)}], + | close=in1df[${pyStringLiteral(close)}] + |)]) + |fig.update_layout(title='Candlestick Chart') + |fig.write_json(outputJson) + |fig.write_html(outputHtml) + |print("Candlestick chart saved to " + outputJson + " and " + outputHtml)""".stripMargin + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/continuousErrorBands/BandConfig.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/continuousErrorBands/BandConfig.scala index 9d360212bb..e0946b810b 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/continuousErrorBands/BandConfig.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/continuousErrorBands/BandConfig.scala @@ -49,7 +49,8 @@ class BandConfig extends LineConfig { @JsonPropertyDescription("must be a valid CSS color or hex color string") @JsonSchemaInject(json = """ { - "pattern": "^\\s*$|^\\s*#(?:\\s*[0-9a-fA-F]){3}(?:(?:\\s*[0-9a-fA-F]){3})?\\s*$|^\\s*(?:[rR]\\s*[gG]\\s*[bB]|[hH]\\s*[sS]\\s*[lL]|[hH]\\s*[sS]\\s*[vV])(?:\\s*[aA])?\\s*\\(\\s*(?:\\s*[0-9.])+(?:\\s*%)?(?:\\s*,(?:\\s*[0-9.])+(?:\\s*%)?){2,3}\\s*\\)\\s*$|^\\s*[vV]\\s*[aA]\\s*[rR]\\s*\\(\\s*-\\s*-[^)]*\\)\\s*$|^\\s*[a-zA-Z][a-zA-Z\\s]*$" + "pattern": "^\\s*$|^\\s*#(?:\\s*[0-9a-fA-F]){3}(?:(?:\\s*[0-9a-fA-F]){3})?\\s*$|^\\s*(?:[rR]\\s*[gG]\\s*[bB]|[hH]\\s*[sS]\\s*[lL]|[hH]\\s*[sS]\\s*[vV])(?:\\s*[aA])?\\s*\\(\\s*(?:\\s*[0-9.])+(?:\\s*%)?(?:\\s*,(?:\\s*[0-9.])+(?:\\s*%)?){2,3}\\s*\\)\\s*$|^\\s*[vV]\\s*[aA]\\s*[rR]\\s*\\(\\s*-\\s*-[^)]*\\)\\s*$|^\\s*[a-zA-Z][a-zA-Z\\s]*$", + "examples": ["red"] } """) var fillColor: EncodableString = "" diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/continuousErrorBands/ContinuousErrorBandsOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/continuousErrorBands/ContinuousErrorBandsOpDesc.scala index 016cf8ea94..cf4d8bf53b 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/continuousErrorBands/ContinuousErrorBandsOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/continuousErrorBands/ContinuousErrorBandsOpDesc.scala @@ -22,17 +22,21 @@ package org.apache.texera.amber.operator.visualization.continuousErrorBands import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder import java.util import javax.validation.constraints.NotEmpty import scala.jdk.CollectionConverters.ListHasAsScala -class ContinuousErrorBandsOpDesc extends PythonOperatorDescriptor { +class ContinuousErrorBandsOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "xLabel", required = false, defaultValue = "X Axis") @JsonSchemaTitle("X Label") @@ -153,4 +157,83 @@ class ContinuousErrorBandsOpDesc extends PythonOperatorDescriptor { |""" finalCode.encode } + + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val traces = + bands.asScala + .map { bandConf => + // Values typed into the UI become escaped Python literals: the runtime + // path splices them as decode expressions, which a standalone script has + // no decoder for, and hand-written quotes break on a value containing one. + val colorLit = pyStringLiteral(bandConf.color) + val colorPart = + if (bandConf.color != "") + s"""line={'color': $colorLit}, marker={'color': $colorLit}, """ + else "" + val fillColorPart = + if (bandConf.fillColor != "") + s"""fillcolor=${pyStringLiteral(bandConf.fillColor)}, """ + else "" + val nameLit = + pyStringLiteral(if (bandConf.name != "") bandConf.name else bandConf.yValue) + val xLit = pyStringLiteral(bandConf.xValue) + + s"""fig.add_trace(go.Scatter( + | x=in1df[$xLit], + | y=in1df[${pyStringLiteral(bandConf.yUpper)}], + | mode='lines', + | marker=dict(color="#444"), + | line=dict(width=0), + | showlegend=False, + | name=$nameLit + |)) + |fig.add_trace(go.Scatter( + | x=in1df[$xLit], + | y=in1df[${pyStringLiteral(bandConf.yLower)}], + | mode='lines', + | marker=dict(color="#444"), + | line=dict(width=0), + | fill='tonexty', + | showlegend=False, + | $fillColorPart + | name=$nameLit + |)) + |fig.add_trace(go.Scatter( + | x=in1df[$xLit], + | y=in1df[${pyStringLiteral(bandConf.yValue)}], + | mode='${bandConf.mode.getModeInPlotly}', + | $colorPart + | name=$nameLit + |))""".stripMargin + } + .mkString("\n") + + // Nested under the `else` below, so every trace line needs the extra level. + val indentedTraces = + traces.linesIterator.map(line => if (line.isEmpty) line else s" $line").mkString("\n") + + // The empty-input branch mirrors generatePythonCode's `if table.empty` + // guard: without it the traces index columns of a frame that has none, and + // an empty input silently renders a blank chart instead of saying why. + s"""def render_error(error_msg): + | return '''<h1>Continuous Error Bands is not available.</h1> + | <p>Reason is: {} </p> + | '''.format(error_msg) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("input table is empty.")) + |else: + | fig = go.Figure() + |$indentedTraces + | fig.update_layout(margin=dict(t=0, b=0, l=0, r=0), + | xaxis_title=${pyStringLiteral(xLabel)}, + | yaxis_title=${pyStringLiteral(yLabel)}, + | hovermode="x") + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Continuous error bands saved to " + outputJson + " and " + outputHtml)""".stripMargin + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ecdfPlot/ECDFPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ecdfPlot/ECDFPlotOpDesc.scala index 8461e87c25..ff5e1328c5 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ecdfPlot/ECDFPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ecdfPlot/ECDFPlotOpDesc.scala @@ -24,18 +24,22 @@ import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchema import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.pybuilder.PythonTemplateBuilder -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import javax.validation.constraints.NotNull @JsonSchemaInject( json = """{"attributeTypeRules":{"valueColumn":{"enum":["integer","long","double"]}}}""" ) -class ECDFPlotOpDesc extends PythonOperatorDescriptor { +class ECDFPlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(required = true) @JsonSchemaTitle("Value Column") @@ -183,4 +187,51 @@ class ECDFPlotOpDesc extends PythonOperatorDescriptor { |""" finalCode.encode } + + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val requiredCols = List(valueColumn, colorColumn, separateBy).filter(_.nonEmpty) + val requiredColsLiteral = requiredCols.map(pyStringLiteral).mkString("[", ", ", "]") + val valueLit = pyStringLiteral(valueColumn) + val args = scala.collection.mutable.ArrayBuffer[String]( + "table", + s"""x=$valueLit""" + ) + if (colorColumn.nonEmpty) args += s"""color=${pyStringLiteral(colorColumn)}""" + if (separateBy.nonEmpty) args += s"""facet_col=${pyStringLiteral(separateBy)}""" + yAxisMode match { + case "count" => args += "ecdfnorm=None" + case "sum" => args += "ecdfnorm=None" + case _ => + } + if (yAxisMode == "sum") args += s"""y=$valueLit""" + if (cdfMode != "standard") args += s"""ecdfmode=${pyStringLiteral(cdfMode)}""" + if (orientation == "horizontal") args += "orientation='h'" + if (showMarkers) args += "markers=True" + if (marginal != "none") args += s"""marginal=${pyStringLiteral(marginal)}""" + + s"""def render_error(error_msg): + | return '''<h1>Empirical cumulative distribution plot is not available.</h1> + | <p>Reason is: {} </p> + | '''.format(error_msg) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("input table is empty.")) + |else: + | table = in1df.dropna(subset=$requiredColsLiteral).copy() + | table[$valueLit] = pd.to_numeric(table[$valueLit], errors='coerce') + | table.dropna(subset=[$valueLit], inplace=True) + | if table.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("no valid rows left after removing missing or non-numeric values.")) + | else: + | fig = px.ecdf(${args.mkString(", ")}) + | fig.update_layout(margin=dict(l=0, r=0, t=30, b=0)) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("ECDF plot saved to " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/funnelPlot/FunnelPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/funnelPlot/FunnelPlotOpDesc.scala index eb83ed5000..805afbde97 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/funnelPlot/FunnelPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/funnelPlot/FunnelPlotOpDesc.scala @@ -22,16 +22,20 @@ package org.apache.texera.amber.operator.visualization.funnelPlot import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder import javax.validation.constraints.NotNull -class FunnelPlotOpDesc extends PythonOperatorDescriptor { +class FunnelPlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(required = true) @JsonSchemaTitle("X Column") @@ -114,4 +118,33 @@ class FunnelPlotOpDesc extends PythonOperatorDescriptor { |""" finalcode.encode } + + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val colorArg = if (color.nonEmpty) s""", color=${pyStringLiteral(color)}""" else "" + val xLit = pyStringLiteral(x) + val yLit = pyStringLiteral(y) + s"""def render_error(error_msg): + | return '''<h1>Chart is not available.</h1> + | <p>Reason is: {} </p> + | '''.format(error_msg) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("input table is empty.")) + |else: + | fig = go.Figure(px.funnel(in1df, x=$xLit, y=$yLit$colorArg)) + | fig.update_layout( + | scene=dict( + | xaxis_title="X: " + $xLit, + | yaxis_title="Y: " + $yLit, + | ), + | margin=dict(t=0, b=0, l=0, r=0) + | ) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Funnel plot saved to " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/heatMap/HeatMapOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/heatMap/HeatMapOpDesc.scala index efe4d7783b..22cf2b4f34 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/heatMap/HeatMapOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/heatMap/HeatMapOpDesc.scala @@ -22,16 +22,20 @@ package org.apache.texera.amber.operator.visualization.heatMap import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder import javax.validation.constraints.NotNull -class HeatMapOpDesc extends PythonOperatorDescriptor { +class HeatMapOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "x", required = true) @JsonSchemaTitle("Value X Column") @@ -111,4 +115,27 @@ class HeatMapOpDesc extends PythonOperatorDescriptor { finalcode.encode } + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val valueLit = pyStringLiteral(value) + val xLit = pyStringLiteral(x) + val yLit = pyStringLiteral(y) + s"""def render_error(error_msg): + | return '''<h1>Chart is not available.</h1> + | <p>Reason is: {} </p> + | '''.format(error_msg) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("input table is empty.")) + |else: + | heatmap = go.Heatmap(z=in1df[$valueLit], x=in1df[$xLit], y=in1df[$yLit]) + | layout = go.Layout(margin=dict(l=0, r=0, b=0, t=0)) + | fig = go.Figure(data=[heatmap], layout=layout) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Heatmap saved to " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/histogram/HistogramChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/histogram/HistogramChartOpDesc.scala index 9c16e43658..bfdfbb2848 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/histogram/HistogramChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/histogram/HistogramChartOpDesc.scala @@ -22,16 +22,20 @@ package org.apache.texera.amber.operator.visualization.histogram import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder import javax.validation.constraints.NotNull -class HistogramChartOpDesc extends PythonOperatorDescriptor { +class HistogramChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "value", required = true) @JsonSchemaTitle("Value Column") @JsonPropertyDescription("Column for counting values.") @@ -130,4 +134,32 @@ class HistogramChartOpDesc extends PythonOperatorDescriptor { Map(operatorInfo.outputPorts.head.id -> outputSchema) } + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val colorParam = if (color.nonEmpty) s""", color=${pyStringLiteral(color)}""" else "" + val categoryParam = + if (separateBy.nonEmpty) s""", facet_col=${pyStringLiteral(separateBy)}""" else "" + val marginalParam = + if (marginal.nonEmpty) s""", marginal=${pyStringLiteral(marginal)}""" + else "" + val patternParam = + if (pattern.nonEmpty) s""", pattern_shape=${pyStringLiteral(pattern)}""" else "" + val valueLit = pyStringLiteral(value) + s"""def render_error(error_msg): + | return '''<h1>Histogram chart is not available.</h1> + | <p>Reason is: {} </p> + | '''.format(error_msg) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("input table is empty.")) + |else: + | fig = px.histogram(in1df, x=$valueLit, text_auto=True$colorParam$categoryParam$marginalParam$patternParam) + | fig.update_layout(margin=dict(l=0, r=0, t=0, b=0)) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Histogram saved to " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/histogram2d/Histogram2DOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/histogram2d/Histogram2DOpDesc.scala index 015dce3d60..1da0ffa4f5 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/histogram2d/Histogram2DOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/histogram2d/Histogram2DOpDesc.scala @@ -21,16 +21,20 @@ package org.apache.texera.amber.operator.visualization.histogram2d import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import javax.validation.constraints.NotNull -class Histogram2DOpDesc extends PythonOperatorDescriptor { +class Histogram2DOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(required = true) @JsonSchemaTitle("X Column") @@ -120,4 +124,42 @@ class Histogram2DOpDesc extends PythonOperatorDescriptor { | yield {"html-content": html} |""".encode } + + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + assert(xBins > 0, s"X Bins must be > 0, but got $xBins") + assert(yBins > 0, s"Y Bins must be > 0, but got $yBins") + val xLit = pyStringLiteral(xColumn) + val yLit = pyStringLiteral(yColumn) + + s"""import plotly.express as px + | + |def render_error(msg): + | return f"<h1>2D Histogram failed</h1><p>{msg}</p>" + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("Input table is empty.")) + |else: + | # Bound to a name of its own: the same frame can feed another branch + | # of the plan, which must still see every row. + | chart_df = in1df.dropna(subset=[$xLit, $yLit]) + | if chart_df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("No rows after dropping nulls.")) + | else: + | fig = px.density_heatmap( + | chart_df, + | x=$xLit, + | y=$yLit, + | nbinsx=$xBins, + | nbinsy=$yBins, + | histnorm='${normalize.getValue}', + | text_auto=True + | ) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("2D histogram saved to " + outputJson + " and " + outputHtml)""".stripMargin + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/lineChart/LineChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/lineChart/LineChartOpDesc.scala index b9fa543ecf..983bfef961 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/lineChart/LineChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/lineChart/LineChartOpDesc.scala @@ -22,10 +22,14 @@ package org.apache.texera.amber.operator.visualization.lineChart import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder @@ -33,7 +37,7 @@ import java.util import javax.validation.constraints.NotEmpty import scala.jdk.CollectionConverters.ListHasAsScala -class LineChartOpDesc extends PythonOperatorDescriptor { +class LineChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "yLabel", required = false, defaultValue = "Y Axis") @JsonSchemaTitle("Y Label") @@ -129,4 +133,54 @@ class LineChartOpDesc extends PythonOperatorDescriptor { finalCode.encode } + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val traces = lines.asScala + .map { lineConf => + // Values typed into the UI become escaped Python literals: the runtime + // path splices them as decode expressions, which a standalone script has + // no decoder for, and hand-written quotes break on a value containing one. + val colorLit = pyStringLiteral(lineConf.color) + val colorPart = + if (lineConf.color != "") + s"""line={'color':$colorLit}, marker={'color':$colorLit}, """ + else "" + val namePart = + if (lineConf.name != "") s"""name=${pyStringLiteral(lineConf.name)}""" + else s"""name=${pyStringLiteral(lineConf.yValue)}""" + + s"""fig.add_trace(go.Scatter( + | x=in1df[${pyStringLiteral(lineConf.xValue)}], + | y=in1df[${pyStringLiteral(lineConf.yValue)}], + | mode='${lineConf.mode.getModeInPlotly}', + | $colorPart + | $namePart + | ))""".stripMargin + } + .mkString("\n") + // The traces sit inside the non-empty branch below, so shift them in. + .linesIterator + .map(line => if (line.isEmpty) line else s" $line") + .mkString("\n") + + s"""def render_error(error_msg) -> str: + | return '''<h1>Line chart is not available.</h1> + | <p>Reason is: {} </p> + | '''.format(error_msg) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("input table is empty.")) + |else: + | fig = go.Figure() + |$traces + | fig.update_layout(margin=dict(t=0, b=0, l=0, r=0), + | xaxis_title=${pyStringLiteral(xLabel)}, + | yaxis_title=${pyStringLiteral(yLabel)}) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Line chart saved to " + outputJson + " and " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/lineChart/LineConfig.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/lineChart/LineConfig.scala index 82b9210c98..c9c481b1b7 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/lineChart/LineConfig.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/lineChart/LineConfig.scala @@ -80,7 +80,8 @@ class LineConfig { @JsonPropertyDescription("must be a valid CSS color or hex color string") @JsonSchemaInject(json = """ { - "pattern": "^\\s*$|^\\s*#(?:\\s*[0-9a-fA-F]){3}(?:(?:\\s*[0-9a-fA-F]){3})?\\s*$|^\\s*(?:[rR]\\s*[gG]\\s*[bB]|[hH]\\s*[sS]\\s*[lL]|[hH]\\s*[sS]\\s*[vV])(?:\\s*[aA])?\\s*\\(\\s*(?:\\s*[0-9.])+(?:\\s*%)?(?:\\s*,(?:\\s*[0-9.])+(?:\\s*%)?){2,3}\\s*\\)\\s*$|^\\s*[vV]\\s*[aA]\\s*[rR]\\s*\\(\\s*-\\s*-[^)]*\\)\\s*$|^\\s*[a-zA-Z][a-zA-Z\\s]*$" + "pattern": "^\\s*$|^\\s*#(?:\\s*[0-9a-fA-F]){3}(?:(?:\\s*[0-9a-fA-F]){3})?\\s*$|^\\s*(?:[rR]\\s*[gG]\\s*[bB]|[hH]\\s*[sS]\\s*[lL]|[hH]\\s*[sS]\\s*[vV])(?:\\s*[aA])?\\s*\\(\\s*(?:\\s*[0-9.])+(?:\\s*%)?(?:\\s*,(?:\\s*[0-9.])+(?:\\s*%)?){2,3}\\s*\\)\\s*$|^\\s*[vV]\\s*[aA]\\s*[rR]\\s*\\(\\s*-\\s*-[^)]*\\)\\s*$|^\\s*[a-zA-Z][a-zA-Z\\s]*$", + "examples": ["red"] } """) var color: EncodableString = "" diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/pieChart/PieChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/pieChart/PieChartOpDesc.scala index 30f582410f..d6c47308d7 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/pieChart/PieChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/pieChart/PieChartOpDesc.scala @@ -22,11 +22,15 @@ package org.apache.texera.amber.operator.visualization.pieChart import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor -import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode +import org.apache.texera.amber.operator.metadata.annotations.{AutofillAttributeName, SampleColumn} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder @@ -42,7 +46,7 @@ import javax.validation.constraints.NotNull } } """) -class PieChartOpDesc extends PythonOperatorDescriptor { +class PieChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "value", required = true) @JsonSchemaTitle("Value Column") @@ -55,6 +59,7 @@ class PieChartOpDesc extends PythonOperatorDescriptor { @JsonSchemaTitle("Name Column") @JsonPropertyDescription("The name of the slice of pie") @AutofillAttributeName + @SampleColumn("a\"b'c\\d\ne_uniq_name") @NotNull(message = "Name Column cannot be empty") var name: EncodableString = "" @@ -128,4 +133,34 @@ class PieChartOpDesc extends PythonOperatorDescriptor { finalcode.encode } + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val nameLit = pyStringLiteral(name) + val valueLit = pyStringLiteral(value) + s"""def render_error(error_msg): + | return '''<h1>PieChart is not available.</h1> + | <p>Reason is: {} </p> + | '''.format(error_msg) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("input table is empty.")) + |else: + | table = in1df.dropna(subset=[$valueLit, $nameLit]) + | if table.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("value column contains only non-positive numbers.")) + | elif table.duplicated(subset=[$nameLit]).any(): + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("duplicates in name column, need to aggregate")) + | else: + | fig = px.pie(table, names=$nameLit, values=$valueLit) + | fig.update_traces(textposition='inside', textinfo='percent+label') + | fig.update_layout(margin=dict(t=0, b=0, l=0, r=0)) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Pie chart saved to " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/scatterplot/ScatterplotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/scatterplot/ScatterplotOpDesc.scala index f576fa895e..29f928dcc6 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/scatterplot/ScatterplotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/scatterplot/ScatterplotOpDesc.scala @@ -26,9 +26,11 @@ import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBui import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import javax.validation.constraints.NotNull @@ -45,7 +47,7 @@ import javax.validation.constraints.NotNull " }" + "}" ) -class ScatterplotOpDesc extends PythonOperatorDescriptor { +class ScatterplotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(required = true) @JsonSchemaTitle("X-Column") @@ -175,4 +177,43 @@ class ScatterplotOpDesc extends PythonOperatorDescriptor { |""" finalCode.encode } + + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val dropCols = List(xColumn, yColumn, colorColumn) + .filter(_.nonEmpty) + .map(pyStringLiteral) + .mkString("[", ", ", "]") + val args = scala.collection.mutable.ArrayBuffer[String]( + s"""x=${pyStringLiteral(xColumn)}""", + s"""y=${pyStringLiteral(yColumn)}""", + s"opacity=$alpha" + ) + if (colorColumn.nonEmpty) args += s"""color=${pyStringLiteral(colorColumn)}""" + if (xLogScale) args += "log_x=True" + if (yLogScale) args += "log_y=True" + if (hoverName.nonEmpty) args += s"""hover_name=${pyStringLiteral(hoverName)}""" + + s"""def render_error(error_msg): + | return '''<h1>Scatter Plot is not available.</h1> + | <p>Reasons are: {} </p> + | '''.format(error_msg) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("Input table is empty.")) + |else: + | table = in1df.dropna(subset=$dropCols).copy() + | if table.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("No valid rows left (every row has at least 1 missing value).")) + | else: + | fig = go.Figure(px.scatter(table, ${args.mkString(", ")})) + | fig.update_layout(margin=dict(l=0, r=0, t=0, b=0)) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Scatter plot saved to " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/stripChart/StripChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/stripChart/StripChartOpDesc.scala index c65781c44c..36507659eb 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/stripChart/StripChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/stripChart/StripChartOpDesc.scala @@ -26,12 +26,14 @@ import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBui import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import javax.validation.constraints.NotNull -class StripChartOpDesc extends PythonOperatorDescriptor { +class StripChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "x", required = true) @JsonSchemaTitle("X-Axis Column") @@ -121,4 +123,36 @@ class StripChartOpDesc extends PythonOperatorDescriptor { | yield {'html-content': html} |""".encode } + + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val xLit = pyStringLiteral(x) + val yLit = pyStringLiteral(y) + val colorByLit = pyStringLiteral(colorBy) + val facetColumnLit = pyStringLiteral(facetColumn) + val colorByParam = if (colorBy != null && colorBy.nonEmpty) s""", color=$colorByLit""" else "" + val facetColParam = + if (facetColumn != null && facetColumn.nonEmpty) s""", facet_col=$facetColumnLit""" else "" + s"""data = { + | $xLit: in1df[$xLit], + | $yLit: in1df[$yLit] + |} + |if $colorByLit: + | data[$colorByLit] = in1df[$colorByLit] + |if $facetColumnLit: + | data[$facetColumnLit] = in1df[$facetColumnLit] + | + |fig = px.strip(data, x=$xLit, y=$yLit$colorByParam$facetColParam) + |fig.update_traces(marker=dict(size=8, line=dict(width=0.5, color='DarkSlateGrey'))) + |fig.update_layout( + | xaxis_title=$xLit, + | yaxis_title=$yLit, + | hovermode='closest' + |) + |fig.write_json(outputJson) + |fig.write_html(outputHtml) + |print("Strip chart saved to " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/histogram2d/Histogram2DOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/histogram2d/Histogram2DOpDescSpec.scala index bc56ab72b1..705684de67 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/histogram2d/Histogram2DOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/histogram2d/Histogram2DOpDescSpec.scala @@ -22,12 +22,23 @@ package org.apache.texera.amber.operator.visualization.histogram2d import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants +import org.apache.texera.amber.operator.tags.IntegrationTest +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.util.concurrent.TimeUnit +import scala.io.Source +import scala.util.Try + class Histogram2DOpDescSpec extends AnyFlatSpec with Matchers { + private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) + private def configured(): Histogram2DOpDesc = { val d = new Histogram2DOpDesc d.xColumn = "x" @@ -90,4 +101,72 @@ class Histogram2DOpDescSpec extends AnyFlatSpec with Matchers { h.yBins shouldBe 5 h.normalize shouldBe NormalizationType.PROBABILITY } + + // The operator answers with a page: the engine emits `html-content`, and the two + // branches that cannot draw write that page themselves. A drawn chart owes the + // same file, and the JSON beside it is what the comparison reads. + "Histogram2DOpDesc.generateStandaloneCode" should "write the page as well as the figure" in { + val code = configured().generateStandaloneCode() + code should include("fig.write_json(outputJson)") + code should include("fig.write_html(outputHtml)") + // The branches that draw nothing, which already wrote the page. + code should include("""output.write(render_error("Input table is empty."))""") + code should include("""output.write(render_error("No rows after dropping nulls."))""") + } + + it should "leave both files behind when it runs" taggedAs NeedsPythonPackages in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandasAndPlotly(python)) + cancel(s"'$python' cannot import pandas and plotly") + + val dir = Files.createTempDirectory("histogram2d-outputs-") + val json = dir.resolve("output.json") + val html = dir.resolve("output.html") + val driver = + s"""import pandas as pd + | + |outputJson = ${pyStringLiteral(json.toString)} + |outputHtml = ${pyStringLiteral(html.toString)} + |in1df = pd.DataFrame({"x": [1.0, 2.0, 3.0, 4.0], "y": [2.0, 1.0, 4.0, 3.0]}) + |${configured().generateStandaloneCode()} + |""".stripMargin + + val script = Files.createTempFile("histogram2d-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(300, TimeUnit.SECONDS) + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + Files.exists(json) shouldBe true + Files.exists(html) shouldBe true + Files.size(json) should be > 0L + Files.size(html) should be > 0L + } + } + + private def resolvePython(): Option[String] = { + def runnable(exe: String): Boolean = + Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption + .exists { p => + if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + + (sys.env.get("UDF_PYTHON_PATH").filter(_.nonEmpty).toList ++ List( + "python3", + "python", + "py" + )).distinct + .find(runnable) + } + + private def canImportPandasAndPlotly(python: String): Boolean = + Try( + new ProcessBuilder(python, "-c", "import pandas, plotly").redirectErrorStream(true).start() + ).toOption.exists { p => + if (!p.waitFor(120, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } }
