This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-8346-a46ddc6eb6588491eb72b27bc975f060c4cb1a75 in repository https://gitbox.apache.org/repos/asf/texera.git
commit d5a7a07673a09d0e140abd682bedd82bf26fc558 Author: Kary Zheng <[email protected]> AuthorDate: Mon Oct 19 21:57:53 2026 +0000 feat(visualization): export the charts drawn on a coordinate system (#8346) ### What changes were proposed in this PR? Eight visualizations implement `StandaloneCodeGenerator`: the polar family (polar, radar in its two forms, wind rose), the ternary pair, and the gauge and bullet indicators. What they share is the question they answer, which is where on an angle, a radius or a simplex a row lands. The ten drawn over a field or a sequence are in #8504. With those, #8344 and this one, every visualization Texera offers can be exported. Each is compared as a figure rather than as a frame, after both paths have drawn. A plot's meaning is in the numbers behind it, so that is what the comparison reads; comparing the rendered picture instead would fail over a layout detail that carries no meaning. ### Any related issues, documentation, discussions? Part of #8325, 16 of 27; that issue lists the set in order. It needs #8327 for the trait, so it does not compile until that lands, and the rows these operators add to the verification runner follow with the harness rather than as whole new files here. Closes #8419, the task this change is the whole of. ### How was this PR tested? Each operator asserts the block it emits in its own spec. Once the verification lands it is also run through the engine and through its generated script, on every configuration its schema offers, and the two answers compared; this branch is cut from main and does not carry that machinery, so those runs are not on this diff's CI. ### Was this PR authored or co-authored using generative AI tooling? Generated-by: Claude Code (Opus 5) --------- Co-authored-by: Claude Opus 5 (1M context) <[email protected]> --- .../bulletChart/BulletChartOpDesc.scala | 136 ++++++++++++++++++++- .../gaugeChart/GaugeChartOpDesc.scala | 118 +++++++++++++++++- .../polarChart/PolarChartOpDesc.scala | 43 ++++++- .../radarChart/RadarChartOpDesc.scala | 64 +++++++++- .../visualization/radarPlot/RadarPlotOpDesc.scala | 110 ++++++++++++++++- .../ternaryContour/TernaryContourOpDesc.scala | 57 ++++++++- .../ternaryPlot/TernaryPlotOpDesc.scala | 39 +++++- .../windRoseChart/WindRoseChartOpDesc.scala | 39 +++++- .../bulletChart/BulletChartOpDescSpec.scala | 7 ++ .../gaugeChart/GaugeChartOpDescSpec.scala | 135 ++++++++++++++++++++ .../polarChart/PolarChartOpDescSpec.scala | 135 ++++++++++++++++++++ 11 files changed, 865 insertions(+), 18 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/bulletChart/BulletChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/bulletChart/BulletChartOpDesc.scala index 19d02af3e5..14277cece2 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/bulletChart/BulletChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/bulletChart/BulletChartOpDesc.scala @@ -23,10 +23,14 @@ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.{EncodableString, PythonLiteral} import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} @@ -48,7 +52,7 @@ import scala.jdk.CollectionConverters._ } } """) -class BulletChartOpDesc extends PythonOperatorDescriptor { +class BulletChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "value", required = true) @JsonSchemaTitle("Value") @@ -92,6 +96,11 @@ class BulletChartOpDesc extends PythonOperatorDescriptor { OperatorGroupConstants.VISUALIZATION_FINANCIAL_GROUP ) + private def assertRequiredFields(): Unit = { + assert(value.nonEmpty) + assert(deltaReference.nonEmpty) + } + override def generatePythonCode(): String = { // The reference keeps the 0 the generated code used to fall back to; an unset // threshold stays absent as None. @@ -237,4 +246,127 @@ class BulletChartOpDesc extends PythonOperatorDescriptor { |""" finalCode.encode } + + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + assertRequiredFields() + + // The column name is typed-in text, so it is emitted as a properly escaped + // Python literal. The numeric settings carry no text and are emitted as numbers, + // the same as the runtime path, so the two scripts agree without either of them + // parsing anything. + val valueLit = pyStringLiteral(value) + val deltaReferenceExpr = deltaReference.getOrElse(0.0).toString + val thresholdExpr = thresholdValue.map(_.toString).getOrElse("None") + val stepsExpr = + Option(steps) + .map(_.asScala.toSeq) + .getOrElse(Seq.empty) + .flatMap(step => step.start.zip(step.end)) + .map { case (start, end) => s"""{"start": $start, "end": $end}""" } + .mkString("[", ", ", "]") + + // render_error's continuation line keeps the runtime path's indentation — the + // HTML is triple-quoted, so those spaces reach the browser. + s"""def render_error(error_msg): + | return '''<h1>Bullet chart is not available.</h1> + | <p>Reason: {} </p>'''.format(error_msg) + | + |def generate_gray_gradient(step_count): + | colors = [] + | for i in range(step_count): + | lightness = 90 - (i * (60 / max(1, step_count - 1))) + | colors.append(f"hsl(0, 0%, {lightness}%)") + | return colors + | + |def generate_valid_steps(steps_data): + | valid_steps = [] + | step_errors = [] + | for index, step in enumerate(steps_data): + | s_val = step["start"] + | e_val = step["end"] + | if s_val < e_val: + | valid_steps.append({"start": s_val, "end": e_val}) + | else: + | step_errors.append(f"Step {index + 1}: start >= end ({s_val} >= {e_val})") + | return valid_steps, step_errors + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("Input table is empty.")) + |else: + | try: + | value_col = $valueLit + | delta_ref = $deltaReferenceExpr + | if value_col not in in1df.columns: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error(f"Column '{value_col}' not found in input table.")) + | else: + | table = in1df.dropna(subset=[value_col]) + | if table.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("No valid data rows found after dropping nulls.")) + | else: + | threshold_val = $thresholdExpr + | valid_steps, step_errors = generate_valid_steps($stepsExpr) + | step_colors = generate_gray_gradient(len(valid_steps)) + | steps_list = [] + | for index, step_data in enumerate(valid_steps): + | steps_list.append({ + | "range": [step_data["start"], step_data["end"]], + | "color": step_colors[index] + | }) + | html_chunks = [] + | figs = [] + | for _, row in table.head(10).iterrows(): + | try: + | actual = float(row[value_col]) + | gauge_config = {'shape': 'bullet'} + | if steps_list: + | gauge_config['steps'] = steps_list + | max_range_values = [actual, delta_ref] + | if threshold_val is not None: + | max_range_values.append(threshold_val) + | for r in steps_list: + | max_range_values.append(r["range"][1]) + | gauge_config['axis'] = {"range": [0, max(max_range_values) * 1.2]} + | if threshold_val is not None: + | gauge_config["threshold"] = { + | "value": threshold_val, + | "line": {"color": "red", "width": 2}, + | "thickness": 1 + | } + | fig = go.Figure(go.Indicator( + | mode="number+gauge+delta", + | value=actual, + | delta={"reference": delta_ref}, + | gauge=gauge_config, + | domain={"x": [0.1, 1], "y": [0.1, 0.9]}, + | title={"text": value_col} + | )) + | fig.update_layout(margin=dict(l=80, r=20, b=40, t=40), height=150) + | figs.append(fig) + | html_chunk = plotly.io.to_html(fig, include_plotlyjs='cdn', auto_play=False) + | if step_errors: + | html_chunk += "<br><b>Step Errors:</b><ul>" + "".join([f"<li>{msg}</li>" for msg in step_errors]) + "</ul>" + | html_chunks.append(html_chunk) + | except Exception as e: + | html_chunks.append(render_error(f"Error generating bullet chart: {str(e)}")) + | # One chart per row, so the file holds the whole sequence. + | # A single figure still writes as a lone object, which is + | # what every one-chart operator writes. + | if len(figs) == 1: + | figs[0].write_json(outputJson) + | elif figs: + | with open(outputJson, "w", encoding="utf-8") as output: + | output.write( + | "[" + ",".join(f.to_json() for f in figs) + "]" + | ) + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write("<div>" + "".join(html_chunks) + "</div>") + | except Exception as e: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error(f"General error: {str(e)}"))""".stripMargin + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/gaugeChart/GaugeChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/gaugeChart/GaugeChartOpDesc.scala index 320e2fc564..324d7c6840 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/gaugeChart/GaugeChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/gaugeChart/GaugeChartOpDesc.scala @@ -22,16 +22,20 @@ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.{EncodableString, PythonLiteral} import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import javax.validation.constraints.NotNull -class GaugeChartOpDesc extends PythonOperatorDescriptor { +class GaugeChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "value", required = true) @JsonSchemaTitle("Gauge Value") @JsonPropertyDescription("The primary value displayed on the gauge chart") @@ -190,4 +194,114 @@ class GaugeChartOpDesc extends PythonOperatorDescriptor { | yield {'html-content': self.render_error(f"General error: {str(e)}")} |""".encode } + + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + // The column name is typed-in text, so it is emitted as a properly escaped + // Python literal — hand-written quotes would let a quote, backslash or newline + // in it break the script the runtime path renders fine. The numeric settings + // carry no text and are emitted as numbers, the same as the runtime path. + val valueLit = pyStringLiteral(value) + val deltaExpr = numberOrNone(delta) + val thresholdExpr = numberOrNone(threshold) + val stepsExpr = stepsLiteral + // render_error's continuation line keeps the runtime path's indentation — the + // HTML is triple-quoted, so those spaces reach the browser. + s"""import plotly.graph_objects as go + |import plotly.io as pio + | + |def render_error(error_msg): + | return '''<h1>Gauge chart is not available.</h1> + | <p>Reason: {} </p>'''.format(error_msg) + | + |def generate_gray_gradient(step_count): + | colors = [] + | for i in range(step_count): + | lightness = 90 - (i * (60 / max(1, step_count - 1))) + | colors.append(f"hsl(0, 0%, {lightness}%)") + | return colors + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("Input table is empty.")) + |else: + | # Everything the operator draws inside its own try, so anything it did + | # not foresee (a column that is not there, a step that is not a number) + | # is a page saying so rather than the end of the exported run. + | try: + | gauge_value = $valueLit + | delta_ref = $deltaExpr + | threshold_val = $thresholdExpr + | table = in1df.dropna(subset=[gauge_value]) + | if table.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("No non-null rows found for the value column.")) + | else: + | valid_steps = $stepsExpr + | step_colors = generate_gray_gradient(len(valid_steps)) + | steps_list = [] + | for index, step_data in enumerate(valid_steps): + | color = step_colors[index] + | steps_list.append({ + | "range": [step_data["start"], step_data["end"]], + | "color": color + | }) + | + | html_chunks = [] + | figs = [] + | for _, row in table.iterrows(): + | try: + | actual = float(row[gauge_value]) + | max_val = actual + | if delta_ref is not None: + | max_val = max(max_val, delta_ref) + | if threshold_val is not None: + | max_val = max(max_val, threshold_val) + | if steps_list: + | for r in steps_list: + | max_val = max(max_val, r["range"][1]) + | gauge_config = {'axis': {'range': [None, max_val * 1.2]}} + | if steps_list: + | gauge_config['steps'] = steps_list + | if threshold_val is not None: + | gauge_config['threshold'] = { + | "value": threshold_val, + | "line": {"color": "red", "width": 3}, + | "thickness": 0.75 + | } + | mode_parts = ["number", "gauge"] + | if delta_ref is not None: + | mode_parts.append("delta") + | mode = "+".join(mode_parts) + | delta_config = {"reference": delta_ref} if delta_ref is not None else None + | fig = go.Figure(go.Indicator( + | mode=mode, + | value=actual, + | delta=delta_config, + | gauge=gauge_config, + | domain={"x": [0, 1], "y": [0, 1]}, + | title={"text": gauge_value} + | )) + | fig.update_layout(margin=dict(l=20, r=20, b=40, t=60), height=250) + | figs.append(fig) + | html_chunks.append(pio.to_html(fig, include_plotlyjs='cdn', auto_play=False)) + | except Exception as e: + | html_chunks.append(render_error(f"Error generating chart: {str(e)}")) + | + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write("<div>" + "".join(html_chunks) + "</div>") + | # One chart per row, so the file holds the whole sequence. A + | # single figure still writes as a lone object, which is what + | # every one-chart operator writes. + | if len(figs) == 1: + | figs[0].write_json(outputJson) + | elif figs: + | with open(outputJson, "w", encoding="utf-8") as output: + | output.write("[" + ",".join(f.to_json() for f in figs) + "]") + | print("Gauge chart saved to " + outputHtml) + | except Exception as e: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error(f"General error: {str(e)}"))""".stripMargin + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/polarChart/PolarChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/polarChart/PolarChartOpDesc.scala index 707350497a..470e58b102 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/polarChart/PolarChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/polarChart/PolarChartOpDesc.scala @@ -25,10 +25,14 @@ import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.workflow.OutputPort.OutputMode import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PortIdentity} import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import javax.validation.constraints.NotNull @@ -43,7 +47,7 @@ import javax.validation.constraints.NotNull } } """) -class PolarChartOpDesc extends PythonOperatorDescriptor { +class PolarChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "r", required = true) @JsonSchemaTitle("r") @@ -125,4 +129,39 @@ class PolarChartOpDesc extends PythonOperatorDescriptor { |""" finalCode.encode } + + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val rLit = pyStringLiteral(r) + val thetaLit = pyStringLiteral(theta) + // np.issubdtype cannot read a nullable pandas dtype such as Int64 and raises; + // the bool exclusion keeps what np.number refused. + s"""from pandas.api.types import is_bool_dtype, is_numeric_dtype + | + |def _is_number(column): + | return is_numeric_dtype(column) and not is_bool_dtype(column) + | + |if in1df is None or in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write('<h3>No data available for Polar Chart</h3>') + |elif $rLit not in in1df.columns or $thetaLit not in in1df.columns: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write('<h3>Selected columns not found in input table</h3>') + |elif not _is_number(in1df[$rLit]) or not _is_number(in1df[$thetaLit]): + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write('<h3>Selected columns must be numeric</h3>') + |else: + | fig = go.Figure(data=go.Scatterpolargl( + | r=in1df[$rLit].values, + | theta=in1df[$thetaLit].values, + | mode='markers', + | marker=dict(size=10, opacity=0.7, line=dict(color='white')) + | )) + | fig.update_layout(title='Polar Chart', showlegend=False) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Polar chart saved to " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/radarChart/RadarChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/radarChart/RadarChartOpDesc.scala index f9fcbc09c2..920909993a 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/radarChart/RadarChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/radarChart/RadarChartOpDesc.scala @@ -24,6 +24,7 @@ import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchema import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.{ AutofillAttributeName, AutofillAttributeNameList @@ -31,7 +32,10 @@ import org.apache.texera.amber.operator.metadata.annotations.{ import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.pybuilder.PythonTemplateBuilder -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import javax.validation.constraints.{NotEmpty, NotNull} @@ -45,7 +49,7 @@ import javax.validation.constraints.{NotEmpty, NotNull} } } """) -class RadarChartOpDesc extends PythonOperatorDescriptor { +class RadarChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "nameColumn", required = true) @JsonSchemaTitle("Name Column") @@ -162,4 +166,60 @@ class RadarChartOpDesc extends PythonOperatorDescriptor { |""" finalcode.encode } + + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val valueColsList = valueColumns.map(pyStringLiteral).mkString("[", ", ", "]") + val requiredCols = + (Seq(pyStringLiteral(nameColumn)) ++ valueColumns.map(pyStringLiteral)) + .mkString("[", ", ", "]") + s"""def render_error(error_msg): + | return '''<h1>RadarChart is not available.</h1> + | <p>Reason is: {} </p> + | '''.format(error_msg) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("input table is empty.")) + |else: + | table = in1df.copy() + | required_cols = $requiredCols + | table.dropna(subset=required_cols, inplace=True) + | value_cols = $valueColsList + | for col in value_cols: + | table[col] = pd.to_numeric(table[col], errors='coerce') + | table.dropna(subset=value_cols, inplace=True) + | if table.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("input table is empty after removing missing values.")) + | else: + | fig = go.Figure() + | categories = $valueColsList + | for idx, row in table.iterrows(): + | values = [row[col] for col in categories] + | values.append(values[0]) + | categories_closed = categories + [categories[0]] + | fig.add_trace(go.Scatterpolar( + | r=values, + | theta=categories_closed, + | fill='toself', + | name=str(row[${pyStringLiteral(nameColumn)}]), + | opacity=$fillOpacity + | )) + | fig.update_layout( + | polar=dict( + | radialaxis=dict( + | visible=True, + | range=[0, None] + | ) + | ), + | showlegend=True, + | margin=dict(t=40, b=40, l=40, r=40) + | ) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Radar chart saved to " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/radarPlot/RadarPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/radarPlot/RadarPlotOpDesc.scala index c21c6f3b31..659e41097d 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/radarPlot/RadarPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/radarPlot/RadarPlotOpDesc.scala @@ -28,10 +28,14 @@ import org.apache.texera.amber.operator.metadata.annotations.{ } import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.pybuilder.PythonTemplateBuilder -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import javax.validation.constraints.NotEmpty @@ -44,7 +48,7 @@ import javax.validation.constraints.NotEmpty } } """) -class RadarPlotOpDesc extends PythonOperatorDescriptor { +class RadarPlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "selectedAttributes", required = true) @JsonSchemaTitle("Axes") @JsonPropertyDescription("Numeric columns to use as radar axes") @@ -231,4 +235,106 @@ class RadarPlotOpDesc extends PythonOperatorDescriptor { |""" finalCode.encode } + + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val attributes = Option(selectedAttributes).getOrElse(Nil) + val attrList = attributes.map(pyStringLiteral).mkString(", ") + def optionalColumn(column: EncodableString): String = + Option(column).filterNot(col => col.isEmpty || col == "No Selection") match { + case Some(col) => pyStringLiteral(col) + case None => "None" + } + val traceNameCol = optionalColumn(traceNameAttribute) + val traceColorCol = optionalColumn(traceColorAttribute) + + // render_error's continuation line keeps the runtime path's indentation — the + // HTML is triple-quoted, so those spaces reach the browser. + s"""import numpy as np + | + |def render_error(error_msg): + | return '''<h1>Radar Plot is not available.</h1> + | <p>Reason is: {} </p> + | '''.format(error_msg) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("Input table is empty.")) + |else: + | categories = [$attrList] + | if not categories: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("No columns selected as axes.")) + | else: + | table = in1df + | trace_name_col = $traceNameCol + | trace_color_col = $traceColorCol + | line_pattern = "${linePattern.getLinePattern}" + | max_normalize = ${toPythonBool(maxNormalize)} + | fill_trace = ${toPythonBool(fillTrace)} + | show_markers = ${toPythonBool(showMarkers)} + | show_legend = ${toPythonBool(showLegend)} + | + | selected_table_df = table[categories].astype(float) + | selected_table = selected_table_df.values + | + | trace_names = ( + | table[trace_name_col].values if trace_name_col + | else np.full(len(table), "", dtype=object) + | ) + | + | trace_colors = [None] * len(table) + | if trace_color_col: + | unique_vals = table[trace_color_col].unique() + | color_map = {val: px.colors.qualitative.Plotly[idx % len(px.colors.qualitative.Plotly)] + | for idx, val in enumerate(unique_vals)} + | nan_color = '#000000' + | trace_colors = table[trace_color_col].map(color_map).fillna(nan_color).values + | + | hover_texts = [] + | for idx, row in enumerate(selected_table): + | name_prefix = str(trace_names[idx]) + "<br>" if trace_names[idx] else "" + | row_hover_texts = [] + | for attr, value in zip(categories, row): + | row_hover_texts.append(name_prefix + attr + ": " + str(value)) + | hover_texts.append(row_hover_texts) + | + | if max_normalize: + | max_vals = selected_table_df.max().values + | max_vals[max_vals == 0] = 1 + | selected_table = selected_table / max_vals + | + | selected_table = np.nan_to_num(selected_table) + | + | fig = go.Figure() + | + | for idx, row in enumerate(selected_table): + | # To connect ensure all points in the radar trace are connected + | closed_row = row.tolist() + [row[0]] + | closed_categories = categories + [categories[0]] + | closed_hover_texts = hover_texts[idx] + [hover_texts[idx][0]] + | + | fig.add_trace(go.Scatterpolar( + | r=closed_row, + | theta=closed_categories, + | fill='toself' if fill_trace else 'none', + | name=str(trace_names[idx]) if trace_names[idx] else "", + | text=closed_hover_texts, + | hoverinfo="text", + | mode="lines+markers" if show_markers else "lines", + | line=dict(dash=line_pattern, color=trace_colors[idx] if trace_colors[idx] else None), + | marker=dict(color=trace_colors[idx]) if trace_colors[idx] else {} + | )) + | + | fig.update_layout( + | polar=dict(radialaxis=dict(visible=True)), + | showlegend=show_legend, + | width=600, + | height=600 + | ) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Radar plot saved to " + outputHtml)""".stripMargin + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ternaryContour/TernaryContourOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ternaryContour/TernaryContourOpDesc.scala index 93771d4e1b..b0b54ccbc4 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ternaryContour/TernaryContourOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ternaryContour/TernaryContourOpDesc.scala @@ -23,11 +23,14 @@ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.workflow.OutputPort.OutputMode -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PortIdentity} -import org.apache.texera.amber.operator.PythonOperatorDescriptor -import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName +import org.apache.texera.amber.operator.{PythonOperatorDescriptor, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.metadata.annotations.{AutofillAttributeName, SampleColumn} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder @@ -52,13 +55,14 @@ import javax.validation.constraints.NotNull } } """) -class TernaryContourOpDesc extends PythonOperatorDescriptor { +class TernaryContourOpDesc extends PythonOperatorDescriptor with StandaloneCodeGenerator { // Add annotations for the first variable @JsonProperty(value = "firstVariable", required = true) @JsonSchemaTitle("Variable 1") @JsonPropertyDescription("First variable data field") @AutofillAttributeName + @SampleColumn("a\"b\\c_simplex_a") @NotNull(message = "Variable 1 cannot be empty") var firstVariable: EncodableString = "" @@ -67,6 +71,7 @@ class TernaryContourOpDesc extends PythonOperatorDescriptor { @JsonSchemaTitle("Variable 2") @JsonPropertyDescription("Second variable data field") @AutofillAttributeName + @SampleColumn("a\"b\\c_simplex_b") @NotNull(message = "Variable 2 cannot be empty") var secondVariable: EncodableString = "" @@ -75,6 +80,7 @@ class TernaryContourOpDesc extends PythonOperatorDescriptor { @JsonSchemaTitle("Variable 3") @JsonPropertyDescription("Third variable data field") @AutofillAttributeName + @SampleColumn("a\"b\\c_simplex_c") @NotNull(message = "Variable 3 cannot be empty") var thirdVariable: EncodableString = "" @@ -171,4 +177,47 @@ class TernaryContourOpDesc extends PythonOperatorDescriptor { finalCode.encode } + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val firstLit = pyStringLiteral(firstVariable) + val secondLit = pyStringLiteral(secondVariable) + val thirdLit = pyStringLiteral(thirdVariable) + val fourthLit = pyStringLiteral(fourthVariable) + s"""import plotly.figure_factory as ff + |import numpy as np + | + |def render_error(error_msg): + | return '''<h1>TernaryContour is not available.</h1> + | <p>Reasons are: {} </p> + | '''.format(error_msg) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("Input table is empty.")) + |else: + | table = in1df.copy() + | table.dropna(subset=[$firstLit, $secondLit, $thirdLit, $fourthLit], inplace=True) + | table = table[(table[[$firstLit, $secondLit, $thirdLit]] >= 0).all(axis=1)] + | s = table[$firstLit] + table[$secondLit] + table[$thirdLit] + | table = table[s > 0] + | if table.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("No valid rows left (every row has at least 1 missing value).")) + | else: + | A = table[$firstLit].to_numpy() + | B = table[$secondLit].to_numpy() + | C = table[$thirdLit].to_numpy() + | Z = table[$fourthLit].to_numpy() + | fig = ff.create_ternary_contour( + | np.array([A, B, C]), + | Z, + | pole_labels=[$firstLit, $secondLit, $thirdLit], + | interp_mode='cartesian' + | ) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Ternary contour saved to " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ternaryPlot/TernaryPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ternaryPlot/TernaryPlotOpDesc.scala index 14e7c7fad1..e3ee55fbe9 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ternaryPlot/TernaryPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ternaryPlot/TernaryPlotOpDesc.scala @@ -26,9 +26,11 @@ import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBui import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor -import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode +import org.apache.texera.amber.operator.metadata.annotations.{AutofillAttributeName, SampleColumn} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import javax.validation.constraints.NotNull @@ -51,13 +53,14 @@ import javax.validation.constraints.NotNull } } """) -class TernaryPlotOpDesc extends PythonOperatorDescriptor { +class TernaryPlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { // Add annotations for the first variable @JsonProperty(value = "firstVariable", required = true) @JsonSchemaTitle("Variable 1") @JsonPropertyDescription("First variable data field") @AutofillAttributeName + @SampleColumn("comp_a") @NotNull(message = "Variable 1 cannot be empty") var firstVariable: EncodableString = "" @@ -66,6 +69,7 @@ class TernaryPlotOpDesc extends PythonOperatorDescriptor { @JsonSchemaTitle("Variable 2") @JsonPropertyDescription("Second variable data field") @AutofillAttributeName + @SampleColumn("comp_b") @NotNull(message = "Variable 2 cannot be empty") var secondVariable: EncodableString = "" @@ -74,6 +78,7 @@ class TernaryPlotOpDesc extends PythonOperatorDescriptor { @JsonSchemaTitle("Variable 3") @JsonPropertyDescription("Third variable data field") @AutofillAttributeName + @SampleColumn("comp_c") @NotNull(message = "Variable 3 cannot be empty") var thirdVariable: EncodableString = "" @@ -160,4 +165,34 @@ class TernaryPlotOpDesc extends PythonOperatorDescriptor { finalCode.encode } + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val colorArg = + if (colorEnabled && colorDataField.nonEmpty) + s""", color=${pyStringLiteral(colorDataField)}""" + else "" + val firstLit = pyStringLiteral(firstVariable) + val secondLit = pyStringLiteral(secondVariable) + val thirdLit = pyStringLiteral(thirdVariable) + s"""def render_error(error_msg): + | return '''<h1>TernaryPlot is not available.</h1> + | <p>Reasons are: {} </p> + | '''.format(error_msg) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("Input table is empty.")) + |else: + | table = in1df.dropna(subset=[$firstLit, $secondLit, $thirdLit]).copy() + | if table.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("No valid rows left (every row has at least 1 missing value).")) + | else: + | fig = px.scatter_ternary(table, a=$firstLit, b=$secondLit, c=$thirdLit$colorArg) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Ternary plot saved to " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/windRoseChart/WindRoseChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/windRoseChart/WindRoseChartOpDesc.scala index 9cb36f5882..8d98ac3909 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/windRoseChart/WindRoseChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/windRoseChart/WindRoseChartOpDesc.scala @@ -27,9 +27,11 @@ import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBui import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PortIdentity} import org.apache.texera.amber.operator.PythonOperatorDescriptor -import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode +import org.apache.texera.amber.operator.metadata.annotations.{AutofillAttributeName, SampleColumn} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import javax.validation.constraints.NotNull // The radial value is the length of each wedge, so it has to be a number: given @@ -42,7 +44,7 @@ import javax.validation.constraints.NotNull } } """) -class WindRoseChartOpDesc extends PythonOperatorDescriptor { +class WindRoseChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "rColumn", required = true) @JsonSchemaTitle("Radial Values (r)") @@ -62,6 +64,7 @@ class WindRoseChartOpDesc extends PythonOperatorDescriptor { @JsonSchemaTitle("Color Group") @JsonPropertyDescription("Optional grouping column (e.g., wind strength)") @AutofillAttributeName + @SampleColumn("node_dst") var colorColumn: EncodableString = _ override def operatorInfo: OperatorInfo = @@ -137,4 +140,36 @@ class WindRoseChartOpDesc extends PythonOperatorDescriptor { finalCode.encode } + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val rLit = pyStringLiteral(rColumn) + val colorArg = + if (colorColumn != null && colorColumn.nonEmpty) s""" + | color=${pyStringLiteral(colorColumn)},""" + else "" + s"""def render_error(error_msg) -> str: + | return '''<h1>Wind Rose chart is not available.</h1> + | <p>Reason is: {} </p> + | '''.format(error_msg) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("input table is empty.")) + |elif in1df[$rLit].dtype.kind not in ["i", "u", "f"]: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("Radial column must be numeric (int, float, or double).")) + |else: + | table = in1df + | fig = px.bar_polar( + | table, + | r=$rLit, + | theta=${pyStringLiteral(thetaColumn)},$colorArg + | color_discrete_sequence=px.colors.sequential.Plasma_r + | ) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Wind rose chart saved to " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/bulletChart/BulletChartOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/bulletChart/BulletChartOpDescSpec.scala index ce6d5d32a7..9f41638b8e 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/bulletChart/BulletChartOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/bulletChart/BulletChartOpDescSpec.scala @@ -133,4 +133,11 @@ class BulletChartOpDescSpec extends AnyFlatSpec with Matchers { op.generatePythonCode() } } + + "BulletChartOpDesc.generateStandaloneCode" should "reject empty required fields" in { + val op = new BulletChartOpDesc + assertThrows[AssertionError] { + op.generateStandaloneCode() + } + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/gaugeChart/GaugeChartOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/gaugeChart/GaugeChartOpDescSpec.scala index 3e92339926..5d55f3e529 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/gaugeChart/GaugeChartOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/gaugeChart/GaugeChartOpDescSpec.scala @@ -19,15 +19,25 @@ package org.apache.texera.amber.operator.visualization.gaugeChart +import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants +import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.util.concurrent.TimeUnit +import scala.util.Try + class GaugeChartOpDescSpec extends AnyFlatSpec with Matchers { + private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) + "GaugeChartOpDesc.operatorInfo" should "advertise the name and Financial visualization group" in { val info = (new GaugeChartOpDesc).operatorInfo @@ -126,4 +136,129 @@ class GaugeChartOpDescSpec extends AnyFlatSpec with Matchers { d.generatePythonCode() should include("valid_steps = []") } + + // The operator draws inside a try and answers anything it did not foresee with + // a page. Without the same catch, a column that is not in the table reaches + // pandas as a KeyError and ends the whole exported run, not just this chart. + "GaugeChartOpDesc.generateStandaloneCode" should + "answer what it cannot draw the way the operator answers it" taggedAs NeedsPythonPackages in { + val python = resolvePythonExecutable().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") + ) + if (!canImportPandasAndPlotly(python)) { + cancel(s"'$python' cannot import pandas and plotly; skipping runtime verification") + } + + val d = new GaugeChartOpDesc + d.value = "score" + val moduleFile = Files.createTempFile("gauge_standalone_", ".py") + val driverFile = Files.createTempFile("gauge_driver_", ".py") + try { + Files.write(moduleFile, d.generateStandaloneCode().getBytes(StandardCharsets.UTF_8)) + Files.write(driverFile, driverScript.getBytes(StandardCharsets.UTF_8)) + + val process = new ProcessBuilder(python, driverFile.toString, moduleFile.toString) + .redirectErrorStream(true) + .start() + if (!process.waitFor(120, TimeUnit.SECONDS)) { + process.destroyForcibly() + fail("Gauge driver timed out after 120s") + } + val output = new String(process.getInputStream.readAllBytes(), StandardCharsets.UTF_8) + withClue(s"Driver output:\n$output\n") { + process.exitValue() shouldBe 0 + val verdicts = "CASE (\\S+) (\\S+)".r + .findAllMatchIn(output) + .map(m => m.group(1) -> m.group(2)) + .toMap + verdicts shouldBe Map( + "missing" -> "GENERAL_ERROR", + "empty" -> "EMPTY", + "nulls" -> "NO_ROWS", + "good" -> "CHART" + ) + } + } finally { + Try(Files.deleteIfExists(moduleFile)) + Try(Files.deleteIfExists(driverFile)) + () + } + } + + // Runs the exported block over one frame per case and reports the page it + // wrote, so a branch that raises instead of answering fails here. + private val driverScript: String = + """import pathlib + |import sys + |import tempfile + | + |import pandas as pd + | + |source = pathlib.Path(sys.argv[1]).read_text(encoding="utf-8") + | + |cases = { + | "good": pd.DataFrame({"score": [42.0]}), + | "missing": pd.DataFrame({"other": [42.0]}), + | "empty": pd.DataFrame({"score": []}), + | "nulls": pd.DataFrame({"score": [None, None]}), + |} + | + |for name, frame in cases.items(): + | directory = tempfile.mkdtemp() + | scope = { + | "in1df": frame, + | "outputHtml": directory + "/chart.html", + | "outputJson": directory + "/chart.json", + | "pd": pd, + | } + | exec(compile(source, "standalone", "exec"), scope) + | page = pathlib.Path(scope["outputHtml"]).read_text(encoding="utf-8") + | if "General error" in page: + | verdict = "GENERAL_ERROR" + | elif "Input table is empty" in page: + | verdict = "EMPTY" + | elif "No non-null rows" in page: + | verdict = "NO_ROWS" + | elif "Gauge chart is not available" in page: + | verdict = "OTHER_PAGE" + | else: + | verdict = "CHART" + | print("CASE %s %s" % (name, verdict)) + |""".stripMargin + + // Python executable resolution, following FilledAreaPlotOpDescSpec: + // udf.conf python.path (UDF_PYTHON_PATH), then python3 / python / py. + private def resolvePythonExecutable(): Option[String] = { + def fromConfig: Option[String] = { + val configOpt = + Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption + .orElse(Try(ConfigFactory.load()).toOption) + configOpt + .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) + .map(_.trim) + .filter(_.nonEmpty) + } + + def isRunnable(exe: String): Boolean = { + val pTry = Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()) + pTry.toOption.exists { p => + val finished = p.waitFor(5, TimeUnit.SECONDS) + if (!finished) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + } + + (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(isRunnable) + } + + private def canImportPandasAndPlotly(python: String): Boolean = { + val pTry = Try( + new ProcessBuilder(python, "-c", "import pandas, plotly").redirectErrorStream(true).start() + ) + pTry.toOption.exists { p => + val finished = p.waitFor(60, TimeUnit.SECONDS) + if (!finished) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/polarChart/PolarChartOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/polarChart/PolarChartOpDescSpec.scala index 79770d6ddd..aebe2e23ce 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/polarChart/PolarChartOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/polarChart/PolarChartOpDescSpec.scala @@ -19,15 +19,25 @@ package org.apache.texera.amber.operator.visualization.polarChart +import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants +import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.util.concurrent.TimeUnit +import scala.util.Try + class PolarChartOpDescSpec extends AnyFlatSpec with Matchers { + private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) + "PolarChartOpDesc.operatorInfo" should "advertise the name and Scientific visualization group" in { val info = (new PolarChartOpDesc).operatorInfo @@ -72,4 +82,129 @@ class PolarChartOpDescSpec extends AnyFlatSpec with Matchers { p.r shouldBe "radius" p.theta shouldBe "angle" } + + // A source read from Arrow hands over nullable dtypes such as Int64, which + // np.issubdtype cannot interpret; the check must draw them, not raise. + "PolarChartOpDesc.generateStandaloneCode" should + "draw nullable numeric columns and refuse non-numeric ones" taggedAs NeedsPythonPackages in { + val python = resolvePythonExecutable().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") + ) + if (!canImportPandasAndPlotly(python)) { + cancel(s"'$python' cannot import pandas and plotly; skipping runtime verification") + } + + val d = new PolarChartOpDesc + d.r = "r" + d.theta = "theta" + val moduleFile = Files.createTempFile("polar_standalone_", ".py") + val driverFile = Files.createTempFile("polar_driver_", ".py") + try { + Files.write(moduleFile, d.generateStandaloneCode().getBytes(StandardCharsets.UTF_8)) + Files.write(driverFile, driverScript.getBytes(StandardCharsets.UTF_8)) + + val process = new ProcessBuilder(python, driverFile.toString, moduleFile.toString) + .redirectErrorStream(true) + .start() + if (!process.waitFor(120, TimeUnit.SECONDS)) { + process.destroyForcibly() + fail("Polar driver timed out after 120s") + } + val output = new String(process.getInputStream.readAllBytes(), StandardCharsets.UTF_8) + withClue(s"Driver output:\n$output\n") { + process.exitValue() shouldBe 0 + val verdicts = "CASE (\\S+) (\\S+)".r + .findAllMatchIn(output) + .map(m => m.group(1) -> m.group(2)) + .toMap + verdicts shouldBe Map( + "numpy" -> "CHART", + "nullable" -> "CHART", + "boolean" -> "NOT_NUMERIC", + "text" -> "NOT_NUMERIC" + ) + } + } finally { + Try(Files.deleteIfExists(moduleFile)) + Try(Files.deleteIfExists(driverFile)) + () + } + } + + // Runs the exported block over one frame per case and reports the page it + // wrote, so a check that raises instead of answering fails here. + private val driverScript: String = + """import pathlib + |import sys + |import tempfile + | + |import pandas as pd + |import plotly.graph_objects as go + | + |source = pathlib.Path(sys.argv[1]).read_text(encoding="utf-8") + | + |cases = { + | "numpy": pd.DataFrame({"r": [1, 2], "theta": [0, 90]}), + | "nullable": pd.DataFrame( + | {"r": pd.array([1, 2], dtype="Int64"), "theta": pd.array([0, 90], dtype="Int64")} + | ), + | "boolean": pd.DataFrame({"r": [True, False], "theta": [0, 90]}), + | "text": pd.DataFrame({"r": [1, 2], "theta": ["a", "b"]}), + |} + | + |for name, frame in cases.items(): + | directory = tempfile.mkdtemp() + | scope = { + | "in1df": frame, + | "outputHtml": directory + "/chart.html", + | "outputJson": directory + "/chart.json", + | "pd": pd, + | "go": go, + | } + | exec(compile(source, "standalone", "exec"), scope) + | page = pathlib.Path(scope["outputHtml"]).read_text(encoding="utf-8") + | if "Selected columns must be numeric" in page: + | verdict = "NOT_NUMERIC" + | elif "<h3>" in page: + | verdict = "OTHER_PAGE" + | else: + | verdict = "CHART" + | print("CASE %s %s" % (name, verdict)) + |""".stripMargin + + // Python executable resolution, following FilledAreaPlotOpDescSpec: + // udf.conf python.path (UDF_PYTHON_PATH), then python3 / python / py. + private def resolvePythonExecutable(): Option[String] = { + def fromConfig: Option[String] = { + val configOpt = + Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption + .orElse(Try(ConfigFactory.load()).toOption) + configOpt + .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) + .map(_.trim) + .filter(_.nonEmpty) + } + + def isRunnable(exe: String): Boolean = { + val pTry = Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()) + pTry.toOption.exists { p => + val finished = p.waitFor(5, TimeUnit.SECONDS) + if (!finished) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + } + + (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(isRunnable) + } + + private def canImportPandasAndPlotly(python: String): Boolean = { + val pTry = Try( + new ProcessBuilder(python, "-c", "import pandas, plotly").redirectErrorStream(true).start() + ) + pTry.toOption.exists { p => + val finished = p.waitFor(60, TimeUnit.SECONDS) + if (!finished) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + } }
