This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-8366-a46ddc6eb6588491eb72b27bc975f060c4cb1a75 in repository https://gitbox.apache.org/repos/asf/texera.git
commit de320aab7115a2534c1ad7aaa36575da7652c6ef Author: Kary Zheng <[email protected]> AuthorDate: Mon Oct 19 21:58:18 2026 +0000 feat(visualization): export the tables, ranges and embedded documents (#8366) ### What changes were proposed in this PR? Nine of them, and what they have in common is that the output is not a plot drawn from numbers: a table rendered as a figure, a range a reader moves, an image or an HTML document passed through, and the waterfall, which reads as a chart but is built by accumulating rows rather than plotting them. The waterfall plots every row and appends the total as a bar of its own, rather than consuming the last row to make one. One of these also changes what the engine does. URL Visualizer wrote the cell straight into the iframe's `src`, so a quote in it closed the attribute and left the rest of the cell standing as attributes of the iframe. Both paths escape it now, the same five characters in the same order. Image Visualizer's exported script reads a string as ready base64 only where the column was declared BINARY, which is where the executor's own `b64encode` would have accepted it. ### Any related issues, documentation, discussions? Part of #8325, 15 of 27; that issue lists the set in order. It needs the Plotly mixin from #8365, so it does not compile until that lands. Closes #7974. Closes #8418, the task this change is the whole of. ### How was this PR tested? Each operator asserts the block it emits in its own spec. All nine have also been run through the engine and through their generated scripts, on every configuration their schemas offer, on a branch that carries this change together with the verification runner from #8362, and the two answers agreed for each of the nine. The rows these operators add to that runner arrive with it rather than as whole new files here. ### Was this PR authored or co-authored using generative AI tooling? Generated-by: Claude Code (Opus 5) --------- Co-authored-by: Claude Opus 5 (1M context) <[email protected]> --- .../ImageViz/ImageVisualizerOpDesc.scala | 63 +++++++- .../FigureFactoryTableOpDesc.scala | 47 +++++- .../visualization/htmlviz/HtmlVizOpDesc.scala | 14 +- .../nestedTable/NestedTableOpDesc.scala | 81 ++++++++++- .../ParallelCoordinatesPlotOpDesc.scala | 42 +++++- .../rangeSlider/RangeSliderOpDesc.scala | 45 +++++- .../tablesChart/TablesPlotOpDesc.scala | 40 +++++- .../visualization/urlviz/UrlVizOpDesc.scala | 35 ++++- .../visualization/urlviz/UrlVizOpExec.scala | 25 +++- .../waterfallChart/WaterfallChartOpDesc.scala | 55 +++++-- .../ImageViz/ImageVisualizerOpDescSpec.scala | 89 +++++++++++- .../FigureFactoryTableOpDescSpec.scala | 115 +++++++++++++-- .../tablesChart/TablesPlotOpDescSpec.scala | 115 +++++++++++++-- .../visualization/urlviz/UrlVizOpDescSpec.scala | 103 ++++++++++++- .../waterfallChart/WaterfallChartOpDescSpec.scala | 160 +++++++++++++++++++++ 15 files changed, 984 insertions(+), 45 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ImageViz/ImageVisualizerOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ImageViz/ImageVisualizerOpDesc.scala index 3f6fe27672..dfb7c2c1d5 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ImageViz/ImageVisualizerOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/ImageViz/ImageVisualizerOpDesc.scala @@ -22,16 +22,19 @@ package org.apache.texera.amber.operator.visualization.ImageViz import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity -import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.{PythonOperatorDescriptor, StandaloneCodeGenerator} import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder import javax.validation.constraints.NotNull -class ImageVisualizerOpDesc extends PythonOperatorDescriptor { +class ImageVisualizerOpDesc extends PythonOperatorDescriptor with StandaloneCodeGenerator { @JsonProperty(required = true) @JsonSchemaTitle("image content column") @@ -98,4 +101,58 @@ class ImageVisualizerOpDesc extends PythonOperatorDescriptor { finalCode.encode } + // Output is an HTML visualization, not a tabular DataFrame. + // The translator skips it in the leaf-DataFrame print block. + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = generateStandaloneCode(Map.empty) + + /** The executor hands `b64encode` whatever the column holds, so a column of + * text ends in the reason page: `b64encode` refuses a `str`. The exported + * script reads the same column out of JSONL, where a BINARY column arrives as + * the base64 text of its bytes, which is already what the tag wants. + * + * So the text is read as an image only where the column was declared BINARY. + * A column of any other type falls through to the same `b64encode` call and + * fails there, which is the page the run showed. + */ + override def generateStandaloneCode(inputSchemas: Map[PortIdentity, Schema]): String = { + val carriesBytes = inputSchemas + .get(operatorInfo.inputPorts.head.id) + .flatMap(_.getAttributes.find(_.getName == binaryContent)) + .forall(_.getType == AttributeType.BINARY) + val textIsTheImage = + if (carriesBytes) "isinstance(binary_image_data, str)" else "False" + s"""import base64 + | + |LT = chr(60) + |GT = chr(62) + | + |def encode_image_to_html(binary_image_data): + | try: + | if $textIsTheImage: + | encoded_image_str = binary_image_data + | else: + | encoded_image_data = base64.b64encode(binary_image_data) + | encoded_image_str = encoded_image_data.decode("utf-8") + | return ( + | f'{LT}img src="data:image;base64,{encoded_image_str}" alt="Image" ' + | f'style="max-width: 100vw; max-height: 90vh; width: auto; height: auto;"{GT}' + | ) + | except Exception: + | return ( + | f'{LT}h1{GT}Image is not available.{LT}/h1{GT}' + | f'{LT}p{GT}Reason: Binary input is not valid{LT}/p{GT}' + | ) + | + |all_images_html = f"{LT}div{GT}" + "".join( + | encode_image_to_html(binary_image_data) + | for binary_image_data in in1df[${pyStringLiteral(binaryContent)}] + |) + f"{LT}/div{GT}" + | + |with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(all_images_html) + |print("Image visualizer saved to " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/figureFactoryTable/FigureFactoryTableOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/figureFactoryTable/FigureFactoryTableOpDesc.scala index f9aa6bcd2b..4c94d322f7 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/figureFactoryTable/FigureFactoryTableOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/figureFactoryTable/FigureFactoryTableOpDesc.scala @@ -22,15 +22,18 @@ package org.apache.texera.amber.operator.visualization.figureFactoryTable import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchemaTitle} import org.apache.texera.amber.core.tuple.{AttributeType, Schema} -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity -import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.{PythonOperatorDescriptor, StandaloneCodeGenerator} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder import javax.validation.constraints.{DecimalMin, NotEmpty} -class FigureFactoryTableOpDesc extends PythonOperatorDescriptor { +class FigureFactoryTableOpDesc extends PythonOperatorDescriptor with StandaloneCodeGenerator { @JsonProperty(required = false) @JsonSchemaTitle("Font Size") @@ -44,7 +47,8 @@ class FigureFactoryTableOpDesc extends PythonOperatorDescriptor { @JsonPropertyDescription("Font color of the Figure Factory Table") @JsonSchemaInject(json = """ { - "pattern": "^\\s*$|^\\s*#(?:\\s*[0-9a-fA-F]){3}(?:(?:\\s*[0-9a-fA-F]){3})?\\s*$|^\\s*(?:[rR]\\s*[gG]\\s*[bB]|[hH]\\s*[sS]\\s*[lL]|[hH]\\s*[sS]\\s*[vV])(?:\\s*[aA])?\\s*\\(\\s*(?:\\s*[0-9.])+(?:\\s*%)?(?:\\s*,(?:\\s*[0-9.])+(?:\\s*%)?){2,3}\\s*\\)\\s*$|^\\s*[vV]\\s*[aA]\\s*[rR]\\s*\\(\\s*-\\s*-[^)]*\\)\\s*$|^\\s*[a-zA-Z][a-zA-Z\\s]*$" + "pattern": "^\\s*$|^\\s*#(?:\\s*[0-9a-fA-F]){3}(?:(?:\\s*[0-9a-fA-F]){3})?\\s*$|^\\s*(?:[rR]\\s*[gG]\\s*[bB]|[hH]\\s*[sS]\\s*[lL]|[hH]\\s*[sS]\\s*[vV])(?:\\s*[aA])?\\s*\\(\\s*(?:\\s*[0-9.])+(?:\\s*%)?(?:\\s*,(?:\\s*[0-9.])+(?:\\s*%)?){2,3}\\s*\\)\\s*$|^\\s*[vV]\\s*[aA]\\s*[rR]\\s*\\(\\s*-\\s*-[^)]*\\)\\s*$|^\\s*[a-zA-Z][a-zA-Z\\s]*$", + "examples": ["red"] } """) var fontColor: EncodableString = "#000000" @@ -147,4 +151,39 @@ class FigureFactoryTableOpDesc extends PythonOperatorDescriptor { .add("html-content", AttributeType.STRING) Map(operatorInfo.outputPorts.head.id -> outputSchema) } + + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val attributes = columns.map(c => pyStringLiteral(c.attributeName)).mkString(", ") + s"""import plotly.figure_factory as ff + | + |def render_error(error_msg): + | return "<h1>Figure Factory Table is not available.</h1><p>Reason is: {}</p>".format( + | error_msg + | ) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("input table is empty.")) + |else: + | table = in1df.dropna(subset=[$attributes]) + | if table.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("value column contains only non-positive numbers or nulls.")) + | else: + | filtered_table = table[[$attributes]] + | headers = filtered_table.columns.tolist() + | cell_values = [filtered_table[col].tolist() for col in headers] + | data = [headers] + list(map(list, zip(*cell_values))) + | fig = ff.create_table(data, height_constant=$rowHeight, font_colors=[${pyStringLiteral( + fontColor + )}]) + | for i in range(len(fig.layout.annotations)): + | fig.layout.annotations[i].font.size = $fontSize + | fig.update_layout(margin=dict(l=0, r=0, b=0, t=0)) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Figure factory table saved to " + outputHtml)""".stripMargin + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/htmlviz/HtmlVizOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/htmlviz/HtmlVizOpDesc.scala index 4775a123d5..4735fbab53 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/htmlviz/HtmlVizOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/htmlviz/HtmlVizOpDesc.scala @@ -25,9 +25,10 @@ import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{PhysicalOp, SchemaPropagationFunc} -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.metadata.annotations.{AutofillAttributeName, SampleColumn} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper import javax.validation.constraints.NotNull @@ -36,10 +37,11 @@ import javax.validation.constraints.NotNull * HTML Visualization operator to render any given HTML code * This is the description of the operator */ -class HtmlVizOpDesc extends LogicalOp { +class HtmlVizOpDesc extends LogicalOp with StandaloneCodeGenerator { @JsonProperty(required = true) @JsonSchemaTitle("HTML content") @AutofillAttributeName + @SampleColumn("short_text") @NotNull(message = "HTML content cannot be empty") var htmlContentAttrName: String = "" @@ -74,4 +76,10 @@ class HtmlVizOpDesc extends LogicalOp { OperatorGroupConstants.VISUALIZATION_MEDIA_GROUP ) + // Mirrors HtmlVizOpExec: emit one row per input row whose single + // "html-content" column is the value of htmlContentAttrName, passed through + // unconverted (the exec does not coerce either). + override def generateStandaloneCode(): String = + s"""out1df = pd.DataFrame({"html-content": in1df[${pyStringLiteral(htmlContentAttrName)}]})""" + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/nestedTable/NestedTableOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/nestedTable/NestedTableOpDesc.scala index b32cd9740a..7da846c1bf 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/nestedTable/NestedTableOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/nestedTable/NestedTableOpDesc.scala @@ -22,15 +22,16 @@ import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext import org.apache.texera.amber.core.workflow.PortIdentity -import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.{PythonOperatorDescriptor, StandaloneCodeGenerator} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import java.util import javax.validation.constraints.NotEmpty import scala.jdk.CollectionConverters.ListHasAsScala -class NestedTableOpDesc extends PythonOperatorDescriptor { +class NestedTableOpDesc extends PythonOperatorDescriptor with StandaloneCodeGenerator { @JsonPropertyDescription( "List of columns to include in the nested table chart and their subgroup" @@ -137,4 +138,80 @@ class NestedTableOpDesc extends PythonOperatorDescriptor { |""" finalcode.encode } + + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val sortedColumns = includedColumns.asScala.sortBy(_.attributeGroup) + + val multiIndexTuples = sortedColumns + .map { config => + val name = + if (config.newName != null && config.newName.nonEmpty) config.newName + else config.originalName + s"""(${pyStringLiteral(config.attributeGroup)}, ${pyStringLiteral(name)})""" + } + .mkString(",\n ") + + val rowValues = sortedColumns + .map(config => s"""row[${pyStringLiteral(config.originalName)}]""") + .mkString(", ") + + s"""import pandas as pd + |import numpy as np + | + |def render_error(error_msg): + | return '''<h1>Nested Table is not available.</h1> + | <p>Reason is: {} </p> + | '''.format(error_msg) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("input table is empty.")) + |else: + | columns = pd.MultiIndex.from_tuples([ + | $multiIndexTuples + | ]) + | + | data = [] + | for _, row in in1df.iterrows(): + | data.append([ + | $rowValues + | ]) + | + | df = pd.DataFrame(data, columns=columns) + | + | styles = [ + | {'selector': 'th', 'props': [('background-color', '#f2f2f2'), + | ('color', 'black'), + | ('font-weight', 'bold'), + | ('border', '1px solid #ddd'), + | ('padding', '8px'), + | ('text-align', 'center')]}, + | {'selector': 'td', 'props': [('border', '1px solid #ddd'), + | ('padding', '8px'), + | ('text-align', 'center')]}, + | {'selector': 'caption', 'props': [('caption-side', 'top'), + | ('font-size', '16pt'), + | ('font-weight', 'bold'), + | ('text-align', 'left'), + | ('padding', '10px')]}, + | {'selector': '.row_heading', 'props': [('text-align', 'left'), + | ('font-weight', 'normal')]}, + | {'selector': '.blank.level0', 'props': [('display', 'none')]} + | ] + | + | styled_table = ( + | df.style + | .set_table_styles(styles) + | .format(precision=2, na_rep="") + | .set_table_attributes('class="dataframe"') + | .hide(axis="index") + | ) + | + | html = styled_table.to_html() + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(html) + | print("Nested table saved to " + outputHtml)""".stripMargin + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/parallelCoordinatesPlot/ParallelCoordinatesPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/parallelCoordinatesPlot/ParallelCoordinatesPlotOpDesc.scala index 4af91c332d..e67217ae62 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/parallelCoordinatesPlot/ParallelCoordinatesPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/parallelCoordinatesPlot/ParallelCoordinatesPlotOpDesc.scala @@ -24,6 +24,7 @@ import com.kjetland.jackson.jsonSchema.annotations.{JsonSchemaInject, JsonSchema import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.{ AutofillAttributeName, AutofillAttributeNameList @@ -31,7 +32,10 @@ import org.apache.texera.amber.operator.metadata.annotations.{ import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.pybuilder.PythonTemplateBuilder -import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.{ + PythonTemplateBuilderStringContext, + pyStringLiteral +} import javax.validation.constraints.{NotNull, Size} @@ -50,7 +54,7 @@ import javax.validation.constraints.{NotNull, Size} } } """) -class ParallelCoordinatesPlotOpDesc extends PythonOperatorDescriptor { +class ParallelCoordinatesPlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "dimensions", required = true) @JsonSchemaTitle("Dimensions") @@ -134,4 +138,38 @@ class ParallelCoordinatesPlotOpDesc extends PythonOperatorDescriptor { |""" finalcode.encode } + + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val dimCols = dimensions.map(pyStringLiteral).mkString("[", ", ", "]") + val colorLit = pyStringLiteral(color) + val colorFilter = + if (color != null && color.nonEmpty) s""" & (in1df[$colorLit].notnull())""" else "" + val colorArg = + if (color != null && color.nonEmpty) s""", color=$colorLit""" else "" + + s"""def render_error(error_msg): + | return '''<h1>Parallel coordinates plot is not available.</h1> + | <p>Reason is: {} </p> + | '''.format(error_msg) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("Input table is empty.")) + |else: + | table = in1df[in1df[$dimCols].notnull().all(axis=1)$colorFilter].copy() + | if table.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("No valid rows after filtering.")) + | else: + | fig = px.parallel_coordinates( + | table, + | dimensions=$dimCols$colorArg + | ) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Parallel coordinates plot saved to " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/rangeSlider/RangeSliderOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/rangeSlider/RangeSliderOpDesc.scala index 6b1425708d..05ea9f91d2 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/rangeSlider/RangeSliderOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/rangeSlider/RangeSliderOpDesc.scala @@ -26,9 +26,11 @@ import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBui import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import javax.validation.constraints.NotNull @@ -43,7 +45,7 @@ import javax.validation.constraints.NotNull } } """) -class RangeSliderOpDesc extends PythonOperatorDescriptor { +class RangeSliderOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "Y-axis", required = true) @JsonSchemaTitle("Y-axis") @JsonPropertyDescription("The name of the column to represent y-axis") @@ -143,4 +145,45 @@ class RangeSliderOpDesc extends PythonOperatorDescriptor { finalcode.encode } + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + val functionType = duplicateType.getFunctionType + val xAxisLit = pyStringLiteral(xAxis) + val yAxisLit = pyStringLiteral(yAxis) + s"""def render_error(error_msg): + | return '''<h1>RangeChart is not available.</h1> + | <p>Reason is: {} </p> + | '''.format(error_msg) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("input table is empty.")) + |elif $yAxisLit.strip() == "" or $xAxisLit.strip() == "": + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("Y-axis or X-axis is empty")) + |else: + | table = in1df + | table = table.dropna(subset=[$xAxisLit, $yAxisLit]) + | functionType = ${pyStringLiteral(functionType)} + | if functionType.lower() == "mean": + | table = table.groupby($xAxisLit)[$yAxisLit].mean().reset_index() + | elif functionType.lower() == "sum": + | table = table.groupby($xAxisLit)[$yAxisLit].sum().reset_index() + | fig = go.Figure() + | fig.add_trace(go.Scatter(x=table[$xAxisLit], y=table[$yAxisLit], mode="markers+lines")) + | fig.update_layout( + | xaxis_title=$xAxisLit, + | yaxis_title=$yAxisLit, + | xaxis=dict( + | rangeslider=dict( + | visible=True + | ) + | ) + | ) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Range slider saved to " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesPlotOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesPlotOpDesc.scala index 7290c057dd..3c5a18d04a 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesPlotOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesPlotOpDesc.scala @@ -24,11 +24,13 @@ import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import javax.validation.constraints.NotEmpty -class TablesPlotOpDesc extends PythonOperatorDescriptor { +class TablesPlotOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonPropertyDescription("List of columns to include in the table chart") @JsonProperty(value = "add attribute", required = true) @@ -114,4 +116,40 @@ class TablesPlotOpDesc extends PythonOperatorDescriptor { .add("html-content", AttributeType.STRING) Map(operatorInfo.outputPorts.head.id -> outputSchema) } + + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + assert(includedColumns.nonEmpty) + // Mirror getAttributes: a Python list literal of the selected column names. + val columnsList = + includedColumns.map(c => pyStringLiteral(c.attributeName)).mkString("[", ", ", "]") + // The two guards mirror generatePythonCode's, which reports both conditions + // rather than rendering a table with no rows in it. + s"""def render_error(error_msg): + | return "<h1>Tables Plot is not available.</h1><p>Reason is: {}</p>".format(error_msg) + | + |attributes = $columnsList + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("input table is empty.")) + |else: + | table = in1df.dropna(subset=attributes) + | if table.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("value column contains only non-positive numbers or nulls.")) + | else: + | filtered_table = table[attributes] + | headers = filtered_table.columns.tolist() + | cell_values = [filtered_table[col].tolist() for col in headers] + | + | fig = go.Figure(data=[go.Table( + | header=dict(values=headers), + | cells=dict(values=cell_values) + | )]) + | fig.update_layout(margin=dict(l=0, r=0, b=0, t=0)) + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Tables plot saved to " + outputJson)""".stripMargin + } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpDesc.scala index 3add49a9e1..4409adbcd8 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpDesc.scala @@ -25,9 +25,10 @@ import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{PhysicalOp, SchemaPropagationFunc} -import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper import javax.validation.constraints.NotNull @@ -45,7 +46,7 @@ import javax.validation.constraints.NotNull } } """) -class UrlVizOpDesc extends LogicalOp { +class UrlVizOpDesc extends LogicalOp with StandaloneCodeGenerator { @JsonProperty(required = true) @JsonSchemaTitle("URL content") @@ -84,4 +85,34 @@ class UrlVizOpDesc extends LogicalOp { OperatorGroupConstants.VISUALIZATION_MEDIA_GROUP ) + // Mirrors UrlVizOpExec: wrap each urlContentAttrName value in the exact same + // iframe HTML document and emit it as the "html-content" column. + override def generateStandaloneCode(): String = { + val urlLit = pyStringLiteral(urlContentAttrName) + s"""import html + | + |def _texera_urlviz_iframe(u): + | # "null", not Python's "None": the operator interpolates the field into a + | # Scala string, and the JVM renders a null that way. + | u = "null" if pd.isna(u) else str(u) + | # The cell lands where an attribute is expected, so a quote in it would + | # close src= and leave the rest of the cell standing as attributes of the + | # iframe. escapeAttribute on the executor writes the same five characters + | # in the same order, so the two paths write the page the same way. + | u = html.escape(u, quote=True) + | return ( + | '<!DOCTYPE html>\\n' + | '<html lang="en">\\n' + | '<body>\\n' + | ' <div class="modal-body">\\n' + | ' <iframe src="' + u + '" frameborder="0"\\n' + | ' style="height:100vh; width:100%; border:none;">\\n' + | ' </iframe>\\n' + | ' </div>\\n' + | '</body>\\n' + | '</html>' + | ) + |out1df = pd.DataFrame({"html-content": in1df[$urlLit].apply(_texera_urlviz_iframe)})""".stripMargin + } + } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpExec.scala index 5e5338aee3..37c76a7500 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpExec.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpExec.scala @@ -29,12 +29,15 @@ import org.apache.texera.amber.util.JSONUtils.objectMapper class UrlVizOpExec(descString: String) extends OperatorExecutor { private val desc: UrlVizOpDesc = objectMapper.readValue(descString, classOf[UrlVizOpDesc]) override def processTuple(tuple: Tuple, port: Int): Iterator[TupleLike] = { + val url = UrlVizOpExec.escapeAttribute( + String.valueOf(tuple.getField[Any](desc.urlContentAttrName)) + ) val iframe = s"""<!DOCTYPE html> |<html lang="en"> |<body> | <div class="modal-body"> - | <iframe src="${tuple.getField(desc.urlContentAttrName)}" frameborder="0" + | <iframe src="$url" frameborder="0" | style="height:100vh; width:100%; border:none;"> | </iframe> | </div> @@ -43,3 +46,23 @@ class UrlVizOpExec(descString: String) extends OperatorExecutor { Iterator(TupleLike(iframe)) } } + +object UrlVizOpExec { + + /** The value written where an attribute is expected. + * + * A cell is a value the workflow carried, not markup: a quote in it used to + * close `src=` and leave the rest of the cell standing as attributes of the + * iframe, which is the page speaking for whoever wrote the row. The five + * characters and their order are Python's `html.escape(quote=True)`, which + * the exported script calls on the same cell, so both paths write the page + * the same way. + */ + def escapeAttribute(value: String): String = + value + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace("\"", """) + .replace("'", "'") +} diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/waterfallChart/WaterfallChartOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/waterfallChart/WaterfallChartOpDesc.scala index 87965a61d6..e2e7b33960 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/waterfallChart/WaterfallChartOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/visualization/waterfallChart/WaterfallChartOpDesc.scala @@ -26,9 +26,11 @@ import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBui import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.PythonOperatorDescriptor +import org.apache.texera.amber.operator.visualization.PlotlyStandaloneCode import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import javax.validation.constraints.NotNull @@ -41,7 +43,7 @@ import javax.validation.constraints.NotNull } } """) -class WaterfallChartOpDesc extends PythonOperatorDescriptor { +class WaterfallChartOpDesc extends PythonOperatorDescriptor with PlotlyStandaloneCode { @JsonProperty(value = "xColumn", required = true) @JsonSchemaTitle("X Axis Values") @@ -76,20 +78,24 @@ class WaterfallChartOpDesc extends PythonOperatorDescriptor { assert(xColumn.nonEmpty, "X Axis Values cannot be empty") assert(yColumn.nonEmpty, "Y Axis Values cannot be empty") pyb""" - | x_values = table[$xColumn] - | y_values = table[$yColumn] + | x_values = list(table[$xColumn]) + | y_values = list(table[$yColumn]) + | total = sum(y_values) | + | # every input row is a step; the total is an extra bar plotly accumulates, + | # so no row is consumed as the summary. A categorical axis keeps the bars in + | # input order, which is the order the running total is computed in. | fig = go.Figure(go.Waterfall( | name="Waterfall", orientation="v", - | measure=["relative"] * (len(y_values) - 1) + ["total"], - | x=x_values, - | y=y_values, + | measure=["relative"] * len(y_values) + ["total"], + | x=x_values + ["Total"], + | y=y_values + [0], | textposition="outside", - | text=[f"{v:+}" for v in y_values], + | text=[f"{v:+}" for v in y_values] + [f"{total}"], | connector={"line": {"color": "rgb(63, 63, 63)"}} | )) | - | fig.update_layout(showlegend=True, waterfallgap=0.3) + | fig.update_layout(showlegend=True, waterfallgap=0.3, xaxis_type="category") |""" } @@ -121,4 +127,37 @@ class WaterfallChartOpDesc extends PythonOperatorDescriptor { finalCode.encode } + override def producesDataFrame(): Boolean = false + + override def generateStandaloneCode(): String = { + s"""def render_error(error_msg) -> str: + | return '''<h1>Waterfall chart is not available.</h1> + | <p>Reason is: {} </p> + | '''.format(error_msg) + | + |if in1df.empty: + | with open(outputHtml, "w", encoding="utf-8") as output: + | output.write(render_error("input table is empty.")) + |else: + | table = in1df + | x_values = list(table[${pyStringLiteral(xColumn)}]) + | y_values = list(table[${pyStringLiteral(yColumn)}]) + | total = sum(y_values) + | + | fig = go.Figure(go.Waterfall( + | name="Waterfall", orientation="v", + | measure=["relative"] * len(y_values) + ["total"], + | x=x_values + ["Total"], + | y=y_values + [0], + | textposition="outside", + | text=[f"{v:+}" for v in y_values] + [f"{total}"], + | connector={"line": {"color": "rgb(63, 63, 63)"}} + | )) + | + | fig.update_layout(showlegend=True, waterfallgap=0.3, xaxis_type="category") + | fig.write_json(outputJson) + | fig.write_html(outputHtml) + | print("Waterfall chart saved to " + outputHtml)""".stripMargin + } + } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/ImageViz/ImageVisualizerOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/ImageViz/ImageVisualizerOpDescSpec.scala index 2283610cd3..e6493d7647 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/ImageViz/ImageVisualizerOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/ImageViz/ImageVisualizerOpDescSpec.scala @@ -19,12 +19,18 @@ package org.apache.texera.amber.operator.visualization.ImageViz -import org.apache.texera.amber.core.tuple.AttributeType +import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.scalatest.BeforeAndAfter import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.util.concurrent.TimeUnit +import scala.io.Source +import scala.util.Try + class ImageVisualizerOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matchers { var opDesc: ImageVisualizerOpDesc = _ before { @@ -103,4 +109,85 @@ class ImageVisualizerOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Mat code should include("encode_image_to_html") code should include("decode_python_template") } + + "ImageVisualizerOpDesc.generateStandaloneCode" should "avoid literal HTML tags that the UI code viewer can render away" in { + opDesc.binaryContent = "image_bytes" + val code = opDesc.generateStandaloneCode() + + code should include("LT = chr(60)") + code should include("GT = chr(62)") + code should include("encoded_image_str") + code should not include "<img" + code should not include "<h1" + code should not include "<p" + code should not include "<div" + } + + // The executor hands b64encode whatever the column holds, and b64encode refuses + // a str, so a column of text ends in the reason page. The exported script reads + // the same column out of JSONL, where a BINARY column arrives as the base64 text + // of its bytes, so text is the image there and only there. + private val inPort = new ImageVisualizerOpDesc().operatorInfo.inputPorts.head.id + + private def standaloneFor(columnType: AttributeType): String = { + opDesc.binaryContent = "image_bytes" + opDesc.generateStandaloneCode(Map(inPort -> Schema().add("image_bytes", columnType))) + } + + it should "read the column's text as the image only where it was declared binary" in { + standaloneFor(AttributeType.BINARY) should include("if isinstance(binary_image_data, str):") + } + + it should "let a text column fail in b64encode, the way the executor does" in { + val code = standaloneFor(AttributeType.STRING) + code should include("if False:") + code should not include "isinstance(binary_image_data, str)" + // The call the executor makes on the same value, left to refuse it here too. + code should include("base64.b64encode(binary_image_data)") + } + + // Both paths run the same two lines on a string, so the answer is one both can + // be held to: b64encode raises, and the reason page is what the viewer sees. + it should "answer a string column with the reason page in both paths" in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + + val body = standaloneFor(AttributeType.STRING).linesIterator + .takeWhile(!_.startsWith("all_images_html")) + .mkString("\n") + val driver = + s"""$body + |print(encode_image_to_html("not an image")) + |""".stripMargin + + val script = Files.createTempFile("image-visualizer-string-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + out should include("Image is not available.") + out should include("Reason: Binary input is not valid") + out should not include "img src" + } + // The executor's own reason page, which the run showed for the same value. + opDesc.generatePythonCode() should include("Image is not available.") + } + + private def resolvePython(): Option[String] = { + def runnable(exe: String): Boolean = + Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption + .exists { p => + if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + + (sys.env.get("UDF_PYTHON_PATH").filter(_.nonEmpty).toList ++ List( + "python3", + "python", + "py" + )).distinct + .find(runnable) + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/figureFactoryTable/FigureFactoryTableOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/figureFactoryTable/FigureFactoryTableOpDescSpec.scala index 9d564fd358..221959265e 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/figureFactoryTable/FigureFactoryTableOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/figureFactoryTable/FigureFactoryTableOpDescSpec.scala @@ -19,15 +19,23 @@ package org.apache.texera.amber.operator.visualization.figureFactoryTable +import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.operator.tags.IntegrationTest import org.scalatest.BeforeAndAfter +import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import java.nio.charset.StandardCharsets +import java.nio.file.Files import java.util.Base64 +import java.util.concurrent.TimeUnit +import scala.util.Try class FigureFactoryTableOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matchers { + private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) + var opDesc: FigureFactoryTableOpDesc = _ before { @@ -100,14 +108,105 @@ class FigureFactoryTableOpDescSpec extends AnyFlatSpec with BeforeAndAfter with code should include("class TableChartOperator(UDFTableOperator)") } - it should "define the render_error the empty-table branches call" in { - // Both empty-table branches call self.render_error; without the definition they - // raised AttributeError instead of rendering the message. - withColumns() - val code = opDesc.generatePythonCode() - code should include("def render_error(self, error_msg) -> str:") - code should include( - """return f"<h1>Figure Factory Table is not available.</h1><p>Reason is: {error_msg}</p>"""" + // Stubs only the pytexera import seam, then hands the operator a table that arrives + // empty, one whose every row has a hole in a chosen column, and one that draws. + // Both empty branches call self.render_error, which once was not defined. + private val runtimeDriverScript: String = + """import base64 + |import sys + |import types + |from typing import Iterator, Optional + | + |import pandas as pd + | + |class UDFTableOperator: + | def decode_python_template(self, data): + | return base64.b64decode(data).decode("utf-8") + | + |stub = types.ModuleType("pytexera") + |stub.UDFTableOperator = UDFTableOperator + |stub.overrides = lambda fn: fn + |stub.Table = pd.DataFrame + |stub.TableLike = object + |stub.Iterator = Iterator + |stub.Optional = Optional + |sys.modules["pytexera"] = stub + | + |ns = {"__name__": "generated_table_chart"} + |with open(sys.argv[1], encoding="utf-8") as f: + | exec(compile(f.read(), sys.argv[1], "exec"), ns) + |op = ns["TableChartOperator"]() + | + |cases = [ + | ("empty", pd.DataFrame({"col_one": [], "col_two": []})), + | ("every_row_holed", pd.DataFrame({"col_one": [1.0, None], "col_two": [None, "b"]})), + | ("filled", pd.DataFrame({"col_one": [1, 2], "col_two": ["a", "b"]})), + |] + |for cid, df in cases: + | page = list(op.process_table(df, 0))[0]["html-content"] + | print("CASE %s %s" % (cid, page if "not available" in page else "CHART")) + |""".stripMargin + + it should "render its message when the table is empty or every row has a hole" taggedAs NeedsPythonPackages in { + val python = resolvePythonExecutable().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) + if (!canImportPandasAndPlotly(python)) cancel(s"'$python' cannot import pandas and plotly") + + withColumns() + val moduleFile = Files.createTempFile("figure_factory_table_op_", ".py") + val driverFile = Files.createTempFile("figure_factory_table_driver_", ".py") + try { + Files.write(moduleFile, opDesc.generatePythonCode().getBytes(StandardCharsets.UTF_8)) + Files.write(driverFile, runtimeDriverScript.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, driverFile.toString, moduleFile.toString) + .redirectErrorStream(true) + .start() + if (!process.waitFor(120, TimeUnit.SECONDS)) { + process.destroyForcibly() + fail("Runtime driver timed out after 120s") + } + val output = new String(process.getInputStream.readAllBytes(), StandardCharsets.UTF_8) + val page = "<h1>Figure Factory Table is not available.</h1><p>Reason is: " + withClue(s"Driver output:\n$output\n") { + process.exitValue() shouldBe 0 + output should include(s"CASE empty ${page}input table is empty.</p>") + output should include( + s"CASE every_row_holed ${page}value column contains only non-positive numbers or nulls.</p>" + ) + output should include("CASE filled CHART") + } + } finally { + Try(Files.deleteIfExists(moduleFile)) + Try(Files.deleteIfExists(driverFile)) + () + } + } + + // udf.conf python.path (UDF_PYTHON_PATH), then python3 / python / py. + private def resolvePythonExecutable(): Option[String] = { + def fromConfig: Option[String] = + Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption + .orElse(Try(ConfigFactory.load()).toOption) + .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) + .map(_.trim) + .filter(_.nonEmpty) + + def isRunnable(exe: String): Boolean = + Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption + .exists { p => + if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + + (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(isRunnable) } + + private def canImportPandasAndPlotly(python: String): Boolean = + Try( + new ProcessBuilder(python, "-c", "import pandas, plotly").redirectErrorStream(true).start() + ).toOption.exists { p => + if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesPlotOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesPlotOpDescSpec.scala index c088cacc67..c54df3f231 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesPlotOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesPlotOpDescSpec.scala @@ -19,15 +19,23 @@ package org.apache.texera.amber.operator.visualization.tablesChart +import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.operator.tags.IntegrationTest import org.scalatest.BeforeAndAfter +import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import java.nio.charset.StandardCharsets +import java.nio.file.Files import java.util.Base64 +import java.util.concurrent.TimeUnit +import scala.util.Try class TablesPlotOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matchers { + private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) + var opDesc: TablesPlotOpDesc = _ before { @@ -89,14 +97,105 @@ class TablesPlotOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matchers code should not include "')','" } - it should "define the render_error the empty-table branches call" in { - // Both empty-table branches call self.render_error; without the definition they - // raised AttributeError instead of rendering the message. - opDesc.includedColumns = List(column("col_one"), column("col_two")) - val code = opDesc.generatePythonCode() - code should include("def render_error(self, error_msg) -> str:") - code should include( - """return f"<h1>Tables Plot is not available.</h1><p>Reason is: {error_msg}</p>"""" + // Stubs only the pytexera import seam, then hands the operator a table that arrives + // empty, one whose every row has a hole in a chosen column, and one that draws. + // Both empty branches call self.render_error, which once was not defined. + private val runtimeDriverScript: String = + """import base64 + |import sys + |import types + |from typing import Iterator, Optional + | + |import pandas as pd + | + |class UDFTableOperator: + | def decode_python_template(self, data): + | return base64.b64decode(data).decode("utf-8") + | + |stub = types.ModuleType("pytexera") + |stub.UDFTableOperator = UDFTableOperator + |stub.overrides = lambda fn: fn + |stub.Table = pd.DataFrame + |stub.TableLike = object + |stub.Iterator = Iterator + |stub.Optional = Optional + |sys.modules["pytexera"] = stub + | + |ns = {"__name__": "generated_table_chart"} + |with open(sys.argv[1], encoding="utf-8") as f: + | exec(compile(f.read(), sys.argv[1], "exec"), ns) + |op = ns["TableChartOperator"]() + | + |cases = [ + | ("empty", pd.DataFrame({"col_one": [], "col_two": []})), + | ("every_row_holed", pd.DataFrame({"col_one": [1.0, None], "col_two": [None, "b"]})), + | ("filled", pd.DataFrame({"col_one": [1, 2], "col_two": ["a", "b"]})), + |] + |for cid, df in cases: + | page = list(op.process_table(df, 0))[0]["html-content"] + | print("CASE %s %s" % (cid, page if "not available" in page else "CHART")) + |""".stripMargin + + it should "render its message when the table is empty or every row has a hole" taggedAs NeedsPythonPackages in { + val python = resolvePythonExecutable().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) + if (!canImportPandasAndPlotly(python)) cancel(s"'$python' cannot import pandas and plotly") + + opDesc.includedColumns = List(column("col_one"), column("col_two")) + val moduleFile = Files.createTempFile("tables_plot_op_", ".py") + val driverFile = Files.createTempFile("tables_plot_driver_", ".py") + try { + Files.write(moduleFile, opDesc.generatePythonCode().getBytes(StandardCharsets.UTF_8)) + Files.write(driverFile, runtimeDriverScript.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, driverFile.toString, moduleFile.toString) + .redirectErrorStream(true) + .start() + if (!process.waitFor(120, TimeUnit.SECONDS)) { + process.destroyForcibly() + fail("Runtime driver timed out after 120s") + } + val output = new String(process.getInputStream.readAllBytes(), StandardCharsets.UTF_8) + val page = "<h1>Tables Plot is not available.</h1><p>Reason is: " + withClue(s"Driver output:\n$output\n") { + process.exitValue() shouldBe 0 + output should include(s"CASE empty ${page}input table is empty.</p>") + output should include( + s"CASE every_row_holed ${page}value column contains only non-positive numbers or nulls.</p>" + ) + output should include("CASE filled CHART") + } + } finally { + Try(Files.deleteIfExists(moduleFile)) + Try(Files.deleteIfExists(driverFile)) + () + } + } + + // udf.conf python.path (UDF_PYTHON_PATH), then python3 / python / py. + private def resolvePythonExecutable(): Option[String] = { + def fromConfig: Option[String] = + Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption + .orElse(Try(ConfigFactory.load()).toOption) + .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) + .map(_.trim) + .filter(_.nonEmpty) + + def isRunnable(exe: String): Boolean = + Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption + .exists { p => + if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + + (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(isRunnable) } + + private def canImportPandasAndPlotly(python: String): Boolean = + Try( + new ProcessBuilder(python, "-c", "import pandas, plotly").redirectErrorStream(true).start() + ).toOption.exists { p => + if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpDescSpec.scala index 2dbbb886d1..f5aad7db0e 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpDescSpec.scala @@ -22,17 +22,28 @@ package org.apache.texera.amber.operator.visualization.urlviz import com.fasterxml.jackson.annotation.JsonProperty import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaInject import org.apache.texera.amber.core.executor.OpExecWithClassName -import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} +import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName +import org.apache.texera.amber.operator.tags.IntegrationTest +import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral +import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.util.concurrent.TimeUnit import javax.validation.constraints.NotNull +import scala.io.Source +import scala.util.Try class UrlVizOpDescSpec extends AnyFlatSpec with Matchers { + private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) + private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -125,4 +136,94 @@ class UrlVizOpDescSpec extends AnyFlatSpec with Matchers { val b = new UrlVizOpDesc a.operatorIdentifier should not equal b.operatorIdentifier } + + // --------------------------------------------------------------------------- + // The cell lands where an attribute is expected + // --------------------------------------------------------------------------- + + // A quote in the cell used to close `src=` and leave the rest of it standing as + // attributes of the iframe, which is the page speaking for whoever wrote the + // row. Both paths are held to the same written page for that value. + private val hostileUrl = """https://example.invalid/a"; onload="alert(1)""" + + /** A configured descriptor: `urlContentAttrName` is a val, so Jackson writes it. */ + private def descFor(urlAttr: String): UrlVizOpDesc = { + val node = objectMapper.createObjectNode() + node.put("operatorType", "URLVisualizer") + node.put("urlContentAttrName", urlAttr) + objectMapper.readValue(node.toString, classOf[UrlVizOpDesc]) + } + + it should "escape a hostile cell rather than letting it add an attribute" in { + val attr = new Attribute("url", AttributeType.STRING) + val tuple = Tuple.builder(Schema().add(attr)).add(attr, hostileUrl).build() + val page = new UrlVizOpExec(objectMapper.writeValueAsString(descFor("url"))) + .processTuple(tuple, port = 0) + .next() + .getFields + .head + .asInstanceOf[String] + + page should include("""src="https://example.invalid/a" onload="alert(1)"""") + page should not include """onload="alert""" + } + + it should "write that cell the same way in the exported script" taggedAs NeedsPythonPackages in { + val python = resolvePython().getOrElse(cancel("No runnable python executable")) + if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + + val desc = descFor("url") + val attr = new Attribute("url", AttributeType.STRING) + val tuple = Tuple.builder(Schema().add(attr)).add(attr, hostileUrl).build() + val fromExecutor = new UrlVizOpExec(objectMapper.writeValueAsString(desc)) + .processTuple(tuple, port = 0) + .next() + .getFields + .head + .asInstanceOf[String] + + val driver = + s"""import pandas as pd + | + |in1df = pd.DataFrame({"url": [${pyStringLiteral(hostileUrl)}]}) + |${desc.generateStandaloneCode()} + | + |print(out1df["html-content"].iloc[0]) + |""".stripMargin + + val script = Files.createTempFile("urlviz-hostile-", ".py") + script.toFile.deleteOnExit() + Files.write(script, driver.getBytes(StandardCharsets.UTF_8)) + val process = new ProcessBuilder(python, script.toString).redirectErrorStream(true).start() + val out = Source.fromInputStream(process.getInputStream).mkString + process.waitFor(120, TimeUnit.SECONDS) + withClue(s"python said:\n$out\nscript:\n$driver") { + process.exitValue() shouldBe 0 + out.trim shouldBe fromExecutor + } + } + + private def resolvePython(): Option[String] = { + def runnable(exe: String): Boolean = + Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption + .exists { p => + if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + + (sys.env.get("UDF_PYTHON_PATH").filter(_.nonEmpty).toList ++ List( + "python3", + "python", + "py" + )).distinct + .find(runnable) + } + + private def canImportPandas(python: String): Boolean = + Try( + new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() + ).toOption.exists { p => + if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/waterfallChart/WaterfallChartOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/waterfallChart/WaterfallChartOpDescSpec.scala index c6a42ef8e3..9c89d9daf0 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/waterfallChart/WaterfallChartOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/waterfallChart/WaterfallChartOpDescSpec.scala @@ -19,15 +19,23 @@ package org.apache.texera.amber.operator.visualization.waterfallChart +import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.operator.tags.IntegrationTest import org.scalatest.BeforeAndAfter +import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import java.nio.charset.StandardCharsets +import java.nio.file.Files import java.util.Base64 +import java.util.concurrent.TimeUnit +import scala.util.Try class WaterfallChartOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matchers { + private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) + var opDesc: WaterfallChartOpDesc = _ before { @@ -77,4 +85,156 @@ class WaterfallChartOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matc assert(carries(code, "y_col")) code should include("class ProcessTableOperator(UDFTableOperator)") } + + private def configured(): WaterfallChartOpDesc = { + opDesc.xColumn = "x_col" + opDesc.yColumn = "y_col" + opDesc + } + + it should "give every input row a relative bar and append the total as an extra bar" in { + val figure = configured().createPlotlyFigure().plain + figure should include("""measure=["relative"] * len(y_values) + ["total"]""") + figure should not include """(len(y_values) - 1)""" + figure should include("""x=x_values + ["Total"]""") + figure should include("y=y_values + [0]") + } + + it should "pin the x axis to categories so the bars keep their input order" in { + configured().createPlotlyFigure().plain should include("""xaxis_type="category"""") + } + + // Python executable resolution, following FilledAreaPlotOpDescSpec: + // udf.conf python.path (UDF_PYTHON_PATH), then python3 / python / py. + private def resolvePythonExecutable(): Option[String] = { + def fromConfig: Option[String] = { + val configOpt = + Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption + .orElse(Try(ConfigFactory.load()).toOption) + configOpt + .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) + .map(_.trim) + .filter(_.nonEmpty) + } + + def isRunnable(exe: String): Boolean = { + val pTry = Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()) + pTry.toOption.exists { p => + val finished = p.waitFor(5, TimeUnit.SECONDS) + if (!finished) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + } + + (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(isRunnable) + } + + private def canImportPandasAndPlotly(python: String): Boolean = { + val pTry = Try( + new ProcessBuilder(python, "-c", "import pandas, plotly").redirectErrorStream(true).start() + ) + pTry.toOption.exists { p => + val finished = p.waitFor(60, TimeUnit.SECONDS) + if (!finished) { p.destroyForcibly(); false } + else p.exitValue() == 0 + } + } + + // Driver executed by the runtime test below. It stubs only the pytexera import seam and + // swaps plotly's html renderer for its json one, so the trace the generated module hands + // plotly can be read back bar by bar. + private val runtimeDriverScript: String = + """import base64 + |import json + |import sys + |import types + |from typing import Iterator, Optional + | + |import pandas as pd + |import plotly.io + | + |plotly.io.to_html = lambda fig, **kwargs: fig.to_json() + | + |class UDFTableOperator: + | def decode_python_template(self, data): + | return base64.b64decode(data).decode("utf-8") + | + |stub = types.ModuleType("pytexera") + |stub.UDFTableOperator = UDFTableOperator + |stub.overrides = lambda fn: fn + |stub.Table = pd.DataFrame + |stub.TableLike = object + |stub.Iterator = Iterator + |stub.Optional = Optional + |sys.modules["pytexera"] = stub + | + |ns = {"__name__": "generated_waterfall_chart"} + |with open(sys.argv[1], encoding="utf-8") as f: + | exec(compile(f.read(), sys.argv[1], "exec"), ns) + |op = ns["ProcessTableOperator"]() + | + |cases = [ + | ("labels", pd.DataFrame({"x_col": ["a", "b", "c", "d"], "y_col": [1, 2, 3, 4]})), + | ("unsorted_numeric_x", pd.DataFrame({"x_col": [30, 10, 20], "y_col": [1, 2, 3]})), + |] + | + |for cid, df in cases: + | fig = json.loads(list(op.process_table(df, 0))[0]["html-content"]) + | trace = fig["data"][0] + | print("CASE %s measure=%s x=%s y=%s text=%s xaxis=%s" % ( + | cid, + | json.dumps(trace["measure"]), + | json.dumps(list(trace["x"])), + | json.dumps(list(trace["y"])), + | json.dumps(list(trace["text"])), + | fig["layout"]["xaxis"]["type"], + | )) + |""".stripMargin + + it should "plot the last row and draw the total over the whole column at runtime" taggedAs NeedsPythonPackages in { + val python = resolvePythonExecutable().getOrElse( + cancel("No runnable python executable (udf.conf python.path, python3, python, py)") + ) + if (!canImportPandasAndPlotly(python)) { + cancel(s"'$python' cannot import pandas and plotly; skipping runtime verification") + } + + val moduleFile = Files.createTempFile("waterfall_chart_op_", ".py") + val driverFile = Files.createTempFile("waterfall_chart_driver_", ".py") + try { + Files.write(moduleFile, configured().generatePythonCode().getBytes(StandardCharsets.UTF_8)) + Files.write(driverFile, runtimeDriverScript.getBytes(StandardCharsets.UTF_8)) + + val process = new ProcessBuilder(python, driverFile.toString, moduleFile.toString) + .redirectErrorStream(true) + .start() + val finished = process.waitFor(120, TimeUnit.SECONDS) + if (!finished) { + process.destroyForcibly() + fail("Runtime verification driver timed out after 120s") + } + val output = new String(process.getInputStream.readAllBytes(), StandardCharsets.UTF_8) + withClue(s"Driver output:\n$output\n") { + process.exitValue() shouldBe 0 + // Four rows of 1, 2, 3, 4 give four bars plus a total of 10. The old measure list + // spent the last row on the total, drawing three bars and a fourth at 6 labelled +4. + output should include( + """CASE labels measure=["relative", "relative", "relative", "relative", "total"] """ + + """x=["a", "b", "c", "d", "Total"] y=[1, 2, 3, 4, 0] """ + + """text=["+1", "+2", "+3", "+4", "10"] xaxis=category""" + ) + // An unsorted numeric x column keeps its input order, so the bar plotly accumulates + // into is the bar the running total was computed from. + output should include( + """CASE unsorted_numeric_x measure=["relative", "relative", "relative", "total"] """ + + """x=[30, 10, 20, "Total"] y=[1, 2, 3, 0] """ + + """text=["+1", "+2", "+3", "6"] xaxis=category""" + ) + } + } finally { + Try(Files.deleteIfExists(moduleFile)) + Try(Files.deleteIfExists(driverFile)) + () + } + } }
