carloea2 commented on code in PR #8705:
URL: https://github.com/apache/texera/pull/8705#discussion_r4252204838
##########
common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDesc.scala:
##########
@@ -68,4 +74,43 @@ class TextInputSourceOpDesc extends SourceOperatorDescriptor
with TextSourceOpDe
inputPorts = List.empty,
outputPorts = List(OutputPort())
)
+
+ override def generateStandaloneCode(): String = {
+ val text = objectMapper.writeValueAsString(textInput)
+ val col = attributeName
+ val colLit = pyStringLiteral(col)
+ val buf = scala.collection.mutable.ArrayBuffer[String]()
+
+ buf += s"_text = $text"
+
+ val isBinary =
+ attributeType == FileAttributeType.BINARY || attributeType ==
FileAttributeType.LARGE_BINARY
+
+ if (attributeType.isSingle) {
+ val valueExpr = if (isBinary) """_text.encode("utf-8")""" else "_text"
+ buf += s"""out1df = pd.DataFrame({$colLit: [$valueExpr]})"""
+ } else {
+ val castExpr = attributeType match {
+ case FileAttributeType.INTEGER => "int(l)"
+ case FileAttributeType.LONG => "int(l)"
+ case FileAttributeType.DOUBLE => "float(l)"
+ case FileAttributeType.BOOLEAN => TextSourceOpDesc.BooleanParserCall
+ case FileAttributeType.TIMESTAMP => "l"
+ case _ => "l"
+ }
+ // The slice applies to the raw lines, as the engine drops and takes
+ // before parsing: a line outside the window is never converted, so an
+ // unparseable one there costs nothing. Taking after dropping also keeps
+ // a large limit from overflowing the end index.
+ val dropped =
+ fileScanOffset.filter(_ > 0).fold("_text.splitlines()")(o =>
s"_text.splitlines()[$o:]")
Review Comment:
splitlines treats U+2028 as a row boundary, but Scala linesIterator keeps it
inside the row. For first followed by U+2028 followed by second, the engine
emits one row and the exported script emits two. Reproduced against
TextInputSourceOpExec with LF and CRLF controls. Please match the engine's line
boundaries and add a parity test.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
