kz930 commented on code in PR #8705:
URL: https://github.com/apache/texera/pull/8705#discussion_r4252752334
##########
common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDesc.scala:
##########
@@ -68,4 +74,43 @@ class TextInputSourceOpDesc extends SourceOperatorDescriptor
with TextSourceOpDe
inputPorts = List.empty,
outputPorts = List(OutputPort())
)
+
+ override def generateStandaloneCode(): String = {
+ val text = objectMapper.writeValueAsString(textInput)
+ val col = attributeName
+ val colLit = pyStringLiteral(col)
+ val buf = scala.collection.mutable.ArrayBuffer[String]()
+
+ buf += s"_text = $text"
+
+ val isBinary =
+ attributeType == FileAttributeType.BINARY || attributeType ==
FileAttributeType.LARGE_BINARY
+
+ if (attributeType.isSingle) {
+ val valueExpr = if (isBinary) """_text.encode("utf-8")""" else "_text"
+ buf += s"""out1df = pd.DataFrame({$colLit: [$valueExpr]})"""
+ } else {
+ val castExpr = attributeType match {
+ case FileAttributeType.INTEGER => "int(l)"
+ case FileAttributeType.LONG => "int(l)"
+ case FileAttributeType.DOUBLE => "float(l)"
+ case FileAttributeType.BOOLEAN => TextSourceOpDesc.BooleanParserCall
+ case FileAttributeType.TIMESTAMP => "l"
+ case _ => "l"
+ }
+ // The slice applies to the raw lines, as the engine drops and takes
+ // before parsing: a line outside the window is never converted, so an
+ // unparseable one there costs nothing. Taking after dropping also keeps
+ // a large limit from overflowing the end index.
+ val dropped =
+ fileScanOffset.filter(_ > 0).fold("_text.splitlines()")(o =>
s"_text.splitlines()[$o:]")
Review Comment:
Fixed in e9d9824a0. The export now splits only at LF, CR and CRLF like
linesIterator, and a new test checks all eleven characters splitlines breaks
at, U+2028 included, against TextInputSourceOpExec.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
