carloea2 commented on code in PR #8340:
URL: https://github.com/apache/texera/pull/8340#discussion_r3975421335
##########
common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDesc.scala:
##########
@@ -147,6 +146,34 @@ class IntervalJoinOpDesc extends LogicalOp {
outputPorts = List(OutputPort())
)
+ // Inner interval join: left point in [rightKey, rightKey + constant], bounds
+ // toggled by include{Left,Right}Bound. Cross-join + mask computes the full
+ // result (no sorted-input assumption, unlike the exec). Runtime dtype check
+ // picks numeric vs pd.DateOffset (unit from timeIntervalType).
+ override def generateStandaloneCode(): String = {
+ val leftLit = objectMapper.writeValueAsString(leftAttributeName)
+ val rightLit = objectMapper.writeValueAsString(rightAttributeName)
+ val loOp = if (includeLeftBound) ">=" else ">"
+ val hiOp = if (includeRightBound) "<=" else "<"
+ val offsetUnit = Option(timeIntervalType).flatten match {
+ case Some(TimeIntervalType.YEAR) => "years"
+ case Some(TimeIntervalType.MONTH) => "months"
+ case Some(TimeIntervalType.HOUR) => "hours"
+ case Some(TimeIntervalType.MINUTE) => "minutes"
+ case Some(TimeIntervalType.SECOND) => "seconds"
+ case _ => "days" // DAY or unset
+ }
+ s"""_l = in1df.assign(_iv_l=in1df[$leftLit])
Review Comment:
Confirmed locally: the original payload-column regression now passes.
##########
common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/typecasting/TypeCastingOpDesc.scala:
##########
@@ -72,4 +74,54 @@ class TypeCastingOpDesc extends MapOpDesc {
List(OutputPort())
)
}
+
+ override def generateStandaloneCode(): String = {
+ val units = Option(typeCastingUnits).getOrElse(List.empty)
+ if (units.isEmpty) return "out1df = in1df.copy()"
+
+ val lines = scala.collection.mutable.ArrayBuffer[String]("out1df =
in1df.copy()")
+ units.foreach { unit =>
+ val colLit = pyStringLiteral(unit.attribute)
+ // Every cast goes through the transcription of AttributeTypeUtils rather
+ // than through Python's own conversions, which answer differently: a
+ // non-empty string is always a true boolean, and a coercing numeric cast
+ // reads "6.7" as an integer the engine refuses.
+ //
+ // A timestamp is the one that stays approximate. The engine reads it
with
+ // DateParserUtils, which accepts a set of formats no single pandas call
+ // states, so this coerces what it cannot read rather than claiming a
+ // match it does not have.
+ val expr = unit.resultType match {
+ case AttributeType.STRING =>
+ // `astype(str)` gets three things wrong against `toString`: an empty
+ // cell renders as the text "nan", a column holding one is a float by
+ // then so 6 reads "6.0", and a boolean capitalises. Each is handled
+ // rather than the column cast wholesale.
+ s"""out1df[$colLit].apply(""" +
+ """lambda x: None if pd.isna(x) """ +
+ """else ("true" if x else "false") if isinstance(x, bool) """ +
+ """else str(int(x)) if isinstance(x, float) and x.is_integer() """
+
Review Comment:
Confirmed: the original DOUBLE regression now passes. I left a separate
finding for nullable INTEGER values arriving from CSV.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
