This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git
The following commit(s) were added to refs/heads/main by this push:
new 72975ce [SPARK-58479] Support math functions in `MathFunctions`
72975ce is described below
commit 72975ce7f663abccbd133935c3432b420d0b1597
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Fri Jul 31 11:48:54 2026 -0700
[SPARK-58479] Support math functions in `MathFunctions`
### What changes were proposed in this pull request?
This PR aims to add math functions in a new file, `MathFunctions.swift`.
The function names and signatures follow Scala's
`org.apache.spark.sql.functions`.
| Category | Functions |
| --- | --- |
| Trigonometric | `sin`, `cos`, `tan`, `asin`, `acos`, `atan`, `atan2`,
`cot`, `sec`, `csc` |
| Hyperbolic | `sinh`, `cosh`, `tanh`, `asinh`, `acosh`, `atanh` |
| Exponential and logarithmic | `exp`, `expm1`, `ln`, `log` (with a
`log(base, column)` overload), `log10`, `log1p`, `log2`, `pow`/`power`, `cbrt`,
`hypot` |
| Rounding and sign | `ceil`/`ceiling`/`floor` (each with a `scale`
overload), `rint`, `bround` (HALF_EVEN, with a `scale` overload),
`sign`/`signum`, `negative`, `positive` |
| Base conversion and bit shift | `bin`, `hex`, `unhex`, `conv`,
`shiftleft`, `shiftright`, `shiftrightunsigned` |
| Others | `factorial`, `pmod`, `degrees`, `radians`, `e`, `pi`,
`width_bucket` |
Following Scala, some Swift APIs map to different SQL function names.
| Swift API | SQL function |
| --- | --- |
| `pow`, `power` | `power` |
| `log(column)` | `ln` |
Deprecated aliases in Scala (`toDegrees`, `toRadians`, `shiftLeft`,
`shiftRight`, `shiftRightUnsigned`) are excluded intentionally.
### Why are the changes needed?
To improve the usability of the column-based `DataFrame` API by providing
math functions in the same shape as Scala/PySpark, so users can write queries
without falling back to `selectExpr` or raw SQL strings.
### Does this PR introduce _any_ user-facing change?
Yes, this adds new public functions. There is no behavior change to
existing APIs.
### How was this patch tested?
Pass the CIs with a newly added test suite, `MathFunctionsTests`.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Fable 5
Closes #460 from dongjoon-hyun/SPARK-58479.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
Sources/SparkConnect/Functions.swift | 2 +-
Sources/SparkConnect/MathFunctions.swift | 474 +++++++++++++++++++++++
Tests/SparkConnectTests/MathFunctionsTests.swift | 202 ++++++++++
3 files changed, 677 insertions(+), 1 deletion(-)
diff --git a/Sources/SparkConnect/Functions.swift
b/Sources/SparkConnect/Functions.swift
index a31fe33..59e1fcf 100644
--- a/Sources/SparkConnect/Functions.swift
+++ b/Sources/SparkConnect/Functions.swift
@@ -243,7 +243,7 @@ public func when(_ condition: Column, _ value: some
SparkLiteral) -> Column {
return when(condition, value.toLiteralColumn)
}
-private func fn(_ name: String, _ args: Column...) -> Column {
+func fn(_ name: String, _ args: Column...) -> Column {
var function = Spark_Connect_Expression.UnresolvedFunction()
function.functionName = name
function.arguments = args.map { $0.expr }
diff --git a/Sources/SparkConnect/MathFunctions.swift
b/Sources/SparkConnect/MathFunctions.swift
new file mode 100644
index 0000000..d5201f6
--- /dev/null
+++ b/Sources/SparkConnect/MathFunctions.swift
@@ -0,0 +1,474 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+// MARK: - Math functions
+
+/// Computes the absolute value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func abs(_ col: Column) -> Column {
+ return fn("abs", col)
+}
+
+/// Computes the inverse cosine of the given value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func acos(_ col: Column) -> Column {
+ return fn("acos", col)
+}
+
+/// Computes the inverse hyperbolic cosine of the given value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func acosh(_ col: Column) -> Column {
+ return fn("acosh", col)
+}
+
+/// Computes the inverse sine of the given value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func asin(_ col: Column) -> Column {
+ return fn("asin", col)
+}
+
+/// Computes the inverse hyperbolic sine of the given value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func asinh(_ col: Column) -> Column {
+ return fn("asinh", col)
+}
+
+/// Computes the inverse tangent of the given value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func atan(_ col: Column) -> Column {
+ return fn("atan", col)
+}
+
+/// Returns the angle theta from the conversion of rectangular coordinates (x,
y) to polar
+/// coordinates (r, theta).
+/// - Parameters:
+/// - y: A coordinate ``Column`` on the y-axis.
+/// - x: A coordinate ``Column`` on the x-axis.
+/// - Returns: A ``Column``.
+public func atan2(_ y: Column, _ x: Column) -> Column {
+ return fn("atan2", y, x)
+}
+
+/// Computes the inverse hyperbolic tangent of the given value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func atanh(_ col: Column) -> Column {
+ return fn("atanh", col)
+}
+
+/// Returns the string representation of the binary value of the given long
column.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func bin(_ col: Column) -> Column {
+ return fn("bin", col)
+}
+
+/// Returns the value of the column rounded to 0 decimal places with HALF_EVEN
round mode.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func bround(_ col: Column) -> Column {
+ return bround(col, 0)
+}
+
+/// Rounds the value of the column to `scale` decimal places with HALF_EVEN
round mode.
+/// - Parameters:
+/// - col: A ``Column``.
+/// - scale: The number of decimal places.
+/// - Returns: A ``Column``.
+public func bround(_ col: Column, _ scale: Int32) -> Column {
+ return fn("bround", col, lit(scale))
+}
+
+/// Computes the cube-root of the given value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func cbrt(_ col: Column) -> Column {
+ return fn("cbrt", col)
+}
+
+/// Computes the ceiling of the given value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func ceil(_ col: Column) -> Column {
+ return fn("ceil", col)
+}
+
+/// Computes the ceiling of the given value of `col` to `scale` decimal places.
+/// - Parameters:
+/// - col: A ``Column``.
+/// - scale: A ``Column`` for the number of decimal places.
+/// - Returns: A ``Column``.
+public func ceil(_ col: Column, _ scale: Column) -> Column {
+ return fn("ceil", col, scale)
+}
+
+/// Computes the ceiling of the given value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func ceiling(_ col: Column) -> Column {
+ return fn("ceiling", col)
+}
+
+/// Computes the ceiling of the given value of `col` to `scale` decimal places.
+/// - Parameters:
+/// - col: A ``Column``.
+/// - scale: A ``Column`` for the number of decimal places.
+/// - Returns: A ``Column``.
+public func ceiling(_ col: Column, _ scale: Column) -> Column {
+ return fn("ceiling", col, scale)
+}
+
+/// Converts a number in a string column from one base to another.
+/// - Parameters:
+/// - num: A ``Column`` to convert.
+/// - fromBase: A base of the given number.
+/// - toBase: A base to convert the number to.
+/// - Returns: A ``Column``.
+public func conv(_ num: Column, _ fromBase: Int32, _ toBase: Int32) -> Column {
+ return fn("conv", num, lit(fromBase), lit(toBase))
+}
+
+/// Computes the cosine of the given value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func cos(_ col: Column) -> Column {
+ return fn("cos", col)
+}
+
+/// Computes the hyperbolic cosine of the given value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func cosh(_ col: Column) -> Column {
+ return fn("cosh", col)
+}
+
+/// Computes the cotangent of the given value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func cot(_ col: Column) -> Column {
+ return fn("cot", col)
+}
+
+/// Computes the cosecant of the given value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func csc(_ col: Column) -> Column {
+ return fn("csc", col)
+}
+
+/// Converts an angle measured in radians to an approximately equivalent angle
measured in
+/// degrees.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func degrees(_ col: Column) -> Column {
+ return fn("degrees", col)
+}
+
+/// Returns Euler's number.
+/// - Returns: A ``Column``.
+public func e() -> Column {
+ return fn("e")
+}
+
+/// Computes the exponential of the given value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func exp(_ col: Column) -> Column {
+ return fn("exp", col)
+}
+
+/// Computes the exponential of the given value minus one.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func expm1(_ col: Column) -> Column {
+ return fn("expm1", col)
+}
+
+/// Computes the factorial of the given value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func factorial(_ col: Column) -> Column {
+ return fn("factorial", col)
+}
+
+/// Computes the floor of the given value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func floor(_ col: Column) -> Column {
+ return fn("floor", col)
+}
+
+/// Computes the floor of the given value of `col` to `scale` decimal places.
+/// - Parameters:
+/// - col: A ``Column``.
+/// - scale: A ``Column`` for the number of decimal places.
+/// - Returns: A ``Column``.
+public func floor(_ col: Column, _ scale: Column) -> Column {
+ return fn("floor", col, scale)
+}
+
+/// Computes hex value of the given column.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func hex(_ col: Column) -> Column {
+ return fn("hex", col)
+}
+
+/// Computes `sqrt(l^2 + r^2)` without intermediate overflow or underflow.
+/// - Parameters:
+/// - l: A ``Column``.
+/// - r: A ``Column``.
+/// - Returns: A ``Column``.
+public func hypot(_ l: Column, _ r: Column) -> Column {
+ return fn("hypot", l, r)
+}
+
+/// Computes the natural logarithm of the given value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func ln(_ col: Column) -> Column {
+ return fn("ln", col)
+}
+
+/// Computes the natural logarithm of the given value. This is an alias of
``ln(_:)``.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func log(_ col: Column) -> Column {
+ return ln(col)
+}
+
+/// Returns the first argument-base logarithm of the second argument.
+/// - Parameters:
+/// - base: A base of the logarithm.
+/// - col: A ``Column``.
+/// - Returns: A ``Column``.
+public func log(_ base: Double, _ col: Column) -> Column {
+ return fn("log", lit(base), col)
+}
+
+/// Computes the logarithm of the given value in base 10.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func log10(_ col: Column) -> Column {
+ return fn("log10", col)
+}
+
+/// Computes the natural logarithm of the given value plus one.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func log1p(_ col: Column) -> Column {
+ return fn("log1p", col)
+}
+
+/// Computes the logarithm of the given column in base 2.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func log2(_ col: Column) -> Column {
+ return fn("log2", col)
+}
+
+/// Returns the negated value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func negative(_ col: Column) -> Column {
+ return fn("negative", col)
+}
+
+/// Returns Pi.
+/// - Returns: A ``Column``.
+public func pi() -> Column {
+ return fn("pi")
+}
+
+/// Returns the positive value of dividend mod divisor.
+/// - Parameters:
+/// - dividend: A dividend ``Column``.
+/// - divisor: A divisor ``Column``.
+/// - Returns: A ``Column``.
+public func pmod(_ dividend: Column, _ divisor: Column) -> Column {
+ return fn("pmod", dividend, divisor)
+}
+
+/// Returns the value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func positive(_ col: Column) -> Column {
+ return fn("positive", col)
+}
+
+/// Returns the value of the first argument raised to the power of the second
argument.
+/// - Parameters:
+/// - l: A base ``Column``.
+/// - r: An exponent ``Column``.
+/// - Returns: A ``Column``.
+public func pow(_ l: Column, _ r: Column) -> Column {
+ return fn("power", l, r)
+}
+
+/// Returns the value of the first argument raised to the power of the second
argument.
+/// This is an alias of ``pow(_:_:)``.
+/// - Parameters:
+/// - l: A base ``Column``.
+/// - r: An exponent ``Column``.
+/// - Returns: A ``Column``.
+public func power(_ l: Column, _ r: Column) -> Column {
+ return fn("power", l, r)
+}
+
+/// Converts an angle measured in degrees to an approximately equivalent angle
measured in
+/// radians.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func radians(_ col: Column) -> Column {
+ return fn("radians", col)
+}
+
+/// Returns the double value that is closest in value to the argument and is
equal to a
+/// mathematical integer.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func rint(_ col: Column) -> Column {
+ return fn("rint", col)
+}
+
+/// Returns the value of the column rounded to 0 decimal places with HALF_UP
round mode.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func round(_ col: Column) -> Column {
+ return round(col, 0)
+}
+
+/// Rounds the value of the column to `scale` decimal places with HALF_UP
round mode.
+/// - Parameters:
+/// - col: A ``Column``.
+/// - scale: The number of decimal places.
+/// - Returns: A ``Column``.
+public func round(_ col: Column, _ scale: Int32) -> Column {
+ return fn("round", col, lit(scale))
+}
+
+/// Computes the secant of the given value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func sec(_ col: Column) -> Column {
+ return fn("sec", col)
+}
+
+/// Shifts the given value `numBits` left.
+/// - Parameters:
+/// - col: A ``Column``.
+/// - numBits: The number of bits to shift.
+/// - Returns: A ``Column``.
+public func shiftleft(_ col: Column, _ numBits: Int32) -> Column {
+ return fn("shiftleft", col, lit(numBits))
+}
+
+/// (Signed) shifts the given value `numBits` right.
+/// - Parameters:
+/// - col: A ``Column``.
+/// - numBits: The number of bits to shift.
+/// - Returns: A ``Column``.
+public func shiftright(_ col: Column, _ numBits: Int32) -> Column {
+ return fn("shiftright", col, lit(numBits))
+}
+
+/// (Unsigned) shifts the given value `numBits` right.
+/// - Parameters:
+/// - col: A ``Column``.
+/// - numBits: The number of bits to shift.
+/// - Returns: A ``Column``.
+public func shiftrightunsigned(_ col: Column, _ numBits: Int32) -> Column {
+ return fn("shiftrightunsigned", col, lit(numBits))
+}
+
+/// Computes the signum of the given value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func sign(_ col: Column) -> Column {
+ return fn("sign", col)
+}
+
+/// Computes the signum of the given value. This is an alias of ``sign(_:)``.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func signum(_ col: Column) -> Column {
+ return fn("signum", col)
+}
+
+/// Computes the sine of the given value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func sin(_ col: Column) -> Column {
+ return fn("sin", col)
+}
+
+/// Computes the hyperbolic sine of the given value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func sinh(_ col: Column) -> Column {
+ return fn("sinh", col)
+}
+
+/// Computes the square root of the specified value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func sqrt(_ col: Column) -> Column {
+ return fn("sqrt", col)
+}
+
+/// Computes the tangent of the given value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func tan(_ col: Column) -> Column {
+ return fn("tan", col)
+}
+
+/// Computes the hyperbolic tangent of the given value.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func tanh(_ col: Column) -> Column {
+ return fn("tanh", col)
+}
+
+/// Inverse of ``hex(_:)``. Interprets each pair of characters as a
hexadecimal number and
+/// converts to the byte representation of number.
+/// - Parameter col: A ``Column``.
+/// - Returns: A ``Column``.
+public func unhex(_ col: Column) -> Column {
+ return fn("unhex", col)
+}
+
+/// Returns the bucket number into which the value of this expression would
fall after being
+/// evaluated.
+/// - Parameters:
+/// - v: A value ``Column`` for which the bucket is computed.
+/// - min: A minimum value ``Column`` of the histogram.
+/// - max: A maximum value ``Column`` of the histogram.
+/// - numBucket: A ``Column`` for the number of buckets.
+/// - Returns: A ``Column``.
+public func width_bucket(_ v: Column, _ min: Column, _ max: Column, _
numBucket: Column) -> Column
+{
+ return fn("width_bucket", v, min, max, numBucket)
+}
diff --git a/Tests/SparkConnectTests/MathFunctionsTests.swift
b/Tests/SparkConnectTests/MathFunctionsTests.swift
new file mode 100644
index 0000000..194397d
--- /dev/null
+++ b/Tests/SparkConnectTests/MathFunctionsTests.swift
@@ -0,0 +1,202 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+import Testing
+
+@testable import SparkConnect
+
+/// A test suite for `MathFunctions`
+@Suite(.serialized)
+struct MathFunctionsTests {
+
+ @Test
+ func mathFunctions() throws {
+ for (column, name) in [
+ (abs(col("a")), "abs"),
+ (acos(col("a")), "acos"),
+ (acosh(col("a")), "acosh"),
+ (asin(col("a")), "asin"),
+ (asinh(col("a")), "asinh"),
+ (atan(col("a")), "atan"),
+ (atanh(col("a")), "atanh"),
+ (bin(col("a")), "bin"),
+ (bround(col("a")), "bround"),
+ (cbrt(col("a")), "cbrt"),
+ (ceil(col("a")), "ceil"),
+ (ceiling(col("a")), "ceiling"),
+ (cos(col("a")), "cos"),
+ (cosh(col("a")), "cosh"),
+ (cot(col("a")), "cot"),
+ (csc(col("a")), "csc"),
+ (degrees(col("a")), "degrees"),
+ (exp(col("a")), "exp"),
+ (expm1(col("a")), "expm1"),
+ (factorial(col("a")), "factorial"),
+ (floor(col("a")), "floor"),
+ (hex(col("a")), "hex"),
+ (ln(col("a")), "ln"),
+ (log(col("a")), "ln"),
+ (log10(col("a")), "log10"),
+ (log1p(col("a")), "log1p"),
+ (log2(col("a")), "log2"),
+ (negative(col("a")), "negative"),
+ (positive(col("a")), "positive"),
+ (radians(col("a")), "radians"),
+ (rint(col("a")), "rint"),
+ (round(col("a")), "round"),
+ (sec(col("a")), "sec"),
+ (sign(col("a")), "sign"),
+ (signum(col("a")), "signum"),
+ (sin(col("a")), "sin"),
+ (sinh(col("a")), "sinh"),
+ (sqrt(col("a")), "sqrt"),
+ (tan(col("a")), "tan"),
+ (tanh(col("a")), "tanh"),
+ (unhex(col("a")), "unhex"),
+ ] {
+ let expr = column.expr
+ #expect(expr.unresolvedFunction.functionName == name)
+
#expect(expr.unresolvedFunction.arguments[0].unresolvedAttribute.unparsedIdentifier
== "a")
+ }
+ }
+
+ @Test
+ func mathFunctionArguments() throws {
+ for (column, name) in [
+ (atan2(col("a"), col("b")), "atan2"),
+ (hypot(col("a"), col("b")), "hypot"),
+ (pmod(col("a"), col("b")), "pmod"),
+ (pow(col("a"), col("b")), "power"),
+ (power(col("a"), col("b")), "power"),
+ (ceil(col("a"), lit(1)), "ceil"),
+ (ceiling(col("a"), lit(1)), "ceiling"),
+ (floor(col("a"), lit(1)), "floor"),
+ ] {
+ let expr = column.expr
+ #expect(expr.unresolvedFunction.functionName == name)
+ #expect(expr.unresolvedFunction.arguments.count == 2)
+ }
+
+ for (column, name, scale) in [
+ (round(col("a"), 2), "round", Int32(2)),
+ (bround(col("a"), 2), "bround", Int32(2)),
+ (shiftleft(col("a"), 1), "shiftleft", Int32(1)),
+ (shiftright(col("a"), 1), "shiftright", Int32(1)),
+ (shiftrightunsigned(col("a"), 1), "shiftrightunsigned", Int32(1)),
+ ] {
+ let expr = column.expr
+ #expect(expr.unresolvedFunction.functionName == name)
+ #expect(expr.unresolvedFunction.arguments[1].literal.integer == scale)
+ }
+
+ let logarithm = log(10.0, col("a")).expr
+ #expect(logarithm.unresolvedFunction.functionName == "log")
+ #expect(logarithm.unresolvedFunction.arguments[0].literal.double == 10.0)
+
+ let converted = conv(col("a"), 2, 16).expr
+ #expect(converted.unresolvedFunction.functionName == "conv")
+ #expect(converted.unresolvedFunction.arguments[1].literal.integer == 2)
+ #expect(converted.unresolvedFunction.arguments[2].literal.integer == 16)
+
+ #expect(e().expr.unresolvedFunction.arguments.isEmpty)
+ #expect(pi().expr.unresolvedFunction.arguments.isEmpty)
+ #expect(
+ width_bucket(col("a"), lit(0), lit(10),
lit(5)).expr.unresolvedFunction.arguments.count == 4)
+ }
+
+ @Test
+ func selectMathFunctions() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let df = try await spark.sql("SELECT double(-4.0) AS a, double(2.345) AS
b, double(2.5) AS c")
+ let rows = try await df.select(
+ abs(col("a")), sqrt(abs(col("a"))), round(col("b")), round(col("b"), 2)
+ ).collect()
+ #expect(rows == [Row(4.0, 2.0, 2.0, 2.35)])
+
+ let rounded = try await df.select(
+ ceil(col("b")), ceiling(col("b")), floor(col("b")), round(col("c")),
bround(col("c")),
+ rint(col("b"))
+ ).collect()
+ #expect(rounded == [Row(3, 3, 2, 3.0, 2.0, 2.0)])
+
+ let signs = try await df.select(
+ sign(col("a")), signum(col("a")), negative(col("a")), positive(col("a")),
+ factorial(lit(5)), pmod(lit(-7), lit(3))
+ ).collect()
+ #expect(signs == [Row(-1.0, -1.0, 4.0, -4.0, 120, 2)])
+
+ let constants = try await df.select(
+ e(), pi(), width_bucket(lit(5.35), lit(0.024), lit(10.06), lit(5))
+ ).collect()
+ #expect(constants == [Row(2.718281828459045, 3.141592653589793, 3)])
+ await spark.stop()
+ }
+
+ @Test
+ func selectTrigonometricFunctions() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let df = try await spark.range(1)
+ let rows = try await df.select(
+ sin(lit(0.0)), cos(lit(0.0)), tan(lit(0.0)), sec(lit(0.0)),
atan2(lit(0.0), lit(1.0))
+ ).collect()
+ #expect(rows == [Row(0.0, 1.0, 0.0, 1.0, 0.0)])
+
+ let inverse = try await df.select(
+ asin(lit(0.0)), acos(lit(1.0)), atan(lit(0.0)), degrees(lit(0.0)),
radians(lit(0.0))
+ ).collect()
+ #expect(inverse == [Row(0.0, 0.0, 0.0, 0.0, 0.0)])
+
+ let hyperbolic = try await df.select(
+ sinh(lit(0.0)), cosh(lit(0.0)), tanh(lit(0.0)), asinh(lit(0.0)),
acosh(lit(1.0)),
+ atanh(lit(0.0))
+ ).collect()
+ #expect(hyperbolic == [Row(0.0, 1.0, 0.0, 0.0, 0.0, 0.0)])
+ await spark.stop()
+ }
+
+ @Test
+ func selectLogarithmicFunctions() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let df = try await spark.range(1)
+ let rows = try await df.select(
+ exp(lit(0.0)), expm1(lit(0.0)), ln(lit(1.0)), log(lit(1.0)),
log1p(lit(0.0)),
+ log10(lit(100.0))
+ ).collect()
+ #expect(rows == [Row(1.0, 0.0, 0.0, 0.0, 0.0, 2.0)])
+
+ let powers = try await df.select(
+ pow(lit(2.0), lit(10.0)), power(lit(3.0), lit(2.0)), hypot(lit(3.0),
lit(4.0)),
+ log2(lit(1.0)), log(2.0, lit(1.0))
+ ).collect()
+ #expect(powers == [Row(1024.0, 9.0, 5.0, 0.0, 0.0)])
+ await spark.stop()
+ }
+
+ @Test
+ func selectBaseConversionFunctions() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let df = try await spark.range(1)
+ let rows = try await df.select(
+ bin(lit(5)), hex(lit(255)), conv(lit("100"), 2, 10),
+ shiftleft(lit(1), 3), shiftright(lit(8), 2), shiftrightunsigned(lit(8),
2)
+ ).collect()
+ #expect(rows == [Row("101", "FF", "4", 8, 2, 2)])
+ await spark.stop()
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]