This is an automated email from the ASF dual-hosted git repository. jmalkin pushed a commit to branch rename_packages in repository https://gitbox.apache.org/repos/asf/datasketches-spark.git
commit 7fd48171a5aa5422261ad8ef4fe04ef90537ba8a Author: Jon Malkin <[email protected]> AuthorDate: Tue Jan 28 16:05:21 2025 -0800 Rename packages to use org.apache.spark.sql.datasketches and update imports as needed --- .../{registrar => common}/DatasketchesFunctionRegistry.scala | 2 +- .../{registrar => common}/DatasketchesScalaFunctionsBase.scala | 3 ++- .../spark/sql/{registrar => kll}/KllFunctionRegistry.scala | 8 +++++--- .../spark/sql/kll/aggregate/KllDoublesSketchAggBuild.scala | 6 ++++-- .../spark/sql/kll/aggregate/KllDoublesSketchAggMerge.scala | 5 +++-- .../sql/kll/expressions/KllDoublesSketchExpressions.scala | 4 ++-- .../functions_datasketches_kll.scala => kll/functions.scala} | 10 ++++++---- .../org/apache/spark/sql/kll/types/KllDoublesSketchType.scala | 3 ++- .../spark/sql/{registrar => theta}/ThetaFunctionRegistry.scala | 4 +++- .../functions.scala} | 7 +++++-- src/test/scala/org/apache/spark/sql/SparkSessionManager.scala | 3 ++- src/test/scala/org/apache/spark/sql/{ => kll}/KllTest.scala | 9 +++++---- .../scala/org/apache/spark/sql/{ => theta}/ThetaTest.scala | 8 +++++--- 13 files changed, 45 insertions(+), 27 deletions(-) diff --git a/src/main/scala/org/apache/spark/sql/registrar/DatasketchesFunctionRegistry.scala b/src/main/scala/org/apache/spark/sql/common/DatasketchesFunctionRegistry.scala similarity index 98% rename from src/main/scala/org/apache/spark/sql/registrar/DatasketchesFunctionRegistry.scala rename to src/main/scala/org/apache/spark/sql/common/DatasketchesFunctionRegistry.scala index 86095d8..bdc8235 100644 --- a/src/main/scala/org/apache/spark/sql/registrar/DatasketchesFunctionRegistry.scala +++ b/src/main/scala/org/apache/spark/sql/common/DatasketchesFunctionRegistry.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.spark.sql.registrar +package org.apache.spark.sql.datasketches.common import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.FunctionIdentifier diff --git a/src/main/scala/org/apache/spark/sql/registrar/DatasketchesScalaFunctionsBase.scala b/src/main/scala/org/apache/spark/sql/common/DatasketchesScalaFunctionsBase.scala similarity index 94% rename from src/main/scala/org/apache/spark/sql/registrar/DatasketchesScalaFunctionsBase.scala rename to src/main/scala/org/apache/spark/sql/common/DatasketchesScalaFunctionsBase.scala index f0ee23a..b060dad 100644 --- a/src/main/scala/org/apache/spark/sql/registrar/DatasketchesScalaFunctionsBase.scala +++ b/src/main/scala/org/apache/spark/sql/common/DatasketchesScalaFunctionsBase.scala @@ -15,10 +15,11 @@ * limitations under the License. */ -package org.apache.spark.sql +package org.apache.spark.sql.datasketches.common import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateFunction +import org.apache.spark.sql.Column // this interfact provides a few helper methods defines and maps all the variants of each function invocation, analagous // to the functions object in core Spark's org.apache.spark.sql.functions diff --git a/src/main/scala/org/apache/spark/sql/registrar/KllFunctionRegistry.scala b/src/main/scala/org/apache/spark/sql/kll/KllFunctionRegistry.scala similarity index 76% rename from src/main/scala/org/apache/spark/sql/registrar/KllFunctionRegistry.scala rename to src/main/scala/org/apache/spark/sql/kll/KllFunctionRegistry.scala index a1fe6a1..550e6c3 100644 --- a/src/main/scala/org/apache/spark/sql/registrar/KllFunctionRegistry.scala +++ b/src/main/scala/org/apache/spark/sql/kll/KllFunctionRegistry.scala @@ -15,13 +15,15 @@ * limitations under the License. */ -package org.apache.spark.sql.registrar +package org.apache.spark.sql.datasketches.kll import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder import org.apache.spark.sql.catalyst.expressions.{ExpressionInfo} -import org.apache.spark.sql.aggregate.{KllDoublesSketchAggBuild, KllDoublesSketchAggMerge} -import org.apache.spark.sql.expressions.{KllDoublesSketchGetMin, KllDoublesSketchGetMax, KllDoublesSketchGetPmf, KllDoublesSketchGetCdf} +import org.apache.spark.sql.datasketches.common.DatasketchesFunctionRegistry +import org.apache.spark.sql.datasketches.kll.aggregate.{KllDoublesSketchAggBuild, KllDoublesSketchAggMerge} +import org.apache.spark.sql.datasketches.kll.expressions.{KllDoublesSketchGetMin, KllDoublesSketchGetMax, KllDoublesSketchGetPmf, KllDoublesSketchGetCdf} +import org.apache.spark.sql.datasketches.common.DatasketchesFunctionRegistry object KllFunctionRegistry extends DatasketchesFunctionRegistry { override val expressions: Map[String, (ExpressionInfo, FunctionBuilder)] = Map( diff --git a/src/main/scala/org/apache/spark/sql/kll/aggregate/KllDoublesSketchAggBuild.scala b/src/main/scala/org/apache/spark/sql/kll/aggregate/KllDoublesSketchAggBuild.scala index 53225ea..7e02240 100644 --- a/src/main/scala/org/apache/spark/sql/kll/aggregate/KllDoublesSketchAggBuild.scala +++ b/src/main/scala/org/apache/spark/sql/kll/aggregate/KllDoublesSketchAggBuild.scala @@ -15,14 +15,16 @@ * limitations under the License. */ -package org.apache.spark.sql.aggregate +package org.apache.spark.sql.datasketches.kll.aggregate import org.apache.datasketches.kll.{KllSketch, KllDoublesSketch} +import org.apache.spark.sql.datasketches.kll.types.KllDoublesSketchType + import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes, Expression, ExpressionDescription, Literal} import org.apache.spark.sql.catalyst.expressions.aggregate.TypedImperativeAggregate import org.apache.spark.sql.catalyst.trees.BinaryLike -import org.apache.spark.sql.types.{AbstractDataType, DataType, IntegerType, LongType, NumericType, FloatType, DoubleType, KllDoublesSketchType} +import org.apache.spark.sql.types.{AbstractDataType, DataType, IntegerType, LongType, NumericType, FloatType, DoubleType} import org.apache.spark.sql.catalyst.analysis.TypeCheckResult /** diff --git a/src/main/scala/org/apache/spark/sql/kll/aggregate/KllDoublesSketchAggMerge.scala b/src/main/scala/org/apache/spark/sql/kll/aggregate/KllDoublesSketchAggMerge.scala index bf4c51c..cf0dc7e 100644 --- a/src/main/scala/org/apache/spark/sql/kll/aggregate/KllDoublesSketchAggMerge.scala +++ b/src/main/scala/org/apache/spark/sql/kll/aggregate/KllDoublesSketchAggMerge.scala @@ -15,16 +15,17 @@ * limitations under the License. */ -package org.apache.spark.sql.aggregate +package org.apache.spark.sql.datasketches.kll.aggregate import org.apache.datasketches.memory.Memory import org.apache.datasketches.kll.{KllSketch, KllDoublesSketch} +import org.apache.spark.sql.datasketches.kll.types.KllDoublesSketchType import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes, Expression, ExpressionDescription, Literal} import org.apache.spark.sql.catalyst.expressions.aggregate.TypedImperativeAggregate import org.apache.spark.sql.catalyst.trees.BinaryLike -import org.apache.spark.sql.types.{AbstractDataType, DataType, IntegerType, KllDoublesSketchType} +import org.apache.spark.sql.types.{AbstractDataType, DataType, IntegerType} import org.apache.spark.sql.catalyst.analysis.TypeCheckResult /** diff --git a/src/main/scala/org/apache/spark/sql/kll/expressions/KllDoublesSketchExpressions.scala b/src/main/scala/org/apache/spark/sql/kll/expressions/KllDoublesSketchExpressions.scala index 0ff03e6..f59b0dd 100644 --- a/src/main/scala/org/apache/spark/sql/kll/expressions/KllDoublesSketchExpressions.scala +++ b/src/main/scala/org/apache/spark/sql/kll/expressions/KllDoublesSketchExpressions.scala @@ -15,12 +15,12 @@ * limitations under the License. */ -package org.apache.spark.sql.expressions +package org.apache.spark.sql.datasketches.kll.expressions import org.apache.datasketches.memory.Memory import org.apache.datasketches.kll.KllDoublesSketch import org.apache.datasketches.quantilescommon.QuantileSearchCriteria -import org.apache.spark.sql.types.KllDoublesSketchType +import org.apache.spark.sql.datasketches.kll.types.KllDoublesSketchType import org.apache.spark.sql.types.{AbstractDataType, ArrayType, BooleanType, DataType, DoubleType} import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription, ExpectsInputTypes, ImplicitCastInputTypes} diff --git a/src/main/scala/org/apache/spark/sql/registrar/functions_datasketches_kll.scala b/src/main/scala/org/apache/spark/sql/kll/functions.scala similarity index 92% rename from src/main/scala/org/apache/spark/sql/registrar/functions_datasketches_kll.scala rename to src/main/scala/org/apache/spark/sql/kll/functions.scala index 734110c..fe48c1c 100644 --- a/src/main/scala/org/apache/spark/sql/registrar/functions_datasketches_kll.scala +++ b/src/main/scala/org/apache/spark/sql/kll/functions.scala @@ -15,16 +15,18 @@ * limitations under the License. */ -package org.apache.spark.sql +package org.apache.spark.sql.datasketches.kll +import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.Literal import org.apache.spark.sql.functions.lit import org.apache.spark.sql.types.{ArrayType, BooleanType, DoubleType} -import org.apache.spark.sql.aggregate.{KllDoublesSketchAggMerge, KllDoublesSketchAggBuild} -import org.apache.spark.sql.expressions.{KllDoublesSketchGetMin, KllDoublesSketchGetMax, KllDoublesSketchGetPmfCdf} +import org.apache.spark.sql.datasketches.common.DatasketchesScalaFunctionBase +import org.apache.spark.sql.datasketches.kll.aggregate.{KllDoublesSketchAggMerge, KllDoublesSketchAggBuild} +import org.apache.spark.sql.datasketches.kll.expressions.{KllDoublesSketchGetMin, KllDoublesSketchGetMax, KllDoublesSketchGetPmfCdf} -object functions_datasketches_kll extends DatasketchesScalaFunctionBase { +object functions extends DatasketchesScalaFunctionBase { // build sketch def kll_sketch_double_agg_build(expr: Column, k: Column): Column = withAggregateFunction { diff --git a/src/main/scala/org/apache/spark/sql/kll/types/KllDoublesSketchType.scala b/src/main/scala/org/apache/spark/sql/kll/types/KllDoublesSketchType.scala index 2fb01e9..a058190 100644 --- a/src/main/scala/org/apache/spark/sql/kll/types/KllDoublesSketchType.scala +++ b/src/main/scala/org/apache/spark/sql/kll/types/KllDoublesSketchType.scala @@ -15,11 +15,12 @@ * limitations under the License. */ -package org.apache.spark.sql.types +package org.apache.spark.sql.datasketches.kll.types import org.apache.spark.sql.functions.udf import org.apache.datasketches.kll.KllDoublesSketch import org.apache.datasketches.memory.Memory +import org.apache.spark.sql.types.{DataType, DataTypes, UDTRegistration, UserDefinedType} class KllDoublesSketchType extends UserDefinedType[KllDoublesSketch] with Serializable { override def sqlType: DataType = DataTypes.BinaryType diff --git a/src/main/scala/org/apache/spark/sql/registrar/ThetaFunctionRegistry.scala b/src/main/scala/org/apache/spark/sql/theta/ThetaFunctionRegistry.scala similarity index 87% rename from src/main/scala/org/apache/spark/sql/registrar/ThetaFunctionRegistry.scala rename to src/main/scala/org/apache/spark/sql/theta/ThetaFunctionRegistry.scala index 0062154..02c00ee 100644 --- a/src/main/scala/org/apache/spark/sql/registrar/ThetaFunctionRegistry.scala +++ b/src/main/scala/org/apache/spark/sql/theta/ThetaFunctionRegistry.scala @@ -15,13 +15,15 @@ * limitations under the License. */ -package org.apache.spark.sql.registrar +package org.apache.spark.sql.datasketches.theta import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder import org.apache.spark.sql.catalyst.expressions.{ExpressionInfo} +import org.apache.spark.sql.datasketches.common.DatasketchesFunctionRegistry import org.apache.spark.sql.aggregate.{ThetaSketchAggBuild, ThetaSketchAggUnion} import org.apache.spark.sql.expressions.ThetaSketchGetEstimate +import org.apache.spark.sql.datasketches.common.DatasketchesFunctionRegistry object ThetaFunctionRegistry extends DatasketchesFunctionRegistry { override val expressions: Map[String, (ExpressionInfo, FunctionBuilder)] = Map( diff --git a/src/main/scala/org/apache/spark/sql/registrar/functions_datasketches_theta.scala b/src/main/scala/org/apache/spark/sql/theta/functions.scala similarity index 88% rename from src/main/scala/org/apache/spark/sql/registrar/functions_datasketches_theta.scala rename to src/main/scala/org/apache/spark/sql/theta/functions.scala index 3c33054..2cdd222 100644 --- a/src/main/scala/org/apache/spark/sql/registrar/functions_datasketches_theta.scala +++ b/src/main/scala/org/apache/spark/sql/theta/functions.scala @@ -15,14 +15,17 @@ * limitations under the License. */ -package org.apache.spark.sql +package org.apache.spark.sql.datasketches.theta +import org.apache.spark.sql.Column import org.apache.spark.sql.functions.lit +import org.apache.spark.sql.datasketches.common.DatasketchesScalaFunctionBase import org.apache.spark.sql.aggregate.{ThetaSketchAggBuild, ThetaSketchAggUnion} import org.apache.spark.sql.expressions.ThetaSketchGetEstimate +import org.apache.spark.sql.datasketches.common.DatasketchesScalaFunctionBase -object functions_datasketches_theta extends DatasketchesScalaFunctionBase { +object functions extends DatasketchesScalaFunctionBase { def theta_sketch_agg_build(column: Column, lgk: Int): Column = withAggregateFunction { new ThetaSketchAggBuild(column.expr, lgk) } diff --git a/src/test/scala/org/apache/spark/sql/SparkSessionManager.scala b/src/test/scala/org/apache/spark/sql/SparkSessionManager.scala index 4c96cb4..8aa20bf 100644 --- a/src/test/scala/org/apache/spark/sql/SparkSessionManager.scala +++ b/src/test/scala/org/apache/spark/sql/SparkSessionManager.scala @@ -15,11 +15,12 @@ * limitations under the License. */ -package org.apache.spark.sql +package org.apache.spark.sql.datasketches.common import org.apache.log4j.{Level, Logger} import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite +import org.apache.spark.sql.SparkSession /** * This class provides a common base for tests. It can perhaps diff --git a/src/test/scala/org/apache/spark/sql/KllTest.scala b/src/test/scala/org/apache/spark/sql/kll/KllTest.scala similarity index 97% rename from src/test/scala/org/apache/spark/sql/KllTest.scala rename to src/test/scala/org/apache/spark/sql/kll/KllTest.scala index 5123774..0a17a9d 100644 --- a/src/test/scala/org/apache/spark/sql/KllTest.scala +++ b/src/test/scala/org/apache/spark/sql/kll/KllTest.scala @@ -15,16 +15,17 @@ * limitations under the License. */ -package org.apache.spark.sql +package org.apache.spark.sql.datasketches.kll import scala.util.Random +import org.apache.spark.sql.Row import org.apache.spark.sql.functions._ import org.apache.spark.sql.types.{StructType, StructField, IntegerType, BinaryType} -import org.apache.spark.sql.functions_datasketches_kll._ import org.apache.datasketches.kll.KllDoublesSketch -import org.apache.spark.sql.types.KllDoublesSketchType -import org.apache.spark.sql.registrar.KllFunctionRegistry +import org.apache.spark.sql.datasketches.kll.functions._ +import org.apache.spark.sql.datasketches.kll.types.KllDoublesSketchType +import org.apache.spark.sql.datasketches.common.SparkSessionManager class KllTest extends SparkSessionManager { import spark.implicits._ diff --git a/src/test/scala/org/apache/spark/sql/ThetaTest.scala b/src/test/scala/org/apache/spark/sql/theta/ThetaTest.scala similarity index 94% rename from src/test/scala/org/apache/spark/sql/ThetaTest.scala rename to src/test/scala/org/apache/spark/sql/theta/ThetaTest.scala index 6c93d25..53bd0aa 100644 --- a/src/test/scala/org/apache/spark/sql/ThetaTest.scala +++ b/src/test/scala/org/apache/spark/sql/theta/ThetaTest.scala @@ -15,10 +15,12 @@ * limitations under the License. */ -package org.apache.spark.sql +package org.apache.spark.sql.theta -import org.apache.spark.sql.functions_datasketches_theta._ -import org.apache.spark.sql.registrar.ThetaFunctionRegistry +import org.apache.spark.sql.Row +import org.apache.spark.sql.datasketches.common.SparkSessionManager +import org.apache.spark.sql.datasketches.theta.functions._ +import org.apache.spark.sql.datasketches.theta.ThetaFunctionRegistry class ThetaTest extends SparkSessionManager { import spark.implicits._ --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
