Repository: incubator-hivemall Updated Branches: refs/heads/master 0861d5703 -> 18ce75f78
Close #42: [HIVEMALL-38][SPARK] Support ChangeFinderUDF in DataFrame/Spark Project: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/commit/18ce75f7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/tree/18ce75f7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/diff/18ce75f7 Branch: refs/heads/master Commit: 18ce75f78f30f7f9ceeaf990ffbc1de280a2668c Parents: 0861d57 Author: Takeshi Yamamuro <[email protected]> Authored: Thu Mar 2 16:03:36 2017 +0900 Committer: myui <[email protected]> Committed: Thu Mar 2 16:03:36 2017 +0900 ---------------------------------------------------------------------- .../scala/org/apache/spark/sql/hive/HivemallOps.scala | 13 +++++++++++++ .../org/apache/spark/sql/hive/HivemallOpsSuite.scala | 3 ++- .../scala/org/apache/spark/sql/hive/HivemallOps.scala | 13 +++++++++++++ .../org/apache/spark/sql/hive/HivemallOpsSuite.scala | 3 ++- 4 files changed, 30 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/18ce75f7/spark/spark-2.0/src/main/scala/org/apache/spark/sql/hive/HivemallOps.scala ---------------------------------------------------------------------- diff --git a/spark/spark-2.0/src/main/scala/org/apache/spark/sql/hive/HivemallOps.scala b/spark/spark-2.0/src/main/scala/org/apache/spark/sql/hive/HivemallOps.scala index 28653a5..7b13892 100644 --- a/spark/spark-2.0/src/main/scala/org/apache/spark/sql/hive/HivemallOps.scala +++ b/spark/spark-2.0/src/main/scala/org/apache/spark/sql/hive/HivemallOps.scala @@ -857,6 +857,19 @@ object HivemallOps { } /** + * @see [[hivemall.anomaly.ChangeFinderUDF]] + * @group anomaly + */ + @scala.annotation.varargs + def changefinder(exprs: Column*): Column = withExpr { + planHiveGenericUDF( + "hivemall.anomaly.ChangeFinderUDF", + "changefinder", + exprs + ) + } + + /** * @see [[hivemall.anomaly.SingularSpectrumTransformUDF]] * @group anomaly */ http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/18ce75f7/spark/spark-2.0/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala ---------------------------------------------------------------------- diff --git a/spark/spark-2.0/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala b/spark/spark-2.0/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala index bab5085..1430f09 100644 --- a/spark/spark-2.0/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala +++ b/spark/spark-2.0/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala @@ -33,7 +33,8 @@ final class HivemallOpsWithFeatureSuite extends HivemallFeatureQueryTest { test("anomaly") { import hiveContext.implicits._ val df = spark.range(1000).selectExpr("id AS time", "rand() AS x") - // TODO: Test results more exactly + // TODO: Test results more strictly + assert(df.sort($"time".asc).select(changefinder($"x")).count === 1000) assert(df.sort($"time".asc).select(sst($"x", lit("-th 0.005"))).count === 1000) } http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/18ce75f7/spark/spark-2.1/src/main/scala/org/apache/spark/sql/hive/HivemallOps.scala ---------------------------------------------------------------------- diff --git a/spark/spark-2.1/src/main/scala/org/apache/spark/sql/hive/HivemallOps.scala b/spark/spark-2.1/src/main/scala/org/apache/spark/sql/hive/HivemallOps.scala index 6913d24..6883ac1 100644 --- a/spark/spark-2.1/src/main/scala/org/apache/spark/sql/hive/HivemallOps.scala +++ b/spark/spark-2.1/src/main/scala/org/apache/spark/sql/hive/HivemallOps.scala @@ -905,6 +905,19 @@ object HivemallOps { } /** + * @see [[hivemall.anomaly.ChangeFinderUDF]] + * @group anomaly + */ + @scala.annotation.varargs + def changefinder(exprs: Column*): Column = withExpr { + planHiveGenericUDF( + "hivemall.anomaly.ChangeFinderUDF", + "changefinder", + exprs + ) + } + + /** * @see [[hivemall.anomaly.SingularSpectrumTransformUDF]] * @group anomaly */ http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/18ce75f7/spark/spark-2.1/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala ---------------------------------------------------------------------- diff --git a/spark/spark-2.1/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala b/spark/spark-2.1/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala index 76195fd..15879e0 100644 --- a/spark/spark-2.1/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala +++ b/spark/spark-2.1/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala @@ -35,7 +35,8 @@ final class HivemallOpsWithFeatureSuite extends HivemallFeatureQueryTest { test("anomaly") { import hiveContext.implicits._ val df = spark.range(1000).selectExpr("id AS time", "rand() AS x") - // TODO: Test results more exactly + // TODO: Test results more strictly + assert(df.sort($"time".asc).select(changefinder($"x")).count === 1000) assert(df.sort($"time".asc).select(sst($"x", lit("-th 0.005"))).count === 1000) }
