Repository: incubator-griffin Updated Branches: refs/heads/master 6b389b316 -> 88ac8f7cd
enable scala test in maven project Author: Lionel Liu <bhlx3l...@163.com> Author: dodobel <1254288...@qq.com> Closes #301 from bhlx3lyx7/spark2. Project: http://git-wip-us.apache.org/repos/asf/incubator-griffin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-griffin/commit/88ac8f7c Tree: http://git-wip-us.apache.org/repos/asf/incubator-griffin/tree/88ac8f7c Diff: http://git-wip-us.apache.org/repos/asf/incubator-griffin/diff/88ac8f7c Branch: refs/heads/master Commit: 88ac8f7cdf69c62a9b8fad931e1d2edea27f0510 Parents: 6b389b3 Author: Lionel Liu <bhlx3l...@163.com> Authored: Thu Jun 14 14:24:25 2018 +0800 Committer: Lionel Liu <bhlx3l...@163.com> Committed: Thu Jun 14 14:24:25 2018 +0800 ---------------------------------------------------------------------- measure/pom.xml | 15 ++++ .../measure/configuration/params/DQConfig.scala | 2 +- .../apache/griffin/measure/launch/DQApp.scala | 6 +- .../griffin/measure/ApplicationTest.scala | 41 ----------- .../measure/context/MetricWrapperTest.scala | 42 +++++++++++ .../datasource/TimestampStorageTest.scala | 75 ++++++++++++++++++++ 6 files changed, 137 insertions(+), 44 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/88ac8f7c/measure/pom.xml ---------------------------------------------------------------------- diff --git a/measure/pom.xml b/measure/pom.xml index 79cf3cb..5b95025 100644 --- a/measure/pom.xml +++ b/measure/pom.xml @@ -186,11 +186,26 @@ under the License. <id>compile</id> <goals> <goal>compile</goal> + <goal>testCompile</goal> </goals> <phase>compile</phase> </execution> </executions> </plugin> + <!-- enable scalatest --> + <plugin> + <groupId>org.scalatest</groupId> + <artifactId>scalatest-maven-plugin</artifactId> + <version>1.0</version> + <executions> + <execution> + <id>test</id> + <goals> + <goal>test</goal> + </goals> + </execution> + </executions> + </plugin> <plugin> <artifactId>maven-compiler-plugin</artifactId> <version>3.5.1</version> http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/88ac8f7c/measure/src/main/scala/org/apache/griffin/measure/configuration/params/DQConfig.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/configuration/params/DQConfig.scala b/measure/src/main/scala/org/apache/griffin/measure/configuration/params/DQConfig.scala index d07ab51..d13f61e 100644 --- a/measure/src/main/scala/org/apache/griffin/measure/configuration/params/DQConfig.scala +++ b/measure/src/main/scala/org/apache/griffin/measure/configuration/params/DQConfig.scala @@ -39,7 +39,7 @@ case class DQConfig(@JsonProperty("name") name: String, @JsonProperty("evaluate.rule") evaluateRule: EvaluateRuleParam ) extends Param { def getName: String = name - def getTimestamp: Long = timestamp + def getTimestampOpt: Option[Long] = if (timestamp != null) Some(timestamp) else None def getProcType: String = procType def getDataSources: Seq[DataSourceParam] = { dataSources.foldLeft((Nil: Seq[DataSourceParam], Set[String]())) { (ret, ds) => http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/88ac8f7c/measure/src/main/scala/org/apache/griffin/measure/launch/DQApp.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/launch/DQApp.scala b/measure/src/main/scala/org/apache/griffin/measure/launch/DQApp.scala index 79cef33..29cee08 100644 --- a/measure/src/main/scala/org/apache/griffin/measure/launch/DQApp.scala +++ b/measure/src/main/scala/org/apache/griffin/measure/launch/DQApp.scala @@ -48,8 +48,10 @@ trait DQApp extends Loggable with Serializable { * timestamp as a key for metrics */ protected def getMeasureTime: Long = { - if (dqParam.timestamp != null && dqParam.timestamp > 0) { dqParam.timestamp } - else { System.currentTimeMillis } + dqParam.getTimestampOpt match { + case Some(t) if t > 0 => dqParam.timestamp + case _ => System.currentTimeMillis + } } } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/88ac8f7c/measure/src/test/scala/org/apache/griffin/measure/ApplicationTest.scala ---------------------------------------------------------------------- diff --git a/measure/src/test/scala/org/apache/griffin/measure/ApplicationTest.scala b/measure/src/test/scala/org/apache/griffin/measure/ApplicationTest.scala deleted file mode 100644 index 9ad9d9e..0000000 --- a/measure/src/test/scala/org/apache/griffin/measure/ApplicationTest.scala +++ /dev/null @@ -1,41 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure - -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -import org.scalatest.{BeforeAndAfter, FunSuite, Matchers} - -import scala.util.{Failure, Success, Try} - -@RunWith(classOf[JUnitRunner]) -class ApplicationTest extends FunSuite with Matchers with BeforeAndAfter { - - val envFile = "src/test/resources/env-batch.json" -// val envFile = "src/test/resources/env-streaming.json" - - val confFile = "src/test/resources/_accuracy-batch-griffindsl.json" -// val confFile = "src/test/resources/_accuracy-streaming-griffindsl.json" - - test("test application") { - val args = Array[String](envFile, confFile) - Application.main(args) - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/88ac8f7c/measure/src/test/scala/org/apache/griffin/measure/context/MetricWrapperTest.scala ---------------------------------------------------------------------- diff --git a/measure/src/test/scala/org/apache/griffin/measure/context/MetricWrapperTest.scala b/measure/src/test/scala/org/apache/griffin/measure/context/MetricWrapperTest.scala new file mode 100644 index 0000000..c835611 --- /dev/null +++ b/measure/src/test/scala/org/apache/griffin/measure/context/MetricWrapperTest.scala @@ -0,0 +1,42 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ +package org.apache.griffin.measure.context + +import org.scalatest._ + +class MetricWrapperTest extends FlatSpec with Matchers { + + "metric wrapper" should "flush empty if no metric inserted" in { + val metricWrapper = MetricWrapper("name") + metricWrapper.flush should be (Map[Long, Map[String, Any]]()) + } + + it should "flush all metrics inserted" in { + val metricWrapper = MetricWrapper("test") + metricWrapper.insertMetric(1, Map("total" -> 10, "miss"-> 2)) + metricWrapper.insertMetric(1, Map("match" -> 8)) + metricWrapper.insertMetric(2, Map("total" -> 20)) + metricWrapper.insertMetric(2, Map("miss" -> 4)) + metricWrapper.flush should be (Map( + 1L -> Map("name" -> "test", "tmst" -> 1, "value" -> Map("total" -> 10, "miss"-> 2, "match" -> 8)), + 2L -> Map("name" -> "test", "tmst" -> 2, "value" -> Map("total" -> 20, "miss"-> 4)) + )) + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/88ac8f7c/measure/src/test/scala/org/apache/griffin/measure/datasource/TimestampStorageTest.scala ---------------------------------------------------------------------- diff --git a/measure/src/test/scala/org/apache/griffin/measure/datasource/TimestampStorageTest.scala b/measure/src/test/scala/org/apache/griffin/measure/datasource/TimestampStorageTest.scala new file mode 100644 index 0000000..9f788b3 --- /dev/null +++ b/measure/src/test/scala/org/apache/griffin/measure/datasource/TimestampStorageTest.scala @@ -0,0 +1,75 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ +package org.apache.griffin.measure.datasource + +import org.scalatest._ + +class TimestampStorageTest extends FlatSpec with Matchers { + + "timestamp storage" should "be able to insert a timestamp" in { + val timestampStorage = TimestampStorage() + timestampStorage.insert(1L) + timestampStorage.all should be (Set(1L)) + } + + it should "be able to insert timestamps" in { + val timestampStorage = TimestampStorage() + timestampStorage.insert(Seq(1L, 2L, 3L)) + timestampStorage.all should be (Set(1L, 2L, 3L)) + } + + it should "be able to remove a timestamp" in { + val timestampStorage = TimestampStorage() + timestampStorage.insert(Seq(1L, 2L, 3L)) + timestampStorage.remove(1L) + timestampStorage.all should be (Set(2L, 3L)) + } + + it should "be able to remove timestamps" in { + val timestampStorage = TimestampStorage() + timestampStorage.insert(Seq(1L, 2L, 3L)) + timestampStorage.remove(Seq(1L, 2L)) + timestampStorage.all should be (Set(3L)) + } + + it should "be able to get timestamps in range [a, b)" in { + val timestampStorage = TimestampStorage() + timestampStorage.insert(Seq(6L, 2L, 3L, 4L, 8L)) + timestampStorage.fromUntil(2L, 6L) should be (Set(2L, 3L, 4L)) + } + + it should "be able to get timestamps in range (a, b]" in { + val timestampStorage = TimestampStorage() + timestampStorage.insert(Seq(6L, 2L, 3L, 4L, 8L)) + timestampStorage.afterTil(2L, 6L) should be (Set(3L, 4L, 6L)) + } + + it should "be able to get timestamps smaller than b" in { + val timestampStorage = TimestampStorage() + timestampStorage.insert(Seq(6L, 2L, 3L, 4L, 8L)) + timestampStorage.until(8L) should be (Set(2L, 3L, 4L, 6L)) + } + + it should "be able to get timestamps bigger than or equal a" in { + val timestampStorage = TimestampStorage() + timestampStorage.insert(Seq(6L, 2L, 3L, 4L, 8L)) + timestampStorage.from(4L) should be (Set(4L, 6L, 8L)) + } + +}
