Repository: incubator-griffin Updated Branches: refs/heads/master 4ca4cdb9b -> 1d7acd57a
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/users_info_src.dat ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/users_info_src.dat b/measure/src/test/resources/users_info_src.dat deleted file mode 100644 index ce49443..0000000 --- a/measure/src/test/resources/users_info_src.dat +++ /dev/null @@ -1,50 +0,0 @@ -10001|Tom001|Jerrya|201 DisneyCity|[email protected]|10000001|94022 -10002|Tom002|Jerrya|202 DisneyCity|[email protected]|10000002|94022 -10003|Tom003|Jerrya|203 DisneyCity|[email protected]|10000003|94022 -10004|Tom004|Jerrya|204 DisneyCity|[email protected]|10000004|94022 -10005|Tom005|Jerrya|205 DisneyCity|[email protected]|10000005|94022 -10006|Tom006|Jerrya|206 DisneyCity|[email protected]|10000006|94022 -10007|Tom007|Jerrya|207 DisneyCity|[email protected]|10000007|94022 -10008|Tom008|Jerrya|208 DisneyCity|[email protected]|10000008|94022 -10009|Tom009|Jerrya|209 DisneyCity|[email protected]|10000009|94022 -10010|Tom010|Jerrya|210 DisneyCity|[email protected]|10000010|94022 -10011|Tom011|Jerrya|211 DisneyCity|[email protected]|10000011|94022 -10012|Tom012|Jerrya|212 DisneyCity|[email protected]|10000012|94022 -10013|Tom013|Jerrya|213 DisneyCity|[email protected]|10000013|94022 -10014|Tom014|Jerrya|214 DisneyCity|[email protected]|10000014|94022 -10015|Tom015|Jerrya|215 DisneyCity|[email protected]|10000015|94022 -10016|Tom016|Jerrya|216 DisneyCity|[email protected]|10000016|94022 -10017|Tom017|Jerrya|217 DisneyCity|[email protected]|10000017|94022 -10018|Tom018|Jerrya|218 DisneyCity|[email protected]|10000018|94022 -10019|Tom019|Jerrya|219 DisneyCity|[email protected]|10000019|94022 -10020|Tom020|Jerrya|220 DisneyCity|[email protected]|10000020|94022 -10021|Tom021|Jerrya|221 DisneyCity|[email protected]|10000021|94022 -10022|Tom022|Jerrya|222 DisneyCity|[email protected]|10000022|94022 -10023|Tom023|Jerrya|223 DisneyCity|[email protected]|10000023|94022 -10024|Tom024|Jerrya|224 DisneyCity|[email protected]|10000024|94022 -10025|Tom025|Jerrya|225 DisneyCity|[email protected]|10000025|94022 -10026|Tom026|Jerrya|226 DisneyCity|[email protected]|10000026|94022 -10027|Tom027|Jerrya|227 DisneyCity|[email protected]|10000027|94022 -10028|Tom028|Jerrya|228 DisneyCity|[email protected]|10000028|94022 -10029|Tom029|Jerrya|229 DisneyCity|[email protected]|10000029|94022 -10030|Tom030|Jerrya|230 DisneyCity|[email protected]|10000030|94022 -10031|Tom031|Jerrya|231 DisneyCity|[email protected]|10000031|94022 -10032|Tom032|Jerrya|232 DisneyCity|[email protected]|10000032|94022 -10033|Tom033|Jerrya|233 DisneyCity|[email protected]|10000033|94022 -10034|Tom034|Jerrya|234 DisneyCity|[email protected]|10000034|94022 -10035|Tom035|Jerrya|235 DisneyCity|[email protected]|10000035|94022 -10036|Tom036|Jerrya|236 DisneyCity|[email protected]|10000036|94022 -10037|Tom037|Jerrya|237 DisneyCity|[email protected]|10000037|94022 -10038|Tom038|Jerrya|238 DisneyCity|[email protected]|10000038|94022 -10039|Tom039|Jerrya|239 DisneyCity|[email protected]|10000039|94022 -10040|Tom040|Jerrya|240 DisneyCity|[email protected]|10000040|94022 -10041|Tom041|Jerrya|241 DisneyCity|[email protected]|10000041|94022 -10042|Tom042|Jerrya|242 DisneyCity|[email protected]|10000042|94022 -10043|Tom043|Jerrya|243 DisneyCity|[email protected]|10000043|94022 -10044|Tom044|Jerrya|244 DisneyCity|[email protected]|10000044|94022 -10045|Tom045|Jerrya|245 DisneyCity|[email protected]|10000045|94022 -10046|Tom046|Jerrya|246 DisneyCity|[email protected]|10000046|94022 -10047|Tom047|Jerrya|247 DisneyCity|[email protected]|10000047|94022 -10048|Tom048|Jerrya|248 DisneyCity|[email protected]|10000048|94022 -10049|Tom049|Jerrya|249 DisneyCity|[email protected]|10000049|94022 -10050|Tom050|Jerrya|250 DisneyCity|[email protected]|10000050|94022 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/users_info_target.dat ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/users_info_target.dat b/measure/src/test/resources/users_info_target.dat deleted file mode 100644 index 07a6b40..0000000 --- a/measure/src/test/resources/users_info_target.dat +++ /dev/null @@ -1,50 +0,0 @@ -10001|Tom001|Jerrya|201 DisneyCity|[email protected]|10000101|94022 -10002|Tom002|Jerrya|202 DisneyCity|[email protected]|10000102|94022 -10003|Tom003|Jerrya|203 DisneyCity|[email protected]|10000003|94022 -10004|Tom004|Jerrya|204 DisneyCity|[email protected]|10000004|94022 -10005|Tom005|Jerrya|205 DisneyCity|[email protected]|10000005|94022 -10006|Tom006|Jerrya|206 DisneyCity|[email protected]|10000006|94022 -10007|Tom007|Jerrya|207 DisneyCity|[email protected]|10000007|94022 -10008|Tom008|Jerrya|208 DisneyCity|[email protected]|10000008|94022 -10009|Tom009|Jerrya|209 DisneyCity|[email protected]|10000009|94022 -10010|Tom010|Jerrya|210 DisneyCity|[email protected]|10000010|94022 -10011|Tom011|Jerrya|211 DisneyCity|[email protected]|10000011|94022 -10012|Tom012|Jerrya|212 DisneyCity|[email protected]|10000012|94022 -10013|Tom013|Jerrya|213 DisneyCity|[email protected]|10000013|94022 -10014|Tom014|Jerrya|214 DisneyCity|[email protected]|10000014|94022 -10015|Tom015|Jerrya|215 DisneyCity|[email protected]|10000015|94022 -10016|Tom016|Jerrya|216 DisneyCity|[email protected]|10000016|94022 -10017|Tom017|Jerrya|217 DisneyCity|[email protected]|10000017|94022 -10018|Tom018|Jerrya|218 DisneyCity|[email protected]|10000018|94022 -10019|Tom019|Jerrya|219 DisneyCity|[email protected]|10000019|94022 -10020|Tom020|Jerrya|220 DisneyCity|[email protected]|10000020|94022 -10021|Tom021|Jerrya|221 DisneyCity|[email protected]|10000021|94022 -10022|Tom022|Jerrya|222 DisneyCity|[email protected]|10000022|94022 -10023|Tom023|Jerrya|223 DisneyCity|[email protected]|10000023|94022 -10024|Tom024|Jerrya|224 DisneyCity|[email protected]|10000024|94022 -10025|Tom025|Jerrya|225 DisneyCity|[email protected]|10000025|94022 -10026|Tom026|Jerrya|226 DisneyCity|[email protected]|10000026|94022 -10027|Tom027|Jerrya|227 DisneyCity|[email protected]|10000027|94022 -10028|Tom028|Jerrya|228 DisneyCity|[email protected]|10000028|94022 -10029|Tom029|Jerrya|229 DisneyCity|[email protected]|10000029|94022 -10030|Tom030|Jerrya|230 DisneyCity|[email protected]|10000030|94022 -10031|Tom031|Jerrya|231 DisneyCity|[email protected]|10000031|94022 -10032|Tom032|Jerrya|232 DisneyCity|[email protected]|10000032|94022 -10033|Tom033|Jerrya|233 DisneyCity|[email protected]|10000033|94022 -10034|Tom034|Jerrya|234 DisneyCity|[email protected]|10000034|94022 -10035|Tom035|Jerrya|235 DisneyCity|[email protected]|10000035|94022 -10036|Tom036|Jerrya|236 DisneyCity|[email protected]|10000036|94022 -10037|Tom037|Jerrya|237 DisneyCity|[email protected]|10000037|94022 -10038|Tom038|Jerrya|238 DisneyCity|[email protected]|10000038|94022 -10039|Tom039|Jerrya|239 DisneyCity|[email protected]|10000039|94022 -10040|Tom040|Jerrya|240 DisneyCity|[email protected]|10000040|94022 -10041|Tom041|Jerrya|241 DisneyCity|[email protected]|10000041|94022 -10042|Tom042|Jerrya|242 DisneyCity|[email protected]|10000042|94022 -10043|Tom043|Jerrya|243 DisneyCity|[email protected]|10000043|94022 -10044|Tom044|Jerrya|244 DisneyCity|[email protected]|10000044|94022 -10045|Tom045|Jerrya|245 DisneyCity|[email protected]|10000045|94022 -10046|Tom046|Jerrya|246 DisneyCity|[email protected]|10000046|94022 -10047|Tom047|Jerrya|247 DisneyCity|[email protected]|10000047|94022 -10048|Tom048|Jerrya|248 DisneyCity|[email protected]|10000048|94022 -10049|Tom049|Jerrya|249 DisneyCity|[email protected]|10000049|94022 -10050|Tom050|Jerrya|250 DisneyCity|[email protected]|10000050|94022 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/scala/org/apache/griffin/measure/ApplicationTest.scala ---------------------------------------------------------------------- diff --git a/measure/src/test/scala/org/apache/griffin/measure/ApplicationTest.scala b/measure/src/test/scala/org/apache/griffin/measure/ApplicationTest.scala new file mode 100644 index 0000000..edf1134 --- /dev/null +++ b/measure/src/test/scala/org/apache/griffin/measure/ApplicationTest.scala @@ -0,0 +1,25 @@ +package org.apache.griffin.measure + +import org.junit.runner.RunWith +import org.scalatest.junit.JUnitRunner +import org.scalatest.{BeforeAndAfter, FunSuite, Matchers} + +import scala.util.{Failure, Success, Try} + +@RunWith(classOf[JUnitRunner]) +class ApplicationTest extends FunSuite with Matchers with BeforeAndAfter { + + val envFile = "src/test/resources/env.json" +// val envFile = "src/test/resources/env-streaming.json" + +// val confFile = "src/test/resources/config.json" + val confFile = "src/test/resources/config-griffindsl.json" +// val confFile = "src/test/resources/config-streaming.json" +// val confFile = "src/test/resources/config-streaming-accuracy.json" + + test("test application") { + val args = Array[String](envFile, confFile) + Application.main(args) + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/scala/org/apache/griffin/measure/config/reader/ParamRawStringReaderTest.scala ---------------------------------------------------------------------- diff --git a/measure/src/test/scala/org/apache/griffin/measure/config/reader/ParamRawStringReaderTest.scala b/measure/src/test/scala/org/apache/griffin/measure/config/reader/ParamRawStringReaderTest.scala deleted file mode 100644 index 9e5d380..0000000 --- a/measure/src/test/scala/org/apache/griffin/measure/config/reader/ParamRawStringReaderTest.scala +++ /dev/null @@ -1,37 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.config.reader - -import org.apache.griffin.measure.config.params.env._ -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -import org.scalatest.{BeforeAndAfter, FunSuite, Matchers} - -@RunWith(classOf[JUnitRunner]) -class ParamRawStringReaderTest extends FunSuite with Matchers with BeforeAndAfter { - - test("read raw config") { - val rawString = """{"type": "hdfs", "config": {"path": "/path/to", "time": 1234567}}""" - - val reader = ParamRawStringReader(rawString) - val paramTry = reader.readConfig[PersistParam] - paramTry.isSuccess should be (true) - paramTry.get should be (PersistParam("hdfs", Map[String, Any](("path" -> "/path/to"), ("time" -> 1234567)))) - } -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/scala/org/apache/griffin/measure/config/validator/AllParamValidatorTest.scala ---------------------------------------------------------------------- diff --git a/measure/src/test/scala/org/apache/griffin/measure/config/validator/AllParamValidatorTest.scala b/measure/src/test/scala/org/apache/griffin/measure/config/validator/AllParamValidatorTest.scala deleted file mode 100644 index 1f2f77c..0000000 --- a/measure/src/test/scala/org/apache/griffin/measure/config/validator/AllParamValidatorTest.scala +++ /dev/null @@ -1,40 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.config.validator - -import org.apache.griffin.measure.config.params._ -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} -import org.scalamock.scalatest.MockFactory - -@RunWith(classOf[JUnitRunner]) -class AllParamValidatorTest extends FlatSpec with Matchers with BeforeAndAfter with MockFactory { - - "validate" should "pass" in { -// val validator = AllParamValidator() -// val paramMock = mock[Param] -// paramMock.validate _ expects () returning (false) -// -// val validateTry = validator.validate(paramMock) -// validateTry.isSuccess should be (true) -// validateTry.get should be (false) - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/scala/org/apache/griffin/measure/data/connector/ConnectorTest.scala ---------------------------------------------------------------------- diff --git a/measure/src/test/scala/org/apache/griffin/measure/data/connector/ConnectorTest.scala b/measure/src/test/scala/org/apache/griffin/measure/data/connector/ConnectorTest.scala deleted file mode 100644 index 30cacc4..0000000 --- a/measure/src/test/scala/org/apache/griffin/measure/data/connector/ConnectorTest.scala +++ /dev/null @@ -1,97 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.data.connector - -import java.util.Date -import java.util.concurrent.TimeUnit - -import kafka.serializer.StringDecoder -import org.apache.griffin.measure.cache.info.InfoCacheInstance -import org.apache.griffin.measure.config.params.env._ -import org.apache.griffin.measure.config.params.user.{DataConnectorParam, EvaluateRuleParam} -import org.apache.griffin.measure.config.reader.ParamRawStringReader -import org.apache.griffin.measure.data.connector.batch.TextDirBatchDataConnector -import org.apache.griffin.measure.process.TimingProcess -import org.apache.griffin.measure.process.engine.DqEngines -import org.apache.griffin.measure.result.{DataInfo, TimeStampInfo} -import org.apache.griffin.measure.rule._ -import org.apache.griffin.measure.utils.{HdfsFileDumpUtil, HdfsUtil, TimeUtil} -import org.apache.spark.rdd.RDD -import org.apache.spark.sql.hive.HiveContext -import org.apache.spark.sql.types.{DataType, StructField, StructType} -import org.apache.spark.sql.{DataFrame, Row, SQLContext} -import org.apache.spark.streaming.dstream.InputDStream -import org.apache.spark.streaming.kafka.KafkaUtils -import org.apache.spark.streaming.{Milliseconds, StreamingContext} -import org.apache.spark.{SparkConf, SparkContext} -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -import org.scalatest.{BeforeAndAfter, FunSuite, Matchers} - -import scala.reflect.ClassTag -import scala.util.{Failure, Success, Try} - -@RunWith(classOf[JUnitRunner]) -class ConnectorTest extends FunSuite with Matchers with BeforeAndAfter { - - test("read config") { - - val a = "java.lang.String" - val at = getClassTag(a) - println(at) - - at match { - case ClassTag(m) => println(m) - case _ => println("no") - } - - } - - private def getClassTag(tp: String): ClassTag[_] = { - val clazz = Class.forName(tp) - ClassTag(clazz) - } - - test ("test sql") { - val concreteTableName = "default.demo_src" - val partitionsString = " dt=1, hr=2; dt=1 ; ;,hr=4; dt=2 ;;dt=5" - val partitions: Array[Array[String]] = partitionsString.split(";").flatMap { s => - val arr = s.trim.split(",").flatMap { t => - t.trim match { - case p if (p.nonEmpty) => Some(p) - case _ => None - } - } - if (arr.size > 0) Some(arr) else None - } - - val tableClause = s"SELECT * FROM ${concreteTableName}" - val validPartitions = partitions.filter(_.size > 0) - val ret = if (validPartitions.size > 0) { - val clauses = validPartitions.map { prtn => - val cls = prtn.mkString(" AND ") - s"${tableClause} WHERE ${cls}" - } - clauses.mkString(" UNION ALL ") - } else tableClause - - println(ret) - } -} - http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/scala/org/apache/griffin/measure/persist/HdfsPersistTest.scala ---------------------------------------------------------------------- diff --git a/measure/src/test/scala/org/apache/griffin/measure/persist/HdfsPersistTest.scala b/measure/src/test/scala/org/apache/griffin/measure/persist/HdfsPersistTest.scala deleted file mode 100644 index 2dfceb3..0000000 --- a/measure/src/test/scala/org/apache/griffin/measure/persist/HdfsPersistTest.scala +++ /dev/null @@ -1,48 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.persist - -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -import org.scalatest.{BeforeAndAfter, FunSuite, Matchers} - -import scala.util.{Try, Failure} - -@RunWith(classOf[JUnitRunner]) -class HdfsPersistTest extends FunSuite with Matchers with BeforeAndAfter { - - val config: Map[String, Any] = Map[String, Any]( - ("path" -> "/path/to"), ("max.persist.lines" -> 100), ("max.lines.per.file" -> 1000)) - val metricName: String = "metric" - val timeStamp: Long = 123456789L - - val hdfsPersist = HdfsPersist(config, metricName, timeStamp) - - test ("constructor") { - hdfsPersist.path should be ("/path/to") - hdfsPersist.maxPersistLines should be (100) - hdfsPersist.maxLinesPerFile should be (1000) - - hdfsPersist.StartFile should be (s"/path/to/${metricName}/${timeStamp}/_START") - } - - test ("avaiable") { - hdfsPersist.available should be (true) - } -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/scala/org/apache/griffin/measure/persist/HttpPersistTest.scala ---------------------------------------------------------------------- diff --git a/measure/src/test/scala/org/apache/griffin/measure/persist/HttpPersistTest.scala b/measure/src/test/scala/org/apache/griffin/measure/persist/HttpPersistTest.scala deleted file mode 100644 index 1b0bc6f..0000000 --- a/measure/src/test/scala/org/apache/griffin/measure/persist/HttpPersistTest.scala +++ /dev/null @@ -1,42 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.persist - -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -import org.scalatest.{BeforeAndAfter, FunSuite, Matchers} - -@RunWith(classOf[JUnitRunner]) -class HttpPersistTest extends FunSuite with Matchers with BeforeAndAfter { - - val config: Map[String, Any] = Map[String, Any](("api" -> "url/api"), ("method" -> "post")) - val metricName: String = "metric" - val timeStamp: Long = 123456789L - - val httpPersist = HttpPersist(config, metricName, timeStamp) - - test ("constructor") { - httpPersist.api should be ("url/api") - httpPersist.method should be ("post") - } - - test("available") { - httpPersist.available should be (true) - } -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/scala/org/apache/griffin/measure/persist/MongoPersistTest.scala ---------------------------------------------------------------------- diff --git a/measure/src/test/scala/org/apache/griffin/measure/persist/MongoPersistTest.scala b/measure/src/test/scala/org/apache/griffin/measure/persist/MongoPersistTest.scala deleted file mode 100644 index 1a0dedd..0000000 --- a/measure/src/test/scala/org/apache/griffin/measure/persist/MongoPersistTest.scala +++ /dev/null @@ -1,47 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.persist - -import org.junit.runner.RunWith -import org.mongodb.scala.{Completed, Document} -import org.mongodb.scala.model.{Filters, UpdateOptions, Updates} -import org.mongodb.scala.result.UpdateResult -import org.scalatest.junit.JUnitRunner -import org.scalatest.{BeforeAndAfter, FunSuite, Matchers} - -import scala.util.Success - -@RunWith(classOf[JUnitRunner]) -class MongoPersistTest extends FunSuite with Matchers with BeforeAndAfter { - - val config = Map[String, Any]( - ("url" -> "mongodb://111.111.111.111"), - ("database" -> "db"), - ("collection" -> "cl") - ) - val metricName: String = "metric" - val timeStamp: Long = 123456789L - - val mongoPersist = MongoPersist(config, metricName, timeStamp) - - test("available") { - mongoPersist.available should be (true) - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/scala/org/apache/griffin/measure/result/AccuracyResultTest.scala ---------------------------------------------------------------------- diff --git a/measure/src/test/scala/org/apache/griffin/measure/result/AccuracyResultTest.scala b/measure/src/test/scala/org/apache/griffin/measure/result/AccuracyResultTest.scala deleted file mode 100644 index 8a45530..0000000 --- a/measure/src/test/scala/org/apache/griffin/measure/result/AccuracyResultTest.scala +++ /dev/null @@ -1,57 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.result - -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -import org.scalatest.{BeforeAndAfter, FunSuite, Matchers} - -@RunWith(classOf[JUnitRunner]) -class AccuracyResultTest extends FunSuite with BeforeAndAfter with Matchers { - - test ("update") { - val result = AccuracyResult(10, 100) - val delta = AccuracyResult(3, 10) - result.update(delta) should be (AccuracyResult(3, 100)) - } - - test ("eventual") { - val result1 = AccuracyResult(10, 100) - result1.eventual should be (false) - - val result2 = AccuracyResult(0, 100) - result2.eventual should be (true) - } - - test ("differsFrom") { - val result = AccuracyResult(10, 100) - result.differsFrom(AccuracyResult(11, 100)) should be (true) - result.differsFrom(AccuracyResult(10, 110)) should be (true) - result.differsFrom(AccuracyResult(10, 100)) should be (false) - } - - test ("matchPercentage") { - val result1 = AccuracyResult(10, 100) - result1.matchPercentage should be (90.0) - - val result2 = AccuracyResult(10, 0) - result2.matchPercentage should be (0.0) - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/scala/org/apache/griffin/measure/result/ProfileResultTest.scala ---------------------------------------------------------------------- diff --git a/measure/src/test/scala/org/apache/griffin/measure/result/ProfileResultTest.scala b/measure/src/test/scala/org/apache/griffin/measure/result/ProfileResultTest.scala deleted file mode 100644 index 2c4fc5f..0000000 --- a/measure/src/test/scala/org/apache/griffin/measure/result/ProfileResultTest.scala +++ /dev/null @@ -1,57 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.result - -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -import org.scalatest.{BeforeAndAfter, FunSuite, Matchers} - -@RunWith(classOf[JUnitRunner]) -class ProfileResultTest extends FunSuite with BeforeAndAfter with Matchers { - - test ("update") { - val result = ProfileResult(10, 100) - val delta = ProfileResult(30, 90) - result.update(delta) should be (ProfileResult(40, 100)) - } - - test ("eventual") { - val result1 = ProfileResult(10, 100) - result1.eventual should be (false) - - val result2 = ProfileResult(100, 100) - result2.eventual should be (true) - } - - test ("differsFrom") { - val result = ProfileResult(10, 100) - result.differsFrom(ProfileResult(11, 100)) should be (true) - result.differsFrom(ProfileResult(10, 110)) should be (true) - result.differsFrom(ProfileResult(10, 100)) should be (false) - } - - test ("matchPercentage") { - val result1 = ProfileResult(90, 100) - result1.matchPercentage should be (90.0) - - val result2 = ProfileResult(10, 0) - result2.matchPercentage should be (0.0) - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/scala/org/apache/griffin/measure/rule/adaptor/GriffinDslAdaptorTest.scala ---------------------------------------------------------------------- diff --git a/measure/src/test/scala/org/apache/griffin/measure/rule/adaptor/GriffinDslAdaptorTest.scala b/measure/src/test/scala/org/apache/griffin/measure/rule/adaptor/GriffinDslAdaptorTest.scala deleted file mode 100644 index d551a4f..0000000 --- a/measure/src/test/scala/org/apache/griffin/measure/rule/adaptor/GriffinDslAdaptorTest.scala +++ /dev/null @@ -1,178 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.adaptor - -import org.apache.griffin.measure.process._ -import org.apache.griffin.measure.process.temp.{TableRegisters, _} -import org.apache.griffin.measure.rule.plan.CalcTimeInfo -import org.apache.griffin.measure.utils.JsonUtil -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -import org.scalatest.{BeforeAndAfter, FunSuite, Matchers} -import org.scalamock.scalatest.MockFactory - -@RunWith(classOf[JUnitRunner]) -class GriffinDslAdaptorTest extends FunSuite with Matchers with BeforeAndAfter with MockFactory { - - test ("profiling groupby") { -// val adaptor = GriffinDslAdaptor("source" :: "target" :: Nil, "count" :: Nil) -// -// val ruleJson = -// """ -// |{ -// | "dsl.type": "griffin-dsl", -// | "dq.type": "accuracy", -// | "name": "accu", -// | "rule": "source.user_id = target.user_id", -// | "details": { -// | "source": "source", -// | "target": "target", -// | "miss": "miss_count", -// | "total": "total_count", -// | "matched": "matched_count" -// | }, -// | "metric": { -// | "name": "accu" -// | }, -// | "record": { -// | "name": "missRecords" -// | } -// |} -// """.stripMargin -// -// // rule: Map[String, Any] -// val rule: Map[String, Any] = JsonUtil.toAnyMap(ruleJson) -// println(rule) -// -//// val dataCheckerMock = mock[DataChecker] -//// dataCheckerMock.existDataSourceName _ expects ("source") returning (true) -//// RuleAdaptorGroup.dataChecker = dataCheckerMock -// -// val dsTmsts = Map[String, Set[Long]](("source" -> Set[Long](1234))) -// -// val timeInfo = CalcTimeInfo(123) -// TableRegisters.registerCompileTempTable(timeInfo.key, "source") -// -// val rp = adaptor.genRulePlan(timeInfo, rule, StreamingProcessType) -// rp.ruleSteps.foreach(println) -// rp.ruleExports.foreach(println) - } - - test ("accuracy") { -// val adaptor = GriffinDslAdaptor("source" :: "target" :: Nil, "count" :: Nil, StreamingProcessType, RunPhase) -// -// val ruleJson = -// """ -// |{ -// | "dsl.type": "griffin-dsl", -// | "dq.type": "accuracy", -// | "name": "accu", -// | "rule": "source.id = target.id and source.name = target.name", -// | "details": { -// | "source": "source", -// | "target": "target", -// | "persist.type": "metric" -// | } -// |} -// """.stripMargin -// -// // rule: Map[String, Any] -// val rule: Map[String, Any] = JsonUtil.toAnyMap(ruleJson) -// println(rule) -// -// val dataCheckerMock = mock[DataChecker] -// dataCheckerMock.existDataSourceName _ expects ("source") returns (true) -// dataCheckerMock.existDataSourceName _ expects ("target") returns (true) -// RuleAdaptorGroup.dataChecker = dataCheckerMock -// -// val dsTmsts = Map[String, Set[Long]](("source" -> Set[Long](1234)), ("target" -> Set[Long](1234))) -// val steps = adaptor.genConcreteRuleStep(TimeInfo(0, 0), rule, dsTmsts) -// -// steps.foreach { step => -// println(s"${step}, ${step.ruleInfo.persistType}") -// } - } - - test ("uniqueness") { -// val adaptor = GriffinDslAdaptor("new" :: "old" :: Nil, "count" :: Nil) -// val ruleJson = -// """ -// |{ -// | "dsl.type": "griffin-dsl", -// | "dq.type": "uniqueness", -// | "name": "dup", -// | "rule": "name, count(age + 1) as ct", -// | "details": { -// | "count": "cnt" -// | }, -// | "metric": { -// | "name": "dup" -// | } -// |} -// """.stripMargin -// val rule: Map[String, Any] = JsonUtil.toAnyMap(ruleJson) -// println(rule) -// -// val timeInfo = CalcTimeInfo(123) -// TableRegisters.registerCompileTempTable(timeInfo.key, "new") -// TableRegisters.registerCompileTempTable(timeInfo.key, "old") -// -// val rp = adaptor.genRulePlan(timeInfo, rule, StreamingProcessType) -// rp.ruleSteps.foreach(println) -// rp.ruleExports.foreach(println) -// -// TableRegisters.unregisterCompileTempTables(timeInfo.key) - } - - test ("timeliness") { -// val adaptor = GriffinDslAdaptor("source" :: Nil, "length" :: Nil) -// val ruleJson = -// """ -// |{ -// | "dsl.type": "griffin-dsl", -// | "dq.type": "timeliness", -// | "name": "timeliness", -// | "rule": "ts", -// | "details": { -// | "source": "source", -// | "latency": "latency", -// | "threshold": "1h" -// | }, -// | "metric": { -// | "name": "timeliness" -// | }, -// | "record": { -// | "name": "lateRecords" -// | } -// |} -// """.stripMargin -// val rule: Map[String, Any] = JsonUtil.toAnyMap(ruleJson) -// println(rule) -// -// val timeInfo = CalcTimeInfo(123) -// TableRegisters.registerCompileTempTable(timeInfo.key, "source") -// -// val rp = adaptor.genRulePlan(timeInfo, rule, StreamingProcessType) -// rp.ruleSteps.foreach(println) -// rp.ruleExports.foreach(println) -// -// TableRegisters.unregisterCompileTempTables(timeInfo.key) - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/scala/org/apache/griffin/measure/rule/adaptor/RuleAdaptorGroupTest.scala ---------------------------------------------------------------------- diff --git a/measure/src/test/scala/org/apache/griffin/measure/rule/adaptor/RuleAdaptorGroupTest.scala b/measure/src/test/scala/org/apache/griffin/measure/rule/adaptor/RuleAdaptorGroupTest.scala deleted file mode 100644 index 23b26d1..0000000 --- a/measure/src/test/scala/org/apache/griffin/measure/rule/adaptor/RuleAdaptorGroupTest.scala +++ /dev/null @@ -1,70 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.adaptor - -import org.apache.griffin.measure.config.params.Param -import org.apache.griffin.measure.config.params.user.UserParam -import org.apache.griffin.measure.config.reader.ParamReaderFactory -import org.apache.griffin.measure.process._ -import org.apache.griffin.measure.process.temp._ -import org.apache.griffin.measure.rule.plan.CalcTimeInfo -import org.apache.griffin.measure.utils.JsonUtil -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -import org.scalatest.{BeforeAndAfter, FunSuite, Matchers} -import org.scalamock.scalatest.MockFactory - -import scala.util.{Failure, Success, Try} - -@RunWith(classOf[JUnitRunner]) -class RuleAdaptorGroupTest extends FunSuite with Matchers with BeforeAndAfter with MockFactory { - - test ("profiling groupby") { - RuleAdaptorGroup.init( - "source" :: "target" :: Nil, - "source", - "coalesce" :: "count" :: "upper" :: Nil - ) - val timeInfo = CalcTimeInfo(123) - TableRegisters.registerCompileTempTable(timeInfo.key, "source") - TableRegisters.registerCompileTempTable(timeInfo.key, "target") - - val confFile = "src/test/resources/config-test-accuracy-new.json" - - val userParam = readParamFile[UserParam](confFile, "local") match { - case Success(p) => p - case Failure(ex) => fail - } - - val dsTmsts = Map[String, Set[Long]](("source" -> Set[Long](111, 222, 333))) - -// val steps = RuleAdaptorGroup.genRuleSteps( -// TmstTimeInfo(123, 321), -// userParam.evaluateRuleParam, -// dsTmsts -// ) -// steps.foreach(println) - } - - private def readParamFile[T <: Param](file: String, fsType: String)(implicit m : Manifest[T]): Try[T] = { - val paramReader = ParamReaderFactory.getParamReader(file, fsType) - paramReader.readConfig[T] - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/scala/org/apache/griffin/measure/rule/adaptor/SparkSqlAdaptorTest.scala ---------------------------------------------------------------------- diff --git a/measure/src/test/scala/org/apache/griffin/measure/rule/adaptor/SparkSqlAdaptorTest.scala b/measure/src/test/scala/org/apache/griffin/measure/rule/adaptor/SparkSqlAdaptorTest.scala deleted file mode 100644 index 42c4f59..0000000 --- a/measure/src/test/scala/org/apache/griffin/measure/rule/adaptor/SparkSqlAdaptorTest.scala +++ /dev/null @@ -1,59 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.adaptor - -import org.apache.griffin.measure.rule.plan.TimeInfo -import org.apache.griffin.measure.utils.JsonUtil -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -import org.scalatest.{BeforeAndAfter, FunSuite, Matchers} -import org.scalamock.scalatest.MockFactory - -@RunWith(classOf[JUnitRunner]) -class SparkSqlAdaptorTest extends FunSuite with Matchers with BeforeAndAfter with MockFactory { - - test ("spark sql adaptor test") { -// val adaptor = SparkSqlAdaptor() -// -// val ruleJson = -// """ -// |{ -// | "dsl.type": "spark-sql", -// | "name": "out", -// | "rule": "count(*)", -// | "details": { -// | "persist.type": "metric", -// | "collect.type": "array" -// | } -// |} -// """.stripMargin -// -// // rule: Map[String, Any] -// val rule: Map[String, Any] = JsonUtil.toAnyMap(ruleJson) -// println(rule) -// -// val dsTmsts = Map[String, Set[Long]](("source" -> Set[Long](1234))) -// val steps = adaptor.genConcreteRuleStep(TimeInfo(1, 2), rule) -// -// steps.foreach { step => -// println(s"${step}") -// } - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/scala/org/apache/griffin/measure/rule/dsl/parser/BasicParserTest.scala ---------------------------------------------------------------------- diff --git a/measure/src/test/scala/org/apache/griffin/measure/rule/dsl/parser/BasicParserTest.scala b/measure/src/test/scala/org/apache/griffin/measure/rule/dsl/parser/BasicParserTest.scala deleted file mode 100644 index 75cda73..0000000 --- a/measure/src/test/scala/org/apache/griffin/measure/rule/dsl/parser/BasicParserTest.scala +++ /dev/null @@ -1,234 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.dsl.parser - -import org.apache.griffin.measure.rule.dsl.ProfilingType -import org.apache.griffin.measure.rule.dsl.expr._ -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -import org.scalatest.{BeforeAndAfter, FunSuite, Matchers} -//import org.scalatest.FlatSpec -//import org.scalamock.scalatest.MockFactory - -@RunWith(classOf[JUnitRunner]) -class BasicParserTest extends FunSuite with Matchers with BeforeAndAfter { - - val parser = new BasicParser{ - val dataSourceNames: Seq[String] = "source" :: "target" :: Nil - val functionNames: Seq[String] = "func" :: "get_json_object" :: Nil - def rootExpression: Parser[Expr] = expression - } - - test("test literal") { - val rule1 = """null""" - val result1 = parser.parseAll(parser.literal, rule1) - result1.successful should be (true) - result1.get.desc should be ("NULL") - - val rule2 = """nan""" - val result2 = parser.parseAll(parser.literal, rule2) - result2.successful should be (true) - result2.get.desc should be ("NaN") - - val rule3 = """'test\'ing'""" - val result3 = parser.parseAll(parser.literal, rule3) - result3.successful should be (true) - result3.get.desc should be ("'test\\'ing'") - - val rule4 = """"test\" again"""" - val result4 = parser.parseAll(parser.literal, rule4) - result4.successful should be (true) - result4.get.desc should be ("\"test\\\" again\"") - - val rule5 = """-1.342""" - val result5 = parser.parseAll(parser.literal, rule5) - result5.successful should be (true) - result5.get.desc should be ("-1.342") - - val rule51 = """33""" - val result51 = parser.parseAll(parser.literal, rule51) - result51.successful should be (true) - result51.get.desc should be ("33") - - val rule6 = """2h""" - val result6 = parser.parseAll(parser.literal, rule6) - result6.successful should be (true) - result6.get.desc should be (s"${2 * 3600 * 1000}") - - val rule7 = """true""" - val result7 = parser.parseAll(parser.literal, rule7) - result7.successful should be (true) - result7.get.desc should be ("true") - } - - test ("test selection") { - val rule1 = """source""" - val result1 = parser.parseAll(parser.selection, rule1) - result1.successful should be (true) - result1.get.desc should be ("`source`") - - val rule2 = """source_not_registered""" - val result2 = parser.parseAll(parser.selection, rule2) - result2.successful should be (false) - - val rule3 = """source[12].age""" - val result3 = parser.parseAll(parser.selection, rule3) - result3.successful should be (true) - result3.get.desc should be ("`source`[12].`age`") - result3.get.alias should be (Some("12_age")) - - val rule4 = """source.name.func(target.name)""" - val result4 = parser.parseAll(parser.selection, rule4) - result4.successful should be (true) - result4.get.desc should be ("func(`source`.`name`, `target`.`name`)") - } - - test ("test math") { - val rule1 = """-1""" - val result1 = parser.parseAll(parser.mathExpression, rule1) - result1.successful should be (true) - result1.get.desc should be ("(-1)") - - val rule2 = "1 + 1" - val result2 = parser.parseAll(parser.mathExpression, rule2) - result2.successful should be (true) - result2.get.desc should be ("1 + 1") - - val rule3 = "source.age + 2 * 5 + target.offset" - val result3 = parser.parseAll(parser.mathExpression, rule3) - result3.successful should be (true) - result3.get.desc should be ("`source`.`age` + 2 * 5 + `target`.`offset`") - - val rule4 = "(source.age + 2) * (5 + target.offset)" - val result4 = parser.parseAll(parser.mathExpression, rule4) - result4.successful should be (true) - result4.get.desc should be ("(`source`.`age` + 2) * (5 + `target`.`offset`)") - } - - test ("test logical") { - val rule1 = "source.age in (12 + 3, 23, 34)" - val result1 = parser.parseAll(parser.logicalExpression, rule1) - result1.successful should be (true) - result1.get.desc should be ("`source`.`age` IN (12 + 3, 23, 34)") - - val rule2 = "source.age between (12 + 3, 23, 34)" - val result2 = parser.parseAll(parser.logicalExpression, rule2) - result2.successful should be (true) - result2.get.desc should be ("`source`.`age` BETWEEN 12 + 3 AND 23") - - val rule3 = "source.age between (12 + 3)" - assertThrows[Exception](parser.parseAll(parser.logicalExpression, rule3)) - - val rule4 = "source.name like '%tk'" - val result4 = parser.parseAll(parser.logicalExpression, rule4) - result4.successful should be (true) - result4.get.desc should be ("`source`.`name` LIKE '%tk'") - - val rule5 = "source.desc is not null" - val result5 = parser.parseAll(parser.logicalExpression, rule5) - result5.successful should be (true) - result5.get.desc should be ("`source`.`desc` IS NOT NULL") - - val rule6 = "source.desc is not nan" - val result6 = parser.parseAll(parser.logicalExpression, rule6) - result6.successful should be (true) - result6.get.desc should be ("NOT isnan(`source`.`desc`)") - - val rule7 = "!source.ok and source.name = target.name && (source.age between 12 and 52) && target.desc is not null" - val result7 = parser.parseAll(parser.logicalExpression, rule7) - result7.successful should be (true) - result7.get.desc should be ("(NOT `source`.`ok`) AND `source`.`name` = `target`.`name` AND (`source`.`age` BETWEEN 12 AND 52) AND `target`.`desc` IS NOT NULL") - - val rule8 = "!(10 != 30 and !(31 > 2) or (45 <= 8 and 33 <> 0))" - val result8 = parser.parseAll(parser.logicalExpression, rule8) - result8.successful should be (true) - result8.get.desc should be ("(NOT (10 != 30 AND (NOT (31 > 2)) OR (45 <= 8 AND 33 <> 0)))") - - } - - test ("test expression") { - val rule3 = "source.age + 2 * 5 + target.offset" - val result3 = parser.parseAll(parser.expression, rule3) - println(result3) - result3.successful should be (true) - result3.get.desc should be ("`source`.`age` + 2 * 5 + `target`.`offset`") - - val rule4 = "(source.age + 2) * (5 + target.offset)" - val result4 = parser.parseAll(parser.expression, rule4) - println(result4) - result4.successful should be (true) - result4.get.desc should be ("(`source`.`age` + 2) * (5 + `target`.`offset`)") - - val rule7 = "!source.ok and source.name = target.name && (source.age between 12 and 52) && target.desc is not null" - val result7 = parser.parseAll(parser.expression, rule7) - result7.successful should be (true) - result7.get.desc should be ("(NOT `source`.`ok`) AND `source`.`name` = `target`.`name` AND (`source`.`age` BETWEEN 12 AND 52) AND `target`.`desc` IS NOT NULL") - - val rule8 = "!(10 != 30 and !(31 > 2) or (45 <= 8 and 33 <> 0))" - val result8 = parser.parseAll(parser.expression, rule8) - result8.successful should be (true) - result8.get.desc should be ("(NOT (10 != 30 AND (NOT (31 > 2)) OR (45 <= 8 AND 33 <> 0)))") - - val rule1 = "source.user_id = target.user_id AND source.first_name = target.first_name AND source.last_name = target.last_name AND source.address = target.address AND source.email = target.email AND source.phone = target.phone AND source.post_code = target.post_code" - val result1 = parser.parseAll(parser.expression, rule1) - result1.successful should be (true) - result1.get.desc should be ("`source`.`user_id` = `target`.`user_id` AND `source`.`first_name` = `target`.`first_name` AND `source`.`last_name` = `target`.`last_name` AND `source`.`address` = `target`.`address` AND `source`.`email` = `target`.`email` AND `source`.`phone` = `target`.`phone` AND `source`.`post_code` = `target`.`post_code`") - } - - test ("test function") { - val rule3 = "source.age + 2 * 5 + target.offset * func('a', source.name)" - val result3 = parser.parseAll(parser.expression, rule3) - result3.successful should be (true) - result3.get.desc should be ("`source`.`age` + 2 * 5 + `target`.`offset` * func('a', `source`.`name`)") - } - - test ("order by clause") { - val rule = "order by source.user_id, item" - val result = parser.parseAll(parser.orderbyClause, rule) - result.successful should be (true) - println(result.get.desc) - } - - test ("select clause") { - val rule = "select `source`.user_id, item, `source`.age.func()" - val result = parser.parseAll(parser.selectClause, rule) - println(result) - result.successful should be (true) - println(result.get.desc) - } - - test ("from clause") { - val rule = "from `source`" - val result = parser.parseAll(parser.fromClause, rule) - result.successful should be (true) - println(result.get.desc) - } - - test ("profiling") { - val griffinParser = GriffinDslParser( - "target" :: Nil, - "max" :: "min" :: Nil - ) - val rule = "source.name, source.name.max(), source.name.min() group by source.name" - val result = griffinParser.parseRule(rule, ProfilingType) - result.successful should be (true) - println(result.get) - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/scala/org/apache/griffin/measure/rule/udf/GriffinUdfsTest.scala ---------------------------------------------------------------------- diff --git a/measure/src/test/scala/org/apache/griffin/measure/rule/udf/GriffinUdfsTest.scala b/measure/src/test/scala/org/apache/griffin/measure/rule/udf/GriffinUdfsTest.scala deleted file mode 100644 index af70bd8..0000000 --- a/measure/src/test/scala/org/apache/griffin/measure/rule/udf/GriffinUdfsTest.scala +++ /dev/null @@ -1,52 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.rule.udf - - -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -import org.scalatest.{BeforeAndAfter, FunSuite, Matchers} -//import org.scalatest.FlatSpec -import org.scalatest.PrivateMethodTester -//import org.scalamock.scalatest.MockFactory - -@RunWith(classOf[JUnitRunner]) -class GriffinUdfsTest extends FunSuite with Matchers with BeforeAndAfter with PrivateMethodTester { - - test ("test indexOf") { - val inv = new Invocation[Int]('indexOf, "a" :: "b" :: "c" :: Nil, "b") - GriffinUdfs.invokePrivate(inv) should be (1) - } - - test ("test matches") { - val inv = new Invocation[Boolean]('matches, "s123", "^s\\d+$") - GriffinUdfs.invokePrivate(inv) should be (true) - } - - test ("test regexSubstr") { - val str = "https://www.abc.com/test/gp/product/B023/ref=sr_1_1/123-456?id=123"; -// val regexStr = """^([^/]+://[^/]+)(?:/[^/]+)?(/dp/[A-Z0-9]+)(?:/.*)?$""" - val regexStr = """^([^/]+://[^/]+)(?:/[^/]+)?(?:/[dg]p(?:/product)?/)([A-Z0-9]+)(?:/.*)?$""" - val replacement = "$1/dp/$2" - val inv = new Invocation[String]('regReplace, str, regexStr, replacement) - println(GriffinUdfs.invokePrivate(inv)) -// GriffinUdfs.invokePrivate(inv) should be ("https://www.abc.com/dp/B023";) - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/scala/org/apache/griffin/measure/rule/udf/MeanUdafTest.scala ---------------------------------------------------------------------- diff --git a/measure/src/test/scala/org/apache/griffin/measure/rule/udf/MeanUdafTest.scala b/measure/src/test/scala/org/apache/griffin/measure/rule/udf/MeanUdafTest.scala deleted file mode 100644 index 1ec75e4..0000000 --- a/measure/src/test/scala/org/apache/griffin/measure/rule/udf/MeanUdafTest.scala +++ /dev/null @@ -1,117 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ - -package org.apache.griffin.measure.rule.udf - - -import org.apache.spark.sql.SQLContext -import org.apache.spark.{SparkConf, SparkContext} -import org.scalatest.FunSuite - - -class MeanUdafTest extends FunSuite { - - val sparkConf = new SparkConf() - sparkConf.setMaster("local") - //sparkConf.setSparkHome(System.getenv("SPARK_HOME")) - sparkConf.setAppName("UDFTest") - - val sparkContext = new SparkContext(sparkConf) - - implicit val sqlContext = new SQLContext(sparkContext) - - import sqlContext.implicits._ - - - test("test the udf"){ - - sqlContext.udf.register("my_mean", new MeanUdaf) - - // the udf will not work with nulls. - val data1 = Seq[(Integer, Integer)]( - (1, 10), - (null, -60), - (1, 20), - (1, 30), - (2, 0), - (null, -10), - (2, -1), - (2, null), - (2, null), - (null, 100), - (3, null), - (null, null), - (3, null)).toDF("key", "value") - data1.registerTempTable("agg1") - - val data2 = Seq[(Integer, Integer)]( - (1, 10), - (1, -60), - (1, 20), - (1, 30), - (2, 0), - (2, -10), - (2, -1), - (2, -5), - (2, 5), - (3, 100), - (3, -10), - (3, 10), - (3, 10)).toDF("key", "value") - data2.registerTempTable("agg2") - - - val nullFreeDf = sqlContext.sql( - - """ - |SELECT - | key, - | my_mean(value) as udfVal, - | avg(value) as defaultVal, - | count(*) as totalCount, - | sum(value) as totalSum - |FROM agg2 - |GROUP BY key - """.stripMargin).toDF() - - - nullFreeDf.registerTempTable("agg0") - nullFreeDf.show() - - val result = sqlContext.sql( - - """ - |SELECT - | SUM(IF(udfVal=defaultVal, 0, 1)) equalCols, - | SUM(IF(udfVal=totalSum/totalCount, 0, 1)) avgCols - |FROM agg0 - """.stripMargin).toDF() - - result.show() - - assert(result.count() === 1) - - assert(result.head().getLong(0) === 0L) - assert(result.head().getLong(1) === 0L) - - } - - - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/scala/org/apache/griffin/measure/utils/HdfsUtilTest.scala ---------------------------------------------------------------------- diff --git a/measure/src/test/scala/org/apache/griffin/measure/utils/HdfsUtilTest.scala b/measure/src/test/scala/org/apache/griffin/measure/utils/HdfsUtilTest.scala deleted file mode 100644 index 5b94901..0000000 --- a/measure/src/test/scala/org/apache/griffin/measure/utils/HdfsUtilTest.scala +++ /dev/null @@ -1,132 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.utils - -import java.io.{BufferedReader, FileReader, InputStreamReader} - -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.{FileSystem, Path} -import org.apache.spark.sql.SQLContext -import org.apache.spark.sql.hive.HiveContext -import org.apache.spark.{SparkConf, SparkContext} -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -import org.scalatest.{BeforeAndAfter, FunSuite, Matchers} - -//@RunWith(classOf[JUnitRunner]) -class HdfsUtilTest extends FunSuite with Matchers with BeforeAndAfter { - - private val seprator = "/" - - private val conf1 = new Configuration() - conf1.addResource(new Path("file:///apache/hadoop/etc/hadoop/core-site.xml")) - conf1.addResource(new Path("file:///apache/hadoop/etc/hadoop/hdfs-site.xml")) - private val dfs1 = FileSystem.get(conf1) - - private val conf2 = new Configuration() - conf2.addResource(new Path("file:///Users/lliu13/test/hadoop/core-site.xml")) - conf2.addResource(new Path("file:///Users/lliu13/test/hadoop/hdfs-site.xml")) - private val dfs2 = FileSystem.get(conf2) - -// val conf = new SparkConf().setAppName("test_hdfs").setMaster("local[*]") -// val sparkContext = new SparkContext(conf) -// sparkContext.setLogLevel("WARN") -// val sqlContext = new HiveContext(sparkContext) - - def listSubPaths(dfs: FileSystem, dirPath: String, subType: String, fullPath: Boolean = false): Iterable[String] = { - val path = new Path(dirPath) - try { - val fileStatusArray = dfs.listStatus(path) - fileStatusArray.filter { fileStatus => - subType match { - case "dir" => fileStatus.isDirectory - case "file" => fileStatus.isFile - case _ => true - } - }.map { fileStatus => - val fname = fileStatus.getPath.getName - if (fullPath) getHdfsFilePath(dirPath, fname) else fname - } - } catch { - case e: Throwable => { - println(s"list path files error: ${e.getMessage}") - Nil - } - } - } - - def getHdfsFilePath(parentPath: String, fileName: String): String = { - if (parentPath.endsWith(seprator)) parentPath + fileName else parentPath + seprator + fileName - } - -// test ("test multiple hdfs") { -// val list1 = listSubPaths(dfs1, "/", "dir", false) -// println(list1) -// -// val list2 = listSubPaths(dfs2, "/", "dir", false) -// println(list2) -// -// val path1 = "/depth/discovery_file_sample.txt" -// val istream1 = dfs1.open(new Path(path1)) -// val reader1 = new BufferedReader(new InputStreamReader(istream1)) -// val seq1 = scala.collection.mutable.MutableList[String]() -// try { -// var line = reader1.readLine() -// while (line != null) { -// val arr = line.split("\u0007") -// seq1 ++= arr -// line = reader1.readLine() -// } -// } finally { -// reader1.close() -// istream1.close() -// } -// -//// val scanner = new java.util.Scanner(istream1,"UTF-8").useDelimiter("\u0007") -//// val theString = if (scanner.hasNext()) scanner.next() else "" -//// println(theString) -//// scanner.close() -// -// println(seq1.size) -// println(seq1.take(10)) -// seq1.take(10).foreach(println) -// -//// val path2 = "/griffin/json/env.json" -//// val istream2 = dfs2.open(new Path(path2)) -//// val reader2 = new BufferedReader(new InputStreamReader(istream2)) -//// val seq2 = scala.collection.mutable.MutableList[String]() -//// try { -//// var line = reader2.readLine() -//// while (line != null) { -//// line = reader2.readLine() -//// seq2 += line -//// } -//// } catch { -//// case e: Throwable => { -//// println("error in reading") -//// } -//// } finally { -//// reader2.close() -//// istream2.close() -//// } -//// println(seq2.size) -//// println(seq2.take(10)) -// } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/scala/org/apache/griffin/measure/utils/ParamUtilTest.scala ---------------------------------------------------------------------- diff --git a/measure/src/test/scala/org/apache/griffin/measure/utils/ParamUtilTest.scala b/measure/src/test/scala/org/apache/griffin/measure/utils/ParamUtilTest.scala deleted file mode 100644 index 5a54b11..0000000 --- a/measure/src/test/scala/org/apache/griffin/measure/utils/ParamUtilTest.scala +++ /dev/null @@ -1,50 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.utils - -import java.io.{BufferedReader, InputStreamReader} - -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.{FileSystem, Path} -import org.apache.spark.sql.hive.HiveContext -import org.apache.spark.{SparkConf, SparkContext} -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -import org.scalatest.{BeforeAndAfter, FunSuite, Matchers} - -@RunWith(classOf[JUnitRunner]) -class ParamUtilTest extends FunSuite with Matchers with BeforeAndAfter { - - test ("test param util") { - val params = Map[String, Any]( - ("a" -> "321"), - ("b" -> 123), - ("c" -> 3.2), - ("d" -> (213 :: 321 :: Nil)) - ) - - import ParamUtil._ - - params.getString("a", "") should be ("321") - params.getInt("b", 0) should be (123) - params.getBoolean("c", false) should be (false) - params.getAnyRef("d", List[Int]()) should be ((213 :: 321 :: Nil)) - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/scala/org/apache/griffin/measure/utils/TimeUtilTest.scala ---------------------------------------------------------------------- diff --git a/measure/src/test/scala/org/apache/griffin/measure/utils/TimeUtilTest.scala b/measure/src/test/scala/org/apache/griffin/measure/utils/TimeUtilTest.scala deleted file mode 100644 index 673eca0..0000000 --- a/measure/src/test/scala/org/apache/griffin/measure/utils/TimeUtilTest.scala +++ /dev/null @@ -1,38 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.griffin.measure.utils - -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -import org.scalatest.{BeforeAndAfter, FunSuite, Matchers} - -@RunWith(classOf[JUnitRunner]) -class TimeUtilTest extends FunSuite with Matchers with BeforeAndAfter { - - test ("milliseconds") { - val ts = "1h" - val res = TimeUtil.milliseconds(ts) - println(res) - - val t = 1200000 - val s = TimeUtil.time2String(t) - println(s) - } - -}
