hive prepared
Project: http://git-wip-us.apache.org/repos/asf/incubator-griffin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-griffin/commit/8a49d889 Tree: http://git-wip-us.apache.org/repos/asf/incubator-griffin/tree/8a49d889 Diff: http://git-wip-us.apache.org/repos/asf/incubator-griffin/diff/8a49d889 Branch: refs/heads/griffin-0.2.0-incubating-rc4 Commit: 8a49d8896f28fec3a21adb083705866a86d49ada Parents: c24fd4a Author: Lionel Liu <[email protected]> Authored: Tue Apr 17 16:05:37 2018 +0800 Committer: Lionel Liu <[email protected]> Committed: Tue Apr 17 16:05:37 2018 +0800 ---------------------------------------------------------------------- .../batch/HiveBatchDataConnector.scala | 6 +-- .../_profiling-batch-griffindsl-hive.json | 48 ++++++++++++++++++++ measure/src/test/resources/env-test.json | 3 +- 3 files changed, 53 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/8a49d889/measure/src/main/scala/org/apache/griffin/measure/data/connector/batch/HiveBatchDataConnector.scala ---------------------------------------------------------------------- diff --git a/measure/src/main/scala/org/apache/griffin/measure/data/connector/batch/HiveBatchDataConnector.scala b/measure/src/main/scala/org/apache/griffin/measure/data/connector/batch/HiveBatchDataConnector.scala index 2c9747e..6eb5391 100644 --- a/measure/src/main/scala/org/apache/griffin/measure/data/connector/batch/HiveBatchDataConnector.scala +++ b/measure/src/main/scala/org/apache/griffin/measure/data/connector/batch/HiveBatchDataConnector.scala @@ -36,9 +36,9 @@ case class HiveBatchDataConnector(sqlContext: SQLContext, dqEngines: DqEngines, val config = dcParam.config - if (!sqlContext.isInstanceOf[HiveContext]) { - throw new Exception("hive context not prepared!") - } +// if (!sqlContext.isInstanceOf[HiveContext]) { +// throw new Exception("hive context not prepared!") +// } val Database = "database" val TableName = "table.name" http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/8a49d889/measure/src/test/resources/_profiling-batch-griffindsl-hive.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/_profiling-batch-griffindsl-hive.json b/measure/src/test/resources/_profiling-batch-griffindsl-hive.json new file mode 100644 index 0000000..03b0405 --- /dev/null +++ b/measure/src/test/resources/_profiling-batch-griffindsl-hive.json @@ -0,0 +1,48 @@ +{ + "name": "prof_batch", + + "process.type": "batch", + + "timestamp": 123456, + + "data.sources": [ + { + "name": "source", + "connectors": [ + { + "type": "hive", + "version": "1.2", + "config": { + "database": "default", + "table.name": "s1" + } + } + ] + } + ], + + "evaluate.rule": { + "rules": [ + { + "dsl.type": "griffin-dsl", + "dq.type": "profiling", + "name": "prof", + "rule": "name, count(*) as cnt from source group by name", + "metric": { + "name": "name_group", + "collect.type": "array" + } + }, + { + "dsl.type": "griffin-dsl", + "dq.type": "profiling", + "name": "grp", + "rule": "age, count(*) as cnt from source group by age order by cnt", + "metric": { + "name": "age_group", + "collect.type": "array" + } + } + ] + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/8a49d889/measure/src/test/resources/env-test.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/env-test.json b/measure/src/test/resources/env-test.json index 4a8e3d0..5a0c573 100644 --- a/measure/src/test/resources/env-test.json +++ b/measure/src/test/resources/env-test.json @@ -5,7 +5,8 @@ "batch.interval": "10s", "process.interval": "10m", "config": { - "spark.master": "local[*]" + "spark.master": "local[*]", + "hive.metastore.uris": "thrift://localhost:9083" }, "init.clear": true },
