Merge branch 'master' of https://github.com/apache/incubator-griffin into apache-master
Project: http://git-wip-us.apache.org/repos/asf/incubator-griffin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-griffin/commit/dc7c3cca Tree: http://git-wip-us.apache.org/repos/asf/incubator-griffin/tree/dc7c3cca Diff: http://git-wip-us.apache.org/repos/asf/incubator-griffin/diff/dc7c3cca Branch: refs/heads/griffin-0.2.0-incubating-rc1 Commit: dc7c3cca6b7a0d00f840abcc24a26536b3d354a0 Parents: d8b371d 9c586ee Author: Lionel Liu <[email protected]> Authored: Mon Jan 8 17:45:07 2018 +0800 Committer: Lionel Liu <[email protected]> Committed: Mon Jan 8 17:45:07 2018 +0800 ---------------------------------------------------------------------- .gitignore | 2 +- .../docker/svc_msr/docker-compose-streaming.yml | 1 + measure/pom.xml | 8 + .../apache/griffin/measure/Application.scala | 9 +- .../measure/cache/info/TimeInfoCache.scala | 2 +- .../cache/result/CacheResultProcesser.scala | 2 +- .../griffin/measure/cache/tmst/TempName.scala | 47 + .../griffin/measure/cache/tmst/TmstCache.scala | 46 + .../config/params/user/DataSourceParam.scala | 5 +- .../measure/config/params/user/UserParam.scala | 35 +- .../config/validator/AllParamValidator.scala | 34 - .../config/validator/ParamValidator.scala | 8 +- .../measure/data/connector/DataConnector.scala | 77 +- .../batch/AvroBatchDataConnector.scala | 5 +- .../batch/HiveBatchDataConnector.scala | 5 +- .../batch/TextDirBatchDataConnector.scala | 5 +- .../streaming/StreamingDataConnector.scala | 2 +- .../measure/data/source/DataSource.scala | 66 +- .../measure/data/source/DataSourceCache.scala | 69 +- .../measure/data/source/DataSourceFactory.scala | 41 +- .../griffin/measure/persist/HdfsPersist.scala | 128 +- .../griffin/measure/persist/HttpPersist.scala | 3 +- .../griffin/measure/persist/LoggerPersist.scala | 53 +- .../griffin/measure/persist/MongoPersist.scala | 119 ++ .../measure/persist/MongoThreadPool.scala | 73 + .../griffin/measure/persist/MultiPersists.scala | 19 +- .../griffin/measure/persist/Persist.scala | 3 +- .../measure/persist/PersistFactory.scala | 2 + .../measure/persist/PersistThreadPool.scala | 2 +- .../measure/process/BatchDqProcess.scala | 110 +- .../griffin/measure/process/DqProcess.scala | 2 +- .../measure/process/StreamingDqProcess.scala | 32 +- .../measure/process/StreamingDqThread.scala | 101 +- .../measure/process/check/DataChecker.scala | 29 - .../process/engine/DataFrameOprEngine.scala | 105 +- .../measure/process/engine/DqEngine.scala | 26 +- .../measure/process/engine/DqEngines.scala | 349 ++++- .../measure/process/engine/SparkDqEngine.scala | 350 ++++- .../measure/process/engine/SparkSqlEngine.scala | 25 +- .../measure/process/temp/DataFrameCaches.scala | 115 ++ .../measure/process/temp/TableRegisters.scala | 153 ++ .../measure/process/temp/TableRegs.scala | 81 + .../griffin/measure/result/AccuracyResult.scala | 6 + .../griffin/measure/result/ProfileResult.scala | 4 + .../apache/griffin/measure/result/Result.scala | 4 + .../rule/adaptor/DataFrameOprAdaptor.scala | 45 +- .../rule/adaptor/GriffinDslAdaptor.scala | 1415 ++++++++++++++---- .../measure/rule/adaptor/InternalColumns.scala | 31 + .../measure/rule/adaptor/RuleAdaptor.scala | 157 +- .../measure/rule/adaptor/RuleAdaptorGroup.scala | 259 +++- .../measure/rule/adaptor/SparkSqlAdaptor.scala | 50 +- .../griffin/measure/rule/dsl/CollectType.scala | 57 + .../griffin/measure/rule/dsl/DqType.scala | 11 +- .../griffin/measure/rule/dsl/DslType.scala | 9 +- .../griffin/measure/rule/dsl/PersistType.scala | 2 + .../rule/dsl/analyzer/BasicAnalyzer.scala | 4 +- .../rule/dsl/analyzer/DuplicateAnalyzer.scala | 46 + .../rule/dsl/analyzer/ProfilingAnalyzer.scala | 22 +- .../rule/dsl/analyzer/TimelinessAnalyzer.scala | 65 + .../rule/dsl/expr/ClauseExpression.scala | 66 +- .../griffin/measure/rule/dsl/expr/Expr.scala | 3 + .../rule/dsl/expr/ExtraConditionExpr.scala | 27 + .../measure/rule/dsl/expr/FunctionExpr.scala | 16 +- .../measure/rule/dsl/expr/LogicalExpr.scala | 34 + .../measure/rule/dsl/expr/MathExpr.scala | 14 + .../measure/rule/dsl/expr/SelectExpr.scala | 39 +- .../measure/rule/dsl/parser/BasicParser.scala | 25 +- .../rule/dsl/parser/GriffinDslParser.scala | 18 + .../griffin/measure/rule/plan/DfOprStep.scala | 32 + .../measure/rule/plan/MetricExport.scala | 28 + .../measure/rule/plan/RecordExport.scala | 27 + .../griffin/measure/rule/plan/RuleExport.scala | 27 + .../griffin/measure/rule/plan/RulePlan.scala | 54 + .../griffin/measure/rule/plan/RuleStep.scala | 40 + .../measure/rule/plan/SparkSqlStep.scala | 32 + .../griffin/measure/rule/plan/TimeInfo.scala | 37 + .../measure/rule/step/ConcreteRuleStep.scala | 37 - .../griffin/measure/rule/step/DfOprStep.scala | 29 - .../measure/rule/step/GriffinDslStep.scala | 28 - .../griffin/measure/rule/step/RuleStep.scala | 31 - .../measure/rule/step/SparkSqlStep.scala | 30 - .../griffin/measure/rule/udf/GriffinUdfs.scala | 5 + .../measure/utils/HdfsFileDumpUtil.scala | 40 +- .../apache/griffin/measure/utils/HdfsUtil.scala | 6 +- .../griffin/measure/utils/ParamUtil.scala | 36 + .../apache/griffin/measure/utils/TimeUtil.scala | 6 +- .../resources/_accuracy-batch-griffindsl.json | 63 + .../resources/_accuracy-batch-sparksql.json | 63 + .../_accuracy-streaming-griffindsl.json | 117 ++ .../resources/_accuracy-streaming-sparksql.json | 142 ++ .../resources/_duplicate-batch-griffindsl.json | 56 + .../_duplicate-streaming-griffindsl.json | 116 ++ .../_duplicate-streaming-sparksql.json | 130 ++ .../resources/_profiling-batch-griffindsl.json | 46 + .../resources/_profiling-batch-sparksql.json | 44 + .../_profiling-streaming-griffindsl.json | 74 + .../_profiling-streaming-sparksql.json | 80 + .../resources/_timeliness-batch-griffindsl.json | 42 + .../resources/_timeliness-batch-sparksql.json | 52 + .../_timeliness-streaming-griffindsl.json | 72 + .../_timeliness-streaming-sparksql.json | 82 + .../resources/config-test-accuracy-new.json | 56 + .../resources/config-test-accuracy-new2.json | 72 + .../config-test-accuracy-streaming-new.json | 117 ++ .../config-test-accuracy-streaming-new2.json | 133 ++ .../test/resources/config-test-accuracy2.json | 64 + .../resources/config-test-profiling-new.json | 80 + .../resources/config-test-profiling-new2.json | 36 + .../config-test-profiling-streaming-new.json | 85 ++ .../config-test-profiling-streaming-new2.json | 72 + .../config-test-profiling-streaming.json | 25 +- .../test/resources/config-test-profiling1.json | 60 + .../test/resources/config-test-profiling2.json | 35 + measure/src/test/resources/env-hdfs-test.json | 45 + .../src/test/resources/env-streaming-mongo.json | 54 + measure/src/test/resources/env-test.json | 2 +- .../resources/performance-test-accuracy.json | 56 + .../resources/performance-test-profiling.json | 34 + measure/src/test/resources/timeliness_data.avro | Bin 0 -> 409 bytes .../validator/AllParamValidatorTest.scala | 14 +- .../measure/persist/MongoPersistTest.scala | 47 + .../rule/adaptor/GriffinDslAdaptorTest.scala | 171 ++- .../rule/adaptor/RuleAdaptorGroupTest.scala | 70 + .../rule/adaptor/SparkSqlAdaptorTest.scala | 59 + .../rule/dsl/parser/BasicParserTest.scala | 44 +- 125 files changed, 6773 insertions(+), 1140 deletions(-) ----------------------------------------------------------------------
