http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/_uniqueness-streaming-griffindsl.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/_uniqueness-streaming-griffindsl.json b/measure/src/test/resources/_uniqueness-streaming-griffindsl.json deleted file mode 100644 index bc5cbd2..0000000 --- a/measure/src/test/resources/_uniqueness-streaming-griffindsl.json +++ /dev/null @@ -1,119 +0,0 @@ -{ - "name": "unique_streaming", - - "process.type": "streaming", - - "data.sources": [ - { - "name": "new", - "baseline": true, - "connectors": [ - { - "type": "kafka", - "version": "0.8", - "config": { - "kafka.config": { - "bootstrap.servers": "10.149.247.156:9092", - "group.id": "new", - "auto.offset.reset": "smallest", - "auto.commit.enable": "false" - }, - "topics": "ttt", - "key.type": "java.lang.String", - "value.type": "java.lang.String" - }, - "pre.proc": [ - { - "dsl.type": "df-opr", - "name": "${s1}", - "rule": "from_json", - "details": { - "df.name": "${this}" - } - }, - { - "dsl.type": "spark-sql", - "name": "${this}", - "rule": "select name, age from ${s1}" - } - ] - } - ], - "cache": { - "file.path": "hdfs://localhost/griffin/streaming/dump/new", - "info.path": "new", - "ready.time.interval": "10s", - "ready.time.delay": "0", - "time.range": ["0", "0"] - } - }, - { - "name": "old", - "connectors": [ - { - "type": "kafka", - "version": "0.8", - "config": { - "kafka.config": { - "bootstrap.servers": "10.149.247.156:9092", - "group.id": "old", - "auto.offset.reset": "smallest", - "auto.commit.enable": "false" - }, - "topics": "ttt", - "key.type": "java.lang.String", - "value.type": "java.lang.String" - }, - "pre.proc": [ - { - "dsl.type": "df-opr", - "name": "${s1}", - "rule": "from_json", - "details": { - "df.name": "${this}" - } - }, - { - "dsl.type": "spark-sql", - "name": "${this}", - "rule": "select name, age from ${s1}" - } - ] - } - ], - "cache": { - "file.path": "hdfs://localhost/griffin/streaming/dump/old", - "info.path": "old", - "ready.time.interval": "10s", - "ready.time.delay": "0", - "time.range": ["-24h", "0"] - } - } - ], - - "evaluate.rule": { - "rules": [ - { - "dsl.type": "griffin-dsl", - "dq.type": "uniqueness", - "name": "dup", - "rule": "name, age", - "details": { - "source": "new", - "target": "old", - "total": "total", - "unique": "unique", - "dup": "dup", - "num": "num", - "duplication.array": "dup" - }, - "metric": { - "name": "unique" - }, - "record": { - "name": "dupRecords" - } - } - ] - } -} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/_uniqueness-streaming-sparksql.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/_uniqueness-streaming-sparksql.json b/measure/src/test/resources/_uniqueness-streaming-sparksql.json deleted file mode 100644 index 7d13215..0000000 --- a/measure/src/test/resources/_uniqueness-streaming-sparksql.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "name": "unique_streaming", - - "process.type": "streaming", - - "data.sources": [ - { - "name": "new", - "baseline": true, - "connectors": [ - { - "type": "kafka", - "version": "0.8", - "config": { - "kafka.config": { - "bootstrap.servers": "10.149.247.156:9092", - "group.id": "new", - "auto.offset.reset": "smallest", - "auto.commit.enable": "false" - }, - "topics": "sss", - "key.type": "java.lang.String", - "value.type": "java.lang.String" - }, - "pre.proc": [ - { - "dsl.type": "df-opr", - "name": "${s1}", - "rule": "from_json", - "details": { - "df.name": "${this}" - } - }, - { - "dsl.type": "spark-sql", - "name": "${this}", - "rule": "select name, age from ${s1}" - } - ] - } - ], - "cache": { - "file.path": "hdfs://localhost/griffin/streaming/dump/new", - "info.path": "new", - "ready.time.interval": "10s", - "ready.time.delay": "0", - "time.range": ["0", "0"] - } - }, - { - "name": "old", - "connectors": [ - { - "type": "kafka", - "version": "0.8", - "config": { - "kafka.config": { - "bootstrap.servers": "10.149.247.156:9092", - "group.id": "old", - "auto.offset.reset": "smallest", - "auto.commit.enable": "false" - }, - "topics": "sss", - "key.type": "java.lang.String", - "value.type": "java.lang.String" - }, - "pre.proc": [ - { - "dsl.type": "df-opr", - "name": "${s1}", - "rule": "from_json", - "details": { - "df.name": "${this}" - } - }, - { - "dsl.type": "spark-sql", - "name": "${this}", - "rule": "select name, age from ${s1}" - } - ] - } - ], - "cache": { - "file.path": "hdfs://localhost/griffin/streaming/dump/old", - "info.path": "old", - "ready.time.interval": "10s", - "ready.time.delay": "0", - "time.range": ["-24h", "0"] - } - } - ], - - "evaluate.rule": { - "rules": [ - { - "dsl.type": "spark-sql", - "name": "dist", - "rule": "SELECT DISTINCT * FROM new" - }, - { - "dsl.type": "spark-sql", - "name": "joined", - "rule": "SELECT dist.* FROM old RIGHT JOIN dist ON coalesce(old.name, '') = coalesce(dist.name, '') AND coalesce(old.age, '') = coalesce(dist.age, '')" - }, - { - "dsl.type": "spark-sql", - "name": "grouped", - "rule": "SELECT `__tmst`, `name`, `age`, count(*) as `dup_cnt` FROM joined GROUP BY `__tmst`, `name`, `age`" - }, - { - "dsl.type": "spark-sql", - "name": "dupRecs", - "cache": true, - "rule": "SELECT * FROM grouped WHERE `dup_cnt` > 1", - "record": { - "name": "dupRecords" - } - }, - { - "dsl.type": "spark-sql", - "name": "dupMetric", - "rule": "SELECT `__tmst`, `dup_cnt`, count(*) as `item_cnt` FROM dupRecs GROUP BY `__tmst`, `dup_cnt`", - "metric": { - "name": "dup" - } - } - ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config-griffindsl.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config-griffindsl.json b/measure/src/test/resources/config-griffindsl.json new file mode 100644 index 0000000..10167cd --- /dev/null +++ b/measure/src/test/resources/config-griffindsl.json @@ -0,0 +1,56 @@ +{ + "name": "accu_batch", + + "process.type": "batch", + + "data.sources": [ + { + "name": "source", + "baseline": true, + "connectors": [ + { + "type": "avro", + "version": "1.7", + "config": { + "file.name": "src/test/resources/users_info_src.avro" + } + } + ] + }, { + "name": "target", + "connectors": [ + { + "type": "avro", + "version": "1.7", + "config": { + "file.name": "src/test/resources/users_info_target.avro" + } + } + ] + } + ], + + "evaluate.rule": { + "rules": [ + { + "dsl.type": "griffin-dsl", + "dq.type": "accuracy", + "name": "accu", + "rule": "source.user_id = target.user_id AND upper(source.first_name) = upper(target.first_name) AND source.last_name = target.last_name AND source.address = target.address AND source.email = target.email AND source.phone = target.phone AND source.post_code = target.post_code", + "details": { + "source": "source", + "target": "target", + "miss": "miss_count", + "total": "total_count", + "matched": "matched_count" + }, + "metric": { + "name": "accu" + }, + "record": { + "name": "missRecords" + } + } + ] + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config-profile.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config-profile.json b/measure/src/test/resources/config-profile.json deleted file mode 100644 index 2529def..0000000 --- a/measure/src/test/resources/config-profile.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "prof1", - "type": "profile", - - "process.type": "batch", - - "source": { - "type": "avro", - "version": "1.7", - "config": { - "file.name": "src/test/resources/users_info_src.avro" - } - }, - - "evaluateRule": { - "sampleRatio": 1, - "rules": "$source.post_code == null" - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config-streaming-accuracy.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config-streaming-accuracy.json b/measure/src/test/resources/config-streaming-accuracy.json new file mode 100644 index 0000000..240d768 --- /dev/null +++ b/measure/src/test/resources/config-streaming-accuracy.json @@ -0,0 +1,121 @@ +{ + "name": "accu_streaming", + + "process.type": "streaming", + + "data.sources": [ + { + "name": "source", + "baseline": true, + "connectors": [ + { + "type": "kafka", + "version": "0.8", + "config": { + "kafka.config": { + "bootstrap.servers": "10.147.177.107:9092", + "group.id": "group1", + "auto.offset.reset": "smallest", + "auto.commit.enable": "false" + }, + "topics": "sss", + "key.type": "java.lang.String", + "value.type": "java.lang.String" + }, + "pre.proc": [ + { + "dsl.type": "df-opr", + "name": "${s1}", + "rule": "from_json", + "details": { + "df.name": "${this}" + } + }, + { + "dsl.type": "spark-sql", + "name": "${this}", + "rule": "select name, age from ${s1}" + } + ] + } + ], + "cache": { + "type": "parquet", + "file.path": "hdfs://localhost/griffin/streaming/dump/source", + "info.path": "source", + "ready.time.interval": "10s", + "ready.time.delay": "0", + "time.range": ["-2m", "0"], + "init.clear": true, + "updatable": true + } + }, { + "name": "target", + "connectors": [ + { + "type": "kafka", + "version": "0.8", + "config": { + "kafka.config": { + "bootstrap.servers": "10.147.177.107:9092", + "group.id": "group1", + "auto.offset.reset": "smallest", + "auto.commit.enable": "false" + }, + "topics": "ttt", + "key.type": "java.lang.String", + "value.type": "java.lang.String" + }, + "pre.proc": [ + { + "dsl.type": "df-opr", + "name": "${t1}", + "rule": "from_json", + "details": { + "df.name": "${this}" + } + }, + { + "dsl.type": "spark-sql", + "name": "${this}", + "rule": "select name, age from ${t1}" + } + ] + } + ], + "cache": { + "type": "parquet", + "file.path": "hdfs://localhost/griffin/streaming/dump/target", + "info.path": "target", + "ready.time.interval": "10s", + "ready.time.delay": "0", + "time.range": ["-2m", "0"], + "init.clear": true + } + } + ], + + "evaluate.rule": { + "rules": [ + { + "dsl.type": "griffin-dsl", + "dq.type": "accuracy", + "name": "accu", + "rule": "source.name = target.name and source.age = target.age", + "details": { + "source": "source", + "target": "target", + "miss": "miss_count", + "total": "total_count", + "matched": "matched_count" + }, + "metric": { + "name": "accu" + }, + "record": { + "name": "missRecords" + } + } + ] + } +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config-streaming.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config-streaming.json b/measure/src/test/resources/config-streaming.json index 697fb7b..243a691 100644 --- a/measure/src/test/resources/config-streaming.json +++ b/measure/src/test/resources/config-streaming.json @@ -1,69 +1,75 @@ { - "name": "accu1", - "type": "accuracy", + "name": "prof_streaming", "process.type": "streaming", - "source": { - "type": "kafka", - "version": "0.8", - "config": { - "kafka.config": { - "bootstrap.servers": "10.149.247.156:9092", - "group.id": "group1", - "auto.offset.reset": "smallest", - "auto.commit.enable": "false" - }, - "topics": "sss", - "key.type": "java.lang.String", - "value.type": "java.lang.String" - }, - "cache": { - "type": "hive", - "config": { - "database": "griffin_cache", - "table.name": "src", - "parent.path": "hdfs://localhost/griffin/streaming/dump", - "info.path": "src", + "data.sources": [ + { + "name": "source", + "connectors": [ + { + "type": "kafka", + "version": "0.8", + "config": { + "kafka.config": { + "bootstrap.servers": "10.147.177.107:9092", + "group.id": "group1", + "auto.offset.reset": "smallest", + "auto.commit.enable": "false" + }, + "topics": "sss", + "key.type": "java.lang.String", + "value.type": "java.lang.String" + }, + "pre.proc": [ + { + "dsl.type": "df-opr", + "name": "${s1}", + "rule": "from_json", + "details": { + "df.name": "${this}" + } + }, + { + "dsl.type": "spark-sql", + "name": "${this}", + "rule": "select name, age from ${s1}" + } + ] + } + ], + "cache": { + "file.path": "hdfs://localhost/griffin/streaming/dump/source", + "info.path": "source", "ready.time.interval": "10s", - "ready.time.delay": "0" - }, - "time.range": ["-2m", "0"] - }, - "match.once": true - }, + "ready.time.delay": "0", + "time.range": ["0", "0"], + "init.clear": true + } + } + ], - "target": { - "type": "kafka", - "version": "0.8", - "config": { - "kafka.config": { - "bootstrap.servers": "10.149.247.156:9092", - "group.id": "group1", - "auto.offset.reset": "smallest", - "auto.commit.enable": "false" + "evaluate.rule": { + "rules": [ + { + "dsl.type": "griffin-dsl", + "dq.type": "profiling", + "name": "prof", + "rule": "select count(name) as `cnt`, max(age) as `max`, min(age) as `min` from source", + "metric": { + "name": "prof" + } }, - "topics": "ttt", - "key.type": "java.lang.String", - "value.type": "java.lang.String" - }, - "cache": { - "type": "hive", - "config": { - "database": "griffin_cache", - "table.name": "tgt", - "parent.path": "hdfs://localhost/griffin/streaming/dump", - "info.path": "tgt", - "ready.time.interval": "10s", - "ready.time.delay": "0" - }, - "time.range": ["-2m", "0"] - }, - "match.once": false - }, - - "evaluateRule": { - "sampleRatio": 0.2, - "rules": "$source.json().name = $target.json().name AND $source.json().age = $target.json().age" + { + "dsl.type": "griffin-dsl", + "dq.type": "profiling", + "name": "grp", + "rule": "select name, count(*) as `cnt` from source group by name", + "metric": { + "name": "name_group", + "collect.type": "array" + } + } + ] } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config-streaming1.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config-streaming1.json b/measure/src/test/resources/config-streaming1.json deleted file mode 100644 index 5465201..0000000 --- a/measure/src/test/resources/config-streaming1.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "name": "accu2", - "type": "accuracy", - - "process.type": "streaming", - - "source": { - "type": "kafka", - "version": "0.8", - "config": { - "kafka.config": { - "bootstrap.servers": "10.149.247.156:9092", - "group.id": "group1", - "auto.offset.reset": "smallest", - "auto.commit.enable": "false" - }, - "topics": "sss", - "key.type": "java.lang.String", - "value.type": "java.lang.String" - }, - "cache": { - "type": "df", - "config": { - "cache.level": "MEMORY_AND_DISK", - "info.path": "src", - "ready.time.interval": "10s", - "ready.time.delay": "0" - }, - "time.range": ["-2m", "0"] - }, - "match.once": true - }, - - "target": { - "type": "kafka", - "version": "0.8", - "config": { - "kafka.config": { - "bootstrap.servers": "10.149.247.156:9092", - "group.id": "group1", - "auto.offset.reset": "smallest", - "auto.commit.enable": "false" - }, - "topics": "ttt", - "key.type": "java.lang.String", - "value.type": "java.lang.String" - }, - "cache": { - "type": "df", - "config": { - "cache.level": "MEMORY_AND_DISK", - "info.path": "tgt", - "ready.time.interval": "10s", - "ready.time.delay": "0" - }, - "time.range": ["-2m", "0"] - }, - "match.once": false - }, - - "evaluateRule": { - "sampleRatio": 0.2, - "rules": "$source.json().name = $target.json().name AND $source.json().age = $target.json().age" - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config-streaming2.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config-streaming2.json b/measure/src/test/resources/config-streaming2.json deleted file mode 100644 index 061382b..0000000 --- a/measure/src/test/resources/config-streaming2.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "name": "accu1", - "type": "accuracy", - - "process.type": "streaming", - - "source": { - "type": "kafka", - "version": "0.8", - "config": { - "kafka.config": { - "bootstrap.servers": "10.149.247.156:9092", - "group.id": "group1", - "auto.offset.reset": "smallest", - "auto.commit.enable": "false" - }, - "topics": "sss", - "key.type": "java.lang.String", - "value.type": "java.lang.String" - }, - "cache": { - "type": "parquet", - "config": { - "file.path": "hdfs://localhost/griffin/streaming/dump/source", - "info.path": "source", - "ready.time.interval": "10s", - "ready.time.delay": "0" - }, - "time.range": ["-2m", "0"] - }, - "match.once": true - }, - - "target": { - "type": "kafka", - "version": "0.8", - "config": { - "kafka.config": { - "bootstrap.servers": "10.149.247.156:9092", - "group.id": "group1", - "auto.offset.reset": "smallest", - "auto.commit.enable": "false" - }, - "topics": "ttt", - "key.type": "java.lang.String", - "value.type": "java.lang.String" - }, - "cache": { - "type": "parquet", - "config": { - "file.path": "hdfs://localhost/griffin/streaming/dump/target", - "info.path": "target", - "ready.time.interval": "10s", - "ready.time.delay": "0" - }, - "time.range": ["-2m", "0"] - }, - "match.once": false - }, - - "evaluateRule": { - "sampleRatio": 0.2, - "rules": "$source.json().name = $target.json().name AND $source.json().age = $target.json().age" - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config-streaming3.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config-streaming3.json b/measure/src/test/resources/config-streaming3.json deleted file mode 100644 index fe3e56f..0000000 --- a/measure/src/test/resources/config-streaming3.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "name": "accu1", - "type": "accuracy", - - "process.type": "streaming", - - "source": { - "type": "kafka", - "version": "0.8", - "config": { - "kafka.config": { - "bootstrap.servers": "10.149.247.156:9092", - "group.id": "group1", - "auto.offset.reset": "smallest", - "auto.commit.enable": "false" - }, - "topics": "sss", - "key.type": "java.lang.String", - "value.type": "java.lang.String" - }, - "cache": { - "type": "text", - "config": { - "file.path": "hdfs://localhost/griffin/streaming/dump/source", - "info.path": "source", - "ready.time.interval": "10s", - "ready.time.delay": "0" - }, - "time.range": ["-2m", "0"] - }, - "match.once": true - }, - - "target": { - "type": "kafka", - "version": "0.8", - "config": { - "kafka.config": { - "bootstrap.servers": "10.149.247.156:9092", - "group.id": "group1", - "auto.offset.reset": "smallest", - "auto.commit.enable": "false" - }, - "topics": "ttt", - "key.type": "java.lang.String", - "value.type": "java.lang.String" - }, - "cache": { - "type": "text", - "config": { - "file.path": "hdfs://localhost/griffin/streaming/dump/target", - "info.path": "target", - "ready.time.interval": "10s", - "ready.time.delay": "0" - }, - "time.range": ["-2m", "0"] - }, - "match.once": false - }, - - "evaluateRule": { - "sampleRatio": 0.2, - "rules": "$source.json().name = $target.json().name AND $source.json().age = $target.json().age" - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config-test-accuracy-new.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config-test-accuracy-new.json b/measure/src/test/resources/config-test-accuracy-new.json deleted file mode 100644 index 80d608b..0000000 --- a/measure/src/test/resources/config-test-accuracy-new.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "name": "accu_batch_test", - - "timestamp": 12124214, - - "process.type": "batch", - - "data.sources": [ - { - "name": "source", - "baseline": true, - "connectors": [ - { - "type": "avro", - "version": "1.7", - "config": { - "file.name": "src/test/resources/users_info_src.avro" - } - } - ] - }, { - "name": "target", - "connectors": [ - { - "type": "avro", - "version": "1.7", - "config": { - "file.name": "src/test/resources/users_info_target.avro" - } - } - ] - } - ], - - "evaluate.rule": { - "rules": [ - { - "dsl.type": "griffin-dsl", - "dq.type": "accuracy", - "name": "accuracy", - "rule": "source.user_id = target.user_id AND upper(source.first_name) = upper(target.first_name) AND source.last_name = target.last_name AND source.address = target.address AND source.email = target.email AND source.phone = target.phone AND source.post_code = target.post_code", - "details": { - "persist.type": "metric", - "source": "source", - "target": "target", - "miss": "miss_count", - "total": "total_count", - "matched": "matched_count", - "missRecords": { - "persist.type": "record" - } - } - } - ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config-test-accuracy-new2.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config-test-accuracy-new2.json b/measure/src/test/resources/config-test-accuracy-new2.json deleted file mode 100644 index 23e42cb..0000000 --- a/measure/src/test/resources/config-test-accuracy-new2.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "name": "accu_batch_test", - - "timestamp": 12124214, - - "process.type": "batch", - - "data.sources": [ - { - "name": "source", - "baseline": true, - "connectors": [ - { - "type": "avro", - "version": "1.7", - "config": { - "file.name": "src/test/resources/users_info_src.avro" - } - } - ] - }, { - "name": "target", - "connectors": [ - { - "type": "avro", - "version": "1.7", - "config": { - "file.name": "src/test/resources/users_info_target.avro" - } - } - ] - } - ], - - "evaluate.rule": { - "rules": [ - { - "dsl.type": "spark-sql", - "name": "miss_records", - "gather.step": true, - "rule": "SELECT source.* FROM source LEFT JOIN target ON coalesce(source.user_id, '') = coalesce(target.user_id, '') AND coalesce(source.first_name, '') = coalesce(target.first_name, '') AND coalesce(source.post_code, '') = coalesce(target.post_code, '') WHERE (NOT (source.user_id IS NULL AND source.user_id IS NULL AND source.post_code IS NULL)) AND (target.user_id IS NULL AND target.user_id IS NULL AND target.post_code IS NULL)", - "details": { - "persist.type": "record" - } - }, - { - "dsl.type": "spark-sql", - "name": "miss_count", - "rule": "SELECT count(*) as miss FROM `miss_records`" - }, - { - "dsl.type": "spark-sql", - "name": "total_count", - "rule": "SELECT count(*) as total FROM source" - }, - { - "dsl.type": "spark-sql", - "name": "accu", - "rule": "SELECT `miss_count`.miss, `total_count`.total, (`total_count`.total - `miss_count`.miss) as matched FROM `miss_count` FULL JOIN `total_count`" - }, - { - "dsl.type": "df-opr", - "name": "accu", - "rule": "accuracy", - "details": { - "persist.type": "metric", - "df.name": "accu" - } - } - ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config-test-accuracy-streaming-multids.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config-test-accuracy-streaming-multids.json b/measure/src/test/resources/config-test-accuracy-streaming-multids.json deleted file mode 100644 index 2107ff6..0000000 --- a/measure/src/test/resources/config-test-accuracy-streaming-multids.json +++ /dev/null @@ -1,142 +0,0 @@ -{ - "name": "accu_streaming", - - "process.type": "streaming", - - "data.sources": [ - { - "name": "source", - "connectors": [ - { - "type": "kafka", - "version": "0.8", - "config": { - "kafka.config": { - "bootstrap.servers": "10.149.247.156:9092", - "group.id": "group1", - "auto.offset.reset": "smallest", - "auto.commit.enable": "false" - }, - "topics": "sss", - "key.type": "java.lang.String", - "value.type": "java.lang.String" - }, - "pre.proc": [ - { - "dsl.type": "df-opr", - "name": "${s1}", - "rule": "from_json", - "persist.type": "cache", - "details": { - "df.name": "${this}" - } - }, - { - "dsl.type": "spark-sql", - "name": "${this}", - "rule": "select name, age from ${s1}" - } - ] - }, - { - "type": "text-dir", - "config": { - "dir.path": "hdfs://localhost/griffin/text", - "data.dir.depth": 0, - "success.file": "_SUCCESS", - "done.file": "_DONE" - }, - "pre.proc": [ - { - "dsl.type": "df-opr", - "name": "${s1}", - "rule": "from_json", - "details": { - "df.name": "${this}" - } - }, - { - "dsl.type": "spark-sql", - "name": "${this}", - "rule": "select name, age from ${s1}" - } - ] - } - ], - "cache": { - "file.path": "hdfs://localhost/griffin/streaming/dump/source", - "info.path": "source", - "ready.time.interval": "10s", - "ready.time.delay": "0", - "time.range": ["-2m", "0"] - } - }, { - "name": "target", - "connectors": [ - { - "type": "kafka", - "version": "0.8", - "config": { - "kafka.config": { - "bootstrap.servers": "10.149.247.156:9092", - "group.id": "group1", - "auto.offset.reset": "smallest", - "auto.commit.enable": "false" - }, - "topics": "ttt", - "key.type": "java.lang.String", - "value.type": "java.lang.String" - }, - "pre.proc": [ - { - "dsl.type": "df-opr", - "name": "${t1}", - "rule": "from_json", - "details": { - "df.name": "${this}" - } - }, - { - "dsl.type": "spark-sql", - "name": "${this}", - "rule": "select name, age from ${t1}" - } - ] - } - ], - "cache": { - "file.path": "hdfs://localhost/griffin/streaming/dump/target", - "info.path": "target", - "ready.time.interval": "10s", - "ready.time.delay": "0", - "time.range": ["-2m", "0"] - } - } - ], - - "evaluateRule": { - "rules": [ - { - "dsl.type": "griffin-dsl", - "dq.type": "accuracy", - "rule": "source.name = target.name and source.age = target.age", - "details": { - "source": "source", - "target": "target", - "miss.records": { - "name": "miss.records", - "persist.type": "record", - "update.data.source": "source" - }, - "accuracy": { - "name": "accu", - "persist.type": "metric" - }, - "miss": "miss_count", - "total": "total_count", - "matched": "matched_count" - } - } - ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config-test-accuracy-streaming-new.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config-test-accuracy-streaming-new.json b/measure/src/test/resources/config-test-accuracy-streaming-new.json deleted file mode 100644 index 66f1081..0000000 --- a/measure/src/test/resources/config-test-accuracy-streaming-new.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "name": "accu_streaming", - - "process.type": "streaming", - - "data.sources": [ - { - "name": "source", - "baseline": true, - "connectors": [ - { - "type": "kafka", - "version": "0.8", - "config": { - "kafka.config": { - "bootstrap.servers": "10.149.247.156:9092", - "group.id": "group1", - "auto.offset.reset": "smallest", - "auto.commit.enable": "false" - }, - "topics": "sss", - "key.type": "java.lang.String", - "value.type": "java.lang.String" - }, - "pre.proc": [ - { - "dsl.type": "df-opr", - "name": "${s1}", - "rule": "from_json", - "details": { - "df.name": "${this}" - } - }, - { - "dsl.type": "spark-sql", - "name": "${this}", - "rule": "select name, age from ${s1}" - } - ] - } - ], - "cache": { - "file.path": "hdfs://localhost/griffin/streaming/dump/source", - "info.path": "source", - "ready.time.interval": "10s", - "ready.time.delay": "0", - "time.range": ["-2m", "0"] - } - }, { - "name": "target", - "connectors": [ - { - "type": "kafka", - "version": "0.8", - "config": { - "kafka.config": { - "bootstrap.servers": "10.149.247.156:9092", - "group.id": "group1", - "auto.offset.reset": "smallest", - "auto.commit.enable": "false" - }, - "topics": "ttt", - "key.type": "java.lang.String", - "value.type": "java.lang.String" - }, - "pre.proc": [ - { - "dsl.type": "df-opr", - "name": "${t1}", - "rule": "from_json", - "details": { - "df.name": "${this}" - } - }, - { - "dsl.type": "spark-sql", - "name": "${this}", - "rule": "select name, age from ${t1}" - } - ] - } - ], - "cache": { - "file.path": "hdfs://localhost/griffin/streaming/dump/target", - "info.path": "target", - "ready.time.interval": "10s", - "ready.time.delay": "0", - "time.range": ["-2m", "0"] - } - } - ], - - "evaluate.rule": { - "rules": [ - { - "dsl.type": "griffin-dsl", - "dq.type": "accuracy", - "name": "accuracy", - "rule": "source.name = target.name and source.age = target.age", - "details": { - "persist.type": "metric", - "source": "source", - "target": "target", - "miss": "miss_count", - "total": "total_count", - "matched": "matched_count", - "missRecords": { - "persist.name": "missRecords", - "persist.type": "record", - "cache.data.source": "source" - }, - "enable.ignore.cache": true - } - } - ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config-test-accuracy-streaming-new2.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config-test-accuracy-streaming-new2.json b/measure/src/test/resources/config-test-accuracy-streaming-new2.json deleted file mode 100644 index feb49e7..0000000 --- a/measure/src/test/resources/config-test-accuracy-streaming-new2.json +++ /dev/null @@ -1,133 +0,0 @@ -{ - "name": "accu_streaming", - - "process.type": "streaming", - - "data.sources": [ - { - "name": "source", - "baseline": true, - "connectors": [ - { - "type": "kafka", - "version": "0.8", - "config": { - "kafka.config": { - "bootstrap.servers": "10.149.247.156:9092", - "group.id": "group1", - "auto.offset.reset": "smallest", - "auto.commit.enable": "false" - }, - "topics": "sss", - "key.type": "java.lang.String", - "value.type": "java.lang.String" - }, - "pre.proc": [ - { - "dsl.type": "df-opr", - "name": "${s1}", - "rule": "from_json", - "details": { - "df.name": "${this}" - } - }, - { - "dsl.type": "spark-sql", - "name": "${this}", - "rule": "select name, age from ${s1}" - } - ] - } - ], - "cache": { - "file.path": "hdfs://localhost/griffin/streaming/dump/source", - "info.path": "source", - "ready.time.interval": "10s", - "ready.time.delay": "0", - "time.range": ["-2m", "0"] - } - }, { - "name": "target", - "connectors": [ - { - "type": "kafka", - "version": "0.8", - "config": { - "kafka.config": { - "bootstrap.servers": "10.149.247.156:9092", - "group.id": "group1", - "auto.offset.reset": "smallest", - "auto.commit.enable": "false" - }, - "topics": "ttt", - "key.type": "java.lang.String", - "value.type": "java.lang.String" - }, - "pre.proc": [ - { - "dsl.type": "df-opr", - "name": "${t1}", - "rule": "from_json", - "details": { - "df.name": "${this}" - } - }, - { - "dsl.type": "spark-sql", - "name": "${this}", - "rule": "select name, age from ${t1}" - } - ] - } - ], - "cache": { - "file.path": "hdfs://localhost/griffin/streaming/dump/target", - "info.path": "target", - "ready.time.interval": "10s", - "ready.time.delay": "0", - "time.range": ["-2m", "0"] - } - } - ], - - "evaluate.rule": { - "rules": [ - { - "dsl.type": "spark-sql", - "name": "missRecords", - "rule": "SELECT source.* FROM source LEFT JOIN target ON coalesce(source.name, '') = coalesce(target.name, '') AND coalesce(source.age, '') = coalesce(target.age, '') WHERE (NOT (source.name IS NULL AND source.age IS NULL)) AND (target.name IS NULL AND target.age IS NULL)", - "details": { - "persist.type": "record", - "cache.data.source": "source" - } - }, - { - "dsl.type": "spark-sql", - "name": "miss_count", - "rule": "SELECT count(*) as miss FROM `missRecords`" - }, - { - "dsl.type": "spark-sql", - "name": "total_count", - "rule": "SELECT count(*) as total FROM source" - }, - { - "dsl.type": "spark-sql", - "name": "accu", - "rule": "SELECT `miss_count`.miss, `total_count`.total FROM `miss_count` FULL JOIN `total_count`" - }, - { - "dsl.type": "df-opr", - "name": "accu", - "rule": "accuracy", - "details": { - "persist.type": "metric", - "df.name": "accu", - "miss": "miss", - "total": "total", - "matched": "matched_count" - } - } - ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config-test-accuracy-streaming.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config-test-accuracy-streaming.json b/measure/src/test/resources/config-test-accuracy-streaming.json deleted file mode 100644 index 453f87e..0000000 --- a/measure/src/test/resources/config-test-accuracy-streaming.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "name": "accu_streaming", - - "process.type": "streaming", - - "data.sources": [ - { - "name": "source", - "connectors": [ - { - "type": "kafka", - "version": "0.8", - "config": { - "kafka.config": { - "bootstrap.servers": "10.149.247.156:9092", - "group.id": "group1", - "auto.offset.reset": "smallest", - "auto.commit.enable": "false" - }, - "topics": "sss", - "key.type": "java.lang.String", - "value.type": "java.lang.String" - }, - "pre.proc": [ - { - "dsl.type": "df-opr", - "name": "${s1}", - "rule": "from_json", - "details": { - "df.name": "${this}" - } - }, - { - "dsl.type": "spark-sql", - "name": "${this}", - "rule": "select name, age from ${s1}" - } - ] - } - ], - "cache": { - "file.path": "hdfs://localhost/griffin/streaming/dump/source", - "info.path": "source", - "ready.time.interval": "10s", - "ready.time.delay": "0", - "time.range": ["-2m", "0"] - } - }, { - "name": "target", - "connectors": [ - { - "type": "kafka", - "version": "0.8", - "config": { - "kafka.config": { - "bootstrap.servers": "10.149.247.156:9092", - "group.id": "group1", - "auto.offset.reset": "smallest", - "auto.commit.enable": "false" - }, - "topics": "ttt", - "key.type": "java.lang.String", - "value.type": "java.lang.String" - }, - "pre.proc": [ - { - "dsl.type": "df-opr", - "name": "${t1}", - "rule": "from_json", - "details": { - "df.name": "${this}" - } - }, - { - "dsl.type": "spark-sql", - "name": "${this}", - "rule": "select name, age from ${t1}" - } - ] - } - ], - "cache": { - "file.path": "hdfs://localhost/griffin/streaming/dump/target", - "info.path": "target", - "ready.time.interval": "10s", - "ready.time.delay": "0", - "time.range": ["-2m", "0"] - } - } - ], - - "evaluateRule": { - "rules": [ - { - "dsl.type": "griffin-dsl", - "dq.type": "accuracy", - "rule": "source.name = target.name and source.age = target.age", - "details": { - "source": "source", - "target": "target", - "miss.records": { - "name": "miss.records", - "persist.type": "record", - "update.data.source": "source" - }, - "accuracy": { - "name": "accu", - "persist.type": "metric" - }, - "miss": "miss_count", - "total": "total_count", - "matched": "matched_count" - } - } - ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config-test-accuracy.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config-test-accuracy.json b/measure/src/test/resources/config-test-accuracy.json deleted file mode 100644 index 7f637a0..0000000 --- a/measure/src/test/resources/config-test-accuracy.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "name": "accu_batch_test", - - "timestamp": 12124214, - - "process.type": "batch", - - "data.sources": [ - { - "name": "src", - "connectors": [ - { - "type": "avro", - "version": "1.7", - "config": { - "file.name": "src/test/resources/users_info_src.avro" - } - } - ] - }, { - "name": "tgt", - "connectors": [ - { - "type": "avro", - "version": "1.7", - "config": { - "file.name": "src/test/resources/users_info_target.avro" - } - } - ] - } - ], - - "evaluateRule": { - "rules": [ - { - "dsl.type": "griffin-dsl", - "dq.type": "accuracy", - "rule": "src.user_id = tgt.user_id AND upper(src.first_name) = upper(tgt.first_name) AND src.last_name = tgt.last_name AND src.address = tgt.address AND src.email = tgt.email AND src.phone = tgt.phone AND src.post_code = tgt.post_code", - "details": { - "source": "src", - "target": "tgt", - "miss.records": { - "name": "miss.records", - "persist.type": "record" - }, - "accuracy": { - "name": "accu", - "persist.type": "metric" - }, - "miss": "miss_count", - "total": "total_count", - "matched": "matched_count" - } - } - ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config-test-accuracy2.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config-test-accuracy2.json b/measure/src/test/resources/config-test-accuracy2.json deleted file mode 100644 index 079baa7..0000000 --- a/measure/src/test/resources/config-test-accuracy2.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "name": "accu_batch_test", - - "timestamp": 12124214, - - "process.type": "batch", - - "data.sources": [ - { - "name": "source", - "connectors": [ - { - "type": "avro", - "version": "1.7", - "config": { - "file.name": "src/test/resources/users_info_src.avro" - } - } - ] - }, { - "name": "target", - "connectors": [ - { - "type": "avro", - "version": "1.7", - "config": { - "file.name": "src/test/resources/users_info_target.avro" - } - } - ] - } - ], - - "evaluateRule": { - "rules": [ - { - "dsl.type": "spark-sql", - "name": "miss-records", - "rule": "SELECT source.* FROM source LEFT JOIN target ON coalesce(source.user_id, '') = coalesce(target.user_id, '') AND coalesce(source.first_name, '') = coalesce(target.first_name, '') AND coalesce(source.post_code, '') = coalesce(target.post_code, '') WHERE (NOT (source.user_id IS NULL AND source.user_id IS NULL AND source.post_code IS NULL)) AND (target.user_id IS NULL AND target.user_id IS NULL AND target.post_code IS NULL)", - "details": { - "persist.type": "record" - } - }, - { - "dsl.type": "spark-sql", - "name": "miss-count", - "rule": "SELECT count(*) as miss FROM `miss-records`" - }, - { - "dsl.type": "spark-sql", - "name": "total-count", - "rule": "SELECT count(*) as total FROM source" - }, - { - "dsl.type": "spark-sql", - "name": "accu", - "rule": "SELECT `miss-count`.miss, `total-count`.total, (`total-count`.total - `miss-count`.miss) as matched FROM `miss-count` FULL JOIN `total-count`", - "details": { - "persist.type": "metric" - } - } - ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config-test-profiling-new.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config-test-profiling-new.json b/measure/src/test/resources/config-test-profiling-new.json deleted file mode 100644 index 47a029e..0000000 --- a/measure/src/test/resources/config-test-profiling-new.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "name": "prof_batch_test", - - "process.type": "batch", - - "timestamp": 123456, - - "data.sources": [ - { - "name": "source", - "connectors": [ - { - "type": "avro", - "version": "1.7", - "config": { - "file.name": "src/test/resources/users_info_src.avro" - }, - "pre.proc": [ - { - "dsl.type": "spark-sql", - "name": "${this}", - "rule": "select * from ${this} where post_code IS NOT NULL" - } - ] - } - ] - } - ], - - "evaluate.rule": { - "rules": [ - { - "dsl.type": "griffin-dsl", - "dq.type": "profiling", - "name": "profiling", - "rule": "select count(*) from source", - "details": { - "persist.type": "metric" - } - }, - { - "dsl.type": "griffin-dsl", - "dq.type": "profiling", - "name": "dist_name", - "rule": "select count ( distinct source.post_code ) as `dis-cnt`, max(source.user_id) from source", - "details": { - "persist.type": "metric" - } - }, - { - "dsl.type": "griffin-dsl", - "dq.type": "profiling", - "name": "pri", - "rule": "source.last_name, count(*) as `cnt` from source group by source.last_name", - "details": { - "persist.type": "metric", - "collect.type": "list" - } - }, - { - "dsl.type": "griffin-dsl", - "dq.type": "profiling", - "name": "temp", - "rule": "select * from source", - "details": { - "persist.type": "none" - } - }, - { - "dsl.type": "griffin-dsl", - "dq.type": "profiling", - "name": "temp_res", - "rule": "select count(distinct user_id) as `id-dist-cnt` from temp", - "details": { - "persist.type": "metric" - } - } - ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config-test-profiling-new2.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config-test-profiling-new2.json b/measure/src/test/resources/config-test-profiling-new2.json deleted file mode 100644 index 16125fa..0000000 --- a/measure/src/test/resources/config-test-profiling-new2.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "name": "prof_batch_test", - - "process.type": "batch", - - "timestamp": 123456, - - "data.sources": [ - { - "name": "source", - "connectors": [ - { - "type": "avro", - "version": "1.7", - "config": { - "file.name": "src/test/resources/users_info_src.avro" - } - } - ] - } - ], - - "evaluate.rule": { - "rules": [ - { - "dsl.type": "spark-sql", - "name": "out", - "rule": "select post_code, count(*) as `dist-cnt` from source group by post_code", - "details": { - "persist.type": "metric", - "collect.type": "array" - } - } - ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config-test-profiling-streaming-new.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config-test-profiling-streaming-new.json b/measure/src/test/resources/config-test-profiling-streaming-new.json deleted file mode 100644 index 20e6289..0000000 --- a/measure/src/test/resources/config-test-profiling-streaming-new.json +++ /dev/null @@ -1,85 +0,0 @@ -{ - "name": "prof_streaming", - - "process.type": "streaming", - - "data.sources": [ - { - "name": "source", - "connectors": [ - { - "type": "kafka", - "version": "0.8", - "config": { - "kafka.config": { - "bootstrap.servers": "10.149.247.156:9092", - "group.id": "group1", - "auto.offset.reset": "smallest", - "auto.commit.enable": "false" - }, - "topics": "sss", - "key.type": "java.lang.String", - "value.type": "java.lang.String" - }, - "pre.proc": [ - { - "dsl.type": "df-opr", - "name": "${s1}", - "rule": "from_json", - "details": { - "df.name": "${this}" - } - }, - { - "dsl.type": "spark-sql", - "name": "${this}", - "rule": "select name, age from ${s1}" - } - ] - } - ], - "cache": { - "file.path": "hdfs://localhost/griffin/streaming/dump/source", - "info.path": "source", - "ready.time.interval": "10s", - "ready.time.delay": "0", - "time.range": ["0", "0"] - } - } - ], - - "evaluate.rule": { - "rules": [ - { - "dsl.type": "griffin-dsl", - "dq.type": "profiling", - "name": "name-group", - "rule": "source.name, source.*.count() from source group by source.name", - "details": { - "source": "source", - "persist.type": "metric" - } - }, - { - "dsl.type": "griffin-dsl", - "dq.type": "profiling", - "name": "profiling", - "rule": "name.count(), source.age.min(), age.avg(), source.age.max()", - "details": { - "source": "source", - "persist.type": "metric" - } - }, - { - "dsl.type": "griffin-dsl", - "dq.type": "profiling", - "name": "null-count", - "rule": "name.count() as `name-null-count` where source.name IS NULL", - "details": { - "source": "source", - "persist.type": "metric" - } - } - ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config-test-profiling-streaming-new2.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config-test-profiling-streaming-new2.json b/measure/src/test/resources/config-test-profiling-streaming-new2.json deleted file mode 100644 index 53c5b49..0000000 --- a/measure/src/test/resources/config-test-profiling-streaming-new2.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "name": "prof_streaming", - - "process.type": "streaming", - - "data.sources": [ - { - "name": "source", - "connectors": [ - { - "type": "kafka", - "version": "0.8", - "config": { - "kafka.config": { - "bootstrap.servers": "10.149.247.156:9092", - "group.id": "group1", - "auto.offset.reset": "smallest", - "auto.commit.enable": "false" - }, - "topics": "sss", - "key.type": "java.lang.String", - "value.type": "java.lang.String" - }, - "pre.proc": [ - { - "dsl.type": "df-opr", - "name": "${s1}", - "rule": "from_json", - "details": { - "df.name": "${this}" - } - }, - { - "dsl.type": "spark-sql", - "name": "${this}", - "rule": "select name, age from ${s1}" - } - ] - } - ], - "cache": { - "file.path": "hdfs://localhost/griffin/streaming/dump/source", - "info.path": "source", - "ready.time.interval": "10s", - "ready.time.delay": "0", - "time.range": ["0", "0"] - } - } - ], - - "evaluate.rule": { - "rules": [ - { - "dsl.type": "spark-sql", - "name": "name-grp", - "rule": "select name, count(*) as `cnt` from source group by name", - "details": { - "persist.type": "metric", - "collect.type": "array" - } - }, - { - "dsl.type": "spark-sql", - "name": "prof", - "rule": "select count(name) as `cnt`, max(age) as `max`, min(age) as `min` from source", - "details": { - "persist.type": "metric" - } - } - ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config-test-profiling-streaming.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config-test-profiling-streaming.json b/measure/src/test/resources/config-test-profiling-streaming.json deleted file mode 100644 index 9f5435e..0000000 --- a/measure/src/test/resources/config-test-profiling-streaming.json +++ /dev/null @@ -1,91 +0,0 @@ -{ - "name": "prof_streaming", - - "process.type": "streaming", - - "data.sources": [ - { - "name": "source", - "connectors": [ - { - "type": "kafka", - "version": "0.8", - "config": { - "kafka.config": { - "bootstrap.servers": "10.149.247.156:9092", - "group.id": "group1", - "auto.offset.reset": "smallest", - "auto.commit.enable": "false" - }, - "topics": "sss", - "key.type": "java.lang.String", - "value.type": "java.lang.String" - }, - "pre.proc": [ - { - "dsl.type": "df-opr", - "name": "${s1}", - "rule": "from_json", - "persist.type": "cache", - "details": { - "df.name": "${this}" - } - }, - { - "dsl.type": "spark-sql", - "name": "${this}", - "rule": "select name, age from ${s1}" - } - ] - } - ], - "cache": { - "file.path": "hdfs://localhost/griffin/streaming/dump/source", - "info.path": "source", - "ready.time.interval": "10s", - "ready.time.delay": "0", - "time.range": ["0", "0"] - } - } - ], - - "evaluateRule": { - "rules": [ - { - "dsl.type": "griffin-dsl", - "dq.type": "profiling", - "rule": "source.name, source.*.count() from source group by source.name", - "details": { - "source": "source", - "profiling": { - "name": "prof", - "persist.type": "metric" - } - } - }, - { - "dsl.type": "griffin-dsl", - "dq.type": "profiling", - "rule": "name.count(), source.age.min(), age.avg(), source.age.max()", - "details": { - "source": "source", - "profiling": { - "persist.type": "metric" - } - } - }, - { - "dsl.type": "griffin-dsl", - "dq.type": "profiling", - "rule": "name.count() as `name-null-count` where source.name IS NULL", - "details": { - "source": "source", - "profiling": { - "name": "null-count", - "persist.type": "metric" - } - } - } - ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config-test-profiling.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config-test-profiling.json b/measure/src/test/resources/config-test-profiling.json deleted file mode 100644 index 7c16f24..0000000 --- a/measure/src/test/resources/config-test-profiling.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "name": "prof_batch_test", - - "process.type": "batch", - - "data.sources": [ - { - "name": "source", - "connectors": [ - { - "type": "avro", - "version": "1.7", - "config": { - "file.name": "src/test/resources/users_info_src.avro" - } - } - ] - } - ], - - "evaluateRule": { - "rules": [ - { - "dsl.type": "griffin-dsl", - "dq.type": "profiling", - "rule": "select user_id as id, user_id.count() as cnt from source group by user_id order by cnt desc, id desc limit 3", - "details": { - "profiling": { - "name": "count", - "persist.type": "metric" - } - } - } - ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config-test-profiling1.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config-test-profiling1.json b/measure/src/test/resources/config-test-profiling1.json deleted file mode 100644 index f1d8788..0000000 --- a/measure/src/test/resources/config-test-profiling1.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "name": "prof_batch_test", - - "process.type": "batch", - - "timestamp": 123456, - - "data.sources": [ - { - "name": "source", - "connectors": [ - { - "type": "avro", - "version": "1.7", - "config": { - "file.name": "src/test/resources/users_info_src.avro" - } - } - ] - } - ], - - "evaluateRule": { - "rules": [ - { - "dsl.type": "griffin-dsl", - "dq.type": "profiling", - "rule": "select count(*) from source", - "details": { - "profiling": { - "persist.type": "metric" - } - } - }, - { - "dsl.type": "griffin-dsl", - "dq.type": "profiling", - "rule": "select count ( distinct source.post_code ) as `dis-cnt` from source", - "details": { - "profiling": { - "name": "dist-name", - "persist.type": "metric" - } - } - }, - { - "dsl.type": "griffin-dsl", - "dq.type": "profiling", - "rule": "source.last_name, count(*) as `cnt` from source group by source.last_name", - "details": { - "profiling": { - "name": "pri", - "persist.type": "metric" - }, - "as.array": true - } - } - ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config-test-profiling2.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config-test-profiling2.json b/measure/src/test/resources/config-test-profiling2.json deleted file mode 100644 index 7a2650f..0000000 --- a/measure/src/test/resources/config-test-profiling2.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "name": "prof_batch_test", - - "process.type": "batch", - - "timestamp": 123456, - - "data.sources": [ - { - "name": "source", - "connectors": [ - { - "type": "avro", - "version": "1.7", - "config": { - "file.name": "src/test/resources/users_info_src.avro" - } - } - ] - } - ], - - "evaluateRule": { - "rules": [ - { - "dsl.type": "spark-sql", - "name": "out", - "rule": "select source.post_code, count(*) as `dist-cnt` from source group by post_code", - "details": { - "persist.type": "metric" - } - } - ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config-test.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config-test.json b/measure/src/test/resources/config-test.json deleted file mode 100644 index 23eb5ff..0000000 --- a/measure/src/test/resources/config-test.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "name": "accu batch test", - - "process.type": "batch", - - "data.sources": [ - { - "name": "source", - "connectors": [ - { - "type": "avro", - "version": "1.7", - "config": { - "file.name": "src/test/resources/users_info_src.avro" - } - } - ] - }, { - "name": "target", - "connectors": [ - { - "type": "avro", - "version": "1.7", - "config": { - "file.name": "src/test/resources/users_info_target.avro" - } - } - ] - } - ], - - "evaluateRule": { - "rules": [ - { - "dsl.type": "spark-sql", - "name": "miss.records", - "rule": "SELECT source.user_id, source.first_name, source.last_name, source.address, source.email, source.phone, source.post_code FROM source LEFT JOIN target ON coalesce(source.user_id, 'null') = coalesce(target.user_id, 'null') AND coalesce(source.first_name, 'null') = coalesce(target.first_name, 'null') AND coalesce(source.last_name, 'null') = coalesce(target.last_name, 'null') AND coalesce(source.address, 'null') = coalesce(target.address, 'null') AND coalesce(source.email, 'null') = coalesce(target.email, 'null') AND coalesce(source.phone, 'null') = coalesce(target.phone, 'null') AND coalesce(source.post_code, 'null') = coalesce(target.post_code, 'null') WHERE (NOT (source.user_id IS NULL AND source.first_name IS NULL AND source.last_name IS NULL AND source.address IS NULL AND source.email IS NULL AND source.phone IS NULL AND source.post_code IS NULL)) AND (target.user_id IS NULL AND target.first_name IS NULL AND target.last_name IS NULL AND target.address IS NULL AND t arget.email IS NULL AND target.phone IS NULL AND target.post_code IS NULL)", - "persist.type": "record" - }, { - "dsl.type": "spark-sql", - "name": "miss", - "rule": "SELECT COUNT(*) AS `miss` FROM `miss.records`", - }, { - "dsl.type": "spark-sql", - "name": "total", - "rule": "SELECT COUNT(*) AS `total` FROM source", - }, { - "dsl.type": "spark-sql", - "name": "accuracy", - "rule": "SELECT `total`.`total` AS `total`, `miss`.`miss` AS `miss`, (`total`.`total` - `miss`.`miss`) AS `matched` FROM total JOIN miss", - "persist.type": "metric" - } - ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config-test1.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config-test1.json b/measure/src/test/resources/config-test1.json deleted file mode 100644 index 53a8765..0000000 --- a/measure/src/test/resources/config-test1.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "name": "accu batch test", - - "process.type": "batch", - - "data.sources": [ - { - "name": "source", - "connectors": [ - { - "type": "hive", - "version": "1.2", - "config": { - "database": "default", - "table.name": "src" - } - } - ] - }, { - "name": "target", - "connectors": [ - { - "type": "hive", - "version": "1.2", - "config": { - "database": "default", - "table.name": "tgt" - } - } - ] - } - ], - - "evaluateRule": { - "rules": [ - { - "dsl.type": "df-opr", - "name": "source", - "rule": "from_json", - "details": { - "df.name": "source" - } - }, - { - "dsl.type": "spark-sql", - "name": "seeds", - "rule": "SELECT explode(seeds) as seed FROM source" - }, - { - "dsl.type": "df-opr", - "name": "seeds", - "rule": "from_json", - "details": { - "df.name": "seeds", - "col.name": "seed" - } - }, - { - "dsl.type": "spark-sql", - "name": "source", - "rule": "SELECT url, get_json_object(metadata, '$.tracker.crawlRequestCreateTS') AS ts FROM seeds" - }, - { - "dsl.type": "spark-opr", - "name": "target", - "rule": "from_json(target.value)" - }, - { - "dsl.type": "spark-sql", - "name": "attrs", - "rule": "SELECT groups[0].attrsList AS attrs FROM target" - }, - { - "dsl.type": "spark-sql", - "name": "target", - "rule": "SELECT attrs.values[index_of(attrs.name, 'URL')][0] AS url, get_json_object(attrs.values[index_of(attrs.name, 'CRAWLMETADATA')][0], '$.tracker.crawlRequestCreateTS') AS ts FROM df2" - }, - { - "dsl.type": "spark-sql", - "name": "miss.record", - "rule": "SELECT source.url, source.ts FROM source LEFT JOIN target ON coalesce(source.url, '') = coalesce(target.url, '') AND coalesce(source.ts, '') = coalesce(target.ts, '') WHERE (NOT (source.url IS NULL AND source.ts IS NULL)) AND (target.url IS NULL AND target.ts IS NULL)", - "persist.type": "record" - }, { - "dsl.type": "spark-sql", - "name": "miss.count", - "rule": "SELECT COUNT(*) AS `miss.count` FROM `miss.record`", - "persist.type": "metric" - }, { - "dsl.type": "spark-sql", - "name": "total.count", - "rule": "SELECT COUNT(*) AS `total.count` FROM source", - "persist.type": "metric" - } - ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config.json b/measure/src/test/resources/config.json index 0a17474..99687b3 100644 --- a/measure/src/test/resources/config.json +++ b/measure/src/test/resources/config.json @@ -1,27 +1,71 @@ { - "name": "accu1", - "type": "accuracy", + "name": "accu_batch", "process.type": "batch", - "source": { - "type": "avro", - "version": "1.7", - "config": { - "file.name": "src/test/resources/users_info_src.avro" + "data.sources": [ + { + "name": "source", + "baseline": true, + "connectors": [ + { + "type": "avro", + "version": "1.7", + "config": { + "file.name": "src/test/resources/users_info_src.avro" + }, + "pre.proc": [ + { + "dsl.type": "spark-sql", + "name": "${this}", + "rule": "select * from ${this} where user_id >= 10044" + } + ] + } + ] + }, { + "name": "target", + "connectors": [ + { + "type": "avro", + "version": "1.7", + "config": { + "file.name": "src/test/resources/users_info_target.avro" + } + } + ] } - }, + ], - "target": { - "type": "avro", - "version": "1.7", - "config": { - "file.name": "src/test/resources/users_info_target.avro" - } - }, - - "evaluateRule": { - "sampleRatio": 1, - "rules": "$source.user_id = $target.user_id AND $source.first_name = $target.first_name AND $source.last_name = $target.last_name AND $source.address = $target.address AND $source.email = $target.email AND $source.phone = $target.phone AND $source.post_code = $target.post_code" + "evaluate.rule": { + "rules": [ + { + "dsl.type": "spark-sql", + "name": "missRecords", + "rule": "SELECT source.* FROM source LEFT JOIN target ON coalesce(source.user_id, '') = coalesce(target.user_id, '') AND coalesce(source.first_name, '') = coalesce(target.first_name, '') AND coalesce(source.post_code, '') = coalesce(target.post_code, '') WHERE (NOT (source.user_id IS NULL AND source.first_name IS NULL AND source.post_code IS NULL)) AND (target.user_id IS NULL AND target.first_name IS NULL AND target.post_code IS NULL)", + "record": { + "name": "miss" + }, + "cache": true + }, + { + "dsl.type": "spark-sql", + "name": "miss_count", + "rule": "SELECT count(*) as miss FROM `missRecords`" + }, + { + "dsl.type": "spark-sql", + "name": "total_count", + "rule": "SELECT count(*) as total FROM source" + }, + { + "dsl.type": "spark-sql", + "name": "accu", + "rule": "SELECT `total_count`.`total` AS `total`, coalesce(`miss_count`.`miss`, 0) AS `miss`, (`total` - `miss`) AS `matched` FROM `total_count` FULL JOIN `miss_count`", + "metric": { + "name": "accu" + } + } + ] } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/config1.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/config1.json b/measure/src/test/resources/config1.json deleted file mode 100644 index 883f4e2..0000000 --- a/measure/src/test/resources/config1.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "name": "accu-test", - "type": "accuracy", - - "process.type": "batch", - - "source": { - "type": "hive", - "version": "1.2", - "config": { - "table.name": "rheos_view_event", - "where": "dt=20170410 AND hour=15" - } - }, - - "target": { - "type": "hive", - "version": "1.2", - "config": { - "table.name": "be_view_event_queue", - "where": "dt=20170410 AND hour=15, dt=20170410 AND hour=16" - } - }, - - "evaluateRule": { - "sampleRatio": 1, - "rules": "@Key ${source}['uid'] === ${target}['uid']; @Key ${source}['eventtimestamp'] === ${target}['eventtimestamp']; ${source}['page_id'] === ${target}['page_id']; ${source}['site_id'] === ${target}['site_id']; ${source}['itm'] === ${target}['itm']" - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/dupdata.avro ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/dupdata.avro b/measure/src/test/resources/dupdata.avro deleted file mode 100644 index f6bd312..0000000 Binary files a/measure/src/test/resources/dupdata.avro and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/empty.avro ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/empty.avro b/measure/src/test/resources/empty.avro deleted file mode 100644 index 1ac3a72..0000000 Binary files a/measure/src/test/resources/empty.avro and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/env-hdfs-test.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/env-hdfs-test.json b/measure/src/test/resources/env-hdfs-test.json deleted file mode 100644 index 2f67e44..0000000 --- a/measure/src/test/resources/env-hdfs-test.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "spark": { - "log.level": "WARN", - "checkpoint.dir": "hdfs:///griffin/batch/cp", - "batch.interval": "10s", - "process.interval": "10m", - "config": { - "spark.master": "local[*]" - } - }, - - "persist": [ - { - "type": "log", - "config": { - "max.log.lines": 10 - } - }, - { - "type": "hdfs", - "config": { - "path": "hdfs://localhost/griffin/test", - "max.lines.per.file": 10000 - } - } - ], - - "info.cache": [ - { - "type": "zk", - "config": { - "hosts": "localhost:2181", - "namespace": "griffin/infocache", - "lock.path": "lock", - "mode": "persist", - "init.clear": true, - "close.clear": false - } - } - ], - - "cleaner": { - - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/env-streaming-mongo.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/env-streaming-mongo.json b/measure/src/test/resources/env-streaming-mongo.json deleted file mode 100644 index 0d50462..0000000 --- a/measure/src/test/resources/env-streaming-mongo.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "spark": { - "log.level": "WARN", - "checkpoint.dir": "hdfs://localhost/test/griffin/cp", - "batch.interval": "2s", - "process.interval": "10s", - "config": { - "spark.master": "local[*]", - "spark.task.maxFailures": 5, - "spark.streaming.kafkaMaxRatePerPartition": 1000, - "spark.streaming.concurrentJobs": 4, - "spark.yarn.maxAppAttempts": 5, - "spark.yarn.am.attemptFailuresValidityInterval": "1h", - "spark.yarn.max.executor.failures": 120, - "spark.yarn.executor.failuresValidityInterval": "1h", - "spark.hadoop.fs.hdfs.impl.disable.cache": true - } - }, - - "persist": [ - { - "type": "log", - "config": { - "max.log.lines": 100 - } - }, - { - "type": "mongo", - "config": { - "url": "10.149.247.156", - "database": "test", - "collection": "sss" - } - } - ], - - "info.cache": [ - { - "type": "zk", - "config": { - "hosts": "localhost:2181", - "namespace": "griffin/infocache", - "lock.path": "lock", - "mode": "persist", - "init.clear": true, - "close.clear": false - } - } - ], - - "cleaner": { - "clean.interval": "2m" - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/env-test.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/env-test.json b/measure/src/test/resources/env-test.json deleted file mode 100644 index 4a8e3d0..0000000 --- a/measure/src/test/resources/env-test.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "spark": { - "log.level": "WARN", - "checkpoint.dir": "hdfs:///griffin/batch/cp", - "batch.interval": "10s", - "process.interval": "10m", - "config": { - "spark.master": "local[*]" - }, - "init.clear": true - }, - - "persist": [ - { - "type": "log", - "config": { - "max.log.lines": 10 - } - } - ], - - "info.cache": [ - { - "type": "zk", - "config": { - "hosts": "localhost:2181", - "namespace": "griffin/infocache", - "lock.path": "lock", - "mode": "persist", - "init.clear": true, - "close.clear": false - } - } - ], - - "cleaner": { - - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/env.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/env.json b/measure/src/test/resources/env.json index 14e3b75..4a8e3d0 100644 --- a/measure/src/test/resources/env.json +++ b/measure/src/test/resources/env.json @@ -1,26 +1,20 @@ { "spark": { - "log.level": "INFO", + "log.level": "WARN", "checkpoint.dir": "hdfs:///griffin/batch/cp", "batch.interval": "10s", "process.interval": "10m", - "config": {} + "config": { + "spark.master": "local[*]" + }, + "init.clear": true }, "persist": [ { - "type": "hdfs", - "config": { - "path": "hdfs:///griffin/streaming/persist", - "max.persist.lines": 10000, - "max.lines.per.file": 10000 - } - }, - { - "type": "http", + "type": "log", "config": { - "method": "post", - "api": "http://HOSTNAME:9200/griffin/accuracy"; + "max.log.lines": 10 } } ], http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/env1.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/env1.json b/measure/src/test/resources/env1.json deleted file mode 100644 index a059715..0000000 --- a/measure/src/test/resources/env1.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "spark": { - "log.level": "INFO", - "checkpoint.dir": "hdfs:///griffin/batch/cp", - "config": {} - }, - - "persist": [ - { - "type": "hdfs", - "config": { - "path": "hdfs:///user/b_des/bark/griffin-batch/test", - "max.lines.per.file": 10000 - } - } - ], - - "cleaner": { - - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/performance-test-accuracy.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/performance-test-accuracy.json b/measure/src/test/resources/performance-test-accuracy.json deleted file mode 100644 index 035e4ac..0000000 --- a/measure/src/test/resources/performance-test-accuracy.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "name": "accu_batch_test", - - "process.type": "batch", - - "data.sources": [ - { - "name": "source", - "baseline": true, - "connectors": [ - { - "type": "HIVE", - "version": "1.2", - "config": { - "table.name": "data_avr_big", - "where": "pt=2" - } - } - ] - }, - { - "name": "target", - "connectors": [ - { - "type": "HIVE", - "version": "1.2", - "config": { - "table.name": "data_rdm" - } - } - ] - } - ], - - "evaluate.rule": { - "rules": [ - { - "dsl.type": "griffin-dsl", - "dq.type": "accuracy", - "name": "accuracy", - "rule": "source.uid = target.uid AND source.uage = target.uage AND source.udes = target.udes", - "details": { - "persist.type": "metric", - "source": "source", - "target": "target", - "miss": "miss_count", - "total": "total_count", - "matched": "matched_count", - "miss.records": { - "persist.type": "record" - } - } - } - ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/performance-test-profiling.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/performance-test-profiling.json b/measure/src/test/resources/performance-test-profiling.json deleted file mode 100644 index 0b22d75..0000000 --- a/measure/src/test/resources/performance-test-profiling.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "name": "prof_batch_test", - - "process.type": "batch", - - "data.sources": [ - { - "name": "source", - "connectors": [ - { - "type": "HIVE", - "version": "1.2", - "config": { - "table.name": "data_avr_big", - "where": "pt <= 100" - } - } - ] - } - ], - - "evaluate.rule": { - "rules": [ - { - "dsl.type": "griffin-dsl", - "dq.type": "profiling", - "rule": "count(*) as `cnt` from source where uid > 100", - "details": { - "persist.type": "metric" - } - } - ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/1d7acd57/measure/src/test/resources/test-data0.json ---------------------------------------------------------------------- diff --git a/measure/src/test/resources/test-data0.json b/measure/src/test/resources/test-data0.json deleted file mode 100644 index 406acb8..0000000 --- a/measure/src/test/resources/test-data0.json +++ /dev/null @@ -1,56 +0,0 @@ -[ - { - "name": "emily", - "age": 5, - "map": { - "a": 1, - "b": 2 - }, - "list": [ - { - "c": 1, - "d": 2 - }, - { - "c": 3, - "d": 4 - } - ] - }, - { - "name": "white", - "age": 15, - "map": { - "a": 11, - "b": 12 - }, - "list": [ - { - "c": 11, - "d": 2 - }, - { - "c": 23, - "d": 4 - } - ] - }, - { - "name": "west", - "age": 25, - "map": { - "a": 21, - "b": 22 - }, - "list": [ - { - "c": 11, - "d": 2 - }, - { - "c": 23, - "d": 4 - } - ] - } -] \ No newline at end of file
