Repository: incubator-griffin Updated Branches: refs/heads/master b82702e00 -> 8e0afa45a
fix livy backquote bug and add rule details Author: ahutsunshine <[email protected]> Closes #177 from ahutsunshine/master. Project: http://git-wip-us.apache.org/repos/asf/incubator-griffin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-griffin/commit/8e0afa45 Tree: http://git-wip-us.apache.org/repos/asf/incubator-griffin/tree/8e0afa45 Diff: http://git-wip-us.apache.org/repos/asf/incubator-griffin/diff/8e0afa45 Branch: refs/heads/master Commit: 8e0afa45a591634160147a80d2beeb00d39f7c38 Parents: b82702e Author: ahutsunshine <[email protected]> Authored: Wed Nov 22 14:37:54 2017 +0800 Committer: Lionel Liu <[email protected]> Committed: Wed Nov 22 14:37:54 2017 +0800 ---------------------------------------------------------------------- griffin-doc/postman/griffin.json | 7 +++++++ .../apache/griffin/core/job/SparkSubmitJob.java | 18 +++++++++++------ .../measure/entity/AbstractAuditableEntity.java | 4 ++-- .../griffin/core/measure/entity/Rule.java | 21 ++++++++++++++++++-- .../griffin/core/measure/MeasureTestHelper.java | 2 +- 5 files changed, 41 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/8e0afa45/griffin-doc/postman/griffin.json ---------------------------------------------------------------------- diff --git a/griffin-doc/postman/griffin.json b/griffin-doc/postman/griffin.json index 7b64552..7c5905d 100644 --- a/griffin-doc/postman/griffin.json +++ b/griffin-doc/postman/griffin.json @@ -1093,10 +1093,17 @@ "tests": null, "currentHelper": "normal", "helperAttributes": {}, +<<<<<<< HEAD + "time": 1509333182624, + "name": "Update measure", + "description": "`PUT /api/v1/measure`\n\n#### Request Header\nkey | value\n--- | ---\nContent-Type | application/json\n\n#### Request Body\n\nname | description | type\n--- | --- | --- \nmeasure | measure entity | Measure\n\n#### Response Body Sample\n```\n{\n \"code\": 204,\n \"description\": \"Update Measure Succeed\"\n}\n```\n\nIt may return failed messages.Such as,\n\n```\n {\n \"code\": 400,\n \"description\": \"Resource Not Found\"\n}\n\n```\n\nThe reason for failure may be that measure id doesn't exist or the measure has been deleted by logically.You should check your measure.", + "collectionId": "689bb3f2-1c6a-b45e-5409-4df1ef07554c", +======= "time": 1509332871323, "name": "Get measure names group by org", "description": "`GET /api/v1/org/measure/names`", "collectionId": "871762c3-97f9-1ac0-f17c-d17bd3446b87", +>>>>>>> b82702e00868eefe216915ff3064fca176a2d18e "responses": [ { "status": "", http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/8e0afa45/service/src/main/java/org/apache/griffin/core/job/SparkSubmitJob.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/job/SparkSubmitJob.java b/service/src/main/java/org/apache/griffin/core/job/SparkSubmitJob.java index d5502e5..d620cac 100644 --- a/service/src/main/java/org/apache/griffin/core/job/SparkSubmitJob.java +++ b/service/src/main/java/org/apache/griffin/core/job/SparkSubmitJob.java @@ -221,17 +221,22 @@ public class SparkSubmitJob implements Job { return currentBlockStartTimestamp; } + private String escapeCharacter(String str, String regex) { + String escapeCh = "\\" + regex; + return str.replaceAll(regex, escapeCh); + } + private void setSparkJobDO() { sparkJobDO.setFile(sparkJobProps.getProperty("sparkJob.file")); sparkJobDO.setClassName(sparkJobProps.getProperty("sparkJob.className")); List<String> args = new ArrayList<>(); args.add(sparkJobProps.getProperty("sparkJob.args_1")); - // measure - String measureJson; measure.setTriggerTimeStamp(System.currentTimeMillis()); - measureJson = JsonUtil.toJsonWithFormat(measure); - args.add(measureJson); + String measureJson = JsonUtil.toJsonWithFormat(measure); + // to fix livy bug: ` will be ignored by livy + String finalMeasureJson = escapeCharacter(measureJson, "\\`"); + args.add(finalMeasureJson); args.add(sparkJobProps.getProperty("sparkJob.args_3")); sparkJobDO.setArgs(args); @@ -257,7 +262,8 @@ public class SparkSubmitJob implements Job { } public void saveJobInstance(String groupName, String jobName, String result) { - TypeReference<HashMap<String, Object>> type = new TypeReference<HashMap<String, Object>>() {}; + TypeReference<HashMap<String, Object>> type = new TypeReference<HashMap<String, Object>>() { + }; try { Map<String, Object> resultMap = JsonUtil.toEntity(result, type); if (resultMap != null) { @@ -271,7 +277,7 @@ public class SparkSubmitJob implements Job { } } - private JobInstance genJobInstance(String groupName, String jobName, Map<String, Object> resultMap) throws IllegalArgumentException{ + private JobInstance genJobInstance(String groupName, String jobName, Map<String, Object> resultMap) throws IllegalArgumentException { JobInstance jobInstance = new JobInstance(); jobInstance.setGroupName(groupName); jobInstance.setJobName(jobName); http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/8e0afa45/service/src/main/java/org/apache/griffin/core/measure/entity/AbstractAuditableEntity.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/measure/entity/AbstractAuditableEntity.java b/service/src/main/java/org/apache/griffin/core/measure/entity/AbstractAuditableEntity.java index 548c4dc..015633e 100644 --- a/service/src/main/java/org/apache/griffin/core/measure/entity/AbstractAuditableEntity.java +++ b/service/src/main/java/org/apache/griffin/core/measure/entity/AbstractAuditableEntity.java @@ -38,10 +38,10 @@ public abstract class AbstractAuditableEntity implements Serializable { private Long id; @JsonIgnore - Timestamp createdDate = new Timestamp(System.currentTimeMillis()); + private Timestamp createdDate = new Timestamp(System.currentTimeMillis()); @JsonIgnore - Timestamp modifiedDate; + private Timestamp modifiedDate; public Long getId() { return id; http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/8e0afa45/service/src/main/java/org/apache/griffin/core/measure/entity/Rule.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/measure/entity/Rule.java b/service/src/main/java/org/apache/griffin/core/measure/entity/Rule.java index ebc35d3..48c65d2 100644 --- a/service/src/main/java/org/apache/griffin/core/measure/entity/Rule.java +++ b/service/src/main/java/org/apache/griffin/core/measure/entity/Rule.java @@ -19,7 +19,9 @@ under the License. package org.apache.griffin.core.measure.entity; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.griffin.core.util.JsonUtil; import javax.persistence.Column; import javax.persistence.Entity; @@ -28,7 +30,9 @@ import javax.persistence.Entity; @Entity public class Rule extends AbstractAuditableEntity { - /**three type:1.griffin-dsl 2.df-opr 3.spark-sql**/ + /** + * three type:1.griffin-dsl 2.df-opr 3.spark-sql + */ private String dslType; private String dqType; @@ -36,6 +40,10 @@ public class Rule extends AbstractAuditableEntity { @Column(length = 1024) private String rule; + @JsonInclude(JsonInclude.Include.NON_NULL) + private String details; + + @JsonProperty("dsl.type") public String getDslType() { return dslType; @@ -64,12 +72,21 @@ public class Rule extends AbstractAuditableEntity { this.rule = rule; } + public String getDetails() { + return details; + } + + public void setDetails(Object details) { + this.details = JsonUtil.toJson(details); + } + public Rule() { } - public Rule(String dslType, String dqType, String rule) { + public Rule(String dslType, String dqType, String rule, String details) { this.dslType = dslType; this.dqType = dqType; this.rule = rule; + this.details = details; } } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/8e0afa45/service/src/test/java/org/apache/griffin/core/measure/MeasureTestHelper.java ---------------------------------------------------------------------- diff --git a/service/src/test/java/org/apache/griffin/core/measure/MeasureTestHelper.java b/service/src/test/java/org/apache/griffin/core/measure/MeasureTestHelper.java index c9f1c7b..ad727f7 100644 --- a/service/src/test/java/org/apache/griffin/core/measure/MeasureTestHelper.java +++ b/service/src/test/java/org/apache/griffin/core/measure/MeasureTestHelper.java @@ -47,7 +47,7 @@ public class MeasureTestHelper { dataSources.add(dataSource); dataSources.add(targetSource); String rules = "source.id=target.id AND source.name=target.name AND source.age=target.age"; - Rule rule = new Rule("griffin-dsl", "accuracy", rules); + Rule rule = new Rule("griffin-dsl", "accuracy", rules,null); EvaluateRule evaluateRule = new EvaluateRule(Arrays.asList(rule)); return new Measure(name, "description", org, "batch", "test", dataSources, evaluateRule); }
