Repository: incubator-griffin Updated Branches: refs/heads/master d063c03ee -> fdcd7f75d
fix predicate path not replace bug and update docker document Author: ahutsunshine <[email protected]> Author: He Wang <[email protected]> Author: dodobel <[email protected]> Closes #215 from ahutsunshine/master. Project: http://git-wip-us.apache.org/repos/asf/incubator-griffin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-griffin/commit/fdcd7f75 Tree: http://git-wip-us.apache.org/repos/asf/incubator-griffin/tree/fdcd7f75 Diff: http://git-wip-us.apache.org/repos/asf/incubator-griffin/diff/fdcd7f75 Branch: refs/heads/master Commit: fdcd7f75da2916d3a234691e586bd24de7401f67 Parents: d063c03 Author: ahutsunshine <[email protected]> Authored: Wed Feb 7 16:17:39 2018 +0800 Committer: Lionel Liu <[email protected]> Committed: Wed Feb 7 16:17:39 2018 +0800 ---------------------------------------------------------------------- griffin-doc/docker/griffin-docker-guide.md | 2 +- .../org/apache/griffin/core/job/JobInstance.java | 17 ++++++++--------- .../apache/griffin/core/job/JobServiceImpl.java | 2 +- .../apache/griffin/core/job/SparkSubmitJob.java | 14 +++++++------- .../griffin/core/job/entity/JobDataSegment.java | 2 +- .../griffin/core/job/entity/JobSchedule.java | 6 +++--- .../griffin/core/measure/entity/DataConnector.java | 2 +- .../org/apache/griffin/core/util/TimeUtil.java | 4 ++-- 8 files changed, 24 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/fdcd7f75/griffin-doc/docker/griffin-docker-guide.md ---------------------------------------------------------------------- diff --git a/griffin-doc/docker/griffin-docker-guide.md b/griffin-doc/docker/griffin-docker-guide.md index 1fb5980..2336743 100644 --- a/griffin-doc/docker/griffin-docker-guide.md +++ b/griffin-doc/docker/griffin-docker-guide.md @@ -54,7 +54,7 @@ Griffin docker images are pre-built on docker hub, users can pull them to try gr ``` docker-compose -f docker-compose-batch.yml up -d ``` -3. Now you can try griffin APIs by using postman after importing the [json files](https://github.com/apache/incubator-griffin/blob/master/griffin-doc/postman). +3. Now you can try griffin APIs by using postman after importing the [json files](https://github.com/apache/incubator-griffin/tree/master/griffin-doc/service/postman). In which you need to modify the environment `BASE_PATH` value into `<your local IP address>:38080`. 4. You can try the api `Basic -> Get griffin version`, to make sure griffin service has started up. 5. Add an accuracy measure through api `Measures -> Add measure`, to create a measure in griffin. http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/fdcd7f75/service/src/main/java/org/apache/griffin/core/job/JobInstance.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/job/JobInstance.java b/service/src/main/java/org/apache/griffin/core/job/JobInstance.java index 5cffceb..8df02af 100644 --- a/service/src/main/java/org/apache/griffin/core/job/JobInstance.java +++ b/service/src/main/java/org/apache/griffin/core/job/JobInstance.java @@ -179,7 +179,7 @@ public class JobInstance implements Job { private void setConnectorPredicates(DataConnector dc, Long[] sampleTs) throws IOException { List<SegmentPredicate> predicates = dc.getPredicates(); for (SegmentPredicate predicate : predicates) { - genConfMap(dc, sampleTs); + genConfMap(predicate.getConfigMap(), sampleTs,dc.getDataTimeZone()); //Do not forget to update origin string config predicate.setConfigMap(predicate.getConfigMap()); mPredicates.add(predicate); @@ -187,19 +187,18 @@ public class JobInstance implements Job { } private void setConnectorConf(DataConnector dc, Long[] sampleTs) throws IOException { - genConfMap(dc, sampleTs); + genConfMap(dc.getConfigMap(), sampleTs,dc.getDataTimeZone()); dc.setConfigMap(dc.getConfigMap()); } /** - * @param dc data connector + * @param conf config map * @param sampleTs collection of data split start timestamp * @return all config data combine,like {"where": "year=2017 AND month=11 AND dt=15 AND hour=09,year=2017 AND month=11 AND dt=15 AND hour=10"} * or like {"path": "/year=2017/month=11/dt=15/hour=09/_DONE,/year=2017/month=11/dt=15/hour=10/_DONE"} */ - private void genConfMap(DataConnector dc, Long[] sampleTs) { - Map<String, String> conf = dc.getConfigMap(); + private void genConfMap(Map<String, String> conf, Long[] sampleTs,String timezone) { if (conf == null) { LOGGER.warn("Predicate config is null."); return; @@ -211,17 +210,17 @@ public class JobInstance implements Job { continue; } for (Long timestamp : sampleTs) { - set.add(TimeUtil.format(value, timestamp, getTimeZone(dc))); + set.add(TimeUtil.format(value, timestamp, getTimeZone(timezone))); } conf.put(entry.getKey(), StringUtils.join(set, PATH_CONNECTOR_CHARACTER)); } } - private TimeZone getTimeZone(DataConnector dc) { - if (StringUtils.isEmpty(dc.getDataTimeZone())) { + private TimeZone getTimeZone(String timezone) { + if (StringUtils.isEmpty(timezone)) { return TimeZone.getDefault(); } - return TimeZone.getTimeZone(dc.getDataTimeZone()); + return TimeZone.getTimeZone(timezone); } private boolean createJobInstance(Map<String, Object> confMap) throws Exception { http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/fdcd7f75/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java b/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java index b2faa72..af1312c 100644 --- a/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java +++ b/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java @@ -480,7 +480,7 @@ public class JobServiceImpl implements JobService { HashMap<String, Object> resultMap = JsonUtil.toEntity(resultStr, type); setJobInstanceIdAndUri(jobInstance, resultMap); } catch (RestClientException e) { - LOGGER.error("Spark session {} has overdue, set state as unknown!\n {}", jobInstance.getSessionId(), e.getMessage()); + LOGGER.warn("Spark session {} has overdue, set state as unknown!\n {}", jobInstance.getSessionId(), e.getMessage()); setJobInstanceUnknownStatus(jobInstance); } catch (IOException e) { LOGGER.error("Job instance json converts to map failed. {}", e.getMessage()); http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/fdcd7f75/service/src/main/java/org/apache/griffin/core/job/SparkSubmitJob.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/job/SparkSubmitJob.java b/service/src/main/java/org/apache/griffin/core/job/SparkSubmitJob.java index f1b254a..88ea11d 100644 --- a/service/src/main/java/org/apache/griffin/core/job/SparkSubmitJob.java +++ b/service/src/main/java/org/apache/griffin/core/job/SparkSubmitJob.java @@ -124,11 +124,11 @@ public class SparkSubmitJob implements Job { livyUri = livyConfProps.getProperty("livy.uri"); jobInstance = jobInstanceRepo.findByPredicateName(jd.getJobDataMap().getString(PREDICATE_JOB_NAME)); measure = JsonUtil.toEntity(jd.getJobDataMap().getString(MEASURE_KEY), GriffinMeasure.class); - setPredicts(jd.getJobDataMap().getString(PREDICATES_KEY)); + setPredicates(jd.getJobDataMap().getString(PREDICATES_KEY)); setMeasureInstanceName(measure, jd); } - private void setPredicts(String json) throws IOException { + private void setPredicates(String json) throws IOException { if (StringUtils.isEmpty(json)) { return; } @@ -197,13 +197,13 @@ public class SparkSubmitJob implements Job { private void saveJobInstance(JobDetail jd) throws SchedulerException, IOException { String result = post2Livy(); - boolean pauseStatus = false; if (result != null) { - jobService.pauseJob(jd.getKey().getGroup(), jd.getKey().getName()); - pauseStatus = true; - LOGGER.info("Delete predicate job {} success"); + String group = jd.getKey().getGroup(); + String name = jd.getKey().getName(); + jobService.pauseJob(group, name); + LOGGER.info("Delete predicate job({},{}) success.", group, name); } - saveJobInstance(result, LivySessionStates.State.found, pauseStatus); + saveJobInstance(result, LivySessionStates.State.found, true); } private void saveJobInstance(String result, LivySessionStates.State state, Boolean pauseStatus) throws IOException { http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/fdcd7f75/service/src/main/java/org/apache/griffin/core/job/entity/JobDataSegment.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/job/entity/JobDataSegment.java b/service/src/main/java/org/apache/griffin/core/job/entity/JobDataSegment.java index e765702..c81c733 100644 --- a/service/src/main/java/org/apache/griffin/core/job/entity/JobDataSegment.java +++ b/service/src/main/java/org/apache/griffin/core/job/entity/JobDataSegment.java @@ -70,7 +70,7 @@ public class JobDataSegment extends AbstractAuditableEntity { @JsonProperty("data.connector.name") public void setDataConnectorName(String dataConnectorName) { if (StringUtils.isEmpty(dataConnectorName)) { - LOGGER.error(" Data connector name is invalid. Please check your connector name."); + LOGGER.warn(" Data connector name is invalid. Please check your connector name."); throw new NullPointerException(); } this.dataConnectorName = dataConnectorName; http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/fdcd7f75/service/src/main/java/org/apache/griffin/core/job/entity/JobSchedule.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/job/entity/JobSchedule.java b/service/src/main/java/org/apache/griffin/core/job/entity/JobSchedule.java index 0a1c2c4..5116805 100644 --- a/service/src/main/java/org/apache/griffin/core/job/entity/JobSchedule.java +++ b/service/src/main/java/org/apache/griffin/core/job/entity/JobSchedule.java @@ -84,7 +84,7 @@ public class JobSchedule extends AbstractAuditableEntity { @JsonProperty("job.name") public void setJobName(String jobName) { if (StringUtils.isEmpty(jobName)) { - LOGGER.error("Job name cannot be empty."); + LOGGER.warn("Job name cannot be empty."); throw new NullPointerException(); } this.jobName = jobName; @@ -98,7 +98,7 @@ public class JobSchedule extends AbstractAuditableEntity { @JsonProperty("cron.expression") public void setCronExpression(String cronExpression) { if (StringUtils.isEmpty(cronExpression) || !isCronExpressionValid(cronExpression)) { - LOGGER.error("Cron expression is invalid.Please check your cron expression."); + LOGGER.warn("Cron expression is invalid.Please check your cron expression."); throw new IllegalArgumentException(); } this.cronExpression = cronExpression; @@ -165,7 +165,7 @@ public class JobSchedule extends AbstractAuditableEntity { private boolean isCronExpressionValid(String cronExpression) { if (!CronExpression.isValidExpression(cronExpression)) { - LOGGER.error("Cron expression {} is invalid.", cronExpression); + LOGGER.warn("Cron expression {} is invalid.", cronExpression); return false; } return true; http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/fdcd7f75/service/src/main/java/org/apache/griffin/core/measure/entity/DataConnector.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/measure/entity/DataConnector.java b/service/src/main/java/org/apache/griffin/core/measure/entity/DataConnector.java index dae35e1..d126f5b 100644 --- a/service/src/main/java/org/apache/griffin/core/measure/entity/DataConnector.java +++ b/service/src/main/java/org/apache/griffin/core/measure/entity/DataConnector.java @@ -136,7 +136,7 @@ public class DataConnector extends AbstractAuditableEntity { public void setName(String name) { if (StringUtils.isEmpty(name)) { - LOGGER.error("Connector name cannot be empty."); + LOGGER.warn("Connector name cannot be empty."); throw new NullPointerException(); } this.name = name; http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/fdcd7f75/service/src/main/java/org/apache/griffin/core/util/TimeUtil.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/util/TimeUtil.java b/service/src/main/java/org/apache/griffin/core/util/TimeUtil.java index 75f068d..5de3889 100644 --- a/service/src/main/java/org/apache/griffin/core/util/TimeUtil.java +++ b/service/src/main/java/org/apache/griffin/core/util/TimeUtil.java @@ -51,7 +51,7 @@ public class TimeUtil { public static Long str2Long(String timeStr) { if (timeStr == null) { - LOGGER.error("Time string can not be empty."); + LOGGER.warn("Time string can not be empty."); return 0L; } String trimTimeStr = timeStr.trim(); @@ -106,7 +106,7 @@ public class TimeUtil { } else if (unit.matches(DAYS_PATTERN)) { return milliseconds(t, TimeUnit.DAYS); } else { - LOGGER.error("Time string format error.It only supports d(day),h(hour),m(minute),s(second),ms(millsecond).Please check your time format."); + LOGGER.warn("Time string format error.It only supports d(day),h(hour),m(minute),s(second),ms(millsecond).Please check your time format."); return 0L; } }
