http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/griffin-doc/postman/griffin_environment.json ---------------------------------------------------------------------- diff --git a/griffin-doc/postman/griffin_environment.json b/griffin-doc/postman/griffin_environment.json new file mode 100644 index 0000000..9a3da5b --- /dev/null +++ b/griffin-doc/postman/griffin_environment.json @@ -0,0 +1,16 @@ +{ + "id": "b0a42a84-0418-4bb6-226d-ca9d6d5f23d7", + "name": "Griffin Environment", + "values": [ + { + "enabled": true, + "key": "BASE_PATH", + "value": "http://localhost:8080";, + "type": "text" + } + ], + "timestamp": 1508998036167, + "_postman_variable_scope": "environment", + "_postman_exported_at": "2017-10-30T01:58:11.275Z", + "_postman_exported_using": "Postman/5.3.2" +} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/common/CacheEvictor.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/common/CacheEvictor.java b/service/src/main/java/org/apache/griffin/core/common/CacheEvictor.java index 2e32e1c..c40f4e1 100644 --- a/service/src/main/java/org/apache/griffin/core/common/CacheEvictor.java +++ b/service/src/main/java/org/apache/griffin/core/common/CacheEvictor.java @@ -23,7 +23,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.annotation.CacheEvict; -import org.springframework.context.annotation.Bean; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; @@ -39,6 +38,6 @@ public class CacheEvictor { public void evictHiveCache() { LOGGER.info("Evict hive cache"); hiveMetaStoreService.getAllTable(); - LOGGER.info("After evict hive cache,automatically get hive tables to cache again."); + LOGGER.info("After evict hive cache,automatically refresh hive tables cache."); } } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/config/SwaggerConfig.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/config/SwaggerConfig.java b/service/src/main/java/org/apache/griffin/core/config/SwaggerConfig.java index 15dce47..02d57c1 100644 --- a/service/src/main/java/org/apache/griffin/core/config/SwaggerConfig.java +++ b/service/src/main/java/org/apache/griffin/core/config/SwaggerConfig.java @@ -19,10 +19,44 @@ under the License. package org.apache.griffin.core.config; +import com.google.common.base.Optional; +import com.google.common.base.Predicates; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import springfox.documentation.builders.PathSelectors; +import springfox.documentation.builders.RequestHandlerSelectors; +import springfox.documentation.service.ApiInfo; +import springfox.documentation.service.Contact; +import springfox.documentation.spi.DocumentationType; +import springfox.documentation.spring.web.plugins.Docket; import springfox.documentation.swagger2.annotations.EnableSwagger2; @Configuration @EnableSwagger2 public class SwaggerConfig { + + /** + * exclude the basic-error-controller from swagger api + */ + @Bean + public Docket excludeSwaggerErrorControllerApi() { + return new Docket(DocumentationType.SWAGGER_2) + .select() + .apis(RequestHandlerSelectors.any()) + .paths(Predicates.not(PathSelectors.regex("/error.*"))) + .build() + .apiInfo(metaData()); + } + + private ApiInfo metaData(){ + ApiInfo apiInfo =new ApiInfo( + "REST API Document", + "Spring Boot REST API for Apache Griffin", + "0.1.0", + "", + new Contact("","",""), + "Apache License Version 2.0", + "https://www.apache.org/licenses/LICENSE-2.0";); + return apiInfo; + } } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/config/jobConfig/SchedulerConfig.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/config/jobConfig/SchedulerConfig.java b/service/src/main/java/org/apache/griffin/core/config/jobConfig/SchedulerConfig.java index 2963b0e..ef71fe1 100644 --- a/service/src/main/java/org/apache/griffin/core/config/jobConfig/SchedulerConfig.java +++ b/service/src/main/java/org/apache/griffin/core/config/jobConfig/SchedulerConfig.java @@ -19,7 +19,8 @@ under the License. package org.apache.griffin.core.config.jobConfig; -import org.apache.griffin.core.util.GriffinUtil; +import org.apache.griffin.core.util.JsonUtil; +import org.apache.griffin.core.util.PropertiesUtil; import org.quartz.spi.JobFactory; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; @@ -53,6 +54,6 @@ public class SchedulerConfig { @Bean public Properties quartzProperties() { - return GriffinUtil.getProperties("/quartz.properties"); + return PropertiesUtil.getProperties("/quartz.properties"); } } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/config/jobConfig/SparkJobConfig.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/config/jobConfig/SparkJobConfig.java b/service/src/main/java/org/apache/griffin/core/config/jobConfig/SparkJobConfig.java index 4e41194..e089872 100644 --- a/service/src/main/java/org/apache/griffin/core/config/jobConfig/SparkJobConfig.java +++ b/service/src/main/java/org/apache/griffin/core/config/jobConfig/SparkJobConfig.java @@ -19,7 +19,8 @@ under the License. package org.apache.griffin.core.config.jobConfig; -import org.apache.griffin.core.util.GriffinUtil; +import org.apache.griffin.core.util.JsonUtil; +import org.apache.griffin.core.util.PropertiesUtil; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -29,6 +30,6 @@ import java.util.Properties; public class SparkJobConfig { @Bean(name = "sparkJobProps") public Properties sparkJobProperties() { - return GriffinUtil.getProperties("/sparkJob.properties"); + return PropertiesUtil.getProperties("/sparkJob.properties"); } } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/info/GriffinInfoController.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/info/GriffinInfoController.java b/service/src/main/java/org/apache/griffin/core/info/GriffinInfoController.java new file mode 100644 index 0000000..e460cff --- /dev/null +++ b/service/src/main/java/org/apache/griffin/core/info/GriffinInfoController.java @@ -0,0 +1,38 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +package org.apache.griffin.core.info; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +@Api(tags = "Basic introduce", description = "griffin version") +@RestController +@RequestMapping("/api/v1") +public class GriffinInfoController { + + @ApiOperation(value = "Get griffin version", response = String.class) + @RequestMapping(value = "/version", method = RequestMethod.GET) + public String greeting() { + return "0.1.0"; + } +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/job/JobController.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/job/JobController.java b/service/src/main/java/org/apache/griffin/core/job/JobController.java index ecc72e2..432f913 100644 --- a/service/src/main/java/org/apache/griffin/core/job/JobController.java +++ b/service/src/main/java/org/apache/griffin/core/job/JobController.java @@ -19,6 +19,9 @@ under the License. package org.apache.griffin.core.job; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; import org.apache.griffin.core.job.entity.JobHealth; import org.apache.griffin.core.job.entity.JobInstance; import org.apache.griffin.core.job.entity.JobRequestBody; @@ -32,6 +35,7 @@ import java.io.Serializable; import java.util.List; import java.util.Map; +@Api(tags = "Jobs",description = "execute your measure periodically") @RestController @RequestMapping("/api/v1/jobs") public class JobController { @@ -40,30 +44,39 @@ public class JobController { @Autowired private JobService jobService; - @RequestMapping(value = "/", method = RequestMethod.GET) + @ApiOperation(value = "Get jobs", response = List.class) + @RequestMapping(value = "", method = RequestMethod.GET) public List<Map<String, Serializable>> getJobs() { return jobService.getAliveJobs(); } + @ApiOperation(value = "Add job", response = GriffinOperationMessage.class) @RequestMapping(value = "", method = RequestMethod.POST) - public GriffinOperationMessage addJob(@RequestParam("group") String groupName, - @RequestParam("jobName") String jobName, - @RequestParam("measureId") Long measureId, - @RequestBody JobRequestBody jobRequestBody) { + public GriffinOperationMessage addJob(@ApiParam(value = "job group name", required = true) @RequestParam("group") String groupName, + @ApiParam(value = "job name", required = true) @RequestParam("jobName") String jobName, + @ApiParam(value = "measure id, required = true") @RequestParam("measureId") Long measureId, + @ApiParam(value = "custom class composed of job key parameters", required = true) + @RequestBody JobRequestBody jobRequestBody) { return jobService.addJob(groupName, jobName, measureId, jobRequestBody); } + @ApiOperation(value = "Delete job", response = GriffinOperationMessage.class) @RequestMapping(value = "", method = RequestMethod.DELETE) - public GriffinOperationMessage deleteJob(@RequestParam("group") String group, @RequestParam("jobName") String jobName) { + public GriffinOperationMessage deleteJob(@ApiParam(value = "job group name", required = true) @RequestParam("group") String group, + @ApiParam(value = "job name", required = true) @RequestParam("jobName") String jobName) { return jobService.deleteJob(group, jobName); } + @ApiOperation(value = "Get job instances", response = List.class) @RequestMapping(value = "/instances", method = RequestMethod.GET) - public List<JobInstance> findInstancesOfJob(@RequestParam("group") String group, @RequestParam("jobName") String jobName, - @RequestParam("page") int page, @RequestParam("size") int size) { + public List<JobInstance> findInstancesOfJob(@ApiParam(value = "job group name", required = true) @RequestParam("group") String group, + @ApiParam(value = "job name", required = true) @RequestParam("jobName") String jobName, + @ApiParam(value = "page you want starting from index 0", required = true) @RequestParam("page") int page, + @ApiParam(value = "instance number per page", required = true) @RequestParam("size") int size) { return jobService.findInstancesOfJob(group, jobName, page, size); } + @ApiOperation(value = "Get job healthy statistics", response = JobHealth.class) @RequestMapping(value = "/health", method = RequestMethod.GET) public JobHealth getHealthInfo() { return jobService.getHealthInfo(); http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java b/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java index 95a294e..251d280 100644 --- a/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java +++ b/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java @@ -30,7 +30,7 @@ import org.apache.griffin.core.job.entity.LivySessionStates; import org.apache.griffin.core.job.repo.JobInstanceRepo; import org.apache.griffin.core.measure.entity.Measure; import org.apache.griffin.core.util.GriffinOperationMessage; -import org.apache.griffin.core.util.GriffinUtil; +import org.apache.griffin.core.util.JsonUtil; import org.quartz.*; import org.quartz.impl.matchers.GroupMatcher; import org.slf4j.Logger; @@ -42,6 +42,7 @@ import org.springframework.data.domain.Sort; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.quartz.SchedulerFactoryBean; import org.springframework.stereotype.Service; +import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; import java.io.IOException; @@ -65,6 +66,7 @@ public class JobServiceImpl implements JobService { @Autowired private Properties sparkJobProps; + public JobServiceImpl() { } @@ -73,12 +75,10 @@ public class JobServiceImpl implements JobService { Scheduler scheduler = factory.getObject(); List<Map<String, Serializable>> list = new ArrayList<>(); try { - for (String groupName : scheduler.getJobGroupNames()) { - for (JobKey jobKey : scheduler.getJobKeys(GroupMatcher.jobGroupEquals(groupName))) { - Map jobInfoMap = getJobInfoMap(scheduler, jobKey); - if (jobInfoMap.size() != 0 && !isJobDeleted(scheduler, jobKey)) { - list.add(jobInfoMap); - } + for (JobKey jobKey : scheduler.getJobKeys(GroupMatcher.anyGroup())) { + Map jobInfoMap = getJobInfoMap(scheduler, jobKey); + if (jobInfoMap.size() != 0 && !isJobDeleted(scheduler, jobKey)) { + list.add(jobInfoMap); } } } catch (SchedulerException e) { @@ -136,48 +136,57 @@ public class JobServiceImpl implements JobService { interval = Integer.parseInt(jobRequestBody.getInterval()); jobStartTime = new Date(Long.parseLong(jobRequestBody.getJobStartTime())); setJobStartTime(jobStartTime, interval); - } catch (Exception e) { - LOGGER.info("jobStartTime or interval format error! {}", e.getMessage()); - return CREATE_JOB_FAIL; - } - try { + Scheduler scheduler = factory.getObject(); TriggerKey triggerKey = triggerKey(jobName, groupName); if (scheduler.checkExists(triggerKey)) { - LOGGER.error("the triggerKey(jobName,groupName) {} has been used.", jobName); + LOGGER.error("the triggerKey({},{}) has been used.", jobName, groupName); return CREATE_JOB_FAIL; } - JobKey jobKey = jobKey(jobName, groupName); - JobDetail jobDetail; - if (scheduler.checkExists(jobKey)) { - jobDetail = scheduler.getJobDetail(jobKey); - setJobData(jobDetail, jobRequestBody, measureId, groupName, jobName); - scheduler.addJob(jobDetail, true); - } else { - jobDetail = newJob(SparkSubmitJob.class) - .storeDurably() - .withIdentity(jobKey) - .build(); - //set JobData - setJobData(jobDetail, jobRequestBody, measureId, groupName, jobName); - scheduler.addJob(jobDetail, false); - } - Trigger trigger = newTrigger() - .withIdentity(triggerKey) - .forJob(jobDetail) - .withSchedule(SimpleScheduleBuilder.simpleSchedule() - .withIntervalInSeconds(interval) - .repeatForever()) - .startAt(jobStartTime) - .build(); - scheduler.scheduleJob(trigger); + + JobDetail jobDetail = addJobDetail(scheduler, groupName, jobName, measureId, jobRequestBody); + scheduler.scheduleJob(newTriggerInstance(triggerKey, jobDetail, interval, jobStartTime)); return GriffinOperationMessage.CREATE_JOB_SUCCESS; + } catch (NumberFormatException e) { + LOGGER.info("jobStartTime or interval format error! {}", e.getMessage()); + return CREATE_JOB_FAIL; } catch (SchedulerException e) { LOGGER.error("SchedulerException when add job. {}", e.getMessage()); return CREATE_JOB_FAIL; } } + private JobDetail addJobDetail(Scheduler scheduler, String groupName, String jobName, Long measureId, JobRequestBody jobRequestBody) throws SchedulerException { + JobKey jobKey = jobKey(jobName, groupName); + JobDetail jobDetail; + if (scheduler.checkExists(jobKey)) { + jobDetail = scheduler.getJobDetail(jobKey); + setJobData(jobDetail, jobRequestBody, measureId, groupName, jobName); + scheduler.addJob(jobDetail, true); + } else { + jobDetail = newJob(SparkSubmitJob.class) + .storeDurably() + .withIdentity(jobKey) + .build(); + //set JobData + setJobData(jobDetail, jobRequestBody, measureId, groupName, jobName); + scheduler.addJob(jobDetail, false); + } + return jobDetail; + } + + private Trigger newTriggerInstance(TriggerKey triggerKey, JobDetail jobDetail, int interval, Date jobStartTime) throws SchedulerException { + Trigger trigger = newTrigger() + .withIdentity(triggerKey) + .forJob(jobDetail) + .withSchedule(SimpleScheduleBuilder.simpleSchedule() + .withIntervalInSeconds(interval) + .repeatForever()) + .startAt(jobStartTime) + .build(); + return trigger; + } + private void setJobStartTime(Date jobStartTime, int interval) { long currentTimestamp = System.currentTimeMillis(); long jobStartTimestamp = jobStartTime.getTime(); @@ -290,10 +299,10 @@ public class JobServiceImpl implements JobService { } /** - * call livy to update jobInstance table in mysql. + * call livy to update part of jobInstance table data associated with group and jobName in mysql. * - * @param group - * @param jobName + * @param group group name of jobInstance + * @param jobName job name of jobInstance */ private void syncInstancesOfJob(String group, String jobName) { //update all instance info belongs to this group and job. @@ -303,38 +312,44 @@ public class JobServiceImpl implements JobService { continue; } String uri = sparkJobProps.getProperty("livy.uri") + "/" + jobInstance.getSessionId(); - RestTemplate restTemplate = new RestTemplate(); - String resultStr; - try { - resultStr = restTemplate.getForObject(uri, String.class); - } catch (Exception e) { - LOGGER.error("spark session {} has overdue, set state as unknown!\n {}", jobInstance.getSessionId(), e.getMessage()); - //if server cannot get session from Livy, set State as unknown. - jobInstance.setState(LivySessionStates.State.unknown); - jobInstanceRepo.save(jobInstance); - continue; - } - TypeReference<HashMap<String, Object>> type = new TypeReference<HashMap<String, Object>>() { - }; - HashMap<String, Object> resultMap; - try { - resultMap = GriffinUtil.toEntity(resultStr, type); - } catch (IOException e) { - LOGGER.error("jobInstance jsonStr convert to map failed. {}", e.getMessage()); - continue; - } - try { - if (resultMap != null && resultMap.size() != 0) { - jobInstance.setState(LivySessionStates.State.valueOf(resultMap.get("state").toString())); - jobInstance.setAppId(resultMap.get("appId").toString()); - jobInstance.setAppUri(sparkJobProps.getProperty("spark.uri") + "/cluster/app/" + resultMap.get("appId").toString()); - } - } catch (Exception e) { - LOGGER.warn("{},{} job Instance has some null field (state or appId). {}", group, jobName, e.getMessage()); - continue; + setJobInstanceInfo(jobInstance, uri, group, jobName); + } + } + + private void setJobInstanceInfo(JobInstance jobInstance, String uri, String group, String jobName) { + RestTemplate restTemplate = new RestTemplate(); + TypeReference<HashMap<String, Object>> type = new TypeReference<HashMap<String, Object>>() { + }; + try { + String resultStr = restTemplate.getForObject(uri, String.class); + HashMap<String, Object> resultMap = JsonUtil.toEntity(resultStr, type); + setJobInstanceIdAndUri(jobInstance, resultMap); + } catch (RestClientException e) { + LOGGER.error("spark session {} has overdue, set state as unknown!\n {}", jobInstance.getSessionId(), e.getMessage()); + setJobInstanceUnknownStatus(jobInstance); + } catch (IOException e) { + LOGGER.error("jobInstance jsonStr convert to map failed. {}", e.getMessage()); + } catch (IllegalArgumentException e) { + LOGGER.warn("Livy status is illegal. {}", group, jobName, e.getMessage()); + } + } + + private void setJobInstanceIdAndUri(JobInstance jobInstance, HashMap<String, Object> resultMap) throws IllegalArgumentException { + if (resultMap != null && resultMap.size() != 0 && resultMap.get("state") != null) { + jobInstance.setState(LivySessionStates.State.valueOf(resultMap.get("state").toString())); + if (resultMap.get("appId") != null) { + jobInstance.setAppId(resultMap.get("appId").toString()); + jobInstance.setAppUri(sparkJobProps.getProperty("spark.uri") + "/cluster/app/" + resultMap.get("appId").toString()); } jobInstanceRepo.save(jobInstance); } + + } + + private void setJobInstanceUnknownStatus(JobInstance jobInstance) { + //if server cannot get session from Livy, set State as unknown. + jobInstance.setState(LivySessionStates.State.unknown); + jobInstanceRepo.save(jobInstance); } /** @@ -348,19 +363,17 @@ public class JobServiceImpl implements JobService { int jobCount = 0; int notHealthyCount = 0; try { - for (String groupName : scheduler.getJobGroupNames()) { - for (JobKey jobKey : scheduler.getJobKeys(GroupMatcher.jobGroupEquals(groupName))) { - jobCount++; - String jobName = jobKey.getName(); - String jobGroup = jobKey.getGroup(); - Pageable pageRequest = new PageRequest(0, 1, Sort.Direction.DESC, "timestamp"); - JobInstance latestJobInstance; - if (jobInstanceRepo.findByGroupNameAndJobName(jobGroup, jobName, pageRequest) != null - && jobInstanceRepo.findByGroupNameAndJobName(jobGroup, jobName, pageRequest).size() > 0) { - latestJobInstance = jobInstanceRepo.findByGroupNameAndJobName(jobGroup, jobName, pageRequest).get(0); - if (!LivySessionStates.isHeathy(latestJobInstance.getState())) { - notHealthyCount++; - } + for (JobKey jobKey : scheduler.getJobKeys(GroupMatcher.anyGroup())) { + jobCount++; + String jobName = jobKey.getName(); + String jobGroup = jobKey.getGroup(); + Pageable pageRequest = new PageRequest(0, 1, Sort.Direction.DESC, "timestamp"); + JobInstance latestJobInstance; + List<JobInstance> jobInstances = jobInstanceRepo.findByGroupNameAndJobName(jobGroup, jobName, pageRequest); + if (jobInstances != null && jobInstances.size() > 0) { + latestJobInstance = jobInstances.get(0); + if (!LivySessionStates.isHeathy(latestJobInstance.getState())) { + notHealthyCount++; } } } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/job/SparkSubmitJob.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/job/SparkSubmitJob.java b/service/src/main/java/org/apache/griffin/core/job/SparkSubmitJob.java index 9be41a4..4590fc8 100644 --- a/service/src/main/java/org/apache/griffin/core/job/SparkSubmitJob.java +++ b/service/src/main/java/org/apache/griffin/core/job/SparkSubmitJob.java @@ -30,7 +30,7 @@ import org.apache.griffin.core.measure.entity.DataConnector; import org.apache.griffin.core.measure.entity.DataSource; import org.apache.griffin.core.measure.entity.Measure; import org.apache.griffin.core.measure.repo.MeasureRepo; -import org.apache.griffin.core.util.GriffinUtil; +import org.apache.griffin.core.util.JsonUtil; import org.quartz.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -229,7 +229,7 @@ public class SparkSubmitJob implements Job { args.add(sparkJobProps.getProperty("sparkJob.args_1")); // measure String measureJson; - measureJson = GriffinUtil.toJsonWithFormat(measure); + measureJson = JsonUtil.toJsonWithFormat(measure); args.add(measureJson); args.add(sparkJobProps.getProperty("sparkJob.args_3")); sparkJobDO.setArgs(args); @@ -255,29 +255,31 @@ public class SparkSubmitJob implements Job { sparkJobDO.setFiles(files); } - private void saveJobInstance(String groupName, String jobName, String result) { - //save JobInstance info into DataBase - Map<String, Object> resultMap = new HashMap<>(); - TypeReference<HashMap<String, Object>> type = new TypeReference<HashMap<String, Object>>() { - }; + public void saveJobInstance(String groupName, String jobName, String result) { + TypeReference<HashMap<String, Object>> type = new TypeReference<HashMap<String, Object>>() {}; try { - resultMap = GriffinUtil.toEntity(result, type); + Map<String, Object> resultMap = JsonUtil.toEntity(result, type); + if (resultMap != null) { + JobInstance jobInstance = genJobInstance(groupName, jobName, resultMap); + jobInstanceRepo.save(jobInstance); + } } catch (IOException e) { LOGGER.error("jobInstance jsonStr convert to map failed. {}", e.getMessage()); + } catch (IllegalArgumentException e) { + LOGGER.warn("Livy status is illegal. {}", e.getMessage()); } + } + + private JobInstance genJobInstance(String groupName, String jobName, Map<String, Object> resultMap) throws IllegalArgumentException{ JobInstance jobInstance = new JobInstance(); - if (resultMap != null) { - jobInstance.setGroupName(groupName); - jobInstance.setJobName(jobName); - try { - jobInstance.setSessionId(Integer.parseInt(resultMap.get("id").toString())); - jobInstance.setState(LivySessionStates.State.valueOf(resultMap.get("state").toString())); - jobInstance.setAppId(resultMap.get("appId").toString()); - } catch (Exception e) { - LOGGER.warn("jobInstance has null field. {}", e.getMessage()); - } - jobInstance.setTimestamp(System.currentTimeMillis()); - jobInstanceRepo.save(jobInstance); + jobInstance.setGroupName(groupName); + jobInstance.setJobName(jobName); + jobInstance.setTimestamp(System.currentTimeMillis()); + jobInstance.setSessionId(Integer.parseInt(resultMap.get("id").toString())); + jobInstance.setState(LivySessionStates.State.valueOf(resultMap.get("state").toString())); + if (resultMap.get("appId") != null) { + jobInstance.setAppId(resultMap.get("appId").toString()); } + return jobInstance; } } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/login/LoginController.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/login/LoginController.java b/service/src/main/java/org/apache/griffin/core/login/LoginController.java index 2e75a81..d189f03 100644 --- a/service/src/main/java/org/apache/griffin/core/login/LoginController.java +++ b/service/src/main/java/org/apache/griffin/core/login/LoginController.java @@ -19,6 +19,9 @@ under the License. package org.apache.griffin.core.login; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -29,8 +32,10 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; +import java.util.List; import java.util.Map; +@Api(tags = "Auth", description = "user authentication") @RestController @RequestMapping("/api/v1/login") public class LoginController { @@ -42,8 +47,11 @@ public class LoginController { @Autowired private Environment env; + @ApiOperation(value = "Get all jobs", response = ResponseEntity.class) @RequestMapping(value = "/authenticate", method = RequestMethod.POST) - public ResponseEntity<Map<String, Object>> login(@RequestBody Map<String, String> map) { + public ResponseEntity<Map<String, Object>> login( + @ApiParam(value = "a map contains user name and password", required = true) + @RequestBody Map<String, String> map) { return loginService.login(map); } } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/measure/MeasureController.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/measure/MeasureController.java b/service/src/main/java/org/apache/griffin/core/measure/MeasureController.java index 2017ecc..aaa2db5 100644 --- a/service/src/main/java/org/apache/griffin/core/measure/MeasureController.java +++ b/service/src/main/java/org/apache/griffin/core/measure/MeasureController.java @@ -19,6 +19,9 @@ under the License. package org.apache.griffin.core.measure; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; import org.apache.griffin.core.measure.entity.Measure; import org.apache.griffin.core.util.GriffinOperationMessage; import org.springframework.beans.factory.annotation.Autowired; @@ -27,41 +30,45 @@ import org.springframework.web.bind.annotation.*; import java.util.List; import java.util.Map; +@Api(tags = "Measures",description = "measure data quality between source and target dataset") @RestController -//@Api("MeasureController") - @RequestMapping(value = "/api/v1") public class MeasureController { @Autowired private MeasureService measureService; + @ApiOperation(value ="Get measures",response = Iterable.class) @RequestMapping(value = "/measures", method = RequestMethod.GET) public Iterable<Measure> getAllAliveMeasures() { return measureService.getAllAliveMeasures(); } + @ApiOperation(value ="Get measure by id",response = Measure.class) @RequestMapping(value = "/measure/{id}", method = RequestMethod.GET) - public Measure getMeasureById(@PathVariable("id") long id) { + public Measure getMeasureById(@ApiParam(value = "measure id", required = true) @PathVariable("id") long id) { return measureService.getMeasureById(id); } + @ApiOperation(value ="Delete measure",response = GriffinOperationMessage.class) @RequestMapping(value = "/measure/{id}", method = RequestMethod.DELETE) - public GriffinOperationMessage deleteMeasureById(@PathVariable("id") Long id) { + public GriffinOperationMessage deleteMeasureById(@ApiParam(value = "measure id", required = true) @PathVariable("id") Long id) { return measureService.deleteMeasureById(id); } - + @ApiOperation(value ="Update measure",response = GriffinOperationMessage.class) @RequestMapping(value = "/measure", method = RequestMethod.PUT) - public GriffinOperationMessage updateMeasure(@RequestBody Measure measure) { + public GriffinOperationMessage updateMeasure(@ApiParam(value = "measure entity", required = true) @RequestBody Measure measure) { return measureService.updateMeasure(measure); } + @ApiOperation(value ="Get measures by org",response = List.class) @RequestMapping(value = "/measures/owner/{owner}", method = RequestMethod.GET) - public List<Map<String, String>> getAllAliveMeasureNameIdByOwner(@PathVariable("owner") String owner) { - return measureService.getAllAliveMeasureNameIdByOwner(owner); + public List<Measure> getAliveMeasuresByOwner(@ApiParam(value = "owner name", required = true) @PathVariable("owner") String owner) { + return measureService.getAliveMeasuresByOwner(owner); } + @ApiOperation(value ="Add measure",response = GriffinOperationMessage.class) @RequestMapping(value = "/measure", method = RequestMethod.POST) - public GriffinOperationMessage createMeasure(@RequestBody Measure measure) { + public GriffinOperationMessage createMeasure(@ApiParam(value = "measure entity", required = true) @RequestBody Measure measure) { return measureService.createMeasure(measure); } } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/measure/MeasureOrgController.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/measure/MeasureOrgController.java b/service/src/main/java/org/apache/griffin/core/measure/MeasureOrgController.java new file mode 100644 index 0000000..5982e0c --- /dev/null +++ b/service/src/main/java/org/apache/griffin/core/measure/MeasureOrgController.java @@ -0,0 +1,72 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +package org.apache.griffin.core.measure; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import org.apache.griffin.core.measure.repo.MeasureRepo; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Api(tags = "Organization Dimension", description = "measure belongs to") +@RestController +@RequestMapping(value = "/api/v1") +public class MeasureOrgController { + @Autowired + private MeasureRepo measureRepo; + + @ApiOperation(value = "Get orgs for measure", response = List.class) + @RequestMapping(value = "/org", method = RequestMethod.GET) + public List<String> getOrgs() { + return measureRepo.findOrganizations(); + } + + /** + * @param org + * @return list of metric name, and a metric is the result of executing the job sharing the same name with + * measure. + */ + @ApiOperation(value = "Get measure names by org", response = List.class) + @RequestMapping(value = "/org/{org}", method = RequestMethod.GET) + public List<String> getMetricNameListByOrg(@ApiParam(value = "organization name") @PathVariable("org") String org) { + return measureRepo.findNameByOrganization(org); + } + + @ApiOperation(value = "Get measure names group by org", response = Map.class) + @RequestMapping(value = "/org/measure/names", method = RequestMethod.GET) + public Map<String, List<String>> getMeasureNamesGroupByOrg() { + Map<String, List<String>> orgWithMetricsMap = new HashMap<>(); + List<String> orgList = measureRepo.findOrganizations(); + for (String org : orgList) { + if (org != null) { + orgWithMetricsMap.put(org, measureRepo.findNameByOrganization(org)); + } + } + return orgWithMetricsMap; + } +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/measure/MeasureService.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/measure/MeasureService.java b/service/src/main/java/org/apache/griffin/core/measure/MeasureService.java index a970752..0e20b4f 100644 --- a/service/src/main/java/org/apache/griffin/core/measure/MeasureService.java +++ b/service/src/main/java/org/apache/griffin/core/measure/MeasureService.java @@ -46,7 +46,7 @@ public interface MeasureService { GriffinOperationMessage updateMeasure(Measure measure); - List<Map<String, String>> getAllAliveMeasureNameIdByOwner(String owner); + List<Measure> getAliveMeasuresByOwner(String owner); GriffinOperationMessage createMeasure(Measure measure); } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/measure/MeasureServiceImpl.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/measure/MeasureServiceImpl.java b/service/src/main/java/org/apache/griffin/core/measure/MeasureServiceImpl.java index ec09f2a..0a880cc 100644 --- a/service/src/main/java/org/apache/griffin/core/measure/MeasureServiceImpl.java +++ b/service/src/main/java/org/apache/griffin/core/measure/MeasureServiceImpl.java @@ -99,15 +99,8 @@ public class MeasureServiceImpl implements MeasureService { } @Override - public List<Map<String, String>> getAllAliveMeasureNameIdByOwner(String owner) { - List<Map<String, String>> res = new ArrayList<>(); - for (Measure measure : measureRepo.findByOwnerAndDeleted(owner, false)) { - HashMap<String, String> map = new HashMap<>(); - map.put("name", measure.getName()); - map.put("id", measure.getId().toString()); - res.add(map); - } - return res; + public List<Measure> getAliveMeasuresByOwner(String owner) { + return measureRepo.findByOwnerAndDeleted(owner, false); } @Override http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/measure/entity/DataConnector.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/measure/entity/DataConnector.java b/service/src/main/java/org/apache/griffin/core/measure/entity/DataConnector.java index 59f611d..a5b80f9 100644 --- a/service/src/main/java/org/apache/griffin/core/measure/entity/DataConnector.java +++ b/service/src/main/java/org/apache/griffin/core/measure/entity/DataConnector.java @@ -22,7 +22,7 @@ package org.apache.griffin.core.measure.entity; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; -import org.apache.griffin.core.util.GriffinUtil; +import org.apache.griffin.core.util.JsonUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,7 +52,7 @@ public class DataConnector extends AbstractAuditableEntity { }; if (this.configInMaps == null) { try { - this.configInMaps = GriffinUtil.toEntity(config, mapType); + this.configInMaps = JsonUtil.toEntity(config, mapType); } catch (IOException e) { LOGGER.error("Error in converting json to map. {}", e.getMessage()); } @@ -61,7 +61,7 @@ public class DataConnector extends AbstractAuditableEntity { } public void setConfig(Map<String, String> configInMaps) throws JsonProcessingException { - this.config = GriffinUtil.toJson(configInMaps); + this.config = JsonUtil.toJson(configInMaps); } public Map<String, String> getConfig() { @@ -95,7 +95,7 @@ public class DataConnector extends AbstractAuditableEntity { TypeReference<Map<String, String>> mapType = new TypeReference<Map<String, String>>() { }; try { - this.configInMaps = GriffinUtil.toEntity(config, mapType); + this.configInMaps = JsonUtil.toEntity(config, mapType); } catch (IOException e) { LOGGER.error("Error in converting json to map. {}", e.getMessage()); } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/measure/entity/DataSource.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/measure/entity/DataSource.java b/service/src/main/java/org/apache/griffin/core/measure/entity/DataSource.java index e967374..b6097c6 100644 --- a/service/src/main/java/org/apache/griffin/core/measure/entity/DataSource.java +++ b/service/src/main/java/org/apache/griffin/core/measure/entity/DataSource.java @@ -20,6 +20,8 @@ under the License. package org.apache.griffin.core.measure.entity; +import io.swagger.annotations.ApiModelProperty; + import javax.persistence.*; import java.util.List; @@ -29,7 +31,7 @@ public class DataSource extends AbstractAuditableEntity { private String name; - @OneToMany(fetch = FetchType.EAGER, cascade = {CascadeType.PERSIST, CascadeType.REMOVE}) + @OneToMany(fetch = FetchType.EAGER, cascade = {CascadeType.PERSIST, CascadeType.REMOVE, CascadeType.MERGE}) @JoinColumn(name = "dataSource_id") private List<DataConnector> connectors; http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/measure/entity/EvaluateRule.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/measure/entity/EvaluateRule.java b/service/src/main/java/org/apache/griffin/core/measure/entity/EvaluateRule.java index 8ef5294..2a70636 100644 --- a/service/src/main/java/org/apache/griffin/core/measure/entity/EvaluateRule.java +++ b/service/src/main/java/org/apache/griffin/core/measure/entity/EvaluateRule.java @@ -31,7 +31,7 @@ import java.util.List; public class EvaluateRule extends AbstractAuditableEntity { private static final long serialVersionUID = 4240072518233967528L; - @OneToMany(fetch = FetchType.EAGER, cascade = {CascadeType.PERSIST, CascadeType.REMOVE}) + @OneToMany(fetch = FetchType.EAGER, cascade = {CascadeType.PERSIST, CascadeType.REMOVE, CascadeType.MERGE}) @JoinColumn(name = "evaluateRule_id") @Fetch(FetchMode.SUBSELECT) private List<Rule> rules; http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/measure/entity/Measure.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/measure/entity/Measure.java b/service/src/main/java/org/apache/griffin/core/measure/entity/Measure.java index 98460d5..60e8147 100644 --- a/service/src/main/java/org/apache/griffin/core/measure/entity/Measure.java +++ b/service/src/main/java/org/apache/griffin/core/measure/entity/Measure.java @@ -37,11 +37,11 @@ public class Measure extends AbstractAuditableEntity { private String processType; - @OneToMany(fetch = FetchType.EAGER, cascade = {CascadeType.PERSIST, CascadeType.REMOVE}) + @OneToMany(fetch = FetchType.EAGER, cascade = {CascadeType.PERSIST, CascadeType.REMOVE, CascadeType.MERGE}) @JoinColumn(name = "measure_id") private List<DataSource> dataSources; - @OneToOne(fetch = FetchType.EAGER, cascade = {CascadeType.PERSIST, CascadeType.REMOVE}) + @OneToOne(fetch = FetchType.EAGER, cascade = {CascadeType.PERSIST, CascadeType.REMOVE, CascadeType.MERGE}) @JoinColumn(name = "evaluateRule_id") private EvaluateRule evaluateRule; http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreController.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreController.java b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreController.java index 319c116..6b446e0 100644 --- a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreController.java +++ b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreController.java @@ -19,6 +19,9 @@ under the License. package org.apache.griffin.core.metastore.hive; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; import org.apache.hadoop.hive.metastore.api.Table; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; @@ -26,6 +29,7 @@ import org.springframework.web.bind.annotation.*; import java.util.List; import java.util.Map; +@Api(tags = "Hive metastore",description = "hive table and database manipulation") @RestController @RequestMapping("/api/v1/metadata/hive") public class HiveMetaStoreController { @@ -33,39 +37,36 @@ public class HiveMetaStoreController { @Autowired private HiveMetaStoreService hiveMetaStoreService; - - @RequestMapping(value = "/db", method = RequestMethod.GET) + @ApiOperation(value = "Get database names", response = Iterable.class) + @RequestMapping(value = "/dbs", method = RequestMethod.GET) public Iterable<String> getAllDatabases() { return hiveMetaStoreService.getAllDatabases(); } - @RequestMapping(value = "/table", method = RequestMethod.GET) - public Iterable<String> getDefAllTables() { - return hiveMetaStoreService.getAllTableNames(""); - } - @RequestMapping(value = "/allTableNames", method = RequestMethod.GET) - public Iterable<String> getAllTableNames(@RequestParam("db") String dbName) { + @ApiOperation(value = "Get table names", response = Iterable.class) + @RequestMapping(value = "/tables/names", method = RequestMethod.GET) + public Iterable<String> getAllTableNames(@ApiParam(value = "hive db name", required = true) @RequestParam("db") String dbName) { return hiveMetaStoreService.getAllTableNames(dbName); } - @RequestMapping(value = "/db/allTables", method = RequestMethod.GET) - public List<Table> getAllTables(@RequestParam("db") String dbName) { + @ApiOperation(value = "Get tables metadata", response = List.class) + @RequestMapping(value = "/tables", method = RequestMethod.GET) + public List<Table> getAllTables(@ApiParam(value = "hive db name", required = true) @RequestParam("db") String dbName) { return hiveMetaStoreService.getAllTable(dbName); } - @RequestMapping(value = "/allTables", method = RequestMethod.GET) + @ApiOperation(value = "Get all database tables metadata", response = Map.class) + @RequestMapping(value = "/dbs/tables", method = RequestMethod.GET) public Map<String, List<Table>> getAllTables() { return hiveMetaStoreService.getAllTable(); } - @RequestMapping(value = "/default/{table}", method = RequestMethod.GET) - public Table getDefTable(@PathVariable("table") String tableName) { - return hiveMetaStoreService.getTable("", tableName); - } - @RequestMapping(value = "", method = RequestMethod.GET) - public Table getTable(@RequestParam("db") String dbName, @RequestParam("table") String tableName) { + @ApiOperation(value = "Get table metadata", response = Table.class) + @RequestMapping(value = "/table", method = RequestMethod.GET) + public Table getTable(@ApiParam(value = "hive database name", required = true) @RequestParam("db") String dbName, + @ApiParam(value = "hive table name", required = true) @RequestParam("table") String tableName) { return hiveMetaStoreService.getTable(dbName, tableName); } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreService.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreService.java b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreService.java index e9a1bbd..952bc39 100644 --- a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreService.java +++ b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreService.java @@ -35,5 +35,4 @@ public interface HiveMetaStoreService { Map<String, List<Table>> getAllTable(); Table getTable(String dbName, String tableName); - } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java index db57e9b..cc1a599 100644 --- a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java +++ b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java @@ -68,8 +68,7 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService { } @Override - @Cacheable - + @Cacheable(key = "#root.methodName") public Iterable<String> getAllDatabases() { Iterable<String> results = null; try { @@ -83,7 +82,7 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService { @Override - @Cacheable + @Cacheable(key = "#root.methodName.concat(#dbName)") public Iterable<String> getAllTableNames(String dbName) { Iterable<String> results = null; try { @@ -97,17 +96,22 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService { @Override - @Cacheable + @Cacheable(key = "#root.methodName.concat(#db)") public List<Table> getAllTable(String db) { return getTables(db); } @Override - @Cacheable + @Cacheable(key = "#root.methodName") public Map<String, List<Table>> getAllTable() { Map<String, List<Table>> results = new HashMap<>(); - Iterable<String> dbs = getAllDatabases(); + Iterable<String> dbs = null; + // if hive.metastore.uris in application.properties configs wrong, client will be injected failure and will be null. + if (client != null) { + dbs = getAllDatabases(); + LOGGER.error("hive client is null.Please check your hive config."); + } //MetaException happens if (dbs == null) { return results; @@ -120,7 +124,7 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService { @Override - @Cacheable + @Cacheable(key = "#root.methodName.concat(#dbName).concat(#tableName)") public Table getTable(String dbName, String tableName) { Table result = null; try { http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/metastore/kafka/KafkaSchemaController.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/metastore/kafka/KafkaSchemaController.java b/service/src/main/java/org/apache/griffin/core/metastore/kafka/KafkaSchemaController.java index acff59b..0ef6159 100644 --- a/service/src/main/java/org/apache/griffin/core/metastore/kafka/KafkaSchemaController.java +++ b/service/src/main/java/org/apache/griffin/core/metastore/kafka/KafkaSchemaController.java @@ -22,9 +22,13 @@ package org.apache.griffin.core.metastore.kafka; import io.confluent.kafka.schemaregistry.client.rest.entities.Config; import io.confluent.kafka.schemaregistry.client.rest.entities.Schema; import io.confluent.kafka.schemaregistry.client.rest.entities.SchemaString; +import io.swagger.annotations.Api; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; +import springfox.documentation.annotations.ApiIgnore; +@Api(tags = "Kafka metastore") +@ApiIgnore @RestController @RequestMapping("/api/v1/metadata/kafka") public class KafkaSchemaController { http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/metric/MetricController.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/metric/MetricController.java b/service/src/main/java/org/apache/griffin/core/metric/MetricController.java index 95b13fe..1b3c3d5 100644 --- a/service/src/main/java/org/apache/griffin/core/metric/MetricController.java +++ b/service/src/main/java/org/apache/griffin/core/metric/MetricController.java @@ -19,6 +19,8 @@ under the License. package org.apache.griffin.core.metric; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -26,11 +28,13 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import springfox.documentation.annotations.ApiIgnore; /** * In griffin, metricName usually equals to measureName, and we only save measureName in server. */ +@ApiIgnore @RestController @RequestMapping("/api/v1/metrics") public class MetricController { @@ -38,8 +42,9 @@ public class MetricController { @Autowired MetricService metricService; + @ApiOperation(value = "Get org by measure name", response = String.class) @RequestMapping(value = "/org", method = RequestMethod.GET) - public String getOrgByMeasureName(@RequestParam("measureName") String measureName) { + public String getOrgByMeasureName(@ApiParam(value = "measure name", required = true) @RequestParam("measureName") String measureName) { return metricService.getOrgByMeasureName(measureName); } } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/service/GriffinController.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/service/GriffinController.java b/service/src/main/java/org/apache/griffin/core/service/GriffinController.java deleted file mode 100644 index 8fb605c..0000000 --- a/service/src/main/java/org/apache/griffin/core/service/GriffinController.java +++ /dev/null @@ -1,78 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ - -package org.apache.griffin.core.service; - - -import org.apache.griffin.core.measure.repo.MeasureRepo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.RestController; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - - -@RestController -@RequestMapping(value = "/api/v1") -public class GriffinController { - private static final Logger LOGGER = LoggerFactory.getLogger(GriffinController.class); - - @Autowired - MeasureRepo measureRepo; - - @RequestMapping(value = "/version", method = RequestMethod.GET) - public String greeting() { - return "0.1.0"; - } - - @RequestMapping(value = "/org", method = RequestMethod.GET) - public List<String> getOrgs() { - return measureRepo.findOrganizations(); - } - - /** - * @param org - * @return list of the name of metric, and a metric is the result of executing the job sharing the same name with - * measure. - */ - @RequestMapping(value = "/org/{org}", method = RequestMethod.GET) - public List<String> getMetricNameListByOrg(@PathVariable("org") String org) { - return measureRepo.findNameByOrganization(org); - } - - @RequestMapping(value = "/orgWithMetricsName", method = RequestMethod.GET) - public Map<String, List<String>> getOrgsWithMetricsName() { - Map<String, List<String>> orgWithMetricsMap = new HashMap<>(); - List<String> orgList = measureRepo.findOrganizations(); - for (String org : orgList) { - if (org != null) { - orgWithMetricsMap.put(org, measureRepo.findNameByOrganization(org)); - } - } - return orgWithMetricsMap; - } - -} - http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/util/GriffinOperationMessage.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/util/GriffinOperationMessage.java b/service/src/main/java/org/apache/griffin/core/util/GriffinOperationMessage.java index dd4d895..982efb6 100644 --- a/service/src/main/java/org/apache/griffin/core/util/GriffinOperationMessage.java +++ b/service/src/main/java/org/apache/griffin/core/util/GriffinOperationMessage.java @@ -26,10 +26,10 @@ import com.fasterxml.jackson.annotation.JsonFormat; public enum GriffinOperationMessage { //success CREATE_MEASURE_SUCCESS(201, "Create Measure Succeed"), - DELETE_MEASURE_BY_ID_SUCCESS(202, "Delete Measures By Name Succeed"), + DELETE_MEASURE_BY_ID_SUCCESS(202, "Delete Measures By Id Succeed"), DELETE_MEASURE_BY_NAME_SUCCESS(203, "Delete Measures By Name Succeed"), UPDATE_MEASURE_SUCCESS(204, "Update Measure Succeed"), - CREATE_JOB_SUCCESS(205, "CREATE Job Succeed"), + CREATE_JOB_SUCCESS(205, "Create Job Succeed"), DELETE_JOB_SUCCESS(206, "Delete Job Succeed"), SET_JOB_DELETED_STATUS_SUCCESS(207, "Set Job Deleted Status Succeed"), PAUSE_JOB_SUCCESS(208, "Pause Job Succeed"), @@ -38,7 +38,7 @@ public enum GriffinOperationMessage { //failed RESOURCE_NOT_FOUND(400, "Resource Not Found"), CREATE_MEASURE_FAIL(401, "Create Measure Failed"), - DELETE_MEASURE_BY_ID_FAIL(402, "Delete Measures By Name Failed"), + DELETE_MEASURE_BY_ID_FAIL(402, "Delete Measures By Id Failed"), DELETE_MEASURE_BY_NAME_FAIL(403, "Delete Measures By Name Failed"), UPDATE_MEASURE_FAIL(404, "Update Measure Failed"), CREATE_JOB_FAIL(405, "Create Job Failed"), http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/util/GriffinUtil.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/util/GriffinUtil.java b/service/src/main/java/org/apache/griffin/core/util/GriffinUtil.java deleted file mode 100644 index 1d5a570..0000000 --- a/service/src/main/java/org/apache/griffin/core/util/GriffinUtil.java +++ /dev/null @@ -1,91 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ - -package org.apache.griffin.core.util; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.config.PropertiesFactoryBean; -import org.springframework.core.io.ClassPathResource; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Properties; - -public class GriffinUtil { - private static final Logger LOGGER = LoggerFactory.getLogger(GriffinUtil.class); - - public static String toJson(Object obj) { - ObjectMapper mapper = new ObjectMapper(); - String jsonStr = null; - try { - jsonStr = mapper.writeValueAsString(obj); - } catch (JsonProcessingException e) { - LOGGER.error("convert to json failed. {}", obj); - } - return jsonStr; - } - - public static String toJsonWithFormat(Object obj) { - ObjectWriter mapper = new ObjectMapper().writer().withDefaultPrettyPrinter(); - String jsonStr = null; - try { - jsonStr = mapper.writeValueAsString(obj); - } catch (JsonProcessingException e) { - LOGGER.error("convert to json failed. {}", obj); - } - return jsonStr; - } - - public static <T> T toEntity(String jsonStr, Class<T> type) throws IOException { - if (jsonStr == null || jsonStr.length() == 0) { - LOGGER.warn("jsonStr {} is empty!", type); - return null; - } - ObjectMapper mapper = new ObjectMapper(); - return mapper.readValue(jsonStr, type); - } - - public static <T> T toEntity(String jsonStr, TypeReference type) throws IOException { - if (jsonStr == null || jsonStr.length() == 0) { - LOGGER.warn("jsonStr {} is empty!", type); - return null; - } - ObjectMapper mapper = new ObjectMapper(); - return mapper.readValue(jsonStr, type); - } - - public static Properties getProperties(String propertiesPath) { - PropertiesFactoryBean propertiesFactoryBean = new PropertiesFactoryBean(); - propertiesFactoryBean.setLocation(new ClassPathResource(propertiesPath)); - Properties properties = null; - try { - propertiesFactoryBean.afterPropertiesSet(); - properties = propertiesFactoryBean.getObject(); - } catch (IOException e) { - LOGGER.error("get properties from {} failed. {}", propertiesPath, e.getMessage()); - } - return properties; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/util/JsonUtil.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/util/JsonUtil.java b/service/src/main/java/org/apache/griffin/core/util/JsonUtil.java new file mode 100644 index 0000000..8a3f686 --- /dev/null +++ b/service/src/main/java/org/apache/griffin/core/util/JsonUtil.java @@ -0,0 +1,77 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +package org.apache.griffin.core.util; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.config.PropertiesFactoryBean; +import org.springframework.core.io.ClassPathResource; + +import java.io.IOException; +import java.util.Properties; + +public class JsonUtil { + private static final Logger LOGGER = LoggerFactory.getLogger(JsonUtil.class); + + public static String toJson(Object obj) { + ObjectMapper mapper = new ObjectMapper(); + String jsonStr = null; + try { + jsonStr = mapper.writeValueAsString(obj); + } catch (JsonProcessingException e) { + LOGGER.error("convert to json failed. {}", obj); + } + return jsonStr; + } + + public static String toJsonWithFormat(Object obj) { + ObjectWriter mapper = new ObjectMapper().writer().withDefaultPrettyPrinter(); + String jsonStr = null; + try { + jsonStr = mapper.writeValueAsString(obj); + } catch (JsonProcessingException e) { + LOGGER.error("convert to json failed. {}", obj); + } + return jsonStr; + } + + public static <T> T toEntity(String jsonStr, Class<T> type) throws IOException { + if (jsonStr == null || jsonStr.length() == 0) { + LOGGER.warn("jsonStr {} is empty!", type); + return null; + } + ObjectMapper mapper = new ObjectMapper(); + return mapper.readValue(jsonStr, type); + } + + public static <T> T toEntity(String jsonStr, TypeReference type) throws IOException { + if (jsonStr == null || jsonStr.length() == 0) { + LOGGER.warn("jsonStr {} is empty!", type); + return null; + } + ObjectMapper mapper = new ObjectMapper(); + return mapper.readValue(jsonStr, type); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/main/java/org/apache/griffin/core/util/PropertiesUtil.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/util/PropertiesUtil.java b/service/src/main/java/org/apache/griffin/core/util/PropertiesUtil.java new file mode 100644 index 0000000..ee57ddd --- /dev/null +++ b/service/src/main/java/org/apache/griffin/core/util/PropertiesUtil.java @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +package org.apache.griffin.core.util; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.config.PropertiesFactoryBean; +import org.springframework.core.io.ClassPathResource; + +import java.io.IOException; +import java.util.Properties; + +public class PropertiesUtil { + private static final Logger LOGGER = LoggerFactory.getLogger(PropertiesUtil.class); + + public static Properties getProperties(String propertiesPath) { + PropertiesFactoryBean propertiesFactoryBean = new PropertiesFactoryBean(); + propertiesFactoryBean.setLocation(new ClassPathResource(propertiesPath)); + Properties properties = null; + try { + propertiesFactoryBean.afterPropertiesSet(); + properties = propertiesFactoryBean.getObject(); + } catch (IOException e) { + LOGGER.error("get properties from {} failed. {}", propertiesPath, e.getMessage()); + } + return properties; + } +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/test/java/org/apache/griffin/core/info/GriffinInfoControllerTest.java ---------------------------------------------------------------------- diff --git a/service/src/test/java/org/apache/griffin/core/info/GriffinInfoControllerTest.java b/service/src/test/java/org/apache/griffin/core/info/GriffinInfoControllerTest.java new file mode 100644 index 0000000..426d705 --- /dev/null +++ b/service/src/test/java/org/apache/griffin/core/info/GriffinInfoControllerTest.java @@ -0,0 +1,48 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +package org.apache.griffin.core.info; + +import org.apache.griffin.core.util.URLHelper; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.MockMvc; + +import static org.hamcrest.CoreMatchers.is; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@RunWith(SpringRunner.class) +@WebMvcTest(value = GriffinInfoController.class, secure = false) +public class GriffinInfoControllerTest { + + @Autowired + private MockMvc mockMvc; + + @Test + public void testGreeting() throws Exception { + mockMvc.perform(get(URLHelper.API_VERSION_PATH + "/version")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", is("0.1.0"))); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/test/java/org/apache/griffin/core/job/JobControllerTest.java ---------------------------------------------------------------------- diff --git a/service/src/test/java/org/apache/griffin/core/job/JobControllerTest.java b/service/src/test/java/org/apache/griffin/core/job/JobControllerTest.java index 0ce227e..a73ba73 100644 --- a/service/src/test/java/org/apache/griffin/core/job/JobControllerTest.java +++ b/service/src/test/java/org/apache/griffin/core/job/JobControllerTest.java @@ -89,7 +89,7 @@ public class JobControllerTest { .content(schedulerRequestBodyJson)) .andExpect(status().isOk()) .andExpect(jsonPath("$.code", is(205))) - .andExpect(jsonPath("$.description", is("CREATE Job Succeed"))) + .andExpect(jsonPath("$.description", is("Create Job Succeed"))) .andDo(print()); } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/test/java/org/apache/griffin/core/job/JobServiceImplTest.java ---------------------------------------------------------------------- diff --git a/service/src/test/java/org/apache/griffin/core/job/JobServiceImplTest.java b/service/src/test/java/org/apache/griffin/core/job/JobServiceImplTest.java index 56567ff..a838933 100644 --- a/service/src/test/java/org/apache/griffin/core/job/JobServiceImplTest.java +++ b/service/src/test/java/org/apache/griffin/core/job/JobServiceImplTest.java @@ -41,6 +41,9 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Sort; import org.springframework.scheduling.quartz.SchedulerFactoryBean; import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestClientException; +import org.springframework.web.client.RestTemplate; import java.util.*; @@ -49,6 +52,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.BDDMockito.given; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; import static org.quartz.TriggerBuilder.newTrigger; @RunWith(SpringRunner.class) @@ -89,7 +93,7 @@ public class JobServiceImplTest { given(scheduler.getJobGroupNames()).willReturn(Arrays.asList("group")); HashSet<JobKey> set = new HashSet<>(); set.add(new JobKey("name", "group")); - given(scheduler.getJobKeys(GroupMatcher.jobGroupEquals("group"))).willReturn(set); + given(scheduler.getJobKeys(GroupMatcher.anyGroup())).willReturn(set); List<Trigger> triggers = Arrays.asList(newTriggerInstance("name", "group", 3000)); JobKey jobKey = set.iterator().next(); given((List<Trigger>) scheduler.getTriggersOfJob(jobKey)).willReturn(triggers); @@ -117,7 +121,7 @@ public class JobServiceImplTest { given(scheduler.getJobGroupNames()).willReturn(Arrays.asList("group")); HashSet<JobKey> set = new HashSet<>(); set.add(new JobKey("name", "group")); - given(scheduler.getJobKeys(GroupMatcher.jobGroupEquals("group"))).willReturn(set); + given(scheduler.getJobKeys(GroupMatcher.anyGroup())).willReturn(set); JobKey jobKey = set.iterator().next(); GriffinException.GetJobsFailureException exception = getTriggersOfJobExpectException(scheduler, jobKey); assertTrue(exception != null); @@ -204,6 +208,20 @@ public class JobServiceImplTest { assertEquals(service.findInstancesOfJob(groupName, jobName, page, size).size(), 1); } +// @Test +// public void testSyncInstancesOfJob() { +// JobInstance instance = newJobInstance(); +// instance.setSessionId(1234564); +// String group = "groupName"; +// String jobName = "jobName"; +// RestTemplate restTemplate = mock(RestTemplate.class); +// given(jobInstanceRepo.findGroupWithJobName()).willReturn(Arrays.asList((Object) (new Object[]{group, jobName}))); +// given(jobInstanceRepo.findByGroupNameAndJobName(group, jobName)).willReturn(Arrays.asList(instance)); +// given(restTemplate.getForObject("uri", String.class)).willThrow(RestClientException.class); +// RestClientException restClientException = getJobInstanceStatusExpectException(); +// assert (restClientException != null); +// } + @Test public void testGetHealthInfoWithHealthy() throws SchedulerException { Scheduler scheduler = Mockito.mock(Scheduler.class); @@ -212,7 +230,7 @@ public class JobServiceImplTest { JobKey jobKey = new JobKey("test"); Set<JobKey> jobKeySet = new HashSet<>(); jobKeySet.add(jobKey); - given(scheduler.getJobKeys(GroupMatcher.jobGroupEquals("BA"))).willReturn((jobKeySet)); + given(scheduler.getJobKeys(GroupMatcher.anyGroup())).willReturn((jobKeySet)); Pageable pageRequest = new PageRequest(0, 1, Sort.Direction.DESC, "timestamp"); List<JobInstance> scheduleStateList = new ArrayList<>(); @@ -248,6 +266,16 @@ public class JobServiceImplTest { .repeatForever()).startAt(new Date()).build(); } + private RestClientException getJobInstanceStatusExpectException() { + RestClientException exception = null; + try { + service.syncInstancesOfAllJobs(); + } catch (RestClientException e) { + exception = e; + } + return exception; + } + private GriffinException.GetJobsFailureException getTriggersOfJobExpectException(Scheduler scheduler, JobKey jobKey) { GriffinException.GetJobsFailureException exception = null; try { http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/test/java/org/apache/griffin/core/job/SparkSubmitJobTest.java ---------------------------------------------------------------------- diff --git a/service/src/test/java/org/apache/griffin/core/job/SparkSubmitJobTest.java b/service/src/test/java/org/apache/griffin/core/job/SparkSubmitJobTest.java index 60ddf3b..130e66d 100644 --- a/service/src/test/java/org/apache/griffin/core/job/SparkSubmitJobTest.java +++ b/service/src/test/java/org/apache/griffin/core/job/SparkSubmitJobTest.java @@ -23,7 +23,8 @@ import org.apache.griffin.core.job.entity.JobInstance; import org.apache.griffin.core.job.entity.SparkJobDO; import org.apache.griffin.core.job.repo.JobInstanceRepo; import org.apache.griffin.core.measure.repo.MeasureRepo; -import org.apache.griffin.core.util.GriffinUtil; +import org.apache.griffin.core.util.JsonUtil; +import org.apache.griffin.core.util.PropertiesUtil; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -57,7 +58,7 @@ public class SparkSubmitJobTest { @Bean public Properties sparkJobProps() { - return GriffinUtil.getProperties("/sparkJob.properties"); + return PropertiesUtil.getProperties("/sparkJob.properties"); } } @@ -90,9 +91,6 @@ public class SparkSubmitJobTest { given(jobInstanceRepo.save(new JobInstance())).willReturn(new JobInstance()); sparkSubmitJob.execute(context); assertTrue(true); - - } - } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/test/java/org/apache/griffin/core/measure/MeasureControllerTest.java ---------------------------------------------------------------------- diff --git a/service/src/test/java/org/apache/griffin/core/measure/MeasureControllerTest.java b/service/src/test/java/org/apache/griffin/core/measure/MeasureControllerTest.java index e1230d4..5b9ca5b 100644 --- a/service/src/test/java/org/apache/griffin/core/measure/MeasureControllerTest.java +++ b/service/src/test/java/org/apache/griffin/core/measure/MeasureControllerTest.java @@ -85,7 +85,7 @@ public class MeasureControllerTest { mvc.perform(delete(URLHelper.API_VERSION_PATH + "/measure/1").contentType(MediaType.APPLICATION_JSON)) .andExpect(status().isOk()) - .andExpect(jsonPath("$.description", is("Delete Measures By Name Succeed"))) + .andExpect(jsonPath("$.description", is("Delete Measures By Id Succeed"))) .andExpect(jsonPath("$.code", is(202))); } @@ -105,7 +105,7 @@ public class MeasureControllerTest { mvc.perform(delete(URLHelper.API_VERSION_PATH + "/measure/1").contentType(MediaType.APPLICATION_JSON)) .andExpect(status().isOk()) - .andExpect(jsonPath("$.description", is("Delete Measures By Name Failed"))) + .andExpect(jsonPath("$.description", is("Delete Measures By Id Failed"))) .andExpect(jsonPath("$.code", is(402))); } @@ -147,14 +147,12 @@ public class MeasureControllerTest { } @Test - public void testGetAllMeasuresOfOwner() throws Exception { + public void testGetAllMeasuresByOwner() throws Exception { String owner = "test"; - List<Map<String, String>> measureList = new LinkedList<>(); - HashMap<String, String> map = new HashMap<>(); - map.put("name", "view_item_hourly"); - map.put("id", "0"); - measureList.add(map); - given(service.getAllAliveMeasureNameIdByOwner(owner)).willReturn(measureList); + List<Measure> measureList = new LinkedList<>(); + Measure measure = createATestMeasure("view_item_hourly", owner); + measureList.add(measure); + given(service.getAliveMeasuresByOwner(owner)).willReturn(measureList); mvc.perform(get(URLHelper.API_VERSION_PATH + "/measures/owner/" + owner).contentType(MediaType.APPLICATION_JSON)) .andExpect(status().isOk()) http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c7334935/service/src/test/java/org/apache/griffin/core/measure/MeasureOrgControllerTest.java ---------------------------------------------------------------------- diff --git a/service/src/test/java/org/apache/griffin/core/measure/MeasureOrgControllerTest.java b/service/src/test/java/org/apache/griffin/core/measure/MeasureOrgControllerTest.java new file mode 100644 index 0000000..33a2ede --- /dev/null +++ b/service/src/test/java/org/apache/griffin/core/measure/MeasureOrgControllerTest.java @@ -0,0 +1,84 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +package org.apache.griffin.core.measure; + +import org.apache.griffin.core.measure.repo.MeasureRepo; +import org.apache.griffin.core.util.URLHelper; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.MockMvc; + +import java.util.Arrays; +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.hasSize; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@RunWith(SpringRunner.class) +@WebMvcTest(value = MeasureOrgController.class, secure = false) +public class MeasureOrgControllerTest { + + @Autowired + private MockMvc mockMvc; + + @MockBean + private MeasureRepo measureRepo; + + + @Test + public void testGetOrgs() throws Exception { + String org = "orgName"; + when(measureRepo.findOrganizations()).thenReturn(Arrays.asList(org)); + + mockMvc.perform(get(URLHelper.API_VERSION_PATH + "/org")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.[0]", is(org))); + } + + @Test + public void testGetMetricNameListByOrg() throws Exception { + String org = "hadoop"; + when(measureRepo.findNameByOrganization(org)).thenReturn(Arrays.asList(org)); + + mockMvc.perform(get(URLHelper.API_VERSION_PATH + "/org/{org}", org)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.[0]", is(org))); + } + + @Test + public void testGetMeasureNamesGroupByOrg() throws Exception { + List<String> orgs = Arrays.asList("orgName"); + when(measureRepo.findOrganizations()).thenReturn(orgs); + when(measureRepo.findNameByOrganization(orgs.get(0))).thenReturn(Arrays.asList("measureName")); + + mockMvc.perform(get(URLHelper.API_VERSION_PATH + "/org/measure/names")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.orgName", hasSize(1))); + } + +}
