http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqScheduleServiceImpl.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqScheduleServiceImpl.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqScheduleServiceImpl.java deleted file mode 100644 index 27cc138..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqScheduleServiceImpl.java +++ /dev/null @@ -1,670 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.service; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStreamReader; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Calendar; -import java.util.Date; -import java.util.List; -import java.util.Properties; - -import org.apache.commons.io.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.PropertySource; -import org.springframework.stereotype.Service; -//import org.springframework.validation.annotation.Validated; - - - - - - - - - - - - -import com.ebay.oss.griffin.common.Pair; -import com.ebay.oss.griffin.common.ScheduleModelSeperator; -import com.ebay.oss.griffin.domain.DataAsset; -import com.ebay.oss.griffin.domain.DqJob; -import com.ebay.oss.griffin.domain.DqMetricsValue; -import com.ebay.oss.griffin.domain.DqModel; -import com.ebay.oss.griffin.domain.DqSchedule; -import com.ebay.oss.griffin.domain.JobStatus; -import com.ebay.oss.griffin.domain.ModelStatus; -import com.ebay.oss.griffin.domain.ModelType; -import com.ebay.oss.griffin.domain.PartitionFormat; -import com.ebay.oss.griffin.domain.SampleFilePathLKP; -import com.ebay.oss.griffin.domain.ScheduleType; -import com.ebay.oss.griffin.domain.SystemType; -import com.ebay.oss.griffin.repo.DataAssetRepo; -import com.ebay.oss.griffin.repo.DqJobRepo; -import com.ebay.oss.griffin.repo.DqMetricsRepo; -import com.ebay.oss.griffin.repo.DqModelRepo; -import com.ebay.oss.griffin.repo.DqScheduleRepo; -import com.ebay.oss.griffin.vo.AccuracyHiveJobConfig; -import com.ebay.oss.griffin.vo.AccuracyHiveJobConfigDetail; -import com.ebay.oss.griffin.vo.PartitionConfig; -import com.ebay.oss.griffin.vo.ValidateHiveJobConfig; -import com.google.gson.Gson; -import com.mongodb.DBObject; - -@PropertySource("classpath:application.properties") -@Service ("scheduleService") -public class DqScheduleServiceImpl implements DqScheduleService { - private static Logger logger = LoggerFactory.getLogger(DqScheduleServiceImpl.class); - - public static String resultFile = "_RESULT"; - public static String startFile = "_START"; - public static String finishFile = "_FINISH"; - public static String logFile = "dqjoblog"; - - @Autowired - DQMetricsService dqMetricsService; - - @Autowired - DqModelService dqModelService; - - @Autowired DqModelRepo dqModelRepo; - - @Autowired DqMetricsRepo metricsRepo; - - @Autowired - private DqScheduleRepo scheduleRepo; - - @Autowired - private DqJobRepo jobRepo; - - @Autowired - private DataAssetRepo dataAssetRepo; - - @Override - public void schedulingJobs() { - logger.info("===========checking new jobs==============="); - - createJobToRunBySchedule(); - - generateAllWaitingJobsRunningConfigs(); - - checkAllJOBSStatus(); - - updateModelStatus(ModelStatus.TESTING, ModelStatus.VERIFIED); - - logger.info("===========checking jobs done==============="); - } - - void createJobToRunBySchedule() { - for (DqSchedule schedule : scheduleRepo.getAll()) { - long now = new Date().getTime(); - long startTime = schedule.getStarttime(); - if (now < startTime) { - continue; - } - - Calendar c = Calendar.getInstance(); - Date date = new Date(startTime); - c.setTime(date); - int type = schedule.getScheduleType(); - if (type == ScheduleType.DAILY) { - c.add(Calendar.DATE, 1); - } else if (type == ScheduleType.HOURLY) { - c.add(Calendar.HOUR, 1); - } else if (type == ScheduleType.WEEKLY) { - c.add(Calendar.DATE, 7); - } else if (type == ScheduleType.MONTHLY) { - c.add(Calendar.MONTH, 1); - } else { - continue; - } - startTime = c.getTime().getTime(); - schedule.setStarttime(startTime); - - DqJob job = new DqJob(); - job.setModelList(schedule.getModelList()); - job.setStarttime(startTime); - job.setStatus(0); - job.setId(schedule.getModelList() + "_" + startTime); // this is the job.id generation logic - job.setJobType(schedule.getJobType()); - int result = jobRepo.newJob(job); - if (result == 0) { - logger.info("===================new model failure"); - continue; - } - - scheduleRepo.save(schedule); - - } - } - - String updateHDFSDirTemplateString(String dir,String dateString,String hourString) { - String result = dir; - result = result.replaceAll("\\[YYYYMMDD\\]", dateString); - result = result.replaceAll("\\[YYYY\\-MM\\-DD\\]", dateString.substring(0,4)+"-"+dateString.substring(4,6)+"-"+dateString.substring(6,8)); - result = result.replaceAll("\\[YYYY\\]", dateString.substring(0,4)); - result = result.replaceAll("\\[YY\\]", dateString.substring(2,4)); - result = result.replaceAll("\\[MM\\]", dateString.substring(4,6)); - result = result.replaceAll("\\[DD\\]", dateString.substring(6,8)); - result = result.replaceAll("\\[HH\\]", hourString); - result = result.replaceAll("\\[yyyymmdd\\]", dateString); - result = result.replaceAll("\\[yyyy\\-mm\\-dd\\]", dateString.substring(0,4)+"-"+dateString.substring(4,6)+"-"+dateString.substring(6,8)); - result = result.replaceAll("\\[yyyy\\]", dateString.substring(0,4)); - result = result.replaceAll("\\[yy\\]", dateString.substring(3,4)); - result = result.replaceAll("\\[mm\\]", dateString.substring(4,6)); - result = result.replaceAll("\\[dd\\]", dateString.substring(6,8)); - result = result.replaceAll("\\[hh\\]", hourString); - return result; - } - - void generateAllWaitingJobsRunningConfigs() { - try{ - logger.info("===========generating running config==============="); - Properties env = new Properties(); - env.load(Thread.currentThread().getContextClassLoader() - .getResourceAsStream("application.properties")); - String environment = env.getProperty("env"); - - for(DqJob eachJob : jobRepo.getByStatus(JobStatus.READY)) { - String jobid = eachJob.getId(); - int jobtype = eachJob.getJobType(); - StringBuffer doneFiles = new StringBuffer(); - StringBuffer runningParameter = new StringBuffer(); - - if(jobtype==ModelType.ACCURACY) { - String modelid = eachJob.getModelList(); - long ts = eachJob.getStarttime(); - Date dt = new Date(ts); - SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd"); - String dateString = formatter.format(dt); - SimpleDateFormat formatter2 = new SimpleDateFormat("HH"); - String hourString = formatter2.format(dt); - - DqModel model = dqModelRepo.findByColumn("modelId", modelid); - if(model==null) { - logger.warn( "===================can not find model "+modelid); - continue; - } - - String content = model.getModelContent(); - - String[] contents = content.split("\\|"); - String srcPlatform = contents[0]; - String srcSystem = contents[1]; - String tgtPlatform = contents[2]; - String tgtSystem = contents[3]; - - String[] attributesArray = contents[4].split(";"); - String[] attributes = attributesArray[0].split(","); - String srcDataset = attributes[0].substring(0,attributes[0].lastIndexOf(".")); - String tgtDataset = attributes[1].substring(0,attributes[1].lastIndexOf(".")); - - // runningParameter.append(System.getProperty("line.separator")+srcPlatform+" "+srcSystem+" "+srcDataset); - // runningParameter.append(System.getProperty("line.separator")+tgtPlatform+" "+tgtSystem+" "+tgtDataset); - - List<Pair> queryList = new ArrayList<Pair>(); - queryList.add(new Pair("platform", srcPlatform)); - queryList.add(new Pair("system", srcSystem)); - queryList.add(new Pair("assetName", srcDataset)); - logger.info( "===================find source object "+srcPlatform+" "+srcSystem+" "+srcDataset); - DBObject srcObj = dataAssetRepo.getByCondition(queryList); - DataAsset srcAsset = new DataAsset(srcObj); - - List<Pair> queryList2 = new ArrayList<Pair>(); - queryList2.add(new Pair("platform", tgtPlatform)); - queryList2.add(new Pair("system", tgtSystem)); - queryList2.add(new Pair("assetName", tgtDataset)); - logger.info( "===================find target object "+tgtPlatform+" "+tgtSystem+" "+tgtDataset); - DBObject tgtObj = dataAssetRepo.getByCondition(queryList2); - DataAsset tgtAsset = new DataAsset(tgtObj); - - doneFiles.append(updateHDFSDirTemplateString(srcAsset.getAssetHDFSPath(),dateString,hourString) - +System.getProperty("line.separator") - +updateHDFSDirTemplateString(tgtAsset.getAssetHDFSPath(),dateString,hourString) - +System.getProperty("line.separator")); - if(model.getSchedule()==ScheduleType.HOURLY && model.getSystem()==SystemType.BULLSEYE) - { - Date dt4be = new Date(ts+3600000); -// SimpleDateFormat formatter4be = new SimpleDateFormat("yyyyMMdd"); - String dateString4be = formatter.format(dt4be); -// SimpleDateFormat formatter24be = new SimpleDateFormat("HH"); - String hourString4be = formatter2.format(dt4be); - doneFiles.append(updateHDFSDirTemplateString(tgtAsset.getAssetHDFSPath(),dateString4be,hourString4be) - +System.getProperty("line.separator")); - } - - - AccuracyHiveJobConfig config = new AccuracyHiveJobConfig(); - List<AccuracyHiveJobConfigDetail> configDetailList = new ArrayList<AccuracyHiveJobConfigDetail>(); - for(String tempAttribute : attributesArray) - { - String[] tempAttributeArray = tempAttribute.split(","); - String srcColName = tempAttributeArray[0].substring(tempAttributeArray[0].lastIndexOf(".")+1); - String tgtColName = tempAttributeArray[1].substring(tempAttributeArray[1].lastIndexOf(".")+1); - configDetailList.add(new AccuracyHiveJobConfigDetail( - srcAsset.getColId(srcColName), srcColName - , tgtAsset.getColId(tgtColName), tgtColName - ,tempAttributeArray[3], Boolean.parseBoolean(tempAttributeArray[2].toUpperCase()) - ) ); - } - config.setAccuracyMapping(configDetailList); - config.setSource(srcAsset.getAssetName()); - config.setTarget(tgtAsset.getAssetName()); - - config.setSrcPartitions(getPartitionList(srcAsset, ts)); - - List<List<PartitionConfig>> tgtPartitions = new ArrayList<List<PartitionConfig>>(); - tgtPartitions.add(getPartitionList(tgtAsset, ts)); - if(model.getSchedule()==ScheduleType.HOURLY - && model.getSystem()==SystemType.BULLSEYE) { - tgtPartitions.add(getPartitionList(tgtAsset, ts+3600000)); - } - - config.setTgtPartitions(tgtPartitions); - - Gson gson = new Gson(); - runningParameter.append(gson.toJson(config)+System.getProperty("line.separator")); - - } else if(jobtype==ModelType.VALIDITY) { - - String modelList = eachJob.getModelList(); - long ts = eachJob.getStarttime(); - Date dt = new Date(ts); - SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd"); - String dateString = formatter.format(dt); - SimpleDateFormat formatter2 = new SimpleDateFormat("HH"); - String hourString = formatter2.format(dt); - - List<String> models = new ArrayList<String>(); - if(!modelList.contains(ScheduleModelSeperator.SEPERATOR)) - { - models.add(modelList); - } - else - { - models = Arrays.asList(modelList.split(ScheduleModelSeperator.SPLIT_SEPERATOR)); - } - - - if(models.size()==0) return; - logger.debug("+++ model id value: " + models.get(0)); - DqModel model = dqModelRepo.findByColumn("modelId", models.get(0)); - logger.debug("--- model: " + model); - if(model == null){ - continue ; - } - DataAsset srcAsset = dataAssetRepo.getById((long)model.getAssetId()); - - - doneFiles.append(updateHDFSDirTemplateString(srcAsset.getAssetHDFSPath(),dateString,hourString) - +System.getProperty("line.separator")); - - ValidateHiveJobConfig config = new ValidateHiveJobConfig(srcAsset.getAssetName()); - config.setTimePartitions(getPartitionList(srcAsset, ts)); - - for(String modelname : models) { - model = dqModelRepo.findByColumn("modelId", modelname); - if(model==null) { - logger.warn("===================can not find model "+modelname); - continue; - } - - String content = model.getModelContent(); - String[] contents = content.split("\\|"); - String calType = contents[2]; - String calColname = contents[3]; - - config.addColumnCalculation(srcAsset.getColId(calColname), calColname, Integer.parseInt(calType)); - } - - Gson gson = new Gson(); - runningParameter.append(gson.toJson(config)+System.getProperty("line.separator")); - } - - - - - logger.info( "===================="+env.getProperty("job.local.folder")+File.separator+jobid+File.separator+"cmd.txt"); - - String dir = env.getProperty("job.local.folder")+File.separator+jobid+File.separator+"cmd.txt"; - createFile(dir); - File file = new File(dir); - FileWriter fw = new FileWriter(file.getAbsoluteFile()); - BufferedWriter bw = new BufferedWriter(fw); - bw.write(runningParameter.toString()); - bw.flush(); - bw.close(); - - String dir2 = env.getProperty("job.local.folder")+File.separator+jobid+File.separator+"watchfile.txt"; - createFile(dir2); - File file2 = new File(dir2); - FileWriter fw2 = new FileWriter(file2.getAbsoluteFile()); - BufferedWriter bw2 = new BufferedWriter(fw2); - bw2.write(doneFiles.toString()); - bw2.flush(); - bw2.close(); - - logger.info("====================create file done"); - - if(environment.equals("prod")) { - String hdfs = env.getProperty("job.hdfs.folder")+"/"+env.getProperty("job.hdfs.runningfoldername"); - Process process1 = Runtime.getRuntime().exec("hadoop fs -mkdir "+hdfs+File.separator+jobid); - logger.info("====================hadoop fs -mkdir "+hdfs+File.separator+jobid); - process1.waitFor();; - Process process2 = Runtime.getRuntime().exec("hadoop fs -put "+dir+" "+hdfs+File.separator+jobid+File.separator); - logger.info( "====================hadoop fs -put "+dir+" "+hdfs+File.separator+jobid+File.separator); - process2.waitFor(); - Process process2_1 = Runtime.getRuntime().exec("hadoop fs -put "+dir2+" "+hdfs+File.separator+jobid+File.separator+"_watchfile"); - logger.info("====================hadoop fs -put "+dir2+" "+hdfs+File.separator+jobid+File.separator+"_watchfile"); - process2_1.waitFor(); - Process process3 = Runtime.getRuntime().exec("hadoop fs -touchz "+hdfs+File.separator+jobid+File.separator+"_type_"+jobtype+".done"); - logger.info( "====================hadoop fs -touchz "+hdfs+File.separator+jobid+File.separator+"_type_"+jobtype+".done"); - process3.waitFor(); - - } - - //file.delete(); - new File(env.getProperty("job.local.folder")+File.separator+jobid).delete(); - logger.info( "====================delete file done"); - - eachJob.setStatus(JobStatus.WAITING); - jobRepo.update(eachJob); - logger.info("====================udpate status done"); - } - - - } catch(Exception e) { - logger.error(e.toString(), e); - } - - } - - List<PartitionConfig> getPartitionList(DataAsset srcAsset, long ts) { - Date dt = new Date(ts); - List<PartitionConfig> partitions = new ArrayList<PartitionConfig>(); - List<PartitionFormat> lv1partitions = srcAsset.getPartitions(); - if(lv1partitions!=null) - { - for(PartitionFormat tempPartitionFormat : lv1partitions) - { - SimpleDateFormat tempFormatter = new SimpleDateFormat(tempPartitionFormat.getFormat()); - String tempdateString = tempFormatter.format(dt); - partitions.add(new PartitionConfig(tempPartitionFormat.getName(), tempdateString)); - } - } - return partitions; - } - - void checkAllJOBSStatus() { - try { - Properties env = new Properties(); - env.load(Thread.currentThread().getContextClassLoader() - .getResourceAsStream("application.properties")); - String hdfsbasedir = env.getProperty("job.hdfs.folder"); - String runningfoldername = env.getProperty("job.hdfs.runningfoldername"); - String historyfoldername = env.getProperty("job.hdfs.historyfoldername"); - String failurefoldername = env.getProperty("job.hdfs.failurefoldername"); - String environment = env.getProperty("env"); - String localdir = env.getProperty("job.local.folder"); - if(!environment.equals("prod")) return; - - int result; - Process processLV1 = Runtime.getRuntime().exec("hadoop fs -ls "+hdfsbasedir+"/"+runningfoldername); - result = processLV1.waitFor(); - if(result != 0) { - logger.info("===================="+"hadoop fs -ls "+hdfsbasedir+"/"+runningfoldername+" error"); - return; - } - - BufferedReader readerLV1 = new BufferedReader(new InputStreamReader(processLV1.getInputStream())); - String lineLV1; - int index; - while ((lineLV1 = readerLV1.readLine()) != null) { - index = lineLV1.indexOf("/"); - if(index==-1) continue; - String runningJobFolderLV1Dir = lineLV1.substring(index); - logger.info("===================checking hdfs folder"+runningJobFolderLV1Dir); - Process processLV2 = Runtime.getRuntime().exec("hadoop fs -ls "+runningJobFolderLV1Dir); - result = processLV2.waitFor(); - if(result != 0) - { - logger.warn("===================="+"hadoop fs -ls "+runningJobFolderLV1Dir+" error"); - continue; - } - BufferedReader readerLV2 = new BufferedReader(new InputStreamReader(processLV2.getInputStream())); -// if(readerLV2==null) return; - - String lineLV2; - int startindi = 0; - int resultindi = 0; - int logindi = 0; - while ((lineLV2 = readerLV2.readLine()) != null) { - index = lineLV2.indexOf("/"); - if(index==-1) continue; - String runningJobContentDir = lineLV2.substring(index); - logger.info("===================checking hdfs folder"+runningJobContentDir); - if(runningJobContentDir.indexOf(startFile)!=-1) - startindi = 1; - else if(runningJobContentDir.indexOf(resultFile)!=-1) - resultindi = 1; - else if(runningJobContentDir.indexOf(logFile)!=-1) - logindi = 1; - } - - String jobID = runningJobFolderLV1Dir.substring(runningJobFolderLV1Dir.indexOf(runningfoldername)+runningfoldername.length()+1); - logger.info("===================job id: "+jobID); - DqJob job = jobRepo.getById(jobID); - if(job==null) { - logger.warn("===================no such job: "+job); - continue; - } - - if(startindi == 1) { - logger.info("===================start"); - if(environment.equals("prod")) { - job.setJobType(JobStatus.STARTED); // FIXME numeric issue???!!! - jobRepo.update(job); - } - logger.info("===================udpate job status to started"); - // Process processChangeStartFile = Runtime.getRuntime().exec("hadoop fs -mv "+runningJobFolderLV1Dir+"/"+startFile+" "+runningJobFolderLV1Dir+"/_RUNNING"); - // result = processChangeStartFile.waitFor(); - } - - if(resultindi ==1) { - logger.info("===================finished"); - - if(environment.equals("prod")) { - String historyJobFolderLV1Dir = runningJobFolderLV1Dir.replaceAll(runningfoldername, historyfoldername); - Process processMoveFolder = Runtime.getRuntime().exec("hadoop fs -mv "+runningJobFolderLV1Dir+" "+historyJobFolderLV1Dir); - result = processMoveFolder.waitFor(); - if(result != 0) - { - logger.warn("===================="+"hadoop fs -mv "+runningJobFolderLV1Dir+" "+historyJobFolderLV1Dir+" error"); - continue; - } - logger.info("===================moved to history folder"); - - logger.info("===================publish metrics."); - - String hdfs = env.getProperty("job.hdfs.folder")+"/"+env.getProperty("job.hdfs.historyfoldername"); - String resultLocalFileDir = localdir+File.separator+jobID+File.separator+resultFile; - createFile(resultLocalFileDir); - new File(resultLocalFileDir).delete(); - Process process1 = Runtime.getRuntime().exec("hadoop fs -get "+hdfs+File.separator+jobID+File.separator+resultFile+" "+resultLocalFileDir); - logger.info("====================hadoop fs -get "+hdfs+File.separator+jobID+File.separator+resultFile+" "+resultLocalFileDir); - process1.waitFor(); - - File rFile = new File(resultLocalFileDir); - BufferedReader reader = new BufferedReader(new FileReader(rFile)); - String resultValue = reader.readLine(); - IOUtils.closeQuietly(reader); - - String metricsNames = jobID.substring(0, jobID.lastIndexOf("_")); - List<String> metricsNameArray = new ArrayList<String>(); - if(!metricsNames.contains(ScheduleModelSeperator.SEPERATOR)) - { - metricsNameArray.add(metricsNames); - } else { - metricsNameArray = Arrays.asList(metricsNames.split(ScheduleModelSeperator.SPLIT_SEPERATOR)); - } - - for(String metricsName : metricsNameArray) { - DqModel model = dqModelRepo.findByName(metricsName); - if(model.getModelType() == ModelType.ACCURACY) { - float floatResultValue = -1; - long ts = -1; - try{ - floatResultValue = Float.parseFloat(resultValue); - ts = Long.parseLong(jobID.substring(jobID.lastIndexOf("_")+1)); - } catch(Exception e) { - logger.error(e.toString(), e); - } - - if(floatResultValue >= 0 && ts>= 0) { - DqMetricsValue newDQMetricsValue = new DqMetricsValue(); - newDQMetricsValue.setMetricName(jobID.substring(0, jobID.lastIndexOf("_"))); - newDQMetricsValue.setTimestamp(ts); - newDQMetricsValue.setValue(floatResultValue); - logger.info("===================new accuracy dq metrics: "+newDQMetricsValue.getMetricName()+" "+newDQMetricsValue.getTimestamp()+" "+newDQMetricsValue.getTimestamp()); - dqMetricsService.insertMetadata(newDQMetricsValue); - -// object.put("endtime", new Date().getTime()); -// object.put("value", value); - job.setEndtime(ts); - job.setValue(ts); - jobRepo.update(job); - } - - //insert missing data path to mongo - - SampleFilePathLKP sfp = new SampleFilePathLKP(); - - sfp.setHdfsPath(historyJobFolderLV1Dir + "/" + "missingRec.txt"); - sfp.setModelName(jobID.substring(0, jobID.lastIndexOf("_"))); - sfp.setTimestamp(ts); - - dqMetricsService.insertSampleFilePath(sfp); - - - } else if(model.getModelType() == ModelType.VALIDITY) { - Gson gson = new Gson(); - ValidateHiveJobConfig resultObject = gson.fromJson(resultValue.toString(), ValidateHiveJobConfig.class); - String content = model.getModelContent(); - String[] contents = content.split("\\|"); - String calType = contents[2]; - String calColname = contents[3]; - long tempValue = resultObject.getValue(calColname, Integer.parseInt(calType)); - - long ts = -1; - try{ - ts = Long.parseLong(jobID.substring(jobID.lastIndexOf("_")+1)); - } catch(Exception e) { - logger.warn(e.toString(), e); - } - - if(tempValue >= 0 && ts>= 0) { - DqMetricsValue newDQMetricsValue = new DqMetricsValue(); - newDQMetricsValue.setMetricName(metricsName); - newDQMetricsValue.setTimestamp(ts); - newDQMetricsValue.setValue(tempValue); - logger.warn("===================new validity dq metrics: "+metricsName+" "+ts+" "+tempValue); - dqMetricsService.insertMetadata(newDQMetricsValue); - - job.setEndtime(ts); - job.setValue(tempValue); - jobRepo.update(job); - } - } - - logger.warn("===================publish metrics done."); - } - } - - job.setJobType(JobStatus.FINISHED); - jobRepo.update(job); - - } else if(logindi == 1 && resultindi == 0) { - if(environment.equals("prod")) { - String failureJobFolderLV1Dir = runningJobFolderLV1Dir.replaceAll(runningfoldername, failurefoldername); - Process processMoveFolder = Runtime.getRuntime().exec("hadoop fs -mv "+runningJobFolderLV1Dir+" "+failureJobFolderLV1Dir); - result = processMoveFolder.waitFor(); - if(result != 0) - { - logger.warn("===================="+"hadoop fs -mv "+runningJobFolderLV1Dir+" "+failureJobFolderLV1Dir+" error"); - continue; - } - logger.warn("===================moved to history folder"); - } - } else { - logger.warn("===================waiting"); - } - - } - - - - } catch (Exception e) { - logger.warn(e.toString(), e); - } - } - - boolean createFile(String destFileName) { - File file = new File(destFileName); - if(file.exists()) { - return false; - } - if (destFileName.endsWith(File.separator)) { - return false; - } - if(!file.getParentFile().exists()) { - if(!file.getParentFile().mkdirs()) { - return false; - } - } - try { - if (file.createNewFile()) { - return true; - } else { - return false; - } - } catch (IOException e) { - e.printStackTrace(); - return false; - } - } - - protected void updateModelStatus(int fromStatus, int toStatus) { - List<DqModel> allmodels = dqModelRepo.getByStatus(fromStatus); - for (DqModel model : allmodels) { - List<DqMetricsValue> allMetrics = metricsRepo.getByMetricsName(model.getModelName()); - if (allMetrics.size() >= DqModelCreator.MIN_TESTING_JOB_NUMBER) { - model.setStatus(toStatus); - dqModelRepo.update(model); - } - } - } - -}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/DummyLoginService.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DummyLoginService.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/DummyLoginService.java deleted file mode 100644 index e0cbe91..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DummyLoginService.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.service; - -import org.springframework.stereotype.Service; - -@Service -public class DummyLoginService implements LoginService { - - @Override - public String login(String ntUser, String password) { - return ntUser; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/LoginService.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/LoginService.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/LoginService.java deleted file mode 100644 index f21e45c..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/LoginService.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.service; - - - -public interface LoginService { - public String login(String ntUser, String password); -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/ModelInputConverter.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/ModelInputConverter.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/ModelInputConverter.java deleted file mode 100644 index 162c46e..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/ModelInputConverter.java +++ /dev/null @@ -1,80 +0,0 @@ -package com.ebay.oss.griffin.service; - -import org.springframework.stereotype.Component; - -import com.ebay.oss.griffin.domain.DqModel; -import com.ebay.oss.griffin.domain.ModelType; -import com.ebay.oss.griffin.vo.ModelBasicInputNew; -import com.ebay.oss.griffin.vo.ModelExtraInputNew; -import com.ebay.oss.griffin.vo.ModelInput; - -@Component("modelInputConverter") -public class ModelInputConverter implements Converter<DqModel, ModelInput> { - - @Override - public ModelInput voOf(DqModel dqModel) { - if(dqModel == null){ - return null; - } - - // Object result = sourceObject; - int modelType = dqModel.getModelType(); - ModelInput result = new ModelInput(); - result.setBasic(getViewModelForFront(dqModel)); - - if (modelType == ModelType.ACCURACY) { - result.parseFromString(dqModel.getModelContent()); - } else if (modelType == ModelType.VALIDITY) { - - ModelExtraInputNew extra = result.getExtra(); - String content = dqModel.getModelContent(); - String[] contents = content.split("\\|"); - extra.setSrcDb(contents[0]); - extra.setSrcDataSet(contents[1]); - extra.setVaType(Integer.parseInt(contents[2])); - extra.setColumn(contents[3]); - - - } else if (modelType == ModelType.ANOMALY) { - - ModelExtraInputNew extra = result.getExtra(); - String content = dqModel.getModelContent(); - String[] contents = content.split("\\|"); - extra.setSrcDb(contents[0]); - extra.setSrcDataSet(contents[1]); - int type = Integer.parseInt(contents[2]); - extra.setAnType(type); - - } else if (modelType == ModelType.PUBLISH) { - - result.getExtra().setPublishUrl(dqModel.getModelContent()); - } - - return result; - } - - ModelBasicInputNew getViewModelForFront(DqModel sourceObject) { - ModelBasicInputNew basic = new ModelBasicInputNew(); - basic.setDesc(sourceObject.getModelDesc()); - basic.setName(sourceObject.getModelName()); - basic.setDataaset(sourceObject.getAssetName()); - basic.setDataasetId(sourceObject.getAssetId()); - basic.setStatus(sourceObject.getStatus()); - basic.setType(sourceObject.getModelType()); - basic.setScheduleType(sourceObject.getSchedule()); - basic.setSystem(sourceObject.getSystem()); - basic.setEmail(sourceObject.getNotificationEmail()); - basic.setOwner(sourceObject.getOwner()); - basic.setThreshold(sourceObject.getThreshold()); - - return basic; - } - - @Override - public DqModel entityOf(ModelInput vo) { - // TODO Auto-generated method stub - throw new RuntimeException("not implemented yet..."); - } - - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/NotificationService.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/NotificationService.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/NotificationService.java deleted file mode 100644 index 816944b..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/NotificationService.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ -package com.ebay.oss.griffin.service; - -import java.util.List; - -import com.ebay.oss.griffin.vo.NotificationRecord; - -public interface NotificationService { - public void insert(NotificationRecord record); - public List<NotificationRecord> getAll(); - public void delete(NotificationRecord record); - public void delete(int id); - public NotificationRecord get(int id); - public List<NotificationRecord> getTop(int limit); - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/NotificationServiceImpl.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/NotificationServiceImpl.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/NotificationServiceImpl.java deleted file mode 100644 index 264e5f8..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/NotificationServiceImpl.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.service; - -import java.util.ArrayList; -import java.util.List; - -import org.springframework.stereotype.Service; - -import com.ebay.oss.griffin.vo.NotificationRecord; - -@Service -public class NotificationServiceImpl implements NotificationService { - private static List<NotificationRecord> records = new ArrayList<NotificationRecord>(); - private static int count = 1; - - @Override - public void insert(NotificationRecord record) { - record.setId(count++); - records.add(0, record); - - } - - @Override - public List<NotificationRecord> getAll() { - return records; - } - - @Override - public void delete(NotificationRecord record) { - int index = records.indexOf(record); - if(index > -1){ - records.remove(index); - }else if(record.getId()>0){ - delete(record.getId()); - } - - } - - @Override - public void delete(int id) { - int length = records.size(); - for(int i = 0; i < length; i ++){ - if(records.get(i).getId() == id){ - records.remove(i); - break; - } - } - - } - - @Override - public NotificationRecord get(int id) { - int length = records.size(); - for(int i = 0; i < length; i ++){ - if(records.get(i).getId() == id){ - return records.get(i); - } - } - return null; - } - - @Override - public List<NotificationRecord> getTop(int limit) { - // TODO Auto-generated method stub - return records.subList(0, limit); - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/RefMetrcsCalc.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/RefMetrcsCalc.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/RefMetrcsCalc.java deleted file mode 100644 index 87b8e02..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/RefMetrcsCalc.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.service; - -import com.ebay.oss.griffin.vo.SystemLevelMetricsList; - -public interface RefMetrcsCalc { - - void calc(SystemLevelMetricsList totalSystemLevelMetricsList); - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/RefMetricsCalcImpl.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/RefMetricsCalcImpl.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/RefMetricsCalcImpl.java deleted file mode 100644 index d1f50f8..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/RefMetricsCalcImpl.java +++ /dev/null @@ -1,347 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.service; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.util.StringUtils; - -import com.ebay.oss.griffin.domain.AnomalyType; -import com.ebay.oss.griffin.domain.DqMetricsValue; -import com.ebay.oss.griffin.domain.DqModel; -import com.ebay.oss.griffin.domain.MetricType; -import com.ebay.oss.griffin.domain.ModelType; -import com.ebay.oss.griffin.domain.ScheduleType; -import com.ebay.oss.griffin.domain.SystemType; -import com.ebay.oss.griffin.repo.DqMetricsRepo; -import com.ebay.oss.griffin.repo.DqModelRepo; -import com.ebay.oss.griffin.vo.AssetLevelMetricsDetail; -import com.ebay.oss.griffin.vo.BollingerBandsEntity; -import com.ebay.oss.griffin.vo.MADEntity; -import com.ebay.oss.griffin.vo.SystemLevelMetricsList; - -@Component -public class RefMetricsCalcImpl implements RefMetrcsCalc { - - private static Logger logger = LoggerFactory.getLogger(RefMetricsCalcImpl.class); - - @Autowired - DqModelService dqModelService; - - @Autowired - DqModelRepo modelRepo; - - @Autowired - DqMetricsRepo metricsRepo; - - Map<String, String> modelName_system; - String getSystemType(String modelName) { - if(modelName_system==null) { - modelName_system = new HashMap<String, String>(); - - for(DqModel model : modelRepo.getAll()) { - modelName_system.put(model.getModelName(), SystemType.val(model.getModelType())); - } - } - return modelName_system.get(modelName); - } - - @Override - public void calc(SystemLevelMetricsList totalSystemLevelMetricsList) { - Map<String, List<String>> references = getReferences(); - - for(String modelName : references.keySet()) { - List<String> refNames = references.get(modelName); - - for (String referencerName : refNames) { - calc(modelName, referencerName, totalSystemLevelMetricsList); - } - } - } - - private void calc(String modelName, String referencerName, SystemLevelMetricsList totalSystemLevelMetricsList) { - logger.info("==============anmoni loop start==================" + referencerName + " " + modelName); - DqModel refModel = dqModelService.getGeneralModel(referencerName); - if (refModel == null) { - logger.warn("==============referencerModel is null================== "+referencerName); -// return false; - return; - } - - DqModel sourceModel = dqModelService.getGeneralModel(modelName); - if (sourceModel == null) { - logger.warn("==============sourceModel is null================== "+sourceModel); -// return false; - return; - } - if (refModel.getModelType() != ModelType.ANOMALY) { - logger.warn("==============non-anomaly model founded================== "+referencerName); -// return true; - return; - } - -// public static int trendLength = 20 * 24; -// public static int trendOffset = 24 * 7; - - List<DqMetricsValue> metricList = metricsRepo.getByMetricsName(modelName); - Collections.sort(metricList); - - String content = refModel.getModelContent(); - String[] contents = content.split("\\|"); - int type = Integer.parseInt(contents[2]); - - if (type == AnomalyType.HISTORY_TREND) { - calcHistoryRefModel(sourceModel, refModel, - metricList,totalSystemLevelMetricsList); - } else if (type == AnomalyType.BOLLINGER_BANDS) { - calcBollingerRefModel(modelName, referencerName, metricList - , totalSystemLevelMetricsList); - } else if (type == AnomalyType.MAD) { - calcMad(modelName, referencerName, refModel, metricList, totalSystemLevelMetricsList); - } - logger.info("==============anmoni loop end==================" + referencerName + " " + modelName); - - } - protected void calcHistoryRefModel(DqModel sourceModel, DqModel refModel - , List<DqMetricsValue> metricList - , SystemLevelMetricsList totalSystemLevelMetricsList) { - int trendLength, trendOffset; - if (sourceModel.getSchedule() == ScheduleType.DAILY) { - trendLength = 20; - trendOffset = 7; - } else { - trendLength = 20 * 24; - trendOffset = 7 * 24; - } - - if (metricList.size() <= trendLength + trendOffset) { - return; -// return false; - } - - String modelName = sourceModel.getModelName(); - String referencerName = refModel.getModelName(); - logger.info("==============trend start==================" - + referencerName - + " " - + modelName - + " " - + trendLength + " " + trendOffset); - - int dqfail = 0; - float threadshold = refModel.getThreshold(); - if (metricList.get(0).getValue() / metricList.get(trendOffset) .getValue() >= 1 + threadshold - || metricList.get(0).getValue() / metricList.get(trendOffset) .getValue() <= 1 - threadshold) { - dqfail = 1; - } - - for (int i = 0; i <= trendLength; i++) { - DqMetricsValue tempDQMetricsValue = metricList.get(i); - float lastValue = metricList.get( i + trendOffset).getValue(); - totalSystemLevelMetricsList.upsertNewAssetExecute( - referencerName, - MetricType.Trend.toString(), - tempDQMetricsValue.getTimestamp(), - tempDQMetricsValue.getValue() - , getSystemType(tempDQMetricsValue.getMetricName()) - , dqfail, - true, new AssetLevelMetricsDetail(lastValue)); - } - - logger.info("==============trend end=================="); - } - - - protected void calcMad(String modelName, String referencerName, DqModel refModel, - List<DqMetricsValue> metricList, - SystemLevelMetricsList totalSystemLevelMetricsList) { - logger.info("==============MAD start==================" + referencerName + " " + modelName); - Collections.reverse(metricList); - List<String> sourceValues = new ArrayList<String>(); - for (int i = 0; i < metricList.size(); i++) { - sourceValues.add((long) metricList.get(i).getValue() + ""); - } - List<MADEntity> madList = createMad(sourceValues); - - logger.info("==============MAD size : "+madList.size() +" metrics size:"+metricList.size()); - if (metricList.size() > 0 && madList.size() > 0) { - int dqfail = 0; - if (metricList.get( metricList.size() - 1).getValue() - < madList.get(madList.size() - 1).getLower()) { - dqfail = 1; - } - - int offset = metricList.size() - madList.size(); - for (int i = offset; i < metricList.size(); i++) { - DqMetricsValue tempDQMetricsValue = metricList.get(i); - - MADEntity mad = madList.get( i - offset).clone(); - AssetLevelMetricsDetail detail = new AssetLevelMetricsDetail( mad); - totalSystemLevelMetricsList.upsertNewAssetExecute( - referencerName, - MetricType.MAD.toString(), - tempDQMetricsValue.getTimestamp(), - tempDQMetricsValue.getValue(), - getSystemType(tempDQMetricsValue.getMetricName()), - dqfail, - true, - detail - ); - } - } - logger.info("==============MAD end=================="); - } - - protected void calcBollingerRefModel(String modelName, String referencerName, - List<DqMetricsValue> metricList, - SystemLevelMetricsList totalSystemLevelMetricsList) { - logger.info("==============Bollinger start==================" - + referencerName + " " + modelName); - Collections.reverse(metricList); - List<String> sourceValues = new ArrayList<String>(); - for (int i = 0; i < metricList.size(); i++) { - sourceValues.add((long) metricList.get(i) - .getValue() + ""); - } - - List<BollingerBandsEntity> bollingers = bollingerBand(sourceValues); - - logger.info("==============Bollinger size : "+bollingers.size() +" metrics size:"+metricList.size()); - if (metricList.size() > 0 && bollingers.size() > 0) { - int dqfail = 0; - if (metricList.get( metricList.size() - 1).getValue() - < bollingers.get(bollingers.size() - 1).getLower()) { - dqfail = 1; - } - - int offset = metricList.size()- bollingers.size(); - for (int i = offset; i < metricList.size(); i++) { - DqMetricsValue tempDQMetricsValue = metricList.get(i); - - BollingerBandsEntity bollinger = bollingers.get(i - offset).clone(); - AssetLevelMetricsDetail detail = new AssetLevelMetricsDetail(bollinger); - totalSystemLevelMetricsList.upsertNewAssetExecute( - referencerName, - MetricType.Bollinger.toString(), - tempDQMetricsValue.getTimestamp(), - tempDQMetricsValue.getValue(), - getSystemType(tempDQMetricsValue.getMetricName()), - dqfail, - true, - detail - ); - } - } - logger.info("==============Bollinger end==================" - + referencerName + " " + modelName); - } - - private List<String> parseRefNames(String reference) { - List<String> refNames = new ArrayList<String>(); - if (reference.indexOf(",") == -1) { - refNames.add(reference); - } else { - refNames = Arrays.asList(reference.split(",")); - } - return refNames; - } - - /** <modelName, [refModelName]> */ - Map<String, List<String>> getReferences() { - Map<String, List<String>> map = new HashMap<>(); - for(DqModel each : modelRepo.getAll()) { - String modelName = each.getModelName(); - String references = each.getReferenceModel(); - if(!map.containsKey(each.getModelName())) { - map.put(modelName, new ArrayList<String>()); - } - if(StringUtils.hasText(references)) { - for(String ref : parseRefNames(references)) { - map.get(modelName).add(ref.trim()); - } - } - } - return map; - } - - List<BollingerBandsEntity> bollingerBand(List<String> list) { - List<BollingerBandsEntity> result = new ArrayList<BollingerBandsEntity>(); - int preparePointNumber = 30; - float up_coff = 1.8f; - float down_coff = 1.8f; - for (int i = preparePointNumber; i < list.size(); i++) { - long total = 0; - for (int j = i - preparePointNumber; j < i; j++) { - long rawNumber = Long.parseLong(list.get(j)); - total = total + rawNumber; - } - long mean = total / preparePointNumber; - long meantotal = 0; - for (int j = i - preparePointNumber; j < i; j++) { - long rawNumber = Integer.parseInt(list.get(j)); - long rawDiff = rawNumber - mean; - meantotal += rawDiff * rawDiff; - } - long mad = (long) Math.sqrt(meantotal / preparePointNumber); - long upper = (long) (mean + mad * up_coff); - long lower = (long) (mean - mad * down_coff); - // .out.println( list.get(i)+"\t"+upper +"\t"+lower); - result.add(new BollingerBandsEntity(upper, lower, mean)); - } - logger.info("bollingerband done"); - return result; - } - - List<MADEntity> createMad(List<String> list) { - List<MADEntity> result = new ArrayList<MADEntity>(); - int preparePointNumber = 15; - float up_coff = 2.3f; - float down_coff = 2.3f; - for (int i = preparePointNumber; i < list.size(); i++) { - long total = 0; - for (int j = i - preparePointNumber; j < i; j++) { - long rawNumber = Long.parseLong(list.get(j)); - total = total + rawNumber; - } - long mean = total / preparePointNumber; - long meantotal = 0; - for (int j = i - preparePointNumber; j < i; j++) { - long rawNumber = Integer.parseInt(list.get(j)); - long rawDiff = rawNumber - mean; - if (rawDiff >= 0) - meantotal = meantotal + rawDiff; - else - meantotal = meantotal - rawDiff; - } - long mad = meantotal / preparePointNumber; - long upper = (long) (mean + mad * up_coff); - long lower = (long) (mean - mad * down_coff); - - result.add(new MADEntity(upper, lower)); - } - logger.info("mad done"); - return result; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/SubscribeService.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/SubscribeService.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/SubscribeService.java deleted file mode 100644 index 8755df0..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/SubscribeService.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.service; - -import com.ebay.oss.griffin.domain.UserSubscription; - - -public interface SubscribeService { - - void subscribe(UserSubscription item); - - UserSubscription getSubscribe(String user); - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/SubscribeServiceImpl.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/SubscribeServiceImpl.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/SubscribeServiceImpl.java deleted file mode 100644 index 562271a..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/SubscribeServiceImpl.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.service; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import com.ebay.oss.griffin.domain.UserSubscription; -import com.ebay.oss.griffin.repo.UserSubscriptionRepo; - -@Service -public class SubscribeServiceImpl implements SubscribeService{ - - final static Logger logger = LoggerFactory.getLogger(SubscribeServiceImpl.class); - - @Autowired - UserSubscriptionRepo subscriptionRepo; - - @Override - public void subscribe(UserSubscription item) { - subscriptionRepo.upsertUserSubscribe(item); - } - - @Override - public UserSubscription getSubscribe(String user) { - return subscriptionRepo.getUserSubscribeItem(user); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/modelcreator/AccurcyModelCreator.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/modelcreator/AccurcyModelCreator.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/modelcreator/AccurcyModelCreator.java deleted file mode 100644 index b6f5ef6..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/modelcreator/AccurcyModelCreator.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.ebay.oss.griffin.service.modelcreator; - -import org.springframework.stereotype.Component; - -import com.ebay.oss.griffin.domain.DqModel; -import com.ebay.oss.griffin.domain.ModelStatus; -import com.ebay.oss.griffin.domain.ModelType; -import com.ebay.oss.griffin.vo.MappingItemInput; -import com.ebay.oss.griffin.vo.ModelInput; - -@Component("accuracyModelCreator") -public class AccurcyModelCreator extends BaseModelCreator { - - @Override - protected void enhance(DqModel entity, ModelInput input) { - entity.setStatus(ModelStatus.TESTING); - - newSampleJob4Model(entity); - } - - @Override - public boolean isSupport(ModelInput input) { - return input.getBasic() != null && input.getBasic().getType() == ModelType.ACCURACY; - } - - protected String contentOf( ModelInput input) { - String content = input.getExtra().getSrcDb() + "|" - + input.getExtra().getSrcDataSet() + "|" - + input.getExtra().getTargetDb() + "|" - + input.getExtra().getTargetDataSet() + "|"; - - String delimeter = ""; - for(MappingItemInput itm : input.getMappings()) { - content += delimeter - + itm.getSrc() + "," - + itm.getTarget() + "," - + itm.isIsPk() + "," - + itm.getMatchMethod(); - delimeter = ";"; - } - - return content; - - } -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/modelcreator/AnomalyModelCreator.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/modelcreator/AnomalyModelCreator.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/modelcreator/AnomalyModelCreator.java deleted file mode 100644 index 1c18162..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/modelcreator/AnomalyModelCreator.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.ebay.oss.griffin.service.modelcreator; - -import org.springframework.stereotype.Component; - -import com.ebay.oss.griffin.domain.DqModel; -import com.ebay.oss.griffin.domain.ModelStatus; -import com.ebay.oss.griffin.domain.ModelType; -import com.ebay.oss.griffin.vo.ModelInput; - -@Component("anomalyModelCreator") -public class AnomalyModelCreator extends BaseModelCreator { - - @Override - public boolean isSupport(ModelInput input) { - return input.getBasic() != null && input.getBasic().getType() == ModelType.ANOMALY; - - } - - @Override - protected void enhance(DqModel entity, ModelInput input) { - entity.setStatus(ModelStatus.DEPLOYED); - - DqModel countModel = createCountModel(input); - dqModelRepo.addReference(countModel, input.getBasic().getName()); - } - - protected String contentOf(ModelInput input) { - return input.getExtra().getSrcDb() + "|" - + input.getExtra().getSrcDataSet() + "|" - + input.getExtra().getAnType(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/modelcreator/BaseModelCreator.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/modelcreator/BaseModelCreator.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/modelcreator/BaseModelCreator.java deleted file mode 100644 index 9a182de..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/modelcreator/BaseModelCreator.java +++ /dev/null @@ -1,157 +0,0 @@ -package com.ebay.oss.griffin.service.modelcreator; - -import java.util.Calendar; -import java.util.Date; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; - -import com.ebay.oss.griffin.domain.DataAsset; -import com.ebay.oss.griffin.domain.DqJob; -import com.ebay.oss.griffin.domain.DqModel; -import com.ebay.oss.griffin.domain.ModelType; -import com.ebay.oss.griffin.domain.ScheduleType; -import com.ebay.oss.griffin.domain.ValidityType; -import com.ebay.oss.griffin.error.BarkDbOperationException; -import com.ebay.oss.griffin.repo.DataAssetRepo; -import com.ebay.oss.griffin.repo.DqJobRepo; -import com.ebay.oss.griffin.repo.DqModelRepo; -import com.ebay.oss.griffin.service.DqModelCreator; -import com.ebay.oss.griffin.vo.ModelBasicInputNew; -import com.ebay.oss.griffin.vo.ModelExtraInputNew; -import com.ebay.oss.griffin.vo.ModelInput; - -public abstract class BaseModelCreator implements DqModelCreator { - - private static Logger logger = LoggerFactory.getLogger(BaseModelCreator.class); - - @Autowired - DqModelRepo dqModelRepo; - - @Autowired - DataAssetRepo dataAssetRepo; - - @Autowired - private DqJobRepo jobRepo; - - boolean hasModelWithName(String name) { - return null != dqModelRepo.findByName(name); - } - @Override - public DqModel newModel(ModelInput input) { - if ( hasModelWithName(input.getBasic().getName()) ) { - throw new BarkDbOperationException("Record already existing"); - } - - try { - DqModel entity = createModel(input); - - String content = contentOf(input); - entity.setModelContent(content); - - enhance(entity, input); - return dqModelRepo.update(entity); - } catch (Exception e) { - logger.error(e.toString()); - throw new BarkDbOperationException("Failed to create a new Model", e); - } - - } - - protected abstract String contentOf(ModelInput input); - - protected abstract void enhance(DqModel entity, ModelInput input); - - protected DqModel createModel(ModelInput input) { - DqModel entity = new DqModel(); - entity.set_id(dqModelRepo.getNextId()); - entity.setModelId(input.getBasic().getName()); - entity.setModelName(input.getBasic().getName()); - entity.setNotificationEmail(input.getBasic().getEmail()); - entity.setOwner(input.getBasic().getOwner()); - entity.setSchedule(input.getBasic().getScheduleType()); - entity.setSystem(input.getBasic().getSystem()); - entity.setThreshold(input.getBasic().getThreshold()); - entity.setModelDesc(input.getBasic().getDesc()); - entity.setTimestamp(new Date().getTime()); - entity.setAssetName(input.getBasic().getDataaset()); - entity.setAssetId(input.getBasic().getDataasetId()); - entity.setReferenceModel(""); - entity.setModelType(input.getBasic().getType()); - - if (input.getBasic().getStarttime() == 0) { - entity.setStarttime(new Date().getTime()); - } else { - entity.setStarttime(input.getBasic().getStarttime()); - } - - return entity; - } - protected DqModel createCountModel(ModelInput input) { - DqModel countModel = dqModelRepo.findCountModelByAssetID(input.getBasic().getDataasetId()); - if (countModel != null) { - return countModel; - } - - DataAsset asset = dataAssetRepo.getById(new Long(input.getBasic().getDataasetId())); - ModelBasicInputNew basic = new ModelBasicInputNew(); - ModelExtraInputNew extra = new ModelExtraInputNew(); - basic.setDataaset(input.getBasic().getDataaset()); - basic.setDataasetId(input.getBasic().getDataasetId()); - basic.setDesc("Count for " + input.getBasic().getDataaset()); - basic.setEmail(input.getBasic().getEmail()); - basic.setName("Count_" + input.getBasic().getName() ); - basic.setOwner(input.getBasic().getOwner()); - basic.setScheduleType(input.getBasic() .getScheduleType()); - basic.setStatus(input.getBasic().getStatus()); - basic.setSystem(input.getBasic().getSystem()); - basic.setType(ModelType.VALIDITY); - - extra.setVaType(ValidityType.TOTAL_COUNT); - extra.setSrcDataSet(asset.getSystem()); - extra.setSrcDb(asset.getPlatform()); - - ModelInput tempCountModel = new ModelInput(); - tempCountModel.setBasic(basic); - tempCountModel.setExtra(extra); - return newModel(tempCountModel); - } - - void newSampleJob4Model(DqModel input) { - int type = input.getSchedule(); - Calendar c = Calendar.getInstance(); - Date date = new Date(); - date.setMinutes(0); - date.setSeconds(0); - c.setTime(date); - - for (int i = 0; i < MIN_TESTING_JOB_NUMBER; i++) { - if (type == ScheduleType.DAILY) - c.add(Calendar.DATE, -1); - else if (type == ScheduleType.HOURLY) - c.add(Calendar.HOUR, -1); - else if (type == ScheduleType.WEEKLY) - c.add(Calendar.DATE, -7); - else if (type == ScheduleType.MONTHLY) - c.add(Calendar.MONTH, -1); - else - continue; - - long starttime = c.getTime().getTime() / 1000 * 1000; - - DqJob job = new DqJob(); - job.setModelList(input.getModelName()); - job.setStarttime(starttime); - job.setStatus(0); - job.setId(input.getModelName() + "_" + starttime); - job.setJobType(input.getModelType()); - - if (jobRepo.newJob(job) == 0) { - logger.warn("===================new job failure"); - continue; - } - } - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/modelcreator/DqModelCreatorChain.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/modelcreator/DqModelCreatorChain.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/modelcreator/DqModelCreatorChain.java deleted file mode 100644 index 6073743..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/modelcreator/DqModelCreatorChain.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.ebay.oss.griffin.service.modelcreator; - -import java.util.List; - -import org.springframework.stereotype.Component; - -import com.ebay.oss.griffin.domain.DqModel; -import com.ebay.oss.griffin.service.DqModelCreator; -import com.ebay.oss.griffin.vo.ModelInput; - -@Component("modelCreatorChain") -public class DqModelCreatorChain implements DqModelCreator { - - private final List<DqModelCreator> list; - - public DqModelCreatorChain(List<DqModelCreator> list) { - this.list = list; - } - - @Override - public DqModel newModel(ModelInput input) { - for(DqModelCreator each : list) { - if(each.isSupport(input)) { - return each.newModel(input); - } - } - throw new RuntimeException("Unsupported ModelInput" + input.getBasic().getType()); - } - - @Override - public boolean isSupport(ModelInput input) { - return true; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/modelcreator/PublishModelCreator.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/modelcreator/PublishModelCreator.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/modelcreator/PublishModelCreator.java deleted file mode 100644 index 6dfe356..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/modelcreator/PublishModelCreator.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.ebay.oss.griffin.service.modelcreator; - -import org.springframework.stereotype.Component; - -import com.ebay.oss.griffin.domain.DqModel; -import com.ebay.oss.griffin.domain.ModelStatus; -import com.ebay.oss.griffin.domain.ModelType; -import com.ebay.oss.griffin.vo.ModelInput; - -@Component("publishModelCreator") -public class PublishModelCreator extends BaseModelCreator { - - @Override - public boolean isSupport(ModelInput input) { - return input.getBasic() != null && input.getBasic().getType() == ModelType.PUBLISH; - } - - @Override - protected void enhance(DqModel entity, ModelInput input) { - entity.setStatus(ModelStatus.DEPLOYED); - } - - - @Override - protected String contentOf(ModelInput input) { - return input.getExtra().getPublishUrl(); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/modelcreator/ValidityModelCreator.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/modelcreator/ValidityModelCreator.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/modelcreator/ValidityModelCreator.java deleted file mode 100644 index 6c5b227..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/modelcreator/ValidityModelCreator.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.ebay.oss.griffin.service.modelcreator; - -import org.springframework.stereotype.Component; - -import com.ebay.oss.griffin.domain.DqModel; -import com.ebay.oss.griffin.domain.ModelStatus; -import com.ebay.oss.griffin.domain.ModelType; -import com.ebay.oss.griffin.domain.ValidityType; -import com.ebay.oss.griffin.vo.ModelInput; - -@Component("validityModelCreator") -public class ValidityModelCreator extends BaseModelCreator { - - @Override - public boolean isSupport(ModelInput input) { - return input.getBasic() != null && input.getBasic().getType() == ModelType.VALIDITY; - } - - @Override - protected void enhance(DqModel entity, ModelInput input) { - if(input.getExtra().getVaType() == ValidityType.TOTAL_COUNT){ - entity.setStatus(input.getBasic().getStatus()); - }else{ - entity.setStatus(ModelStatus.TESTING); - } - - newSampleJob4Model(entity); - } - - protected String contentOf(ModelInput input) { - return input.getExtra().getSrcDb() + "|" - + input.getExtra().getSrcDataSet() + "|" - + input.getExtra().getVaType() + "|" - + input.getExtra().getColumn(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/vo/AccuracyHiveJobConfig.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/vo/AccuracyHiveJobConfig.java b/griffin-core/src/main/java/com/ebay/oss/griffin/vo/AccuracyHiveJobConfig.java deleted file mode 100644 index 676279d..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/vo/AccuracyHiveJobConfig.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.vo; - -import java.util.ArrayList; -import java.util.List; - -public class AccuracyHiveJobConfig { - public String source; - public String target; - public List<AccuracyHiveJobConfigDetail> accuracyMapping = new ArrayList<AccuracyHiveJobConfigDetail>(); - public List<PartitionConfig> srcPartitions = new ArrayList<PartitionConfig>(); - public List<List<PartitionConfig>> tgtPartitions = new ArrayList<List<PartitionConfig>>(); - - public AccuracyHiveJobConfig() { } - - public AccuracyHiveJobConfig(String source, String target) { - this.source = source; - this.target = target; - } - - public String getSource() { - return source; - } - - public void setSource(String source) { - this.source = source; - } - - public String getTarget() { - return target; - } - - public void setTarget(String target) { - this.target = target; - } - - public List<AccuracyHiveJobConfigDetail> getAccuracyMapping() { - return accuracyMapping; - } - - public void setAccuracyMapping(List<AccuracyHiveJobConfigDetail> accuracyMapping) { - this.accuracyMapping = accuracyMapping; - } - - public List<PartitionConfig> getSrcPartitions() { - return srcPartitions; - } - - public void setSrcPartitions(List<PartitionConfig> srcPartitions) { - if(srcPartitions!=null) this.srcPartitions = srcPartitions; - } - - public List<List<PartitionConfig>> getTgtPartitions() { - return tgtPartitions; - } - - public void setTgtPartitions(List<List<PartitionConfig>> tgtPartitions) { - if(tgtPartitions!=null) this.tgtPartitions = tgtPartitions; - } - - - - - - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/vo/AccuracyHiveJobConfigDetail.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/vo/AccuracyHiveJobConfigDetail.java b/griffin-core/src/main/java/com/ebay/oss/griffin/vo/AccuracyHiveJobConfigDetail.java deleted file mode 100644 index c85152d..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/vo/AccuracyHiveJobConfigDetail.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.vo; - - -public class AccuracyHiveJobConfigDetail { - public int sourceColId; - public String sourceColName; - public int targetColId; - public String targetColName; - public String matchFunction; - public boolean isPK; - - public AccuracyHiveJobConfigDetail() { } - - public AccuracyHiveJobConfigDetail(int sourceColId, String sourceColName, int targetColId, String targetColName, String matchFunction, boolean isPK) - { - this.sourceColId = sourceColId; - this.sourceColName = sourceColName; - this.targetColId = targetColId; - this.targetColName = targetColName; - this.matchFunction = matchFunction; - this.isPK = isPK; - } - - public int getSourceColId() { - return sourceColId; - } - - public void setSourceColId(int sourceColId) { - this.sourceColId = sourceColId; - } - - public String getSourceColName() { - return sourceColName; - } - - public void setSourceColName(String sourceColName) { - this.sourceColName = sourceColName; - } - - public int getTargetColId() { - return targetColId; - } - - public void setTargetColId(int targetColId) { - this.targetColId = targetColId; - } - - public String getTargetColName() { - return targetColName; - } - - public void setTargetColName(String targetColName) { - this.targetColName = targetColName; - } - - public String getMatchFunction() { - return matchFunction; - } - - public void setMatchFunction(String matchFunction) { - this.matchFunction = matchFunction; - } - - public boolean isPK() { - return isPK; - } - - public void setPK(boolean isPK) { - this.isPK = isPK; - } - - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/vo/AssetLevelMetrics.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/vo/AssetLevelMetrics.java b/griffin-core/src/main/java/com/ebay/oss/griffin/vo/AssetLevelMetrics.java deleted file mode 100644 index 05d8fa3..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/vo/AssetLevelMetrics.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.vo; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -public class AssetLevelMetrics { - private String name; - private float dq; - private int dqfail; - private long timestamp; - private String metricType; - private String assetName; - - private List<AssetLevelMetricsDetail> details = new ArrayList<AssetLevelMetricsDetail>(); - - public AssetLevelMetrics() { } - - public AssetLevelMetrics(String name) { - this.name = name; - } - - public AssetLevelMetrics(String name, float dq, long timestamp) { - this.name = name; - this.dq = dq; - this.timestamp = timestamp; - } - - public AssetLevelMetrics(String name, String metricType, float dq, long timestamp, int dqfail) { - this.name = name; - this.dq = dq; - this.timestamp = timestamp; - this.dqfail = dqfail; - this.metricType = metricType; - } - - public AssetLevelMetrics(AssetLevelMetrics other, int count) { - this.name = other.getName(); - this.dq = other.getDq(); - this.timestamp = other.getTimestamp(); - this.dqfail = other.getDqfail(); - this.metricType = other.getMetricType(); - List<AssetLevelMetricsDetail> otherDetail = other.getDetails(); - if(count == -1) count = other.getDetails().size(); - if(other.getDetails().size()<count) count = other.getDetails().size(); - Collections.sort(otherDetail); - for(int i=0;i<count;i++) { - AssetLevelMetricsDetail tempAssetLevelMetricsDetail = otherDetail.get(i); - details.add(new AssetLevelMetricsDetail(tempAssetLevelMetricsDetail)); - } - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public float getDq() { - return dq; - } - - public void setDq(float dq) { - this.dq = dq; - } - - public long getTimestamp() { - return timestamp; - } - - public void setTimestamp(long timestamp) { - this.timestamp = timestamp; - } - - public List<AssetLevelMetricsDetail> getDetails() { - return details; - } - - public void setDetails(List<AssetLevelMetricsDetail> details) { - this.details = details; - } - - public void addAssetLevelMetricsDetail(AssetLevelMetricsDetail dq) - { - this.details.add(dq); - } - - public int getDqfail() { - return dqfail; - } - - public void setDqfail(int dqfail) { - this.dqfail = dqfail; - } - - public String getMetricType() { - return metricType; - } - - public void setMetricType(String metricType) { - this.metricType = metricType; - } - - public String getAssetName() { - return assetName; - } - - public void setAssetName(String asseetName) { - this.assetName = asseetName; - } - - - - - - - - - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/vo/AssetLevelMetricsDetail.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/vo/AssetLevelMetricsDetail.java b/griffin-core/src/main/java/com/ebay/oss/griffin/vo/AssetLevelMetricsDetail.java deleted file mode 100644 index 59f515e..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/vo/AssetLevelMetricsDetail.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.vo; - - -public class AssetLevelMetricsDetail implements Comparable<AssetLevelMetricsDetail> { - - private long timestamp; - private float value; - private BollingerBandsEntity bolling; - private MADEntity MAD; - private float comparisionValue; - - public AssetLevelMetricsDetail() { } - - public AssetLevelMetricsDetail(long timestamp, float value) { - this.timestamp = timestamp; - this.value = value; - } - - public AssetLevelMetricsDetail(long timestamp, float value, - BollingerBandsEntity bolling) { - this.timestamp = timestamp; - this.value = value; - this.bolling = bolling; - } - - public AssetLevelMetricsDetail(long timestamp, float value, - float comparisionValue) { - this.timestamp = timestamp; - this.value = value; - this.comparisionValue = comparisionValue; - } - - public AssetLevelMetricsDetail(long timestamp, float value, MADEntity MAD) { - this.timestamp = timestamp; - this.value = value; - this.MAD = MAD; - } - - public AssetLevelMetricsDetail(BollingerBandsEntity bolling) { - this.bolling = bolling; - } - - public AssetLevelMetricsDetail(MADEntity MAD) { - this.MAD = MAD; - } - - public AssetLevelMetricsDetail(AssetLevelMetricsDetail other) { - this.timestamp = other.getTimestamp(); - this.value = other.getValue(); - this.comparisionValue = other.getComparisionValue(); - if (other.getBolling() != null) - this.bolling = new BollingerBandsEntity(other.getBolling() - .getUpper(), other.getBolling().getLower(), other - .getBolling().getMean()); - if (other.getMAD() != null) - this.MAD = new MADEntity(other.getMAD().getUpper(), other.getMAD() - .getLower()); - } - - public AssetLevelMetricsDetail(float comparisionValue) { - this.comparisionValue = comparisionValue; - } - - public long getTimestamp() { - return timestamp; - } - - public void setTimestamp(long timestamp) { - this.timestamp = timestamp; - } - - public float getValue() { - return value; - } - - public void setValue(float value) { - this.value = value; - } - - public BollingerBandsEntity getBolling() { - return bolling; - } - - public void setBolling(BollingerBandsEntity bolling) { - this.bolling = bolling; - } - - public float getComparisionValue() { - return comparisionValue; - } - - public void setComparisionValue(float comparisionValue) { - this.comparisionValue = comparisionValue; - } - - public MADEntity getMAD() { - return MAD; - } - - public void setMAD(MADEntity mAD) { - MAD = mAD; - } - - @Override - public int compareTo(AssetLevelMetricsDetail o) { - return (int) Math.signum(o.getTimestamp() - this.getTimestamp()); -// return o.getTimestamp() == this.getTimestamp() ? 0 -// : (o.getTimestamp() > this.getTimestamp() ? 1 : -1); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/vo/BaseObj.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/vo/BaseObj.java b/griffin-core/src/main/java/com/ebay/oss/griffin/vo/BaseObj.java deleted file mode 100644 index 39225d1..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/vo/BaseObj.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.vo; - -import java.util.Set; - -import javax.validation.ConstraintViolation; -import javax.validation.Validation; -import javax.validation.Validator; -import javax.validation.ValidatorFactory; - -import com.ebay.oss.griffin.error.ErrorMessage; - -public class BaseObj { - - public ErrorMessage validate() { - ErrorMessage msg = null; - - ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); - Validator validator = factory.getValidator(); - Set<ConstraintViolation<BaseObj>> constraintViolations = validator - .validate(this); - - if (constraintViolations.size() > 0) { - msg = new ErrorMessage(); - StringBuffer error = new StringBuffer(); - for (ConstraintViolation<BaseObj> violation : constraintViolations) { - error.append(violation.getPropertyPath() + " " + violation.getMessage() + ", "); - } - - msg.setMessage(error.toString()); - - } - - return msg; - - } -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/vo/BollingerBandsEntity.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/vo/BollingerBandsEntity.java b/griffin-core/src/main/java/com/ebay/oss/griffin/vo/BollingerBandsEntity.java deleted file mode 100644 index 2067c0e..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/vo/BollingerBandsEntity.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.vo; - - -public class BollingerBandsEntity { - private long upper; - private long lower; - private long mean; - - public BollingerBandsEntity() {} - - public BollingerBandsEntity(long upper, long lower, long mean) { - this.upper = upper; - this.lower = lower; - this.mean = mean; - } - - public long getUpper() { - return upper; - } - - public void setUpper(long upper) { - this.upper = upper; - } - - public long getLower() { - return lower; - } - - public void setLower(long lower) { - this.lower = lower; - } - - public long getMean() { - return mean; - } - - public void setMean(long mean) { - this.mean = mean; - } - - - public BollingerBandsEntity clone() { - return new BollingerBandsEntity(getUpper(), getLower(), getMean()); - } - - -}
