http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqSchedule.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqSchedule.java b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqSchedule.java new file mode 100644 index 0000000..5ead6f1 --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqSchedule.java @@ -0,0 +1,123 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.ebay.oss.griffin.domain; + +import com.google.code.morphia.annotations.Property; + +/** + * This is a representation of model execution(s) schedule. <p/> + * + * A DqSchedule has muliple DqJob, based on the ScheduleType. + * + * A DqModel defines what DQ should be calculated, there + * will be a corresponding algo model (defined in bark-models) describing how to calculate. + * + * @see DqJob + * @see ScheduleType + */ +public class DqSchedule extends IdEntity { + + /** a list of modelName, seprated by ScheduleModelSeperator.SEPERATOR. + * if ModelType.VALIDITY, there will be multiple model included, or only one model. + */ + @Property("modelList") + private String modelList; + + @Property("assetId") + private long assetId; + + /** Inherits from DqModel.modelType. In case of modelList has more models, they should share a + * same modelType, specifically the VALIDITY. + * + * @see ModelType + * */ + @Property("jobType") + private int jobType; + + // @see ScheduleType + @Property("scheduleType") + private int scheduleType; + + /** @see JobStatus */ + @Property("status") + private int status; + + @Property("starttime") + private long starttime; + + @Property("content") + private String content; + + public int getStatus() { + return status; + } + + public void setStatus(int status) { + this.status = status; + } + + public long getStarttime() { + return starttime; + } + + public void setStarttime(long starttime) { + this.starttime = starttime; + } + + public String getContent() { + return content; + } + + public void setContent(String content) { + this.content = content; + } + + public int getScheduleType() { + return scheduleType; + } + + public void setScheduleType(int scheduleType) { + this.scheduleType = scheduleType; + } + + public String getModelList() { + return modelList; + } + + public void setModelList(String modelList) { + this.modelList = modelList; + } + + public long getAssetId() { + return assetId; + } + + public void setAssetId(long assetId) { + this.assetId = assetId; + } + + public int getJobType() { + return jobType; + } + + public void setJobType(int jobType) { + this.jobType = jobType; + } + + + + + +}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/domain/IdEntity.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/domain/IdEntity.java b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/IdEntity.java new file mode 100644 index 0000000..e42b5ad --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/IdEntity.java @@ -0,0 +1,28 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.ebay.oss.griffin.domain; + +import com.google.code.morphia.annotations.Id; + +public class IdEntity { + + @Id + private Long _id; // table id primary key + + public Long get_id() { return _id; } + + public void set_id(Long _id) { this._id = _id; } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/domain/JobStatus.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/domain/JobStatus.java b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/JobStatus.java new file mode 100644 index 0000000..1bc1c9d --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/JobStatus.java @@ -0,0 +1,24 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.ebay.oss.griffin.domain; + +public class JobStatus { + + public static final int READY = 0; + public static final int WAITING = 1; + public static final int STARTED = 2; + public static final int FINISHED = 3; + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/domain/MetricType.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/domain/MetricType.java b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/MetricType.java new file mode 100644 index 0000000..78b0691 --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/MetricType.java @@ -0,0 +1,25 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.ebay.oss.griffin.domain; + +/** conceptually same as AnomalyType, should be unified. + * @see AnomalType + */ +public enum MetricType { + NORMAL, + Bollinger, + Trend, + MAD +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/domain/ModelStatus.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/domain/ModelStatus.java b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/ModelStatus.java new file mode 100644 index 0000000..4f83251 --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/ModelStatus.java @@ -0,0 +1,24 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.ebay.oss.griffin.domain; + +public class ModelStatus { + + public static final int TESTING = 0; + public static final int VERIFIED = 1; + public static final int DEPLOYED = 2; + + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/domain/ModelType.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/domain/ModelType.java b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/ModelType.java new file mode 100644 index 0000000..12c32a9 --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/ModelType.java @@ -0,0 +1,24 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.ebay.oss.griffin.domain; + +public class ModelType { + + public static final int ACCURACY = 0; + public static final int VALIDITY = 1; + public static final int ANOMALY = 2; + public static final int PUBLISH = 3; + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/domain/PartitionFormat.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/domain/PartitionFormat.java b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/PartitionFormat.java new file mode 100644 index 0000000..27fa1fe --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/PartitionFormat.java @@ -0,0 +1,59 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.ebay.oss.griffin.domain; + +import com.google.code.morphia.annotations.Embedded; +import com.google.code.morphia.annotations.Property; + +/** + * In some cases, such as tera-data or hive, partition is used to distribute the data in different + * storage(file). This is an abstract of naming pattern of partition. + */ +@Embedded +public class PartitionFormat { + + @Property("name") + private String name; + + @Property("format") + private String format; + + public PartitionFormat() { + } + + public PartitionFormat(String name, String format) { + this.name = name; + this.format = format; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getFormat() { + return format; + } + + public void setFormat(String format) { + this.format = format; + } + + + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/domain/SampleFilePathLKP.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/domain/SampleFilePathLKP.java b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/SampleFilePathLKP.java new file mode 100644 index 0000000..ee7a214 --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/SampleFilePathLKP.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2016 eBay Software Foundation. Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable + * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License + * for the specific language governing permissions and limitations under the License. + */ +package com.ebay.oss.griffin.domain; + +import com.google.code.morphia.annotations.Entity; +import com.google.code.morphia.annotations.Property; + +/** + * In Accuracy DQ, the mismatched data is actually stored to file. <p/> + * + * Each of this has a corresponding DqMetricsValue, by {modelName, timestamp} + */ +// uniq{model, timestamp} +@Entity("dq_missed_file_path_lkp") +public class SampleFilePathLKP extends IdEntity { + + @Property("modelName") + private String modelName; + + @Property("hdfsPath") + private String hdfsPath; + + @Property("timestamp") + private long timestamp; + + public String getModelName() { + return modelName; + } + + public void setModelName(String modelName) { + this.modelName = modelName; + } + + public String getHdfsPath() { + return hdfsPath; + } + + public void setHdfsPath(String hdfsPath) { + this.hdfsPath = hdfsPath; + } + + public long getTimestamp() { + return timestamp; + } + + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/domain/ScheduleType.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/domain/ScheduleType.java b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/ScheduleType.java new file mode 100644 index 0000000..dc3c2f7 --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/ScheduleType.java @@ -0,0 +1,24 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.ebay.oss.griffin.domain; + +public class ScheduleType { + + public static final int DAILY = 0; + public static final int WEEKLY = 1; + public static final int MONTHLY = 2; + public static final int HOURLY = 3; + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/domain/SystemType.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/domain/SystemType.java b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/SystemType.java new file mode 100644 index 0000000..20f322b --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/SystemType.java @@ -0,0 +1,69 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.ebay.oss.griffin.domain; + + +/** + * A system is actually an application (or sth else, like data container), managing data which + * involved the data quality. A System could be considered a set of DataAssets. + * <p/> + * here lists the different type of bark involved System. + */ +public class SystemType { + + public static final int BULLSEYE = 0; + public static final int GPS = 1; + public static final int HADOOP = 2; + public static final int PDS = 3; + public static final int IDLS = 4; + public static final int PULSAR = 5; + public static final int KAFKA = 6; + public static final int SOJOURNER= 7; + public static final int SITESPEED = 8; + public static final int EDW = 9; + +// private final int value; +// +// private final String desc; +// +// public SystemType(int value, String desc) { +// super(); +// this.value = value; +// this.desc = desc; +// } + + private static final String[] array = {"Bullseye", "GPS", "Hadoop", "PDS", "IDLS", "Pulsar", "Kafka", "Sojourner", "SiteSpeed", "EDW"}; + + public static String val(int type){ + if(type < array.length && type >=0){ + return array[type]; + }else{ + return type + ""; + } + + + } + + public static int indexOf(String desc){ + for(int i = 0;i < array.length; i ++){ + if(array[i].equals(desc)){ + return i; + } + } + + return -1; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/domain/UserSubscription.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/domain/UserSubscription.java b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/UserSubscription.java new file mode 100644 index 0000000..192ce8c --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/UserSubscription.java @@ -0,0 +1,101 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.ebay.oss.griffin.domain; + +import java.util.List; + +import com.ebay.oss.griffin.vo.PlatformSubscription; +import com.ebay.oss.griffin.vo.SystemSubscription; + +/** + * User could define the model/metrics by subscribe the DataAssets. + */ +// FIXME uid+List<asset> is enough, group by platform/system is just a rendering issue, which should +// not impact the model design. +public class UserSubscription { + // same as ntaccount? + String _id; + + String ntaccount; + + List<PlatformSubscription> subscribes; + + public UserSubscription() { } + + public UserSubscription(String user) { + ntaccount = user; + } + + public String getId() { + return _id; + } + + public void setId(String _id) { + this._id = _id; + } + + public String getNtaccount() { + return ntaccount; + } + + public void setNtaccount(String ntaccount) { + this.ntaccount = ntaccount; + } + + public List<PlatformSubscription> getSubscribes() { + return subscribes; + } + + public void setSubscribes(List<PlatformSubscription> subscribes) { + this.subscribes = subscribes; + } + + public boolean isPlatformSelected(String platform) { + for(PlatformSubscription item : subscribes) { + if(item.getPlatform().equals(platform) && item.isSelectAll()) { + return true; + } + } + return false; + } + + public boolean isSystemSelected(String platform, String system) { + for(PlatformSubscription item : subscribes) { + if(item.getPlatform().equals(platform)) { + for(SystemSubscription eachSystem : item.getSystems()) { + if(eachSystem.getSystem().equals(system) && eachSystem.isSelectAll()) { + return true; + } + } + } + } + return false; + } + + public boolean isDataAssetSelected(String platform, String system, String dataasset) { + for(PlatformSubscription item : subscribes) { + if(item.getPlatform().equals(platform)) { + for(SystemSubscription eachSystem : item.getSystems()) { + if(eachSystem.getSystem().equals(system) + && eachSystem.getDataassets().contains(dataasset)) { + return true; + } + } + } + } + return false; + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/domain/ValidityType.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/domain/ValidityType.java b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/ValidityType.java new file mode 100644 index 0000000..f834579 --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/ValidityType.java @@ -0,0 +1,31 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.ebay.oss.griffin.domain; + +/** the type of Validity model */ +public class ValidityType { + public static final int DEFAULT_COUNT = 0; + public static final int TOTAL_COUNT = 1; + public static final int NULL_COUNT = 2; + public static final int UNIQUE_COUNT = 3; + public static final int DUPLICATE_COUNT = 4; + public static final int MAXIMUM = 5; + public static final int MINIMUM = 6; + public static final int MEAN = 7; + public static final int MEDIAN = 8; + public static final int REGULAR_EXPRESSION_MATCHING = 9; + public static final int PATTERN_FREQUENCY = 10; + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/error/BarkDbOperationException.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/error/BarkDbOperationException.java b/griffin-core/src/main/java/com/ebay/oss/griffin/error/BarkDbOperationException.java new file mode 100644 index 0000000..88d88f2 --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/error/BarkDbOperationException.java @@ -0,0 +1,105 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.ebay.oss.griffin.error; + + + +public class BarkDbOperationException extends RuntimeException { + + /** + * + */ + private static final long serialVersionUID = 2137866000039215687L; + + + /** + * Constructs a new exception with {@code null} as its detail message. + * The cause is not initialized, and may subsequently be initialized by a + * call to {@link #initCause}. + */ + public BarkDbOperationException() { + super(); + } + + + + /** + * Constructs a new exception with the specified detail message. The + * cause is not initialized, and may subsequently be initialized by + * a call to {@link #initCause}. + * + * @param message the detail message. The detail message is saved for + * later retrieval by the {@link #getMessage()} method. + */ + public BarkDbOperationException(String message) { + super(message); + } + + /** + * Constructs a new exception with the specified detail message and + * cause. <p>Note that the detail message associated with + * {@code cause} is <i>not</i> automatically incorporated in + * this exception's detail message. + * + * @param message the detail message (which is saved for later retrieval + * by the {@link #getMessage()} method). + * @param cause the cause (which is saved for later retrieval by the + * {@link #getCause()} method). (A <tt>null</tt> value is + * permitted, and indicates that the cause is nonexistent or + * unknown.) + * @since 1.4 + */ + public BarkDbOperationException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructs a new exception with the specified cause and a detail + * message of <tt>(cause==null ? null : cause.toString())</tt> (which + * typically contains the class and detail message of <tt>cause</tt>). + * This constructor is useful for exceptions that are little more than + * wrappers for other throwables (for example, {@link + * java.security.PrivilegedActionException}). + * + * @param cause the cause (which is saved for later retrieval by the + * {@link #getCause()} method). (A <tt>null</tt> value is + * permitted, and indicates that the cause is nonexistent or + * unknown.) + * @since 1.4 + */ + public BarkDbOperationException(Throwable cause) { + super(cause); + } + + /** + * Constructs a new exception with the specified detail message, + * cause, suppression enabled or disabled, and writable stack + * trace enabled or disabled. + * + * @param message the detail message. + * @param cause the cause. (A {@code null} value is permitted, + * and indicates that the cause is nonexistent or unknown.) + * @param enableSuppression whether or not suppression is enabled + * or disabled + * @param writableStackTrace whether or not the stack trace should + * be writable + * @since 1.7 + */ + protected BarkDbOperationException(String message, Throwable cause, + boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/error/BarkWebException.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/error/BarkWebException.java b/griffin-core/src/main/java/com/ebay/oss/griffin/error/BarkWebException.java new file mode 100644 index 0000000..7bba6f5 --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/error/BarkWebException.java @@ -0,0 +1,136 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.ebay.oss.griffin.error; + + + +public class BarkWebException extends RuntimeException { + + /** + * + */ + private static final long serialVersionUID = -4108849854769426970L; + + + private int status; + + private ErrorMessage err; + + public ErrorMessage getErr() { + return err; + } + + public void setErr(ErrorMessage err) { + this.err = err; + } + + public int getStatus() { + return status; + } + + public void setStatus(int status) { + this.status = status; + } + + /** + * Constructs a new exception with {@code null} as its detail message. + * The cause is not initialized, and may subsequently be initialized by a + * call to {@link #initCause}. + */ + public BarkWebException() { + super(); + } + + public BarkWebException(int status, String message) { + super(message); + this.status = status; + + } + + public BarkWebException(int status, Throwable cause) { + super(cause); + this.status = status; + + } + + + /** + * Constructs a new exception with the specified detail message. The + * cause is not initialized, and may subsequently be initialized by + * a call to {@link #initCause}. + * + * @param message the detail message. The detail message is saved for + * later retrieval by the {@link #getMessage()} method. + */ + public BarkWebException(String message) { + super(message); + } + + /** + * Constructs a new exception with the specified detail message and + * cause. <p>Note that the detail message associated with + * {@code cause} is <i>not</i> automatically incorporated in + * this exception's detail message. + * + * @param message the detail message (which is saved for later retrieval + * by the {@link #getMessage()} method). + * @param cause the cause (which is saved for later retrieval by the + * {@link #getCause()} method). (A <tt>null</tt> value is + * permitted, and indicates that the cause is nonexistent or + * unknown.) + * @since 1.4 + */ + public BarkWebException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructs a new exception with the specified cause and a detail + * message of <tt>(cause==null ? null : cause.toString())</tt> (which + * typically contains the class and detail message of <tt>cause</tt>). + * This constructor is useful for exceptions that are little more than + * wrappers for other throwables (for example, {@link + * java.security.PrivilegedActionException}). + * + * @param cause the cause (which is saved for later retrieval by the + * {@link #getCause()} method). (A <tt>null</tt> value is + * permitted, and indicates that the cause is nonexistent or + * unknown.) + * @since 1.4 + */ + public BarkWebException(Throwable cause) { + super(cause); + } + + /** + * Constructs a new exception with the specified detail message, + * cause, suppression enabled or disabled, and writable stack + * trace enabled or disabled. + * + * @param message the detail message. + * @param cause the cause. (A {@code null} value is permitted, + * and indicates that the cause is nonexistent or unknown.) + * @param enableSuppression whether or not suppression is enabled + * or disabled + * @param writableStackTrace whether or not the stack trace should + * be writable + * @since 1.7 + */ + protected BarkWebException(String message, Throwable cause, + boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/error/BarkWebExceptionMapper.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/error/BarkWebExceptionMapper.java b/griffin-core/src/main/java/com/ebay/oss/griffin/error/BarkWebExceptionMapper.java new file mode 100644 index 0000000..a3a0595 --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/error/BarkWebExceptionMapper.java @@ -0,0 +1,43 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.ebay.oss.griffin.error; + +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.ext.ExceptionMapper; +import javax.ws.rs.ext.Provider; + +import org.springframework.stereotype.Component; + +@Provider +@Component +public class BarkWebExceptionMapper implements +ExceptionMapper<BarkWebException> { + + @Override + public Response toResponse(BarkWebException ex) { + int status = ex.getStatus(); + if (status == 0) { + status = 500; + } + + return Response.status(status) + .entity(new ErrorMessage(status, ex.getMessage())) + .type(MediaType.APPLICATION_JSON) // this has to be set to get + // the generated JSON + .build(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/error/ErrorMessage.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/error/ErrorMessage.java b/griffin-core/src/main/java/com/ebay/oss/griffin/error/ErrorMessage.java new file mode 100644 index 0000000..3f633f0 --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/error/ErrorMessage.java @@ -0,0 +1,101 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.ebay.oss.griffin.error; + +import javax.ws.rs.core.Response; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement +public class ErrorMessage { + /** contains the same HTTP Status code returned by the server */ + @XmlElement(name = "status") + int status; + + /** application specific error code */ + @XmlElement(name = "code") + int code; + + /** message describing the error */ + @XmlElement(name = "message") + String message; + + /** link point to page where the error message is documented */ + @XmlElement(name = "link") + String link; + + /** extra information that might useful for developers */ + @XmlElement(name = "developerMessage") + String developerMessage; + + public int getStatus() { + return status; + } + + public void setStatus(int status) { + this.status = status; + } + + public int getCode() { + return code; + } + + public void setCode(int code) { + this.code = code; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public String getDeveloperMessage() { + return developerMessage; + } + + public void setDeveloperMessage(String developerMessage) { + this.developerMessage = developerMessage; + } + + public String getLink() { + return link; + } + + public void setLink(String link) { + this.link = link; + } + + public ErrorMessage(Throwable ex) { + this.status = Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(); + this.message = ex.getMessage(); + + } + + public ErrorMessage() { + } + + public ErrorMessage(String msg) { + this.message = msg; + } + + public ErrorMessage(int status, String msg) { + this.status = status; + this.message = msg; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/repo/BarkIdRepo.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/repo/BarkIdRepo.java b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/BarkIdRepo.java new file mode 100644 index 0000000..2a3c7df --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/BarkIdRepo.java @@ -0,0 +1,11 @@ +package com.ebay.oss.griffin.repo; + +public interface BarkIdRepo<T> extends BarkRepo<T> { + + Long getNextId(); + + T getById(Long id); + + void delete(Long id); + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/repo/BarkRepo.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/repo/BarkRepo.java b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/BarkRepo.java new file mode 100644 index 0000000..c003e84 --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/BarkRepo.java @@ -0,0 +1,31 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.ebay.oss.griffin.repo; + +import java.util.List; + +import com.ebay.oss.griffin.common.Pair; +import com.mongodb.DBObject; + +public interface BarkRepo<T> { + + List<T> getAll(); + + DBObject getByCondition(List<Pair> queryList); + + void save(T t); + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/repo/BaseIdRepo.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/repo/BaseIdRepo.java b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/BaseIdRepo.java new file mode 100644 index 0000000..c6f8b53 --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/BaseIdRepo.java @@ -0,0 +1,53 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.ebay.oss.griffin.repo; + +import org.springframework.beans.factory.annotation.Autowired; + +import com.mongodb.BasicDBObject; +import com.mongodb.DBObject; + +public abstract class BaseIdRepo<T> extends BaseRepo<T> implements BarkIdRepo<T> { + + @Autowired + private SequenceRepo seqRepo; + + private final String idKey; + + protected BaseIdRepo(String collectionName, String idKey, Class<T> clz) { + super(collectionName, clz); + this.idKey = idKey; + } + + @Override + synchronized final public T getById(Long id) { + DBObject o = dbCollection.findOne(new BasicDBObject("_id", id)); + return o != null ? toEntity(o) : null; + } + + @Override + final public void delete(Long id) { + DBObject temp = dbCollection.findOne(new BasicDBObject("_id", id)); + if (temp != null) { + dbCollection.remove(temp); + } + } + + @Override + final public Long getNextId() { + return seqRepo.getNextSequence(idKey); + } +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/repo/BaseRepo.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/repo/BaseRepo.java b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/BaseRepo.java new file mode 100644 index 0000000..0d34d78 --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/BaseRepo.java @@ -0,0 +1,95 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.ebay.oss.griffin.repo; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.ebay.oss.griffin.common.Pair; +import com.google.gson.Gson; +import com.mongodb.BasicDBObject; +import com.mongodb.DB; +import com.mongodb.DBCollection; +import com.mongodb.DBObject; +import com.mongodb.MongoClient; +import com.mongodb.util.JSON; + +public abstract class BaseRepo<T> implements BarkRepo<T> { + + protected static Logger logger = LoggerFactory.getLogger(BaseRepo.class); + + protected final DBCollection dbCollection; + + private final Class<T> clz; + + protected BaseRepo(String collectionName, Class<T> clz) throws RuntimeException { + this.clz = clz; + Properties env = new Properties(); + try { + env.load(Thread.currentThread().getContextClassLoader() + .getResourceAsStream("application.properties")); + String mongoServer = env.getProperty("spring.data.mongodb.host"); + int mongoPort = Integer.parseInt(env + .getProperty("spring.data.mongodb.port")); + + DB db = new MongoClient(mongoServer, mongoPort).getDB("unitdb0"); + + dbCollection = db.getCollection(collectionName); + + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + + @Override + final public List<T> getAll() { + List<T> result = new LinkedList<T>(); + for(DBObject dbo : dbCollection.find()) { + result.add(toEntity(dbo)); + } + return result; + + } + + protected T toEntity(DBObject dbo) { + Gson gson = new Gson(); + return gson.fromJson(dbo.toString(), clz); + } + + @Override + final public void save(T t) { + Gson gson = new Gson(); + DBObject t1 = (DBObject) JSON.parse(gson.toJson(t)); + dbCollection.save(t1); + } + + @Override + final public DBObject getByCondition(List<Pair> queryList) { + BasicDBObject query = new BasicDBObject(); + for (Pair k : queryList) { + query.put(k.key, k.value); + } + + return dbCollection.findOne(query); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DataAssetRepo.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DataAssetRepo.java b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DataAssetRepo.java new file mode 100644 index 0000000..d3003d5 --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DataAssetRepo.java @@ -0,0 +1,26 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.ebay.oss.griffin.repo; + +import com.ebay.oss.griffin.domain.DataAsset; +import com.mongodb.DBObject; + +public interface DataAssetRepo extends BarkIdRepo<DataAsset> { + + // FIXME why DBO here, how save & update + void update(DataAsset asset, DBObject dbo); + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DataAssetRepoImpl.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DataAssetRepoImpl.java b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DataAssetRepoImpl.java new file mode 100644 index 0000000..dde275c --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DataAssetRepoImpl.java @@ -0,0 +1,47 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.ebay.oss.griffin.repo; + +import org.springframework.stereotype.Repository; + +import com.ebay.oss.griffin.domain.DataAsset; +import com.google.gson.Gson; +import com.mongodb.DBObject; +import com.mongodb.util.JSON; + +@Repository +public class DataAssetRepoImpl extends BaseIdRepo<DataAsset> implements DataAssetRepo { + + public DataAssetRepoImpl() { + super("data_assets", "DQ_DATA_ASSET_SEQ_NO", DataAsset.class); + } + + @Override + public void update(DataAsset entity, DBObject old) { + Gson gson = new Gson(); + // DBObject t1 = gson.fromJson(gson.toJson(entity), + // BasicDBObject.class); + DBObject t1 = (DBObject) JSON.parse(gson.toJson(entity)); + dbCollection.remove(old); + dbCollection.save(t1); + } + + @Override + protected DataAsset toEntity(DBObject o) { + return new DataAsset(o); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqJobRepo.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqJobRepo.java b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqJobRepo.java new file mode 100644 index 0000000..837cbd1 --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqJobRepo.java @@ -0,0 +1,33 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.ebay.oss.griffin.repo; + +import java.util.List; + +import com.ebay.oss.griffin.domain.DqJob; + +public interface DqJobRepo extends BarkRepo<DqJob> { + + int newJob(DqJob job); + + void update(DqJob DqJob); + + /** return null if not found. */ + DqJob getById(String jobID); + + List<DqJob> getByStatus(int ready); + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqJobRepoImpl.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqJobRepoImpl.java b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqJobRepoImpl.java new file mode 100644 index 0000000..2900dff --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqJobRepoImpl.java @@ -0,0 +1,102 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.ebay.oss.griffin.repo; + +import java.util.ArrayList; +import java.util.List; + +import org.springframework.stereotype.Repository; + +import com.ebay.oss.griffin.common.NumberUtils; +import com.ebay.oss.griffin.domain.DqJob; +import com.google.gson.Gson; +import com.mongodb.BasicDBObject; +import com.mongodb.DBObject; +import com.mongodb.util.JSON; + +// FIXME why DqJob use a string id +@Repository +public class DqJobRepoImpl extends BaseRepo<DqJob> implements DqJobRepo { + + public DqJobRepoImpl() { + super("dq_job", DqJob.class); + } + + @Override + public void update(DqJob job) { + DBObject dbo = dboOf(job); + if (dbo != null) + dbCollection.remove(dbo); + + save(job); + } + + DBObject dboOf(DqJob job) { + return dbCollection.findOne(new BasicDBObject("_id", job.getId())); + } + @Override + public int newJob(DqJob job) { + try { + DBObject temp = dboOf(job); + if (temp != null) + return 0; + Gson gson = new Gson(); + DBObject t1 = (DBObject) JSON.parse(gson.toJson(job)); + dbCollection.save(t1); + return 1; + } catch (Exception e) { + logger.warn("===========insert new job error===============" + + e.getMessage()); + return 0; + } + } + + @Override + public DqJob getById(String jobId) { + DBObject dbo = dbCollection.findOne(new BasicDBObject("_id", jobId)); + if (dbo == null) { + return null; + } + return toEntity(dbo); + } + + @Override + protected DqJob toEntity(DBObject o) { + DqJob entity = new DqJob(); + entity.setId((String)o.get("_id")); + entity.setModelList((String)o.get("modelList")); + entity.setJobType(NumberUtils.parseInt( o.get("jobType"))); + entity.setStatus(NumberUtils.parseInt( o.get("status"))); + entity.setStarttime(NumberUtils.parseLong(o.get("starttime"))); + entity.setContent((String) o.get("content")); + entity.setEndtime(NumberUtils.parseLong( o.get("endtime"))); + entity.setValue(NumberUtils.parseLong(o.get("value"))); + + return entity; + } + + @Override + public List<DqJob> getByStatus(int status) { + List<DqJob> list = new ArrayList<>(); + + List<DBObject> dboList = dbCollection.find(new BasicDBObject("status", status)).toArray(); + for (DBObject dbo : dboList) { + list.add(toEntity(dbo)); + } + + return list; + } +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqMetricsRepo.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqMetricsRepo.java b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqMetricsRepo.java new file mode 100644 index 0000000..829824d --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqMetricsRepo.java @@ -0,0 +1,31 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.ebay.oss.griffin.repo; + +import java.util.List; + +import com.ebay.oss.griffin.domain.DqMetricsValue; +import com.mongodb.DBObject; + +public interface DqMetricsRepo extends BarkIdRepo<DqMetricsValue> { + + List<DqMetricsValue> getByMetricsName(String modelName); + + void update(DqMetricsValue metrics, DBObject item); + + DqMetricsValue getLatestByAssetId(String assetId); + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqMetricsRepoImpl.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqMetricsRepoImpl.java b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqMetricsRepoImpl.java new file mode 100644 index 0000000..9baba2a --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqMetricsRepoImpl.java @@ -0,0 +1,85 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.ebay.oss.griffin.repo; + +import java.util.ArrayList; +import java.util.List; + +import org.springframework.stereotype.Repository; + +import com.ebay.oss.griffin.domain.DqMetricsValue; +import com.google.gson.Gson; +import com.mongodb.BasicDBObject; +import com.mongodb.DBCursor; +import com.mongodb.DBObject; +import com.mongodb.util.JSON; + +@Repository +public class DqMetricsRepoImpl extends BaseIdRepo<DqMetricsValue> implements DqMetricsRepo { + + public DqMetricsRepoImpl() { + super("dq_metrics_values", "DQ_METRICS_SEQ_NO", DqMetricsValue.class); + } + + @Override + public void update(DqMetricsValue entity, DBObject old) { + Gson gson = new Gson(); + // DBObject t1 = gson.fromJson(gson.toJson(entity), + // BasicDBObject.class); + DBObject t1 = (DBObject) JSON.parse(gson.toJson(entity)); + dbCollection.update(old, t1); + } + + @Override + public DqMetricsValue getLatestByAssetId(String assetId) { + DBCursor temp = dbCollection.find(new BasicDBObject("assetId", + assetId)); + List<DBObject> all = temp.toArray(); + long latest = 0L; + DBObject latestObject = null; + for (DBObject o : all) { + if (Long.parseLong(o.get("timestamp").toString()) - latest > 0) { + latest = Long.parseLong(o.get("timestamp").toString()); + latestObject = o; + } + } + + if (latestObject == null) { + return null; + } + return new DqMetricsValue( + latestObject.get("metricName").toString(), + Long.parseLong(latestObject.get("timestamp").toString()), + Float.parseFloat(latestObject.get("value").toString())); + } + + @Override + public List<DqMetricsValue> getByMetricsName(String name) { + DBCursor temp = dbCollection.find(new BasicDBObject("metricName", + name)); + List<DBObject> all = temp.toArray(); + List<DqMetricsValue> result = new ArrayList<DqMetricsValue>(); + for (DBObject o : all) { + result.add( toEntity(o)); + } + return result; + } + + +// new DqMetricsValue(o.get("metricName").toString(), Long +// .parseLong(o.get("timestamp").toString()), Float +// .parseFloat(o.get("value").toString()))); +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqModelRepo.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqModelRepo.java b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqModelRepo.java new file mode 100644 index 0000000..3800508 --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqModelRepo.java @@ -0,0 +1,40 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.ebay.oss.griffin.repo; + +import java.util.List; + +import com.ebay.oss.griffin.domain.DataAsset; +import com.ebay.oss.griffin.domain.DqModel; + +public interface DqModelRepo extends BarkIdRepo<DqModel> { + + DqModel update(DqModel model); + + /////////// + DqModel findCountModelByAssetID(long dataasetId); + + List<DqModel> getByDataAsset(DataAsset da); + + List<DqModel> getByStatus(int testing); + + DqModel findByColumn(String string, String modelid); + + DqModel findByName(String name); + + void addReference(DqModel countModel, String name); + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqModelRepoImpl.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqModelRepoImpl.java b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqModelRepoImpl.java new file mode 100644 index 0000000..722821a --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqModelRepoImpl.java @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2016 eBay Software Foundation. Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable + * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License + * for the specific language governing permissions and limitations under the License. + */ + +package com.ebay.oss.griffin.repo; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.springframework.stereotype.Repository; + +import com.ebay.oss.griffin.domain.DataAsset; +import com.ebay.oss.griffin.domain.DqModel; +import com.ebay.oss.griffin.domain.ModelType; +import com.ebay.oss.griffin.domain.ValidityType; +import com.google.gson.Gson; +import com.mongodb.BasicDBObject; +import com.mongodb.DBCursor; +import com.mongodb.DBObject; +import com.mongodb.util.JSON; + +@Repository +public class DqModelRepoImpl extends BaseIdRepo<DqModel> implements DqModelRepo { + + public DqModelRepoImpl() { + super("dq_model", "DQ_MODEL_NO", DqModel.class); + } + + @Override + public List<DqModel> getByStatus(int status) { + List<DqModel> result = new ArrayList<DqModel>(); + DBCursor cursor = dbCollection.find(new BasicDBObject("status", status)); + for (DBObject dbo : cursor) { + result.add(toEntity(dbo)); + } + return result; + } + + // FIXME what's the difference with save(T) + @Override + public DqModel update(DqModel entity) { + DBObject temp = dbCollection.findOne(new BasicDBObject("modelId", entity.getModelName())); + if (temp != null) { + dbCollection.remove(temp); + } + + Gson gson = new Gson(); + DBObject t1 = (DBObject) JSON.parse(gson.toJson(entity)); + dbCollection.save(t1); + + return toEntity(t1); + } + + // FIXME concerned could be removed + // get models concerned with data asset + // allConcerned: false- only the models directly concerned with the data asset + // true - the models directly concerned and non-directly concerned(eg. as the source asset of + // accuracy model) +// +// } else if (allConcerned) { // check the non-directly concerned models +// if (me.getModelType() == ModelType.ACCURACY) { // accuracy +// // find model +// DqModel entity = findByName(me.getModelName()); +// ModelInput mi = new ModelInput(); +// mi.parseFromString(entity.getModelContent()); +// +// // get mapping list to get the asset name +// String otherAsset = ""; +// List<MappingItemInput> mappingList = mi.getMappings(); +// Iterator<MappingItemInput> mpitr = mappingList.iterator(); +// while (mpitr.hasNext()) { +// MappingItemInput mapping = mpitr.next(); +// // since the target data asset is directly concerned, we should get source +// // as the other one +// String col = mapping.getSrc(); +// otherAsset = col.replaceFirst("\\..+", ""); // delete from the first .xxxx +// if (!otherAsset.isEmpty()) +// break; +// } +// +// // check the other asset name equals to this asst or not +// if (otherAsset.equals(da.getAssetName())) { // concerned non-directly +// result.add(me); +// } +// } + @Override + public List<DqModel> getByDataAsset(DataAsset da) { + List<DqModel> result = new ArrayList<DqModel>(); + List<DqModel> allModels = getAll(); + Iterator<DqModel> itr = allModels.iterator(); + while (itr.hasNext()) { + DqModel me = itr.next(); + if (me.getAssetId() == da.get_id()) { // concerned directly + result.add(me); + } + } + return result; + } + + @Override + public DqModel findByColumn(String colName, String value) { + + DBObject temp = dbCollection.findOne(new BasicDBObject(colName, value)); + + if (temp == null) { + return null; + } else { + Gson gson = new Gson(); + return gson.fromJson(temp.toString(), DqModel.class); + } + + } + + @Override + public DqModel findByName(String name) { + + DBObject temp = dbCollection.findOne(new BasicDBObject("modelName", name)); + + if (temp == null) { + return null; + } else { + return new Gson().fromJson(temp.toString(), DqModel.class); + } + + } + + @Override + public DqModel findCountModelByAssetID(long assetID) { + + DBCursor cursor = dbCollection.find(new BasicDBObject("assetId", assetID)); + for (DBObject tempDBObject : cursor) { + if (tempDBObject.get("modelType").equals(ModelType.VALIDITY)) { + String content = tempDBObject.get("modelContent").toString(); + String[] contents = content.split("\\|"); + if (contents[2].equals(ValidityType.TOTAL_COUNT + "")) { + return toEntity(tempDBObject); + } + } + } + + return null; + + } + + @SuppressWarnings("deprecation") + @Override + public void addReference(DqModel dqModel, String reference) { + if (!StringUtils.isBlank(dqModel.getReferenceModel())) { + reference = dqModel.getReferenceModel() + "," + reference; + } + dqModel.setReferenceModel(reference); + save(dqModel); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqScheduleRepo.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqScheduleRepo.java b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqScheduleRepo.java new file mode 100644 index 0000000..f38bd81 --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqScheduleRepo.java @@ -0,0 +1,33 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.ebay.oss.griffin.repo; + +import com.ebay.oss.griffin.domain.DqSchedule; +import com.mongodb.DBObject; + +public interface DqScheduleRepo extends BarkIdRepo<DqSchedule> { + + /////////// + void updateByModelType(DqSchedule schedule, int modelType); + + // is it required??? + void updateModelType(DBObject currentSchedule, int modelType); + + void deleteByModelList(String name); + + DBObject getValiditySchedule(long assetId); + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqScheudleRepoImpl.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqScheudleRepoImpl.java b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqScheudleRepoImpl.java new file mode 100644 index 0000000..6b55674 --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/DqScheudleRepoImpl.java @@ -0,0 +1,82 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.ebay.oss.griffin.repo; + +import org.springframework.stereotype.Repository; + +import com.ebay.oss.griffin.domain.DqSchedule; +import com.ebay.oss.griffin.domain.ModelType; +import com.google.gson.Gson; +import com.mongodb.BasicDBObject; +import com.mongodb.DBObject; +import com.mongodb.util.JSON; + +@Repository +public class DqScheudleRepoImpl extends BaseIdRepo<DqSchedule> implements DqScheduleRepo { + + public DqScheudleRepoImpl() { + super("dq_schedule", "DQ_JOB_SEQ_NO", DqSchedule.class); + } + + @Override + public DBObject getValiditySchedule(long assetId) { + BasicDBObject document = new BasicDBObject(); + document.put("assetId", assetId); + document.put("jobType", ModelType.VALIDITY); + return dbCollection.findOne(document); + } + + @Override + public void updateByModelType(DqSchedule schedule, int type) { + DBObject dbo = null; + if (type == ModelType.ACCURACY) { + dbo = dbCollection.findOne(new BasicDBObject("modelList", schedule.getModelList())); + } else if (type == ModelType.VALIDITY) { + dbo = getValiditySchedule(schedule.getAssetId()); + } + + if (dbo != null) { + dbCollection.remove(dbo); + } + + Gson gson = new Gson(); + DBObject t1 = (DBObject) JSON.parse(gson.toJson(schedule)); + dbCollection.save(t1); + } + + @Override + public void updateModelType(DBObject schedule, int type) { + DBObject dbo = null; + if (type == ModelType.ACCURACY) { + dbo = dbCollection.findOne(new BasicDBObject("modelList", schedule + .get("modelList"))); + } else if (type == ModelType.VALIDITY) { + dbo = getValiditySchedule(Integer.parseInt(schedule.get("assetId").toString())); + } + if (dbo != null) + dbCollection.remove(dbo); + dbCollection.save(schedule); + } + + @Override + public void deleteByModelList(String modelList) { + DBObject dbo = dbCollection.findOne(new BasicDBObject("modelList", modelList)); + if (dbo != null) { + dbCollection.remove(dbo); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/repo/SampleFilePathRepo.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/repo/SampleFilePathRepo.java b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/SampleFilePathRepo.java new file mode 100644 index 0000000..0b03023 --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/SampleFilePathRepo.java @@ -0,0 +1,27 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.ebay.oss.griffin.repo; + +import java.util.List; + +import com.ebay.oss.griffin.domain.SampleFilePathLKP; +import com.mongodb.DBObject; + +public interface SampleFilePathRepo extends BarkIdRepo<SampleFilePathLKP> { + + List<DBObject> findByModelName(String modelName); + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/repo/SampleFilePathRepoImpl.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/repo/SampleFilePathRepoImpl.java b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/SampleFilePathRepoImpl.java new file mode 100644 index 0000000..e4595be --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/SampleFilePathRepoImpl.java @@ -0,0 +1,43 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.ebay.oss.griffin.repo; + +import java.util.List; + +import org.springframework.stereotype.Repository; + +import com.ebay.oss.griffin.domain.SampleFilePathLKP; +import com.mongodb.BasicDBObject; +import com.mongodb.DBCursor; +import com.mongodb.DBObject; + +@Repository +public class SampleFilePathRepoImpl extends BaseIdRepo<SampleFilePathLKP> implements SampleFilePathRepo { + + public SampleFilePathRepoImpl() { + super("dq_missed_file_path_lkp", "DQ_MISSED_FILE_SEQ_NO", SampleFilePathLKP.class); + } + + @Override + public List<DBObject> findByModelName(String name) { + + DBCursor temp = dbCollection + .find(new BasicDBObject("modelName", name)).limit(20) + .sort(new BasicDBObject("timestamp", -1)); + return temp.toArray(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/repo/SequenceRepo.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/repo/SequenceRepo.java b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/SequenceRepo.java new file mode 100644 index 0000000..44c2e79 --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/SequenceRepo.java @@ -0,0 +1,22 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.ebay.oss.griffin.repo; + +public interface SequenceRepo { + + Long getNextSequence(String seq); + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/repo/SequenceRepoImpl.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/repo/SequenceRepoImpl.java b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/SequenceRepoImpl.java new file mode 100644 index 0000000..46fba24 --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/SequenceRepoImpl.java @@ -0,0 +1,44 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.ebay.oss.griffin.repo; + +import org.springframework.stereotype.Repository; + +import com.mongodb.BasicDBObject; +import com.mongodb.DBObject; + +@SuppressWarnings({"unchecked", "rawtypes"}) +@Repository +public class SequenceRepoImpl extends BaseRepo implements SequenceRepo { + + public SequenceRepoImpl() { + super("SEQUENCES", Object.class); + } + + @Override + public synchronized Long getNextSequence(String seq) { + DBObject temp = dbCollection.findOne(new BasicDBObject("_id", seq)); + + BasicDBObject document = new BasicDBObject(); + document.put("_id", seq); + document.put(seq, Long.parseLong(temp.get(seq).toString()) + 1); + + dbCollection.save(document); + + return new Long((Long.parseLong(temp.get(seq).toString()) + 1)); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/repo/UserSubscriptionRepo.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/repo/UserSubscriptionRepo.java b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/UserSubscriptionRepo.java new file mode 100644 index 0000000..fee43d7 --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/UserSubscriptionRepo.java @@ -0,0 +1,26 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.ebay.oss.griffin.repo; + +import com.ebay.oss.griffin.domain.UserSubscription; + +public interface UserSubscriptionRepo { + + void upsertUserSubscribe(UserSubscription item); + + UserSubscription getUserSubscribeItem(String user); + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/repo/UserSubscriptionRepoImpl.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/repo/UserSubscriptionRepoImpl.java b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/UserSubscriptionRepoImpl.java new file mode 100644 index 0000000..6a50e46 --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/UserSubscriptionRepoImpl.java @@ -0,0 +1,60 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.ebay.oss.griffin.repo; + +import org.springframework.stereotype.Repository; + +import com.ebay.oss.griffin.domain.UserSubscription; +import com.google.gson.Gson; +import com.mongodb.BasicDBObject; +import com.mongodb.DBObject; +import com.mongodb.util.JSON; + +@Repository +public class UserSubscriptionRepoImpl extends BaseRepo<UserSubscription> + implements UserSubscriptionRepo { + + private UserSubscriptionRepoImpl() { + super("user_subscribe", UserSubscription.class); + } + + @Override + public void upsertUserSubscribe(UserSubscription item) { + if (item.getNtaccount() == null) + return; + + item.setId(item.getNtaccount());// user_subscribe + + DBObject find = dbCollection.findOne(new BasicDBObject("_id", item.getId())); + if (find != null) + dbCollection.remove(find); + + Gson gson = new Gson(); + DBObject t1 = (DBObject) JSON.parse(gson.toJson(item)); + dbCollection.save(t1); + } + + @Override + public UserSubscription getUserSubscribeItem(String user) { + Gson gson = new Gson(); + DBObject find = dbCollection.findOne(new BasicDBObject("_id", user)); + + if (find == null) + return null; + else + return gson.fromJson(find.toString(), UserSubscription.class); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DQMetricsController.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DQMetricsController.java b/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DQMetricsController.java new file mode 100644 index 0000000..44c2acb --- /dev/null +++ b/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DQMetricsController.java @@ -0,0 +1,253 @@ +/* + Copyright (c) 2016 eBay Software Foundation. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.ebay.oss.griffin.resources; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.validation.Valid; +import javax.ws.rs.Consumes; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.ResponseBuilder; +import javax.ws.rs.core.StreamingOutput; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.ebay.oss.griffin.domain.DqMetricsValue; +import com.ebay.oss.griffin.domain.SampleFilePathLKP; +import com.ebay.oss.griffin.error.BarkDbOperationException; +import com.ebay.oss.griffin.error.BarkWebException; +import com.ebay.oss.griffin.error.ErrorMessage; +import com.ebay.oss.griffin.service.DQMetricsService; +import com.ebay.oss.griffin.service.DqModelService; +import com.ebay.oss.griffin.vo.AssetLevelMetrics; +import com.ebay.oss.griffin.vo.DqModelVo; +import com.ebay.oss.griffin.vo.OverViewStatistics; +import com.ebay.oss.griffin.vo.SampleOut; +import com.ebay.oss.griffin.vo.SystemLevelMetrics; + +@Component +// @Scope("request") +@Path("/metrics") +public class DQMetricsController { + + // + + @Autowired + private DQMetricsService dqMetricsService; + + @Autowired + private DqModelService dqModelService; + + @POST + @Path("/") + @Consumes({ "application/json" }) + public void insertMetadata(@Valid DqMetricsValue dq) { + if (dq != null) { + ErrorMessage err = dq.validate(); + if (err != null) { + + throw new BarkWebException(Response.Status.BAD_REQUEST.getStatusCode(), + err.getMessage()); + } + } + + try{ + dqMetricsService.insertMetadata(dq); + }catch(BarkDbOperationException e){ + throw new BarkWebException(Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), + e.getMessage()); + } + + + } + + @GET + @Path("/{asset_id}/latest") + @Produces(MediaType.APPLICATION_JSON) + public DqMetricsValue getLatestMetricsValueById( + @PathParam("asset_id") String assetId) { + return dqMetricsService.getLatestlMetricsbyId(assetId); + } + + @GET + @Path("/heatmap") + @Produces(MediaType.APPLICATION_JSON) + public List<SystemLevelMetrics> getHeatMap() { + return dqMetricsService.heatMap(); + } + + @GET + @Path("/briefmetrics") + @Produces(MediaType.APPLICATION_JSON) + public List<SystemLevelMetrics> getAllBriefMetrics() { + return dqMetricsService.briefMetrics("all"); + } + + @GET + @Path("/briefmetrics/{system}") + @Produces(MediaType.APPLICATION_JSON) + public List<SystemLevelMetrics> getBriefMetrics( + @PathParam("system") String system) { + return dqMetricsService.briefMetrics(system); + } + + @GET + @Path("/dashboard") + @Produces(MediaType.APPLICATION_JSON) + public List<SystemLevelMetrics> getAllDashboard() { + List<DqModelVo> models = dqModelService.getAllModles(); + Map<String, String> modelMap = new HashMap<String, String>(); + + for (DqModelVo model : models) { + modelMap.put( + model.getName(), + model.getAssetName() == null ? "unknow" : model + .getAssetName()); + } + + List<SystemLevelMetrics> sysMetricsList = dqMetricsService + .dashboard("all"); + for (SystemLevelMetrics sys : sysMetricsList) { + List<AssetLevelMetrics> assetList = sys.getMetrics(); + if (assetList != null && assetList.size() > 0) { + for (AssetLevelMetrics metrics : assetList) { + metrics.setAssetName(modelMap.get(metrics.getName())); + } + } + } + + return sysMetricsList; + } + + @GET + @Path("/dashboard/{system}") + @Produces(MediaType.APPLICATION_JSON) + public List<SystemLevelMetrics> getDashboard( + @PathParam("system") String system) { + return dqMetricsService.dashboard(system); + } + + @GET + @Path("/mydashboard/{user}") + public List<SystemLevelMetrics> getAllDashboard( + @PathParam("user") String user) { + return dqMetricsService.mydashboard(user); + } + + @GET + @Path("/complete/{name}") + @Produces(MediaType.APPLICATION_JSON) + public AssetLevelMetrics getCompelete(@PathParam("name") String name) { + return dqMetricsService.oneDataCompleteDashboard(name); + } + + @GET + @Path("/brief/{name}") + @Produces(MediaType.APPLICATION_JSON) + public AssetLevelMetrics getBrief(@PathParam("name") String name) { + return dqMetricsService.oneDataBriefDashboard(name); + } + + @GET + @Path("/refresh") + public void refreshDQ() { + dqMetricsService.updateLatestDQList(); + } + + @GET + @Path("/statics") + @Produces(MediaType.APPLICATION_JSON) + public OverViewStatistics getOverViewStats() { + return dqMetricsService.getOverViewStats(); + } + + @POST + @Path("/sample") + @Consumes({ "application/json" }) + public Response insertSamplePath(SampleFilePathLKP sample) { + dqMetricsService.insertSampleFilePath(sample); + return Response.status(Response.Status.CREATED).build(); + } + + @GET + @Path("/sample/{name}") + @Produces(MediaType.APPLICATION_JSON) + public List<SampleOut> getSampleList(@PathParam("name") String name) { + return dqMetricsService.listSampleFile(name); + } + + @GET + @Path("/download/{path:.+}") + @Produces(MediaType.APPLICATION_OCTET_STREAM) + public Response downloadFile(@PathParam("path") final String path, + @DefaultValue("100") @QueryParam("count") final Long count) { + + StreamingOutput fileStream = new StreamingOutput() { + @Override + public void write(OutputStream output) throws IOException, + WebApplicationException { + try { + Process pro = Runtime.getRuntime().exec( + "hadoop fs -cat /" + path); + + BufferedReader buff = new BufferedReader( + new InputStreamReader(pro.getInputStream())); + + int cnt = 0; + String line = null; + while ((line = buff.readLine()) != null) { + if (cnt < count) { + output.write((line.getBytes())); + output.write("\n".getBytes()); + output.flush(); + } else { + break; + } + cnt++; + } + buff.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + }; + + String[] pathTokens = path.split("/"); + String fileName = pathTokens[pathTokens.length-2] + "_" + pathTokens[pathTokens.length-1]; + + ResponseBuilder response = Response.ok(fileStream); + response.header("Content-Disposition", + "attachment; filename=" + fileName); + return response.build(); + + } + +}
