http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/repo/UserSubscriptionRepoImpl.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/repo/UserSubscriptionRepoImpl.java b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/UserSubscriptionRepoImpl.java deleted file mode 100644 index 6a50e46..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/repo/UserSubscriptionRepoImpl.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.repo; - -import org.springframework.stereotype.Repository; - -import com.ebay.oss.griffin.domain.UserSubscription; -import com.google.gson.Gson; -import com.mongodb.BasicDBObject; -import com.mongodb.DBObject; -import com.mongodb.util.JSON; - -@Repository -public class UserSubscriptionRepoImpl extends BaseRepo<UserSubscription> - implements UserSubscriptionRepo { - - private UserSubscriptionRepoImpl() { - super("user_subscribe", UserSubscription.class); - } - - @Override - public void upsertUserSubscribe(UserSubscription item) { - if (item.getNtaccount() == null) - return; - - item.setId(item.getNtaccount());// user_subscribe - - DBObject find = dbCollection.findOne(new BasicDBObject("_id", item.getId())); - if (find != null) - dbCollection.remove(find); - - Gson gson = new Gson(); - DBObject t1 = (DBObject) JSON.parse(gson.toJson(item)); - dbCollection.save(t1); - } - - @Override - public UserSubscription getUserSubscribeItem(String user) { - Gson gson = new Gson(); - DBObject find = dbCollection.findOne(new BasicDBObject("_id", user)); - - if (find == null) - return null; - else - return gson.fromJson(find.toString(), UserSubscription.class); - } - -}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DQMetricsController.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DQMetricsController.java b/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DQMetricsController.java deleted file mode 100644 index 44c2acb..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DQMetricsController.java +++ /dev/null @@ -1,253 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.resources; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import javax.validation.Valid; -import javax.ws.rs.Consumes; -import javax.ws.rs.DefaultValue; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.WebApplicationException; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.Response.ResponseBuilder; -import javax.ws.rs.core.StreamingOutput; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import com.ebay.oss.griffin.domain.DqMetricsValue; -import com.ebay.oss.griffin.domain.SampleFilePathLKP; -import com.ebay.oss.griffin.error.BarkDbOperationException; -import com.ebay.oss.griffin.error.BarkWebException; -import com.ebay.oss.griffin.error.ErrorMessage; -import com.ebay.oss.griffin.service.DQMetricsService; -import com.ebay.oss.griffin.service.DqModelService; -import com.ebay.oss.griffin.vo.AssetLevelMetrics; -import com.ebay.oss.griffin.vo.DqModelVo; -import com.ebay.oss.griffin.vo.OverViewStatistics; -import com.ebay.oss.griffin.vo.SampleOut; -import com.ebay.oss.griffin.vo.SystemLevelMetrics; - -@Component -// @Scope("request") -@Path("/metrics") -public class DQMetricsController { - - // - - @Autowired - private DQMetricsService dqMetricsService; - - @Autowired - private DqModelService dqModelService; - - @POST - @Path("/") - @Consumes({ "application/json" }) - public void insertMetadata(@Valid DqMetricsValue dq) { - if (dq != null) { - ErrorMessage err = dq.validate(); - if (err != null) { - - throw new BarkWebException(Response.Status.BAD_REQUEST.getStatusCode(), - err.getMessage()); - } - } - - try{ - dqMetricsService.insertMetadata(dq); - }catch(BarkDbOperationException e){ - throw new BarkWebException(Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), - e.getMessage()); - } - - - } - - @GET - @Path("/{asset_id}/latest") - @Produces(MediaType.APPLICATION_JSON) - public DqMetricsValue getLatestMetricsValueById( - @PathParam("asset_id") String assetId) { - return dqMetricsService.getLatestlMetricsbyId(assetId); - } - - @GET - @Path("/heatmap") - @Produces(MediaType.APPLICATION_JSON) - public List<SystemLevelMetrics> getHeatMap() { - return dqMetricsService.heatMap(); - } - - @GET - @Path("/briefmetrics") - @Produces(MediaType.APPLICATION_JSON) - public List<SystemLevelMetrics> getAllBriefMetrics() { - return dqMetricsService.briefMetrics("all"); - } - - @GET - @Path("/briefmetrics/{system}") - @Produces(MediaType.APPLICATION_JSON) - public List<SystemLevelMetrics> getBriefMetrics( - @PathParam("system") String system) { - return dqMetricsService.briefMetrics(system); - } - - @GET - @Path("/dashboard") - @Produces(MediaType.APPLICATION_JSON) - public List<SystemLevelMetrics> getAllDashboard() { - List<DqModelVo> models = dqModelService.getAllModles(); - Map<String, String> modelMap = new HashMap<String, String>(); - - for (DqModelVo model : models) { - modelMap.put( - model.getName(), - model.getAssetName() == null ? "unknow" : model - .getAssetName()); - } - - List<SystemLevelMetrics> sysMetricsList = dqMetricsService - .dashboard("all"); - for (SystemLevelMetrics sys : sysMetricsList) { - List<AssetLevelMetrics> assetList = sys.getMetrics(); - if (assetList != null && assetList.size() > 0) { - for (AssetLevelMetrics metrics : assetList) { - metrics.setAssetName(modelMap.get(metrics.getName())); - } - } - } - - return sysMetricsList; - } - - @GET - @Path("/dashboard/{system}") - @Produces(MediaType.APPLICATION_JSON) - public List<SystemLevelMetrics> getDashboard( - @PathParam("system") String system) { - return dqMetricsService.dashboard(system); - } - - @GET - @Path("/mydashboard/{user}") - public List<SystemLevelMetrics> getAllDashboard( - @PathParam("user") String user) { - return dqMetricsService.mydashboard(user); - } - - @GET - @Path("/complete/{name}") - @Produces(MediaType.APPLICATION_JSON) - public AssetLevelMetrics getCompelete(@PathParam("name") String name) { - return dqMetricsService.oneDataCompleteDashboard(name); - } - - @GET - @Path("/brief/{name}") - @Produces(MediaType.APPLICATION_JSON) - public AssetLevelMetrics getBrief(@PathParam("name") String name) { - return dqMetricsService.oneDataBriefDashboard(name); - } - - @GET - @Path("/refresh") - public void refreshDQ() { - dqMetricsService.updateLatestDQList(); - } - - @GET - @Path("/statics") - @Produces(MediaType.APPLICATION_JSON) - public OverViewStatistics getOverViewStats() { - return dqMetricsService.getOverViewStats(); - } - - @POST - @Path("/sample") - @Consumes({ "application/json" }) - public Response insertSamplePath(SampleFilePathLKP sample) { - dqMetricsService.insertSampleFilePath(sample); - return Response.status(Response.Status.CREATED).build(); - } - - @GET - @Path("/sample/{name}") - @Produces(MediaType.APPLICATION_JSON) - public List<SampleOut> getSampleList(@PathParam("name") String name) { - return dqMetricsService.listSampleFile(name); - } - - @GET - @Path("/download/{path:.+}") - @Produces(MediaType.APPLICATION_OCTET_STREAM) - public Response downloadFile(@PathParam("path") final String path, - @DefaultValue("100") @QueryParam("count") final Long count) { - - StreamingOutput fileStream = new StreamingOutput() { - @Override - public void write(OutputStream output) throws IOException, - WebApplicationException { - try { - Process pro = Runtime.getRuntime().exec( - "hadoop fs -cat /" + path); - - BufferedReader buff = new BufferedReader( - new InputStreamReader(pro.getInputStream())); - - int cnt = 0; - String line = null; - while ((line = buff.readLine()) != null) { - if (cnt < count) { - output.write((line.getBytes())); - output.write("\n".getBytes()); - output.flush(); - } else { - break; - } - cnt++; - } - buff.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - }; - - String[] pathTokens = path.split("/"); - String fileName = pathTokens[pathTokens.length-2] + "_" + pathTokens[pathTokens.length-1]; - - ResponseBuilder response = Response.ok(fileStream); - response.header("Content-Disposition", - "attachment; filename=" + fileName); - return response.build(); - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DQModelController.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DQModelController.java b/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DQModelController.java deleted file mode 100644 index 5e8c479..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DQModelController.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.resources; - -import java.util.Date; -import java.util.List; - -import javax.ws.rs.DELETE; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.web.bind.annotation.RequestBody; - -import com.ebay.oss.griffin.error.BarkDbOperationException; -import com.ebay.oss.griffin.error.BarkWebException; -import com.ebay.oss.griffin.error.ErrorMessage; -import com.ebay.oss.griffin.service.DqModelService; -import com.ebay.oss.griffin.service.NotificationService; -import com.ebay.oss.griffin.vo.DqModelVo; -import com.ebay.oss.griffin.vo.ModelInput; -import com.ebay.oss.griffin.vo.NotificationRecord; -import com.sun.jersey.api.Responses; - -@Component -// @Scope("request") -@Path("/models") -public class DQModelController { - - @Autowired - private DqModelService dqModelService; - - @Autowired - private NotificationService notificationService; - - @GET - @Path("/") - @Produces(MediaType.APPLICATION_JSON) - public List<DqModelVo> allModels() { - return dqModelService.getAllModles(); - } - - @GET - @Path("/{name}") - @Produces(MediaType.APPLICATION_JSON) - public ModelInput getModel(@PathParam("name") String name) { - - ModelInput model = dqModelService.getModelByName(name); - if (model == null) { - throw new BarkWebException(Responses.NOT_FOUND, - "The model you are looking for doesn't exist!"); - } - return model; - } - - @GET - @Path("/enableModel/{name}") - public void enableModel(@PathParam("name") String name) { - dqModelService.enableSchedule4Model(dqModelService.getGeneralModel(name)); - } - - - @DELETE - @Path("/{name}") - public void deleteModel(@PathParam("name") String name) throws BarkDbOperationException { - - try { - dqModelService.deleteModel(name); - } catch (BarkDbOperationException e) { - throw new BarkWebException(Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), e.getMessage()); - } - } - - @POST - @Path("/") - public void newModel(@RequestBody ModelInput input) { - if (input != null) { - - ErrorMessage err = input.validate(); - if (err != null) { - throw new BarkWebException(Response.Status.BAD_REQUEST.getStatusCode(), err.getMessage()); - } - - try { - dqModelService.newModel(input); - notificationService.insert(new NotificationRecord(new Date().getTime(), input.getBasic().getOwner(), - "create", "model", input.getBasic().getName())); - - } catch (BarkDbOperationException e) { - throw new BarkWebException(Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), e.getMessage()); - } - } - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DataAssetController.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DataAssetController.java b/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DataAssetController.java deleted file mode 100644 index 2386dc0..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DataAssetController.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.resources; - -import java.util.Date; -import java.util.List; - -import javax.ws.rs.DELETE; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.PUT; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import com.ebay.oss.griffin.domain.DataAsset; -import com.ebay.oss.griffin.error.BarkDbOperationException; -import com.ebay.oss.griffin.error.BarkWebException; -import com.ebay.oss.griffin.service.DataAssetService; -import com.ebay.oss.griffin.service.NotificationService; -import com.ebay.oss.griffin.vo.DataAssetInput; -import com.ebay.oss.griffin.vo.NotificationRecord; -import com.ebay.oss.griffin.vo.PlatformMetadata; -import com.sun.jersey.api.Responses; - -@Component -//@Scope("request") -@Path("/dataassets") -public class DataAssetController { - - @Autowired - private DataAssetService dataAssetService; - @Autowired - private NotificationService notificationService; - - /** - * Get data asset metadata - * - * @return - */ - @GET - @Path("/metadata") - @Produces(MediaType.APPLICATION_JSON) - public List<PlatformMetadata> getSrcTree() { - return dataAssetService.getSourceTree(); - } - - /** - * Register a new data asset - * - * @param input - * @return - * @throws BarkWebException - */ - @POST - @Path("/") - public void registerNewDataAsset(DataAssetInput input) - throws BarkWebException { - try { - dataAssetService.createDataAsset(input); - notificationService.insert(new NotificationRecord(new Date() - .getTime(), input.getOwner(), "create", "dataasset", input - .getAssetName())); - // return Response.status(Response.Status.CREATED).build(); - } catch (BarkDbOperationException e) { - - throw new BarkWebException( - Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), - e.getMessage()); - } - - } - - /** - * Get all data asset list - * - * @return - */ - @GET - @Path("/") - @Produces(MediaType.APPLICATION_JSON) - public List<DataAsset> getAssetList() { - return dataAssetService.getAllDataAssets(); - } - - /** - * Get one data asset by id - * - * @return - * @throws BarkWebException - */ - @GET - @Path("/{id}") - @Produces(MediaType.APPLICATION_JSON) - public DataAsset getOneAsset(@PathParam("id") Long id) - throws BarkWebException { - DataAsset da = dataAssetService.getDataAssetById(id); - if (da == null) { - throw new BarkWebException(Responses.NOT_FOUND, - "The asset you are looking for doesn't exist"); - } else { - return da; - } - } - - /** - * Remove a data asset by id - * - * @param id - * @return - */ - @DELETE - @Path("/{id}") - public void removeAssetById(@PathParam("id") Long id) throws BarkWebException { - - try { - dataAssetService.removeAssetById(id); - - } catch (BarkDbOperationException e) { - throw new BarkWebException( - Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), - e.getMessage()); - } - - } - - /** - * Update a new data asset - * - * @param input - * @return - */ - @PUT - @Path("/") - public void updateDataAsset(DataAssetInput input) throws BarkWebException{ - - try { - dataAssetService.updateDataAsset(input); - - } catch (BarkDbOperationException e) { - throw new BarkWebException( - Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), - e.getMessage()); - } - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/resources/LoginController.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/LoginController.java b/griffin-core/src/main/java/com/ebay/oss/griffin/resources/LoginController.java deleted file mode 100644 index 7f4d4df..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/LoginController.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.resources; - -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.web.bind.annotation.RequestBody; - -import com.ebay.oss.griffin.service.LoginService; -import com.ebay.oss.griffin.vo.LoginUser; - -@Component -//@Scope("request") -@Path("/login") -public class LoginController { - - @Autowired - private LoginService loginService; - - @POST - @Path("/authenticate") - @Produces(MediaType.APPLICATION_JSON) - public String authenticate(@RequestBody LoginUser input) { - if("test".equals(input.getUsername())){ - return "{\"status\": 0, \"ntAccount\": \"test\", \"fullName\": \"test\"}"; - } - - String result = loginService.login(input.getUsername(), - input.getPassword()); - if (result != null) - return "{\"status\": 0, \"ntAccount\": \"" + input.getUsername() - + "\", \"fullName\": \"" + result + "\"}"; - else - return "{\"status\": -1, \"ntAccount\": \"" + input.getUsername() - + "\", \"fullName\": \"\",\"errMsg\":\"logon failed.\"}"; - // return "{\"status\": -1, \"errMsg\": \"Invalid username\"}"; - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/resources/NotificationController.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/NotificationController.java b/griffin-core/src/main/java/com/ebay/oss/griffin/resources/NotificationController.java deleted file mode 100644 index e2ffb59..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/NotificationController.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.resources; - -import java.util.List; - -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import com.ebay.oss.griffin.service.NotificationService; -import com.ebay.oss.griffin.vo.NotificationRecord; - -@Component -//@Scope("request") -@Path("/notifications") -public class NotificationController { - @Autowired - private NotificationService notificationService; - - @GET - @Path("/") - @Produces(MediaType.APPLICATION_JSON) - public List<NotificationRecord> getAll(){ - return notificationService.getAll(); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/resources/ScheduleController.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/ScheduleController.java b/griffin-core/src/main/java/com/ebay/oss/griffin/resources/ScheduleController.java deleted file mode 100644 index d3dfc1a..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/ScheduleController.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.resources; - -import javax.ws.rs.GET; -import javax.ws.rs.Path; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import com.ebay.oss.griffin.service.DqScheduleService; - -@Component -//@Scope("request") -@Path("/schedule") -public class ScheduleController { - - @Autowired - private DqScheduleService dqJobSchedulingService; - - @GET - @Path("/") - public void schedule() { - dqJobSchedulingService.schedulingJobs(); - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/resources/SubscribeController.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/SubscribeController.java b/griffin-core/src/main/java/com/ebay/oss/griffin/resources/SubscribeController.java deleted file mode 100644 index 91e3f81..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/SubscribeController.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.resources; - -import javax.ws.rs.Consumes; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.web.bind.annotation.RequestBody; - -import com.ebay.oss.griffin.domain.UserSubscription; -import com.ebay.oss.griffin.service.SubscribeService; - - -@Component -//@Scope("request") -@Path("/subscribe") -public class SubscribeController { - - @Autowired - private SubscribeService subscribeService; - - @GET - @Path("/{user}") - @Produces(MediaType.APPLICATION_JSON) - public UserSubscription get(@PathParam("user") String user) { - return subscribeService.getSubscribe(user); - } - - @POST - @Path("/") - @Consumes({"application/json"}) - public void set(@RequestBody UserSubscription input) { - subscribeService.subscribe(input); - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/Converter.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/Converter.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/Converter.java deleted file mode 100644 index 14a827c..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/Converter.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.service; - -public interface Converter<E, V> { - - V voOf(E e); - - E entityOf(V vo); - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/DQMetricsService.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DQMetricsService.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/DQMetricsService.java deleted file mode 100644 index aa19cbc..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DQMetricsService.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.service; - -import java.util.List; - -import com.ebay.oss.griffin.domain.DqMetricsValue; -import com.ebay.oss.griffin.domain.SampleFilePathLKP; -import com.ebay.oss.griffin.vo.AssetLevelMetrics; -import com.ebay.oss.griffin.vo.OverViewStatistics; -import com.ebay.oss.griffin.vo.SampleOut; -import com.ebay.oss.griffin.vo.SystemLevelMetrics; - -public interface DQMetricsService { - - public void insertMetadata(DqMetricsValue dq); - - public DqMetricsValue getLatestlMetricsbyId(String assetId); - - public List<SystemLevelMetrics> briefMetrics(String system); - - public void updateLatestDQList(); - - public OverViewStatistics getOverViewStats(); - - ////////////// sample-file - public List<SampleOut> listSampleFile(String modelName); - -// public void downloadSample(String filePath); - - public void insertSampleFilePath(SampleFilePathLKP samplePath); - - ////////////// HeatMap - public List<SystemLevelMetrics> heatMap(); - - ///////////// dashboard - public List<SystemLevelMetrics> dashboard(String system); - - public AssetLevelMetrics oneDataCompleteDashboard(String name); - - public AssetLevelMetrics oneDataBriefDashboard(String name); - - public AssetLevelMetrics metricsForReport(String name); - - public List<SystemLevelMetrics> mydashboard(String user); - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/DQMetricsServiceImpl.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DQMetricsServiceImpl.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/DQMetricsServiceImpl.java deleted file mode 100644 index e7ee1d8..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DQMetricsServiceImpl.java +++ /dev/null @@ -1,321 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.service; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import com.ebay.oss.griffin.common.Pair; -import com.ebay.oss.griffin.domain.DqMetricsValue; -import com.ebay.oss.griffin.domain.DqModel; -import com.ebay.oss.griffin.domain.SampleFilePathLKP; -import com.ebay.oss.griffin.domain.SystemType; -import com.ebay.oss.griffin.error.BarkDbOperationException; -import com.ebay.oss.griffin.repo.DataAssetRepo; -import com.ebay.oss.griffin.repo.DqMetricsRepo; -import com.ebay.oss.griffin.repo.DqModelRepo; -import com.ebay.oss.griffin.repo.SampleFilePathRepo; -import com.ebay.oss.griffin.vo.AssetLevelMetrics; -import com.ebay.oss.griffin.vo.DQHealthStats; -import com.ebay.oss.griffin.vo.DqModelVo; -import com.ebay.oss.griffin.vo.OverViewStatistics; -import com.ebay.oss.griffin.vo.SampleOut; -import com.ebay.oss.griffin.vo.SystemLevelMetrics; -import com.ebay.oss.griffin.vo.SystemLevelMetricsList; -import com.mongodb.DBObject; - -@Service("dqmetrics") -public class DQMetricsServiceImpl implements DQMetricsService { - - private static Logger logger = LoggerFactory.getLogger(DQMetricsServiceImpl.class); - - @Autowired - private DqModelService dqModelService; - - @Autowired - private SubscribeService subscribeService; - - @Autowired - private DqMetricsRepo metricsRepo; - - @Autowired - private DqModelRepo modelRepo; - - @Autowired - private DataAssetRepo dataAssetRepo; - - @Autowired - private SampleFilePathRepo missedFileRepo; - - @Autowired - private RefMetrcsCalc refMetricCalc; - - public static List<DqMetricsValue> cacheValues = new ArrayList<DqMetricsValue>();; - - public static SystemLevelMetricsList totalSystemLevelMetricsList; - - public void insertMetadata(DqMetricsValue metrics) { - - List<Pair> queryList = new ArrayList<Pair>(); - queryList.add(new Pair("metricName", metrics.getMetricName())); - // queryList.add(new KeyValue("metricType", dq.getMetricType())); - // queryList.add(new KeyValue("assetId", dq.getAssetId())); - queryList.add(new Pair("timestamp", metrics.getTimestamp())); - - List<Pair> updateValues = new ArrayList<Pair>(); - updateValues.add(new Pair("value", metrics.getValue())); - - DBObject item = metricsRepo.getByCondition(queryList); - - try { - if (item == null) { - long seq = metricsRepo.getNextId(); - logger.info("log: new record inserted" + seq); - metrics.set_id(seq); - metricsRepo.save(metrics); - } else { - logger.info("log: updated record"); - metricsRepo.update(metrics, item); - } - }catch(Exception e) { - throw new BarkDbOperationException("Failed to save metrics value!", e); - } - - } - - public DqMetricsValue getLatestlMetricsbyId(String assetId) { - return metricsRepo.getLatestByAssetId(assetId); - } - - void refreshAllDQMetricsValuesinCache() { - refreshModelSystemCache(); - - cacheValues.clear(); - for (DqMetricsValue each : metricsRepo.getAll()) { - cacheValues.add(each); - } - } - Map<String, String> modelSystem = new HashMap<String, String>(); - void refreshModelSystemCache() { - for (DqModel model : modelRepo.getAll()) { - modelSystem.put(model.getModelName(), SystemType.val(model.getSystem())); - } - } - - public synchronized void updateLatestDQList() { - try { - logger.info("==============updating all latest dq metrics=================="); - refreshAllDQMetricsValuesinCache(); - - totalSystemLevelMetricsList = new SystemLevelMetricsList(); - for (DqMetricsValue temp : cacheValues) { - // totalSystemLevelMetricsList.upsertNewAsset(temp, assetSystem, - // 1); - totalSystemLevelMetricsList.upsertNewAssetExecute( - temp.getMetricName(), "", temp.getTimestamp(), - temp.getValue(), modelSystem.get(temp.getMetricName()), - 0, true, null); - } - - totalSystemLevelMetricsList.updateDQFail(getThresholds()); - refMetricCalc.calc(totalSystemLevelMetricsList); - - logger.info("==============update all latest dq metrics done=================="); - } catch (Exception e) { - logger.error("{}", e); - } - } - - Map<String, String> getThresholds() { - Map<String, String> thresHolds = new HashMap<>(); - for(DqModel each : modelRepo.getAll()) { - thresHolds.put(each.getModelName(), "" + each.getThreshold()); - } - return thresHolds; - } - - List<SystemLevelMetrics> addAssetNames( - List<SystemLevelMetrics> result) { - List<DqModelVo> models = dqModelService.getAllModles(); - Map<String, String> modelMap = new HashMap<String, String>(); - - for (DqModelVo model : models) { - modelMap.put( - model.getName(), - model.getAssetName() == null ? "unknow" : model - .getAssetName()); - } - - for (SystemLevelMetrics sys : result) { - List<AssetLevelMetrics> assetList = sys.getMetrics(); - if (assetList != null && assetList.size() > 0) { - for (AssetLevelMetrics metrics : assetList) { - metrics.setAssetName(modelMap.get(metrics.getName())); - } - } - } - - return result; - } - - Map<String, String> getAssetMap() { - Map<String, String> modelMap = new HashMap<String, String>(); - - for (DqModelVo model : dqModelService.getAllModles()) { - modelMap.put( model.getName(), - model.getAssetName() == null ? "unknow" : model.getAssetName()); - } - - return modelMap; - } - - @Override - public List<SystemLevelMetrics> briefMetrics(String system) { - if (totalSystemLevelMetricsList == null) - updateLatestDQList(); - return totalSystemLevelMetricsList.getListWithLatestNAssets(24, system, null, null); - } - - @Override - public List<SystemLevelMetrics> heatMap() { - if (totalSystemLevelMetricsList == null) - updateLatestDQList(); - return totalSystemLevelMetricsList.getHeatMap(getThresholds()); - } - - @Override - public List<SystemLevelMetrics> dashboard(String system) { - if (totalSystemLevelMetricsList == null) - updateLatestDQList(); - return addAssetNames(totalSystemLevelMetricsList - .getListWithLatestNAssets(30, system, null, null)); - } - - @Override - public List<SystemLevelMetrics> mydashboard(String user) { - if (totalSystemLevelMetricsList == null) - updateLatestDQList(); - return addAssetNames(totalSystemLevelMetricsList - .getListWithLatestNAssets(30, "all", - subscribeService.getSubscribe(user), getAssetMap())); - } - - @Override - public AssetLevelMetrics oneDataCompleteDashboard(String name) { - if (totalSystemLevelMetricsList == null) - updateLatestDQList(); - return totalSystemLevelMetricsList.getListWithSpecificAssetName(name); - } - - @Override - public AssetLevelMetrics oneDataBriefDashboard(String name) { - if (totalSystemLevelMetricsList == null) - updateLatestDQList(); - return totalSystemLevelMetricsList.getListWithSpecificAssetName(name, 30); - } - - public OverViewStatistics getOverViewStats() { - - OverViewStatistics os = new OverViewStatistics(); - - os.setAssets(dataAssetRepo.getAll().size()); - os.setMetrics(modelRepo.getAll().size()); - - DQHealthStats health = new DQHealthStats(); - - if (totalSystemLevelMetricsList == null) - updateLatestDQList(); - - List<SystemLevelMetrics> allMetrics = totalSystemLevelMetricsList - .getLatestDQList(); - - int healthCnt = 0; - int invalidCnt = 0; - - for (SystemLevelMetrics metricS : allMetrics) { - - List<AssetLevelMetrics> metricsA = metricS.getMetrics(); - - for (AssetLevelMetrics m : metricsA) { - if (m.getDqfail() == 0) { - healthCnt++; - } else { - invalidCnt++; - } - } - } - - health.setHealth(healthCnt); - health.setInvalid(invalidCnt); - - health.setWarn(0); - os.setStatus(health); - - return os; - - } - - /** - * Get the metrics for 24 hours - */ - @Override - public AssetLevelMetrics metricsForReport(String name) { - if (totalSystemLevelMetricsList == null) - updateLatestDQList(); - return totalSystemLevelMetricsList.getListWithSpecificAssetName(name, 24); - } - - @Override - public List<SampleOut> listSampleFile(String modelName) { - - List<SampleOut> samples = new ArrayList<SampleOut>(); - - List<DBObject> dbos = missedFileRepo.findByModelName(modelName); - - for (DBObject dbo : dbos) { - - SampleOut so = new SampleOut(); - - so.setDate(Long.parseLong(dbo.get("timestamp").toString())); - so.setPath(dbo.get("hdfsPath").toString()); - - samples.add(so); - } - - return samples; - - } - - @Override - public void insertSampleFilePath(SampleFilePathLKP samplePath) { - SampleFilePathLKP entity = new SampleFilePathLKP(); - - entity.set_id(missedFileRepo.getNextId()); - entity.setModelName(samplePath.getModelName()); - entity.setTimestamp(samplePath.getTimestamp()); - entity.setHdfsPath(samplePath.getHdfsPath()); - - missedFileRepo.save(entity); - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/DataAssetService.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DataAssetService.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/DataAssetService.java deleted file mode 100644 index 118061a..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DataAssetService.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ -package com.ebay.oss.griffin.service; - -import java.util.List; - -import com.ebay.oss.griffin.domain.DataAsset; -import com.ebay.oss.griffin.error.BarkDbOperationException; -import com.ebay.oss.griffin.vo.DataAssetInput; -import com.ebay.oss.griffin.vo.PlatformMetadata; - -public interface DataAssetService { - public List<DataAsset> getAllDataAssets(); - - public int createDataAsset(DataAssetInput input) throws BarkDbOperationException; - - public void updateDataAsset(DataAssetInput input) throws BarkDbOperationException; - - public DataAsset getDataAssetById(Long id) throws BarkDbOperationException; - - public List<PlatformMetadata> getSourceTree(); - - public void removeAssetById(Long id) throws BarkDbOperationException; -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/DataAssetServiceImpl.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DataAssetServiceImpl.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/DataAssetServiceImpl.java deleted file mode 100644 index b6a9f28..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DataAssetServiceImpl.java +++ /dev/null @@ -1,363 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.service; - -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Service; -//import org.springframework.validation.annotation.Validated; - - - - - - - - - - - - - - - - - - - -import com.ebay.oss.griffin.common.HDFSUtils; -import com.ebay.oss.griffin.common.Pair; -import com.ebay.oss.griffin.domain.DataAsset; -import com.ebay.oss.griffin.domain.DqModel; -import com.ebay.oss.griffin.domain.ModelStatus; -import com.ebay.oss.griffin.domain.ModelType; -import com.ebay.oss.griffin.domain.ScheduleType; -import com.ebay.oss.griffin.domain.SystemType; -import com.ebay.oss.griffin.domain.ValidityType; -import com.ebay.oss.griffin.error.BarkDbOperationException; -import com.ebay.oss.griffin.repo.DataAssetRepo; -import com.ebay.oss.griffin.repo.DqModelRepo; -import com.ebay.oss.griffin.vo.DataAssetIndex; -import com.ebay.oss.griffin.vo.DataAssetInput; -import com.ebay.oss.griffin.vo.DqModelVo; -import com.ebay.oss.griffin.vo.ModelBasicInputNew; -import com.ebay.oss.griffin.vo.ModelExtraInputNew; -import com.ebay.oss.griffin.vo.ModelInput; -import com.ebay.oss.griffin.vo.PlatformMetadata; -import com.ebay.oss.griffin.vo.SystemMetadata; -import com.mongodb.DBObject; - -@Service -public class DataAssetServiceImpl implements DataAssetService { - - private static Logger logger = LoggerFactory.getLogger(DataAssetServiceImpl.class); - - @Autowired - private DataAssetRepo dataAssetRepo; - - @Autowired - private DqModelRepo dqModelRepo; - - @Autowired - private DqModelService dqModelService; - - @Autowired - private Environment env; - - // FIXME ??? - public static List<DqModelVo> allModels; - - // FIXME ??? - public static HashMap<String, String> thresholds; - - @Override - public List<DataAsset> getAllDataAssets() { - return dataAssetRepo.getAll(); - } - - protected DataAsset ofEntity(DBObject o) { - return new DataAsset(o); - } - @Override - public int createDataAsset(DataAssetInput input) - throws BarkDbOperationException { - - DataAsset da = new DataAsset(); - - List<Pair> queryList = new ArrayList<Pair>(); - queryList.add(new Pair("assetName", input.getAssetName())); - queryList.add(new Pair("assetType", input.getAssetType())); - queryList.add(new Pair("system", input.getSystem())); - - List<Pair> updateValues = new ArrayList<Pair>(); - updateValues.add(new Pair("schema", input.getSchema())); - updateValues.add(new Pair("platform", input.getPlatform())); - updateValues - .add(new Pair("assetHDFSPath", input.getAssetHDFSPath())); - updateValues.add(new Pair("owner", input.getOwner())); - - DBObject item = dataAssetRepo.getByCondition(queryList); - - if (item != null) { - throw new BarkDbOperationException("Record already existing"); - } - - String hdfsPath = input.getAssetHDFSPath(); - - String[] subs = hdfsPath.split("/"); - - StringBuilder sb = new StringBuilder(); - - for (String s : subs) { - if(!s.contains("[")){ - sb.append("/"); - sb.append(s); - } - } - - hdfsPath = sb.toString().replaceFirst("/", ""); - - - logger.info("HDFS Path: " + hdfsPath); - - if("prod".equals(env.getProperty("env"))){//in prod environment, need to validate the hdfs path - - if (!HDFSUtils.checkHDFSFolder(hdfsPath)) { - - - throw new BarkDbOperationException( - "Hdfs Path is invalid, please input valid hdfs path!"); - } - } - - try { - - long seq = dataAssetRepo.getNextId(); - da.set_id(seq); - da.setAssetName(input.getAssetName()); - da.setAssetType(input.getAssetType()); - da.setPlatform(input.getPlatform()); - da.setSystem(input.getSystem()); - da.setAssetHDFSPath(input.getAssetHDFSPath()); - da.setSchema(input.getSchema()); - da.setOwner(input.getOwner()); - da.setPartitions(input.getPartitions()); - da.setTimestamp(new Date()); - - logger.debug("log: new record inserted" + seq); - dataAssetRepo.save(da); - - //create new total count dqmodel - { - ModelInput tempCountModel = new ModelInput(); - ModelBasicInputNew basic = new ModelBasicInputNew(); - ModelExtraInputNew extra = new ModelExtraInputNew(); - basic.setDataaset(input.getAssetName()); - basic.setDataasetId(seq); - basic.setDesc("Count for " + input.getAssetName()); - basic.setEmail(input.getOwner() + "@ebay.com"); - basic.setName("TotalCount_" + input.getAssetName()); - basic.setOwner(input.getOwner()); - basic.setScheduleType(ScheduleType.DAILY); - basic.setStatus(ModelStatus.DEPLOYED); - basic.setSystem(SystemType.indexOf(input.getSystem())); - basic.setType(ModelType.VALIDITY); - extra.setVaType(ValidityType.TOTAL_COUNT); - extra.setColumn("wholeDataSet"); - extra.setSrcDataSet(input.getAssetName()); - extra.setSrcDb(input.getPlatform()); - tempCountModel.setBasic(basic); - tempCountModel.setExtra(extra); - dqModelService.newModel(tempCountModel); - } - - return 0; - - } catch (Exception e) { - throw new BarkDbOperationException( - "Failed to create a new data asset", e); - } - - } - - @Override - public void updateDataAsset(DataAssetInput input) - throws BarkDbOperationException { - try { - - DataAsset da = new DataAsset(); - - List<Pair> queryList = new ArrayList<Pair>(); - queryList.add(new Pair("assetName", input.getAssetName())); - queryList.add(new Pair("assetType", input.getAssetType())); - queryList.add(new Pair("system", input.getSystem())); - - List<Pair> updateValues = new ArrayList<Pair>(); - updateValues.add(new Pair("schema", input.getSchema())); - updateValues.add(new Pair("platform", input.getPlatform())); - updateValues.add(new Pair("assetHDFSPath", input - .getAssetHDFSPath())); - updateValues.add(new Pair("owner", input.getOwner())); - - DBObject item = dataAssetRepo.getByCondition(queryList); - if (item == null) { - throw new BarkDbOperationException( "The data asset doesn't exist"); - } - - da.setAssetName(input.getAssetName()); - da.setAssetType(input.getAssetType()); - da.setPlatform(input.getPlatform()); - da.setSystem(input.getSystem()); - da.setAssetHDFSPath(input.getAssetHDFSPath()); - da.setSchema(input.getSchema()); - da.setOwner(input.getOwner()); - da.setPartitions(input.getPartitions()); - da.setTimestamp(new Date()); - - logger.warn( "log: updated record, id is: " - + (long) Double.parseDouble(item.get("_id").toString())); - da.set_id(new Long((long) Double.parseDouble(item.get("_id") - .toString()))); - dataAssetRepo.update(da, item); - } catch (Exception e) { - throw new BarkDbOperationException("Failed to update data asset", e); - } - - } - - @Override - public DataAsset getDataAssetById(Long id) throws BarkDbOperationException { - return dataAssetRepo.getById(id); - } - - @Override - public List<PlatformMetadata> getSourceTree() { - - List<PlatformMetadata> alltree = new ArrayList<PlatformMetadata>(); - - List<DataAsset> allAssets = dataAssetRepo.getAll(); - - - for (String platform : getAllPlatforms(allAssets)) { - - PlatformMetadata plat = new PlatformMetadata(); - plat.setPlatform(platform); - - - List<SystemMetadata> d = new ArrayList<SystemMetadata>(); - - for (String dataset : getSystemsByPlatform( platform, allAssets)) { - - SystemMetadata db = new SystemMetadata(); - db.setName(dataset); - - List<DataAssetIndex> assets = new ArrayList<DataAssetIndex>(); - - Map<Long, String> tableB = getAssetsBySystem(platform, dataset, allAssets); - for (Entry<Long, String> entry : tableB.entrySet()) { - DataAssetIndex dai = new DataAssetIndex(); - dai.setId(entry.getKey()); - dai.setName(entry.getValue()); - - assets.add(dai); - } - - db.setAssets(assets); - d.add(db); - - } - - plat.setSystems(d); - alltree.add(plat); - - } - - return alltree; - - } - - private Set<String> getAllPlatforms(List<DataAsset> records) { - Set<String> p = new LinkedHashSet<>(); - for (DataAsset o : records) { - p.add(o.getPlatform()); - } - return p; - } - - public Set<String> getSystemsByPlatform(String platform, List<DataAsset> records) { - Set<String> p = new HashSet<String>(); - for (DataAsset o : records) { - if (o.getPlatform().equals(platform)) { - p.add(o.getSystem()); - } - } - return p; - - } - - public Map<Long, String> getAssetsBySystem(String platform, String system, List<DataAsset> records) { - Map<Long, String> p = new HashMap<Long, String>(); - for (DataAsset o : records) { - if (o.getPlatform().equals(platform) && o.getSystem().equals(system)) { - p.put(o.get_id(), o.getAssetName()); - } - } - return p; - - } - - @Override - public void removeAssetById(Long id) throws BarkDbOperationException { - try { - removeRelatedModels(id); - - dataAssetRepo.delete(id); - } catch (Exception e) { - throw new BarkDbOperationException("Failed to delete data asset with id of '" + id + "'", e); - } - - } - - //remove all the models related with the given data asset - private int removeRelatedModels(Long dataAssetId) { - try { - DataAsset da = dataAssetRepo.getById(dataAssetId); - if (da != null) { - //delete all the accuracy models with this given source asset - List<DqModel> models = dqModelRepo.getByDataAsset(da); - for(DqModel each : models) { - dqModelService.deleteModel(each.getModelName()); - } - } - } catch (Exception e) { - e.printStackTrace(); - } - - return 0; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelConverter.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelConverter.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelConverter.java deleted file mode 100644 index 82b3fd6..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelConverter.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.service; - -import java.util.Date; - -import org.springframework.stereotype.Component; - -import com.ebay.oss.griffin.domain.DqModel; -import com.ebay.oss.griffin.vo.DqModelVo; - -@Component("modelVoConverter") -public class DqModelConverter implements Converter<DqModel, DqModelVo>{ - - @Override - public DqModelVo voOf(DqModel o) { - if(o == null) { - return null; - } - - DqModelVo vo = new DqModelVo(); - vo.setName(o.getModelName()); - vo.setSystem(o.getSystem()); - vo.setDescription(o.getModelDesc()); - vo.setType(o.getModelType()); - vo.setCreateDate(new Date(o.getTimestamp())); - vo.setStatus("" + o.getStatus()); - vo.setAssetName(o.getAssetName()); - vo.setOwner(o.getOwner()); - return vo; - } - - @Override - public DqModel entityOf(DqModelVo vo) { - // TODO Auto-generated method stub - throw new RuntimeException("not implemented yet..."); - } - - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelCreator.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelCreator.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelCreator.java deleted file mode 100644 index 6404d39..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelCreator.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.service; - -import com.ebay.oss.griffin.domain.DqModel; -import com.ebay.oss.griffin.vo.ModelInput; - -public interface DqModelCreator { - - /** the minimum number of Jobs executed in the ModelStatus.TESTING, and then it could be shifted - * to ModelStatus.VERIFIED. - */ - int MIN_TESTING_JOB_NUMBER = 1; // here set only 1 time to execute a job, for test - - boolean isSupport(ModelInput input); - - // FIXME the return value here doesn't mean anything - DqModel newModel(ModelInput input); - -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelService.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelService.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelService.java deleted file mode 100644 index c5d19a8..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelService.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.service; - -import java.util.List; - -import com.ebay.oss.griffin.domain.DqModel; -import com.ebay.oss.griffin.error.BarkDbOperationException; -import com.ebay.oss.griffin.vo.DqModelVo; -import com.ebay.oss.griffin.vo.ModelInput; - -public interface DqModelService { - - public List<DqModelVo> getAllModles(); - - // FIXME what the return value means - // HACK why need to delete a model - public int deleteModel(String name) throws BarkDbOperationException; - - public DqModel getGeneralModel(String name); - - - public ModelInput getModelByName(String name) throws BarkDbOperationException; - - public void enableSchedule4Model(DqModel input); - - /** - * Create a new Model - * @param input - * @return 0 if successful, -1 if already existing, other negative values for other reasons - */ - public DqModel newModel(ModelInput input) throws BarkDbOperationException; -} http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelServiceImpl.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelServiceImpl.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelServiceImpl.java deleted file mode 100644 index 4a369da..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelServiceImpl.java +++ /dev/null @@ -1,181 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.service; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -import javax.annotation.Resource; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import com.ebay.oss.griffin.common.ScheduleModelSeperator; -import com.ebay.oss.griffin.domain.DqModel; -import com.ebay.oss.griffin.domain.DqSchedule; -import com.ebay.oss.griffin.domain.ModelStatus; -import com.ebay.oss.griffin.domain.ModelType; -import com.ebay.oss.griffin.error.BarkDbOperationException; -import com.ebay.oss.griffin.repo.DqMetricsRepo; -import com.ebay.oss.griffin.repo.DqModelRepo; -import com.ebay.oss.griffin.repo.DqScheduleRepo; -import com.ebay.oss.griffin.vo.DqModelVo; -import com.ebay.oss.griffin.vo.ModelInput; -import com.mongodb.DBObject; - -@Service -public class DqModelServiceImpl implements DqModelService { - - private static Logger logger = LoggerFactory.getLogger(DqModelServiceImpl.class); - - @Autowired - private DqModelRepo dqModelRepo; - - @Autowired - private DqScheduleRepo scheduleRepo; - - @Resource(name = "modelVoConverter") - private Converter<DqModel, DqModelVo> converter; - - @Resource(name = "modelInputConverter") - Converter<DqModel, ModelInput> modelInputConverter; - - @Autowired - private DqMetricsRepo metricsRepo; - - @Override - public List<DqModelVo> getAllModles() { - List<DqModelVo> allModels = new ArrayList<>(); - for(DqModel each : dqModelRepo.getAll()) { - allModels.add( converter.voOf(each)); - } - return allModels; - } - - - @Override - public int deleteModel(String name) throws BarkDbOperationException { - try { - DqModel dqModel = dqModelRepo.findByName(name); - - // TODO need to mark related metrics as deleted, instead of real deletion - // markMetricsDeleted(dqModel); - - dqModelRepo.delete(dqModel.get_id()); - - if (dqModel.getModelType() == ModelType.ACCURACY) { - scheduleRepo.deleteByModelList(name); - } else if (dqModel.getModelType() == ModelType.VALIDITY) { - DBObject currentSchedule = scheduleRepo.getValiditySchedule(dqModel.getAssetId()); - if (currentSchedule == null || currentSchedule.get("modelList") == null) { - return -1; - } - - String rawModelList = currentSchedule.get("modelList") .toString(); - String newModelList = ""; - if (rawModelList.contains(ScheduleModelSeperator.SEPERATOR)) { - String[] rawModelArray = rawModelList.split(ScheduleModelSeperator.SPLIT_SEPERATOR); - for (int i = 0; i < rawModelArray.length; i++) { - if (!rawModelArray[i].equals(name)) - newModelList = newModelList + ScheduleModelSeperator.SEPERATOR + rawModelArray[i]; - } - newModelList = newModelList.substring(ScheduleModelSeperator.SEPERATOR.length()); - currentSchedule.put("modelList", newModelList); - scheduleRepo.updateModelType(currentSchedule, dqModel.getModelType()); - - } else if (rawModelList.equals(name)) { - scheduleRepo.deleteByModelList(name); - } - } - - return 0; - } catch (Exception e) { - logger.warn(e.toString()); - throw new BarkDbOperationException("Failed to delete model of '" - + name + "'", e); - } - } - - // FIXME to be removed - @Override - public DqModel getGeneralModel(String name) { - return dqModelRepo.findByName(name); - } - - @Override - public ModelInput getModelByName(String name) throws BarkDbOperationException { - try { - DqModel dqModel = dqModelRepo.findByName(name); - return modelInputConverter.voOf(dqModel); - } catch (Exception e) { - logger.error(e.toString()); - throw new BarkDbOperationException("Failed to find model with name of '" + name + "'", e); - } - } - - @Override - public void enableSchedule4Model(DqModel dqModel) { - if(dqModel==null) return; - - dqModel.setStatus(ModelStatus.DEPLOYED); - dqModelRepo.update(dqModel); - - if (dqModel.getModelType() == ModelType.ACCURACY - || dqModel.getModelType() == ModelType.VALIDITY) { - DqSchedule schedule = new DqSchedule(); - schedule.set_id(scheduleRepo.getNextId()); - schedule.setScheduleType(dqModel.getSchedule()); - Date d = new Date(dqModel.getStarttime()); - d.setMinutes(0); - d.setSeconds(0); - schedule.setStarttime(d.getTime() / 1000 * 1000); // FIXME ???? - schedule.setStatus(0); - schedule.setAssetId(dqModel.getAssetId()); - schedule.setJobType(dqModel.getModelType()); - - String modellist = ""; - if (dqModel.getModelType() == ModelType.VALIDITY) { - DBObject currentSchedule = scheduleRepo.getValiditySchedule( - dqModel.getAssetId()); - if (currentSchedule != null) { - if (currentSchedule.get("modelList") != null - && !currentSchedule.get("modelList").toString().equals("")) { - modellist = currentSchedule.get("modelList") - .toString(); - modellist = modellist - + ScheduleModelSeperator.SEPERATOR - + dqModel.getModelName(); - } - } - if (modellist.equals("")) - modellist = dqModel.getModelName(); - } else - modellist = dqModel.getModelName(); - schedule.setModelList(modellist); - - scheduleRepo.updateByModelType(schedule, dqModel.getModelType()); - } - } - - @Resource(name = "modelCreatorChain") - DqModelCreator modelCreator; - - @Override - public DqModel newModel(ModelInput input) throws BarkDbOperationException { - return modelCreator.newModel(input); - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqScheduleService.java ---------------------------------------------------------------------- diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqScheduleService.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqScheduleService.java deleted file mode 100644 index 1cca030..0000000 --- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqScheduleService.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - Copyright (c) 2016 eBay Software Foundation. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.ebay.oss.griffin.service; - - - -/** - * TODO: to decouple the schedule/job from model/metrics. schedule/job depends on model/metric, but - * not vice versa. - */ -public interface DqScheduleService { - - void schedulingJobs(); - -}
