Repository: ranger Updated Branches: refs/heads/master 41e220381 -> ad2731811
RANGER-1848:Implement getLinkList/getJobList in SqoopClient for Ranger Sqoop2 plugin Signed-off-by: peng.jianhua <peng.jian...@zte.com.cn> Project: http://git-wip-us.apache.org/repos/asf/ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/ad273181 Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/ad273181 Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/ad273181 Branch: refs/heads/master Commit: ad2731811d56a2fa357cf06bea5e5c2f1fc1b98d Parents: 41e2203 Author: zhangqiang2 <zhangqia...@zte.com.cn> Authored: Tue Nov 21 11:01:02 2017 +0800 Committer: peng.jianhua <peng.jian...@zte.com.cn> Committed: Wed Nov 22 10:34:11 2017 +0800 ---------------------------------------------------------------------- .../services/sqoop/client/SqoopClient.java | 139 +++++++++++++++---- .../client/json/model/SqoopJobResponse.java | 104 ++++++++++++++ .../client/json/model/SqoopJobsResponse.java | 37 +++++ .../client/json/model/SqoopLinkResponse.java | 70 ++++++++++ .../client/json/model/SqoopLinksResponse.java | 37 +++++ 5 files changed, 358 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ranger/blob/ad273181/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/SqoopClient.java ---------------------------------------------------------------------- diff --git a/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/SqoopClient.java b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/SqoopClient.java index 640d5db..3563b31 100644 --- a/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/SqoopClient.java +++ b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/SqoopClient.java @@ -21,6 +21,7 @@ package org.apache.ranger.services.sqoop.client; import java.security.PrivilegedAction; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -38,6 +39,10 @@ import org.apache.ranger.plugin.client.BaseClient; import org.apache.ranger.plugin.client.HadoopException; import org.apache.ranger.services.sqoop.client.json.model.SqoopConnectorResponse; import org.apache.ranger.services.sqoop.client.json.model.SqoopConnectorsResponse; +import org.apache.ranger.services.sqoop.client.json.model.SqoopJobResponse; +import org.apache.ranger.services.sqoop.client.json.model.SqoopJobsResponse; +import org.apache.ranger.services.sqoop.client.json.model.SqoopLinkResponse; +import org.apache.ranger.services.sqoop.client.json.model.SqoopLinksResponse; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -53,6 +58,10 @@ public class SqoopClient extends BaseClient { private static final String SQOOP_CONNECTOR_API_ENDPOINT = "/sqoop/v1/connector/all"; + private static final String SQOOP_LINK_API_ENDPOINT = "/sqoop/v1/link/all"; + + private static final String SQOOP_JOB_API_ENDPOINT = "/sqoop/v1/job/all"; + private static final String ERROR_MESSAGE = " You can still save the repository and start creating " + "policies, but you would not be able to use autocomplete for " + "resource names. Check ranger_admin.log for more info."; @@ -80,12 +89,12 @@ public class SqoopClient extends BaseClient { public List<String> getConnectorList(final String connectorMatching, final List<String> existingConnectors) { if (LOG.isDebugEnabled()) { - LOG.debug("Getting sqoop connector list for connectorMatching: " + connectorMatching - + ", existingConnectors: " + existingConnectors); + LOG.debug("Get sqoop connector list for connectorMatching: " + connectorMatching + ", existingConnectors: " + + existingConnectors); } Subject subj = getLoginSubject(); if (subj == null) { - return null; + return Collections.emptyList(); } List<String> ret = Subject.doAs(subj, new PrivilegedAction<List<String>>() { @@ -95,33 +104,106 @@ public class SqoopClient extends BaseClient { ClientResponse response = getClientResponse(sqoopUrl, SQOOP_CONNECTOR_API_ENDPOINT, userName); - SqoopConnectorsResponse sqoopConnectorsResponse = getSqoopConnectorResponse(response); - if (sqoopConnectorsResponse == null) { - return null; + SqoopConnectorsResponse sqoopConnectorsResponse = getSqoopResourceResponse(response, + SqoopConnectorsResponse.class); + if (sqoopConnectorsResponse == null || CollectionUtils.isEmpty(sqoopConnectorsResponse.getConnectors())) { + return Collections.emptyList(); + } + List<String> connectorResponses = new ArrayList<>(); + for (SqoopConnectorResponse sqoopConnectorResponse : sqoopConnectorsResponse.getConnectors()) { + connectorResponses.add(sqoopConnectorResponse.getName()); } - List<SqoopConnectorResponse> connectorResponses = sqoopConnectorsResponse.getConnectors(); + List<String> connectors = null; if (CollectionUtils.isNotEmpty(connectorResponses)) { - connectors = getConnectorFromResponse(connectorMatching, existingConnectors, connectorResponses); + connectors = filterResourceFromResponse(connectorMatching, existingConnectors, connectorResponses); } return connectors; } }); if (LOG.isDebugEnabled()) { - LOG.debug("Getting sqoop project connector result: " + ret); + LOG.debug("Get sqoop connector list result: " + ret); } return ret; } public List<String> getLinkList(final String linkMatching, final List<String> existingLinks) { - // TODO - return null; + if (LOG.isDebugEnabled()) { + LOG.debug("Get sqoop link list for linkMatching: " + linkMatching + ", existingLinks: " + existingLinks); + } + Subject subj = getLoginSubject(); + if (subj == null) { + return Collections.emptyList(); + } + + List<String> ret = Subject.doAs(subj, new PrivilegedAction<List<String>>() { + + @Override + public List<String> run() { + + ClientResponse response = getClientResponse(sqoopUrl, SQOOP_LINK_API_ENDPOINT, userName); + + SqoopLinksResponse sqoopLinksResponse = getSqoopResourceResponse(response, SqoopLinksResponse.class); + if (sqoopLinksResponse == null || CollectionUtils.isEmpty(sqoopLinksResponse.getLinks())) { + return Collections.emptyList(); + } + List<String> linkResponses = new ArrayList<>(); + for (SqoopLinkResponse sqoopLinkResponse : sqoopLinksResponse.getLinks()) { + linkResponses.add(sqoopLinkResponse.getName()); + } + + List<String> links = null; + if (CollectionUtils.isNotEmpty(linkResponses)) { + links = filterResourceFromResponse(linkMatching, existingLinks, linkResponses); + } + return links; + } + }); + + if (LOG.isDebugEnabled()) { + LOG.debug("Get sqoop link list result: " + ret); + } + return ret; } public List<String> getJobList(final String jobMatching, final List<String> existingJobs) { - // TODO - return null; + if (LOG.isDebugEnabled()) { + LOG.debug("Get sqoop job list for jobMatching: " + jobMatching + ", existingJobs: " + existingJobs); + } + Subject subj = getLoginSubject(); + if (subj == null) { + return Collections.emptyList(); + } + + List<String> ret = Subject.doAs(subj, new PrivilegedAction<List<String>>() { + + @Override + public List<String> run() { + + ClientResponse response = getClientResponse(sqoopUrl, SQOOP_JOB_API_ENDPOINT, userName); + + SqoopJobsResponse sqoopJobsResponse = getSqoopResourceResponse(response, SqoopJobsResponse.class); + if (sqoopJobsResponse == null || CollectionUtils.isEmpty(sqoopJobsResponse.getJobs())) { + return Collections.emptyList(); + } + List<String> jobResponses = new ArrayList<>(); + for (SqoopJobResponse sqoopJobResponse : sqoopJobsResponse.getJobs()) { + jobResponses.add(sqoopJobResponse.getName()); + } + + List<String> jobs = null; + if (CollectionUtils.isNotEmpty(jobResponses)) { + jobs = filterResourceFromResponse(jobMatching, existingJobs, jobResponses); + } + return jobs; + } + }); + + if (LOG.isDebugEnabled()) { + LOG.debug("Get sqoop job list result: " + ret); + } + return ret; } private static ClientResponse getClientResponse(String sqoopUrl, String sqoopApi, String userName) { @@ -182,14 +264,14 @@ public class SqoopClient extends BaseClient { return response; } - private SqoopConnectorsResponse getSqoopConnectorResponse(ClientResponse response) { - SqoopConnectorsResponse sqoopConnectorsResponse = null; + private <T> T getSqoopResourceResponse(ClientResponse response, Class<T> classOfT) { + T resource = null; try { if (response != null && response.getStatus() == HttpStatus.SC_OK) { String jsonString = response.getEntity(String.class); Gson gson = new GsonBuilder().setPrettyPrinting().create(); - sqoopConnectorsResponse = gson.fromJson(jsonString, SqoopConnectorsResponse.class); + resource = gson.fromJson(jsonString, classOfT); } else { String msgDesc = "Unable to get a valid response for " + "expected mime type : [" + EXPECTED_MIME_TYPE + "], sqoopUrl: " + sqoopUrl + " - got null response."; @@ -201,7 +283,7 @@ public class SqoopClient extends BaseClient { } catch (HadoopException he) { throw he; } catch (Throwable t) { - String msgDesc = "Exception while getting sqoop project response, sqoopUrl: " + sqoopUrl; + String msgDesc = "Exception while getting sqoop resource response, sqoopUrl: " + sqoopUrl; HadoopException hdpException = new HadoopException(msgDesc, t); LOG.error(msgDesc, t); @@ -214,26 +296,25 @@ public class SqoopClient extends BaseClient { response.close(); } } - return sqoopConnectorsResponse; + return resource; } - private static List<String> getConnectorFromResponse(String connectorMatching, List<String> existingConnectors, - List<SqoopConnectorResponse> connectorResponses) { - List<String> connectors = new ArrayList<String>(); - for (SqoopConnectorResponse connectorResponse : connectorResponses) { - String connectorName = connectorResponse.getName(); - if (CollectionUtils.isNotEmpty(existingConnectors) && existingConnectors.contains(connectorName)) { + private static List<String> filterResourceFromResponse(String resourceMatching, List<String> existingResources, + List<String> resourceResponses) { + List<String> resources = new ArrayList<String>(); + for (String resourceResponse : resourceResponses) { + if (CollectionUtils.isNotEmpty(existingResources) && existingResources.contains(resourceResponse)) { continue; } - if (StringUtils.isEmpty(connectorMatching) || connectorMatching.startsWith("*") - || connectorName.toLowerCase().startsWith(connectorMatching.toLowerCase())) { + if (StringUtils.isEmpty(resourceMatching) || resourceMatching.startsWith("*") + || resourceResponse.toLowerCase().startsWith(resourceMatching.toLowerCase())) { if (LOG.isDebugEnabled()) { - LOG.debug("getConnectorFromResponse(): Adding sqoop connector " + connectorName); + LOG.debug("filterResourceFromResponse(): Adding sqoop resource " + resourceResponse); } - connectors.add(connectorName); + resources.add(resourceResponse); } } - return connectors; + return resources; } public static Map<String, Object> connectionTest(String serviceName, Map<String, String> configs) { http://git-wip-us.apache.org/repos/asf/ranger/blob/ad273181/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopJobResponse.java ---------------------------------------------------------------------- diff --git a/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopJobResponse.java b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopJobResponse.java new file mode 100644 index 0000000..b78ca16 --- /dev/null +++ b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopJobResponse.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.services.sqoop.client.json.model; + +import com.google.gson.annotations.SerializedName; + +public class SqoopJobResponse { + private Long id; + + private String name; + + @SerializedName("from-link-name") + private String fromLinkName; + + @SerializedName("to-link-name") + private String toLinkName; + + @SerializedName("from-connector-name") + private String fromConnectorName; + + @SerializedName("to-connector-name") + private String toConnectorName; + + @SerializedName("creation-user") + private String creationUser; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getFromLinkName() { + return fromLinkName; + } + + public void setFromLinkName(String fromLinkName) { + this.fromLinkName = fromLinkName; + } + + public String getToLinkName() { + return toLinkName; + } + + public void setToLinkName(String toLinkName) { + this.toLinkName = toLinkName; + } + + public String getFromConnectorName() { + return fromConnectorName; + } + + public void setFromConnectorName(String fromConnectorName) { + this.fromConnectorName = fromConnectorName; + } + + public String getToConnectorName() { + return toConnectorName; + } + + public void setToConnectorName(String toConnectorName) { + this.toConnectorName = toConnectorName; + } + + public String getCreationUser() { + return creationUser; + } + + public void setCreationUser(String creationUser) { + this.creationUser = creationUser; + } + + @Override + public String toString() { + return "SqoopJobResponse [id=" + id + ", name=" + name + ", fromLinkName=" + fromLinkName + ", toLinkName=" + + toLinkName + ", fromConnectorName=" + fromConnectorName + ", toConnectorName=" + toConnectorName + + ", creationUser=" + creationUser + "]"; + } +} http://git-wip-us.apache.org/repos/asf/ranger/blob/ad273181/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopJobsResponse.java ---------------------------------------------------------------------- diff --git a/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopJobsResponse.java b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopJobsResponse.java new file mode 100644 index 0000000..a2de043 --- /dev/null +++ b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopJobsResponse.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.services.sqoop.client.json.model; + +import java.util.List; + +public class SqoopJobsResponse { + private List<SqoopJobResponse> jobs; + + public List<SqoopJobResponse> getJobs() { + return jobs; + } + + public void setJobs(List<SqoopJobResponse> jobs) { + this.jobs = jobs; + } + + @Override + public String toString() { + return "SqoopJobsResponse [jobs=" + jobs + "]"; + } +} http://git-wip-us.apache.org/repos/asf/ranger/blob/ad273181/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopLinkResponse.java ---------------------------------------------------------------------- diff --git a/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopLinkResponse.java b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopLinkResponse.java new file mode 100644 index 0000000..0866a3e --- /dev/null +++ b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopLinkResponse.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.services.sqoop.client.json.model; + +import com.google.gson.annotations.SerializedName; + +public class SqoopLinkResponse { + private Long id; + + private String name; + + @SerializedName("connector-name") + private String connectorName; + + @SerializedName("creation-user") + private String creationUser; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getConnectorName() { + return connectorName; + } + + public void setConnectorName(String connectorName) { + this.connectorName = connectorName; + } + + public String getCreationUser() { + return creationUser; + } + + public void setCreationUser(String creationUser) { + this.creationUser = creationUser; + } + + @Override + public String toString() { + return "SqoopLinkResponse [id=" + id + ", name=" + name + ", connectorName=" + connectorName + + ", creationUser=" + creationUser + "]"; + } +} http://git-wip-us.apache.org/repos/asf/ranger/blob/ad273181/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopLinksResponse.java ---------------------------------------------------------------------- diff --git a/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopLinksResponse.java b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopLinksResponse.java new file mode 100644 index 0000000..9fda8c2 --- /dev/null +++ b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopLinksResponse.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.services.sqoop.client.json.model; + +import java.util.List; + +public class SqoopLinksResponse { + private List<SqoopLinkResponse> links; + + public List<SqoopLinkResponse> getLinks() { + return links; + } + + public void setLinks(List<SqoopLinkResponse> links) { + this.links = links; + } + + @Override + public String toString() { + return "SqoopLinksResponse [links=" + links + "]"; + } +}
