Repository: sqoop Updated Branches: refs/heads/sqoop2 5fcdd3baf -> 615defb10
SQOOP-2583: Sqoop2: Encode object name in the http request (Dian Fu via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/615defb1 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/615defb1 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/615defb1 Branch: refs/heads/sqoop2 Commit: 615defb10fd0a3c861db2ebbc64fef3cbbba6cca Parents: 5fcdd3b Author: Jarek Jarcec Cecho <[email protected]> Authored: Mon Sep 21 19:16:16 2015 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Mon Sep 21 19:16:16 2015 -0700 ---------------------------------------------------------------------- .../request/AuthorizationResourceRequest.java | 17 ++-- .../client/request/JobResourceRequest.java | 17 ++-- .../client/request/LinkResourceRequest.java | 5 +- .../request/SubmissionResourceRequest.java | 3 +- .../org/apache/sqoop/utils/UrlSafeUtils.java | 17 ++++ .../org/apache/sqoop/server/RequestContext.java | 15 ++- .../server/InformalObjectNameTest.java | 97 ++++++++++++++++++++ 7 files changed, 149 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/615defb1/client/src/main/java/org/apache/sqoop/client/request/AuthorizationResourceRequest.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/sqoop/client/request/AuthorizationResourceRequest.java b/client/src/main/java/org/apache/sqoop/client/request/AuthorizationResourceRequest.java index 0bb9a7f..ca05d1b 100644 --- a/client/src/main/java/org/apache/sqoop/client/request/AuthorizationResourceRequest.java +++ b/client/src/main/java/org/apache/sqoop/client/request/AuthorizationResourceRequest.java @@ -23,6 +23,7 @@ import org.apache.sqoop.model.MPrincipal; import org.apache.sqoop.model.MPrivilege; import org.apache.sqoop.model.MResource; import org.apache.sqoop.model.MRole; +import org.apache.sqoop.utils.UrlSafeUtils; import org.json.simple.JSONObject; import java.util.List; @@ -67,7 +68,7 @@ public class AuthorizationResourceRequest extends ResourceRequest { } public void dropRole(String serverUrl, MRole role) { - super.delete(serverUrl + RESOURCE + ROLES + "/" + role.getName()); + super.delete(serverUrl + RESOURCE + ROLES + "/" + UrlSafeUtils.urlPathEncode(role.getName())); } public void grantRevokeRole(String serverUrl, List<MRole> roles, List<MPrincipal> principals, boolean isGrant) { @@ -86,8 +87,8 @@ public class AuthorizationResourceRequest extends ResourceRequest { public RolesBean readRolesByPrincipal(String serverUrl, MPrincipal principal) { String response = super.get(serverUrl + RESOURCE + ROLES - + "?principal_name=" + principal.getName() - + "&principal_type=" + principal.getType()); + + "?principal_name=" + UrlSafeUtils.urlEncode(principal.getName()) + + "&principal_type=" + UrlSafeUtils.urlEncode(principal.getType())); JSONObject jsonObject = JSONUtils.parse(response); RolesBean bean = new RolesBean(); bean.restore(jsonObject); @@ -96,7 +97,7 @@ public class AuthorizationResourceRequest extends ResourceRequest { public PrincipalsBean readPrincipalsByRole(String serverUrl, MRole role) { String response = super.get(serverUrl + RESOURCE + PRINCIPALS - + "?role_name=" + role.getName()); + + "?role_name=" + UrlSafeUtils.urlEncode(role.getName())); JSONObject jsonObject = JSONUtils.parse(response); PrincipalsBean bean = new PrincipalsBean(); bean.restore(jsonObject); @@ -123,11 +124,11 @@ public class AuthorizationResourceRequest extends ResourceRequest { public PrivilegesBean readPrivilegesByPrincipal(String serverUrl, MPrincipal principal, MResource resource) { String url = serverUrl + RESOURCE + PRIVILEGES - + "?principal_name=" + principal.getName() - + "&principal_type=" + principal.getType(); + + "?principal_name=" + UrlSafeUtils.urlEncode(principal.getName()) + + "&principal_type=" + UrlSafeUtils.urlEncode(principal.getType()); if (resource != null) { - url += "&resource_name=" + resource.getName(); - url += "&resource_type=" + resource.getType(); + url += "&resource_name=" + UrlSafeUtils.urlEncode(resource.getName()); + url += "&resource_type=" + UrlSafeUtils.urlEncode(resource.getType()); } String response = super.get(url); JSONObject jsonObject = JSONUtils.parse(response); http://git-wip-us.apache.org/repos/asf/sqoop/blob/615defb1/client/src/main/java/org/apache/sqoop/client/request/JobResourceRequest.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/sqoop/client/request/JobResourceRequest.java b/client/src/main/java/org/apache/sqoop/client/request/JobResourceRequest.java index 1094ae8..488318e 100644 --- a/client/src/main/java/org/apache/sqoop/client/request/JobResourceRequest.java +++ b/client/src/main/java/org/apache/sqoop/client/request/JobResourceRequest.java @@ -24,6 +24,7 @@ import org.apache.sqoop.json.JobsBean; import org.apache.sqoop.json.SubmissionBean; import org.apache.sqoop.json.ValidationResultBean; import org.apache.sqoop.model.MJob; +import org.apache.sqoop.utils.UrlSafeUtils; import org.json.simple.JSONObject; /** @@ -51,7 +52,7 @@ public class JobResourceRequest extends ResourceRequest { public JobBean readByConnector(String serverUrl, String cArg) { JobsBean bean = new JobsBean(); if (cArg != null) { - String response = super.get(serverUrl + RESOURCE + "?cname=" + cArg); + String response = super.get(serverUrl + RESOURCE + "?cname=" + UrlSafeUtils.urlEncode(cArg)); JSONObject jsonObject = JSONUtils.parse(response); bean.restore(jsonObject); } @@ -63,7 +64,7 @@ public class JobResourceRequest extends ResourceRequest { if (jobArg == null) { response = super.get(serverUrl + RESOURCE + "all"); } else { - response = super.get(serverUrl + RESOURCE + jobArg); + response = super.get(serverUrl + RESOURCE + UrlSafeUtils.urlPathEncode(jobArg)); } JSONObject jsonObject = JSONUtils.parse(response); // defaults to all @@ -97,29 +98,29 @@ public class JobResourceRequest extends ResourceRequest { } public void delete(String serverUrl, String jobArg) { - super.delete(serverUrl + RESOURCE + jobArg); + super.delete(serverUrl + RESOURCE + UrlSafeUtils.urlPathEncode(jobArg)); } public void enable(String serverUrl, String jobArg, Boolean enabled) { if (enabled) { - super.put(serverUrl + RESOURCE + jobArg + ENABLE, null); + super.put(serverUrl + RESOURCE + UrlSafeUtils.urlPathEncode(jobArg) + ENABLE, null); } else { - super.put(serverUrl + RESOURCE + jobArg + DISABLE, null); + super.put(serverUrl + RESOURCE + UrlSafeUtils.urlPathEncode(jobArg) + DISABLE, null); } } public SubmissionBean start(String serverUrl, String jobArg) { - String response = super.put(serverUrl + RESOURCE + jobArg + START, null); + String response = super.put(serverUrl + RESOURCE + UrlSafeUtils.urlPathEncode(jobArg) + START, null); return createJobSubmissionResponse(response); } public SubmissionBean stop(String serverUrl, String jobArg) { - String response = super.put(serverUrl + RESOURCE + jobArg + STOP, null); + String response = super.put(serverUrl + RESOURCE + UrlSafeUtils.urlPathEncode(jobArg) + STOP, null); return createJobSubmissionResponse(response); } public SubmissionBean status(String serverUrl, String jobArg) { - String response = super.get(serverUrl + RESOURCE + jobArg + STATUS); + String response = super.get(serverUrl + RESOURCE + UrlSafeUtils.urlPathEncode(jobArg) + STATUS); return createJobSubmissionResponse(response); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/615defb1/client/src/main/java/org/apache/sqoop/client/request/LinkResourceRequest.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/sqoop/client/request/LinkResourceRequest.java b/client/src/main/java/org/apache/sqoop/client/request/LinkResourceRequest.java index 8391621..bc6e0a9 100644 --- a/client/src/main/java/org/apache/sqoop/client/request/LinkResourceRequest.java +++ b/client/src/main/java/org/apache/sqoop/client/request/LinkResourceRequest.java @@ -23,6 +23,7 @@ import org.apache.sqoop.json.LinkBean; import org.apache.sqoop.json.LinksBean; import org.apache.sqoop.json.ValidationResultBean; import org.apache.sqoop.model.MLink; +import org.apache.sqoop.utils.UrlSafeUtils; import org.json.simple.JSONObject; /** @@ -49,7 +50,7 @@ public class LinkResourceRequest extends ResourceRequest { if (linkArg == null) { response = super.get(serverUrl + LINK_RESOURCE + "all"); } else { - response = super.get(serverUrl + LINK_RESOURCE + linkArg); + response = super.get(serverUrl + LINK_RESOURCE + UrlSafeUtils.urlPathEncode(linkArg)); } JSONObject jsonObject = JSONUtils.parse(response); // defaults to all @@ -82,7 +83,7 @@ public class LinkResourceRequest extends ResourceRequest { } public void delete(String serverUrl, String arg) { - super.delete(serverUrl + LINK_RESOURCE + arg); + super.delete(serverUrl + LINK_RESOURCE + UrlSafeUtils.urlPathEncode(arg)); } public void enable(String serverUrl, String lArg, Boolean enabled) { http://git-wip-us.apache.org/repos/asf/sqoop/blob/615defb1/client/src/main/java/org/apache/sqoop/client/request/SubmissionResourceRequest.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/sqoop/client/request/SubmissionResourceRequest.java b/client/src/main/java/org/apache/sqoop/client/request/SubmissionResourceRequest.java index f0435ce..0317b93 100644 --- a/client/src/main/java/org/apache/sqoop/client/request/SubmissionResourceRequest.java +++ b/client/src/main/java/org/apache/sqoop/client/request/SubmissionResourceRequest.java @@ -20,6 +20,7 @@ package org.apache.sqoop.client.request; import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL; import org.apache.sqoop.json.JSONUtils; import org.apache.sqoop.json.SubmissionsBean; +import org.apache.sqoop.utils.UrlSafeUtils; import org.json.simple.JSONObject; /** @@ -45,7 +46,7 @@ public class SubmissionResourceRequest extends ResourceRequest { response = super.get(serverUrl + RESOURCE); } else { // submission per job ( name preferred, we fall back to id) - response = super.get(serverUrl + RESOURCE + "?jname=" + jArg); + response = super.get(serverUrl + RESOURCE + "?jname=" + UrlSafeUtils.urlEncode(jArg)); } JSONObject jsonObject = JSONUtils.parse(response); SubmissionsBean submissionBean = new SubmissionsBean(); http://git-wip-us.apache.org/repos/asf/sqoop/blob/615defb1/common/src/main/java/org/apache/sqoop/utils/UrlSafeUtils.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/utils/UrlSafeUtils.java b/common/src/main/java/org/apache/sqoop/utils/UrlSafeUtils.java index 369037a..75defd7 100644 --- a/common/src/main/java/org/apache/sqoop/utils/UrlSafeUtils.java +++ b/common/src/main/java/org/apache/sqoop/utils/UrlSafeUtils.java @@ -49,6 +49,23 @@ public final class UrlSafeUtils { } } + public static String urlPathEncode(String path) { + try { + //return URLEncoder.encode(path, ENCODING_UTF8).replaceAll("\\+", "%20"); + return URLEncoder.encode(URLEncoder.encode(path, ENCODING_UTF8), ENCODING_UTF8); + } catch (UnsupportedEncodingException uee) { + throw new RuntimeException(uee); + } + } + + public static String urlPathDecode(String path) { + try { + //return URLDecoder.decode(path.replaceAll("%20", "\\+"), ENCODING_UTF8); + return URLDecoder.decode(URLDecoder.decode(path, ENCODING_UTF8), ENCODING_UTF8); + } catch (UnsupportedEncodingException uee) { + throw new RuntimeException(uee); + } + } private UrlSafeUtils() { // Disable explicit object creation http://git-wip-us.apache.org/repos/asf/sqoop/blob/615defb1/server/src/main/java/org/apache/sqoop/server/RequestContext.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/sqoop/server/RequestContext.java b/server/src/main/java/org/apache/sqoop/server/RequestContext.java index f8401aa..2beac2b 100644 --- a/server/src/main/java/org/apache/sqoop/server/RequestContext.java +++ b/server/src/main/java/org/apache/sqoop/server/RequestContext.java @@ -24,6 +24,7 @@ import org.apache.sqoop.core.SqoopConfiguration; import org.apache.sqoop.security.AuthenticationManager; import org.apache.sqoop.security.SecurityConstants; import org.apache.sqoop.server.common.ServerError; +import org.apache.sqoop.utils.UrlSafeUtils; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -86,14 +87,18 @@ public class RequestContext { public String getLastURLElement() { String uri = getRequest().getRequestURI(); int slash = uri.lastIndexOf('/'); - return uri.substring(slash + 1); + return UrlSafeUtils.urlPathDecode(uri.substring(slash + 1)); } /** * Return all elements in the url as an array */ public String[] getUrlElements() { - return getRequest().getRequestURI().split("/"); + String[] elements = getRequest().getRequestURI().split("/"); + for(int i = 0; i < elements.length; i++) { + elements[i] = UrlSafeUtils.urlPathDecode(elements[i]); + } + return elements; } /** @@ -101,7 +106,11 @@ public class RequestContext { */ public String getParameterValue(String name) { String[] values = getRequest().getParameterValues(name); - return values != null ? values[0] : null; + String value = values != null ? values[0] : null; + if (value != null) { + value = UrlSafeUtils.urlDecode(value); + } + return value; } /** http://git-wip-us.apache.org/repos/asf/sqoop/blob/615defb1/test/src/test/java/org/apache/sqoop/integration/server/InformalObjectNameTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/server/InformalObjectNameTest.java b/test/src/test/java/org/apache/sqoop/integration/server/InformalObjectNameTest.java new file mode 100644 index 0000000..811315a --- /dev/null +++ b/test/src/test/java/org/apache/sqoop/integration/server/InformalObjectNameTest.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.integration.server; + +import static org.testng.Assert.assertEquals; + +import org.apache.sqoop.connector.hdfs.configuration.ToFormat; +import org.apache.sqoop.model.MJob; +import org.apache.sqoop.model.MLink; +import org.apache.sqoop.test.infrastructure.Infrastructure; +import org.apache.sqoop.test.infrastructure.SqoopTestCase; +import org.apache.sqoop.test.infrastructure.providers.DatabaseInfrastructureProvider; +import org.apache.sqoop.test.infrastructure.providers.HadoopInfrastructureProvider; +import org.apache.sqoop.test.infrastructure.providers.SqoopInfrastructureProvider; +import org.testng.annotations.Test; + +@Infrastructure(dependencies = {HadoopInfrastructureProvider.class, SqoopInfrastructureProvider.class, DatabaseInfrastructureProvider.class}) +public class InformalObjectNameTest extends SqoopTestCase { + private static final String LINK_NAME_CONTAINS_WHITESPACE = "link name"; + private static final String LINK_NAME_CONTAINS_SLASH = "link/name"; + + private static final String JOB_NAME_CONTAINS_WHITESPACE = "job name"; + private static final String JOB_NAME_CONTAINS_SLASH= "job/name"; + + public InformalObjectNameTest() { + } + + @Test + public void testInformalLinkName() throws Exception { + // RDBMS link + MLink rdbmsLink = getClient().createLink("generic-jdbc-connector"); + fillRdbmsLinkConfig(rdbmsLink); + rdbmsLink.setName(LINK_NAME_CONTAINS_WHITESPACE); + saveLink(rdbmsLink); + assertEquals(rdbmsLink, getClient().getLink(LINK_NAME_CONTAINS_WHITESPACE)); + + rdbmsLink = getClient().createLink("generic-jdbc-connector"); + fillRdbmsLinkConfig(rdbmsLink); + rdbmsLink.setName(LINK_NAME_CONTAINS_SLASH); + saveLink(rdbmsLink); + assertEquals(rdbmsLink, getClient().getLink(LINK_NAME_CONTAINS_SLASH)); + } + + @Test + public void testInformalJobName() throws Exception { + // RDBMS link + MLink rdbmsLink = getClient().createLink("generic-jdbc-connector"); + fillRdbmsLinkConfig(rdbmsLink); + saveLink(rdbmsLink); + + // HDFS link + MLink hdfsLink = getClient().createLink("hdfs-connector"); + fillHdfsLinkConfig(hdfsLink); + saveLink(hdfsLink); + + // Job creation + MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); + + // rdms "FROM" config + fillRdbmsFromConfig(job, "id"); + + // hdfs "TO" config + fillHdfsToConfig(job, ToFormat.TEXT_FILE); + + job.setName(JOB_NAME_CONTAINS_WHITESPACE); + saveJob(job); + assertEquals(job, getClient().getJob(JOB_NAME_CONTAINS_WHITESPACE)); + + job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); + + // rdms "FROM" config + fillRdbmsFromConfig(job, "id"); + + // hdfs "TO" config + fillHdfsToConfig(job, ToFormat.TEXT_FILE); + + job.setName(JOB_NAME_CONTAINS_SLASH); + saveJob(job); + + assertEquals(job, getClient().getJob(JOB_NAME_CONTAINS_SLASH)); + } +}
