Updated Branches: refs/heads/master 2a2e2139b -> a0e893e15
OOZIE-1609 Amendment, adding missing new files Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/a0e893e1 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/a0e893e1 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/a0e893e1 Branch: refs/heads/master Commit: a0e893e152d5371b9dd9d0397106ab0c039fa0ca Parents: 2a2e213 Author: Robert Kanter <[email protected]> Authored: Wed Jan 29 15:12:17 2014 -0800 Committer: Robert Kanter <[email protected]> Committed: Wed Jan 29 15:12:17 2014 -0800 ---------------------------------------------------------------------- .../org/apache/oozie/util/AuthUrlClient.java | 134 ++++++++++++++ .../oozie/service/DummyV2AdminServlet.java | 39 ++++ .../oozie/service/TestHAShareLibService.java | 176 +++++++++++++++++++ 3 files changed, 349 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/a0e893e1/core/src/main/java/org/apache/oozie/util/AuthUrlClient.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/util/AuthUrlClient.java b/core/src/main/java/org/apache/oozie/util/AuthUrlClient.java new file mode 100644 index 0000000..79f8883 --- /dev/null +++ b/core/src/main/java/org/apache/oozie/util/AuthUrlClient.java @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.util; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.security.PrivilegedExceptionAction; + +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.client.AuthenticatedURL; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.authentication.client.Authenticator; +import org.apache.hadoop.security.authentication.client.KerberosAuthenticator; +import org.apache.hadoop.security.authentication.client.PseudoAuthenticator; +import org.apache.oozie.service.Services; + +public class AuthUrlClient { + + static private Class<? extends Authenticator> AuthenticatorClass = null; + + static private String errorMsg = null; + + static { + try { + AuthenticatorClass = determineAuthenticatorClassType(); + } + catch (Exception e) { + errorMsg = e.getMessage(); + } + } + + private static HttpURLConnection getConnection(URL url) throws IOException { + AuthenticatedURL.Token token = new AuthenticatedURL.Token(); + HttpURLConnection conn; + try { + conn = new AuthenticatedURL(AuthenticatorClass.newInstance()).openConnection(url, token); + } + catch (AuthenticationException ex) { + throw new IOException("Could not authenticate, " + ex.getMessage(), ex); + } + catch (InstantiationException ex) { + throw new IOException("Could not authenticate, " + ex.getMessage(), ex); + } + catch (IllegalAccessException ex) { + throw new IOException("Could not authenticate, " + ex.getMessage(), ex); + } + if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) { + throw new IOException("Unexpected response code [" + conn.getResponseCode() + "], message [" + + conn.getResponseMessage() + "]"); + } + return conn; + } + + @SuppressWarnings("unchecked") + private static Class<? extends Authenticator> determineAuthenticatorClassType() throws Exception { + // Adapted from + // org.apache.hadoop.security.authentication.server.AuthenticationFilter#init + Class<? extends Authenticator> authClass; + String authName = Services.get().getConf().get("oozie.authentication.type"); + String authClassName; + if (authName == null) { + throw new IOException("Authentication type must be specified: simple|kerberos|<class>"); + } + authName = authName.trim(); + if (authName.equals("simple")) { + authClassName = PseudoAuthenticator.class.getName(); + } + else if (authName.equals("kerberos")) { + authClassName = KerberosAuthenticator.class.getName(); + } + else { + authClassName = authName; + } + + authClass = (Class<? extends Authenticator>) Thread.currentThread().getContextClassLoader() + .loadClass(authClassName); + return authClass; + } + + /** + * Calls other Oozie server over HTTP. + * + * @param server The URL of the other Oozie server + * @return BufferedReader of inputstream. + * @throws IOException Signals that an I/O exception has occurred. + */ + public static BufferedReader callServer(String server) throws IOException { + + if (AuthenticatorClass == null) { + throw new IOException(errorMsg); + } + + final URL url = new URL(server); + BufferedReader reader = null; + try { + reader = UserGroupInformation.getLoginUser().doAs(new PrivilegedExceptionAction<BufferedReader>() { + @Override + public BufferedReader run() throws IOException { + HttpURLConnection conn = getConnection(url); + BufferedReader reader = null; + if ((conn.getResponseCode() == HttpURLConnection.HTTP_OK)) { + InputStream is = conn.getInputStream(); + reader = new BufferedReader(new InputStreamReader(is)); + } + return reader; + } + }); + } + catch (InterruptedException ie) { + throw new IOException(ie); + } + return reader; + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/a0e893e1/core/src/test/java/org/apache/oozie/service/DummyV2AdminServlet.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/DummyV2AdminServlet.java b/core/src/test/java/org/apache/oozie/service/DummyV2AdminServlet.java new file mode 100644 index 0000000..ec6bc22 --- /dev/null +++ b/core/src/test/java/org/apache/oozie/service/DummyV2AdminServlet.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.service; + +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.oozie.servlet.V2AdminServlet; + +public class DummyV2AdminServlet extends HttpServlet { + V2AdminServlet adminServlet = null; + public DummyV2AdminServlet() { + adminServlet = new V2AdminServlet(); + } + + public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + adminServlet.updateShareLib(request, response); + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/a0e893e1/core/src/test/java/org/apache/oozie/service/TestHAShareLibService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestHAShareLibService.java b/core/src/test/java/org/apache/oozie/service/TestHAShareLibService.java new file mode 100644 index 0000000..dbaf8cc --- /dev/null +++ b/core/src/test/java/org/apache/oozie/service/TestHAShareLibService.java @@ -0,0 +1,176 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.service; + +import java.io.InputStreamReader; +import java.io.Reader; +import java.net.HttpURLConnection; +import java.net.URI; +import java.util.Date; + +import org.apache.commons.httpclient.HttpClient; +import org.apache.commons.httpclient.methods.GetMethod; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.oozie.client.rest.JsonTags; +import org.apache.oozie.client.rest.RestConstants; +import org.apache.oozie.servlet.HostnameFilter; +import org.apache.oozie.servlet.V2AdminServlet; +import org.apache.oozie.test.EmbeddedServletContainer; +import org.apache.oozie.test.ZKXTestCase; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; + +public class TestHAShareLibService extends ZKXTestCase { + + EmbeddedServletContainer container; + + FileSystem fs; + + static { + new V2AdminServlet(); + + } + + protected void setUp() throws Exception { + super.setUp(); + container = new EmbeddedServletContainer("oozie"); + container.addServletEndpoint("/v2/admin/*", V2AdminServlet.class); + container.addServletEndpoint("/other-oozie-server/*", DummyV2AdminServlet.class); + container.addFilter("*", HostnameFilter.class); + container.start(); + Services.get().setService(ShareLibService.class); + Services.get().getConf().setBoolean(AuthorizationService.CONF_SECURITY_ENABLED, false); + + Services.get().setService(ZKJobsConcurrencyService.class); + + Path launcherlibPath = Services.get().get(WorkflowAppService.class).getSystemLibPath(); + HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); + URI uri = launcherlibPath.toUri(); + fs = FileSystem.get(has.createJobConf(uri.getAuthority())); + Date time = new Date(System.currentTimeMillis()); + + Path basePath = new Path(Services.get().getConf().get(WorkflowAppService.SYSTEM_LIB_PATH)); + Path libpath = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX + ShareLibService.dateFormat.format(time)); + fs.mkdirs(libpath); + + Path pigPath = new Path(libpath.toString() + Path.SEPARATOR + "pig"); + Path pigPath1 = new Path(libpath.toString() + Path.SEPARATOR + "pig_9"); + Path pigPath2 = new Path(libpath.toString() + Path.SEPARATOR + "pig_10"); + fs.mkdirs(pigPath); + fs.mkdirs(pigPath1); + fs.mkdirs(pigPath2); + fs.create(new Path(libpath.toString() + Path.SEPARATOR + "pig_10" + Path.SEPARATOR + "pig-10.jar")).close(); + + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + } + + public void testShareLibWithHA() throws Exception { + ZKJobsConcurrencyService zkjcs = new ZKJobsConcurrencyService(); + zkjcs.init(Services.get()); + DummyZKOozie dummyOozie_1 = null; + DummyZKOozie dummyOozie_2 = null; + try { + dummyOozie_1 = new DummyZKOozie("9876", container.getServletURL("/other-oozie-server/*")); + String url = container.getServletURL("/v2/admin/*") + "update_sharelib?" + RestConstants.ALL_SERVER_REQUEST + + "=true"; + HttpClient client = new HttpClient(); + GetMethod method = new GetMethod(url); + int statusCode = client.executeMethod(method); + assertEquals(HttpURLConnection.HTTP_OK, statusCode); + Reader reader = new InputStreamReader(method.getResponseBodyAsStream()); + JSONArray sharelib = (JSONArray) JSONValue.parse(reader); + assertEquals(2, sharelib.size()); + // 1st server update is successful + JSONObject obj = (JSONObject) sharelib.get(0); + assertEquals("Successful", + ((JSONObject) obj.get(JsonTags.SHARELIB_LIB_UPDATE)).get(JsonTags.SHARELIB_UPDATE_STATUS)); + + // 2nd server update is successful. + obj = (JSONObject) sharelib.get(1); + assertEquals("Successful", + ((JSONObject) obj.get(JsonTags.SHARELIB_LIB_UPDATE)).get(JsonTags.SHARELIB_UPDATE_STATUS)); + + // 3rd server not defined.should throw exception. + dummyOozie_2 = new DummyZKOozie("9873", container.getServletURL("/") + "not-defined/"); + + statusCode = client.executeMethod(method); + assertEquals(HttpURLConnection.HTTP_OK, statusCode); + reader = new InputStreamReader(method.getResponseBodyAsStream()); + sharelib = (JSONArray) JSONValue.parse(reader); + assertEquals(3, sharelib.size()); + + obj = (JSONObject) sharelib.get(0); + String status1 = ((JSONObject) obj.get(JsonTags.SHARELIB_LIB_UPDATE)).get(JsonTags.SHARELIB_UPDATE_STATUS) + .toString(); + + obj = (JSONObject) sharelib.get(1); + String status2 = ((JSONObject) obj.get(JsonTags.SHARELIB_LIB_UPDATE)).get(JsonTags.SHARELIB_UPDATE_STATUS) + .toString(); + + obj = (JSONObject) sharelib.get(2); + String status3 = ((JSONObject) obj.get(JsonTags.SHARELIB_LIB_UPDATE)).get(JsonTags.SHARELIB_UPDATE_STATUS) + .toString(); + + int success = 0; + int notSuccess = 0; + + if (status1.equals("Successful")) { + success++; + } + else { + notSuccess++; + } + + if (status2.equals("Successful")) { + success++; + } + else { + notSuccess++; + } + if (status3.equals("Successful")) { + success++; + } + else { + notSuccess++; + } + // 1 fails and other 2 succeed. + assertEquals(1, notSuccess); + assertEquals(2, success); + } + finally { + if (dummyOozie_1 != null) { + dummyOozie_1.teardown(); + } + + if (dummyOozie_2 != null) { + dummyOozie_2.teardown(); + } + zkjcs.destroy(); + container.stop(); + } + + } + +}
