Author: ryota
Date: Fri Nov 22 00:50:48 2013
New Revision: 1544387
URL: http://svn.apache.org/r1544387
Log:
OOZIE-1519 Admin command to update the sharelib (puru via ryota)
Modified:
oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
oozie/trunk/client/src/main/java/org/apache/oozie/client/OozieClient.java
oozie/trunk/client/src/main/java/org/apache/oozie/client/rest/JsonTags.java
oozie/trunk/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java
oozie/trunk/core/src/main/java/org/apache/oozie/service/ShareLibService.java
oozie/trunk/core/src/main/java/org/apache/oozie/servlet/BaseAdminServlet.java
oozie/trunk/core/src/main/java/org/apache/oozie/servlet/V1AdminServlet.java
oozie/trunk/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java
oozie/trunk/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
oozie/trunk/core/src/test/java/org/apache/oozie/servlet/TestAdminServlet.java
oozie/trunk/docs/src/site/twiki/DG_CommandLineTool.twiki
oozie/trunk/docs/src/site/twiki/WebServicesAPI.twiki
oozie/trunk/release-log.txt
Modified: oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java?rev=1544387&r1=1544386&r2=1544387&view=diff
==============================================================================
--- oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
(original)
+++ oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java Fri Nov
22 00:50:48 2013
@@ -129,6 +129,12 @@ public class OozieCLI {
public static final String RERUN_REFRESH_OPTION = "refresh";
public static final String RERUN_NOCLEANUP_OPTION = "nocleanup";
+ public static final String UPDATE_SHARELIB_OPTION = "sharelibupdate";
+
+ public static final String LIST_SHARELIB_LIB_OPTION = "shareliblist";
+
+
+
public static final String AUTH_OPTION = "auth";
public static final String VERBOSE_OPTION = "verbose";
@@ -226,6 +232,12 @@ public class OozieCLI {
Option doAs = new Option(DO_AS_OPTION, true, "doAs user, impersonates
as the specified user");
Option availServers = new Option(AVAILABLE_SERVERS_OPTION, false,
"list available Oozie servers"
+ " (more than one only if HA is enabled)");
+ Option sharelibUpdate = new Option(UPDATE_SHARELIB_OPTION, false,
"Update server to use a newer version of sharelib");
+
+ Option sharelib = new Option(LIST_SHARELIB_LIB_OPTION, false,
+ "List available sharelib that can be specified in a workflow
action");
+ sharelib.setOptionalArg(true);
+
Options adminOptions = new Options();
adminOptions.addOption(oozie);
adminOptions.addOption(doAs);
@@ -235,6 +247,8 @@ public class OozieCLI {
group.addOption(version);
group.addOption(queuedump);
group.addOption(availServers);
+ group.addOption(sharelibUpdate);
+ group.addOption(sharelib);
adminOptions.addOptionGroup(group);
addAuthOptions(adminOptions);
return adminOptions;
@@ -1443,6 +1457,19 @@ public class OozieCLI {
status = wc.getSystemMode();
System.out.println("System mode: " + status);
}
+
+ else if (options.contains(UPDATE_SHARELIB_OPTION)) {
+ System.out.println(wc.updateShareLib());
+ }
+
+ else if (options.contains(LIST_SHARELIB_LIB_OPTION)) {
+ String sharelibKey = null;
+ if (commandLine.getArgList().size() > 0) {
+ sharelibKey = (String) commandLine.getArgList().get(0);
+ }
+ System.out.println(wc.listShareLib(sharelibKey));
+ }
+
else if (options.contains(QUEUE_DUMP_OPTION)) {
List<String> list = wc.getQueueDump();
Modified:
oozie/trunk/client/src/main/java/org/apache/oozie/client/OozieClient.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/client/src/main/java/org/apache/oozie/client/OozieClient.java?rev=1544387&r1=1544386&r2=1544387&view=diff
==============================================================================
--- oozie/trunk/client/src/main/java/org/apache/oozie/client/OozieClient.java
(original)
+++ oozie/trunk/client/src/main/java/org/apache/oozie/client/OozieClient.java
Fri Nov 22 00:50:48 2013
@@ -1489,6 +1489,14 @@ public class OozieClient {
return new GetSystemMode().call();
}
+ public String updateShareLib() throws OozieClientException {
+ return new UpdateSharelib().call();
+ }
+
+ public String listShareLib(String sharelibKey) throws OozieClientException
{
+ return new ListShareLib(sharelibKey).call();
+ }
+
private class GetBuildVersion extends ClientCallable<String> {
GetBuildVersion() {
@@ -1509,6 +1517,73 @@ public class OozieClient {
}
}
+
+ private class UpdateSharelib extends ClientCallable<String> {
+
+ UpdateSharelib() {
+ super("GET", RestConstants.ADMIN,
RestConstants.ADMIN_UPDATE_SHARELIB, prepareParams());
+ }
+
+ @Override
+ protected String call(HttpURLConnection conn) throws IOException,
OozieClientException {
+ StringBuffer bf = new StringBuffer();
+ if ((conn.getResponseCode() == HttpURLConnection.HTTP_OK)) {
+ Reader reader = new InputStreamReader(conn.getInputStream());
+ JSONObject json = (JSONObject) ((JSONObject)
JSONValue.parse(reader))
+ .get(JsonTags.SHARELIB_LIB_UPDATE);
+ bf.append("[ShareLib update
status]").append(System.getProperty("line.separator"));
+ for (Object key : json.keySet()) {
+ bf.append(" ").append(key).append(" =
").append(json.get(key))
+ .append(System.getProperty("line.separator"));
+ }
+
+ return bf.toString();
+ }
+ else {
+ handleError(conn);
+ }
+ return null;
+ }
+ }
+
+ private class ListShareLib extends ClientCallable<String> {
+
+ ListShareLib(String sharelibKey) {
+ super("GET", RestConstants.ADMIN,
RestConstants.ADMIN_LIST_SHARELIB, prepareParams(
+ RestConstants.SHARE_LIB_REQUEST_KEY, sharelibKey));
+ }
+
+ @Override
+ protected String call(HttpURLConnection conn) throws IOException,
OozieClientException {
+
+ if ((conn.getResponseCode() == HttpURLConnection.HTTP_OK)) {
+ StringBuffer bf = new StringBuffer();
+ Reader reader = new InputStreamReader(conn.getInputStream());
+ JSONObject json = (JSONObject) JSONValue.parse(reader);
+ Object sharelib = json.get(JsonTags.SHARELIB_LIB);
+ bf.append("[Available
ShareLib]").append(System.getProperty("line.separator"));
+ if (sharelib instanceof JSONArray) {
+ for (Object o : ((JSONArray) sharelib)) {
+ JSONObject obj = (JSONObject) o;
+ bf.append(obj.get(JsonTags.SHARELIB_LIB_NAME))
+ .append(System.getProperty("line.separator"));
+ if (obj.get(JsonTags.SHARELIB_LIB_FILES) != null) {
+ for (Object file : ((JSONArray)
obj.get(JsonTags.SHARELIB_LIB_FILES))) {
+
bf.append("\t").append(file).append(System.getProperty("line.separator"));
+ }
+ }
+ }
+ return bf.toString();
+ }
+ }
+ else {
+ handleError(conn);
+ }
+ return null;
+ }
+
+ }
+
/**
* Return the Oozie server build version.
*
Modified:
oozie/trunk/client/src/main/java/org/apache/oozie/client/rest/JsonTags.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/client/src/main/java/org/apache/oozie/client/rest/JsonTags.java?rev=1544387&r1=1544386&r2=1544387&view=diff
==============================================================================
--- oozie/trunk/client/src/main/java/org/apache/oozie/client/rest/JsonTags.java
(original)
+++ oozie/trunk/client/src/main/java/org/apache/oozie/client/rest/JsonTags.java
Fri Nov 22 00:50:48 2013
@@ -30,6 +30,13 @@ public interface JsonTags {
public static final String UNIQUE_MAP_DUMP = "uniqueMapDump";
public static final String UNIQUE_ENTRY_DUMP = "uniqueEntryDump";
+ public static final String SHARELIB_LIB_UPDATE = "sharelibUpdate";
+ public static final String SHARELIB_LIB = "sharelib";
+ public static final String SHARELIB_LIB_NAME = "name";
+ public static final String SHARELIB_LIB_FILES = "files";
+ public static final String SHARELIB_UPDATE_HOST = "host";
+ public static final String SHARELIB_UPDATE_STATUS = "status";
+
public static final String JOB_ID = "id";
public static final String WORKFLOW_APP_PATH = "appPath";
Modified:
oozie/trunk/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java?rev=1544387&r1=1544386&r2=1544387&view=diff
==============================================================================
---
oozie/trunk/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java
(original)
+++
oozie/trunk/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java
Fri Nov 22 00:50:48 2013
@@ -159,4 +159,13 @@ public interface RestConstants {
public static final String JOB_SHOW_JMS_TOPIC = "jmstopic";
public static final String ADMIN_AVAILABLE_OOZIE_SERVERS_RESOURCE =
"available-oozie-servers";
+
+ public static final String ADMIN_UPDATE_SHARELIB = "update_sharelib";
+
+ public static final String ADMIN_LIST_SHARELIB = "list_sharelib";
+
+ public static final String SHARE_LIB_REQUEST_KEY = "lib";
+
+ public static final String SHARE_LIB_ALLSERVER_REQUEST = "allservers";
+
}
Modified:
oozie/trunk/core/src/main/java/org/apache/oozie/service/ShareLibService.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/service/ShareLibService.java?rev=1544387&r1=1544386&r2=1544387&view=diff
==============================================================================
---
oozie/trunk/core/src/main/java/org/apache/oozie/service/ShareLibService.java
(original)
+++
oozie/trunk/core/src/main/java/org/apache/oozie/service/ShareLibService.java
Fri Nov 22 00:50:48 2013
@@ -32,12 +32,11 @@ import java.util.Date;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import java.util.Set;
import java.util.TimeZone;
-
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
@@ -47,7 +46,8 @@ import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.oozie.action.ActionExecutor;
import org.apache.oozie.action.hadoop.JavaActionExecutor;
-import org.apache.oozie.util.XConfiguration;
+import org.apache.oozie.client.rest.JsonUtils;
+
import org.apache.oozie.util.XLog;
import com.google.common.annotations.VisibleForTesting;
@@ -84,6 +84,10 @@ public class ShareLibService implements
private boolean shareLibLoadAttempted = false;
+ private String sharelibMetaFileOldTimeStamp;
+
+ private String sharelibDirOld;
+
FileSystem fs;
@Override
@@ -116,6 +120,12 @@ public class ShareLibService implements
private void updateLauncherLib() throws IOException {
if (isShipLauncherEnabled) {
+ if (fs == null) {
+ Path launcherlibPath = getLauncherlibPath();
+ HadoopAccessorService has =
Services.get().get(HadoopAccessorService.class);
+ URI uri = launcherlibPath.toUri();
+ fs = FileSystem.get(has.createJobConf(uri.getAuthority()));
+ }
Path launcherlibPath = getLauncherlibPath();
setupLauncherLibPath(fs, launcherlibPath);
recursiveChangePermissions(fs, launcherlibPath,
FsPermission.valueOf(PERMISSION_STRING));
@@ -238,6 +248,10 @@ public class ShareLibService implements
}
}
+ public Map<String, List<Path>> getShareLib() {
+ return shareLibMap;
+ }
+
/**
* Gets the action system lib common jars.
*
@@ -389,19 +403,44 @@ public class ShareLibService implements
/**
* Update share lib cache.
*
+ * @return the map
* @throws IOException Signals that an I/O exception has occurred.
*/
- public void updateShareLib() throws IOException {
+ public Map<String, String> updateShareLib() throws IOException {
+ Map<String, String> status = new HashMap<String, String>();
+
+ if (fs == null) {
+ Path launcherlibPath = getLauncherlibPath();
+ HadoopAccessorService has =
Services.get().get(HadoopAccessorService.class);
+ URI uri = launcherlibPath.toUri();
+ fs = FileSystem.get(has.createJobConf(uri.getAuthority()));
+ }
+
Map<String, List<Path>> tempShareLibMap = new HashMap<String,
List<Path>>();
if (!StringUtils.isEmpty(sharelibMappingFile)) {
+ String sharelibMetaFileNewTimeStamp =
JsonUtils.formatDateRfc822(new Date(fs.getFileStatus(
+ new
Path(sharelibMappingFile)).getModificationTime()),"GMT");
loadShareLibMetaFile(tempShareLibMap, sharelibMappingFile);
+ status.put("sharelibMetaFile", sharelibMappingFile);
+ status.put("sharelibMetaFileNewTimeStamp",
sharelibMetaFileNewTimeStamp);
+ status.put("sharelibMetaFileOldTimeStamp",
sharelibMetaFileOldTimeStamp);
+ sharelibMetaFileOldTimeStamp = sharelibMetaFileNewTimeStamp;
}
else {
- loadShareLibfromDFS(tempShareLibMap);
+ Path shareLibpath =
getLatestLibPath(services.get(WorkflowAppService.class).getSystemLibPath(),
+ SHARED_LIB_PREFIX);
+ loadShareLibfromDFS(tempShareLibMap, shareLibpath);
+
+ if (shareLibpath != null) {
+ status.put("sharelibDirNew", shareLibpath.toString());
+ status.put("sharelibDirOld", sharelibDirOld);
+ sharelibDirOld = shareLibpath.toString();
+ }
+
}
shareLibMap = tempShareLibMap;
-
+ return status;
}
/**
@@ -409,11 +448,10 @@ public class ShareLibService implements
* directory is a action key
*
* @param shareLibMap the share lib jar map
+ * @param shareLibpath the share libpath
* @throws IOException Signals that an I/O exception has occurred.
*/
- private void loadShareLibfromDFS(Map<String, List<Path>> shareLibMap)
throws IOException {
- Path shareLibpath =
getLatestLibPath(services.get(WorkflowAppService.class).getSystemLibPath(),
- SHARED_LIB_PREFIX);
+ private void loadShareLibfromDFS(Map<String, List<Path>> shareLibMap, Path
shareLibpath) throws IOException {
if (shareLibpath == null) {
LOG.info("No share lib directory found");
@@ -428,10 +466,13 @@ public class ShareLibService implements
}
for (FileStatus dir : dirList) {
+ if (!dir.isDir()) {
+ continue;
+ }
List<Path> listOfPaths = new ArrayList<Path>();
getPathRecursively(fs, dir.getPath(), listOfPaths);
shareLibMap.put(dir.getPath().getName(), listOfPaths);
- LOG.info("Share lib for " + dir.getPath().getName() + ":" +
listOfPaths);
+ LOG.info("Share lib for " + dir.getPath().getName() + ":" +
listOfPaths);
}
@@ -450,30 +491,27 @@ public class ShareLibService implements
private void loadShareLibMetaFile(Map<String, List<Path>> shareLibMap,
String sharelibFileMapping)
throws IOException {
-
- Path shareFileMappingPath= new Path(sharelibFileMapping);
+ Path shareFileMappingPath = new Path(sharelibFileMapping);
HadoopAccessorService has =
Services.get().get(HadoopAccessorService.class);
FileSystem filesystem =
FileSystem.get(has.createJobConf(shareFileMappingPath.toUri().getAuthority()));
- XConfiguration conf = new
XConfiguration(filesystem.open(shareFileMappingPath));
- Iterator<Map.Entry<String, String>> it = conf.iterator();
+ Properties prop = new Properties();
+ prop.load(filesystem.open(new Path(sharelibFileMapping)));
-
- while (it.hasNext()) {
- Map.Entry<String, String> en = it.next();
- if (en.getKey().toLowerCase().startsWith(SHARE_LIB_CONF_PREFIX)) {
- String key =
en.getKey().substring(SHARE_LIB_CONF_PREFIX.length() + 1);
- String pathList[] = en.getValue().split(",");
+ for (Object keyObject : prop.keySet()) {
+ String key = (String) keyObject;
+ if (key.toLowerCase().startsWith(SHARE_LIB_CONF_PREFIX)) {
+ String mapKey = key.substring(SHARE_LIB_CONF_PREFIX.length() +
1);
+ String pathList[] = ((String) prop.get(key)).split(",");
List<Path> listOfPaths = new ArrayList<Path>();
for (String dfsPath : pathList) {
getPathRecursively(fs, new Path(dfsPath), listOfPaths);
}
- shareLibMap.put(key, listOfPaths);
- LOG.info("Share lib for " + en.getKey() + ":" + listOfPaths);
-
+ shareLibMap.put(mapKey, listOfPaths);
+ LOG.info("Share lib for " + mapKey + ":" + listOfPaths);
}
else {
- LOG.info(" Not adding " + en.getKey() + " to sharelib, not
prefix with " + SHARE_LIB_CONF_PREFIX);
+ LOG.info(" Not adding " + key + " to sharelib, not prefix with
" + SHARE_LIB_CONF_PREFIX);
}
}
Modified:
oozie/trunk/core/src/main/java/org/apache/oozie/servlet/BaseAdminServlet.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/servlet/BaseAdminServlet.java?rev=1544387&r1=1544386&r2=1544387&view=diff
==============================================================================
---
oozie/trunk/core/src/main/java/org/apache/oozie/servlet/BaseAdminServlet.java
(original)
+++
oozie/trunk/core/src/main/java/org/apache/oozie/servlet/BaseAdminServlet.java
Fri Nov 22 00:50:48 2013
@@ -18,13 +18,14 @@
package org.apache.oozie.servlet;
import java.io.IOException;
+import java.util.List;
import java.util.Map;
import java.util.TimeZone;
-
+import java.util.regex.Pattern;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-
+import org.apache.hadoop.fs.Path;
import org.apache.oozie.BuildInfo;
import org.apache.oozie.client.rest.JsonBean;
import org.apache.oozie.client.rest.JsonTags;
@@ -33,6 +34,7 @@ import org.apache.oozie.service.Authoriz
import org.apache.oozie.service.AuthorizationService;
import org.apache.oozie.service.InstrumentationService;
import org.apache.oozie.service.Services;
+import org.apache.oozie.service.ShareLibService;
import org.apache.oozie.util.Instrumentation;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
@@ -42,6 +44,7 @@ public abstract class BaseAdminServlet e
private static final long serialVersionUID = 1L;
protected String modeTag;
+
public BaseAdminServlet(String instrumentationName, ResourceInfo[]
RESOURCES_INFO) {
super(instrumentationName, RESOURCES_INFO);
setAllowSafeModeChanges(true);
@@ -56,13 +59,7 @@ public abstract class BaseAdminServlet e
request.setAttribute(AUDIT_OPERATION, resourceName);
request.setAttribute(AUDIT_PARAM, request.getParameter(modeTag));
- try {
- AuthorizationService auth =
Services.get().get(AuthorizationService.class);
- auth.authorizeForAdmin(getUser(request), true);
- }
- catch (AuthorizationException ex) {
- throw new XServletException(HttpServletResponse.SC_UNAUTHORIZED,
ex);
- }
+ authorizeRequest(request);
setOozieMode(request, response, resourceName);
/*if (resourceName.equals(RestConstants.ADMIN_STATUS_RESOURCE)) {
@@ -147,8 +144,161 @@ public abstract class BaseAdminServlet e
json.putAll(getOozieURLs());
sendJsonResponse(response, HttpServletResponse.SC_OK, json);
}
+ else if (resource.equals(RestConstants.ADMIN_UPDATE_SHARELIB)) {
+ authorizeRequest(request);
+ updateShareLib(request, response);
+ }
+ else if (resource.equals(RestConstants.ADMIN_LIST_SHARELIB)) {
+ String sharelibKey =
request.getParameter(RestConstants.SHARE_LIB_REQUEST_KEY);
+ sendJsonResponse(response, HttpServletResponse.SC_OK,
getShareLib(sharelibKey));
+ }
+ }
+
+ /**
+ * Gets the list of share lib.
+ *
+ * @param sharelibKey the sharelib key
+ * @return the list of supported share lib
+ * @throws IOException
+ */
+ @SuppressWarnings("unchecked")
+ private JSONObject getShareLib(String sharelibKey) throws IOException {
+ JSONObject json = new JSONObject();
+
+ ShareLibService shareLibService =
Services.get().get(ShareLibService.class);
+
+ // for testcases.
+ if (shareLibService == null) {
+ return json;
+ }
+ JSONArray shareLibList = new JSONArray();
+
+ Map<String, List<Path>> shareLibLauncherMap =
shareLibService.getShareLib();
+ if (sharelibKey != null && !sharelibKey.isEmpty()) {
+ Pattern pattern = Pattern.compile(sharelibKey);
+ for (String key : shareLibLauncherMap.keySet()) {
+ if (pattern.matcher(key).matches() == true) {
+ JSONObject object = new JSONObject();
+ JSONArray fileList = new JSONArray();
+ List<Path> pathList = shareLibLauncherMap.get(key);
+
+ for (Path file : pathList) {
+ fileList.add(file.toString());
+ }
+ object.put(JsonTags.SHARELIB_LIB_NAME, key);
+ object.put(JsonTags.SHARELIB_LIB_FILES, fileList);
+ shareLibList.add(object);
+
+ }
+ }
+ }
+ else {
+ for (String key : shareLibLauncherMap.keySet()) {
+ JSONObject object = new JSONObject();
+ object.put(JsonTags.SHARELIB_LIB_NAME, key);
+ shareLibList.add(object);
+ }
+
+ }
+ json.put(JsonTags.SHARELIB_LIB, shareLibList);
+
+ return json;
}
+ /**
+ * Update share lib.
+ *
+ * @param request the request
+ * @param response the response
+ * @throws IOException Signals that an I/O exception has occurred.
+ */
+ @SuppressWarnings("unchecked")
+ public void updateShareLib(HttpServletRequest request, HttpServletResponse
response) throws IOException {
+
+ //TODO sharelib HA.
+// if
(Boolean.parseBoolean(request.getParameter(RestConstants.SHARE_LIB_ALLSERVER_REQUEST)))
{
+// JSONArray jsonArray = new JSONArray();
+// JSONObject json = new JSONObject();
+// try {
+// JobsConcurrencyService jc =
Services.get().get(JobsConcurrencyService.class);
+// Map<String, String> serverList = jc.getServerUrls();
+// for (String server : serverList.values()) {
+// String serverUrl = server + "/v2/admin/" +
RestConstants.ADMIN_UPDATE_SHARELIB + "?"
+// + RestConstants.SHARE_LIB_ALLSERVER_REQUEST +
"=false";
+// try {
+// jsonArray.add(callServer(serverUrl, request));
+// }
+// catch (Exception e) {
+// JSONObject errorJson = new JSONObject();
+// errorJson.put(JsonTags.SHARELIB_UPDATE_HOST, server);
+// errorJson.put(JsonTags.SHARELIB_UPDATE_STATUS,
e.getMessage());
+// jsonArray.add(errorJson);
+//
+// }
+// }
+// json.put(JsonTags.SHARELIB_LIB_UPDATE, jsonArray);
+// sendJsonResponse(response, HttpServletResponse.SC_OK, json);
+// }
+//
+// catch (Exception e) {
+// sendErrorResponse(response,
HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "internal Error",
+// e.getMessage());
+// }
+//
+// }
+// else {
+ ShareLibService shareLibService =
Services.get().get(ShareLibService.class);
+ JSONObject status = new JSONObject();
+
+ JSONObject json = new JSONObject();
+ json.put(JsonTags.SHARELIB_UPDATE_HOST, request.getServerName() + ":"
+ request.getServerPort());
+ try {
+ json.putAll(shareLibService.updateShareLib());
+ json.put(JsonTags.SHARELIB_UPDATE_STATUS, "Successful");
+ }
+ catch (Exception e) {
+ json.put(JsonTags.SHARELIB_UPDATE_STATUS, e.getClass().getName() +
": " + e.getMessage());
+ }
+ status.put(JsonTags.SHARELIB_LIB_UPDATE, json);
+ sendJsonResponse(response, HttpServletResponse.SC_OK, status);
+
+ // }
+
+ }
+
+// private JSONObject callServer(String url, HttpServletRequest request)
throws MalformedURLException, IOException {
+// HttpURLConnection conn = (HttpURLConnection) new
URL(url).openConnection();
+// conn.setRequestMethod("GET");
+//
+// Enumeration headerNames = request.getHeaderNames();
+// while (headerNames.hasMoreElements()) {
+// String headerName = (String) headerNames.nextElement();
+//
+// conn.setRequestProperty(headerName,
request.getHeader(headerName));
+// }
+// conn.connect();
+//
+// Reader reader = new InputStreamReader(conn.getInputStream());
+// JSONObject json = (JSONObject) JSONValue.parse(reader);
+// return (JSONObject) json.get(JsonTags.SHARELIB_LIB_UPDATE);
+//
+// }
+
+ /**
+ * Authorize request.
+ *
+ * @param request the HttpServletRequest
+ * @throws XServletException the x servlet exception
+ */
+ private void authorizeRequest(HttpServletRequest request) throws
XServletException {
+ try {
+ AuthorizationService auth =
Services.get().get(AuthorizationService.class);
+ auth.authorizeForAdmin(getUser(request), true);
+ }
+ catch (AuthorizationException ex) {
+ throw new XServletException(HttpServletResponse.SC_UNAUTHORIZED,
ex);
+ }
+ }
@Override
protected void doPost(HttpServletRequest request, HttpServletResponse
response) throws ServletException,
Modified:
oozie/trunk/core/src/main/java/org/apache/oozie/servlet/V1AdminServlet.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/servlet/V1AdminServlet.java?rev=1544387&r1=1544386&r2=1544387&view=diff
==============================================================================
--- oozie/trunk/core/src/main/java/org/apache/oozie/servlet/V1AdminServlet.java
(original)
+++ oozie/trunk/core/src/main/java/org/apache/oozie/servlet/V1AdminServlet.java
Fri Nov 22 00:50:48 2013
@@ -46,7 +46,7 @@ public class V1AdminServlet extends Base
private static final long serialVersionUID = 1L;
private static final String INSTRUMENTATION_NAME = "v1admin";
- private static final ResourceInfo RESOURCES_INFO[] = new ResourceInfo[10];
+ private static final ResourceInfo RESOURCES_INFO[] = new ResourceInfo[12];
static {
RESOURCES_INFO[0] = new
ResourceInfo(RestConstants.ADMIN_STATUS_RESOURCE, Arrays.asList("PUT", "GET"),
@@ -70,6 +70,11 @@ public class V1AdminServlet extends Base
Collections.EMPTY_LIST);
RESOURCES_INFO[9] = new
ResourceInfo(RestConstants.ADMIN_AVAILABLE_OOZIE_SERVERS_RESOURCE,
Arrays.asList("GET"),
Collections.EMPTY_LIST);
+ RESOURCES_INFO[10] = new
ResourceInfo(RestConstants.ADMIN_UPDATE_SHARELIB, Arrays.asList("GET"),
+ Collections.EMPTY_LIST);
+ RESOURCES_INFO[11] = new
ResourceInfo(RestConstants.ADMIN_LIST_SHARELIB, Arrays.asList("GET"),
+ Collections.EMPTY_LIST);
+
}
protected V1AdminServlet(String name) {
Modified:
oozie/trunk/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java?rev=1544387&r1=1544386&r2=1544387&view=diff
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java
(original)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java
Fri Nov 22 00:50:48 2013
@@ -1018,5 +1018,66 @@ public class TestOozieCLI extends DagSer
return null;
}
});
- }
+ }
+
+ public void testshareLibUpdate() throws Exception {
+ runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new
Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ HeaderTestingVersionServlet.OOZIE_HEADERS.clear();
+
+ String oozieUrl = getContextURL();
+ String[] args = new String[] { "admin", "-sharelibupdate",
"-oozie", oozieUrl };
+ assertEquals(0, new OozieCLI().run(args));
+
+ return null;
+ }
+ });
+ }
+
+ public void testshareLibUpdate_withSecurity() throws Exception {
+ runTest(END_POINTS, SERVLET_CLASSES, true, new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ HeaderTestingVersionServlet.OOZIE_HEADERS.clear();
+
+ String oozieUrl = getContextURL();
+ String[] args = new String[] { "admin", "-sharelibupdate",
"-oozie", oozieUrl };
+ assertEquals(-1, new OozieCLI().run(args));
+
+ return null;
+ }
+ });
+ }
+
+ public void testGetShareLib() throws Exception {
+ runTest(END_POINTS, SERVLET_CLASSES, false, new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ HeaderTestingVersionServlet.OOZIE_HEADERS.clear();
+
+ String oozieUrl = getContextURL();
+ String[] args = new String[] { "admin", "-shareliblist",
"-oozie", oozieUrl };
+ assertEquals(0, new OozieCLI().run(args));
+
+ return null;
+ }
+ });
+ }
+
+ public void testGetShareLib_withKey() throws Exception {
+ runTest(END_POINTS, SERVLET_CLASSES, false, new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ HeaderTestingVersionServlet.OOZIE_HEADERS.clear();
+
+ String oozieUrl = getContextURL();
+ String[] args = new String[] { "admin", "-shareliblist",
"pig", "-oozie", oozieUrl };
+ assertEquals(0, new OozieCLI().run(args));
+
+ return null;
+ }
+ });
+ }
+
}
Modified:
oozie/trunk/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/service/TestShareLibService.java?rev=1544387&r1=1544386&r2=1544387&view=diff
==============================================================================
---
oozie/trunk/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
(original)
+++
oozie/trunk/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
Fri Nov 22 00:50:48 2013
@@ -24,6 +24,7 @@ import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
+import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
@@ -447,7 +448,7 @@ public class TestShareLibService extends
}
public void creatTempshareLibKeymap(FileSystem fs) {
- XConfiguration prop = new XConfiguration();
+ Properties prop = new Properties();
try {
@@ -462,12 +463,12 @@ public class TestShareLibService extends
createFile(basePath.toString() + Path.SEPARATOR + "pig" +
Path.SEPARATOR + "pig.jar");
createFile(somethingNew.toString() + Path.SEPARATOR +
"somethingNew" + Path.SEPARATOR + "somethingNew.jar");
- prop.set(ShareLibService.SHARE_LIB_CONF_PREFIX + ".pig",
"/user/test/" + basePath.toString());
- prop.set(ShareLibService.SHARE_LIB_CONF_PREFIX + ".something_new",
"/user/test/" + somethingNew.toString());
+ prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".pig",
"/user/test/" + basePath.toString());
+ prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".something_new",
"/user/test/" + somethingNew.toString());
FSDataOutputStream out = fs.create(new
Path("/user/test/config.properties"));
- prop.writeXml(out);
+ prop.store(out, null);
out.close();
}
@@ -478,7 +479,7 @@ public class TestShareLibService extends
public void creatTempshareLibKeymap_multipleFile(FileSystem fs) {
- XConfiguration prop = new XConfiguration();
+ Properties prop = new Properties();
try {
String testPath = "shareLibPath/";
@@ -491,13 +492,13 @@ public class TestShareLibService extends
createFile(basePath.toString() + Path.SEPARATOR + "pig" +
Path.SEPARATOR + "pig.jar");
createFile(somethingNew.toString() + Path.SEPARATOR +
"somethingNew" + Path.SEPARATOR + "somethingNew.jar");
- prop.set(ShareLibService.SHARE_LIB_CONF_PREFIX + ".pig",
"/user/test/" + basePath.toString()
+ prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".pig",
"/user/test/" + basePath.toString()
+ Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig.jar," +
"/user/test/" + somethingNew.toString()
+ Path.SEPARATOR + "somethingNew" + Path.SEPARATOR +
"somethingNew.jar");
FSDataOutputStream out = fs.create(new
Path("/user/test/config.properties"));
- prop.writeXml(out);
+ prop.store(out, null);
out.close();
}
Modified:
oozie/trunk/core/src/test/java/org/apache/oozie/servlet/TestAdminServlet.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/servlet/TestAdminServlet.java?rev=1544387&r1=1544386&r2=1544387&view=diff
==============================================================================
---
oozie/trunk/core/src/test/java/org/apache/oozie/servlet/TestAdminServlet.java
(original)
+++
oozie/trunk/core/src/test/java/org/apache/oozie/servlet/TestAdminServlet.java
Fri Nov 22 00:50:48 2013
@@ -220,4 +220,50 @@ public class TestAdminServlet extends Da
});
}
+ public void testShareLibUpdate() throws Exception {
+ runTest("/v2/admin/*", V2AdminServlet.class, IS_SECURITY_ENABLED, new
Callable<Void>() {
+ public Void call() throws Exception {
+ URL url = createURL(RestConstants.ADMIN_UPDATE_SHARELIB,
Collections.EMPTY_MAP);
+ HttpURLConnection conn = (HttpURLConnection)
url.openConnection();
+ conn.setRequestMethod("GET");
+ assertEquals(HttpServletResponse.SC_OK,
conn.getResponseCode());
+
assertTrue(conn.getHeaderField("content-type").startsWith(RestConstants.JSON_CONTENT_TYPE));
+ JSONObject json = (JSONObject) JSONValue.parse(new
InputStreamReader(conn.getInputStream()));
+ assertEquals(1, json.entrySet().size());
+ return null;
+ }
+ });
+ }
+
+ public void testShareLib() throws Exception {
+ runTest("/v2/admin/*", V2AdminServlet.class, IS_SECURITY_ENABLED, new
Callable<Void>() {
+ public Void call() throws Exception {
+ URL url = createURL(RestConstants.ADMIN_LIST_SHARELIB,
Collections.EMPTY_MAP);
+ HttpURLConnection conn = (HttpURLConnection)
url.openConnection();
+ conn.setRequestMethod("GET");
+ assertEquals(HttpServletResponse.SC_OK,
conn.getResponseCode());
+
assertTrue(conn.getHeaderField("content-type").startsWith(RestConstants.JSON_CONTENT_TYPE));
+ JSONObject json = (JSONObject) JSONValue.parse(new
InputStreamReader(conn.getInputStream()));
+ assertEquals(null, json.get(JsonTags.SHARELIB_LIB));
+ return null;
+ }
+ });
+ }
+
+ public void testShareLib_withKey() throws Exception {
+ runTest("/v2/admin/*", V2AdminServlet.class, IS_SECURITY_ENABLED, new
Callable<Void>() {
+ public Void call() throws Exception {
+ Map<String, String> map = new HashMap<String, String>();
+ map.put(RestConstants.SHARE_LIB_REQUEST_KEY, "pig");
+ URL url = createURL(RestConstants.ADMIN_LIST_SHARELIB, map);
+ HttpURLConnection conn = (HttpURLConnection)
url.openConnection();
+ conn.setRequestMethod("GET");
+ assertEquals(HttpServletResponse.SC_OK,
conn.getResponseCode());
+
assertTrue(conn.getHeaderField("content-type").startsWith(RestConstants.JSON_CONTENT_TYPE));
+ JSONObject json = (JSONObject) JSONValue.parse(new
InputStreamReader(conn.getInputStream()));
+ assertEquals(null, json.get(JsonTags.SHARELIB_LIB));
+ return null;
+ }
+ });
+ }
}
Modified: oozie/trunk/docs/src/site/twiki/DG_CommandLineTool.twiki
URL:
http://svn.apache.org/viewvc/oozie/trunk/docs/src/site/twiki/DG_CommandLineTool.twiki?rev=1544387&r1=1544386&r2=1544387&view=diff
==============================================================================
--- oozie/trunk/docs/src/site/twiki/DG_CommandLineTool.twiki (original)
+++ oozie/trunk/docs/src/site/twiki/DG_CommandLineTool.twiki Fri Nov 22
00:50:48 2013
@@ -80,6 +80,9 @@ usage:
-systemmode <arg> Supported in Oozie-2.0 or later versions
ONLY. Change oozie
system mode
[NORMAL|NOWEBSERVICE|SAFEMODE]
-version show Oozie server build version
+ -shareliblist List available sharelib that can be
specified in a workflow
+ action
+ -sharelibupdate Update server to use a newer version of
sharelib
.
oozie validate <ARGS> : validate a workflow XML file
.
@@ -813,6 +816,54 @@ Error: Invalid workflow-app, org.xml.sax
It performs an XML Schema validation on the specified workflow XML file.
+---+++ Getting list of available sharelib
+This command is used to get list of available sharelib.
+If the name of the sharelib is passed as an argument (regex supported) then
all corresponding files are also listed.
+
+<verbatim>
+$ oozie admin -oozie http://localhost:11000/oozie -shareliblist
+[Available ShareLib]
+ oozie
+ hive
+ distcp
+ hcatalog
+ sqoop
+ mapreduce-streaming
+ pig
+
+$ oozie admin -oozie http://localhost:11000/oozie -sharelib pig*
+[Available ShareLib]
+ pig
+
hdfs://localhost:9000/user/purushah/share/lib/lib_20131114095729/pig/pig.jar
+
hdfs://localhost:9000/user/purushah/share/lib/lib_20131114095729/pig/piggybank.jar
+</verbatim>
+
+---+++ Update system sharelib
+This command makes the oozie server to pick up the latest version of sharelib
present
+under oozie.service.WorkflowAppService.system.libpath directory based on the
sharelib directory timestamp or reloads
+the sharelib metafile if one is configured. The main purpose is to update the
sharelib on the oozie server without restarting.
+
+<verbatim>
+$ oozie admin -oozie http://localhost:11000/oozie -sharelibupdate
+[ShareLib update status]
+ShareLib update status]
+ host = localhost:11000
+ status = Successful
+ sharelibDirOld =
hdfs://localhost:9000/user/purushah/share/lib/lib_20131114095729
+ sharelibDirNew =
hdfs://localhost:9000/user/purushah/share/lib/lib_20131120163343
+</verbatim>
+
+Sharelib update for metafile configuration.
+<verbatim>
+$ oozie admin -oozie http://localhost:11000/oozie -sharelibupdate
+[ShareLib update status]
+ host = localhost:11000
+ status = Successful
+ sharelibMetaFile =
hdfs://localhost:9000/user/purushah/sharelib_metafile.property
+ sharelibMetaFileOldTimeStamp = Thu, 21 Nov 2013 00:40:04 GMT
+ sharelibMetaFileNewTimeStamp = Thu, 21 Nov 2013 01:01:25 GMT
+</verbatim>
+
#SLAOperations
---++ SLA Operations
Modified: oozie/trunk/docs/src/site/twiki/WebServicesAPI.twiki
URL:
http://svn.apache.org/viewvc/oozie/trunk/docs/src/site/twiki/WebServicesAPI.twiki?rev=1544387&r1=1544386&r2=1544387&view=diff
==============================================================================
--- oozie/trunk/docs/src/site/twiki/WebServicesAPI.twiki (original)
+++ oozie/trunk/docs/src/site/twiki/WebServicesAPI.twiki Fri Nov 22 00:50:48
2013
@@ -381,6 +381,95 @@ Content-Type: application/json;charset=U
}
</verbatim>
+---++++ List available sharelib
+A HTTP GET request to get list of available sharelib.
+If the name of the sharelib is passed as an argument (regex supported) then
all corresponding files are also listed.
+
+*Request:*
+
+<verbatim>
+GET /oozie/v2/admin/list_sharelib
+</verbatim>
+
+*Response:*
+
+<verbatim>
+HTTP/1.1 200 OK
+Content-Type: application/json;charset=UTF-8
+{
+ "sharelib":
+ [
+ "oozie",
+ "hive",
+ "distcp",
+ "hcatalog",
+ "sqoop",
+ "mapreduce-streaming",
+ "pig"
+ ]
+}
+</verbatim>
+
+*Request:*
+
+<verbatim>
+GET /oozie/v2/admin/list_sharelib?lib=pig*
+</verbatim>
+
+*Response:*
+
+<verbatim>
+HTTP/1.1 200 OK
+Content-Type: application/json;charset=UTF-8
+
+{
+ "sharelib":
+ [
+ {
+ "pig":
+ {
+ "sharelibFiles":
+ [
+
hdfs://localhost:9000/user/purushah/share/lib/lib_20131114095729/pig/pig.jar
+
hdfs://localhost:9000/user/purushah/share/lib/lib_20131114095729/pig/piggybank.jar
+ ]
+ }
+ }
+ ]
+}
+</verbatim>
+
+
+---++++ Update system sharelib
+This webservice call makes the oozie server to pick up the latest version of
sharelib present
+under oozie.service.WorkflowAppService.system.libpath directory based on the
sharelib directory timestamp or reloads
+the sharelib metafile if one is configured. The main purpose is to update the
sharelib on the oozie server without restarting.
+
+
+*Request:*
+
+<verbatim>
+GET /oozie/v2/admin/update_sharelib
+</verbatim>
+
+*Response:*
+
+<verbatim>
+HTTP/1.1 200 OK
+Content-Type: application/json;charset=UTF-8
+
+{
+ {
+ "sharelibUpdate":
+ {
+
"sharelibDirOld":"hdfs://localhost:9000/user/purushah/share/lib/lib_20131114095729",
+
"sharelibDirNew":"hdfs://localhost:9000/user/purushah/share/lib/lib_20131120163343",
+ "host":"localhost:11000",
+ "status":"Successful"
+ }
+ }
+}
+
---+++ Job and Jobs End-Points
_Modified in Oozie v1 WS API_
@@ -1241,8 +1330,8 @@ Content-Type: application/json;charset=U
** bulkresponses: [
** {
bulkbundle:
- {
- bundleJobName: "my-bundle-app",
+ {
+ bundleJobName: "my-bundle-app",
bundleJobId: "0-200905191240-oozie-B",
status: "SUSPENDED",
},
Modified: oozie/trunk/release-log.txt
URL:
http://svn.apache.org/viewvc/oozie/trunk/release-log.txt?rev=1544387&r1=1544386&r2=1544387&view=diff
==============================================================================
--- oozie/trunk/release-log.txt (original)
+++ oozie/trunk/release-log.txt Fri Nov 22 00:50:48 2013
@@ -1,5 +1,6 @@
-- Oozie 4.1.0 release (trunk - unreleased)
+OOZIE-1519 Admin command to update the sharelib (puru via ryota)
OOZIE-1604 <java-opts> and <java-opt> not added to Application Master property
in uber mode (ryota)
OOZIE-1584 Setup sharelib using script and pickup latest(honor ship.launcher)
and remove DFS dependency at startup (puru via ryota)
OOZIE-1550 Create a safeguard to kill errant recursive workflows before they
bring down oozie (rkanter)
