http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AMSecurityManagerImpl.java ---------------------------------------------------------------------- diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AMSecurityManagerImpl.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AMSecurityManagerImpl.java new file mode 100644 index 0000000..d31690e --- /dev/null +++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AMSecurityManagerImpl.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.yarn.appMaster.http; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.scanner.persistence.ScanResult; +import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.exception.DrillbitStartupException; +import org.apache.drill.exec.rpc.user.security.UserAuthenticationException; +import org.apache.drill.exec.rpc.user.security.UserAuthenticator; +import org.apache.drill.exec.rpc.user.security.UserAuthenticatorFactory; +import org.apache.drill.exec.util.ImpersonationUtil; +import org.apache.drill.yarn.appMaster.AMWrapperException; +import org.apache.drill.yarn.core.DoYUtil; +import org.apache.drill.yarn.core.DrillOnYarnConfig; + +import com.typesafe.config.Config; + +/** + * Implements the three supported AM security models: Drill, + * hard-coded user and password, and open access. + */ + +public class AMSecurityManagerImpl implements AMSecurityManager { + private static final Log LOG = LogFactory.getLog(AMSecurityManagerImpl.class); + + /** + * Thin layer around the Drill authentication system to adapt from + * Drill-on-YARN's environment to that expected by the Drill classes. + */ + private static class DrillSecurityManager implements AMSecurityManager { + private UserAuthenticator authenticator; + + @Override + public void init() { + try { + DrillOnYarnConfig doyConfig = DrillOnYarnConfig.instance(); + DrillConfig config = doyConfig.getDrillConfig(); + ScanResult classpathScan = doyConfig.getClassPathScan(); + if (config.getBoolean(ExecConstants.USER_AUTHENTICATION_ENABLED)) { + authenticator = UserAuthenticatorFactory.createAuthenticator(config, + classpathScan); + } else { + authenticator = null; + } + } catch (DrillbitStartupException e) { + LOG.info("Authentication initialization failed", e); + throw new AMWrapperException("Security init failed", e); + } + } + + @Override + public boolean login(String user, String password) { + if (authenticator == null) { + return true; + } + try { + authenticator.authenticate(user, password); + } catch (UserAuthenticationException e) { + LOG.info("Authentication failed for user " + user, e); + return false; + } + return ImpersonationUtil.getProcessUserName().equals(user); + } + + @Override + public void close() { + try { + if (authenticator != null) { + authenticator.close(); + } + } catch (IOException e) { + LOG.info("Ignoring error on authenticator close", e); + } + } + + @Override + public boolean requiresLogin() { + return authenticator != null; + } + } + + /** + * Simple security manager: user name and password reside in the DoY config + * file. + */ + + private static class SimpleSecurityManager implements AMSecurityManager { + + private String userName; + private String password; + + @Override + public void init() { + Config config = DrillOnYarnConfig.config(); + userName = config.getString(DrillOnYarnConfig.HTTP_USER_NAME); + password = config.getString(DrillOnYarnConfig.HTTP_PASSWORD); + if (DoYUtil.isBlank(userName)) { + LOG.warn("Simple HTTP authentication is enabled, but " + + DrillOnYarnConfig.HTTP_USER_NAME + " is blank."); + } + if (DoYUtil.isBlank(userName)) { + LOG.warn("Simple HTTP authentication is enabled, but " + + DrillOnYarnConfig.HTTP_PASSWORD + " is blank."); + } + } + + @Override + public boolean requiresLogin() { + return !DoYUtil.isBlank(userName); + } + + @Override + public boolean login(String user, String pwd) { + if (!requiresLogin()) { + return true; + } + boolean ok = userName.equals(user) && password.equals(pwd); + if (!ok) { + LOG.info( + "Failed login attempt with simple authorization for user " + user); + } + return ok; + } + + @Override + public void close() { + // Nothing to do + } + + } + + private static AMSecurityManagerImpl instance; + + private AMSecurityManager managerImpl; + + private AMSecurityManagerImpl() { + } + + public static void setup() { + instance = new AMSecurityManagerImpl(); + instance.init(); + } + + /** + * Look at the DoY config file to decide which security system (if any) to + * use. + */ + + @Override + public void init() { + Config config = DrillOnYarnConfig.config(); + String authType = config.getString(DrillOnYarnConfig.HTTP_AUTH_TYPE); + if (DrillOnYarnConfig.AUTH_TYPE_DRILL.equals(authType)) { + // Drill authentication. Requires both DoY to select Drill + // auth, and for Drill's auth to be enabled. + if(config.getBoolean(ExecConstants.USER_AUTHENTICATION_ENABLED)) { + managerImpl = new DrillSecurityManager(); + managerImpl.init(); + } + } else if (DrillOnYarnConfig.AUTH_TYPE_SIMPLE.equals(authType)) { + managerImpl = new SimpleSecurityManager(); + managerImpl.init(); + } else if (DoYUtil.isBlank(authType) + || DrillOnYarnConfig.AUTH_TYPE_NONE.equals(authType)) { + ; + } else { + LOG.error("Unrecognized authorization type for " + + DrillOnYarnConfig.HTTP_AUTH_TYPE + ": " + authType + + " - assuming no auth."); + } + } + + @Override + public boolean login(String user, String password) { + if (managerImpl == null) { + return true; + } + return managerImpl.login(user, password); + } + + @Override + public void close() { + if (managerImpl != null) { + managerImpl.close(); + managerImpl = null; + } + } + + @Override + public boolean requiresLogin() { + return managerImpl != null; + } + + public static AMSecurityManager instance() { + return instance; + } + + public static boolean isEnabled() { + return instance != null && instance.managerImpl != null; + } +}
http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AbstractTasksModel.java ---------------------------------------------------------------------- diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AbstractTasksModel.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AbstractTasksModel.java new file mode 100644 index 0000000..bc4e8d6 --- /dev/null +++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AbstractTasksModel.java @@ -0,0 +1,380 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.yarn.appMaster.http; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; +import org.apache.drill.yarn.appMaster.ClusterController; +import org.apache.drill.yarn.appMaster.ClusterControllerImpl; +import org.apache.drill.yarn.appMaster.ControllerVisitor; +import org.apache.drill.yarn.appMaster.Task; +import org.apache.drill.yarn.appMaster.Task.TrackingState; +import org.apache.drill.yarn.appMaster.TaskState; +import org.apache.drill.yarn.appMaster.TaskVisitor; +import org.apache.drill.yarn.core.DoYUtil; +import org.apache.drill.yarn.core.DrillOnYarnConfig; +import org.apache.drill.yarn.zk.ZKRegistry; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.Resource; + +public abstract class AbstractTasksModel { + public static class TaskModel { + public int id; + protected String groupName; + protected boolean isLive; + protected TaskState taskState; + protected String taskStateHint; + protected String state; + protected boolean cancelled; + protected String trackingState; + protected String trackingStateHint; + protected Container container; + protected DrillbitEndpoint endpoint; + protected long startTime; + protected int memoryMb; + protected int vcores; + protected double disks; + protected String containerId; + protected String nmLink; + protected long endTime; + protected String disposition; + protected int tryCount; + + private Map<TaskState,String> stateHints = makeStateHints( ); + private Map<TrackingState,String> trackingStateHints = makeTrackingStateHints( ); + + public TaskModel(Task task, boolean live) { + id = task.taskId; + groupName = task.scheduler.getName(); + taskState = task.getState(); + taskStateHint = stateHints.get(taskState); + state = taskState.getLabel(); + cancelled = task.isCancelled(); + isLive = live && taskState == TaskState.RUNNING; + TrackingState tState = task.getTrackingState(); + trackingState = tState.getDisplayName(); + trackingStateHint = trackingStateHints.get(tState); + container = task.container; + startTime = task.launchTime; + if (task.container != null) { + containerId = task.container.getId().toString(); + Resource resource = task.container.getResource(); + memoryMb = resource.getMemory(); + vcores = resource.getVirtualCores(); + disks = task.getContainerSpec().disks; + + // Emulate the NM link. Used for debugging, gets us to + // the page on the NM UI for this container so we can see + // logs, etc. + + nmLink = "http://"; + task.container.getNodeHttpAddress(); + } else { + memoryMb = task.scheduler.getResource().memoryMb; + vcores = task.scheduler.getResource().vCores; + } + endpoint = (DrillbitEndpoint) task.properties + .get(ZKRegistry.ENDPOINT_PROPERTY); + if (!live) { + endTime = task.completionTime; + tryCount = task.tryCount; + + // Determine disposition from most general to most + // specific sources of information. + + disposition = state; + if (task.disposition != null) { + disposition = task.disposition.toString(); + } + if (task.completionStatus != null) { + disposition = reformatDiagnostics( task.completionStatus.getDiagnostics() ); + } + if (task.error != null) { + disposition = task.error.getMessage(); + } + } + } + + private enum FormatState { PRE_STACK, IN_STACK, POST_STACK }; + + /** + * YARN diagnostics are verbose: they contain a stack trace of the YARN node + * manager thread (not Drill), and contain blank lines, the container ID, + * etc. Remove unnecessary cruft to make the diagnostics simpler and smaller + * in the Web UI. + * + * @param orig YARN diagnostics + * @return cleaned-up version. + */ + + public static String reformatDiagnostics( String orig ) { + try { + StringBuilder buf = new StringBuilder( ); + BufferedReader reader = new BufferedReader( new StringReader( orig ) ); + String line; + FormatState state = FormatState.PRE_STACK; + while ( (line = reader.readLine()) != null ) { + switch( state ) { + case PRE_STACK: + if ( line.startsWith( "Container id:") ) { + continue; + } + if ( line.startsWith( "Stack trace:" ) ) { + state = FormatState.IN_STACK; + continue; + } + break; + case IN_STACK: + if ( line.trim().isEmpty() ) { + state = FormatState.POST_STACK; + } + continue; + case POST_STACK: + default: + break; + } + if ( line.trim().isEmpty() ) { + continue; + } + buf.append( line ); + buf.append( "<br/>\n" ); + } + buf.append( "See log file for details." ); + return buf.toString(); + } catch (IOException e) { + // Will never occur. But, if the impossible happens, just return + // the original diagnostics. + + return orig.replace("\n", "<br>\n"); + } + } + + private Map<TaskState, String> makeStateHints() { + Map<TaskState, String> hints = new HashMap<>(); + hints.put(TaskState.START, "Queued to send a container request to YARN."); + hints.put(TaskState.REQUESTING, "Container request sent to YARN."); + hints.put(TaskState.LAUNCHING, + "YARN provided a container, send launch request."); + hints.put(TaskState.WAIT_START_ACK, + "Drillbit launched, waiting for ZooKeeper registration."); + hints.put(TaskState.RUNNING, "Drillbit is running normally."); + hints.put(TaskState.ENDING, + "Graceful shutdown request sent to the Drillbit."); + hints.put(TaskState.KILLING, + "Sent the YARN Node Manager a request to forcefully kill the Drillbit."); + hints.put(TaskState.WAIT_END_ACK, + "Drillbit has shut down; waiting for ZooKeeper to confirm."); + // The UI will never display the END state. + hints.put(TaskState.END, "The Drillbit has shut down."); + return hints; + } + + private Map<TrackingState, String> makeTrackingStateHints() { + Map<TrackingState, String> hints = new HashMap<>(); + // UNTRACKED state is not used by Drillbits. + hints.put(TrackingState.UNTRACKED, "Task is not tracked in ZooKeeper."); + hints.put(TrackingState.NEW, + "Drillbit has not yet registered with ZooKeeper."); + hints.put(TrackingState.START_ACK, + "Drillbit has registered normally with ZooKeeper."); + hints.put(TrackingState.END_ACK, + "Drillbit is no longer registered with ZooKeeper."); + return hints; + } + + public String getTaskId() { + return Integer.toString(id); + } + + public String getGroupName( ) { return groupName; } + + public boolean isLive( ) { + return isLive; + } + + public String getHost( ) { + if ( container == null ) { + return ""; } + return container.getNodeId().getHost(); + } + + public String getLink( ) { + if ( endpoint == null ) { + return ""; } + String port = DrillOnYarnConfig.config( ).getString( DrillOnYarnConfig.DRILLBIT_HTTP_PORT ); + String protocol = "http:"; + if ( DrillOnYarnConfig.config().getBoolean( DrillOnYarnConfig.DRILLBIT_USE_HTTPS ) ) { + protocol = "https:"; + } + return protocol + "//" + endpoint.getAddress() + ":" + port + "/"; + } + + public String getState( ) { return state.toString(); } + public String getStateHint( ) { return taskStateHint; } + public boolean isCancelled( ) { return cancelled; } + + public boolean isCancellable( ) { + return ! cancelled && taskState.isCancellable( ); + } + + public String getTrackingState( ) { return trackingState; } + public String getTrackingStateHint( ) { return trackingStateHint; } + + public String getStartTime( ) { + if ( startTime == 0 ) { + return ""; } + return DoYUtil.toIsoTime( startTime ); + } + + public int getMemory( ) { return memoryMb; } + public int getVcores( ) { return vcores; } + public String getDisks( ) { + return String.format( "%.2f", disks ); + } + public boolean hasContainer( ) { return containerId != null; } + public String getContainerId( ) { return displayString( containerId ); } + public String getNmLink( ) { return displayString( nmLink ); } + public String getDisposition( ) { return displayString( disposition ); } + public int getTryCount( ) { return tryCount; } + public String displayString( String value ) { return (value == null) ? "" : value; } + + public String getEndTime( ) { + if ( endTime == 0 ) { + return ""; } + return DoYUtil.toIsoTime( endTime ); + } + } + + public static class UnmanagedDrillbitModel + { + protected String host; + protected String ports; + + public UnmanagedDrillbitModel( String endpoint ) { + String parts[] = endpoint.split( ":" ); + if ( parts.length < 4 ) { + // Should never occur, but better save than sorry. + + host = endpoint; + ports = ""; + } + else { + host = parts[0]; + List<String> thePorts = new ArrayList<>( ); + thePorts.add( parts[1] ); + thePorts.add( parts[2] ); + thePorts.add( parts[3] ); + ports = DoYUtil.join( ", ", thePorts ); + } + } + + public String getHost( ) { return host; } + public String getPorts( ) { return ports; } + } + + protected boolean supportsDisks; + protected List<TaskModel> results = new ArrayList<>( ); + + public List<TaskModel> getTasks( ) { return results; } + public boolean hasTasks( ) { return ! results.isEmpty(); } + public boolean supportsDiskResource( ) { return supportsDisks; } + + public static class TasksModel extends AbstractTasksModel implements TaskVisitor + { + protected List<UnmanagedDrillbitModel> unmanaged; + protected List<String> blacklist; + + @Override + public void visit(Task task) { + results.add( new TaskModel( task, true ) ); + } + + /** + * Sort tasks by Task ID. + */ + + public void sortTasks() { + Collections.sort( results, new Comparator<TaskModel>( ) { + @Override + public int compare(TaskModel t1, TaskModel t2) { + return Integer.compare( t1.id, t2.id ); + } + }); + } + + /** + * List any anomalies: either stray Drillbits (those in ZK but not launched by DoY), + * or blacklisted nodes. + * <p> + * To avoid race conditions, do not use the controller visitor to invoke this method, + * we want to leave the controller unlocked and instead lock only the ZK registry. + * + * @param controller + */ + + public void listAnomalies(ClusterController controller) { + listUnmanaged(controller); + synchronized( controller ) { + blacklist = ((ClusterControllerImpl) controller).getNodeInventory().getBlacklist(); + } + Collections.sort( blacklist ); + } + + private void listUnmanaged(ClusterController controller) { + ZKRegistry zkRegistry = (ZKRegistry) controller.getProperty( ZKRegistry.CONTROLLER_PROPERTY ); + if ( zkRegistry == null ) { + return; + } + List<String> endpoints = zkRegistry.listUnmanagedDrillits( ); + if ( endpoints.isEmpty() ) { + return; } + unmanaged = new ArrayList<>( ); + for ( String endpoint : endpoints ) { + unmanaged.add( new UnmanagedDrillbitModel( endpoint ) ); + } + } + + public List<UnmanagedDrillbitModel>getUnnamaged( ) { return unmanaged; } + public boolean hasUnmanagedDrillbits( ) { return unmanaged != null; } + public int getUnmanagedDrillbitCount( ) { + return (unmanaged == null) ? 0 : unmanaged.size( ); + } + public boolean hasBlacklist( ) { return ! blacklist.isEmpty(); } + public int getBlacklistCount( ) { return blacklist.size( ); } + public List<String> getBlacklist( ) { return blacklist; } + } + + public static class HistoryModel extends AbstractTasksModel implements ControllerVisitor + { + @Override + public void visit(ClusterController controller) { + ClusterControllerImpl impl = (ClusterControllerImpl) controller; + for ( Task task : impl.getHistory( ) ) { + results.add( new TaskModel( task, false ) ); + } + } + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AmRestApi.java ---------------------------------------------------------------------- diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AmRestApi.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AmRestApi.java new file mode 100644 index 0000000..21ddc4b --- /dev/null +++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AmRestApi.java @@ -0,0 +1,296 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.yarn.appMaster.http; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.annotation.security.PermitAll; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.MediaType; + +import org.apache.drill.yarn.appMaster.Dispatcher; +import org.apache.drill.yarn.appMaster.http.AbstractTasksModel.TaskModel; +import org.apache.drill.yarn.appMaster.http.ControllerModel.ClusterGroupModel; +import org.apache.drill.yarn.core.DoYUtil; +import org.apache.drill.yarn.core.DrillOnYarnConfig; +import org.apache.drill.yarn.core.NameValuePair; +import org.apache.drill.yarn.zk.ZKClusterCoordinatorDriver; + +public class AmRestApi extends PageTree +{ + @Path("/config") + @PermitAll + public static class ConfigResource + { + @GET + @Produces(MediaType.APPLICATION_JSON) + public Map<String,Object> getConfig( ) { + Map<String, Object> map = new HashMap<>(); + for (NameValuePair pair : DrillOnYarnConfig.instance().getPairs()) { + map.put(pair.getName(), pair.getValue()); + } + return map; + } + } + + /** + * Returns cluster status as a tree of JSON objects. Done as explicitly-defined + * maps to specify the key names (which must not change to avoid breaking + * compatibility) and to handle type conversions. + */ + + @Path("/status") + @PermitAll + public static class StatusResource + { + @GET + @Produces(MediaType.APPLICATION_JSON) + public Map<String,Object> getStatus( ) { + ControllerModel model = new ControllerModel( ); + dispatcher.getController().visit( model ); + + Map<String,Object> root = new HashMap<>( ); + root.put( "state", model.state.toString() ); + + Map<String, Object> summary = new HashMap<>(); + summary.put("drillMemoryMb", model.totalDrillMemory); + summary.put("drillVcores", model.totalDrillVcores); + summary.put("yarnMemoryMb", model.yarnMemory); + summary.put("yarnVcores", model.yarnVcores); + summary.put("liveBitCount", model.liveCount); + summary.put("totalBitCount", model.taskCount); + summary.put("targetBitCount", model.targetCount); + summary.put("unmanagedCount", model.getUnmanagedCount()); + summary.put("blackListCount", model.getBlacklistCount()); + summary.put("freeNodeCount", model.getFreeNodeCount()); + root.put("summary", summary); + + List<Map<String, Object>> pools = new ArrayList<>(); + for (ClusterGroupModel pool : model.groups) { + Map<String, Object> poolObj = new HashMap<>(); + poolObj.put("name", pool.name); + poolObj.put("type", pool.type); + poolObj.put("liveBitCount", pool.liveCount); + poolObj.put("targetBitCount", pool.targetCount); + poolObj.put("totalBitCount", pool.taskCount); + poolObj.put("totalMemoryMb", pool.memory); + poolObj.put("totalVcores", pool.vcores); + pools.add(poolObj); + } + root.put("pools", pools); + + AbstractTasksModel.TasksModel tasksModel = new AbstractTasksModel.TasksModel(); + dispatcher.getController().visitTasks(tasksModel); + List<Map<String, Object>> bits = new ArrayList<>(); + for (TaskModel task : tasksModel.results) { + Map<String, Object> bitObj = new HashMap<>(); + bitObj.put("containerId", task.container.getId().toString()); + bitObj.put("host", task.getHost()); + bitObj.put("id", task.id); + bitObj.put("live", task.isLive()); + bitObj.put("memoryMb", task.memoryMb); + bitObj.put("vcores", task.vcores); + bitObj.put("pool", task.groupName); + bitObj.put("state", task.state); + bitObj.put("trackingState", task.trackingState); + bitObj.put("endpoint", + ZKClusterCoordinatorDriver.asString(task.endpoint)); + bitObj.put("link", task.getLink()); + bitObj.put("startTime", task.getStartTime()); + bits.add(bitObj); + } + root.put("drillbits", bits); + + return root; + } + } + + /** + * Stop the cluster. Uses a key to validate the request. The value of the key is + * set in the Drill-on-YARN configuration file. The purpose is simply to prevent + * accidental cluster shutdown when experimenting with the REST API; this is + * not meant to be a security mechanism. + * + * @param key + * @return + */ + + @Path("/stop") + @PermitAll + public static class StopResource + { + @DefaultValue( "" ) + @QueryParam( "key" ) + String key; + + @POST + @Produces(MediaType.APPLICATION_JSON) + public Map<String,String> postStop( ) + { + Map<String, String> error = checkKey(key); + if (error != null) { + return error; + } + + dispatcher.getController().shutDown(); + return successResponse("Shutting down"); + } + } + + @Path("/resize/{quantity}") + @PermitAll + public static class ResizeResource + { + @PathParam(value = "quantity") + String quantity; + @DefaultValue( "" ) + @QueryParam( "key" ) + String key; + + @POST + @Produces(MediaType.APPLICATION_JSON) + public Map<String,String> postResize( ) + { + ResizeRequest request = new ResizeRequest(key, quantity); + if (request.error != null) { + return request.error; + } + + int curSize = dispatcher.getController().getTargetCount(); + dispatcher.getController().resizeTo(request.n); + return successResponse("Resizing from " + curSize + " to " + request.n); + } + } + + protected static class ResizeRequest + { + Map<String,String> error; + int n; + + public ResizeRequest( String key, String quantity ) { + error = checkKey(key); + if (error != null) { + return; + } + try { + n = Integer.parseInt(quantity); + } catch (NumberFormatException e) { + error = errorResponse("Invalid argument: " + quantity); + } + if (n < 0) { + error = errorResponse("Invalid argument: " + quantity); + } + } + } + + @Path("/grow/{quantity}") + @PermitAll + public static class GrowResource + { + @PathParam(value = "quantity") + @DefaultValue( "1" ) + String quantity; + @DefaultValue( "" ) + @QueryParam( "key" ) + String key; + + @POST + @Produces(MediaType.APPLICATION_JSON) + public Map<String,String> postResize( ) + { + ResizeRequest request = new ResizeRequest(key, quantity); + if (request.error != null) { + return request.error; + } + + int curSize = dispatcher.getController().getTargetCount(); + int newSize = curSize + request.n; + dispatcher.getController().resizeTo(newSize); + return successResponse("Growing by " + request.n + " to " + newSize); + } + } + + @Path("/shrink/{quantity}") + @PermitAll + public static class ShrinkResource + { + @PathParam(value = "quantity") + @DefaultValue( "1" ) + String quantity; + @DefaultValue( "" ) + @QueryParam( "key" ) + String key; + + @POST + @Produces(MediaType.APPLICATION_JSON) + public Map<String,String> postResize( ) + { + ResizeRequest request = new ResizeRequest(key, quantity); + if (request.error != null) { + return request.error; + } + int curSize = dispatcher.getController().getTargetCount(); + int newSize = Math.max(curSize - request.n, 0); + dispatcher.getController().resizeTo(newSize); + return successResponse("Shrinking by " + request.n + " to " + newSize); + } + } + + private static Map<String, String> checkKey(String key) { + String masterKey = DrillOnYarnConfig.config() + .getString(DrillOnYarnConfig.HTTP_REST_KEY); + if (!DoYUtil.isBlank(masterKey) && !masterKey.equals(key)) { + return errorResponse("Invalid Key"); + } + return null; + } + + private static Map<String, String> errorResponse(String msg) { + Map<String, String> resp = new HashMap<>(); + resp.put("status", "error"); + resp.put("message", msg); + return resp; + } + + private static Map<String, String> successResponse(String msg) { + Map<String, String> resp = new HashMap<>(); + resp.put("status", "ok"); + resp.put("message", msg); + return resp; + } + + public AmRestApi(Dispatcher dispatcher) { + super(dispatcher); + + register(ConfigResource.class); + register(StatusResource.class); + register(StopResource.class); + register(ResizeResource.class); + register(GrowResource.class); + register(ShrinkResource.class); + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AuthDynamicFeature.java ---------------------------------------------------------------------- diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AuthDynamicFeature.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AuthDynamicFeature.java new file mode 100644 index 0000000..55cd59a --- /dev/null +++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AuthDynamicFeature.java @@ -0,0 +1,114 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.yarn.appMaster.http; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.drill.yarn.appMaster.http.WebUiPageTree.LogInLogOutPages; +import org.glassfish.jersey.server.model.AnnotatedMethod; + +import javax.annotation.Priority; +import javax.annotation.security.PermitAll; +import javax.annotation.security.RolesAllowed; +import javax.ws.rs.Priorities; +import javax.ws.rs.container.ContainerRequestContext; +import javax.ws.rs.container.ContainerRequestFilter; +import javax.ws.rs.container.DynamicFeature; +import javax.ws.rs.container.ResourceInfo; +import javax.ws.rs.core.FeatureContext; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.SecurityContext; +import java.io.IOException; +import java.net.URI; +import java.net.URLEncoder; + +/** + * Implementation of {@link DynamicFeature}. As part of the setup it adds the + * auth check filter {@link AuthCheckFilter} for resources that need to have + * user authenticated. If authentication is not done, request is forwarded to + * login page. + * <p> + * Shameless copy of + * {@link org.apache.drill.exec.server.rest.auth.DynamicFeature}; the two + * implementations should be merged at some point. The difference is only the + * log in/log out constant references. + */ + +public class AuthDynamicFeature implements DynamicFeature { + private static final Log LOG = LogFactory.getLog(AuthDynamicFeature.class); + + @Override + public void configure(final ResourceInfo resourceInfo, + final FeatureContext configuration) { + AnnotatedMethod am = new AnnotatedMethod(resourceInfo.getResourceMethod()); + + // RolesAllowed on the method takes precedence over PermitAll + RolesAllowed ra = am.getAnnotation(RolesAllowed.class); + if (ra != null) { + configuration.register(AuthCheckFilter.INSTANCE); + return; + } + + // PermitAll takes precedence over RolesAllowed on the class + if (am.isAnnotationPresent(PermitAll.class)) { + // Do nothing. + return; + } + + // RolesAllowed on the class takes precedence over PermitAll + ra = resourceInfo.getResourceClass().getAnnotation(RolesAllowed.class); + if (ra != null) { + configuration.register(AuthCheckFilter.INSTANCE); + } + } + + @Priority(Priorities.AUTHENTICATION) // authentication filter - should go + // first before all other filters. + private static class AuthCheckFilter implements ContainerRequestFilter { + private static AuthCheckFilter INSTANCE = new AuthCheckFilter(); + + @Override + public void filter(ContainerRequestContext requestContext) + throws IOException { + final SecurityContext sc = requestContext.getSecurityContext(); + if (!isUserLoggedIn(sc)) { + try { + final String destResource = URLEncoder.encode( + requestContext.getUriInfo().getRequestUri().toString(), "UTF-8"); + final URI loginURI = requestContext.getUriInfo().getBaseUriBuilder() + .path(LogInLogOutPages.LOGIN_RESOURCE) + .queryParam(LogInLogOutPages.REDIRECT_QUERY_PARM, destResource) + .build(); + requestContext + .abortWith(Response.temporaryRedirect(loginURI).build()); + } catch (final Exception ex) { + final String errMsg = String.format( + "Failed to forward the request to login page: %s", + ex.getMessage()); + LOG.error(errMsg, ex); + requestContext + .abortWith(Response.serverError().entity(errMsg).build()); + } + } + } + } + + public static boolean isUserLoggedIn(final SecurityContext sc) { + return sc != null && sc.getUserPrincipal() != null; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/ControllerModel.java ---------------------------------------------------------------------- diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/ControllerModel.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/ControllerModel.java new file mode 100644 index 0000000..8947df5 --- /dev/null +++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/ControllerModel.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.yarn.appMaster.http; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.drill.yarn.appMaster.AMYarnFacade.YarnAppHostReport; +import org.apache.drill.yarn.appMaster.ClusterController; +import org.apache.drill.yarn.appMaster.ClusterControllerImpl; +import org.apache.drill.yarn.appMaster.ClusterControllerImpl.State; +import org.apache.drill.yarn.appMaster.ControllerVisitor; +import org.apache.drill.yarn.appMaster.Scheduler; +import org.apache.drill.yarn.appMaster.SchedulerStateActions; +import org.apache.drill.yarn.core.ContainerRequestSpec; +import org.apache.drill.yarn.core.DrillOnYarnConfig; +import org.apache.drill.yarn.zk.ZKRegistry; + +import com.typesafe.config.Config; + +@XmlRootElement +public class ControllerModel implements ControllerVisitor { + public static class ClusterGroupModel { + protected String name; + protected String type; + protected int targetCount; + protected int taskCount; + protected int liveCount; + protected int memory; + protected int vcores; + protected double disks; + + public String getName( ) { return name; } + public String getType( ) { return type; } + public int getTargetCount( ) { return targetCount; } + public int getTaskCount( ) { return taskCount; } + public int getLiveCount( ) { return liveCount; } + public int getMemory( ) { return memory; } + public int getVcores( ) { return vcores; } + public String getDisks( ) { + return String.format( "%.02f", disks ); + } + } + + protected String zkConnectStr; + protected String zkRoot; + protected String zkClusterId; + protected ClusterControllerImpl.State state; + protected String stateHint; + protected boolean supportsDisks; + protected int yarnMemory; + protected int yarnVcores; + protected int yarnNodeCount; + protected int taskCount; + protected int liveCount; + protected int unmanagedCount; + protected int targetCount; + protected int totalDrillMemory; + protected int totalDrillVcores; + protected double totalDrillDisks; + protected int blacklistCount; + protected int freeNodeCount; + protected YarnAppHostReport appRpt; + protected int refreshSecs; + protected List<ClusterGroupModel> groups = new ArrayList<>( ); + + public boolean supportsDiskResource( ) { return supportsDisks; } + public int getRefreshSecs( ) { return refreshSecs; } + public String getZkConnectionStr( ) { return zkConnectStr; } + public String getZkRoot( ) { return zkRoot; } + public String getZkClusterId( ) { return zkClusterId; } + public String getAppId( ) { return appRpt.appId; } + public String getRmHost( ) { return appRpt.rmHost; } + public String getRmLink( ) { return appRpt.rmUrl; } + public String getNmHost( ) { return appRpt.nmHost; } + public String getNmLink( ) { return appRpt.nmUrl; } + public String getRmAppLink( ) { return appRpt.rmAppUrl; } + public String getNmAppLink( ) { return appRpt.nmAppUrl; } + public String getState( ) { return state.toString( ); } + public String getStateHint( ) { return stateHint; } + public int getYarnMemory( ) { return yarnMemory; } + public int getYarnVcores( ) { return yarnVcores; } + public int getDrillTotalMemory( ) { return totalDrillMemory; } + public int getDrillTotalVcores( ) { return totalDrillVcores; } + public String getDrillTotalDisks( ) { + return String.format( "%.2f", totalDrillDisks ); + } + public int getYarnNodeCount( ) { return yarnNodeCount; } + public int getTaskCount( ) { return taskCount; } + public int getLiveCount( ) { return liveCount; } + public int getUnmanagedCount( ) { return unmanagedCount; } + public int getTargetCount( ) { return targetCount; } + public List<ClusterGroupModel> getGroups( ) { return groups; } + public int getBlacklistCount( ) { return blacklistCount; } + public int getFreeNodeCount( ) { return freeNodeCount; } + + private static Map<ClusterControllerImpl.State,String> stateHints = makeStateHints( ); + + @Override + public void visit(ClusterController controller) { + Config config = DrillOnYarnConfig.config(); + refreshSecs = config.getInt( DrillOnYarnConfig.HTTP_REFRESH_SECS ); + zkConnectStr = config.getString( DrillOnYarnConfig.ZK_CONNECT ); + zkRoot = config.getString( DrillOnYarnConfig.ZK_ROOT ); + zkClusterId = config.getString( DrillOnYarnConfig.CLUSTER_ID ); + + ClusterControllerImpl impl = (ClusterControllerImpl) controller; + appRpt = impl.getYarn().getAppHostReport(); + + state = impl.getState( ); + stateHint = stateHints.get( state ); + + // Removed based on feedback. Users should check the + // YARN RM UI instead. + +// if ( state == State.LIVE ) { +// RegisterApplicationMasterResponse resp = impl.getYarn( ).getRegistrationResponse(); +// yarnVcores = resp.getMaximumResourceCapability().getVirtualCores(); +// yarnMemory = resp.getMaximumResourceCapability().getMemory(); +// yarnNodeCount = impl.getYarn( ).getNodeCount(); +// } + capturePools( impl ); + supportsDisks = impl.supportsDiskResource(); + + blacklistCount = impl.getNodeInventory( ).getBlacklist( ).size( ); + freeNodeCount = impl.getFreeNodeCount(); + } + + private void capturePools(ClusterControllerImpl impl) { + for ( SchedulerStateActions pool : impl.getPools( ) ) { + ControllerModel.ClusterGroupModel poolModel = new ControllerModel.ClusterGroupModel( ); + Scheduler sched = pool.getScheduler(); + ContainerRequestSpec containerSpec = sched.getResource( ); + poolModel.name = sched.getName(); + poolModel.type = sched.getType( ); + poolModel.targetCount = sched.getTarget(); + poolModel.memory = containerSpec.memoryMb; + poolModel.vcores = containerSpec.vCores; + poolModel.disks = containerSpec.disks; + poolModel.taskCount = pool.getTaskCount(); + poolModel.liveCount = pool.getLiveCount( ); + targetCount += poolModel.targetCount; + taskCount += poolModel.taskCount; + liveCount += poolModel.liveCount; + totalDrillMemory += poolModel.liveCount * poolModel.memory; + totalDrillVcores += poolModel.liveCount * poolModel.vcores; + totalDrillDisks += poolModel.liveCount * poolModel.disks; + groups.add( poolModel ); + } + if ( state != State.LIVE ) { + targetCount = 0; + } + } + + /** + * Count the unmanaged drillbits. Do this as a separate call, not via the + * {@link #visit(ClusterController) visit} method, to avoid locking both + * the cluster controller and ZK registry. + * + * @param controller + */ + + public void countStrayDrillbits(ClusterController controller) { + ZKRegistry zkRegistry = (ZKRegistry) controller.getProperty( ZKRegistry.CONTROLLER_PROPERTY ); + if ( zkRegistry != null ) { + unmanagedCount = zkRegistry.listUnmanagedDrillits().size(); + } + } + + /** + * Create a table of user-visible descriptions for each controller state. + * + * @return + */ + + private static Map<State, String> makeStateHints() { + Map<ClusterControllerImpl.State,String> hints = new HashMap<>( ); + // UI likely will never display the FAILED state. + hints.put( ClusterControllerImpl.State.START, "AM is starting up." ); + hints.put( ClusterControllerImpl.State.LIVE, "AM is operating normally." ); + hints.put( ClusterControllerImpl.State.ENDING, "AM is shutting down." ); + // UI will never display the ENDED state. + hints.put( ClusterControllerImpl.State.ENDED, "AM is about to exit." ); + // UI will never display the FAILED state. + hints.put( ClusterControllerImpl.State.FAILED, "AM failed to start and is about to exit." ); + return hints; + } + +} http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/PageTree.java ---------------------------------------------------------------------- diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/PageTree.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/PageTree.java new file mode 100644 index 0000000..e4d5dc1 --- /dev/null +++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/PageTree.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.yarn.appMaster.http; + +import java.util.HashMap; +import java.util.Map; + +import javax.ws.rs.core.SecurityContext; + +import org.apache.drill.exec.server.rest.auth.DrillUserPrincipal; +import org.apache.drill.yarn.appMaster.Dispatcher; +import org.apache.drill.yarn.core.DrillOnYarnConfig; +import org.glassfish.jersey.server.ResourceConfig; + +import com.typesafe.config.Config; + +/** + * Base class for a tree of web pages (or REST resources) represented + * as POJOs. Since the AM web UI is simple, this is the most convenient, + * compact way to implement the UI. + */ + +public class PageTree extends ResourceConfig { + // These items are a bit clumsy. We need them, but we can't make them + // instance variables without a bunch of messiness in the page classes. + // So, we let them be static. No harm in setting them multiple times. + + static Dispatcher dispatcher; + static Config config; + + public PageTree(Dispatcher dispatcher) { + PageTree.dispatcher = dispatcher; + config = DrillOnYarnConfig.config(); + } + + /** + * Creates a FreeMarker model that contains two top-level items: + * the model itself (as in the default implementation) and the + * cluster name (used as a title on each UI page.) + * + * @param base + * @return + */ + + public static Map<String, Object> toModel(SecurityContext sc, Object base) { + Map<String, Object> model = new HashMap<>(); + model.put("model", base); + return toMapModel(sc, model); + } + + public static Map<String, Object> toMapModel(SecurityContext sc, + Map<String, Object> model) { + model.put("clusterName", config.getString(DrillOnYarnConfig.APP_NAME)); + boolean useAuth = AMSecurityManagerImpl.isEnabled(); + final boolean isUserLoggedIn = (useAuth) + ? AuthDynamicFeature.isUserLoggedIn(sc) : false; + model.put("showLogin", useAuth && !isUserLoggedIn); + model.put("showLogout", isUserLoggedIn); + model.put("docsLink", config.getString(DrillOnYarnConfig.HTTP_DOCS_LINK)); + String userName = isUserLoggedIn ? sc.getUserPrincipal().getName() + : DrillUserPrincipal.ANONYMOUS_USER; + model.put("loggedInUserName", userName); + return model; + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/WebServer.java ---------------------------------------------------------------------- diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/WebServer.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/WebServer.java new file mode 100644 index 0000000..aeeafde --- /dev/null +++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/WebServer.java @@ -0,0 +1,467 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.yarn.appMaster.http; + +import static org.apache.drill.exec.server.rest.auth.DrillUserPrincipal.ADMIN_ROLE; + +import java.math.BigInteger; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.KeyStore; +import java.security.Principal; +import java.security.SecureRandom; +import java.security.cert.X509Certificate; +import java.util.Collections; +import java.util.Date; +import java.util.Set; + +import javax.servlet.http.HttpSession; +import javax.servlet.http.HttpSessionEvent; +import javax.servlet.http.HttpSessionListener; + +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.drill.yarn.appMaster.Dispatcher; +import org.apache.drill.yarn.core.DrillOnYarnConfig; +import org.bouncycastle.asn1.x500.X500NameBuilder; +import org.bouncycastle.asn1.x500.style.BCStyle; +import org.bouncycastle.cert.X509v3CertificateBuilder; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder; +import org.bouncycastle.operator.ContentSigner; +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; +import org.eclipse.jetty.http.HttpVersion; +import org.eclipse.jetty.security.ConstraintMapping; +import org.eclipse.jetty.security.ConstraintSecurityHandler; +import org.eclipse.jetty.security.DefaultIdentityService; +import org.eclipse.jetty.security.DefaultUserIdentity; +import org.eclipse.jetty.security.IdentityService; +import org.eclipse.jetty.security.LoginService; +import org.eclipse.jetty.security.SecurityHandler; +import org.eclipse.jetty.security.authentication.FormAuthenticator; +import org.eclipse.jetty.security.authentication.SessionAuthentication; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.SecureRequestCustomizer; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.SessionManager; +import org.eclipse.jetty.server.SslConnectionFactory; +import org.eclipse.jetty.server.UserIdentity; +import org.eclipse.jetty.server.handler.ErrorHandler; +import org.eclipse.jetty.server.session.HashSessionManager; +import org.eclipse.jetty.server.session.SessionHandler; +import org.eclipse.jetty.servlet.DefaultServlet; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.util.resource.Resource; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.glassfish.jersey.servlet.ServletContainer; +import org.joda.time.DateTime; + +import com.google.common.collect.ImmutableSet; +import com.typesafe.config.Config; + +/** + * Wrapper around the Jetty web server. + * <p> + * Adapted from Drill's drill.exec.WebServer class. Would be good to create a + * common base class later, but the goal for the initial project is to avoid + * Drill code changes. + * + * @see <a href= + * "http://www.eclipse.org/jetty/documentation/current/embedding-jetty.html";> + * Jetty Embedding documentation</a> + */ + +public class WebServer implements AutoCloseable { + private static final Log LOG = LogFactory.getLog(WebServer.class); + private final Server jettyServer; + private Dispatcher dispatcher; + + public WebServer(Dispatcher dispatcher) { + this.dispatcher = dispatcher; + Config config = DrillOnYarnConfig.config(); + if (config.getBoolean(DrillOnYarnConfig.HTTP_ENABLED)) { + jettyServer = new Server(); + } else { + jettyServer = null; + } + } + + /** + * Start the web server including setup. + * + * @throws Exception + */ + public void start() throws Exception { + if (jettyServer == null) { + return; + } + + build(); + jettyServer.start(); + } + + private void build() throws Exception { + Config config = DrillOnYarnConfig.config(); + buildConnector(config); + buildServlets(config); + } + + private void buildConnector(Config config) throws Exception { + final ServerConnector serverConnector; + if (config.getBoolean(DrillOnYarnConfig.HTTP_ENABLE_SSL)) { + serverConnector = createHttpsConnector(config); + } else { + serverConnector = createHttpConnector(config); + } + jettyServer.addConnector(serverConnector); + } + + /** + * Build the web app with embedded servlets. + * <p> + * <b>ServletContextHandler</b>: is a Jetty-provided handler that add the + * extra bits needed to set up the context that servlets expect. Think of it + * as an adapter between the (simple) Jetty handler and the (more complex) + * servlet API. + * + */ + private void buildServlets(Config config) { + + final ServletContextHandler servletContextHandler = new ServletContextHandler( + null, "/"); + servletContextHandler.setErrorHandler(createErrorHandler()); + jettyServer.setHandler(servletContextHandler); + + // Servlet holder for the pages of the Drill AM web app. The web app is a + // javax.ws application driven from annotations. The servlet holder "does + // the right thing" to drive the application, which is rooted at "/". + // The servlet container comes from Jersey, and manages the servlet + // lifecycle. + + final ServletHolder servletHolder = new ServletHolder( + new ServletContainer(new WebUiPageTree(dispatcher))); + servletHolder.setInitOrder(1); + servletContextHandler.addServlet(servletHolder, "/*"); + + final ServletHolder restHolder = new ServletHolder( + new ServletContainer(new AmRestApi(dispatcher))); + restHolder.setInitOrder(2); + servletContextHandler.addServlet(restHolder, "/rest/*"); + + // Static resources (CSS, images, etc.) + + setupStaticResources(servletContextHandler); + + // Security, if requested. + + if (AMSecurityManagerImpl.isEnabled()) { + servletContextHandler.setSecurityHandler(createSecurityHandler(config)); + servletContextHandler.setSessionHandler(createSessionHandler(config, + servletContextHandler.getSecurityHandler())); + } + } + + private ErrorHandler createErrorHandler() { + // Error handler to show detailed errors. + // Should probably be turned off in production. + final ErrorHandler errorHandler = new ErrorHandler(); + errorHandler.setShowStacks(true); + errorHandler.setShowMessageInTitle(true); + return errorHandler; + } + + private void setupStaticResources( + ServletContextHandler servletContextHandler) { + + // Access to static resources (JS pages, images, etc.) + // The static resources themselves come from Drill exec sub-project + // and the Drill-on-YARN project. + // + // We handle static content this way because we want to do it + // in the context of a servlet app, so we need the Jetty "default servlet" + // that handles static content. That servlet is designed to take its + // properties + // from the web.xml, file; but can also take them programmatically as done + // here. (The Jetty manual suggests a simpler handler, but that is a + // non-Servlet + // version.) + + final ServletHolder staticHolder = new ServletHolder("static", + DefaultServlet.class); + staticHolder.setInitParameter("resourceBase", + Resource.newClassPathResource("/rest/static").toString()); + staticHolder.setInitParameter("dirAllowed", "false"); + staticHolder.setInitParameter("pathInfoOnly", "true"); + servletContextHandler.addServlet(staticHolder, "/static/*"); + + final ServletHolder amStaticHolder = new ServletHolder("am-static", + DefaultServlet.class); + amStaticHolder.setInitParameter("resourceBase", + Resource.newClassPathResource("/drill-am/static").toString()); + amStaticHolder.setInitParameter("dirAllowed", "false"); + amStaticHolder.setInitParameter("pathInfoOnly", "true"); + servletContextHandler.addServlet(amStaticHolder, "/drill-am/static/*"); + } + + public static class AMUserPrincipal implements Principal { + public final String userName; + + public AMUserPrincipal(String userName) { + this.userName = userName; + } + + @Override + public String getName() { + return userName; + } + } + + public static class AmLoginService implements LoginService { + private AMSecurityManager securityMgr; + protected IdentityService identityService = new DefaultIdentityService(); + + public AmLoginService(AMSecurityManager securityMgr) { + this.securityMgr = securityMgr; + } + + @Override + public String getName() { + return "drill-am"; + } + + @Override + public UserIdentity login(String username, Object credentials) { + if (!securityMgr.login(username, (String) credentials)) { + return null; + } + return new DefaultUserIdentity(null, new AMUserPrincipal(username), + new String[] { ADMIN_ROLE }); + } + + @Override + public boolean validate(UserIdentity user) { + return true; + } + + @Override + public IdentityService getIdentityService() { + return identityService; + } + + @Override + public void setIdentityService(IdentityService service) { + this.identityService = service; + } + + @Override + public void logout(UserIdentity user) { + } + + // @Override + // protected UserIdentity loadUser(String username) { + // // TODO Auto-generated method stub + // return null; + // } + // + // @Override + // protected void loadUsers() throws IOException { + // putUser( "fred", new Password( "wilma" ), new String[] { ADMIN_ROLE } ); + // } + + } + + /** + * @return + * @return + * @see http://www.eclipse.org/jetty/documentation/current/embedded-examples.html + */ + + private ConstraintSecurityHandler createSecurityHandler(Config config) { + ConstraintSecurityHandler security = new ConstraintSecurityHandler(); + + Set<String> knownRoles = ImmutableSet.of(ADMIN_ROLE); + security.setConstraintMappings(Collections.<ConstraintMapping> emptyList(), + knownRoles); + + security.setAuthenticator(new FormAuthenticator("/login", "/login", true)); + security + .setLoginService(new AmLoginService(AMSecurityManagerImpl.instance())); + + return security; + } + + /** + * @return A {@link SessionHandler} which contains a + * {@link HashSessionManager} + */ + private SessionHandler createSessionHandler(Config config, + final SecurityHandler securityHandler) { + SessionManager sessionManager = new HashSessionManager(); + sessionManager.setMaxInactiveInterval( + config.getInt(DrillOnYarnConfig.HTTP_SESSION_MAX_IDLE_SECS)); + sessionManager.addEventListener(new HttpSessionListener() { + @Override + public void sessionCreated(HttpSessionEvent se) { + // No-op + } + + @Override + public void sessionDestroyed(HttpSessionEvent se) { + final HttpSession session = se.getSession(); + if (session == null) { + return; + } + + final Object authCreds = session + .getAttribute(SessionAuthentication.__J_AUTHENTICATED); + if (authCreds != null) { + final SessionAuthentication sessionAuth = (SessionAuthentication) authCreds; + securityHandler.logout(sessionAuth); + session.removeAttribute(SessionAuthentication.__J_AUTHENTICATED); + } + } + }); + + return new SessionHandler(sessionManager); + } + + /** + * Create HTTP connector. + * + * @return Initialized {@link ServerConnector} instance for HTTP connections. + * @throws Exception + */ + private ServerConnector createHttpConnector(Config config) throws Exception { + LOG.info("Setting up HTTP connector for web server"); + final HttpConfiguration httpConfig = new HttpConfiguration(); + final ServerConnector httpConnector = new ServerConnector(jettyServer, + new HttpConnectionFactory(httpConfig)); + httpConnector.setPort(config.getInt(DrillOnYarnConfig.HTTP_PORT)); + + return httpConnector; + } + + /** + * Create an HTTPS connector for given jetty server instance. If the admin has + * specified keystore/truststore settings they will be used else a self-signed + * certificate is generated and used. + * <p> + * This is a shameless copy of + * {@link org.apache.drill.exec.server.rest.Webserver#createHttpsConnector( )}. + * The two should be merged at some point. The primary issue is that the Drill + * version is tightly coupled to Drillbit configuration. + * + * @return Initialized {@link ServerConnector} for HTTPS connections. + * @throws Exception + */ + + private ServerConnector createHttpsConnector(Config config) throws Exception { + LOG.info("Setting up HTTPS connector for web server"); + + final SslContextFactory sslContextFactory = new SslContextFactory(); + + // if (config.hasPath(ExecConstants.HTTP_KEYSTORE_PATH) && + // !Strings.isNullOrEmpty(config.getString(ExecConstants.HTTP_KEYSTORE_PATH))) + // { + // LOG.info("Using configured SSL settings for web server"); + // sslContextFactory.setKeyStorePath(config.getString(ExecConstants.HTTP_KEYSTORE_PATH)); + // sslContextFactory.setKeyStorePassword(config.getString(ExecConstants.HTTP_KEYSTORE_PASSWORD)); + // + // // TrustStore and TrustStore password are optional + // if (config.hasPath(ExecConstants.HTTP_TRUSTSTORE_PATH)) { + // sslContextFactory.setTrustStorePath(config.getString(ExecConstants.HTTP_TRUSTSTORE_PATH)); + // if (config.hasPath(ExecConstants.HTTP_TRUSTSTORE_PASSWORD)) { + // sslContextFactory.setTrustStorePassword(config.getString(ExecConstants.HTTP_TRUSTSTORE_PASSWORD)); + // } + // } + // } else { + LOG.info("Using generated self-signed SSL settings for web server"); + final SecureRandom random = new SecureRandom(); + + // Generate a private-public key pair + final KeyPairGenerator keyPairGenerator = KeyPairGenerator + .getInstance("RSA"); + keyPairGenerator.initialize(1024, random); + final KeyPair keyPair = keyPairGenerator.generateKeyPair(); + + final DateTime now = DateTime.now(); + + // Create builder for certificate attributes + final X500NameBuilder nameBuilder = new X500NameBuilder(BCStyle.INSTANCE) + .addRDN(BCStyle.OU, "Apache Drill (auth-generated)") + .addRDN(BCStyle.O, "Apache Software Foundation (auto-generated)") + .addRDN(BCStyle.CN, "Drill AM"); + + final Date notBefore = now.minusMinutes(1).toDate(); + final Date notAfter = now.plusYears(5).toDate(); + final BigInteger serialNumber = new BigInteger(128, random); + + // Create a certificate valid for 5years from now. + final X509v3CertificateBuilder certificateBuilder = new JcaX509v3CertificateBuilder( + nameBuilder.build(), // attributes + serialNumber, notBefore, notAfter, nameBuilder.build(), + keyPair.getPublic()); + + // Sign the certificate using the private key + final ContentSigner contentSigner = new JcaContentSignerBuilder( + "SHA256WithRSAEncryption").build(keyPair.getPrivate()); + final X509Certificate certificate = new JcaX509CertificateConverter() + .getCertificate(certificateBuilder.build(contentSigner)); + + // Check the validity + certificate.checkValidity(now.toDate()); + + // Make sure the certificate is self-signed. + certificate.verify(certificate.getPublicKey()); + + // Generate a random password for keystore protection + final String keyStorePasswd = RandomStringUtils.random(20); + final KeyStore keyStore = KeyStore.getInstance("JKS"); + keyStore.load(null, null); + keyStore.setKeyEntry("DrillAutoGeneratedCert", keyPair.getPrivate(), + keyStorePasswd.toCharArray(), + new java.security.cert.Certificate[] { certificate }); + + sslContextFactory.setKeyStore(keyStore); + sslContextFactory.setKeyStorePassword(keyStorePasswd); + // } + + final HttpConfiguration httpsConfig = new HttpConfiguration(); + httpsConfig.addCustomizer(new SecureRequestCustomizer()); + + // SSL Connector + final ServerConnector sslConnector = new ServerConnector(jettyServer, + new SslConnectionFactory(sslContextFactory, + HttpVersion.HTTP_1_1.asString()), + new HttpConnectionFactory(httpsConfig)); + sslConnector.setPort(config.getInt(DrillOnYarnConfig.HTTP_PORT)); + + return sslConnector; + } + + @Override + public void close() throws Exception { + if (jettyServer != null) { + jettyServer.stop(); + } + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/WebUiPageTree.java ---------------------------------------------------------------------- diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/WebUiPageTree.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/WebUiPageTree.java new file mode 100644 index 0000000..fc44e45 --- /dev/null +++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/WebUiPageTree.java @@ -0,0 +1,527 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.yarn.appMaster.http; + +import static org.apache.drill.exec.server.rest.auth.DrillUserPrincipal.ADMIN_ROLE; + +import java.net.URI; +import java.net.URLDecoder; +import java.util.HashMap; +import java.util.Map; + +import javax.annotation.security.PermitAll; +import javax.annotation.security.RolesAllowed; +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpSession; +import javax.ws.rs.Consumes; +import javax.ws.rs.FormParam; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.SecurityContext; +import javax.ws.rs.core.UriBuilder; +import javax.ws.rs.core.UriInfo; + +import org.apache.commons.lang3.StringUtils; +import org.apache.drill.yarn.appMaster.Dispatcher; +import org.apache.drill.yarn.core.DoYUtil; +import org.apache.drill.yarn.core.DrillOnYarnConfig; +import org.eclipse.jetty.security.authentication.FormAuthenticator; +import org.glassfish.jersey.server.filter.RolesAllowedDynamicFeature; +import org.glassfish.jersey.server.mvc.Viewable; +import org.glassfish.jersey.server.mvc.freemarker.FreemarkerMvcFeature; + +/** + * The Drill AM web UI. The format is highly compact. We use javax.ws.rs to mark + * up a Pojo with page path, permissions and HTTP methods. The ADMIN_ROLE is + * reused from Drill's web UI. + * <p> + * In general, all pages require admin role, except for two: the login page and + * the redirect page which the YARN web UI follows to start the AM UI. + */ + +public class WebUiPageTree extends PageTree { + + /** + * Main DoY page that displays cluster status, and the status of + * the resource groups. Available only to the admin user when + * DoY is secured. + */ + + @Path("/") + @RolesAllowed(ADMIN_ROLE) + public static class RootPage { + @Inject + SecurityContext sc; + + @GET + @Produces(MediaType.TEXT_HTML) + public Viewable getRoot() { + ControllerModel model = new ControllerModel(); + dispatcher.getController().visit(model); + model.countStrayDrillbits(dispatcher.getController()); + return new Viewable("/drill-am/index.ftl", toModel(sc, model)); + } + } + + /** + * Pages, adapted from Drill, that display the login and logout pages. + * Login uses the security mechanism, again borrowed from Drill, to + * validate the user against either the simple user/password + * configured in DoY, or the user who launched DoY using the + * Drill security mechanism. + */ + + @Path("/") + @PermitAll + public static class LogInLogOutPages { + @Inject + SecurityContext sc; + + public static final String REDIRECT_QUERY_PARM = "redirect"; + public static final String LOGIN_RESOURCE = "login"; + + @GET + @Path("/login") + @Produces(MediaType.TEXT_HTML) + public Viewable getLoginPage(@Context HttpServletRequest request, + @Context HttpServletResponse response, @Context SecurityContext sc, + @Context UriInfo uriInfo, + @QueryParam(REDIRECT_QUERY_PARM) String redirect) throws Exception { + + if (!StringUtils.isEmpty(redirect)) { + // If the URL has redirect in it, set the redirect URI in session, so + // that after the login is successful, request + // is forwarded to the redirect page. + final HttpSession session = request.getSession(true); + final URI destURI = UriBuilder + .fromUri(URLDecoder.decode(redirect, "UTF-8")).build(); + session.setAttribute(FormAuthenticator.__J_URI, destURI.toString()); + } + + return new Viewable("/drill-am/login.ftl", toModel(sc, (Object) null)); + } + + // Request type is POST because POST request which contains the login + // credentials are invalid and the request is + // dispatched here directly. + @POST + @Path("/login") + @Produces(MediaType.TEXT_HTML) + public Viewable getLoginPageAfterValidationError() { + return new Viewable("/drill-am/login.ftl", + toModel(sc, "Invalid user name or password.")); + } + + @GET + @Path("/logout") + public Viewable logout(@Context HttpServletRequest req, + @Context HttpServletResponse resp) throws Exception { + final HttpSession session = req.getSession(); + if (session != null) { + session.invalidate(); + } + + req.getRequestDispatcher("/login").forward(req, resp); + return null; + } + } + + /** + * DoY provides a link to YARN to display the AM UI. YARN wants to display the + * linked page in a frame, which does not play well with the DoY UI. To avoid + * this, we give YARN a link to this redirect page which does nothing other + * than to redirect the browser to the (full) DoY main UI. + */ + + @Path("/redirect") + @PermitAll + public static class RedirectPage { + @GET + @Produces(MediaType.TEXT_HTML) + public Viewable getRoot() { + Map<String, String> map = new HashMap<>(); + String baseUrl = DoYUtil.unwrapAmUrl(dispatcher.getTrackingUrl()); + map.put("amLink", baseUrl); + map.put("clusterName", config.getString(DrillOnYarnConfig.APP_NAME)); + return new Viewable("/drill-am/redirect.ftl", map); + } + } + + /** + * Display the configuration page which displays the contents of + * DoY and selected Drill config as name/value pairs. Visible only + * to the admin when DoY is secure. + */ + + @Path("/config") + @RolesAllowed(ADMIN_ROLE) + public static class ConfigPage { + @Inject + private SecurityContext sc; + + @GET + @Produces(MediaType.TEXT_HTML) + public Viewable getRoot() { + return new Viewable("/drill-am/config.ftl", + toModel(sc, DrillOnYarnConfig.instance().getPairs())); + } + } + + /** + * Displays the list of Drillbits showing details for each Drillbit. + * (DoY uses the generic term "task", but, at present, the only + * task that DoY runs is a Drillbit. + */ + + @Path("/drillbits") + @RolesAllowed(ADMIN_ROLE) + public static class DrillbitsPage { + @Inject + private SecurityContext sc; + + @GET + @Produces(MediaType.TEXT_HTML) + public Viewable getRoot() { + AbstractTasksModel.TasksModel model = new AbstractTasksModel.TasksModel(); + dispatcher.getController().visitTasks(model); + model.listAnomalies(dispatcher.getController()); + model.sortTasks(); + + // Done this funky way because FreeMarker only understands lists if they + // are members of a hash (grumble, grumble...) + + Map<String, Object> map = new HashMap<>(); + map.put("model", model); + map.put("tasks", model.getTasks()); + if (model.hasUnmanagedDrillbits()) { + map.put("strays", model.getUnnamaged()); + } + if (model.hasBlacklist()) { + map.put("blacklist", model.getBlacklist()); + } + map.put("showDisks", dispatcher.getController().supportsDiskResource()); + map.put("refreshSecs", DrillOnYarnConfig.config() + .getInt(DrillOnYarnConfig.HTTP_REFRESH_SECS)); + return new Viewable("/drill-am/tasks.ftl", toMapModel(sc, map)); + } + } + + /** + * Displays a warning page to ask the user if they want to cancel + * a Drillbit. This is a bit old-school; we display this as a + * separate page. A good future enhancement is to do this as + * a pop-up in Javascript. The GET request display the confirmation + * page, the PUT request confirms cancellation and does the deed. + * The task to be cancelled appears as a query parameter: + * <pre>.../cancel?id=<task id></pre> + */ + + @Path("/cancel/") + @RolesAllowed(ADMIN_ROLE) + public static class CancelDrillbitPage { + @Inject + private SecurityContext sc; + + @QueryParam("id") + private int id; + + @GET + @Produces(MediaType.TEXT_HTML) + public Viewable getPage() { + ConfirmShrink confirm; + if (dispatcher.getController().isTaskLive(id)) { + confirm = new ConfirmShrink(ConfirmShrink.Mode.KILL); + } else { + confirm = new ConfirmShrink(ConfirmShrink.Mode.CANCEL); + } + confirm.id = id; + return new Viewable("/drill-am/shrink-warning.ftl", toModel(sc, confirm)); + } + + @POST + @Produces(MediaType.TEXT_HTML) + public Viewable postPage() { + Acknowledge ack; + if (dispatcher.getController().cancelTask(id)) { + ack = new Acknowledge(Acknowledge.Mode.CANCELLED); + } else { + ack = new Acknowledge(Acknowledge.Mode.INVALID_TASK); + } + ack.value = id; + return new Viewable("/drill-am/confirm.ftl", toModel(sc, ack)); + } + } + + /** + * Displays a history of completed tasks which indicates failed or cancelled + * Drillbits. Helps the admin to understand what has been happening on the + * cluster if Drillbits have died. + */ + + @Path("/history") + @RolesAllowed(ADMIN_ROLE) + public static class HistoryPage { + @Inject + SecurityContext sc; + + @GET + @Produces(MediaType.TEXT_HTML) + public Viewable getRoot() { + AbstractTasksModel.HistoryModel model = new AbstractTasksModel.HistoryModel(); + dispatcher.getController().visit(model); + Map<String, Object> map = new HashMap<>(); + map.put("model", model.results); + map.put("refreshSecs", DrillOnYarnConfig.config() + .getInt(DrillOnYarnConfig.HTTP_REFRESH_SECS)); + return new Viewable("/drill-am/history.ftl", toMapModel(sc, map)); + } + } + + /** + * Page that lets the admin change the cluster size or shut down the cluster. + */ + + @Path("/manage") + @RolesAllowed(ADMIN_ROLE) + public static class ManagePage { + @Inject + SecurityContext sc; + + @GET + @Produces(MediaType.TEXT_HTML) + public Viewable getRoot() { + ControllerModel model = new ControllerModel(); + dispatcher.getController().visit(model); + return new Viewable("/drill-am/manage.ftl", toModel(sc, model)); + } + } + + /** + * Passes information to the acknowledgement page. + */ + + public static class Acknowledge { + public enum Mode { + STOPPED, INVALID_RESIZE, INVALID_ACTION, NULL_RESIZE, RESIZED, CANCELLED, INVALID_TASK + }; + + Mode mode; + Object value; + + public Acknowledge(Mode mode) { + this.mode = mode; + } + + public String getType() { + return mode.toString(); + } + + public Object getValue() { + return value; + } + } + + /** + * Passes information to the confirmation page. + */ + + public static class ConfirmShrink { + public enum Mode { + SHRINK, STOP, CANCEL, KILL + }; + + Mode mode; + int value; + int id; + + public ConfirmShrink(Mode mode) { + this.mode = mode; + } + + public boolean isStop() { + return mode == Mode.STOP; + } + + public boolean isCancel() { + return mode == Mode.CANCEL; + } + + public boolean isKill() { + return mode == Mode.KILL; + } + + public boolean isShrink() { + return mode == Mode.SHRINK; + } + + public int getCount() { + return value; + } + + public int getId() { + return id; + } + } + + /** + * Confirm that the user wants to resize the cluster. Displays a warning if + * the user wants to shrink the cluster, since, at present, doing so will + * kill any in-flight queries. The GET request display the warning, + * the POST request confirms the action. The action itself is provided + * as query parameters: + * <pre>.../resize?type=<type>&n=<quantity></pre> + * Where the type is one of "resize", "grow", "shrink" or + * "force-shrink" and n is the associated quantity. + * <p> + * Note that the manage page only provides the "resize" option; the + * grow and shrink options were removed from the Web UI and are only + * visible through the REST API. + */ + + @Path("/resize") + @RolesAllowed(ADMIN_ROLE) + public static class ResizePage { + @Inject + SecurityContext sc; + + @FormParam("n") + int n; + @FormParam("type") + String type; + + @POST + @Consumes(MediaType.APPLICATION_FORM_URLENCODED) + @Produces(MediaType.TEXT_HTML) + public Viewable resize() { + int curSize = dispatcher.getController().getTargetCount(); + if (n <= 0) { + Acknowledge confirm = new Acknowledge(Acknowledge.Mode.INVALID_RESIZE); + confirm.value = n; + return new Viewable("/drill-am/confirm.ftl", toModel(sc, confirm)); + } + if (type == null) { + type = "null"; + } + int newSize; + boolean confirmed = false; + if (type.equalsIgnoreCase("resize")) { + newSize = n; + } else if (type.equalsIgnoreCase("grow")) { + newSize = curSize + n; + } else if (type.equalsIgnoreCase("shrink")) { + newSize = curSize - n; + } else if (type.equalsIgnoreCase("force-shrink")) { + newSize = curSize - n; + confirmed = true; + } else { + Acknowledge confirm = new Acknowledge(Acknowledge.Mode.INVALID_ACTION); + confirm.value = type; + return new Viewable("/drill-am/confirm.ftl", toModel(sc, confirm)); + } + + if (curSize == newSize) { + Acknowledge confirm = new Acknowledge(Acknowledge.Mode.NULL_RESIZE); + confirm.value = newSize; + return new Viewable("/drill-am/confirm.ftl", toModel(sc, confirm)); + } else if (confirmed || curSize < newSize) { + Acknowledge confirm = new Acknowledge(Acknowledge.Mode.RESIZED); + confirm.value = dispatcher.getController().resizeTo(newSize); + return new Viewable("/drill-am/confirm.ftl", toModel(sc, confirm)); + } else { + ConfirmShrink confirm = new ConfirmShrink(ConfirmShrink.Mode.SHRINK); + confirm.value = curSize - newSize; + return new Viewable("/drill-am/shrink-warning.ftl", + toModel(sc, confirm)); + } + } + } + + /** + * Confirmation page when the admin asks to stop the cluster. + * The GET request displays the confirmation, the POST does + * the deed. As for other confirmation pages, this is an old-style, + * quick & dirty solution. A more modern solution would be to use JavaScript + * to pop up a confirmation dialog. + */ + + @Path("/stop/") + @RolesAllowed(ADMIN_ROLE) + public static class StopPage { + @Inject + SecurityContext sc; + + @GET + @Produces(MediaType.TEXT_HTML) + public Viewable requestStop() { + ConfirmShrink confirm = new ConfirmShrink(ConfirmShrink.Mode.STOP); + return new Viewable("/drill-am/shrink-warning.ftl", toModel(sc, confirm)); + } + + @POST + @Produces(MediaType.TEXT_HTML) + public Viewable doStop() { + dispatcher.getController().shutDown(); + Acknowledge confirm = new Acknowledge(Acknowledge.Mode.STOPPED); + return new Viewable("/drill-am/confirm.ftl", toModel(sc, confirm)); + } + } + + /** + * Build the pages for the Web UI using Freemarker to implement the + * MVC mechanism. This class builds on a rather complex mechanism; understand + * that to understand what the lines of code below are doing. + * + * @param dispatcher the DoY AM dispatcher that receives requests for + * information about, or requests to change the state of, the Drill clutser + */ + + public WebUiPageTree(Dispatcher dispatcher) { + super(dispatcher); + + // Markup engine + register(FreemarkerMvcFeature.class); + + // Web UI Pages + register(RootPage.class); + register(RedirectPage.class); + register(ConfigPage.class); + register(DrillbitsPage.class); + register(CancelDrillbitPage.class); + register(HistoryPage.class); + register(ManagePage.class); + register(ResizePage.class); + register(StopPage.class); + + // Authorization + // See: https://jersey.java.net/documentation/latest/security.html + + if (AMSecurityManagerImpl.isEnabled()) { + register(LogInLogOutPages.class); + register(AuthDynamicFeature.class); + register(RolesAllowedDynamicFeature.class); + } + } + +} http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/package-info.java ---------------------------------------------------------------------- diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/package-info.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/package-info.java new file mode 100644 index 0000000..13f1bd8 --- /dev/null +++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/package-info.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + */ +package org.apache.drill.yarn.appMaster.http; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/package-info.java ---------------------------------------------------------------------- diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/package-info.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/package-info.java new file mode 100644 index 0000000..0ff835d --- /dev/null +++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/package-info.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Implements the Drill Application Master for YARN. + * <p> + * Note that AM implementation classes use org.apache.commons.logging + * to be consistent with the logging used within YARN itself. However, + * the AM uses Drill's class path which uses logback logging. To enable + * logging, modify + * <code>$DRILL_HOME/conf/logback.xml</code> and add a section something + * like this: + * <pre><code> + * <logger name="org.apache.drill.yarn" additivity="false"> + * <level value="trace" /> + * <appender-ref ref="STDOUT" /> + * </logger> + * </code></pre> + */ + +package org.apache.drill.yarn.appMaster; \ No newline at end of file
