http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java deleted file mode 100644 index efec676..0000000 --- a/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.exceptions; - -import com.sun.jersey.api.client.ClientHandlerException; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.UniformInterfaceException; -import org.apache.hadoop.fs.InvalidRequestException; -import org.apache.hadoop.fs.PathAccessDeniedException; -import org.apache.hadoop.fs.PathIOException; -import org.apache.hadoop.yarn.webapp.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.servlet.http.HttpServletResponse; -import java.io.FileNotFoundException; -import java.io.IOException; - -/** - * static methods to convert exceptions into different types, including - * extraction of details and finer-grained conversions. - */ -public class ExceptionConverter { - private static final Logger - log = LoggerFactory.getLogger(ExceptionConverter.class); - - /** - * Uprate error codes 400 and up into faults; - * 404 is converted to a {@link FileNotFoundException}, - * 401 to {@link ForbiddenException} - * FileNotFoundException for an unknown resource - * PathAccessDeniedException for access denied - * PathIOException for anything else - * @param verb HTTP Verb used - * @param targetURL URL being targeted - * @param exception original exception - * @return a new exception, the original one nested as a cause - */ - public static IOException convertJerseyException(String verb, - String targetURL, - UniformInterfaceException exception) { - - IOException ioe = null; - ClientResponse response = exception.getResponse(); - if (response != null) { - int status = response.getStatus(); - String body = ""; - try { - if (response.hasEntity()) { - body = response.getEntity(String.class); - log.error("{} {} returned status {} and body\n{}", - verb, targetURL, status, body); - } else { - log.error("{} {} returned status {} and empty body", - verb, targetURL, status); - } - } catch (Exception e) { - log.warn("Failed to extract body from client response", e); - } - - if (status == HttpServletResponse.SC_UNAUTHORIZED - || status == HttpServletResponse.SC_FORBIDDEN) { - ioe = new PathAccessDeniedException(targetURL); - } else if (status == HttpServletResponse.SC_BAD_REQUEST - || status == HttpServletResponse.SC_NOT_ACCEPTABLE - || status == HttpServletResponse.SC_UNSUPPORTED_MEDIA_TYPE) { - // bad request - ioe = new InvalidRequestException( - String.format("Bad %s request: status code %d against %s", - verb, status, targetURL)); - } else if (status > 400 && status < 500) { - ioe = new FileNotFoundException(targetURL); - } - if (ioe == null) { - ioe = new PathIOException(targetURL, - verb + " " + targetURL - + " failed with status code : " + status - + ":" + exception); - } - } else { - ioe = new PathIOException(targetURL, - verb + " " + targetURL + " failed: " + exception); - } - ioe.initCause(exception); - return ioe; - } - - /** - * Handle a client-side Jersey exception. - * <p> - * If there's an inner IOException, return that. - * <p> - * Otherwise: create a new wrapper IOE including verb and target details - * @param verb HTTP Verb used - * @param targetURL URL being targeted - * @param exception original exception - * @return an exception to throw - */ - public static IOException convertJerseyException(String verb, - String targetURL, - ClientHandlerException exception) { - if (exception.getCause() instanceof IOException) { - return (IOException)exception.getCause(); - } else { - IOException ioe = new IOException( - verb + " " + targetURL + " failed: " + exception); - ioe.initCause(exception); - return ioe; - } - } - -}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/exceptions/NoSuchNodeException.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/NoSuchNodeException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/NoSuchNodeException.java deleted file mode 100644 index ad2f1a4..0000000 --- a/slider-core/src/main/java/org/apache/slider/core/exceptions/NoSuchNodeException.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.exceptions; - -import java.io.IOException; - -/** - * Exception raised when a node cannot be found in the structure - * that is being examined. - */ -public class NoSuchNodeException extends IOException { - - public NoSuchNodeException(String uuid) { - super(uuid); - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/exceptions/NotFoundException.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/NotFoundException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/NotFoundException.java deleted file mode 100644 index 40cb94d..0000000 --- a/slider-core/src/main/java/org/apache/slider/core/exceptions/NotFoundException.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.exceptions; - - -/** - * Whatever was being resolved: it was not found - */ -public class NotFoundException extends SliderException { - public NotFoundException(String message, - Object... args) { - super(EXIT_NOT_FOUND, message, args); - } - - public NotFoundException(Throwable throwable, - String message, Object... args) { - super(EXIT_NOT_FOUND, throwable, message, args); - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/exceptions/ServiceNotReadyException.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/ServiceNotReadyException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/ServiceNotReadyException.java deleted file mode 100644 index 435bc1a..0000000 --- a/slider-core/src/main/java/org/apache/slider/core/exceptions/ServiceNotReadyException.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.exceptions; - -import java.io.IOException; - -/** - * This is an exception raised when the service does not consider itself - * live (yet) - */ -public class ServiceNotReadyException extends IOException { - - public static final String E_NOT_READY = - "Service not ready for access: please retry"; - - public ServiceNotReadyException(String message) { - super(message); - } - - public ServiceNotReadyException(String message, Throwable cause) { - super(message, cause); - } - - public ServiceNotReadyException(Throwable cause) { - super(cause); - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java deleted file mode 100644 index 7f3134a..0000000 --- a/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.exceptions; - -import org.apache.slider.common.SliderExitCodes; -import org.apache.slider.core.main.ServiceLaunchException; - -public class SliderException extends ServiceLaunchException implements - SliderExitCodes { - public SliderException() { - super(EXIT_EXCEPTION_THROWN, "SliderException"); - } - - public SliderException(int code, String message) { - super(code, message); - } - - public SliderException(String s) { - super(EXIT_EXCEPTION_THROWN, s); - } - - public SliderException(String s, Throwable throwable) { - super(EXIT_EXCEPTION_THROWN, s, throwable); - } - - /** - * Format the exception as you create it - * @param code exit code - * @param message exception message -sprintf formatted - * @param args arguments for the formatting - */ - public SliderException(int code, String message, Object... args) { - super(code, String.format(message, args)); - } - - /** - * Format the exception, include a throwable. - * The throwable comes before the message so that it is out of the varargs - * @param code exit code - * @param throwable thrown - * @param message message - * @param args arguments - */ - public SliderException(int code, - Throwable throwable, - String message, - Object... args) { - super(code, String.format(message, args), throwable); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderInternalStateException.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderInternalStateException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderInternalStateException.java deleted file mode 100644 index deddbbc..0000000 --- a/slider-core/src/main/java/org/apache/slider/core/exceptions/SliderInternalStateException.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.exceptions; - -public class SliderInternalStateException extends SliderException { - public SliderInternalStateException(String s) { - super(EXIT_INTERNAL_ERROR, s); - } - - public SliderInternalStateException(String s, Throwable throwable) { - super(EXIT_INTERNAL_ERROR, throwable, s); - } - - public SliderInternalStateException(String message, - Object... args) { - super(EXIT_INTERNAL_ERROR, message, args); - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/exceptions/TriggerClusterTeardownException.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/TriggerClusterTeardownException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/TriggerClusterTeardownException.java deleted file mode 100644 index bb9f430..0000000 --- a/slider-core/src/main/java/org/apache/slider/core/exceptions/TriggerClusterTeardownException.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.exceptions; - -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; - -/** - * An Exception to be thrown for an explicit "shut down the cluster" operation - * raised by the application state or other parts of the AM - */ -public class TriggerClusterTeardownException extends SliderException { - - private final FinalApplicationStatus finalApplicationStatus; - - public TriggerClusterTeardownException(int code, - FinalApplicationStatus finalApplicationStatus, String message, - Object... args) { - super(code, message, args); - this.finalApplicationStatus = finalApplicationStatus; - } - - public FinalApplicationStatus getFinalApplicationStatus() { - return finalApplicationStatus; - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/exceptions/UnknownApplicationInstanceException.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/UnknownApplicationInstanceException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/UnknownApplicationInstanceException.java deleted file mode 100644 index a1f8ae9..0000000 --- a/slider-core/src/main/java/org/apache/slider/core/exceptions/UnknownApplicationInstanceException.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.exceptions; - -public class UnknownApplicationInstanceException extends SliderException { - public UnknownApplicationInstanceException(String s) { - super(EXIT_UNKNOWN_INSTANCE, s); - } - - public UnknownApplicationInstanceException(String s, Throwable throwable) { - super(EXIT_UNKNOWN_INSTANCE, throwable, s); - } - - public UnknownApplicationInstanceException(String message, - Object... args) { - super(EXIT_UNKNOWN_INSTANCE, message, args); - } - - /** - * Create an instance with the standard exception name - * @param name name - * @return an instance to throw - */ - public static UnknownApplicationInstanceException unknownInstance(String name) { - return new UnknownApplicationInstanceException(ErrorStrings.E_UNKNOWN_INSTANCE - + ": " + name); - } - public static UnknownApplicationInstanceException unknownInstance(String name, - Throwable throwable) { - UnknownApplicationInstanceException exception = - unknownInstance(name); - exception.initCause(throwable); - return exception; - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/exceptions/UsageException.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/UsageException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/UsageException.java deleted file mode 100644 index 8684294..0000000 --- a/slider-core/src/main/java/org/apache/slider/core/exceptions/UsageException.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.exceptions; - -/** - * Used to raise a usage exception ... this has the exit code - * {@link #EXIT_USAGE} - */ -public class UsageException extends SliderException { - public UsageException(String s, Object... args) { - super(EXIT_USAGE, s, args); - } - - public UsageException(Throwable throwable, String message, - Object... args) { - super(EXIT_USAGE, throwable, message, args); - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/exceptions/WaitTimeoutException.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/WaitTimeoutException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/WaitTimeoutException.java deleted file mode 100644 index 5ad3fdc..0000000 --- a/slider-core/src/main/java/org/apache/slider/core/exceptions/WaitTimeoutException.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.exceptions; - -import java.io.IOException; - -/** - * Called when some spinning operation timed out - */ -public class WaitTimeoutException extends IOException { - public WaitTimeoutException(String message) { - super(message); - } - - public WaitTimeoutException(String message, Throwable cause) { - super(message, cause); - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java deleted file mode 100644 index 5a3eb3d..0000000 --- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java +++ /dev/null @@ -1,528 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.launch; - -import com.google.common.base.Preconditions; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.LogAggregationContext; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.util.Records; -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.api.RoleKeys; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.tools.CoreFileSystem; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.conf.MapOperations; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.lang.reflect.Method; -import java.lang.reflect.InvocationTargetException; -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; - -/** - * Launcher of applications: base class - */ -public abstract class AbstractLauncher extends Configured { - private static final Logger log = - LoggerFactory.getLogger(AbstractLauncher.class); - public static final String CLASSPATH = "CLASSPATH"; - /** - * Filesystem to use for the launch - */ - protected final CoreFileSystem coreFileSystem; - /** - * Env vars; set up at final launch stage - */ - protected final Map<String, String> envVars = new HashMap<>(); - protected final MapOperations env = new MapOperations("env", envVars); - protected final ContainerLaunchContext containerLaunchContext = - Records.newRecord(ContainerLaunchContext.class); - protected final List<String> commands = new ArrayList<>(20); - protected final Map<String, LocalResource> localResources = new HashMap<>(); - protected final Map<String, String> mountPaths = new HashMap<>(); - private final Map<String, ByteBuffer> serviceData = new HashMap<>(); - // security - protected final Credentials credentials; - protected LogAggregationContext logAggregationContext; - protected boolean yarnDockerMode = false; - protected String dockerImage; - protected String yarnContainerMountPoints; - protected String runPrivilegedContainer; - - - /** - * Create instance. - * @param conf configuration - * @param coreFileSystem filesystem - * @param credentials initial set of credentials -null is permitted - */ - protected AbstractLauncher(Configuration conf, - CoreFileSystem coreFileSystem, - Credentials credentials) { - super(conf); - this.coreFileSystem = coreFileSystem; - this.credentials = credentials != null ? credentials: new Credentials(); - } - - /** - * Get the container. Until "completed", this isn't valid to launch. - * @return the container to launch - */ - public ContainerLaunchContext getContainerLaunchContext() { - return containerLaunchContext; - } - - public void setYarnDockerMode(boolean yarnDockerMode){ - this.yarnDockerMode = yarnDockerMode; - } - - /** - * Get the env vars to work on - * @return env vars - */ - public MapOperations getEnv() { - return env; - } - - /** - * Get the launch commands. - * @return the live list of commands - */ - public List<String> getCommands() { - return commands; - } - - /** - * Get the map of local resources. - * @return the live map of local resources. - */ - public Map<String, LocalResource> getLocalResources() { - return localResources; - } - - public void addLocalResource(String subPath, LocalResource resource) { - localResources.put(subPath, resource); - } - - public void addLocalResource(String subPath, LocalResource resource, String mountPath) { - localResources.put(subPath, resource); - mountPaths.put(subPath, mountPath); - } - - /** - * Add a set of local resources - * @param resourceMap map of name:resource to add - */ - public void addLocalResources(Map<String, LocalResource> resourceMap) { - localResources.putAll(resourceMap); - } - - public Map<String, ByteBuffer> getServiceData() { - return serviceData; - } - - /** - * Accessor to the credentials - * @return the credentials associated with this launcher - */ - public Credentials getCredentials() { - return credentials; - } - - /** - * Add a command line. It is converted to a single command before being - * added. - * @param cmd - */ - public void addCommandLine(CommandLineBuilder cmd) { - commands.add(cmd.build()); - } - - public void addCommand(String cmd) { - commands.add(cmd); - } - - /** - * Add a list of commands. Each element in the list becomes a single command - * @param commandList list of commands - */ - public void addCommands(List<String> commandList) { - commands.addAll(commandList); - } - - /** - * Get all commands as a string, separated by ";". This is for diagnostics - * @return a string description of the commands - */ - public String getCommandsAsString() { - return SliderUtils.join(getCommands(), "; "); - } - - /** - * Complete the launch context (copy in env vars, etc). - * @return the container to launch - */ - public ContainerLaunchContext completeContainerLaunch() throws IOException { - - String cmdStr = SliderUtils.join(commands, " ", false); - log.debug("Completed setting up container command {}", cmdStr); - containerLaunchContext.setCommands(commands); - - //env variables - if (log.isDebugEnabled()) { - log.debug("Environment variables"); - for (Map.Entry<String, String> envPair : envVars.entrySet()) { - log.debug(" \"{}\"=\"{}\"", envPair.getKey(), envPair.getValue()); - } - } - containerLaunchContext.setEnvironment(env); - - //service data - if (log.isDebugEnabled()) { - log.debug("Service Data size"); - for (Map.Entry<String, ByteBuffer> entry : serviceData.entrySet()) { - log.debug("\"{}\"=> {} bytes of data", entry.getKey(), - entry.getValue().array().length); - } - } - containerLaunchContext.setServiceData(serviceData); - - // resources - dumpLocalResources(); - containerLaunchContext.setLocalResources(localResources); - - //tokens - log.debug("{} tokens", credentials.numberOfTokens()); - containerLaunchContext.setTokens(CredentialUtils.marshallCredentials( - credentials)); - - if(yarnDockerMode){ - Map<String, String> env = containerLaunchContext.getEnvironment(); - env.put("YARN_CONTAINER_RUNTIME_TYPE", "docker"); - env.put("YARN_CONTAINER_RUNTIME_DOCKER_IMAGE", dockerImage);//if yarnDockerMode, then dockerImage is set - env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER", runPrivilegedContainer); - StringBuilder sb = new StringBuilder(); - for (Entry<String,String> mount : mountPaths.entrySet()) { - if (sb.length() > 0) { - sb.append(","); - } - sb.append(mount.getKey()); - sb.append(":"); - sb.append(mount.getValue()); - } - env.put("YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS", sb.toString()); - log.info("yarn docker env var has been set {}", containerLaunchContext.getEnvironment().toString()); - } - - return containerLaunchContext; - } - - /** - * Dump local resources at debug level - */ - private void dumpLocalResources() { - if (log.isDebugEnabled()) { - log.debug("{} resources: ", localResources.size()); - for (Map.Entry<String, LocalResource> entry : localResources.entrySet()) { - - String key = entry.getKey(); - LocalResource val = entry.getValue(); - log.debug(key + "=" + SliderUtils.stringify(val.getResource())); - } - } - } - - /** - * This is critical for an insecure cluster -it passes - * down the username to YARN, and so gives the code running - * in containers the rights it needs to work with - * data. - * @throws IOException problems working with current user - */ - protected void propagateUsernameInInsecureCluster() throws IOException { - //insecure cluster: propagate user name via env variable - String userName = UserGroupInformation.getCurrentUser().getUserName(); - env.put(SliderKeys.HADOOP_USER_NAME, userName); - } - - /** - * Extract any resource requirements from this component's settings. - * All fields that are set will override the existing values -if - * unset that resource field will be left unchanged. - * - * Important: the configuration must already be fully resolved - * in order to pick up global options. - * @param resource resource to configure - * @param map map of options - */ - public void extractResourceRequirements(Resource resource, - Map<String, String> map) { - - if (map != null) { - MapOperations options = new MapOperations("", map); - resource.setMemory(options.getOptionInt(ResourceKeys.YARN_MEMORY, - resource.getMemory())); - resource.setVirtualCores(options.getOptionInt(ResourceKeys.YARN_CORES, - resource.getVirtualCores())); - } - } - - /** - * Extract the value for option - * {@code yarn.resourcemanager.am.retry-count-window-ms} - * and set it on the ApplicationSubmissionContext. Use the default value - * if option is not set. - * - * @param submissionContext - * @param map - */ - public void extractAmRetryCount(ApplicationSubmissionContext submissionContext, - Map<String, String> map) { - - if (map != null) { - MapOperations options = new MapOperations("", map); - long amRetryCountWindow = options.getOptionLong(ResourceKeys - .YARN_RESOURCEMANAGER_AM_RETRY_COUNT_WINDOW_MS, - ResourceKeys.DEFAULT_AM_RETRY_COUNT_WINDOW_MS); - log.info("Setting {} to {}", - ResourceKeys.YARN_RESOURCEMANAGER_AM_RETRY_COUNT_WINDOW_MS, - amRetryCountWindow); - submissionContext.setAttemptFailuresValidityInterval(amRetryCountWindow); - } - } - - public void extractLogAggregationContext(Map<String, String> map) { - if (map != null) { - String logPatternSepStr = "\\|"; - String logPatternJoinStr = "|"; - MapOperations options = new MapOperations("", map); - - List<String> logIncludePatterns = new ArrayList<>(); - String includePatternExpression = options.getOption( - ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, "").trim(); - if (!includePatternExpression.isEmpty()) { - String[] includePatterns = includePatternExpression - .split(logPatternSepStr); - for (String includePattern : includePatterns) { - String trimmedIncludePattern = includePattern.trim(); - if (!trimmedIncludePattern.isEmpty()) { - logIncludePatterns.add(trimmedIncludePattern); - } - } - } - String logIncludePattern = StringUtils.join(logIncludePatterns, - logPatternJoinStr); - log.info("Log include patterns: {}", logIncludePattern); - - List<String> logExcludePatterns = new ArrayList<>(); - String excludePatternExpression = options.getOption( - ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, "").trim(); - if (!excludePatternExpression.isEmpty()) { - String[] excludePatterns = excludePatternExpression - .split(logPatternSepStr); - for (String excludePattern : excludePatterns) { - String trimmedExcludePattern = excludePattern.trim(); - if (!trimmedExcludePattern.isEmpty()) { - logExcludePatterns.add(trimmedExcludePattern); - } - } - } - String logExcludePattern = StringUtils.join(logExcludePatterns, - logPatternJoinStr); - log.info("Log exclude patterns: {}", logExcludePattern); - - // SLIDER-810/YARN-3154 - hadoop 2.7.0 onwards a new instance method has - // been added for log aggregation for LRS. Existing newInstance method's - // behavior has changed and is used for log aggregation only after the - // application has finished. This forces Slider users to move to hadoop - // 2.7.0+ just for log aggregation, which is not very desirable. So we - // decided to use reflection here to find out if the new 2.7.0 newInstance - // method is available. If yes, then we use it, so log aggregation will - // work in hadoop 2.7.0+ env. If no, then we fallback to the pre-2.7.0 - // newInstance method, which means log aggregation will work as expected - // in hadoop 2.6 as well. - // TODO: At some point, say 2-3 Slider releases down, when most users are - // running hadoop 2.7.0, we should get rid of the reflection code here. - try { - Method logAggregationContextMethod = LogAggregationContext.class - .getMethod("newInstance", String.class, String.class, String.class, - String.class); - // Need to set include/exclude patterns appropriately since by default - // rolled log aggregation is not done for any files, so defaults are - // - include pattern set to "" - // - exclude pattern set to "*" - // For Slider we want all logs to be uploaded if include/exclude - // patterns are left empty by the app owner in resources file - if (StringUtils.isEmpty(logIncludePattern) - && StringUtils.isEmpty(logExcludePattern)) { - logIncludePattern = ".*"; - logExcludePattern = ""; - } else if (StringUtils.isEmpty(logIncludePattern) - && StringUtils.isNotEmpty(logExcludePattern)) { - logIncludePattern = ".*"; - } else if (StringUtils.isNotEmpty(logIncludePattern) - && StringUtils.isEmpty(logExcludePattern)) { - logExcludePattern = ""; - } - log.debug("LogAggregationContext newInstance method for rolled logs " - + "include/exclude patterns is available"); - log.info("Modified log include patterns: {}", logIncludePattern); - log.info("Modified log exclude patterns: {}", logExcludePattern); - logAggregationContext = (LogAggregationContext) logAggregationContextMethod - .invoke(null, null, null, logIncludePattern, logExcludePattern); - } catch (NoSuchMethodException | SecurityException - | IllegalAccessException | IllegalArgumentException - | InvocationTargetException e) { - log.debug("LogAggregationContext newInstance method for rolled logs " - + "include/exclude patterns is not available - fallback to old one"); - log.debug(e.toString()); - logAggregationContext = LogAggregationContext.newInstance( - logIncludePattern, logExcludePattern); - } - } - } - - /** - * Utility method to set up the classpath - * @param classpath classpath to use - */ - public void setClasspath(ClasspathConstructor classpath) { - setEnv(CLASSPATH, classpath.buildClasspath()); - } - - /** - * Set an environment variable in the launch context - * @param var variable name - * @param value value (must be non null) - */ - public void setEnv(String var, String value) { - Preconditions.checkArgument(var != null, "null variable name"); - Preconditions.checkArgument(value != null, "null value"); - env.put(var, value); - } - - /** - * Set an environment variable if its value is non-null. - * @param var variable name - * @param value value (may be null) - */ - public void maybeSetEnv(String var, String value) { - if (value != null) { - setEnv(var, value); - } - } - - public void putEnv(Map<String, String> map) { - env.putAll(map); - } - - /** - * Important: the configuration must already be fully resolved - * in order to pick up global options - * Copy env vars into the launch context. - */ - public boolean copyEnvVars(MapOperations options) { - if (options == null) { - return false; - } - for (Map.Entry<String, String> entry : options.entrySet()) { - String key = entry.getKey(); - if (key.startsWith(RoleKeys.ENV_PREFIX)) { - key = key.substring(RoleKeys.ENV_PREFIX.length()); - env.put(key, entry.getValue()); - } - } - return true; - } - - public String[] dumpEnvToString() { - - List<String> nodeEnv = new ArrayList<>(); - - for (Map.Entry<String, String> entry : env.entrySet()) { - String envElt = String.format("%s=\"%s\"", - entry.getKey(), - entry.getValue()); - log.debug(envElt); - nodeEnv.add(envElt); - } - String[] envDescription = nodeEnv.toArray(new String[nodeEnv.size()]); - - return envDescription; - } - - /** - * Submit an entire directory - * @param srcDir src path in filesystem - * @param destRelativeDir relative path under destination local dir - * @throws IOException IO problems - */ - public void submitDirectory(Path srcDir, String destRelativeDir) - throws IOException { - //add the configuration resources - Map<String, LocalResource> confResources; - confResources = coreFileSystem.submitDirectory( - srcDir, - destRelativeDir); - addLocalResources(confResources); - } - - /** - * Return the label expression and if not set null - * @param map map to look up - * @return extracted label or null - */ - public String extractLabelExpression(Map<String, String> map) { - if (map != null) { - MapOperations options = new MapOperations("", map); - return options.getOption(ResourceKeys.YARN_LABEL_EXPRESSION, null); - } - return null; - } - - public void setDockerImage(String dockerImage) { - this.dockerImage = dockerImage; - } - - public void setYarnContainerMountPoints(String yarnContainerMountPoints) { - this.yarnContainerMountPoints = yarnContainerMountPoints; - } - - public void setRunPrivilegedContainer(String runPrivilegedContainer) { - this.runPrivilegedContainer = runPrivilegedContainer; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java deleted file mode 100644 index 7190c3a..0000000 --- a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java +++ /dev/null @@ -1,233 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.launch; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.client.api.YarnClientApplication; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.util.Records; -import org.apache.slider.client.SliderYarnClientImpl; -import org.apache.slider.common.tools.CoreFileSystem; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Map; -import java.util.Set; - -public class AppMasterLauncher extends AbstractLauncher { - - - private static final Logger log = - LoggerFactory.getLogger(AppMasterLauncher.class); - - public final YarnClientApplication application; - public final String name; - public final String type; - public final ApplicationSubmissionContext submissionContext; - public final ApplicationId appId; - public final boolean secureCluster; - private int maxAppAttempts = 0; - private boolean keepContainersOverRestarts = true; - private String queue = YarnConfiguration.DEFAULT_QUEUE_NAME; - private int priority = 1; - private final Resource resource = Records.newRecord(Resource.class); - private final SliderYarnClientImpl yarnClient; - private Long submitTime; - - /** - * Build the AM Launcher - * @param name app name - * @param type application type - * @param conf hadoop config - * @param fs filesystem binding - * @param yarnClient yarn client - * @param secureCluster flag to indicate secure cluster - * @param options map of options. All values are extracted in this constructor only - * @param resourceGlobalOptions global options - * @param applicationTags any app tags - * @param credentials initial set of credentials - * @throws IOException - * @throws YarnException - */ - public AppMasterLauncher(String name, - String type, - Configuration conf, - CoreFileSystem fs, - SliderYarnClientImpl yarnClient, - boolean secureCluster, - Map<String, String> options, - Map<String, String> resourceGlobalOptions, - Set<String> applicationTags, - Credentials credentials) throws IOException, YarnException { - super(conf, fs, credentials); - this.yarnClient = yarnClient; - this.application = yarnClient.createApplication(); - this.name = name; - this.type = type; - this.secureCluster = secureCluster; - - submissionContext = application.getApplicationSubmissionContext(); - appId = submissionContext.getApplicationId(); - // set the application name; - submissionContext.setApplicationName(name); - // app type used in service enum; - submissionContext.setApplicationType(type); - if (!applicationTags.isEmpty()) { - submissionContext.setApplicationTags(applicationTags); - } - submissionContext.setNodeLabelExpression(extractLabelExpression(options)); - - extractAmRetryCount(submissionContext, resourceGlobalOptions); - extractResourceRequirements(resource, options); - extractLogAggregationContext(resourceGlobalOptions); - } - - public void setMaxAppAttempts(int maxAppAttempts) { - this.maxAppAttempts = maxAppAttempts; - } - - public void setKeepContainersOverRestarts(boolean keepContainersOverRestarts) { - this.keepContainersOverRestarts = keepContainersOverRestarts; - } - - - public Resource getResource() { - return resource; - } - - public void setMemory(int memory) { - resource.setMemory(memory); - } - - public void setVirtualCores(int cores) { - resource.setVirtualCores(cores); - } - - public ApplicationId getApplicationId() { - return appId; - } - - public int getMaxAppAttempts() { - return maxAppAttempts; - } - - public boolean isKeepContainersOverRestarts() { - return keepContainersOverRestarts; - } - - public String getQueue() { - return queue; - } - - public int getPriority() { - return priority; - } - - public void setQueue(String queue) { - this.queue = queue; - } - - public void setPriority(int priority) { - this.priority = priority; - } - - /** - * Complete the launch context (copy in env vars, etc). - * @return the container to launch - */ - public ApplicationSubmissionContext completeAppMasterLaunch() - throws IOException { - - //queue priority - Priority pri = Records.newRecord(Priority.class); - pri.setPriority(priority); - submissionContext.setPriority(pri); - - // Set the queue to which this application is to be submitted in the RM - // Queue for App master - - submissionContext.setQueue(queue); - - - //container requirements - submissionContext.setResource(resource); - submissionContext.setLogAggregationContext(logAggregationContext); - - if (keepContainersOverRestarts) { - log.debug("Requesting cluster stays running over AM failure"); - submissionContext.setKeepContainersAcrossApplicationAttempts(true); - } - - if (maxAppAttempts > 0) { - log.debug("Setting max AM attempts to {}", maxAppAttempts); - submissionContext.setMaxAppAttempts(maxAppAttempts); - } - - if (secureCluster) { - //tokens - log.debug("Credentials: {}", - CredentialUtils.dumpTokens(getCredentials(), "\n")); - - } else { - propagateUsernameInInsecureCluster(); - } - completeContainerLaunch(); - submissionContext.setAMContainerSpec(containerLaunchContext); - return submissionContext; - } - - /** - * Submit the application. - * @return a launched application representing the submitted application - * @throws IOException - * @throws YarnException - */ - public LaunchedApplication submitApplication() throws IOException, YarnException { - completeAppMasterLaunch(); - log.info("Submitting application to Resource Manager"); - ApplicationId applicationId = - yarnClient.submitApplication(submissionContext); - // implicit success; record the time - submitTime = System.currentTimeMillis(); - return new LaunchedApplication(applicationId, yarnClient); - } - - /** - * Build a serializable application report. This is a very minimal - * report that contains the application Id, name and type âthe information - * available - * @return a data structure which can be persisted - */ - public SerializedApplicationReport createSerializedApplicationReport() { - SerializedApplicationReport sar = new SerializedApplicationReport(); - sar.applicationId = appId.toString(); - sar.name = name; - sar.applicationType = type; - sar.queue = queue; - sar.submitTime = submitTime; - return sar; - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java deleted file mode 100644 index 6eb4058..0000000 --- a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.launch; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.slider.common.tools.SliderUtils; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; - -/** - * build a classpath -allows for entries to be injected in front of - * YARN classpath as well as behind, adds appropriate separators, - * extraction of local classpath, etc. - */ -public class ClasspathConstructor { - - public static final String CLASS_PATH_SEPARATOR = ApplicationConstants.CLASS_PATH_SEPARATOR; - private final List<String> pathElements = new ArrayList<>(); - - public ClasspathConstructor() { - } - - - /** - * Get the list of JARs from the YARN settings - * @param config configuration - */ - public List<String> yarnApplicationClasspath(Configuration config) { - String[] cp = config.getTrimmedStrings( - YarnConfiguration.YARN_APPLICATION_CLASSPATH, - YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH); - return cp != null ? Arrays.asList(cp) : new ArrayList<String>(0); - - } - - - @Override - public String toString() { - return buildClasspath(); - } - - public String buildClasspath() { - return SliderUtils.join(pathElements, - CLASS_PATH_SEPARATOR, - false); - } - - /** - * Get a copy of the path list - * @return the JARs - */ - public List<String> getPathElements() { - return Collections.unmodifiableList(pathElements); - } - - /** - * Append an entry - * @param path path - */ - public void append(String path) { - pathElements.add(path); - } - - /** - * Insert a path at the front of the list. This places it ahead of - * the standard YARN artifacts - * @param path path to the JAR. Absolute or relative -on the target - * system - */ - public void insert(String path) { - pathElements.add(0, path); - } - - public void appendAll(Collection<String> paths) { - pathElements.addAll(paths); - } - - public void insertAll(Collection<String> paths) { - pathElements.addAll(0, paths); - } - - - public void addLibDir(String pathToLibDir) { - append(buildLibDir(pathToLibDir)); - } - - public void insertLibDir(String pathToLibDir) { - insert(buildLibDir(pathToLibDir)); - } - - public void addClassDirectory(String pathToDir) { - append(appendDirectoryTerminator(pathToDir)); - } - - public void insertClassDirectory(String pathToDir) { - insert(buildLibDir(appendDirectoryTerminator(pathToDir))); - } - - - public void addRemoteClasspathEnvVar() { - append(ApplicationConstants.Environment.CLASSPATH.$$()); - } - - - public void insertRemoteClasspathEnvVar() { - append(ApplicationConstants.Environment.CLASSPATH.$$()); - } - - - /** - * Build a lib dir path - * @param pathToLibDir path to the directory; may or may not end with a - * trailing space - * @return a path to a lib dir that is compatible with the java classpath - */ - public String buildLibDir(String pathToLibDir) { - String dir = appendDirectoryTerminator(pathToLibDir); - dir += "*"; - return dir; - } - - private String appendDirectoryTerminator(String pathToLibDir) { - String dir = pathToLibDir.trim(); - if (!dir.endsWith("/")) { - dir += "/"; - } - return dir; - } - - /** - * Split a classpath. This uses the local path separator so MUST NOT - * be used to work with remote classpaths - * @param localpath local path - * @return a splite - */ - public Collection<String> splitClasspath(String localpath) { - String separator = System.getProperty("path.separator"); - return StringUtils.getStringCollection(localpath, separator); - } - - /** - * Get the local JVM classpath split up - * @return the list of entries on the JVM classpath env var - */ - public Collection<String> localJVMClasspath() { - return splitClasspath(System.getProperty("java.class.path")); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java b/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java deleted file mode 100644 index dbaa981..0000000 --- a/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.launch; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.slider.common.tools.SliderUtils; - -import java.util.ArrayList; -import java.util.List; - -/** - * Build a single command line to include in the container commands; - * Special support for JVM command buildup. - */ -public class CommandLineBuilder { - protected final List<String> argumentList = new ArrayList<>(20); - - - /** - * Add an entry to the command list - * @param args arguments -these will be converted strings - */ - public void add(Object... args) { - for (Object arg : args) { - argumentList.add(arg.toString()); - } - } - - /** - * Get the value at an offset - * @param offset offset - * @return the value at that point - */ - public String elt(int offset) { - return argumentList.get(offset); - } - - /** - * Get the number of arguments - * @return an integer >= 0 - */ - public int size() { - return argumentList.size(); - } - - /** - * Append the output and error files to the tail of the command - * @param stdout out - * @param stderr error. Set this to null to append into stdout - */ - public void addOutAndErrFiles(String stdout, String stderr) { - Preconditions.checkNotNull(stdout, "Null output file"); - Preconditions.checkState(!stdout.isEmpty(), "output filename invalid"); - // write out the path output - argumentList.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + - stdout); - if (stderr != null) { - argumentList.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + - stderr); - } else { - argumentList.add("2>&1"); - } - } - - /** - * This just returns the command line - * @see #build() - * @return the command line - */ - @Override - public String toString() { - return build(); - } - - /** - * Build the command line - * @return the command line - */ - public String build() { - return SliderUtils.join(argumentList, " "); - } - - public List<String> getArgumentList() { - return argumentList; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java deleted file mode 100644 index d220928..0000000 --- a/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.launch; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; -import org.apache.hadoop.yarn.util.ConverterUtils; -import org.apache.slider.common.tools.CoreFileSystem; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.net.InetSocketAddress; - -/** - * Code to ease launching of any container - */ -public class ContainerLauncher extends AbstractLauncher { - private static final Logger log = - LoggerFactory.getLogger(ContainerLauncher.class); - // Allocated container - public final Container container; - - public ContainerLauncher(Configuration conf, - CoreFileSystem coreFileSystem, - Container container, - Credentials credentials) { - super(conf, coreFileSystem, credentials); - this.container = container; - } - - /** - * This code is in the dist shell examples -it's been moved here - * so that if it is needed, it's still here - * @return a remote user with a token to access the container. - */ - public UserGroupInformation setupUGI() { - UserGroupInformation user = - UserGroupInformation.createRemoteUser(container.getId().toString()); - String cmIpPortStr = container.getNodeId().getHost() + ":" + container.getNodeId().getPort(); - final InetSocketAddress cmAddress = NetUtils.createSocketAddr(cmIpPortStr); - - org.apache.hadoop.yarn.api.records.Token containerToken = container.getContainerToken(); - if (containerToken != null) { - Token<ContainerTokenIdentifier> token = - ConverterUtils.convertFromYarn(containerToken, cmAddress); - user.addToken(token); - } - return user; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java b/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java deleted file mode 100644 index 5357cc4..0000000 --- a/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java +++ /dev/null @@ -1,379 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.launch; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.io.DataOutputBuffer; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.SecurityUtil; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; -import org.apache.hadoop.yarn.client.ClientRMProxy; -import org.apache.hadoop.yarn.client.api.TimelineClient; -import org.apache.hadoop.yarn.client.api.YarnClient; -import org.apache.hadoop.yarn.conf.HAUtil; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; -import org.apache.hadoop.yarn.util.ConverterUtils; -import org.apache.slider.common.SliderXmlConfKeys; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.Serializable; -import java.nio.ByteBuffer; -import java.text.DateFormat; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import static org.apache.hadoop.security.UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION; -import static org.apache.hadoop.yarn.conf.YarnConfiguration.*; - -/** - * Utils to work with credentials and tokens. - * - * Designed to be movable to Hadoop core - */ -public final class CredentialUtils { - - public static final String JOB_CREDENTIALS_BINARY - = SliderXmlConfKeys.MAPREDUCE_JOB_CREDENTIALS_BINARY; - - private CredentialUtils() { - } - - private static final Logger LOG = - LoggerFactory.getLogger(CredentialUtils.class); - - /** - * Save credentials to a byte buffer. Returns null if there were no - * credentials to save - * @param credentials credential set - * @return a byte buffer of serialized tokens - * @throws IOException if the credentials could not be written to the stream - */ - public static ByteBuffer marshallCredentials(Credentials credentials) throws IOException { - ByteBuffer buffer = null; - if (!credentials.getAllTokens().isEmpty()) { - DataOutputBuffer dob = new DataOutputBuffer(); - try { - credentials.writeTokenStorageToStream(dob); - } finally { - dob.close(); - } - buffer = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); - } - return buffer; - } - - public static File locateEnvCredentials(Map<String, String> env, - Configuration conf, - StringBuffer sourceTextOut) throws FileNotFoundException { - String tokenFilename = env.get(HADOOP_TOKEN_FILE_LOCATION); - String source = "environment variable " + HADOOP_TOKEN_FILE_LOCATION; - if (tokenFilename == null) { - tokenFilename = conf.get(JOB_CREDENTIALS_BINARY); - source = "configuration option " + JOB_CREDENTIALS_BINARY; - } - if (tokenFilename != null) { - // use delegation tokens, i.e. from Oozie - File file = new File(tokenFilename.trim()); - String details = String.format( - "Token File %s from %s", - file, - source); - if (!file.exists()) { - throw new FileNotFoundException("No " + details); - } - if (!file.isFile() && !file.canRead()) { - throw new FileNotFoundException("Cannot read " + details); - } - sourceTextOut.append(details); - return file; - } else { - return null; - } - } - - /** - * Load the credentials from the environment. This looks at - * the value of {@link UserGroupInformation#HADOOP_TOKEN_FILE_LOCATION} - * and attempts to read in the value - * @param env environment to resolve the variable from - * @param conf configuration use when reading the tokens - * @return a set of credentials, or null if the environment did not - * specify any - * @throws IOException if a location for credentials was defined, but - * the credentials could not be loaded. - */ - public static Credentials loadTokensFromEnvironment(Map<String, String> env, - Configuration conf) - throws IOException { - StringBuffer origin = new StringBuffer(); - File file = locateEnvCredentials(env, conf, origin); - if (file != null) { - LOG.debug("Using {}", origin); - return Credentials.readTokenStorageFile(file, conf); - } else { - return null; - } - } - - /** - * Save credentials to a file - * @param file file to save to (will be overwritten) - * @param credentials credentials to write - * @throws IOException - */ - public static void saveTokens(File file, - Credentials credentials) throws IOException { - try(DataOutputStream daos = new DataOutputStream( - new FileOutputStream(file))) { - credentials.writeTokenStorageToStream(daos); - } - } - - /** - * Look up and return the resource manager's principal. This method - * automatically does the <code>_HOST</code> replacement in the principal and - * correctly handles HA resource manager configurations. - * - * From: YARN-4629 - * @param conf the {@link Configuration} file from which to read the - * principal - * @return the resource manager's principal string - * @throws IOException thrown if there's an error replacing the host name - */ - public static String getRMPrincipal(Configuration conf) throws IOException { - String principal = conf.get(RM_PRINCIPAL, ""); - String hostname; - Preconditions.checkState(!principal.isEmpty(), "Not set: " + RM_PRINCIPAL); - - if (HAUtil.isHAEnabled(conf)) { - YarnConfiguration yarnConf = new YarnConfiguration(conf); - if (yarnConf.get(RM_HA_ID) == null) { - // If RM_HA_ID is not configured, use the first of RM_HA_IDS. - // Any valid RM HA ID should work. - String[] rmIds = yarnConf.getStrings(RM_HA_IDS); - Preconditions.checkState((rmIds != null) && (rmIds.length > 0), - "Not set " + RM_HA_IDS); - yarnConf.set(RM_HA_ID, rmIds[0]); - } - - hostname = yarnConf.getSocketAddr( - RM_ADDRESS, - DEFAULT_RM_ADDRESS, - DEFAULT_RM_PORT).getHostName(); - } else { - hostname = conf.getSocketAddr( - RM_ADDRESS, - DEFAULT_RM_ADDRESS, - DEFAULT_RM_PORT).getHostName(); - } - return SecurityUtil.getServerPrincipal(principal, hostname); - } - - /** - * Create and add any filesystem delegation tokens with - * the RM(s) configured to be able to renew them. Returns null - * on an insecure cluster (i.e. harmless) - * @param conf configuration - * @param fs filesystem - * @param credentials credentials to update - * @return a list of all added tokens. - * @throws IOException - */ - public static Token<?>[] addRMRenewableFSDelegationTokens(Configuration conf, - FileSystem fs, - Credentials credentials) throws IOException { - Preconditions.checkArgument(conf != null); - Preconditions.checkArgument(credentials != null); - if (UserGroupInformation.isSecurityEnabled()) { - return fs.addDelegationTokens(CredentialUtils.getRMPrincipal(conf), - credentials); - } - return null; - } - - /** - * Add an FS delegation token which can be renewed by the current user - * @param fs filesystem - * @param credentials credentials to update - * @throws IOException problems. - */ - public static void addSelfRenewableFSDelegationTokens( - FileSystem fs, - Credentials credentials) throws IOException { - Preconditions.checkArgument(fs != null); - Preconditions.checkArgument(credentials != null); - fs.addDelegationTokens( - getSelfRenewer(), - credentials); - } - - public static String getSelfRenewer() throws IOException { - return UserGroupInformation.getLoginUser().getShortUserName(); - } - - /** - * Create and add an RM delegation token to the credentials - * @param yarnClient Yarn Client - * @param credentials to add token to - * @return the token which was added - * @throws IOException - * @throws YarnException - */ - public static Token<TokenIdentifier> addRMDelegationToken(YarnClient yarnClient, - Credentials credentials) - throws IOException, YarnException { - Configuration conf = yarnClient.getConfig(); - Text rmPrincipal = new Text(CredentialUtils.getRMPrincipal(conf)); - Text rmDTService = ClientRMProxy.getRMDelegationTokenService(conf); - Token<TokenIdentifier> rmDelegationToken = - ConverterUtils.convertFromYarn( - yarnClient.getRMDelegationToken(rmPrincipal), - rmDTService); - credentials.addToken(rmDelegationToken.getService(), rmDelegationToken); - return rmDelegationToken; - } - - public static Token<TimelineDelegationTokenIdentifier> maybeAddTimelineToken( - Configuration conf, - Credentials credentials) - throws IOException, YarnException { - if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) { - LOG.debug("Timeline service enabled -fetching token"); - - try(TimelineClient timelineClient = TimelineClient.createTimelineClient()) { - timelineClient.init(conf); - timelineClient.start(); - Token<TimelineDelegationTokenIdentifier> token = - timelineClient.getDelegationToken( - CredentialUtils.getRMPrincipal(conf)); - credentials.addToken(token.getService(), token); - return token; - } - } else { - LOG.debug("Timeline service is disabled"); - return null; - } - } - - /** - * Filter a list of tokens from a set of credentials - * @param credentials credential source (a new credential set os re - * @param filter List of tokens to strip out - * @return a new, filtered, set of credentials - */ - public static Credentials filterTokens(Credentials credentials, - List<Text> filter) { - Credentials result = new Credentials(credentials); - Iterator<Token<? extends TokenIdentifier>> iter = - result.getAllTokens().iterator(); - while (iter.hasNext()) { - Token<? extends TokenIdentifier> token = iter.next(); - LOG.debug("Token {}", token.getKind()); - if (filter.contains(token.getKind())) { - LOG.debug("Filtering token {}", token.getKind()); - iter.remove(); - } - } - return result; - } - - public static String dumpTokens(Credentials credentials, String separator) { - ArrayList<Token<? extends TokenIdentifier>> sorted = - new ArrayList<>(credentials.getAllTokens()); - Collections.sort(sorted, new TokenComparator()); - StringBuilder buffer = new StringBuilder(sorted.size()* 128); - for (Token<? extends TokenIdentifier> token : sorted) { - buffer.append(tokenToString(token)).append(separator); - } - return buffer.toString(); - } - - /** - * Create a string for people to look at - * @param token token to convert to a string form - * @return a printable view of the token - */ - public static String tokenToString(Token<? extends TokenIdentifier> token) { - DateFormat df = DateFormat.getDateTimeInstance( - DateFormat.SHORT, DateFormat.SHORT); - StringBuilder buffer = new StringBuilder(128); - buffer.append(token.toString()); - try { - TokenIdentifier ti = token.decodeIdentifier(); - buffer.append("; ").append(ti); - if (ti instanceof AbstractDelegationTokenIdentifier) { - // details in human readable form, and compensate for information HDFS DT omits - AbstractDelegationTokenIdentifier dt = (AbstractDelegationTokenIdentifier) ti; - buffer.append("; Renewer: ").append(dt.getRenewer()); - buffer.append("; Issued: ") - .append(df.format(new Date(dt.getIssueDate()))); - buffer.append("; Max Date: ") - .append(df.format(new Date(dt.getMaxDate()))); - } - } catch (IOException e) { - //marshall problem; not ours - LOG.debug("Failed to decode {}: {}", token, e, e); - } - return buffer.toString(); - } - - /** - * Get the expiry time of a token. - * @param token token to examine - * @return the time in milliseconds after which the token is invalid. - * @throws IOException - */ - public static long getTokenExpiryTime(Token token) throws IOException { - TokenIdentifier identifier = token.decodeIdentifier(); - Preconditions.checkState(identifier instanceof AbstractDelegationTokenIdentifier, - "Token %s of type: %s has an identifier which cannot be examined: %s", - token, token.getClass(), identifier); - AbstractDelegationTokenIdentifier id = - (AbstractDelegationTokenIdentifier) identifier; - return id.getMaxDate(); - } - - private static class TokenComparator - implements Comparator<Token<? extends TokenIdentifier>>, Serializable { - @Override - public int compare(Token<? extends TokenIdentifier> left, - Token<? extends TokenIdentifier> right) { - return left.getKind().toString().compareTo(right.getKind().toString()); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/51c2b92c/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java b/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java deleted file mode 100644 index b8aa4c6..0000000 --- a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.launch; - - -import com.google.common.base.Preconditions; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.exceptions.BadConfigException; - -import java.util.Iterator; -import java.util.Map; - -/** - * Command line builder purely for the Java CLI. - * Some of the <code>define</code> methods are designed to work with Hadoop tool and - * Slider launcher applications. - */ -public class JavaCommandLineBuilder extends CommandLineBuilder { - - public JavaCommandLineBuilder() { - add(getJavaBinary()); - } - - /** - * Get the java binary. This is called in the constructor so don't try and - * do anything other than return a constant. - * @return the path to the Java binary - */ - protected String getJavaBinary() { - return ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java"; - } - - /** - * Set the size of the heap if a non-empty heap is passed in. - * @param heap empty string or something like "128M" ,"1G" etc. The value is - * trimmed. - */ - public void setJVMHeap(String heap) { - if (SliderUtils.isSet(heap)) { - add("-Xmx" + heap.trim()); - } - } - - /** - * Turn Java assertions on - */ - public void enableJavaAssertions() { - add("-ea"); - add("-esa"); - } - - /** - * Add a system property definition -must be used before setting the main entry point - * @param property - * @param value - */ - public void sysprop(String property, String value) { - Preconditions.checkArgument(property != null, "null property name"); - Preconditions.checkArgument(value != null, "null value"); - add("-D" + property + "=" + value); - } - - public JavaCommandLineBuilder forceIPv4() { - sysprop("java.net.preferIPv4Stack", "true"); - return this; - } - - public JavaCommandLineBuilder headless() { - sysprop("java.awt.headless", "true"); - return this; - } - - public boolean addConfOption(Configuration conf, String key) { - return defineIfSet(key, conf.get(key)); - } - - /** - * Add a varargs list of configuration parameters âif they are present - * @param conf configuration source - * @param keys keys - */ - public void addConfOptions(Configuration conf, String... keys) { - for (String key : keys) { - addConfOption(conf, key); - } - } - - /** - * Add all configuration options which match the prefix - * @param conf configuration - * @param prefix prefix, e.g {@code "slider."} - * @return the number of entries copied - */ - public int addPrefixedConfOptions(Configuration conf, String prefix) { - int copied = 0; - for (Map.Entry<String, String> entry : conf) { - if (entry.getKey().startsWith(prefix)) { - define(entry.getKey(), entry.getValue()); - copied++; - } - } - return copied; - } - - /** - * Ass a configuration option to the command line of the application - * @param conf configuration - * @param key key - * @param defVal default value - * @return the resolved configuration option - * @throws IllegalArgumentException if key is null or the looked up value - * is null (that is: the argument is missing and devVal was null. - */ - public String addConfOptionToCLI(Configuration conf, - String key, - String defVal) { - Preconditions.checkArgument(key != null, "null key"); - String val = conf.get(key, defVal); - define(key, val); - return val; - } - - /** - * Add a <code>-D key=val</code> command to the CLI. This is very Hadoop API - * @param key key - * @param val value - * @throws IllegalArgumentException if either argument is null - */ - public void define(String key, String val) { - Preconditions.checkArgument(key != null, "null key"); - Preconditions.checkArgument(val != null, "null value"); - add("-D", key + "=" + val); - } - - /** - * Add a <code>-D key=val</code> command to the CLI if <code>val</code> - * is not null - * @param key key - * @param val value - */ - public boolean defineIfSet(String key, String val) { - Preconditions.checkArgument(key != null, "null key"); - if (val != null) { - define(key, val); - return true; - } else { - return false; - } - } - - /** - * Add a mandatory config option - * @param conf configuration - * @param key key - * @throws BadConfigException if the key is missing - */ - public void addMandatoryConfOption(Configuration conf, - String key) throws BadConfigException { - if (!addConfOption(conf, key)) { - throw new BadConfigException("Missing configuration option: " + key); - } - } - -}
