http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java new file mode 100644 index 0000000..5140059 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.params; + +import com.beust.jcommander.Parameter; +import org.apache.slider.core.exceptions.BadCommandArgumentsException; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class ComponentArgsDelegate extends AbstractArgsDelegate { + + /** + * This is a listing of the roles to create + */ + @Parameter(names = {ARG_COMPONENT, ARG_COMPONENT_SHORT, ARG_ROLE}, + arity = 2, + description = "--component <name> <count> e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3", + splitter = DontSplitArguments.class) + public List<String> componentTuples = new ArrayList<>(0); + + + /** + * Get the role mapping (may be empty, but never null) + * @return role mapping + * @throws BadCommandArgumentsException parse problem + */ + public Map<String, String> getComponentMap() throws BadCommandArgumentsException { + return convertTupleListToMap("component", componentTuples); + } + + public List<String> getComponentTuples() { + return componentTuples; + } +}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java new file mode 100644 index 0000000..0344305 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.params; + +import com.beust.jcommander.converters.IParameterSplitter; + +import java.util.ArrayList; +import java.util.List; + +public class DontSplitArguments implements IParameterSplitter { + + @Override + public List<String> split(String value) { + List<String> list = new ArrayList<>(1); + list.add(value); + return list; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java new file mode 100644 index 0000000..7524053 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.params; + +import java.io.File; + +/** + * Launch args for create and start and anything else that can start something + */ +public interface LaunchArgsAccessor extends WaitTimeAccessor { + String getRmAddress(); + + File getOutputFile(); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java new file mode 100644 index 0000000..bc7e94c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.params; + +import com.beust.jcommander.Parameter; + +import java.io.File; + +/** + * Any launch-time args + */ +public class LaunchArgsDelegate extends WaitArgsDelegate implements + LaunchArgsAccessor { + + + //TODO: do we need this? + @Parameter(names = ARG_RESOURCE_MANAGER, + description = "Resource manager hostname:port ", + required = false) + private String rmAddress; + + @Override + public String getRmAddress() { + return rmAddress; + } + + @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT}, + description = "output file for any application report") + public File outputFile; + + @Override + public File getOutputFile() { + return outputFile; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java new file mode 100644 index 0000000..ccb526c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.params; + +import com.beust.jcommander.converters.BaseConverter; +import org.apache.hadoop.fs.Path; + +public class PathArgumentConverter extends BaseConverter<Path> { + + public PathArgumentConverter(String optionName) { + super(optionName); + } + + @Override + public Path convert(String value) { + return new Path(value); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java new file mode 100644 index 0000000..f9516d1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.slider.common.params; + +/** + * Parameters sent by the Client to the AM + */ +public class SliderAMArgs extends CommonArgs { + + SliderAMCreateAction createAction = new SliderAMCreateAction(); + + public SliderAMArgs(String[] args) { + super(args); + } + + @Override + protected void addActionArguments() { + addActions(createAction); + } + + public String getImage() { + return createAction.image; + } + + /** + * This is the URI in the FS to the Slider cluster; the conf file (and any + * other cluster-specifics) can be picked up here + */ + public String getSliderClusterURI() { + return createAction.sliderClusterURI; + } + + /** + * Am binding is simple: there is only one action + */ + @Override + public void applyAction() { + bindCoreAction(createAction); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java new file mode 100644 index 0000000..197c22b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.params; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.beust.jcommander.ParametersDelegate; + +import java.io.File; + + +@Parameters(commandNames = {SliderActions.ACTION_CREATE}, + commandDescription = SliderActions.DESCRIBE_ACTION_CREATE) + +public class SliderAMCreateAction extends AbstractActionArgs implements + LaunchArgsAccessor { + + + @Override + public String getActionName() { + return SliderActions.ACTION_CREATE; + } + + @Parameter(names = ARG_IMAGE, description = "image", required = false) + public String image; + + /** + * This is the URI in the FS to the Slider cluster; the conf file (and any + * other cluster-specifics) can be picked up here + */ + @Parameter(names = ARG_CLUSTER_URI, + description = "URI to the Slider cluster", required = true) + public String sliderClusterURI; + + @ParametersDelegate + LaunchArgsDelegate launchArgs = new LaunchArgsDelegate(); + + @Override + public String getRmAddress() { + return launchArgs.getRmAddress(); + } + + @Override + public int getWaittime() { + return launchArgs.getWaittime(); + } + + @Override + public void setWaittime(int waittime) { + launchArgs.setWaittime(waittime); + } + + @Override + public File getOutputFile() { + return launchArgs.getOutputFile(); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java new file mode 100644 index 0000000..204ad9a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.params; + +/** + * Actions. + * Only some of these are supported by specific Slider Services; they + * are listed here to ensure the names are consistent + */ +public interface SliderActions { + String ACTION_AM_SUICIDE = "am-suicide"; + String ACTION_BUILD = "build"; + String ACTION_CLIENT = "client"; + String ACTION_CREATE = "create"; + String ACTION_DIAGNOSTICS = "diagnostics"; + String ACTION_DEPENDENCY = "dependency"; + String ACTION_UPDATE = "update"; + String ACTION_UPGRADE = "upgrade"; + String ACTION_DESTROY = "destroy"; + String ACTION_ECHO = "echo"; + String ACTION_EXISTS = "exists"; + String ACTION_FLEX = "flex"; + String ACTION_FREEZE = "stop"; + String ACTION_HELP = "help"; + String ACTION_INSTALL_KEYTAB = "install-keytab"; + String ACTION_INSTALL_PACKAGE = "install-package"; + String ACTION_KDIAG = "kdiag"; + String ACTION_KEYTAB = "keytab"; + String ACTION_KILL_CONTAINER = "kill-container"; + String ACTION_LIST = "list"; + String ACTION_LOOKUP = "lookup"; + String ACTION_NODES = "nodes"; + String ACTION_PACKAGE = "package"; + String ACTION_PREFLIGHT = "preflight"; + String ACTION_RECONFIGURE = "reconfigure"; + String ACTION_REGISTRY = "registry"; + String ACTION_RESOLVE = "resolve"; + String ACTION_RESOURCE = "resource"; + String ACTION_STATUS = "status"; + String ACTION_THAW = "start"; + String ACTION_TOKENS = "tokens"; + + String ACTION_VERSION = "version"; + String DESCRIBE_ACTION_AM_SUICIDE = + "Tell the Slider Application Master to simulate a process failure by terminating itself"; + String DESCRIBE_ACTION_BUILD = + "Build a Slider cluster specification, but do not start it"; + String DESCRIBE_ACTION_CREATE = + "Create a live Slider application"; + String DESCRIBE_ACTION_DEPENDENCY = + "Slider AM and agent dependency (libraries) management"; + String DESCRIBE_ACTION_UPDATE = + "Update template for a Slider application"; + String DESCRIBE_ACTION_UPGRADE = + "Rolling upgrade/downgrade the application to a newer/previous version"; + String DESCRIBE_ACTION_DESTROY = + "Destroy a stopped Slider application"; + String DESCRIBE_ACTION_EXISTS = + "Probe for an application running"; + String DESCRIBE_ACTION_FLEX = "Flex a Slider application"; + String DESCRIBE_ACTION_FREEZE = + "Stop a running application"; + String DESCRIBE_ACTION_GETCONF = + "Get the configuration of an application"; + String DESCRIBE_ACTION_KDIAG = "Diagnose Kerberos problems"; + String DESCRIBE_ACTION_KILL_CONTAINER = + "Kill a container in the application"; + String DESCRIBE_ACTION_HELP = "Print help information"; + String DESCRIBE_ACTION_LIST = + "List running Slider applications"; + String DESCRIBE_ACTION_LOOKUP = + "look up a YARN application"; + String DESCRIBE_ACTION_NODES = "List the node information for the YARN cluster or a running application"; + String DESCRIBE_ACTION_MONITOR = + "Monitor a running application"; + String DESCRIBE_ACTION_REGISTRY = + "Query the registry of a YARN application"; + String DESCRIBE_ACTION_RESOLVE = + "Resolve or list records in the YARN registry"; + String DESCRIBE_ACTION_STATUS = + "Get the status of an application"; + String DESCRIBE_ACTION_THAW = + "Start a stopped application"; + String DESCRIBE_ACTION_VERSION = + "Print the Slider version information"; + String DESCRIBE_ACTION_INSTALL_PACKAGE = "Install application package." + + " Deprecated, use '" + ACTION_PACKAGE + " " + ClientArgs.ARG_INSTALL + "'."; + String DESCRIBE_ACTION_PACKAGE = "Install/list/delete application packages and list app instances that use the packages"; + String DESCRIBE_ACTION_CLIENT = "Install the application client in the specified directory or obtain a client keystore or truststore"; + String DESCRIBE_ACTION_INSTALL_KEYTAB = "Install the Kerberos keytab." + + " Deprecated, use '" + ACTION_KEYTAB + " " + ClientArgs.ARG_INSTALL + "'."; + String DESCRIBE_ACTION_KEYTAB = "Manage a Kerberos keytab file (install, delete, list) in the sub-folder 'keytabs' of the user's Slider base directory"; + String DESCRIBE_ACTION_DIAGNOSTIC = "Diagnose the configuration of the running slider application and slider client"; + String DESCRIBE_ACTION_RESOURCE = "Manage a file (install, delete, list) in the 'resources' sub-folder of the user's Slider base directory"; + +} + http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java new file mode 100644 index 0000000..b0d1ebf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.params; + +import com.beust.jcommander.converters.BaseConverter; + +import java.net.URI; +import java.net.URISyntaxException; + +public class URIArgumentConverter extends BaseConverter<URI> { + + public URIArgumentConverter(String optionName) { + super(optionName); + } + + @Override + public URI convert(String value) { + try { + return new URI(value); + } catch (URISyntaxException e) { + throw new RuntimeException("Cannot make a URI from " + value); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java new file mode 100644 index 0000000..8894309 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.slider.common.params; + +import com.beust.jcommander.converters.BaseConverter; + +import java.net.MalformedURLException; +import java.net.URL; + +public class URLArgumentConverter extends BaseConverter<URL> { + public URLArgumentConverter(String optionName) { + super(optionName); + } + + @Override + public URL convert(String value) { + try { + return new URL(value); + } catch (MalformedURLException e) { + throw new RuntimeException("Cannot make a URL from " + value); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java new file mode 100644 index 0000000..1c27c01 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.params; + +import com.beust.jcommander.Parameter; + +public class WaitArgsDelegate extends AbstractArgsDelegate implements + WaitTimeAccessor { + + + //--wait [timeout] + @Parameter(names = {ARG_WAIT}, + description = "time to wait for an action to complete") + public int waittime = 0; + + + @Override + public int getWaittime() { + return waittime; + } + + @Override + public void setWaittime(int waittime) { + this.waittime = waittime; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java new file mode 100644 index 0000000..13d4d5a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.params; + +public interface WaitTimeAccessor { + int getWaittime(); + void setWaittime(int waittime); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java new file mode 100644 index 0000000..6380d0c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.tools; + +import java.io.Serializable; +import java.util.Comparator; + +/** + * Some general comparators + */ +public class Comparators { + + public static class LongComparator implements Comparator<Long>, Serializable { + @Override + public int compare(Long o1, Long o2) { + long result = o1 - o2; + // need to comparisons with a diff greater than integer size + if (result < 0 ) { + return -1; + } else if (result > 0) { + return 1; + } + return 0; + } + } +public static class InvertedLongComparator implements Comparator<Long>, Serializable { + private static final LongComparator inner = new LongComparator(); + @Override + public int compare(Long o1, Long o2) { + return -inner.compare(o1, o2); + } + } + + + /** + * Little template class to reverse any comparitor + * @param <CompareType> the type that is being compared + */ + public static class ComparatorReverser<CompareType> implements Comparator<CompareType>, + Serializable { + + final Comparator<CompareType> instance; + + public ComparatorReverser(Comparator<CompareType> instance) { + this.instance = instance; + } + + @Override + public int compare(CompareType first, CompareType second) { + return instance.compare(second, first); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java new file mode 100644 index 0000000..9db241d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java @@ -0,0 +1,658 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.tools; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.registry.client.api.RegistryConstants; +import org.apache.slider.common.SliderKeys; +import org.apache.slider.common.SliderXmlConfKeys; +import org.apache.slider.core.exceptions.BadConfigException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.StringWriter; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Methods to aid in config, both in the Configuration class and + * with other parts of setting up Slider-initated processes. + * + * Some of the methods take an argument of a map iterable for their sources; this allows + * the same method + */ +public class ConfigHelper { + private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class); + + private static AtomicBoolean sliderResourceInjected = + new AtomicBoolean(false); + private static AtomicBoolean sliderResourceInjectionAttempted = + new AtomicBoolean(false); + + /** + * Dump the (sorted) configuration + * @param conf config + * @return the sorted keyset + */ + public static Set<String> dumpConf(Configuration conf) { + Set<String> keys = sortedConfigKeys(conf); + for (String key : keys) { + log.info("{}={}", key, conf.get(key)); + } + return keys; + } + + /** + * Take a configuration and return a sorted set + * @param conf config + * @return the sorted keyset + + */ + public static Set<String> sortedConfigKeys(Iterable<Map.Entry<String, String>> conf) { + TreeSet<String> sorted = new TreeSet<String>(); + for (Map.Entry<String, String> entry : conf) { + sorted.add(entry.getKey()); + } + return sorted; + } + + /** + * Set an entire map full of values + * + * @param config config to patch + * @param map map of data + * @param origin origin data + */ + public static void addConfigMap(Configuration config, + Map<String, String> map, + String origin) throws BadConfigException { + addConfigMap(config, map.entrySet(), origin); + } + + /** + * Set an entire map full of values + * + * @param config config to patch + * @param map map of data + * @param origin origin data + */ + public static void addConfigMap(Configuration config, + Iterable<Map.Entry<String, String>> map, + String origin) throws BadConfigException { + for (Map.Entry<String, String> mapEntry : map) { + String key = mapEntry.getKey(); + String value = mapEntry.getValue(); + if (value == null) { + throw new BadConfigException("Null value for property " + key); + } + config.set(key, value, origin); + } + } + + + /** + * Save a config file in a destination directory on a given filesystem + * @param systemConf system conf used for creating filesystems + * @param confToSave config to save + * @param confdir the directory path where the file is to go + * @param filename the filename + * @return the destination path where the file was saved + * @throws IOException IO problems + */ + public static Path saveConfig(Configuration systemConf, + Configuration confToSave, + Path confdir, + String filename) throws IOException { + FileSystem fs = FileSystem.get(confdir.toUri(), systemConf); + Path destPath = new Path(confdir, filename); + saveConfig(fs, destPath, confToSave); + return destPath; + } + + /** + * Save a config + * @param fs filesystem + * @param destPath dest to save + * @param confToSave config to save + * @throws IOException IO problems + */ + public static void saveConfig(FileSystem fs, + Path destPath, + Configuration confToSave) throws + IOException { + FSDataOutputStream fos = fs.create(destPath); + try { + confToSave.writeXml(fos); + } finally { + IOUtils.closeStream(fos); + } + } + + /** + * Convert to an XML string + * @param conf configuration + * @return conf + * @throws IOException + */ + public static String toXml(Configuration conf) throws IOException { + StringWriter writer = new StringWriter(); + conf.writeXml(writer); + return writer.toString(); + } + + /** + * This will load and parse a configuration to an XML document + * @param fs filesystem + * @param path path + * @return an XML document + * @throws IOException IO failure + */ + public Document parseConfiguration(FileSystem fs, + Path path) throws + IOException { + + + byte[] data = loadBytes(fs, path); + //this is here to track down a parse issue + //related to configurations + String s = new String(data, 0, data.length); + log.debug("XML resource {} is \"{}\"", path, s); +/* JDK7 + try (ByteArrayInputStream in = new ByteArrayInputStream(data)) { + Document document = parseConfigXML(in); + return document; + } catch (ParserConfigurationException | SAXException e) { + throw new IOException(e); + } +*/ + ByteArrayInputStream in= null; + try { + in = new ByteArrayInputStream(data); + Document document = parseConfigXML(in); + return document; + } catch (ParserConfigurationException e) { + throw new IOException(e); + } catch (SAXException e) { + throw new IOException(e); + } finally { + IOUtils.closeStream(in); + } + } + + public static byte[] loadBytes(FileSystem fs, Path path) throws IOException { + int len = (int) fs.getLength(path); + byte[] data = new byte[len]; + /* JDK7 + try(FSDataInputStream in = fs.open(path)) { + in.readFully(0, data); + } +*/ + FSDataInputStream in = null; + in = fs.open(path); + try { + in.readFully(0, data); + } finally { + IOUtils.closeStream(in); + } + return data; + } + + /** + * Load a configuration from ANY FS path. The normal Configuration + * loader only works with file:// URIs + * @param fs filesystem + * @param path path + * @return a loaded resource + * @throws IOException + */ + public static Configuration loadConfiguration(FileSystem fs, + Path path) throws IOException { + byte[] data = loadBytes(fs, path); + + ByteArrayInputStream in2; + + in2 = new ByteArrayInputStream(data); + Configuration conf1 = new Configuration(false); + conf1.addResource(in2); + //now clone it while dropping all its sources + Configuration conf2 = new Configuration(false); + String src = path.toString(); + for (Map.Entry<String, String> entry : conf1) { + String key = entry.getKey(); + String value = entry.getValue(); + conf2.set(key, value, src); + } + return conf2; + } + + + /** + * Generate a config file in a destination directory on the local filesystem + * @param confdir the directory path where the file is to go + * @param filename the filename + * @return the destination path + */ + public static File saveConfig(Configuration generatingConf, + File confdir, + String filename) throws IOException { + + + File destPath = new File(confdir, filename); + OutputStream fos = new FileOutputStream(destPath); + try { + generatingConf.writeXml(fos); + } finally { + IOUtils.closeStream(fos); + } + return destPath; + } + + /** + * Parse an XML Hadoop configuration into an XML document. x-include + * is supported, but as the location isn't passed in, relative + * URIs are out. + * @param in instream + * @return a document + * @throws ParserConfigurationException parser feature problems + * @throws IOException IO problems + * @throws SAXException XML is invalid + */ + public static Document parseConfigXML(InputStream in) throws + ParserConfigurationException, + IOException, + SAXException { + DocumentBuilderFactory docBuilderFactory + = DocumentBuilderFactory.newInstance(); + //ignore all comments inside the xml file + docBuilderFactory.setIgnoringComments(true); + + //allow includes in the xml file + docBuilderFactory.setNamespaceAware(true); + docBuilderFactory.setXIncludeAware(true); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + return builder.parse(in); + } + + /** + * Load a Hadoop configuration from a local file. + * @param file file to load + * @return a configuration which hasn't actually had the load triggered + * yet. + * @throws FileNotFoundException file is not there + * @throws IOException any other IO problem + */ + public static Configuration loadConfFromFile(File file) throws + IOException { + return loadConfFromFile(file, false); + } + + /** + * + * Load a Hadoop configuration from a local file. + * @param file file to load + * @param loadDefaults flag to indicate if the defaults should be loaded yet + * @return a configuration which hasn't actually had the load triggered + * yet. + * @throws FileNotFoundException file is not there + * @throws IOException any other IO problem + */ + public static Configuration loadConfFromFile(File file, + boolean loadDefaults) throws IOException { + if (!file.exists()) { + throw new FileNotFoundException("File not found :" + + file.getAbsoluteFile()); + } + Configuration conf = new Configuration(loadDefaults); + try { + conf.addResource(file.toURI().toURL()); + } catch (MalformedURLException e) { + // should never happen... + throw new IOException( + "File " + file.toURI() + " doesn't have a valid URL"); + } + return conf; + } + + /** + * Add a configuration from a file to an existing configuration + * @param conf existing configuration + * @param file file to load + * @param overwrite flag to indicate new values should overwrite the predecessor + * @return the merged configuration + * @throws IOException + */ + public static Configuration addConfigurationFile(Configuration conf, + File file, boolean overwrite) + throws IOException { + Configuration c2 = loadConfFromFile(file, false); + mergeConfigurations(conf, c2, file.getAbsolutePath(), overwrite); + return conf; + } + + /** + * Add the system env variables with the given prefix (by convention, env.) + * @param conf existing configuration + * @param prefix prefix + */ + public static void addEnvironmentVariables(Configuration conf, String prefix) { + Map<String, String> env = System.getenv(); + for (Map.Entry<String, String> entry : env.entrySet()) { + conf.set(prefix + entry.getKey(),entry.getValue(), "env"); + } + } + + /** + * looks for the config under $confdir/$templateFilename; if not there + * loads it from /conf/templateFile. + * The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the + * origin to help debug what's happening + * @param systemConf system conf + * @param confdir conf dir in FS + * @param templateFilename filename in the confdir + * @param fallbackResource resource to fall back on + * @return loaded conf + * @throws IOException IO problems + */ + public static Configuration loadTemplateConfiguration(Configuration systemConf, + Path confdir, + String templateFilename, + String fallbackResource) throws + IOException { + FileSystem fs = FileSystem.get(confdir.toUri(), systemConf); + + Path templatePath = new Path(confdir, templateFilename); + return loadTemplateConfiguration(fs, templatePath, fallbackResource); + } + + /** + * looks for the config under $confdir/$templateFilename; if not there + * loads it from /conf/templateFile. + * The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the + * origin to help debug what's happening. + * @param fs Filesystem + * @param templatePath HDFS path for template + * @param fallbackResource resource to fall back on, or "" for no fallback + * @return loaded conf + * @throws IOException IO problems + * @throws FileNotFoundException if the path doesn't have a file and there + * was no fallback. + */ + public static Configuration loadTemplateConfiguration(FileSystem fs, + Path templatePath, + String fallbackResource) + throws IOException { + Configuration conf; + String origin; + if (fs.exists(templatePath)) { + log.debug("Loading template configuration {}", templatePath); + conf = loadConfiguration(fs, templatePath); + origin = templatePath.toString(); + } else { + if (fallbackResource.isEmpty()) { + throw new FileNotFoundException("No config file found at " + templatePath); + } + log.debug("Template {} not found" + + " -reverting to classpath resource {}", templatePath, fallbackResource); + conf = new Configuration(false); + conf.addResource(fallbackResource); + origin = "Resource " + fallbackResource; + } + //force a get + conf.get(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN); + //now set the origin + conf.set(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN, origin); + return conf; + } + + + /** + * For testing: dump a configuration + * @param conf configuration + * @return listing in key=value style + */ + public static String dumpConfigToString(Configuration conf) { + Set<String> sorted = sortedConfigKeys(conf); + + StringBuilder builder = new StringBuilder(); + for (String key : sorted) { + + builder.append(key) + .append("=") + .append(conf.get(key)) + .append("\n"); + } + return builder.toString(); + } + + /** + * Merge in one configuration above another + * @param base base config + * @param merge one to merge. This MUST be a non-default-load config to avoid + * merge origin confusion + * @param origin description of the origin for the put operation + * @param overwrite flag to indicate new values should overwrite the predecessor + * @return the base with the merged values + */ + public static Configuration mergeConfigurations(Configuration base, + Iterable<Map.Entry<String, String>> merge, + String origin, + boolean overwrite) { + for (Map.Entry<String, String> entry : merge) { + String key = entry.getKey(); + if (overwrite || base.get(key) == null) { + base.set(key, entry.getValue(), origin); + } + } + return base; + } + + /** + * Register a resource as a default resource. + * Do not attempt to use this unless you understand that the + * order in which default resources are loaded affects the outcome, + * and that subclasses of Configuration often register new default + * resources + * @param resource the resource name + * @return the URL or null + */ + public static URL registerDefaultResource(String resource) { + URL resURL = getResourceUrl(resource); + if (resURL != null) { + Configuration.addDefaultResource(resource); + } + return resURL; + } + + /** + * Load a configuration from a resource on this classpath. + * If the resource is not found, an empty configuration is returned + * @param resource the resource name + * @return the loaded configuration. + */ + public static Configuration loadFromResource(String resource) { + Configuration conf = new Configuration(false); + URL resURL = getResourceUrl(resource); + if (resURL != null) { + log.debug("loaded resources from {}", resURL); + conf.addResource(resource); + } else{ + log.debug("failed to find {} on the classpath", resource); + } + return conf; + + } + + /** + * Get the URL to a resource, null if not on the CP + * @param resource resource to look for + * @return the URL or null + */ + public static URL getResourceUrl(String resource) { + return ConfigHelper.class.getClassLoader() + .getResource(resource); + } + + /** + * Load a resource that must be on the classpath + * @param resource the resource name + * @return the loaded configuration + * @throws FileNotFoundException if the resource is missing + */ + public static Configuration loadMandatoryResource(String resource) + throws FileNotFoundException { + Configuration conf = new Configuration(false); + URL resURL = getResourceUrl(resource); + if (resURL != null) { + log.debug("loaded resources from {}", resURL); + conf.addResource(resource); + } else { + throw new FileNotFoundException(resource); + } + return conf; + } + + /** + * Propagate a property from a source to a dest config, with a best-effort + * attempt at propagating the origin. + * If the + * @param dest destination + * @param src source + * @param key key to try to copy + * @return true if the key was found and propagated + */ + public static boolean propagate(Configuration dest, + Configuration src, + String key) { + String val = src.get(key); + if (val != null) { + String[] origin = src.getPropertySources(key); + if (origin != null && origin.length > 0) { + dest.set(key, val, origin[0]); + } else { + dest.set(key, val); + return true; + } + } + return false; + } + + + /** + * Take a configuration, return a hash map + * @param conf conf + * @return hash map + */ + public static Map<String, String> buildMapFromConfiguration(Configuration conf) { + Map<String, String> map = new HashMap<String, String>(); + return SliderUtils.mergeEntries(map, conf); + } + + /** + * This goes through the keyset of one configuration and retrieves each value + * from a value source -a different or the same configuration. This triggers + * the property resolution process of the value, resolving any variables against + * in-config or inherited configurations + * @param keysource source of keys + * @param valuesource the source of values + * @return a new configuration where <code>foreach key in keysource, get(key)==valuesource.get(key)</code> + */ + public static Configuration resolveConfiguration( + Iterable<Map.Entry<String, String>> keysource, + Configuration valuesource) { + Configuration result = new Configuration(false); + for (Map.Entry<String, String> entry : keysource) { + String key = entry.getKey(); + String value = valuesource.get(key); + Preconditions.checkState(value != null, + "no reference for \"%s\" in values", key); + result.set(key, value); + } + return result; + } + + /** + * Register anything we consider deprecated + */ + public static void registerDeprecatedConfigItems() { + Configuration.addDeprecation( + SliderXmlConfKeys.REGISTRY_ZK_QUORUM, + RegistryConstants.KEY_REGISTRY_ZK_QUORUM); + Configuration.addDeprecation( + SliderXmlConfKeys.REGISTRY_PATH, + RegistryConstants.KEY_REGISTRY_ZK_ROOT); + + } + + /** + * Load a configuration with the {@link SliderKeys#SLIDER_XML} resource + * included + * @return a configuration instance + */ + public static Configuration loadSliderConfiguration() { + Configuration conf = new Configuration(); + conf.addResource(SliderKeys.SLIDER_XML); + return conf; + } + + /** + * Inject the {@link SliderKeys#SLIDER_XML} resource + * into the configuration resources <i>of all configurations</i>. + * <p> + * This operation is idempotent. + * <p> + * If the resource is not on the classpath, downgrades, rather than + * fails. + * @return true if the resource was found and loaded. + */ + public static synchronized boolean injectSliderXMLResource() { + if (sliderResourceInjectionAttempted.getAndSet(true)) { + return sliderResourceInjected.get(); + } + URL resourceUrl = getResourceUrl(SliderKeys.SLIDER_XML); + if (resourceUrl != null) { + Configuration.addDefaultResource(SliderKeys.SLIDER_XML); + sliderResourceInjected.set(true); + } + return sliderResourceInjected.get(); + } + +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org